1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* Copyright (C) 2016 Oracle. All Rights Reserved.
4	* Author: Darrick J. Wong <darrick.wong@oracle.com>
5	*/
6	#include "xfs.h"
7	#include "xfs_fs.h"
8	#include "xfs_shared.h"
9	#include "xfs_format.h"
10	#include "xfs_log_format.h"
11	#include "xfs_trans_resv.h"
12	#include "xfs_mount.h"
13	#include "xfs_defer.h"
14	#include "xfs_inode.h"
15	#include "xfs_trans.h"
16	#include "xfs_bmap.h"
17	#include "xfs_bmap_util.h"
18	#include "xfs_trace.h"
19	#include "xfs_icache.h"
20	#include "xfs_btree.h"
21	#include "xfs_refcount_btree.h"
22	#include "xfs_refcount.h"
23	#include "xfs_bmap_btree.h"
24	#include "xfs_trans_space.h"
25	#include "xfs_bit.h"
26	#include "xfs_alloc.h"
27	#include "xfs_quota.h"
28	#include "xfs_reflink.h"
29	#include "xfs_iomap.h"
30	#include "xfs_ag.h"
31	#include "xfs_ag_resv.h"
32	#include "xfs_health.h"
33
34	/*
35	* Copy on Write of Shared Blocks
36	*
37	* XFS must preserve "the usual" file semantics even when two files share
38	* the same physical blocks. This means that a write to one file must not
39	* alter the blocks in a different file; the way that we'll do that is
40	* through the use of a copy-on-write mechanism. At a high level, that
41	* means that when we want to write to a shared block, we allocate a new
42	* block, write the data to the new block, and if that succeeds we map the
43	* new block into the file.
44	*
45	* XFS provides a "delayed allocation" mechanism that defers the allocation
46	* of disk blocks to dirty-but-not-yet-mapped file blocks as long as
47	* possible. This reduces fragmentation by enabling the filesystem to ask
48	* for bigger chunks less often, which is exactly what we want for CoW.
49	*
50	* The delalloc mechanism begins when the kernel wants to make a block
51	* writable (write_begin or page_mkwrite). If the offset is not mapped, we
52	* create a delalloc mapping, which is a regular in-core extent, but without
53	* a real startblock. (For delalloc mappings, the startblock encodes both
54	* a flag that this is a delalloc mapping, and a worst-case estimate of how
55	* many blocks might be required to put the mapping into the BMBT.) delalloc
56	* mappings are a reservation against the free space in the filesystem;
57	* adjacent mappings can also be combined into fewer larger mappings.
58	*
59	* As an optimization, the CoW extent size hint (cowextsz) creates
60	* outsized aligned delalloc reservations in the hope of landing out of
61	* order nearby CoW writes in a single extent on disk, thereby reducing
62	* fragmentation and improving future performance.
63	*
64	* D: --RRRRRRSSSRRRRRRRR--- (data fork)
65	* C: ------DDDDDDD--------- (CoW fork)
66	*
67	* When dirty pages are being written out (typically in writepage), the
68	* delalloc reservations are converted into unwritten mappings by
69	* allocating blocks and replacing the delalloc mapping with real ones.
70	* A delalloc mapping can be replaced by several unwritten ones if the
71	* free space is fragmented.
72	*
73	* D: --RRRRRRSSSRRRRRRRR---
74	* C: ------UUUUUUU---------
75	*
76	* We want to adapt the delalloc mechanism for copy-on-write, since the
77	* write paths are similar. The first two steps (creating the reservation
78	* and allocating the blocks) are exactly the same as delalloc except that
79	* the mappings must be stored in a separate CoW fork because we do not want
80	* to disturb the mapping in the data fork until we're sure that the write
81	* succeeded. IO completion in this case is the process of removing the old
82	* mapping from the data fork and moving the new mapping from the CoW fork to
83	* the data fork. This will be discussed shortly.
84	*
85	* For now, unaligned directio writes will be bounced back to the page cache.
86	* Block-aligned directio writes will use the same mechanism as buffered
87	* writes.
88	*
89	* Just prior to submitting the actual disk write requests, we convert
90	* the extents representing the range of the file actually being written
91	* (as opposed to extra pieces created for the cowextsize hint) to real
92	* extents. This will become important in the next step:
93	*
94	* D: --RRRRRRSSSRRRRRRRR---
95	* C: ------UUrrUUU---------
96	*
97	* CoW remapping must be done after the data block write completes,
98	* because we don't want to destroy the old data fork map until we're sure
99	* the new block has been written. Since the new mappings are kept in a
100	* separate fork, we can simply iterate these mappings to find the ones
101	* that cover the file blocks that we just CoW'd. For each extent, simply
102	* unmap the corresponding range in the data fork, map the new range into
103	* the data fork, and remove the extent from the CoW fork. Because of
104	* the presence of the cowextsize hint, however, we must be careful
105	* only to remap the blocks that we've actually written out -- we must
106	* never remap delalloc reservations nor CoW staging blocks that have
107	* yet to be written. This corresponds exactly to the real extents in
108	* the CoW fork:
109	*
110	* D: --RRRRRRrrSRRRRRRRR---
111	* C: ------UU--UUU---------
112	*
113	* Since the remapping operation can be applied to an arbitrary file
114	* range, we record the need for the remap step as a flag in the ioend
115	* instead of declaring a new IO type. This is required for direct io
116	* because we only have ioend for the whole dio, and we have to be able to
117	* remember the presence of unwritten blocks and CoW blocks with a single
118	* ioend structure. Better yet, the more ground we can cover with one
119	* ioend, the better.
120	*/
121
122	/*
123	* Given an AG extent, find the lowest-numbered run of shared blocks
124	* within that range and return the range in fbno/flen. If
125	* find_end_of_shared is true, return the longest contiguous extent of
126	* shared blocks. If there are no shared extents, fbno and flen will
127	* be set to NULLAGBLOCK and 0, respectively.
128	*/
129	static int
130	xfs_reflink_find_shared(
131	struct xfs_perag *pag,
132	struct xfs_trans *tp,
133	xfs_agblock_t agbno,
134	xfs_extlen_t aglen,
135	xfs_agblock_t *fbno,
136	xfs_extlen_t *flen,
137	bool find_end_of_shared)
138	{
139	struct xfs_buf *agbp;
140	struct xfs_btree_cur *cur;
141	int error;
142
143	error = xfs_alloc_read_agf(pag, tp, 0, &agbp);
144	if (error)
145	return error;
146
147	cur = xfs_refcountbt_init_cursor(pag->pag_mount, tp, agbp, pag);
148
149	error = xfs_refcount_find_shared(cur, agbno, aglen, fbno, flen,
150	find_end_of_shared);
151
152	xfs_btree_del_cursor(cur, error);
153
154	xfs_trans_brelse(tp, agbp);
155	return error;
156	}
157
158	/*
159	* Trim the mapping to the next block where there's a change in the
160	* shared/unshared status. More specifically, this means that we
161	* find the lowest-numbered extent of shared blocks that coincides with
162	* the given block mapping. If the shared extent overlaps the start of
163	* the mapping, trim the mapping to the end of the shared extent. If
164	* the shared region intersects the mapping, trim the mapping to the
165	* start of the shared extent. If there are no shared regions that
166	* overlap, just return the original extent.
167	*/
168	int
169	xfs_reflink_trim_around_shared(
170	struct xfs_inode *ip,
171	struct xfs_bmbt_irec *irec,
172	bool *shared)
173	{
174	struct xfs_mount *mp = ip->i_mount;
175	struct xfs_perag *pag;
176	xfs_agblock_t agbno;
177	xfs_extlen_t aglen;
178	xfs_agblock_t fbno;
179	xfs_extlen_t flen;
180	int error = 0;
181
182	/* Holes, unwritten, and delalloc extents cannot be shared */
183	if (!xfs_is_cow_inode(ip) || !xfs_bmap_is_written_extent(irec)) {
184	*shared = false;
185	return 0;
186	}
187
188	trace_xfs_reflink_trim_around_shared(ip, irec);
189
190	pag = xfs_perag_get(mp, XFS_FSB_TO_AGNO(mp, irec->br_startblock));
191	agbno = XFS_FSB_TO_AGBNO(mp, irec->br_startblock);
192	aglen = irec->br_blockcount;
193
194	error = xfs_reflink_find_shared(pag, NULL, agbno, aglen, &fbno, &flen,
195	true);
196	xfs_perag_put(pag);
197	if (error)
198	return error;
199
200	*shared = false;
201	if (fbno == NULLAGBLOCK) {
202	/* No shared blocks at all. */
203	return 0;
204	}
205
206	if (fbno == agbno) {
207	/*
208	* The start of this extent is shared. Truncate the
209	* mapping at the end of the shared region so that a
210	* subsequent iteration starts at the start of the
211	* unshared region.
212	*/
213	irec->br_blockcount = flen;
214	*shared = true;
215	return 0;
216	}
217
218	/*
219	* There's a shared extent midway through this extent.
220	* Truncate the mapping at the start of the shared
221	* extent so that a subsequent iteration starts at the
222	* start of the shared region.
223	*/
224	irec->br_blockcount = fbno - agbno;
225	return 0;
226	}
227
228	int
229	xfs_bmap_trim_cow(
230	struct xfs_inode *ip,
231	struct xfs_bmbt_irec *imap,
232	bool *shared)
233	{
234	/* We can't update any real extents in always COW mode. */
235	if (xfs_is_always_cow_inode(ip) &&
236	!isnullstartblock(imap->br_startblock)) {
237	*shared = true;
238	return 0;
239	}
240
241	/* Trim the mapping to the nearest shared extent boundary. */
242	return xfs_reflink_trim_around_shared(ip, irec: imap, shared);
243	}
244
245	static int
246	xfs_reflink_convert_cow_locked(
247	struct xfs_inode *ip,
248	xfs_fileoff_t offset_fsb,
249	xfs_filblks_t count_fsb)
250	{
251	struct xfs_iext_cursor icur;
252	struct xfs_bmbt_irec got;
253	struct xfs_btree_cur *dummy_cur = NULL;
254	int dummy_logflags;
255	int error = 0;
256
257	if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got))
258	return 0;
259
260	do {
261	if (got.br_startoff >= offset_fsb + count_fsb)
262	break;
263	if (got.br_state == XFS_EXT_NORM)
264	continue;
265	if (WARN_ON_ONCE(isnullstartblock(got.br_startblock)))
266	return -EIO;
267
268	xfs_trim_extent(&got, offset_fsb, count_fsb);
269	if (!got.br_blockcount)
270	continue;
271
272	got.br_state = XFS_EXT_NORM;
273	error = xfs_bmap_add_extent_unwritten_real(NULL, ip,
274	XFS_COW_FORK, &icur, &dummy_cur, &got,
275	&dummy_logflags);
276	if (error)
277	return error;
278	} while (xfs_iext_next_extent(ip->i_cowfp, &icur, &got));
279
280	return error;
281	}
282
283	/* Convert all of the unwritten CoW extents in a file's range to real ones. */
284	int
285	xfs_reflink_convert_cow(
286	struct xfs_inode *ip,
287	xfs_off_t offset,
288	xfs_off_t count)
289	{
290	struct xfs_mount *mp = ip->i_mount;
291	xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
292	xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count);
293	xfs_filblks_t count_fsb = end_fsb - offset_fsb;
294	int error;
295
296	ASSERT(count != 0);
297
298	xfs_ilock(ip, XFS_ILOCK_EXCL);
299	error = xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb);
300	xfs_iunlock(ip, XFS_ILOCK_EXCL);
301	return error;
302	}
303
304	/*
305	* Find the extent that maps the given range in the COW fork. Even if the extent
306	* is not shared we might have a preallocation for it in the COW fork. If so we
307	* use it that rather than trigger a new allocation.
308	*/
309	static int
310	xfs_find_trim_cow_extent(
311	struct xfs_inode *ip,
312	struct xfs_bmbt_irec *imap,
313	struct xfs_bmbt_irec *cmap,
314	bool *shared,
315	bool *found)
316	{
317	xfs_fileoff_t offset_fsb = imap->br_startoff;
318	xfs_filblks_t count_fsb = imap->br_blockcount;
319	struct xfs_iext_cursor icur;
320
321	*found = false;
322
323	/*
324	* If we don't find an overlapping extent, trim the range we need to
325	* allocate to fit the hole we found.
326	*/
327	if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, cmap))
328	cmap->br_startoff = offset_fsb + count_fsb;
329	if (cmap->br_startoff > offset_fsb) {
330	xfs_trim_extent(imap, imap->br_startoff,
331	cmap->br_startoff - imap->br_startoff);
332	return xfs_bmap_trim_cow(ip, imap, shared);
333	}
334
335	*shared = true;
336	if (isnullstartblock(cmap->br_startblock)) {
337	xfs_trim_extent(imap, cmap->br_startoff, cmap->br_blockcount);
338	return 0;
339	}
340
341	/* real extent found - no need to allocate */
342	xfs_trim_extent(cmap, offset_fsb, count_fsb);
343	*found = true;
344	return 0;
345	}
346
347	static int
348	xfs_reflink_convert_unwritten(
349	struct xfs_inode *ip,
350	struct xfs_bmbt_irec *imap,
351	struct xfs_bmbt_irec *cmap,
352	bool convert_now)
353	{
354	xfs_fileoff_t offset_fsb = imap->br_startoff;
355	xfs_filblks_t count_fsb = imap->br_blockcount;
356	int error;
357
358	/*
359	* cmap might larger than imap due to cowextsize hint.
360	*/
361	xfs_trim_extent(cmap, offset_fsb, count_fsb);
362
363	/*
364	* COW fork extents are supposed to remain unwritten until we're ready
365	* to initiate a disk write. For direct I/O we are going to write the
366	* data and need the conversion, but for buffered writes we're done.
367	*/
368	if (!convert_now || cmap->br_state == XFS_EXT_NORM)
369	return 0;
370
371	trace_xfs_reflink_convert_cow(ip, irec: cmap);
372
373	error = xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb);
374	if (!error)
375	cmap->br_state = XFS_EXT_NORM;
376
377	return error;
378	}
379
380	static int
381	xfs_reflink_fill_cow_hole(
382	struct xfs_inode *ip,
383	struct xfs_bmbt_irec *imap,
384	struct xfs_bmbt_irec *cmap,
385	bool *shared,
386	uint *lockmode,
387	bool convert_now)
388	{
389	struct xfs_mount *mp = ip->i_mount;
390	struct xfs_trans *tp;
391	xfs_filblks_t resaligned;
392	xfs_extlen_t resblks;
393	int nimaps;
394	int error;
395	bool found;
396
397	resaligned = xfs_aligned_fsb_count(imap->br_startoff,
398	imap->br_blockcount, xfs_get_cowextsz_hint(ip));
399	resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
400
401	xfs_iunlock(ip, *lockmode);
402	*lockmode = 0;
403
404	error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 0,
405	false, &tp);
406	if (error)
407	return error;
408
409	*lockmode = XFS_ILOCK_EXCL;
410
411	error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, found: &found);
412	if (error || !*shared)
413	goto out_trans_cancel;
414
415	if (found) {
416	xfs_trans_cancel(tp);
417	goto convert;
418	}
419
420	/* Allocate the entire reservation as unwritten blocks. */
421	nimaps = 1;
422	error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount,
423	XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, 0, cmap,
424	&nimaps);
425	if (error)
426	goto out_trans_cancel;
427
428	xfs_inode_set_cowblocks_tag(ip);
429	error = xfs_trans_commit(tp);
430	if (error)
431	return error;
432
433	/*
434	* Allocation succeeded but the requested range was not even partially
435	* satisfied? Bail out!
436	*/
437	if (nimaps == 0)
438	return -ENOSPC;
439
440	convert:
441	return xfs_reflink_convert_unwritten(ip, imap, cmap, convert_now);
442
443	out_trans_cancel:
444	xfs_trans_cancel(tp);
445	return error;
446	}
447
448	static int
449	xfs_reflink_fill_delalloc(
450	struct xfs_inode *ip,
451	struct xfs_bmbt_irec *imap,
452	struct xfs_bmbt_irec *cmap,
453	bool *shared,
454	uint *lockmode,
455	bool convert_now)
456	{
457	struct xfs_mount *mp = ip->i_mount;
458	struct xfs_trans *tp;
459	int nimaps;
460	int error;
461	bool found;
462
463	do {
464	xfs_iunlock(ip, *lockmode);
465	*lockmode = 0;
466
467	error = xfs_trans_alloc_inode(ip, resv: &M_RES(mp)->tr_write, dblocks: 0, rblocks: 0,
468	force: false, tpp: &tp);
469	if (error)
470	return error;
471
472	*lockmode = XFS_ILOCK_EXCL;
473
474	error = xfs_find_trim_cow_extent(ip, imap, cmap, shared,
475	found: &found);
476	if (error || !*shared)
477	goto out_trans_cancel;
478
479	if (found) {
480	xfs_trans_cancel(tp);
481	break;
482	}
483
484	ASSERT(isnullstartblock(cmap->br_startblock) ||
485	cmap->br_startblock == DELAYSTARTBLOCK);
486
487	/*
488	* Replace delalloc reservation with an unwritten extent.
489	*/
490	nimaps = 1;
491	error = xfs_bmapi_write(tp, ip, cmap->br_startoff,
492	cmap->br_blockcount,
493	XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, 0,
494	cmap, &nimaps);
495	if (error)
496	goto out_trans_cancel;
497
498	xfs_inode_set_cowblocks_tag(ip);
499	error = xfs_trans_commit(tp);
500	if (error)
501	return error;
502
503	/*
504	* Allocation succeeded but the requested range was not even
505	* partially satisfied? Bail out!
506	*/
507	if (nimaps == 0)
508	return -ENOSPC;
509	} while (cmap->br_startoff + cmap->br_blockcount <= imap->br_startoff);
510
511	return xfs_reflink_convert_unwritten(ip, imap, cmap, convert_now);
512
513	out_trans_cancel:
514	xfs_trans_cancel(tp);
515	return error;
516	}
517
518	/* Allocate all CoW reservations covering a range of blocks in a file. */
519	int
520	xfs_reflink_allocate_cow(
521	struct xfs_inode *ip,
522	struct xfs_bmbt_irec *imap,
523	struct xfs_bmbt_irec *cmap,
524	bool *shared,
525	uint *lockmode,
526	bool convert_now)
527	{
528	int error;
529	bool found;
530
531	xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
532	if (!ip->i_cowfp) {
533	ASSERT(!xfs_is_reflink_inode(ip));
534	xfs_ifork_init_cow(ip);
535	}
536
537	error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, found: &found);
538	if (error || !*shared)
539	return error;
540
541	/* CoW fork has a real extent */
542	if (found)
543	return xfs_reflink_convert_unwritten(ip, imap, cmap,
544	convert_now);
545
546	/*
547	* CoW fork does not have an extent and data extent is shared.
548	* Allocate a real extent in the CoW fork.
549	*/
550	if (cmap->br_startoff > imap->br_startoff)
551	return xfs_reflink_fill_cow_hole(ip, imap, cmap, shared,
552	lockmode, convert_now);
553
554	/*
555	* CoW fork has a delalloc reservation. Replace it with a real extent.
556	* There may or may not be a data fork mapping.
557	*/
558	if (isnullstartblock(cmap->br_startblock) ||
559	cmap->br_startblock == DELAYSTARTBLOCK)
560	return xfs_reflink_fill_delalloc(ip, imap, cmap, shared,
561	lockmode, convert_now);
562
563	/* Shouldn't get here. */
564	ASSERT(0);
565	return -EFSCORRUPTED;
566	}
567
568	/*
569	* Cancel CoW reservations for some block range of an inode.
570	*
571	* If cancel_real is true this function cancels all COW fork extents for the
572	* inode; if cancel_real is false, real extents are not cleared.
573	*
574	* Caller must have already joined the inode to the current transaction. The
575	* inode will be joined to the transaction returned to the caller.
576	*/
577	int
578	xfs_reflink_cancel_cow_blocks(
579	struct xfs_inode *ip,
580	struct xfs_trans **tpp,
581	xfs_fileoff_t offset_fsb,
582	xfs_fileoff_t end_fsb,
583	bool cancel_real)
584	{
585	struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK);
586	struct xfs_bmbt_irec got, del;
587	struct xfs_iext_cursor icur;
588	int error = 0;
589
590	if (!xfs_inode_has_cow_data(ip))
591	return 0;
592	if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got))
593	return 0;
594
595	/* Walk backwards until we're out of the I/O range... */
596	while (got.br_startoff + got.br_blockcount > offset_fsb) {
597	del = got;
598	xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb);
599
600	/* Extent delete may have bumped ext forward */
601	if (!del.br_blockcount) {
602	xfs_iext_prev(ifp, &icur);
603	goto next_extent;
604	}
605
606	trace_xfs_reflink_cancel_cow(ip, irec: &del);
607
608	if (isnullstartblock(del.br_startblock)) {
609	error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK,
610	&icur, &got, &del);
611	if (error)
612	break;
613	} else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) {
614	ASSERT((*tpp)->t_highest_agno == NULLAGNUMBER);
615
616	/* Free the CoW orphan record. */
617	xfs_refcount_free_cow_extent(*tpp, del.br_startblock,
618	del.br_blockcount);
619
620	error = xfs_free_extent_later(*tpp, del.br_startblock,
621	del.br_blockcount, NULL,
622	XFS_AG_RESV_NONE, false);
623	if (error)
624	break;
625
626	/* Roll the transaction */
627	error = xfs_defer_finish(tpp);
628	if (error)
629	break;
630
631	/* Remove the mapping from the CoW fork. */
632	xfs_bmap_del_extent_cow(ip, &icur, &got, &del);
633
634	/* Remove the quota reservation */
635	error = xfs_quota_unreserve_blkres(ip,
636	blocks: del.br_blockcount);
637	if (error)
638	break;
639	} else {
640	/* Didn't do anything, push cursor back. */
641	xfs_iext_prev(ifp, &icur);
642	}
643	next_extent:
644	if (!xfs_iext_get_extent(ifp, &icur, &got))
645	break;
646	}
647
648	/* clear tag if cow fork is emptied */
649	if (!ifp->if_bytes)
650	xfs_inode_clear_cowblocks_tag(ip);
651	return error;
652	}
653
654	/*
655	* Cancel CoW reservations for some byte range of an inode.
656	*
657	* If cancel_real is true this function cancels all COW fork extents for the
658	* inode; if cancel_real is false, real extents are not cleared.
659	*/
660	int
661	xfs_reflink_cancel_cow_range(
662	struct xfs_inode *ip,
663	xfs_off_t offset,
664	xfs_off_t count,
665	bool cancel_real)
666	{
667	struct xfs_trans *tp;
668	xfs_fileoff_t offset_fsb;
669	xfs_fileoff_t end_fsb;
670	int error;
671
672	trace_xfs_reflink_cancel_cow_range(ip, offset, count);
673	ASSERT(ip->i_cowfp);
674
675	offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
676	if (count == NULLFILEOFF)
677	end_fsb = NULLFILEOFF;
678	else
679	end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count);
680
681	/* Start a rolling transaction to remove the mappings */
682	error = xfs_trans_alloc(mp: ip->i_mount, resp: &M_RES(ip->i_mount)->tr_write,
683	blocks: 0, rtextents: 0, flags: 0, tpp: &tp);
684	if (error)
685	goto out;
686
687	xfs_ilock(ip, XFS_ILOCK_EXCL);
688	xfs_trans_ijoin(tp, ip, 0);
689
690	/* Scrape out the old CoW reservations */
691	error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb,
692	cancel_real);
693	if (error)
694	goto out_cancel;
695
696	error = xfs_trans_commit(tp);
697
698	xfs_iunlock(ip, XFS_ILOCK_EXCL);
699	return error;
700
701	out_cancel:
702	xfs_trans_cancel(tp);
703	xfs_iunlock(ip, XFS_ILOCK_EXCL);
704	out:
705	trace_xfs_reflink_cancel_cow_range_error(ip, error, _RET_IP_);
706	return error;
707	}
708
709	/*
710	* Remap part of the CoW fork into the data fork.
711	*
712	* We aim to remap the range starting at @offset_fsb and ending at @end_fsb
713	* into the data fork; this function will remap what it can (at the end of the
714	* range) and update @end_fsb appropriately. Each remap gets its own
715	* transaction because we can end up merging and splitting bmbt blocks for
716	* every remap operation and we'd like to keep the block reservation
717	* requirements as low as possible.
718	*/
719	STATIC int
720	xfs_reflink_end_cow_extent(
721	struct xfs_inode *ip,
722	xfs_fileoff_t *offset_fsb,
723	xfs_fileoff_t end_fsb)
724	{
725	struct xfs_iext_cursor icur;
726	struct xfs_bmbt_irec got, del, data;
727	struct xfs_mount *mp = ip->i_mount;
728	struct xfs_trans *tp;
729	struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK);
730	unsigned int resblks;
731	int nmaps;
732	int error;
733
734	/* No COW extents? That's easy! */
735	if (ifp->if_bytes == 0) {
736	*offset_fsb = end_fsb;
737	return 0;
738	}
739
740	resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK);
741	error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0,
742	XFS_TRANS_RESERVE, &tp);
743	if (error)
744	return error;
745
746	/*
747	* Lock the inode. We have to ijoin without automatic unlock because
748	* the lead transaction is the refcountbt record deletion; the data
749	* fork update follows as a deferred log item.
750	*/
751	xfs_ilock(ip, XFS_ILOCK_EXCL);
752	xfs_trans_ijoin(tp, ip, 0);
753
754	error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK,
755	XFS_IEXT_REFLINK_END_COW_CNT);
756	if (error == -EFBIG)
757	error = xfs_iext_count_upgrade(tp, ip,
758	XFS_IEXT_REFLINK_END_COW_CNT);
759	if (error)
760	goto out_cancel;
761
762	/*
763	* In case of racing, overlapping AIO writes no COW extents might be
764	* left by the time I/O completes for the loser of the race. In that
765	* case we are done.
766	*/
767	if (!xfs_iext_lookup_extent(ip, ifp, *offset_fsb, &icur, &got) ||
768	got.br_startoff >= end_fsb) {
769	*offset_fsb = end_fsb;
770	goto out_cancel;
771	}
772
773	/*
774	* Only remap real extents that contain data. With AIO, speculative
775	* preallocations can leak into the range we are called upon, and we
776	* need to skip them. Preserve @got for the eventual CoW fork
777	* deletion; from now on @del represents the mapping that we're
778	* actually remapping.
779	*/
780	while (!xfs_bmap_is_written_extent(&got)) {
781	if (!xfs_iext_next_extent(ifp, &icur, &got) ||
782	got.br_startoff >= end_fsb) {
783	*offset_fsb = end_fsb;
784	goto out_cancel;
785	}
786	}
787	del = got;
788	xfs_trim_extent(&del, *offset_fsb, end_fsb - *offset_fsb);
789
790	/* Grab the corresponding mapping in the data fork. */
791	nmaps = 1;
792	error = xfs_bmapi_read(ip, del.br_startoff, del.br_blockcount, &data,
793	&nmaps, 0);
794	if (error)
795	goto out_cancel;
796
797	/* We can only remap the smaller of the two extent sizes. */
798	data.br_blockcount = min(data.br_blockcount, del.br_blockcount);
799	del.br_blockcount = data.br_blockcount;
800
801	trace_xfs_reflink_cow_remap_from(ip, irec: &del);
802	trace_xfs_reflink_cow_remap_to(ip, irec: &data);
803
804	if (xfs_bmap_is_real_extent(&data)) {
805	/*
806	* If the extent we're remapping is backed by storage (written
807	* or not), unmap the extent and drop its refcount.
808	*/
809	xfs_bmap_unmap_extent(tp, ip, XFS_DATA_FORK, &data);
810	xfs_refcount_decrease_extent(tp, &data);
811	xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT,
812	-data.br_blockcount);
813	} else if (data.br_startblock == DELAYSTARTBLOCK) {
814	int done;
815
816	/*
817	* If the extent we're remapping is a delalloc reservation,
818	* we can use the regular bunmapi function to release the
819	* incore state. Dropping the delalloc reservation takes care
820	* of the quota reservation for us.
821	*/
822	error = xfs_bunmapi(NULL, ip, data.br_startoff,
823	data.br_blockcount, 0, 1, &done);
824	if (error)
825	goto out_cancel;
826	ASSERT(done);
827	}
828
829	/* Free the CoW orphan record. */
830	xfs_refcount_free_cow_extent(tp, del.br_startblock, del.br_blockcount);
831
832	/* Map the new blocks into the data fork. */
833	xfs_bmap_map_extent(tp, ip, XFS_DATA_FORK, &del);
834
835	/* Charge this new data fork mapping to the on-disk quota. */
836	xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_DELBCOUNT,
837	(long)del.br_blockcount);
838
839	/* Remove the mapping from the CoW fork. */
840	xfs_bmap_del_extent_cow(ip, &icur, &got, &del);
841
842	error = xfs_trans_commit(tp);
843	xfs_iunlock(ip, XFS_ILOCK_EXCL);
844	if (error)
845	return error;
846
847	/* Update the caller about how much progress we made. */
848	*offset_fsb = del.br_startoff + del.br_blockcount;
849	return 0;
850
851	out_cancel:
852	xfs_trans_cancel(tp);
853	xfs_iunlock(ip, XFS_ILOCK_EXCL);
854	return error;
855	}
856
857	/*
858	* Remap parts of a file's data fork after a successful CoW.
859	*/
860	int
861	xfs_reflink_end_cow(
862	struct xfs_inode *ip,
863	xfs_off_t offset,
864	xfs_off_t count)
865	{
866	xfs_fileoff_t offset_fsb;
867	xfs_fileoff_t end_fsb;
868	int error = 0;
869
870	trace_xfs_reflink_end_cow(ip, offset, count);
871
872	offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
873	end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count);
874
875	/*
876	* Walk forwards until we've remapped the I/O range. The loop function
877	* repeatedly cycles the ILOCK to allocate one transaction per remapped
878	* extent.
879	*
880	* If we're being called by writeback then the pages will still
881	* have PageWriteback set, which prevents races with reflink remapping
882	* and truncate. Reflink remapping prevents races with writeback by
883	* taking the iolock and mmaplock before flushing the pages and
884	* remapping, which means there won't be any further writeback or page
885	* cache dirtying until the reflink completes.
886	*
887	* We should never have two threads issuing writeback for the same file
888	* region. There are also have post-eof checks in the writeback
889	* preparation code so that we don't bother writing out pages that are
890	* about to be truncated.
891	*
892	* If we're being called as part of directio write completion, the dio
893	* count is still elevated, which reflink and truncate will wait for.
894	* Reflink remapping takes the iolock and mmaplock and waits for
895	* pending dio to finish, which should prevent any directio until the
896	* remap completes. Multiple concurrent directio writes to the same
897	* region are handled by end_cow processing only occurring for the
898	* threads which succeed; the outcome of multiple overlapping direct
899	* writes is not well defined anyway.
900	*
901	* It's possible that a buffered write and a direct write could collide
902	* here (the buffered write stumbles in after the dio flushes and
903	* invalidates the page cache and immediately queues writeback), but we
904	* have never supported this 100%. If either disk write succeeds the
905	* blocks will be remapped.
906	*/
907	while (end_fsb > offset_fsb && !error)
908	error = xfs_reflink_end_cow_extent(ip, &offset_fsb, end_fsb);
909
910	if (error)
911	trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_);
912	return error;
913	}
914
915	/*
916	* Free all CoW staging blocks that are still referenced by the ondisk refcount
917	* metadata. The ondisk metadata does not track which inode created the
918	* staging extent, so callers must ensure that there are no cached inodes with
919	* live CoW staging extents.
920	*/
921	int
922	xfs_reflink_recover_cow(
923	struct xfs_mount *mp)
924	{
925	struct xfs_perag *pag;
926	xfs_agnumber_t agno;
927	int error = 0;
928
929	if (!xfs_has_reflink(mp))
930	return 0;
931
932	for_each_perag(mp, agno, pag) {
933	error = xfs_refcount_recover_cow_leftovers(mp, pag);
934	if (error) {
935	xfs_perag_rele(pag);
936	break;
937	}
938	}
939
940	return error;
941	}
942
943	/*
944	* Reflinking (Block) Ranges of Two Files Together
945	*
946	* First, ensure that the reflink flag is set on both inodes. The flag is an
947	* optimization to avoid unnecessary refcount btree lookups in the write path.
948	*
949	* Now we can iteratively remap the range of extents (and holes) in src to the
950	* corresponding ranges in dest. Let drange and srange denote the ranges of
951	* logical blocks in dest and src touched by the reflink operation.
952	*
953	* While the length of drange is greater than zero,
954	* - Read src's bmbt at the start of srange ("imap")
955	* - If imap doesn't exist, make imap appear to start at the end of srange
956	* with zero length.
957	* - If imap starts before srange, advance imap to start at srange.
958	* - If imap goes beyond srange, truncate imap to end at the end of srange.
959	* - Punch (imap start - srange start + imap len) blocks from dest at
960	* offset (drange start).
961	* - If imap points to a real range of pblks,
962	* > Increase the refcount of the imap's pblks
963	* > Map imap's pblks into dest at the offset
964	* (drange start + imap start - srange start)
965	* - Advance drange and srange by (imap start - srange start + imap len)
966	*
967	* Finally, if the reflink made dest longer, update both the in-core and
968	* on-disk file sizes.
969	*
970	* ASCII Art Demonstration:
971	*
972	* Let's say we want to reflink this source file:
973	*
974	* ----SSSSSSS-SSSSS----SSSSSS (src file)
975	* <-------------------->
976	*
977	* into this destination file:
978	*
979	* --DDDDDDDDDDDDDDDDDDD--DDD (dest file)
980	* <-------------------->
981	* '-' means a hole, and 'S' and 'D' are written blocks in the src and dest.
982	* Observe that the range has different logical offsets in either file.
983	*
984	* Consider that the first extent in the source file doesn't line up with our
985	* reflink range. Unmapping and remapping are separate operations, so we can
986	* unmap more blocks from the destination file than we remap.
987	*
988	* ----SSSSSSS-SSSSS----SSSSSS
989	* <------->
990	* --DDDDD---------DDDDD--DDD
991	* <------->
992	*
993	* Now remap the source extent into the destination file:
994	*
995	* ----SSSSSSS-SSSSS----SSSSSS
996	* <------->
997	* --DDDDD--SSSSSSSDDDDD--DDD
998	* <------->
999	*
1000	* Do likewise with the second hole and extent in our range. Holes in the
1001	* unmap range don't affect our operation.
1002	*
1003	* ----SSSSSSS-SSSSS----SSSSSS
1004	* <---->
1005	* --DDDDD--SSSSSSS-SSSSS-DDD
1006	* <---->
1007	*
1008	* Finally, unmap and remap part of the third extent. This will increase the
1009	* size of the destination file.
1010	*
1011	* ----SSSSSSS-SSSSS----SSSSSS
1012	* <----->
1013	* --DDDDD--SSSSSSS-SSSSS----SSS
1014	* <----->
1015	*
1016	* Once we update the destination file's i_size, we're done.
1017	*/
1018
1019	/*
1020	* Ensure the reflink bit is set in both inodes.
1021	*/
1022	STATIC int
1023	xfs_reflink_set_inode_flag(
1024	struct xfs_inode *src,
1025	struct xfs_inode *dest)
1026	{
1027	struct xfs_mount *mp = src->i_mount;
1028	int error;
1029	struct xfs_trans *tp;
1030
1031	if (xfs_is_reflink_inode(ip: src) && xfs_is_reflink_inode(ip: dest))
1032	return 0;
1033
1034	error = xfs_trans_alloc(mp, resp: &M_RES(mp)->tr_ichange, blocks: 0, rtextents: 0, flags: 0, tpp: &tp);
1035	if (error)
1036	goto out_error;
1037
1038	/* Lock both files against IO */
1039	if (src->i_ino == dest->i_ino)
1040	xfs_ilock(src, XFS_ILOCK_EXCL);
1041	else
1042	xfs_lock_two_inodes(ip0: src, XFS_ILOCK_EXCL, ip1: dest, XFS_ILOCK_EXCL);
1043
1044	if (!xfs_is_reflink_inode(ip: src)) {
1045	trace_xfs_reflink_set_inode_flag(ip: src);
1046	xfs_trans_ijoin(tp, src, XFS_ILOCK_EXCL);
1047	src->i_diflags2 |= XFS_DIFLAG2_REFLINK;
1048	xfs_trans_log_inode(tp, src, XFS_ILOG_CORE);
1049	xfs_ifork_init_cow(src);
1050	} else
1051	xfs_iunlock(src, XFS_ILOCK_EXCL);
1052
1053	if (src->i_ino == dest->i_ino)
1054	goto commit_flags;
1055
1056	if (!xfs_is_reflink_inode(ip: dest)) {
1057	trace_xfs_reflink_set_inode_flag(ip: dest);
1058	xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL);
1059	dest->i_diflags2 |= XFS_DIFLAG2_REFLINK;
1060	xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE);
1061	xfs_ifork_init_cow(dest);
1062	} else
1063	xfs_iunlock(dest, XFS_ILOCK_EXCL);
1064
1065	commit_flags:
1066	error = xfs_trans_commit(tp);
1067	if (error)
1068	goto out_error;
1069	return error;
1070
1071	out_error:
1072	trace_xfs_reflink_set_inode_flag_error(ip: dest, error, _RET_IP_);
1073	return error;
1074	}
1075
1076	/*
1077	* Update destination inode size & cowextsize hint, if necessary.
1078	*/
1079	int
1080	xfs_reflink_update_dest(
1081	struct xfs_inode *dest,
1082	xfs_off_t newlen,
1083	xfs_extlen_t cowextsize,
1084	unsigned int remap_flags)
1085	{
1086	struct xfs_mount *mp = dest->i_mount;
1087	struct xfs_trans *tp;
1088	int error;
1089
1090	if (newlen <= i_size_read(inode: VFS_I(ip: dest)) && cowextsize == 0)
1091	return 0;
1092
1093	error = xfs_trans_alloc(mp, resp: &M_RES(mp)->tr_ichange, blocks: 0, rtextents: 0, flags: 0, tpp: &tp);
1094	if (error)
1095	goto out_error;
1096
1097	xfs_ilock(dest, XFS_ILOCK_EXCL);
1098	xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL);
1099
1100	if (newlen > i_size_read(inode: VFS_I(ip: dest))) {
1101	trace_xfs_reflink_update_inode_size(dest, newlen);
1102	i_size_write(inode: VFS_I(ip: dest), i_size: newlen);
1103	dest->i_disk_size = newlen;
1104	}
1105
1106	if (cowextsize) {
1107	dest->i_cowextsize = cowextsize;
1108	dest->i_diflags2 |= XFS_DIFLAG2_COWEXTSIZE;
1109	}
1110
1111	xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE);
1112
1113	error = xfs_trans_commit(tp);
1114	if (error)
1115	goto out_error;
1116	return error;
1117
1118	out_error:
1119	trace_xfs_reflink_update_inode_size_error(ip: dest, error, _RET_IP_);
1120	return error;
1121	}
1122
1123	/*
1124	* Do we have enough reserve in this AG to handle a reflink? The refcount
1125	* btree already reserved all the space it needs, but the rmap btree can grow
1126	* infinitely, so we won't allow more reflinks when the AG is down to the
1127	* btree reserves.
1128	*/
1129	static int
1130	xfs_reflink_ag_has_free_space(
1131	struct xfs_mount *mp,
1132	xfs_agnumber_t agno)
1133	{
1134	struct xfs_perag *pag;
1135	int error = 0;
1136
1137	if (!xfs_has_rmapbt(mp))
1138	return 0;
1139
1140	pag = xfs_perag_get(mp, agno);
1141	if (xfs_ag_resv_critical(pag, XFS_AG_RESV_RMAPBT) ||
1142	xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA))
1143	error = -ENOSPC;
1144	xfs_perag_put(pag);
1145	return error;
1146	}
1147
1148	/*
1149	* Remap the given extent into the file. The dmap blockcount will be set to
1150	* the number of blocks that were actually remapped.
1151	*/
1152	STATIC int
1153	xfs_reflink_remap_extent(
1154	struct xfs_inode *ip,
1155	struct xfs_bmbt_irec *dmap,
1156	xfs_off_t new_isize)
1157	{
1158	struct xfs_bmbt_irec smap;
1159	struct xfs_mount *mp = ip->i_mount;
1160	struct xfs_trans *tp;
1161	xfs_off_t newlen;
1162	int64_t qdelta = 0;
1163	unsigned int resblks;
1164	bool quota_reserved = true;
1165	bool smap_real;
1166	bool dmap_written = xfs_bmap_is_written_extent(dmap);
1167	int iext_delta = 0;
1168	int nimaps;
1169	int error;
1170
1171	/*
1172	* Start a rolling transaction to switch the mappings.
1173	*
1174	* Adding a written extent to the extent map can cause a bmbt split,
1175	* and removing a mapped extent from the extent can cause a bmbt split.
1176	* The two operations cannot both cause a split since they operate on
1177	* the same index in the bmap btree, so we only need a reservation for
1178	* one bmbt split if either thing is happening. However, we haven't
1179	* locked the inode yet, so we reserve assuming this is the case.
1180	*
1181	* The first allocation call tries to reserve enough space to handle
1182	* mapping dmap into a sparse part of the file plus the bmbt split. We
1183	* haven't locked the inode or read the existing mapping yet, so we do
1184	* not know for sure that we need the space. This should succeed most
1185	* of the time.
1186	*
1187	* If the first attempt fails, try again but reserving only enough
1188	* space to handle a bmbt split. This is the hard minimum requirement,
1189	* and we revisit quota reservations later when we know more about what
1190	* we're remapping.
1191	*/
1192	resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK);
1193	error = xfs_trans_alloc_inode(ip, resv: &M_RES(mp)->tr_write,
1194	dblocks: resblks + dmap->br_blockcount, rblocks: 0, force: false, tpp: &tp);
1195	if (error == -EDQUOT || error == -ENOSPC) {
1196	quota_reserved = false;
1197	error = xfs_trans_alloc_inode(ip, resv: &M_RES(mp)->tr_write,
1198	dblocks: resblks, rblocks: 0, force: false, tpp: &tp);
1199	}
1200	if (error)
1201	goto out;
1202
1203	/*
1204	* Read what's currently mapped in the destination file into smap.
1205	* If smap isn't a hole, we will have to remove it before we can add
1206	* dmap to the destination file.
1207	*/
1208	nimaps = 1;
1209	error = xfs_bmapi_read(ip, dmap->br_startoff, dmap->br_blockcount,
1210	&smap, &nimaps, 0);
1211	if (error)
1212	goto out_cancel;
1213	ASSERT(nimaps == 1 && smap.br_startoff == dmap->br_startoff);
1214	smap_real = xfs_bmap_is_real_extent(&smap);
1215
1216	/*
1217	* We can only remap as many blocks as the smaller of the two extent
1218	* maps, because we can only remap one extent at a time.
1219	*/
1220	dmap->br_blockcount = min(dmap->br_blockcount, smap.br_blockcount);
1221	ASSERT(dmap->br_blockcount == smap.br_blockcount);
1222
1223	trace_xfs_reflink_remap_extent_dest(ip, irec: &smap);
1224
1225	/*
1226	* Two extents mapped to the same physical block must not have
1227	* different states; that's filesystem corruption. Move on to the next
1228	* extent if they're both holes or both the same physical extent.
1229	*/
1230	if (dmap->br_startblock == smap.br_startblock) {
1231	if (dmap->br_state != smap.br_state) {
1232	xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
1233	error = -EFSCORRUPTED;
1234	}
1235	goto out_cancel;
1236	}
1237
1238	/* If both extents are unwritten, leave them alone. */
1239	if (dmap->br_state == XFS_EXT_UNWRITTEN &&
1240	smap.br_state == XFS_EXT_UNWRITTEN)
1241	goto out_cancel;
1242
1243	/* No reflinking if the AG of the dest mapping is low on space. */
1244	if (dmap_written) {
1245	error = xfs_reflink_ag_has_free_space(mp,
1246	XFS_FSB_TO_AGNO(mp, dmap->br_startblock));
1247	if (error)
1248	goto out_cancel;
1249	}
1250
1251	/*
1252	* Increase quota reservation if we think the quota block counter for
1253	* this file could increase.
1254	*
1255	* If we are mapping a written extent into the file, we need to have
1256	* enough quota block count reservation to handle the blocks in that
1257	* extent. We log only the delta to the quota block counts, so if the
1258	* extent we're unmapping also has blocks allocated to it, we don't
1259	* need a quota reservation for the extent itself.
1260	*
1261	* Note that if we're replacing a delalloc reservation with a written
1262	* extent, we have to take the full quota reservation because removing
1263	* the delalloc reservation gives the block count back to the quota
1264	* count. This is suboptimal, but the VFS flushed the dest range
1265	* before we started. That should have removed all the delalloc
1266	* reservations, but we code defensively.
1267	*
1268	* xfs_trans_alloc_inode above already tried to grab an even larger
1269	* quota reservation, and kicked off a blockgc scan if it couldn't.
1270	* If we can't get a potentially smaller quota reservation now, we're
1271	* done.
1272	*/
1273	if (!quota_reserved && !smap_real && dmap_written) {
1274	error = xfs_trans_reserve_quota_nblks(tp, ip,
1275	dblocks: dmap->br_blockcount, rblocks: 0, force: false);
1276	if (error)
1277	goto out_cancel;
1278	}
1279
1280	if (smap_real)
1281	++iext_delta;
1282
1283	if (dmap_written)
1284	++iext_delta;
1285
1286	error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, iext_delta);
1287	if (error == -EFBIG)
1288	error = xfs_iext_count_upgrade(tp, ip, iext_delta);
1289	if (error)
1290	goto out_cancel;
1291
1292	if (smap_real) {
1293	/*
1294	* If the extent we're unmapping is backed by storage (written
1295	* or not), unmap the extent and drop its refcount.
1296	*/
1297	xfs_bmap_unmap_extent(tp, ip, XFS_DATA_FORK, &smap);
1298	xfs_refcount_decrease_extent(tp, &smap);
1299	qdelta -= smap.br_blockcount;
1300	} else if (smap.br_startblock == DELAYSTARTBLOCK) {
1301	int done;
1302
1303	/*
1304	* If the extent we're unmapping is a delalloc reservation,
1305	* we can use the regular bunmapi function to release the
1306	* incore state. Dropping the delalloc reservation takes care
1307	* of the quota reservation for us.
1308	*/
1309	error = xfs_bunmapi(NULL, ip, smap.br_startoff,
1310	smap.br_blockcount, 0, 1, &done);
1311	if (error)
1312	goto out_cancel;
1313	ASSERT(done);
1314	}
1315
1316	/*
1317	* If the extent we're sharing is backed by written storage, increase
1318	* its refcount and map it into the file.
1319	*/
1320	if (dmap_written) {
1321	xfs_refcount_increase_extent(tp, dmap);
1322	xfs_bmap_map_extent(tp, ip, XFS_DATA_FORK, dmap);
1323	qdelta += dmap->br_blockcount;
1324	}
1325
1326	xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, qdelta);
1327
1328	/* Update dest isize if needed. */
1329	newlen = XFS_FSB_TO_B(mp, dmap->br_startoff + dmap->br_blockcount);
1330	newlen = min_t(xfs_off_t, newlen, new_isize);
1331	if (newlen > i_size_read(inode: VFS_I(ip))) {
1332	trace_xfs_reflink_update_inode_size(ip, newlen);
1333	i_size_write(inode: VFS_I(ip), i_size: newlen);
1334	ip->i_disk_size = newlen;
1335	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
1336	}
1337
1338	/* Commit everything and unlock. */
1339	error = xfs_trans_commit(tp);
1340	goto out_unlock;
1341
1342	out_cancel:
1343	xfs_trans_cancel(tp);
1344	out_unlock:
1345	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1346	out:
1347	if (error)
1348	trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_);
1349	return error;
1350	}
1351
1352	/* Remap a range of one file to the other. */
1353	int
1354	xfs_reflink_remap_blocks(
1355	struct xfs_inode *src,
1356	loff_t pos_in,
1357	struct xfs_inode *dest,
1358	loff_t pos_out,
1359	loff_t remap_len,
1360	loff_t *remapped)
1361	{
1362	struct xfs_bmbt_irec imap;
1363	struct xfs_mount *mp = src->i_mount;
1364	xfs_fileoff_t srcoff = XFS_B_TO_FSBT(mp, pos_in);
1365	xfs_fileoff_t destoff = XFS_B_TO_FSBT(mp, pos_out);
1366	xfs_filblks_t len;
1367	xfs_filblks_t remapped_len = 0;
1368	xfs_off_t new_isize = pos_out + remap_len;
1369	int nimaps;
1370	int error = 0;
1371
1372	len = min_t(xfs_filblks_t, XFS_B_TO_FSB(mp, remap_len),
1373	XFS_MAX_FILEOFF);
1374
1375	trace_xfs_reflink_remap_blocks(src, srcoff, len, dest, destoff);
1376
1377	while (len > 0) {
1378	unsigned int lock_mode;
1379
1380	/* Read extent from the source file */
1381	nimaps = 1;
1382	lock_mode = xfs_ilock_data_map_shared(src);
1383	error = xfs_bmapi_read(src, srcoff, len, &imap, &nimaps, 0);
1384	xfs_iunlock(src, lock_mode);
1385	if (error)
1386	break;
1387	/*
1388	* The caller supposedly flushed all dirty pages in the source
1389	* file range, which means that writeback should have allocated
1390	* or deleted all delalloc reservations in that range. If we
1391	* find one, that's a good sign that something is seriously
1392	* wrong here.
1393	*/
1394	ASSERT(nimaps == 1 && imap.br_startoff == srcoff);
1395	if (imap.br_startblock == DELAYSTARTBLOCK) {
1396	ASSERT(imap.br_startblock != DELAYSTARTBLOCK);
1397	xfs_bmap_mark_sick(src, XFS_DATA_FORK);
1398	error = -EFSCORRUPTED;
1399	break;
1400	}
1401
1402	trace_xfs_reflink_remap_extent_src(ip: src, irec: &imap);
1403
1404	/* Remap into the destination file at the given offset. */
1405	imap.br_startoff = destoff;
1406	error = xfs_reflink_remap_extent(ip: dest, dmap: &imap, new_isize);
1407	if (error)
1408	break;
1409
1410	if (fatal_signal_pending(current)) {
1411	error = -EINTR;
1412	break;
1413	}
1414
1415	/* Advance drange/srange */
1416	srcoff += imap.br_blockcount;
1417	destoff += imap.br_blockcount;
1418	len -= imap.br_blockcount;
1419	remapped_len += imap.br_blockcount;
1420	}
1421
1422	if (error)
1423	trace_xfs_reflink_remap_blocks_error(ip: dest, error, _RET_IP_);
1424	*remapped = min_t(loff_t, remap_len,
1425	XFS_FSB_TO_B(src->i_mount, remapped_len));
1426	return error;
1427	}
1428
1429	/*
1430	* If we're reflinking to a point past the destination file's EOF, we must
1431	* zero any speculative post-EOF preallocations that sit between the old EOF
1432	* and the destination file offset.
1433	*/
1434	static int
1435	xfs_reflink_zero_posteof(
1436	struct xfs_inode *ip,
1437	loff_t pos)
1438	{
1439	loff_t isize = i_size_read(inode: VFS_I(ip));
1440
1441	if (pos <= isize)
1442	return 0;
1443
1444	trace_xfs_zero_eof(ip, offset: isize, count: pos - isize);
1445	return xfs_zero_range(ip, pos: isize, len: pos - isize, NULL);
1446	}
1447
1448	/*
1449	* Prepare two files for range cloning. Upon a successful return both inodes
1450	* will have the iolock and mmaplock held, the page cache of the out file will
1451	* be truncated, and any leases on the out file will have been broken. This
1452	* function borrows heavily from xfs_file_aio_write_checks.
1453	*
1454	* The VFS allows partial EOF blocks to "match" for dedupe even though it hasn't
1455	* checked that the bytes beyond EOF physically match. Hence we cannot use the
1456	* EOF block in the source dedupe range because it's not a complete block match,
1457	* hence can introduce a corruption into the file that has it's block replaced.
1458	*
1459	* In similar fashion, the VFS file cloning also allows partial EOF blocks to be
1460	* "block aligned" for the purposes of cloning entire files. However, if the
1461	* source file range includes the EOF block and it lands within the existing EOF
1462	* of the destination file, then we can expose stale data from beyond the source
1463	* file EOF in the destination file.
1464	*
1465	* XFS doesn't support partial block sharing, so in both cases we have check
1466	* these cases ourselves. For dedupe, we can simply round the length to dedupe
1467	* down to the previous whole block and ignore the partial EOF block. While this
1468	* means we can't dedupe the last block of a file, this is an acceptible
1469	* tradeoff for simplicity on implementation.
1470	*
1471	* For cloning, we want to share the partial EOF block if it is also the new EOF
1472	* block of the destination file. If the partial EOF block lies inside the
1473	* existing destination EOF, then we have to abort the clone to avoid exposing
1474	* stale data in the destination file. Hence we reject these clone attempts with
1475	* -EINVAL in this case.
1476	*/
1477	int
1478	xfs_reflink_remap_prep(
1479	struct file *file_in,
1480	loff_t pos_in,
1481	struct file *file_out,
1482	loff_t pos_out,
1483	loff_t *len,
1484	unsigned int remap_flags)
1485	{
1486	struct inode *inode_in = file_inode(f: file_in);
1487	struct xfs_inode *src = XFS_I(inode: inode_in);
1488	struct inode *inode_out = file_inode(f: file_out);
1489	struct xfs_inode *dest = XFS_I(inode: inode_out);
1490	int ret;
1491
1492	/* Lock both files against IO */
1493	ret = xfs_ilock2_io_mmap(ip1: src, ip2: dest);
1494	if (ret)
1495	return ret;
1496
1497	/* Check file eligibility and prepare for block sharing. */
1498	ret = -EINVAL;
1499	/* Don't reflink realtime inodes */
1500	if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest))
1501	goto out_unlock;
1502
1503	/* Don't share DAX file data with non-DAX file. */
1504	if (IS_DAX(inode_in) != IS_DAX(inode_out))
1505	goto out_unlock;
1506
1507	if (!IS_DAX(inode_in))
1508	ret = generic_remap_file_range_prep(file_in, pos_in, file_out,
1509	pos_out, count: len, remap_flags);
1510	else
1511	ret = dax_remap_file_range_prep(file_in, pos_in, file_out,
1512	pos_out, len, remap_flags, ops: &xfs_read_iomap_ops);
1513	if (ret || *len == 0)
1514	goto out_unlock;
1515
1516	/* Attach dquots to dest inode before changing block map */
1517	ret = xfs_qm_dqattach(dest);
1518	if (ret)
1519	goto out_unlock;
1520
1521	/*
1522	* Zero existing post-eof speculative preallocations in the destination
1523	* file.
1524	*/
1525	ret = xfs_reflink_zero_posteof(ip: dest, pos: pos_out);
1526	if (ret)
1527	goto out_unlock;
1528
1529	/* Set flags and remap blocks. */
1530	ret = xfs_reflink_set_inode_flag(src, dest);
1531	if (ret)
1532	goto out_unlock;
1533
1534	/*
1535	* If pos_out > EOF, we may have dirtied blocks between EOF and
1536	* pos_out. In that case, we need to extend the flush and unmap to cover
1537	* from EOF to the end of the copy length.
1538	*/
1539	if (pos_out > XFS_ISIZE(dest)) {
1540	loff_t flen = *len + (pos_out - XFS_ISIZE(dest));
1541	ret = xfs_flush_unmap_range(ip: dest, offset: XFS_ISIZE(dest), len: flen);
1542	} else {
1543	ret = xfs_flush_unmap_range(ip: dest, offset: pos_out, len: *len);
1544	}
1545	if (ret)
1546	goto out_unlock;
1547
1548	xfs_iflags_set(ip: src, XFS_IREMAPPING);
1549	if (inode_in != inode_out)
1550	xfs_ilock_demote(src, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL);
1551
1552	return 0;
1553	out_unlock:
1554	xfs_iunlock2_io_mmap(ip1: src, ip2: dest);
1555	return ret;
1556	}
1557
1558	/* Does this inode need the reflink flag? */
1559	int
1560	xfs_reflink_inode_has_shared_extents(
1561	struct xfs_trans *tp,
1562	struct xfs_inode *ip,
1563	bool *has_shared)
1564	{
1565	struct xfs_bmbt_irec got;
1566	struct xfs_mount *mp = ip->i_mount;
1567	struct xfs_ifork *ifp;
1568	struct xfs_iext_cursor icur;
1569	bool found;
1570	int error;
1571
1572	ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK);
1573	error = xfs_iread_extents(tp, ip, XFS_DATA_FORK);
1574	if (error)
1575	return error;
1576
1577	*has_shared = false;
1578	found = xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got);
1579	while (found) {
1580	struct xfs_perag *pag;
1581	xfs_agblock_t agbno;
1582	xfs_extlen_t aglen;
1583	xfs_agblock_t rbno;
1584	xfs_extlen_t rlen;
1585
1586	if (isnullstartblock(got.br_startblock) ||
1587	got.br_state != XFS_EXT_NORM)
1588	goto next;
1589
1590	pag = xfs_perag_get(mp, XFS_FSB_TO_AGNO(mp, got.br_startblock));
1591	agbno = XFS_FSB_TO_AGBNO(mp, got.br_startblock);
1592	aglen = got.br_blockcount;
1593	error = xfs_reflink_find_shared(pag, tp, agbno, aglen,
1594	&rbno, &rlen, false);
1595	xfs_perag_put(pag);
1596	if (error)
1597	return error;
1598
1599	/* Is there still a shared block here? */
1600	if (rbno != NULLAGBLOCK) {
1601	*has_shared = true;
1602	return 0;
1603	}
1604	next:
1605	found = xfs_iext_next_extent(ifp, &icur, &got);
1606	}
1607
1608	return 0;
1609	}
1610
1611	/*
1612	* Clear the inode reflink flag if there are no shared extents.
1613	*
1614	* The caller is responsible for joining the inode to the transaction passed in.
1615	* The inode will be joined to the transaction that is returned to the caller.
1616	*/
1617	int
1618	xfs_reflink_clear_inode_flag(
1619	struct xfs_inode *ip,
1620	struct xfs_trans **tpp)
1621	{
1622	bool needs_flag;
1623	int error = 0;
1624
1625	ASSERT(xfs_is_reflink_inode(ip));
1626
1627	error = xfs_reflink_inode_has_shared_extents(tp: *tpp, ip, has_shared: &needs_flag);
1628	if (error || needs_flag)
1629	return error;
1630
1631	/*
1632	* We didn't find any shared blocks so turn off the reflink flag.
1633	* First, get rid of any leftover CoW mappings.
1634	*/
1635	error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, XFS_MAX_FILEOFF,
1636	true);
1637	if (error)
1638	return error;
1639
1640	/* Clear the inode flag. */
1641	trace_xfs_reflink_unset_inode_flag(ip);
1642	ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK;
1643	xfs_inode_clear_cowblocks_tag(ip);
1644	xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE);
1645
1646	return error;
1647	}
1648
1649	/*
1650	* Clear the inode reflink flag if there are no shared extents and the size
1651	* hasn't changed.
1652	*/
1653	STATIC int
1654	xfs_reflink_try_clear_inode_flag(
1655	struct xfs_inode *ip)
1656	{
1657	struct xfs_mount *mp = ip->i_mount;
1658	struct xfs_trans *tp;
1659	int error = 0;
1660
1661	/* Start a rolling transaction to remove the mappings */
1662	error = xfs_trans_alloc(mp, resp: &M_RES(mp)->tr_write, blocks: 0, rtextents: 0, flags: 0, tpp: &tp);
1663	if (error)
1664	return error;
1665
1666	xfs_ilock(ip, XFS_ILOCK_EXCL);
1667	xfs_trans_ijoin(tp, ip, 0);
1668
1669	error = xfs_reflink_clear_inode_flag(ip, tpp: &tp);
1670	if (error)
1671	goto cancel;
1672
1673	error = xfs_trans_commit(tp);
1674	if (error)
1675	goto out;
1676
1677	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1678	return 0;
1679	cancel:
1680	xfs_trans_cancel(tp);
1681	out:
1682	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1683	return error;
1684	}
1685
1686	/*
1687	* Pre-COW all shared blocks within a given byte range of a file and turn off
1688	* the reflink flag if we unshare all of the file's blocks.
1689	*/
1690	int
1691	xfs_reflink_unshare(
1692	struct xfs_inode *ip,
1693	xfs_off_t offset,
1694	xfs_off_t len)
1695	{
1696	struct inode *inode = VFS_I(ip);
1697	int error;
1698
1699	if (!xfs_is_reflink_inode(ip))
1700	return 0;
1701
1702	trace_xfs_reflink_unshare(ip, offset, count: len);
1703
1704	inode_dio_wait(inode);
1705
1706	if (IS_DAX(inode))
1707	error = dax_file_unshare(inode, pos: offset, len,
1708	ops: &xfs_dax_write_iomap_ops);
1709	else
1710	error = iomap_file_unshare(inode, pos: offset, len,
1711	ops: &xfs_buffered_write_iomap_ops);
1712	if (error)
1713	goto out;
1714
1715	error = filemap_write_and_wait_range(mapping: inode->i_mapping, lstart: offset,
1716	lend: offset + len - 1);
1717	if (error)
1718	goto out;
1719
1720	/* Turn off the reflink flag if possible. */
1721	error = xfs_reflink_try_clear_inode_flag(ip);
1722	if (error)
1723	goto out;
1724	return 0;
1725
1726	out:
1727	trace_xfs_reflink_unshare_error(ip, error, _RET_IP_);
1728	return error;
1729	}
1730