1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (c) 2000-2006 Silicon Graphics, Inc.
4	* Copyright (c) 2016-2018 Christoph Hellwig.
5	* All Rights Reserved.
6	*/
7	#include "xfs.h"
8	#include "xfs_fs.h"
9	#include "xfs_shared.h"
10	#include "xfs_format.h"
11	#include "xfs_log_format.h"
12	#include "xfs_trans_resv.h"
13	#include "xfs_mount.h"
14	#include "xfs_inode.h"
15	#include "xfs_btree.h"
16	#include "xfs_bmap_btree.h"
17	#include "xfs_bmap.h"
18	#include "xfs_bmap_util.h"
19	#include "xfs_errortag.h"
20	#include "xfs_error.h"
21	#include "xfs_trans.h"
22	#include "xfs_trans_space.h"
23	#include "xfs_inode_item.h"
24	#include "xfs_iomap.h"
25	#include "xfs_trace.h"
26	#include "xfs_quota.h"
27	#include "xfs_rtgroup.h"
28	#include "xfs_dquot_item.h"
29	#include "xfs_dquot.h"
30	#include "xfs_reflink.h"
31	#include "xfs_health.h"
32	#include "xfs_rtbitmap.h"
33	#include "xfs_icache.h"
34	#include "xfs_zone_alloc.h"
35
36	#define XFS_ALLOC_ALIGN(mp, off) \
37	(((off) >> mp->m_allocsize_log) << mp->m_allocsize_log)
38
39	static int
40	xfs_alert_fsblock_zero(
41	xfs_inode_t *ip,
42	xfs_bmbt_irec_t *imap)
43	{
44	xfs_alert_tag(ip->i_mount, XFS_PTAG_FSBLOCK_ZERO,
45	"Access to block zero in inode %llu "
46	"start_block: %llx start_off: %llx "
47	"blkcnt: %llx extent-state: %x",
48	(unsigned long long)ip->i_ino,
49	(unsigned long long)imap->br_startblock,
50	(unsigned long long)imap->br_startoff,
51	(unsigned long long)imap->br_blockcount,
52	imap->br_state);
53	xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
54	return -EFSCORRUPTED;
55	}
56
57	u64
58	xfs_iomap_inode_sequence(
59	struct xfs_inode *ip,
60	u16 iomap_flags)
61	{
62	u64 cookie = 0;
63
64	if (iomap_flags & IOMAP_F_XATTR)
65	return READ_ONCE(ip->i_af.if_seq);
66	if ((iomap_flags & IOMAP_F_SHARED) && ip->i_cowfp)
67	cookie = (u64)READ_ONCE(ip->i_cowfp->if_seq) << 32;
68	return cookie | READ_ONCE(ip->i_df.if_seq);
69	}
70
71	/*
72	* Check that the iomap passed to us is still valid for the given offset and
73	* length.
74	*/
75	static bool
76	xfs_iomap_valid(
77	struct inode *inode,
78	const struct iomap *iomap)
79	{
80	struct xfs_inode *ip = XFS_I(inode);
81
82	if (iomap->type == IOMAP_HOLE)
83	return true;
84
85	if (iomap->validity_cookie !=
86	xfs_iomap_inode_sequence(ip, iomap_flags: iomap->flags)) {
87	trace_xfs_iomap_invalid(ip, iomap);
88	return false;
89	}
90
91	XFS_ERRORTAG_DELAY(ip->i_mount, XFS_ERRTAG_WRITE_DELAY_MS);
92	return true;
93	}
94
95	const struct iomap_write_ops xfs_iomap_write_ops = {
96	.iomap_valid = xfs_iomap_valid,
97	};
98
99	int
100	xfs_bmbt_to_iomap(
101	struct xfs_inode *ip,
102	struct iomap *iomap,
103	struct xfs_bmbt_irec *imap,
104	unsigned int mapping_flags,
105	u16 iomap_flags,
106	u64 sequence_cookie)
107	{
108	struct xfs_mount *mp = ip->i_mount;
109	struct xfs_buftarg *target = xfs_inode_buftarg(ip);
110
111	if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) {
112	xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
113	return xfs_alert_fsblock_zero(ip, imap);
114	}
115
116	if (imap->br_startblock == HOLESTARTBLOCK) {
117	iomap->addr = IOMAP_NULL_ADDR;
118	iomap->type = IOMAP_HOLE;
119	} else if (imap->br_startblock == DELAYSTARTBLOCK ||
120	isnullstartblock(imap->br_startblock)) {
121	iomap->addr = IOMAP_NULL_ADDR;
122	iomap->type = IOMAP_DELALLOC;
123	} else {
124	xfs_daddr_t daddr = xfs_fsb_to_db(ip, imap->br_startblock);
125
126	iomap->addr = BBTOB(daddr);
127	if (mapping_flags & IOMAP_DAX)
128	iomap->addr += target->bt_dax_part_off;
129
130	if (imap->br_state == XFS_EXT_UNWRITTEN)
131	iomap->type = IOMAP_UNWRITTEN;
132	else
133	iomap->type = IOMAP_MAPPED;
134
135	/*
136	* Mark iomaps starting at the first sector of a RTG as merge
137	* boundary so that each I/O completions is contained to a
138	* single RTG.
139	*/
140	if (XFS_IS_REALTIME_INODE(ip) && xfs_has_rtgroups(mp) &&
141	xfs_rtbno_is_group_start(mp, imap->br_startblock))
142	iomap->flags |= IOMAP_F_BOUNDARY;
143	}
144	iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff);
145	iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount);
146	if (mapping_flags & IOMAP_DAX)
147	iomap->dax_dev = target->bt_daxdev;
148	else
149	iomap->bdev = target->bt_bdev;
150	iomap->flags = iomap_flags;
151
152	/*
153	* If the inode is dirty for datasync purposes, let iomap know so it
154	* doesn't elide the IO completion journal flushes on O_DSYNC IO.
155	*/
156	if (ip->i_itemp) {
157	struct xfs_inode_log_item *iip = ip->i_itemp;
158
159	spin_lock(lock: &iip->ili_lock);
160	if (iip->ili_datasync_seq)
161	iomap->flags |= IOMAP_F_DIRTY;
162	spin_unlock(lock: &iip->ili_lock);
163	}
164
165	iomap->validity_cookie = sequence_cookie;
166	return 0;
167	}
168
169	static void
170	xfs_hole_to_iomap(
171	struct xfs_inode *ip,
172	struct iomap *iomap,
173	xfs_fileoff_t offset_fsb,
174	xfs_fileoff_t end_fsb)
175	{
176	struct xfs_buftarg *target = xfs_inode_buftarg(ip);
177
178	iomap->addr = IOMAP_NULL_ADDR;
179	iomap->type = IOMAP_HOLE;
180	iomap->offset = XFS_FSB_TO_B(ip->i_mount, offset_fsb);
181	iomap->length = XFS_FSB_TO_B(ip->i_mount, end_fsb - offset_fsb);
182	iomap->bdev = target->bt_bdev;
183	iomap->dax_dev = target->bt_daxdev;
184	}
185
186	static inline xfs_fileoff_t
187	xfs_iomap_end_fsb(
188	struct xfs_mount *mp,
189	loff_t offset,
190	loff_t count)
191	{
192	ASSERT(offset <= mp->m_super->s_maxbytes);
193	return min(XFS_B_TO_FSB(mp, offset + count),
194	XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
195	}
196
197	static xfs_extlen_t
198	xfs_eof_alignment(
199	struct xfs_inode *ip)
200	{
201	struct xfs_mount *mp = ip->i_mount;
202	xfs_extlen_t align = 0;
203
204	if (!XFS_IS_REALTIME_INODE(ip)) {
205	/*
206	* Round up the allocation request to a stripe unit
207	* (m_dalign) boundary if the file size is >= stripe unit
208	* size, and we are allocating past the allocation eof.
209	*
210	* If mounted with the "-o swalloc" option the alignment is
211	* increased from the strip unit size to the stripe width.
212	*/
213	if (mp->m_swidth && xfs_has_swalloc(mp))
214	align = mp->m_swidth;
215	else if (mp->m_dalign)
216	align = mp->m_dalign;
217
218	if (align && XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, align))
219	align = 0;
220	}
221
222	return align;
223	}
224
225	/*
226	* Check if last_fsb is outside the last extent, and if so grow it to the next
227	* stripe unit boundary.
228	*/
229	xfs_fileoff_t
230	xfs_iomap_eof_align_last_fsb(
231	struct xfs_inode *ip,
232	xfs_fileoff_t end_fsb)
233	{
234	struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK);
235	xfs_extlen_t extsz = xfs_get_extsz_hint(ip);
236	xfs_extlen_t align = xfs_eof_alignment(ip);
237	struct xfs_bmbt_irec irec;
238	struct xfs_iext_cursor icur;
239
240	ASSERT(!xfs_need_iread_extents(ifp));
241
242	/*
243	* Always round up the allocation request to the extent hint boundary.
244	*/
245	if (extsz) {
246	if (align)
247	align = roundup_64(align, extsz);
248	else
249	align = extsz;
250	}
251
252	if (align) {
253	xfs_fileoff_t aligned_end_fsb = roundup_64(end_fsb, align);
254
255	xfs_iext_last(ifp, &icur);
256	if (!xfs_iext_get_extent(ifp, &icur, &irec) ||
257	aligned_end_fsb >= irec.br_startoff + irec.br_blockcount)
258	return aligned_end_fsb;
259	}
260
261	return end_fsb;
262	}
263
264	int
265	xfs_iomap_write_direct(
266	struct xfs_inode *ip,
267	xfs_fileoff_t offset_fsb,
268	xfs_fileoff_t count_fsb,
269	unsigned int flags,
270	struct xfs_bmbt_irec *imap,
271	u64 *seq)
272	{
273	struct xfs_mount *mp = ip->i_mount;
274	struct xfs_trans *tp;
275	xfs_filblks_t resaligned;
276	int nimaps;
277	unsigned int dblocks, rblocks;
278	bool force = false;
279	int error;
280	int bmapi_flags = XFS_BMAPI_PREALLOC;
281	int nr_exts = XFS_IEXT_ADD_NOSPLIT_CNT;
282
283	ASSERT(count_fsb > 0);
284
285	resaligned = xfs_aligned_fsb_count(offset_fsb, count_fsb,
286	xfs_get_extsz_hint(ip));
287	if (unlikely(XFS_IS_REALTIME_INODE(ip))) {
288	dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0);
289	rblocks = resaligned;
290	} else {
291	dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
292	rblocks = 0;
293	}
294
295	error = xfs_qm_dqattach(ip);
296	if (error)
297	return error;
298
299	/*
300	* For DAX, we do not allocate unwritten extents, but instead we zero
301	* the block before we commit the transaction. Ideally we'd like to do
302	* this outside the transaction context, but if we commit and then crash
303	* we may not have zeroed the blocks and this will be exposed on
304	* recovery of the allocation. Hence we must zero before commit.
305	*
306	* Further, if we are mapping unwritten extents here, we need to zero
307	* and convert them to written so that we don't need an unwritten extent
308	* callback for DAX. This also means that we need to be able to dip into
309	* the reserve block pool for bmbt block allocation if there is no space
310	* left but we need to do unwritten extent conversion.
311	*/
312	if (flags & IOMAP_DAX) {
313	bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO;
314	if (imap->br_state == XFS_EXT_UNWRITTEN) {
315	force = true;
316	nr_exts = XFS_IEXT_WRITE_UNWRITTEN_CNT;
317	dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
318	}
319	}
320
321	error = xfs_trans_alloc_inode(ip, resv: &M_RES(mp)->tr_write, dblocks,
322	rblocks, force, tpp: &tp);
323	if (error)
324	return error;
325
326	error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, nr_exts);
327	if (error)
328	goto out_trans_cancel;
329
330	/*
331	* From this point onwards we overwrite the imap pointer that the
332	* caller gave to us.
333	*/
334	nimaps = 1;
335	error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, bmapi_flags, 0,
336	imap, &nimaps);
337	if (error)
338	goto out_trans_cancel;
339
340	/*
341	* Complete the transaction
342	*/
343	error = xfs_trans_commit(tp);
344	if (error)
345	goto out_unlock;
346
347	if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) {
348	xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
349	error = xfs_alert_fsblock_zero(ip, imap);
350	}
351
352	out_unlock:
353	*seq = xfs_iomap_inode_sequence(ip, iomap_flags: 0);
354	xfs_iunlock(ip, XFS_ILOCK_EXCL);
355	return error;
356
357	out_trans_cancel:
358	xfs_trans_cancel(tp);
359	goto out_unlock;
360	}
361
362	STATIC bool
363	xfs_quota_need_throttle(
364	struct xfs_inode *ip,
365	xfs_dqtype_t type,
366	xfs_fsblock_t alloc_blocks)
367	{
368	struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
369	struct xfs_dquot_res *res;
370	struct xfs_dquot_pre *pre;
371
372	if (!dq || !xfs_this_quota_on(ip->i_mount, type))
373	return false;
374
375	if (XFS_IS_REALTIME_INODE(ip)) {
376	res = &dq->q_rtb;
377	pre = &dq->q_rtb_prealloc;
378	} else {
379	res = &dq->q_blk;
380	pre = &dq->q_blk_prealloc;
381	}
382
383	/* no hi watermark, no throttle */
384	if (!pre->q_prealloc_hi_wmark)
385	return false;
386
387	/* under the lo watermark, no throttle */
388	if (res->reserved + alloc_blocks < pre->q_prealloc_lo_wmark)
389	return false;
390
391	return true;
392	}
393
394	STATIC void
395	xfs_quota_calc_throttle(
396	struct xfs_inode *ip,
397	xfs_dqtype_t type,
398	xfs_fsblock_t *qblocks,
399	int *qshift,
400	int64_t *qfreesp)
401	{
402	struct xfs_dquot *dq = xfs_inode_dquot(ip, type);
403	struct xfs_dquot_res *res;
404	struct xfs_dquot_pre *pre;
405	int64_t freesp;
406	int shift = 0;
407
408	if (!dq) {
409	res = NULL;
410	pre = NULL;
411	} else if (XFS_IS_REALTIME_INODE(ip)) {
412	res = &dq->q_rtb;
413	pre = &dq->q_rtb_prealloc;
414	} else {
415	res = &dq->q_blk;
416	pre = &dq->q_blk_prealloc;
417	}
418
419	/* no dq, or over hi wmark, squash the prealloc completely */
420	if (!res || res->reserved >= pre->q_prealloc_hi_wmark) {
421	*qblocks = 0;
422	*qfreesp = 0;
423	return;
424	}
425
426	freesp = pre->q_prealloc_hi_wmark - res->reserved;
427	if (freesp < pre->q_low_space[XFS_QLOWSP_5_PCNT]) {
428	shift = 2;
429	if (freesp < pre->q_low_space[XFS_QLOWSP_3_PCNT])
430	shift += 2;
431	if (freesp < pre->q_low_space[XFS_QLOWSP_1_PCNT])
432	shift += 2;
433	}
434
435	if (freesp < *qfreesp)
436	*qfreesp = freesp;
437
438	/* only overwrite the throttle values if we are more aggressive */
439	if ((freesp >> shift) < (*qblocks >> *qshift)) {
440	*qblocks = freesp;
441	*qshift = shift;
442	}
443	}
444
445	static int64_t
446	xfs_iomap_freesp(
447	struct xfs_mount *mp,
448	unsigned int idx,
449	uint64_t low_space[XFS_LOWSP_MAX],
450	int *shift)
451	{
452	int64_t freesp;
453
454	freesp = xfs_estimate_freecounter(mp, idx);
455	if (freesp < low_space[XFS_LOWSP_5_PCNT]) {
456	*shift = 2;
457	if (freesp < low_space[XFS_LOWSP_4_PCNT])
458	(*shift)++;
459	if (freesp < low_space[XFS_LOWSP_3_PCNT])
460	(*shift)++;
461	if (freesp < low_space[XFS_LOWSP_2_PCNT])
462	(*shift)++;
463	if (freesp < low_space[XFS_LOWSP_1_PCNT])
464	(*shift)++;
465	}
466	return freesp;
467	}
468
469	/*
470	* If we don't have a user specified preallocation size, dynamically increase
471	* the preallocation size as the size of the file grows. Cap the maximum size
472	* at a single extent or less if the filesystem is near full. The closer the
473	* filesystem is to being full, the smaller the maximum preallocation.
474	*/
475	STATIC xfs_fsblock_t
476	xfs_iomap_prealloc_size(
477	struct xfs_inode *ip,
478	int whichfork,
479	loff_t offset,
480	loff_t count,
481	struct xfs_iext_cursor *icur)
482	{
483	struct xfs_iext_cursor ncur = *icur;
484	struct xfs_bmbt_irec prev, got;
485	struct xfs_mount *mp = ip->i_mount;
486	struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork);
487	xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
488	int64_t freesp;
489	xfs_fsblock_t qblocks;
490	xfs_fsblock_t alloc_blocks = 0;
491	xfs_extlen_t plen;
492	int shift = 0;
493	int qshift = 0;
494
495	/*
496	* As an exception we don't do any preallocation at all if the file is
497	* smaller than the minimum preallocation and we are using the default
498	* dynamic preallocation scheme, as it is likely this is the only write
499	* to the file that is going to be done.
500	*/
501	if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_allocsize_blocks))
502	return 0;
503
504	/*
505	* Use the minimum preallocation size for small files or if we are
506	* writing right after a hole.
507	*/
508	if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) ||
509	!xfs_iext_prev_extent(ifp, &ncur, &prev) ||
510	prev.br_startoff + prev.br_blockcount < offset_fsb)
511	return mp->m_allocsize_blocks;
512
513	/*
514	* Take the size of the preceding data extents as the basis for the
515	* preallocation size. Note that we don't care if the previous extents
516	* are written or not.
517	*/
518	plen = prev.br_blockcount;
519	while (xfs_iext_prev_extent(ifp, &ncur, &got)) {
520	if (plen > XFS_MAX_BMBT_EXTLEN / 2 ||
521	isnullstartblock(got.br_startblock) ||
522	got.br_startoff + got.br_blockcount != prev.br_startoff ||
523	got.br_startblock + got.br_blockcount != prev.br_startblock)
524	break;
525	plen += got.br_blockcount;
526	prev = got;
527	}
528
529	/*
530	* If the size of the extents is greater than half the maximum extent
531	* length, then use the current offset as the basis. This ensures that
532	* for large files the preallocation size always extends to
533	* XFS_BMBT_MAX_EXTLEN rather than falling short due to things like stripe
534	* unit/width alignment of real extents.
535	*/
536	alloc_blocks = plen * 2;
537	if (alloc_blocks > XFS_MAX_BMBT_EXTLEN)
538	alloc_blocks = XFS_B_TO_FSB(mp, offset);
539	qblocks = alloc_blocks;
540
541	/*
542	* XFS_BMBT_MAX_EXTLEN is not a power of two value but we round the prealloc
543	* down to the nearest power of two value after throttling. To prevent
544	* the round down from unconditionally reducing the maximum supported
545	* prealloc size, we round up first, apply appropriate throttling, round
546	* down and cap the value to XFS_BMBT_MAX_EXTLEN.
547	*/
548	alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(XFS_MAX_BMBT_EXTLEN),
549	alloc_blocks);
550
551	if (unlikely(XFS_IS_REALTIME_INODE(ip)))
552	freesp = xfs_rtbxlen_to_blen(mp,
553	xfs_iomap_freesp(mp, XC_FREE_RTEXTENTS,
554	mp->m_low_rtexts, &shift));
555	else
556	freesp = xfs_iomap_freesp(mp, XC_FREE_BLOCKS, mp->m_low_space,
557	&shift);
558
559	/*
560	* Check each quota to cap the prealloc size, provide a shift value to
561	* throttle with and adjust amount of available space.
562	*/
563	if (xfs_quota_need_throttle(ip, XFS_DQTYPE_USER, alloc_blocks))
564	xfs_quota_calc_throttle(ip, XFS_DQTYPE_USER, &qblocks, &qshift,
565	&freesp);
566	if (xfs_quota_need_throttle(ip, XFS_DQTYPE_GROUP, alloc_blocks))
567	xfs_quota_calc_throttle(ip, XFS_DQTYPE_GROUP, &qblocks, &qshift,
568	&freesp);
569	if (xfs_quota_need_throttle(ip, XFS_DQTYPE_PROJ, alloc_blocks))
570	xfs_quota_calc_throttle(ip, XFS_DQTYPE_PROJ, &qblocks, &qshift,
571	&freesp);
572
573	/*
574	* The final prealloc size is set to the minimum of free space available
575	* in each of the quotas and the overall filesystem.
576	*
577	* The shift throttle value is set to the maximum value as determined by
578	* the global low free space values and per-quota low free space values.
579	*/
580	alloc_blocks = min(alloc_blocks, qblocks);
581	shift = max(shift, qshift);
582
583	if (shift)
584	alloc_blocks >>= shift;
585	/*
586	* rounddown_pow_of_two() returns an undefined result if we pass in
587	* alloc_blocks = 0.
588	*/
589	if (alloc_blocks)
590	alloc_blocks = rounddown_pow_of_two(alloc_blocks);
591	if (alloc_blocks > XFS_MAX_BMBT_EXTLEN)
592	alloc_blocks = XFS_MAX_BMBT_EXTLEN;
593
594	/*
595	* If we are still trying to allocate more space than is
596	* available, squash the prealloc hard. This can happen if we
597	* have a large file on a small filesystem and the above
598	* lowspace thresholds are smaller than XFS_BMBT_MAX_EXTLEN.
599	*/
600	while (alloc_blocks && alloc_blocks >= freesp)
601	alloc_blocks >>= 4;
602	if (alloc_blocks < mp->m_allocsize_blocks)
603	alloc_blocks = mp->m_allocsize_blocks;
604	trace_xfs_iomap_prealloc_size(ip, alloc_blocks, shift,
605	mp->m_allocsize_blocks);
606	return alloc_blocks;
607	}
608
609	int
610	xfs_iomap_write_unwritten(
611	xfs_inode_t *ip,
612	xfs_off_t offset,
613	xfs_off_t count,
614	bool update_isize)
615	{
616	xfs_mount_t *mp = ip->i_mount;
617	xfs_fileoff_t offset_fsb;
618	xfs_filblks_t count_fsb;
619	xfs_filblks_t numblks_fsb;
620	int nimaps;
621	xfs_trans_t *tp;
622	xfs_bmbt_irec_t imap;
623	struct inode *inode = VFS_I(ip);
624	xfs_fsize_t i_size;
625	uint resblks;
626	int error;
627
628	trace_xfs_unwritten_convert(ip, offset, count);
629
630	offset_fsb = XFS_B_TO_FSBT(mp, offset);
631	count_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
632	count_fsb = (xfs_filblks_t)(count_fsb - offset_fsb);
633
634	/*
635	* Reserve enough blocks in this transaction for two complete extent
636	* btree splits. We may be converting the middle part of an unwritten
637	* extent and in this case we will insert two new extents in the btree
638	* each of which could cause a full split.
639	*
640	* This reservation amount will be used in the first call to
641	* xfs_bmbt_split() to select an AG with enough space to satisfy the
642	* rest of the operation.
643	*/
644	resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1;
645
646	/* Attach dquots so that bmbt splits are accounted correctly. */
647	error = xfs_qm_dqattach(ip);
648	if (error)
649	return error;
650
651	do {
652	/*
653	* Set up a transaction to convert the range of extents
654	* from unwritten to real. Do allocations in a loop until
655	* we have covered the range passed in.
656	*
657	* Note that we can't risk to recursing back into the filesystem
658	* here as we might be asked to write out the same inode that we
659	* complete here and might deadlock on the iolock.
660	*/
661	error = xfs_trans_alloc_inode(ip, resv: &M_RES(mp)->tr_write, dblocks: resblks,
662	rblocks: 0, force: true, tpp: &tp);
663	if (error)
664	return error;
665
666	error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK,
667	XFS_IEXT_WRITE_UNWRITTEN_CNT);
668	if (error)
669	goto error_on_bmapi_transaction;
670
671	/*
672	* Modify the unwritten extent state of the buffer.
673	*/
674	nimaps = 1;
675	error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb,
676	XFS_BMAPI_CONVERT, resblks, &imap,
677	&nimaps);
678	if (error)
679	goto error_on_bmapi_transaction;
680
681	/*
682	* Log the updated inode size as we go. We have to be careful
683	* to only log it up to the actual write offset if it is
684	* halfway into a block.
685	*/
686	i_size = XFS_FSB_TO_B(mp, offset_fsb + count_fsb);
687	if (i_size > offset + count)
688	i_size = offset + count;
689	if (update_isize && i_size > i_size_read(inode))
690	i_size_write(inode, i_size);
691	i_size = xfs_new_eof(ip, i_size);
692	if (i_size) {
693	ip->i_disk_size = i_size;
694	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
695	}
696
697	error = xfs_trans_commit(tp);
698	xfs_iunlock(ip, XFS_ILOCK_EXCL);
699	if (error)
700	return error;
701
702	if (unlikely(!xfs_valid_startblock(ip, imap.br_startblock))) {
703	xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
704	return xfs_alert_fsblock_zero(ip, &imap);
705	}
706
707	if ((numblks_fsb = imap.br_blockcount) == 0) {
708	/*
709	* The numblks_fsb value should always get
710	* smaller, otherwise the loop is stuck.
711	*/
712	ASSERT(imap.br_blockcount);
713	break;
714	}
715	offset_fsb += numblks_fsb;
716	count_fsb -= numblks_fsb;
717	} while (count_fsb > 0);
718
719	return 0;
720
721	error_on_bmapi_transaction:
722	xfs_trans_cancel(tp);
723	xfs_iunlock(ip, XFS_ILOCK_EXCL);
724	return error;
725	}
726
727	static inline bool
728	imap_needs_alloc(
729	struct inode *inode,
730	unsigned flags,
731	struct xfs_bmbt_irec *imap,
732	int nimaps)
733	{
734	/* don't allocate blocks when just zeroing */
735	if (flags & IOMAP_ZERO)
736	return false;
737	if (!nimaps ||
738	imap->br_startblock == HOLESTARTBLOCK ||
739	imap->br_startblock == DELAYSTARTBLOCK)
740	return true;
741	/* we convert unwritten extents before copying the data for DAX */
742	if ((flags & IOMAP_DAX) && imap->br_state == XFS_EXT_UNWRITTEN)
743	return true;
744	return false;
745	}
746
747	static inline bool
748	imap_needs_cow(
749	struct xfs_inode *ip,
750	unsigned int flags,
751	struct xfs_bmbt_irec *imap,
752	int nimaps)
753	{
754	if (!xfs_is_cow_inode(ip))
755	return false;
756
757	/* when zeroing we don't have to COW holes or unwritten extents */
758	if (flags & (IOMAP_UNSHARE | IOMAP_ZERO)) {
759	if (!nimaps ||
760	imap->br_startblock == HOLESTARTBLOCK ||
761	imap->br_state == XFS_EXT_UNWRITTEN)
762	return false;
763	}
764
765	return true;
766	}
767
768	/*
769	* Extents not yet cached requires exclusive access, don't block for
770	* IOMAP_NOWAIT.
771	*
772	* This is basically an opencoded xfs_ilock_data_map_shared() call, but with
773	* support for IOMAP_NOWAIT.
774	*/
775	static int
776	xfs_ilock_for_iomap(
777	struct xfs_inode *ip,
778	unsigned flags,
779	unsigned *lockmode)
780	{
781	if (flags & IOMAP_NOWAIT) {
782	if (xfs_need_iread_extents(&ip->i_df))
783	return -EAGAIN;
784	if (!xfs_ilock_nowait(ip, *lockmode))
785	return -EAGAIN;
786	} else {
787	if (xfs_need_iread_extents(&ip->i_df))
788	*lockmode = XFS_ILOCK_EXCL;
789	xfs_ilock(ip, *lockmode);
790	}
791
792	return 0;
793	}
794
795	/*
796	* Check that the imap we are going to return to the caller spans the entire
797	* range that the caller requested for the IO.
798	*/
799	static bool
800	imap_spans_range(
801	struct xfs_bmbt_irec *imap,
802	xfs_fileoff_t offset_fsb,
803	xfs_fileoff_t end_fsb)
804	{
805	if (imap->br_startoff > offset_fsb)
806	return false;
807	if (imap->br_startoff + imap->br_blockcount < end_fsb)
808	return false;
809	return true;
810	}
811
812	static bool
813	xfs_bmap_hw_atomic_write_possible(
814	struct xfs_inode *ip,
815	struct xfs_bmbt_irec *imap,
816	xfs_fileoff_t offset_fsb,
817	xfs_fileoff_t end_fsb)
818	{
819	struct xfs_mount *mp = ip->i_mount;
820	xfs_fsize_t len = XFS_FSB_TO_B(mp, end_fsb - offset_fsb);
821
822	/*
823	* atomic writes are required to be naturally aligned for disk blocks,
824	* which ensures that we adhere to block layer rules that we won't
825	* straddle any boundary or violate write alignment requirement.
826	*/
827	if (!IS_ALIGNED(imap->br_startblock, imap->br_blockcount))
828	return false;
829
830	/*
831	* Spanning multiple extents would mean that multiple BIOs would be
832	* issued, and so would lose atomicity required for REQ_ATOMIC-based
833	* atomics.
834	*/
835	if (!imap_spans_range(imap, offset_fsb, end_fsb))
836	return false;
837
838	/*
839	* The ->iomap_begin caller should ensure this, but check anyway.
840	*/
841	return len <= xfs_inode_buftarg(ip)->bt_awu_max;
842	}
843
844	static int
845	xfs_direct_write_iomap_begin(
846	struct inode *inode,
847	loff_t offset,
848	loff_t length,
849	unsigned flags,
850	struct iomap *iomap,
851	struct iomap *srcmap)
852	{
853	struct xfs_inode *ip = XFS_I(inode);
854	struct xfs_mount *mp = ip->i_mount;
855	struct xfs_bmbt_irec imap, cmap;
856	xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
857	xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, length);
858	xfs_fileoff_t orig_end_fsb = end_fsb;
859	int nimaps = 1, error = 0;
860	bool shared = false;
861	u16 iomap_flags = 0;
862	bool needs_alloc;
863	unsigned int lockmode;
864	u64 seq;
865
866	ASSERT(flags & (IOMAP_WRITE | IOMAP_ZERO));
867
868	if (xfs_is_shutdown(mp))
869	return -EIO;
870
871	/*
872	* Writes that span EOF might trigger an IO size update on completion,
873	* so consider them to be dirty for the purposes of O_DSYNC even if
874	* there is no other metadata changes pending or have been made here.
875	*/
876	if (offset + length > i_size_read(inode))
877	iomap_flags |= IOMAP_F_DIRTY;
878
879	/* HW-offload atomics are always used in this path */
880	if (flags & IOMAP_ATOMIC)
881	iomap_flags |= IOMAP_F_ATOMIC_BIO;
882
883	/*
884	* COW writes may allocate delalloc space or convert unwritten COW
885	* extents, so we need to make sure to take the lock exclusively here.
886	*/
887	if (xfs_is_cow_inode(ip))
888	lockmode = XFS_ILOCK_EXCL;
889	else
890	lockmode = XFS_ILOCK_SHARED;
891
892	relock:
893	error = xfs_ilock_for_iomap(ip, flags, lockmode: &lockmode);
894	if (error)
895	return error;
896
897	/*
898	* The reflink iflag could have changed since the earlier unlocked
899	* check, check if it again and relock if needed.
900	*/
901	if (xfs_is_cow_inode(ip) && lockmode == XFS_ILOCK_SHARED) {
902	xfs_iunlock(ip, lockmode);
903	lockmode = XFS_ILOCK_EXCL;
904	goto relock;
905	}
906
907	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
908	&nimaps, 0);
909	if (error)
910	goto out_unlock;
911
912	if (imap_needs_cow(ip, flags, imap: &imap, nimaps)) {
913	error = -EAGAIN;
914	if (flags & IOMAP_NOWAIT)
915	goto out_unlock;
916
917	/* may drop and re-acquire the ilock */
918	error = xfs_reflink_allocate_cow(ip, imap: &imap, cmap: &cmap, shared: &shared,
919	lockmode: &lockmode,
920	convert_now: (flags & IOMAP_DIRECT) || IS_DAX(inode));
921	if (error)
922	goto out_unlock;
923	if (shared) {
924	if ((flags & IOMAP_ATOMIC) &&
925	!xfs_bmap_hw_atomic_write_possible(ip, &cmap,
926	offset_fsb, end_fsb)) {
927	error = -ENOPROTOOPT;
928	goto out_unlock;
929	}
930	goto out_found_cow;
931	}
932	end_fsb = imap.br_startoff + imap.br_blockcount;
933	length = XFS_FSB_TO_B(mp, end_fsb) - offset;
934	}
935
936	needs_alloc = imap_needs_alloc(inode, flags, imap: &imap, nimaps);
937
938	if (flags & IOMAP_ATOMIC) {
939	error = -ENOPROTOOPT;
940	/*
941	* If we allocate less than what is required for the write
942	* then we may end up with multiple extents, which means that
943	* REQ_ATOMIC-based cannot be used, so avoid this possibility.
944	*/
945	if (needs_alloc && orig_end_fsb - offset_fsb > 1)
946	goto out_unlock;
947
948	if (!xfs_bmap_hw_atomic_write_possible(ip, &imap, offset_fsb,
949	orig_end_fsb))
950	goto out_unlock;
951	}
952
953	if (needs_alloc)
954	goto allocate_blocks;
955
956	/*
957	* NOWAIT and OVERWRITE I/O needs to span the entire requested I/O with
958	* a single map so that we avoid partial IO failures due to the rest of
959	* the I/O range not covered by this map triggering an EAGAIN condition
960	* when it is subsequently mapped and aborting the I/O.
961	*/
962	if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) {
963	error = -EAGAIN;
964	if (!imap_spans_range(&imap, offset_fsb, end_fsb))
965	goto out_unlock;
966	}
967
968	/*
969	* For overwrite only I/O, we cannot convert unwritten extents without
970	* requiring sub-block zeroing. This can only be done under an
971	* exclusive IOLOCK, hence return -EAGAIN if this is not a written
972	* extent to tell the caller to try again.
973	*/
974	if (flags & IOMAP_OVERWRITE_ONLY) {
975	error = -EAGAIN;
976	if (imap.br_state != XFS_EXT_NORM &&
977	((offset | length) & mp->m_blockmask))
978	goto out_unlock;
979	}
980
981	seq = xfs_iomap_inode_sequence(ip, iomap_flags);
982	xfs_iunlock(ip, lockmode);
983	trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
984	return xfs_bmbt_to_iomap(ip, iomap, imap: &imap, mapping_flags: flags, iomap_flags, sequence_cookie: seq);
985
986	allocate_blocks:
987	error = -EAGAIN;
988	if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY))
989	goto out_unlock;
990
991	/*
992	* We cap the maximum length we map to a sane size to keep the chunks
993	* of work done where somewhat symmetric with the work writeback does.
994	* This is a completely arbitrary number pulled out of thin air as a
995	* best guess for initial testing.
996	*
997	* Note that the values needs to be less than 32-bits wide until the
998	* lower level functions are updated.
999	*/
1000	length = min_t(loff_t, length, 1024 * PAGE_SIZE);
1001	end_fsb = xfs_iomap_end_fsb(mp, offset, length);
1002
1003	if (offset + length > XFS_ISIZE(ip))
1004	end_fsb = xfs_iomap_eof_align_last_fsb(ip, end_fsb);
1005	else if (nimaps && imap.br_startblock == HOLESTARTBLOCK)
1006	end_fsb = min(end_fsb, imap.br_startoff + imap.br_blockcount);
1007	xfs_iunlock(ip, lockmode);
1008
1009	error = xfs_iomap_write_direct(ip, offset_fsb, end_fsb - offset_fsb,
1010	flags, &imap, &seq);
1011	if (error)
1012	return error;
1013
1014	trace_xfs_iomap_alloc(ip, offset, length, XFS_DATA_FORK, &imap);
1015	return xfs_bmbt_to_iomap(ip, iomap, imap: &imap, mapping_flags: flags,
1016	iomap_flags: iomap_flags | IOMAP_F_NEW, sequence_cookie: seq);
1017
1018	out_found_cow:
1019	length = XFS_FSB_TO_B(mp, cmap.br_startoff + cmap.br_blockcount);
1020	trace_xfs_iomap_found(ip, offset, length - offset, XFS_COW_FORK, &cmap);
1021	if (imap.br_startblock != HOLESTARTBLOCK) {
1022	seq = xfs_iomap_inode_sequence(ip, iomap_flags: 0);
1023	error = xfs_bmbt_to_iomap(ip, iomap: srcmap, imap: &imap, mapping_flags: flags, iomap_flags: 0, sequence_cookie: seq);
1024	if (error)
1025	goto out_unlock;
1026	}
1027	seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
1028	xfs_iunlock(ip, lockmode);
1029	return xfs_bmbt_to_iomap(ip, iomap, imap: &cmap, mapping_flags: flags, IOMAP_F_SHARED, sequence_cookie: seq);
1030
1031	out_unlock:
1032	if (lockmode)
1033	xfs_iunlock(ip, lockmode);
1034	return error;
1035	}
1036
1037	const struct iomap_ops xfs_direct_write_iomap_ops = {
1038	.iomap_begin = xfs_direct_write_iomap_begin,
1039	};
1040
1041	#ifdef CONFIG_XFS_RT
1042	/*
1043	* This is really simple. The space has already been reserved before taking the
1044	* IOLOCK, the actual block allocation is done just before submitting the bio
1045	* and only recorded in the extent map on I/O completion.
1046	*/
1047	static int
1048	xfs_zoned_direct_write_iomap_begin(
1049	struct inode *inode,
1050	loff_t offset,
1051	loff_t length,
1052	unsigned flags,
1053	struct iomap *iomap,
1054	struct iomap *srcmap)
1055	{
1056	struct xfs_inode *ip = XFS_I(inode);
1057	int error;
1058
1059	ASSERT(!(flags & IOMAP_OVERWRITE_ONLY));
1060
1061	/*
1062	* Needs to be pushed down into the allocator so that only writes into
1063	* a single zone can be supported.
1064	*/
1065	if (flags & IOMAP_NOWAIT)
1066	return -EAGAIN;
1067
1068	/*
1069	* Ensure the extent list is in memory in so that we don't have to do
1070	* read it from the I/O completion handler.
1071	*/
1072	if (xfs_need_iread_extents(&ip->i_df)) {
1073	xfs_ilock(ip, XFS_ILOCK_EXCL);
1074	error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
1075	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1076	if (error)
1077	return error;
1078	}
1079
1080	iomap->type = IOMAP_MAPPED;
1081	iomap->flags = IOMAP_F_DIRTY;
1082	iomap->bdev = ip->i_mount->m_rtdev_targp->bt_bdev;
1083	iomap->offset = offset;
1084	iomap->length = length;
1085	iomap->flags = IOMAP_F_ANON_WRITE;
1086	return 0;
1087	}
1088
1089	const struct iomap_ops xfs_zoned_direct_write_iomap_ops = {
1090	.iomap_begin = xfs_zoned_direct_write_iomap_begin,
1091	};
1092	#endif /* CONFIG_XFS_RT */
1093
1094	#ifdef DEBUG
1095	static void
1096	xfs_check_atomic_cow_conversion(
1097	struct xfs_inode *ip,
1098	xfs_fileoff_t offset_fsb,
1099	xfs_filblks_t count_fsb,
1100	const struct xfs_bmbt_irec *cmap)
1101	{
1102	struct xfs_iext_cursor icur;
1103	struct xfs_bmbt_irec cmap2 = { };
1104
1105	if (xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap2))
1106	xfs_trim_extent(&cmap2, offset_fsb, count_fsb);
1107
1108	ASSERT(cmap2.br_startoff == cmap->br_startoff);
1109	ASSERT(cmap2.br_blockcount == cmap->br_blockcount);
1110	ASSERT(cmap2.br_startblock == cmap->br_startblock);
1111	ASSERT(cmap2.br_state == cmap->br_state);
1112	}
1113	#else
1114	# define xfs_check_atomic_cow_conversion(...) ((void)0)
1115	#endif
1116
1117	static int
1118	xfs_atomic_write_cow_iomap_begin(
1119	struct inode *inode,
1120	loff_t offset,
1121	loff_t length,
1122	unsigned flags,
1123	struct iomap *iomap,
1124	struct iomap *srcmap)
1125	{
1126	struct xfs_inode *ip = XFS_I(inode);
1127	struct xfs_mount *mp = ip->i_mount;
1128	const xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
1129	const xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length);
1130	const xfs_filblks_t count_fsb = end_fsb - offset_fsb;
1131	xfs_filblks_t hole_count_fsb;
1132	int nmaps = 1;
1133	xfs_filblks_t resaligned;
1134	struct xfs_bmbt_irec cmap;
1135	struct xfs_iext_cursor icur;
1136	struct xfs_trans *tp;
1137	unsigned int dblocks = 0, rblocks = 0;
1138	int error;
1139	u64 seq;
1140
1141	ASSERT(flags & IOMAP_WRITE);
1142	ASSERT(flags & IOMAP_DIRECT);
1143
1144	if (xfs_is_shutdown(mp))
1145	return -EIO;
1146
1147	if (!xfs_can_sw_atomic_write(mp)) {
1148	ASSERT(xfs_can_sw_atomic_write(mp));
1149	return -EINVAL;
1150	}
1151
1152	/* blocks are always allocated in this path */
1153	if (flags & IOMAP_NOWAIT)
1154	return -EAGAIN;
1155
1156	trace_xfs_iomap_atomic_write_cow(ip, offset, count: length);
1157	retry:
1158	xfs_ilock(ip, XFS_ILOCK_EXCL);
1159
1160	if (!ip->i_cowfp) {
1161	ASSERT(!xfs_is_reflink_inode(ip));
1162	xfs_ifork_init_cow(ip);
1163	}
1164
1165	if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap))
1166	cmap.br_startoff = end_fsb;
1167	if (cmap.br_startoff <= offset_fsb) {
1168	if (isnullstartblock(cmap.br_startblock))
1169	goto convert_delay;
1170
1171	/*
1172	* cmap could extend outside the write range due to previous
1173	* speculative preallocations. We must trim cmap to the write
1174	* range because the cow fork treats written mappings to mean
1175	* "write in progress".
1176	*/
1177	xfs_trim_extent(&cmap, offset_fsb, count_fsb);
1178	goto found;
1179	}
1180
1181	hole_count_fsb = cmap.br_startoff - offset_fsb;
1182
1183	resaligned = xfs_aligned_fsb_count(offset_fsb, hole_count_fsb,
1184	xfs_get_cowextsz_hint(ip));
1185	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1186
1187	if (XFS_IS_REALTIME_INODE(ip)) {
1188	dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0);
1189	rblocks = resaligned;
1190	} else {
1191	dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
1192	rblocks = 0;
1193	}
1194
1195	error = xfs_trans_alloc_inode(ip, resv: &M_RES(mp)->tr_write, dblocks,
1196	rblocks, force: false, tpp: &tp);
1197	if (error)
1198	return error;
1199
1200	/* extent layout could have changed since the unlock, so check again */
1201	if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap))
1202	cmap.br_startoff = end_fsb;
1203	if (cmap.br_startoff <= offset_fsb) {
1204	xfs_trans_cancel(tp);
1205	if (isnullstartblock(cmap.br_startblock))
1206	goto convert_delay;
1207	xfs_trim_extent(&cmap, offset_fsb, count_fsb);
1208	goto found;
1209	}
1210
1211	/*
1212	* Allocate the entire reservation as unwritten blocks.
1213	*
1214	* Use XFS_BMAPI_EXTSZALIGN to hint at aligning new extents according to
1215	* extszhint, such that there will be a greater chance that future
1216	* atomic writes to that same range will be aligned (and don't require
1217	* this COW-based method).
1218	*/
1219	error = xfs_bmapi_write(tp, ip, offset_fsb, hole_count_fsb,
1220	XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC |
1221	XFS_BMAPI_EXTSZALIGN, 0, &cmap, &nmaps);
1222	if (error) {
1223	xfs_trans_cancel(tp);
1224	goto out_unlock;
1225	}
1226
1227	xfs_inode_set_cowblocks_tag(ip);
1228	error = xfs_trans_commit(tp);
1229	if (error)
1230	goto out_unlock;
1231
1232	/*
1233	* cmap could map more blocks than the range we passed into bmapi_write
1234	* because of EXTSZALIGN or adjacent pre-existing unwritten mappings
1235	* that were merged. Trim cmap to the original write range so that we
1236	* don't convert more than we were asked to do for this write.
1237	*/
1238	xfs_trim_extent(&cmap, offset_fsb, count_fsb);
1239
1240	found:
1241	if (cmap.br_state != XFS_EXT_NORM) {
1242	error = xfs_reflink_convert_cow_locked(ip, cmap.br_startoff,
1243	cmap.br_blockcount);
1244	if (error)
1245	goto out_unlock;
1246	cmap.br_state = XFS_EXT_NORM;
1247	xfs_check_atomic_cow_conversion(ip, offset_fsb, count_fsb,
1248	&cmap);
1249	}
1250
1251	trace_xfs_iomap_found(ip, offset, length, XFS_COW_FORK, &cmap);
1252	seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
1253	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1254	return xfs_bmbt_to_iomap(ip, iomap, imap: &cmap, mapping_flags: flags, IOMAP_F_SHARED, sequence_cookie: seq);
1255
1256	convert_delay:
1257	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1258	error = xfs_bmapi_convert_delalloc(ip, XFS_COW_FORK, offset, iomap,
1259	NULL);
1260	if (error)
1261	return error;
1262
1263	/*
1264	* Try the lookup again, because the delalloc conversion might have
1265	* turned the COW mapping into unwritten, but we need it to be in
1266	* written state.
1267	*/
1268	goto retry;
1269	out_unlock:
1270	xfs_iunlock(ip, XFS_ILOCK_EXCL);
1271	return error;
1272	}
1273
1274	const struct iomap_ops xfs_atomic_write_cow_iomap_ops = {
1275	.iomap_begin = xfs_atomic_write_cow_iomap_begin,
1276	};
1277
1278	static int
1279	xfs_dax_write_iomap_end(
1280	struct inode *inode,
1281	loff_t pos,
1282	loff_t length,
1283	ssize_t written,
1284	unsigned flags,
1285	struct iomap *iomap)
1286	{
1287	struct xfs_inode *ip = XFS_I(inode);
1288
1289	if (!xfs_is_cow_inode(ip))
1290	return 0;
1291
1292	if (!written)
1293	return xfs_reflink_cancel_cow_range(ip, offset: pos, count: length, cancel_real: true);
1294
1295	return xfs_reflink_end_cow(ip, offset: pos, count: written);
1296	}
1297
1298	const struct iomap_ops xfs_dax_write_iomap_ops = {
1299	.iomap_begin = xfs_direct_write_iomap_begin,
1300	.iomap_end = xfs_dax_write_iomap_end,
1301	};
1302
1303	/*
1304	* Convert a hole to a delayed allocation.
1305	*/
1306	static void
1307	xfs_bmap_add_extent_hole_delay(
1308	struct xfs_inode *ip, /* incore inode pointer */
1309	int whichfork,
1310	struct xfs_iext_cursor *icur,
1311	struct xfs_bmbt_irec *new) /* new data to add to file extents */
1312	{
1313	struct xfs_ifork *ifp; /* inode fork pointer */
1314	xfs_bmbt_irec_t left; /* left neighbor extent entry */
1315	xfs_filblks_t newlen=0; /* new indirect size */
1316	xfs_filblks_t oldlen=0; /* old indirect size */
1317	xfs_bmbt_irec_t right; /* right neighbor extent entry */
1318	uint32_t state = xfs_bmap_fork_to_state(whichfork);
1319	xfs_filblks_t temp; /* temp for indirect calculations */
1320
1321	ifp = xfs_ifork_ptr(ip, whichfork);
1322	ASSERT(isnullstartblock(new->br_startblock));
1323
1324	/*
1325	* Check and set flags if this segment has a left neighbor
1326	*/
1327	if (xfs_iext_peek_prev_extent(ifp, icur, &left)) {
1328	state |= BMAP_LEFT_VALID;
1329	if (isnullstartblock(left.br_startblock))
1330	state |= BMAP_LEFT_DELAY;
1331	}
1332
1333	/*
1334	* Check and set flags if the current (right) segment exists.
1335	* If it doesn't exist, we're converting the hole at end-of-file.
1336	*/
1337	if (xfs_iext_get_extent(ifp, icur, &right)) {
1338	state |= BMAP_RIGHT_VALID;
1339	if (isnullstartblock(right.br_startblock))
1340	state |= BMAP_RIGHT_DELAY;
1341	}
1342
1343	/*
1344	* Set contiguity flags on the left and right neighbors.
1345	* Don't let extents get too large, even if the pieces are contiguous.
1346	*/
1347	if ((state & BMAP_LEFT_VALID) && (state & BMAP_LEFT_DELAY) &&
1348	left.br_startoff + left.br_blockcount == new->br_startoff &&
1349	left.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN)
1350	state |= BMAP_LEFT_CONTIG;
1351
1352	if ((state & BMAP_RIGHT_VALID) && (state & BMAP_RIGHT_DELAY) &&
1353	new->br_startoff + new->br_blockcount == right.br_startoff &&
1354	new->br_blockcount + right.br_blockcount <= XFS_MAX_BMBT_EXTLEN &&
1355	(!(state & BMAP_LEFT_CONTIG) ||
1356	(left.br_blockcount + new->br_blockcount +
1357	right.br_blockcount <= XFS_MAX_BMBT_EXTLEN)))
1358	state |= BMAP_RIGHT_CONTIG;
1359
1360	/*
1361	* Switch out based on the contiguity flags.
1362	*/
1363	switch (state & (BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG)) {
1364	case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
1365	/*
1366	* New allocation is contiguous with delayed allocations
1367	* on the left and on the right.
1368	* Merge all three into a single extent record.
1369	*/
1370	temp = left.br_blockcount + new->br_blockcount +
1371	right.br_blockcount;
1372
1373	oldlen = startblockval(left.br_startblock) +
1374	startblockval(new->br_startblock) +
1375	startblockval(right.br_startblock);
1376	newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
1377	oldlen);
1378	left.br_startblock = nullstartblock(newlen);
1379	left.br_blockcount = temp;
1380
1381	xfs_iext_remove(ip, icur, state);
1382	xfs_iext_prev(ifp, icur);
1383	xfs_iext_update_extent(ip, state, icur, &left);
1384	break;
1385
1386	case BMAP_LEFT_CONTIG:
1387	/*
1388	* New allocation is contiguous with a delayed allocation
1389	* on the left.
1390	* Merge the new allocation with the left neighbor.
1391	*/
1392	temp = left.br_blockcount + new->br_blockcount;
1393
1394	oldlen = startblockval(left.br_startblock) +
1395	startblockval(new->br_startblock);
1396	newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
1397	oldlen);
1398	left.br_blockcount = temp;
1399	left.br_startblock = nullstartblock(newlen);
1400
1401	xfs_iext_prev(ifp, icur);
1402	xfs_iext_update_extent(ip, state, icur, &left);
1403	break;
1404
1405	case BMAP_RIGHT_CONTIG:
1406	/*
1407	* New allocation is contiguous with a delayed allocation
1408	* on the right.
1409	* Merge the new allocation with the right neighbor.
1410	*/
1411	temp = new->br_blockcount + right.br_blockcount;
1412	oldlen = startblockval(new->br_startblock) +
1413	startblockval(right.br_startblock);
1414	newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
1415	oldlen);
1416	right.br_startoff = new->br_startoff;
1417	right.br_startblock = nullstartblock(newlen);
1418	right.br_blockcount = temp;
1419	xfs_iext_update_extent(ip, state, icur, &right);
1420	break;
1421
1422	case 0:
1423	/*
1424	* New allocation is not contiguous with another
1425	* delayed allocation.
1426	* Insert a new entry.
1427	*/
1428	oldlen = newlen = 0;
1429	xfs_iext_insert(ip, icur, new, state);
1430	break;
1431	}
1432	if (oldlen != newlen) {
1433	ASSERT(oldlen > newlen);
1434	xfs_add_fdblocks(ip->i_mount, oldlen - newlen);
1435
1436	/*
1437	* Nothing to do for disk quota accounting here.
1438	*/
1439	xfs_mod_delalloc(ip, 0, (int64_t)newlen - oldlen);
1440	}
1441	}
1442
1443	/*
1444	* Add a delayed allocation extent to an inode. Blocks are reserved from the
1445	* global pool and the extent inserted into the inode in-core extent tree.
1446	*
1447	* On entry, got refers to the first extent beyond the offset of the extent to
1448	* allocate or eof is specified if no such extent exists. On return, got refers
1449	* to the extent record that was inserted to the inode fork.
1450	*
1451	* Note that the allocated extent may have been merged with contiguous extents
1452	* during insertion into the inode fork. Thus, got does not reflect the current
1453	* state of the inode fork on return. If necessary, the caller can use lastx to
1454	* look up the updated record in the inode fork.
1455	*/
1456	static int
1457	xfs_bmapi_reserve_delalloc(
1458	struct xfs_inode *ip,
1459	int whichfork,
1460	xfs_fileoff_t off,
1461	xfs_filblks_t len,
1462	xfs_filblks_t prealloc,
1463	struct xfs_bmbt_irec *got,
1464	struct xfs_iext_cursor *icur,
1465	int eof)
1466	{
1467	struct xfs_mount *mp = ip->i_mount;
1468	struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork);
1469	xfs_extlen_t alen;
1470	xfs_extlen_t indlen;
1471	uint64_t fdblocks;
1472	int error;
1473	xfs_fileoff_t aoff;
1474	bool use_cowextszhint =
1475	whichfork == XFS_COW_FORK && !prealloc;
1476
1477	retry:
1478	/*
1479	* Cap the alloc length. Keep track of prealloc so we know whether to
1480	* tag the inode before we return.
1481	*/
1482	aoff = off;
1483	alen = XFS_FILBLKS_MIN(len + prealloc, XFS_MAX_BMBT_EXTLEN);
1484	if (!eof)
1485	alen = XFS_FILBLKS_MIN(alen, got->br_startoff - aoff);
1486	if (prealloc && alen >= len)
1487	prealloc = alen - len;
1488
1489	/*
1490	* If we're targetting the COW fork but aren't creating a speculative
1491	* posteof preallocation, try to expand the reservation to align with
1492	* the COW extent size hint if there's sufficient free space.
1493	*
1494	* Unlike the data fork, the CoW cancellation functions will free all
1495	* the reservations at inactivation, so we don't require that every
1496	* delalloc reservation have a dirty pagecache.
1497	*/
1498	if (use_cowextszhint) {
1499	struct xfs_bmbt_irec prev;
1500	xfs_extlen_t extsz = xfs_get_cowextsz_hint(ip);
1501
1502	if (!xfs_iext_peek_prev_extent(ifp, icur, &prev))
1503	prev.br_startoff = NULLFILEOFF;
1504
1505	error = xfs_bmap_extsize_align(mp, got, &prev, extsz, 0, eof,
1506	1, 0, &aoff, &alen);
1507	ASSERT(!error);
1508	}
1509
1510	/*
1511	* Make a transaction-less quota reservation for delayed allocation
1512	* blocks. This number gets adjusted later. We return if we haven't
1513	* allocated blocks already inside this loop.
1514	*/
1515	error = xfs_quota_reserve_blkres(ip, alen);
1516	if (error)
1517	goto out;
1518
1519	/*
1520	* Split changing sb for alen and indlen since they could be coming
1521	* from different places.
1522	*/
1523	indlen = (xfs_extlen_t)xfs_bmap_worst_indlen(ip, alen);
1524	ASSERT(indlen > 0);
1525
1526	fdblocks = indlen;
1527	if (XFS_IS_REALTIME_INODE(ip)) {
1528	ASSERT(!xfs_is_zoned_inode(ip));
1529	error = xfs_dec_frextents(mp, xfs_blen_to_rtbxlen(mp, alen));
1530	if (error)
1531	goto out_unreserve_quota;
1532	} else {
1533	fdblocks += alen;
1534	}
1535
1536	error = xfs_dec_fdblocks(mp, delta: fdblocks, reserved: false);
1537	if (error)
1538	goto out_unreserve_frextents;
1539
1540	ip->i_delayed_blks += alen;
1541	xfs_mod_delalloc(ip, alen, indlen);
1542
1543	got->br_startoff = aoff;
1544	got->br_startblock = nullstartblock(indlen);
1545	got->br_blockcount = alen;
1546	got->br_state = XFS_EXT_NORM;
1547
1548	xfs_bmap_add_extent_hole_delay(ip, whichfork, icur, new: got);
1549
1550	/*
1551	* Tag the inode if blocks were preallocated. Note that COW fork
1552	* preallocation can occur at the start or end of the extent, even when
1553	* prealloc == 0, so we must also check the aligned offset and length.
1554	*/
1555	if (whichfork == XFS_DATA_FORK && prealloc)
1556	xfs_inode_set_eofblocks_tag(ip);
1557	if (whichfork == XFS_COW_FORK && (prealloc || aoff < off || alen > len))
1558	xfs_inode_set_cowblocks_tag(ip);
1559
1560	return 0;
1561
1562	out_unreserve_frextents:
1563	if (XFS_IS_REALTIME_INODE(ip))
1564	xfs_add_frextents(mp, xfs_blen_to_rtbxlen(mp, alen));
1565	out_unreserve_quota:
1566	if (XFS_IS_QUOTA_ON(mp))
1567	xfs_quota_unreserve_blkres(ip, alen);
1568	out:
1569	if (error == -ENOSPC || error == -EDQUOT) {
1570	trace_xfs_delalloc_enospc(ip, offset: off, count: len);
1571
1572	if (prealloc || use_cowextszhint) {
1573	/* retry without any preallocation */
1574	use_cowextszhint = false;
1575	prealloc = 0;
1576	goto retry;
1577	}
1578	}
1579	return error;
1580	}
1581
1582	static int
1583	xfs_zoned_buffered_write_iomap_begin(
1584	struct inode *inode,
1585	loff_t offset,
1586	loff_t count,
1587	unsigned flags,
1588	struct iomap *iomap,
1589	struct iomap *srcmap)
1590	{
1591	struct iomap_iter *iter =
1592	container_of(iomap, struct iomap_iter, iomap);
1593	struct xfs_zone_alloc_ctx *ac = iter->private;
1594	struct xfs_inode *ip = XFS_I(inode);
1595	struct xfs_mount *mp = ip->i_mount;
1596	xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
1597	xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, count);
1598	u16 iomap_flags = IOMAP_F_SHARED;
1599	unsigned int lockmode = XFS_ILOCK_EXCL;
1600	xfs_filblks_t count_fsb;
1601	xfs_extlen_t indlen;
1602	struct xfs_bmbt_irec got;
1603	struct xfs_iext_cursor icur;
1604	int error = 0;
1605
1606	ASSERT(!xfs_get_extsz_hint(ip));
1607	ASSERT(!(flags & IOMAP_UNSHARE));
1608	ASSERT(ac);
1609
1610	if (xfs_is_shutdown(mp))
1611	return -EIO;
1612
1613	error = xfs_qm_dqattach(ip);
1614	if (error)
1615	return error;
1616
1617	error = xfs_ilock_for_iomap(ip, flags, lockmode: &lockmode);
1618	if (error)
1619	return error;
1620
1621	if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) ||
1622	XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) {
1623	xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
1624	error = -EFSCORRUPTED;
1625	goto out_unlock;
1626	}
1627
1628	XFS_STATS_INC(mp, xs_blk_mapw);
1629
1630	error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
1631	if (error)
1632	goto out_unlock;
1633
1634	/*
1635	* For zeroing operations check if there is any data to zero first.
1636	*
1637	* For regular writes we always need to allocate new blocks, but need to
1638	* provide the source mapping when the range is unaligned to support
1639	* read-modify-write of the whole block in the page cache.
1640	*
1641	* In either case we need to limit the reported range to the boundaries
1642	* of the source map in the data fork.
1643	*/
1644	if (!IS_ALIGNED(offset, mp->m_sb.sb_blocksize) ||
1645	!IS_ALIGNED(offset + count, mp->m_sb.sb_blocksize) ||
1646	(flags & IOMAP_ZERO)) {
1647	struct xfs_bmbt_irec smap;
1648	struct xfs_iext_cursor scur;
1649
1650	if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &scur,
1651	&smap))
1652	smap.br_startoff = end_fsb; /* fake hole until EOF */
1653	if (smap.br_startoff > offset_fsb) {
1654	/*
1655	* We never need to allocate blocks for zeroing a hole.
1656	*/
1657	if (flags & IOMAP_ZERO) {
1658	xfs_hole_to_iomap(ip, iomap, offset_fsb,
1659	smap.br_startoff);
1660	goto out_unlock;
1661	}
1662	end_fsb = min(end_fsb, smap.br_startoff);
1663	} else {
1664	end_fsb = min(end_fsb,
1665	smap.br_startoff + smap.br_blockcount);
1666	xfs_trim_extent(&smap, offset_fsb,
1667	end_fsb - offset_fsb);
1668	error = xfs_bmbt_to_iomap(ip, iomap: srcmap, imap: &smap, mapping_flags: flags, iomap_flags: 0,
1669	sequence_cookie: xfs_iomap_inode_sequence(ip, iomap_flags: 0));
1670	if (error)
1671	goto out_unlock;
1672	}
1673	}
1674
1675	if (!ip->i_cowfp)
1676	xfs_ifork_init_cow(ip);
1677
1678	if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got))
1679	got.br_startoff = end_fsb;
1680	if (got.br_startoff <= offset_fsb) {
1681	trace_xfs_reflink_cow_found(ip, irec: &got);
1682	goto done;
1683	}
1684
1685	/*
1686	* Cap the maximum length to keep the chunks of work done here somewhat
1687	* symmetric with the work writeback does.
1688	*/
1689	end_fsb = min(end_fsb, got.br_startoff);
1690	count_fsb = min3(end_fsb - offset_fsb, XFS_MAX_BMBT_EXTLEN,
1691	XFS_B_TO_FSB(mp, 1024 * PAGE_SIZE));
1692
1693	/*
1694	* The block reservation is supposed to cover all blocks that the
1695	* operation could possible write, but there is a nasty corner case
1696	* where blocks could be stolen from underneath us:
1697	*
1698	* 1) while this thread iterates over a larger buffered write,
1699	* 2) another thread is causing a write fault that calls into
1700	* ->page_mkwrite in range this thread writes to, using up the
1701	* delalloc reservation created by a previous call to this function.
1702	* 3) another thread does direct I/O on the range that the write fault
1703	* happened on, which causes writeback of the dirty data.
1704	* 4) this then set the stale flag, which cuts the current iomap
1705	* iteration short, causing the new call to ->iomap_begin that gets
1706	* us here again, but now without a sufficient reservation.
1707	*
1708	* This is a very unusual I/O pattern, and nothing but generic/095 is
1709	* known to hit it. There's not really much we can do here, so turn this
1710	* into a short write.
1711	*/
1712	if (count_fsb > ac->reserved_blocks) {
1713	xfs_warn_ratelimited(mp,
1714	"Short write on ino 0x%llx comm %.20s due to three-way race with write fault and direct I/O",
1715	ip->i_ino, current->comm);
1716	count_fsb = ac->reserved_blocks;
1717	if (!count_fsb) {
1718	error = -EIO;
1719	goto out_unlock;
1720	}
1721	}
1722
1723	error = xfs_quota_reserve_blkres(ip, count_fsb);
1724	if (error)
1725	goto out_unlock;
1726
1727	indlen = xfs_bmap_worst_indlen(ip, count_fsb);
1728	error = xfs_dec_fdblocks(mp, indlen, false);
1729	if (error)
1730	goto out_unlock;
1731	ip->i_delayed_blks += count_fsb;
1732	xfs_mod_delalloc(ip, count_fsb, indlen);
1733
1734	got.br_startoff = offset_fsb;
1735	got.br_startblock = nullstartblock(indlen);
1736	got.br_blockcount = count_fsb;
1737	got.br_state = XFS_EXT_NORM;
1738	xfs_bmap_add_extent_hole_delay(ip, XFS_COW_FORK, &icur, &got);
1739	ac->reserved_blocks -= count_fsb;
1740	iomap_flags |= IOMAP_F_NEW;
1741
1742	trace_xfs_iomap_alloc(ip, offset, XFS_FSB_TO_B(mp, count_fsb),
1743	XFS_COW_FORK, &got);
1744	done:
1745	error = xfs_bmbt_to_iomap(ip, iomap, imap: &got, mapping_flags: flags, iomap_flags,
1746	sequence_cookie: xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED));
1747	out_unlock:
1748	xfs_iunlock(ip, lockmode);
1749	return error;
1750	}
1751
1752	static int
1753	xfs_buffered_write_iomap_begin(
1754	struct inode *inode,
1755	loff_t offset,
1756	loff_t count,
1757	unsigned flags,
1758	struct iomap *iomap,
1759	struct iomap *srcmap)
1760	{
1761	struct iomap_iter *iter = container_of(iomap, struct iomap_iter,
1762	iomap);
1763	struct xfs_inode *ip = XFS_I(inode);
1764	struct xfs_mount *mp = ip->i_mount;
1765	xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
1766	xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, count);
1767	struct xfs_bmbt_irec imap, cmap;
1768	struct xfs_iext_cursor icur, ccur;
1769	xfs_fsblock_t prealloc_blocks = 0;
1770	bool eof = false, cow_eof = false, shared = false;
1771	int allocfork = XFS_DATA_FORK;
1772	int error = 0;
1773	unsigned int lockmode = XFS_ILOCK_EXCL;
1774	unsigned int iomap_flags = 0;
1775	u64 seq;
1776
1777	if (xfs_is_shutdown(mp))
1778	return -EIO;
1779
1780	if (xfs_is_zoned_inode(ip))
1781	return xfs_zoned_buffered_write_iomap_begin(inode, offset,
1782	count, flags, iomap, srcmap);
1783
1784	/* we can't use delayed allocations when using extent size hints */
1785	if (xfs_get_extsz_hint(ip))
1786	return xfs_direct_write_iomap_begin(inode, offset, length: count,
1787	flags, iomap, srcmap);
1788
1789	error = xfs_qm_dqattach(ip);
1790	if (error)
1791	return error;
1792
1793	error = xfs_ilock_for_iomap(ip, flags, lockmode: &lockmode);
1794	if (error)
1795	return error;
1796
1797	if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) ||
1798	XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) {
1799	xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
1800	error = -EFSCORRUPTED;
1801	goto out_unlock;
1802	}
1803
1804	XFS_STATS_INC(mp, xs_blk_mapw);
1805
1806	error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
1807	if (error)
1808	goto out_unlock;
1809
1810	/*
1811	* Search the data fork first to look up our source mapping. We
1812	* always need the data fork map, as we have to return it to the
1813	* iomap code so that the higher level write code can read data in to
1814	* perform read-modify-write cycles for unaligned writes.
1815	*/
1816	eof = !xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap);
1817	if (eof)
1818	imap.br_startoff = end_fsb; /* fake hole until the end */
1819
1820	/* We never need to allocate blocks for zeroing or unsharing a hole. */
1821	if ((flags & (IOMAP_UNSHARE | IOMAP_ZERO)) &&
1822	imap.br_startoff > offset_fsb) {
1823	xfs_hole_to_iomap(ip, iomap, offset_fsb, imap.br_startoff);
1824	goto out_unlock;
1825	}
1826
1827	/*
1828	* For zeroing, trim extents that extend beyond the EOF block. If a
1829	* delalloc extent starts beyond the EOF block, convert it to an
1830	* unwritten extent.
1831	*/
1832	if (flags & IOMAP_ZERO) {
1833	xfs_fileoff_t eof_fsb = XFS_B_TO_FSB(mp, XFS_ISIZE(ip));
1834
1835	if (isnullstartblock(imap.br_startblock) &&
1836	offset_fsb >= eof_fsb)
1837	goto convert_delay;
1838	if (offset_fsb < eof_fsb && end_fsb > eof_fsb)
1839	end_fsb = eof_fsb;
1840
1841	/*
1842	* Look up dirty folios for unwritten mappings within EOF.
1843	* Providing this bypasses the flush iomap uses to trigger
1844	* extent conversion when unwritten mappings have dirty
1845	* pagecache in need of zeroing.
1846	*
1847	* Trim the mapping to the end pos of the lookup, which in turn
1848	* was trimmed to the end of the batch if it became full before
1849	* the end of the mapping.
1850	*/
1851	if (imap.br_state == XFS_EXT_UNWRITTEN &&
1852	offset_fsb < eof_fsb) {
1853	loff_t foffset = offset, fend;
1854
1855	fend = offset +
1856	min(count, XFS_FSB_TO_B(mp, imap.br_blockcount));
1857	iomap_fill_dirty_folios(iter, start: &foffset, end: fend,
1858	iomap_flags: &iomap_flags);
1859	end_fsb = min_t(xfs_fileoff_t, end_fsb,
1860	XFS_B_TO_FSB(mp, foffset));
1861	}
1862
1863	xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);
1864	}
1865
1866	/*
1867	* Search the COW fork extent list even if we did not find a data fork
1868	* extent. This serves two purposes: first this implements the
1869	* speculative preallocation using cowextsize, so that we also unshare
1870	* block adjacent to shared blocks instead of just the shared blocks
1871	* themselves. Second the lookup in the extent list is generally faster
1872	* than going out to the shared extent tree.
1873	*/
1874	if (xfs_is_cow_inode(ip)) {
1875	if (!ip->i_cowfp) {
1876	ASSERT(!xfs_is_reflink_inode(ip));
1877	xfs_ifork_init_cow(ip);
1878	}
1879	cow_eof = !xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb,
1880	&ccur, &cmap);
1881	if (!cow_eof && cmap.br_startoff <= offset_fsb) {
1882	trace_xfs_reflink_cow_found(ip, irec: &cmap);
1883	goto found_cow;
1884	}
1885	}
1886
1887	if (imap.br_startoff <= offset_fsb) {
1888	/*
1889	* For reflink files we may need a delalloc reservation when
1890	* overwriting shared extents. This includes zeroing of
1891	* existing extents that contain data.
1892	*/
1893	if (!xfs_is_cow_inode(ip) ||
1894	((flags & IOMAP_ZERO) && imap.br_state != XFS_EXT_NORM)) {
1895	trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
1896	&imap);
1897	goto found_imap;
1898	}
1899
1900	xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);
1901
1902	/* Trim the mapping to the nearest shared extent boundary. */
1903	error = xfs_bmap_trim_cow(ip, imap: &imap, shared: &shared);
1904	if (error)
1905	goto out_unlock;
1906
1907	/* Not shared? Just report the (potentially capped) extent. */
1908	if (!shared) {
1909	trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK,
1910	&imap);
1911	goto found_imap;
1912	}
1913
1914	/*
1915	* Fork all the shared blocks from our write offset until the
1916	* end of the extent.
1917	*/
1918	allocfork = XFS_COW_FORK;
1919	end_fsb = imap.br_startoff + imap.br_blockcount;
1920	} else {
1921	/*
1922	* We cap the maximum length we map here to MAX_WRITEBACK_PAGES
1923	* pages to keep the chunks of work done where somewhat
1924	* symmetric with the work writeback does. This is a completely
1925	* arbitrary number pulled out of thin air.
1926	*
1927	* Note that the values needs to be less than 32-bits wide until
1928	* the lower level functions are updated.
1929	*/
1930	count = min_t(loff_t, count, 1024 * PAGE_SIZE);
1931	end_fsb = xfs_iomap_end_fsb(mp, offset, count);
1932
1933	if (xfs_is_always_cow_inode(ip))
1934	allocfork = XFS_COW_FORK;
1935	}
1936
1937	if (eof && offset + count > XFS_ISIZE(ip)) {
1938	/*
1939	* Determine the initial size of the preallocation.
1940	* We clean up any extra preallocation when the file is closed.
1941	*/
1942	if (xfs_has_allocsize(mp))
1943	prealloc_blocks = mp->m_allocsize_blocks;
1944	else if (allocfork == XFS_DATA_FORK)
1945	prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork,
1946	offset, count, &icur);
1947	else
1948	prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork,
1949	offset, count, &ccur);
1950	if (prealloc_blocks) {
1951	xfs_extlen_t align;
1952	xfs_off_t end_offset;
1953	xfs_fileoff_t p_end_fsb;
1954
1955	end_offset = XFS_ALLOC_ALIGN(mp, offset + count - 1);
1956	p_end_fsb = XFS_B_TO_FSBT(mp, end_offset) +
1957	prealloc_blocks;
1958
1959	align = xfs_eof_alignment(ip);
1960	if (align)
1961	p_end_fsb = roundup_64(p_end_fsb, align);
1962
1963	p_end_fsb = min(p_end_fsb,
1964	XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes));
1965	ASSERT(p_end_fsb > offset_fsb);
1966	prealloc_blocks = p_end_fsb - end_fsb;
1967	}
1968	}
1969
1970	/*
1971	* Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch
1972	* them out if the write happens to fail.
1973	*/
1974	iomap_flags |= IOMAP_F_NEW;
1975	if (allocfork == XFS_COW_FORK) {
1976	error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb,
1977	end_fsb - offset_fsb, prealloc_blocks, &cmap,
1978	&ccur, cow_eof);
1979	if (error)
1980	goto out_unlock;
1981
1982	trace_xfs_iomap_alloc(ip, offset, count, whichfork: allocfork, irec: &cmap);
1983	goto found_cow;
1984	}
1985
1986	error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb,
1987	end_fsb - offset_fsb, prealloc_blocks, &imap, &icur,
1988	eof);
1989	if (error)
1990	goto out_unlock;
1991
1992	trace_xfs_iomap_alloc(ip, offset, count, whichfork: allocfork, irec: &imap);
1993	found_imap:
1994	seq = xfs_iomap_inode_sequence(ip, iomap_flags);
1995	xfs_iunlock(ip, lockmode);
1996	return xfs_bmbt_to_iomap(ip, iomap, imap: &imap, mapping_flags: flags, iomap_flags, sequence_cookie: seq);
1997
1998	convert_delay:
1999	xfs_iunlock(ip, lockmode);
2000	truncate_pagecache(inode, new: offset);
2001	error = xfs_bmapi_convert_delalloc(ip, XFS_DATA_FORK, offset,
2002	iomap, NULL);
2003	if (error)
2004	return error;
2005
2006	trace_xfs_iomap_alloc(ip, offset, count, XFS_DATA_FORK, &imap);
2007	return 0;
2008
2009	found_cow:
2010	if (imap.br_startoff <= offset_fsb) {
2011	error = xfs_bmbt_to_iomap(ip, iomap: srcmap, imap: &imap, mapping_flags: flags, iomap_flags: 0,
2012	sequence_cookie: xfs_iomap_inode_sequence(ip, iomap_flags: 0));
2013	if (error)
2014	goto out_unlock;
2015	} else {
2016	xfs_trim_extent(&cmap, offset_fsb,
2017	imap.br_startoff - offset_fsb);
2018	}
2019
2020	iomap_flags |= IOMAP_F_SHARED;
2021	seq = xfs_iomap_inode_sequence(ip, iomap_flags);
2022	xfs_iunlock(ip, lockmode);
2023	return xfs_bmbt_to_iomap(ip, iomap, imap: &cmap, mapping_flags: flags, iomap_flags, sequence_cookie: seq);
2024
2025	out_unlock:
2026	xfs_iunlock(ip, lockmode);
2027	return error;
2028	}
2029
2030	static void
2031	xfs_buffered_write_delalloc_punch(
2032	struct inode *inode,
2033	loff_t offset,
2034	loff_t length,
2035	struct iomap *iomap)
2036	{
2037	struct iomap_iter *iter =
2038	container_of(iomap, struct iomap_iter, iomap);
2039
2040	xfs_bmap_punch_delalloc_range(XFS_I(inode),
2041	(iomap->flags & IOMAP_F_SHARED) ?
2042	XFS_COW_FORK : XFS_DATA_FORK,
2043	offset, offset + length, iter->private);
2044	}
2045
2046	static int
2047	xfs_buffered_write_iomap_end(
2048	struct inode *inode,
2049	loff_t offset,
2050	loff_t length,
2051	ssize_t written,
2052	unsigned flags,
2053	struct iomap *iomap)
2054	{
2055	loff_t start_byte, end_byte;
2056
2057	/* If we didn't reserve the blocks, we're not allowed to punch them. */
2058	if (iomap->type != IOMAP_DELALLOC || !(iomap->flags & IOMAP_F_NEW))
2059	return 0;
2060
2061	/*
2062	* iomap_page_mkwrite() will never fail in a way that requires delalloc
2063	* extents that it allocated to be revoked. Hence never try to release
2064	* them here.
2065	*/
2066	if (flags & IOMAP_FAULT)
2067	return 0;
2068
2069	/* Nothing to do if we've written the entire delalloc extent */
2070	start_byte = iomap_last_written_block(inode, pos: offset, written);
2071	end_byte = round_up(offset + length, i_blocksize(inode));
2072	if (start_byte >= end_byte)
2073	return 0;
2074
2075	/* For zeroing operations the callers already hold invalidate_lock. */
2076	if (flags & (IOMAP_UNSHARE | IOMAP_ZERO)) {
2077	rwsem_assert_held_write(sem: &inode->i_mapping->invalidate_lock);
2078	iomap_write_delalloc_release(inode, start_byte, end_byte, flags,
2079	iomap, punch: xfs_buffered_write_delalloc_punch);
2080	} else {
2081	filemap_invalidate_lock(mapping: inode->i_mapping);
2082	iomap_write_delalloc_release(inode, start_byte, end_byte, flags,
2083	iomap, punch: xfs_buffered_write_delalloc_punch);
2084	filemap_invalidate_unlock(mapping: inode->i_mapping);
2085	}
2086
2087	return 0;
2088	}
2089
2090	const struct iomap_ops xfs_buffered_write_iomap_ops = {
2091	.iomap_begin = xfs_buffered_write_iomap_begin,
2092	.iomap_end = xfs_buffered_write_iomap_end,
2093	};
2094
2095	static int
2096	xfs_read_iomap_begin(
2097	struct inode *inode,
2098	loff_t offset,
2099	loff_t length,
2100	unsigned flags,
2101	struct iomap *iomap,
2102	struct iomap *srcmap)
2103	{
2104	struct xfs_inode *ip = XFS_I(inode);
2105	struct xfs_mount *mp = ip->i_mount;
2106	struct xfs_bmbt_irec imap;
2107	xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
2108	xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, length);
2109	int nimaps = 1, error = 0;
2110	bool shared = false;
2111	unsigned int lockmode = XFS_ILOCK_SHARED;
2112	u64 seq;
2113
2114	ASSERT(!(flags & (IOMAP_WRITE | IOMAP_ZERO)));
2115
2116	if (xfs_is_shutdown(mp))
2117	return -EIO;
2118
2119	error = xfs_ilock_for_iomap(ip, flags, lockmode: &lockmode);
2120	if (error)
2121	return error;
2122	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
2123	&nimaps, 0);
2124	if (!error && ((flags & IOMAP_REPORT) || IS_DAX(inode)))
2125	error = xfs_reflink_trim_around_shared(ip, irec: &imap, shared: &shared);
2126	seq = xfs_iomap_inode_sequence(ip, iomap_flags: shared ? IOMAP_F_SHARED : 0);
2127	xfs_iunlock(ip, lockmode);
2128
2129	if (error)
2130	return error;
2131	trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap);
2132	return xfs_bmbt_to_iomap(ip, iomap, imap: &imap, mapping_flags: flags,
2133	iomap_flags: shared ? IOMAP_F_SHARED : 0, sequence_cookie: seq);
2134	}
2135
2136	const struct iomap_ops xfs_read_iomap_ops = {
2137	.iomap_begin = xfs_read_iomap_begin,
2138	};
2139
2140	static int
2141	xfs_seek_iomap_begin(
2142	struct inode *inode,
2143	loff_t offset,
2144	loff_t length,
2145	unsigned flags,
2146	struct iomap *iomap,
2147	struct iomap *srcmap)
2148	{
2149	struct xfs_inode *ip = XFS_I(inode);
2150	struct xfs_mount *mp = ip->i_mount;
2151	xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
2152	xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length);
2153	xfs_fileoff_t cow_fsb = NULLFILEOFF, data_fsb = NULLFILEOFF;
2154	struct xfs_iext_cursor icur;
2155	struct xfs_bmbt_irec imap, cmap;
2156	int error = 0;
2157	unsigned lockmode;
2158	u64 seq;
2159
2160	if (xfs_is_shutdown(mp))
2161	return -EIO;
2162
2163	lockmode = xfs_ilock_data_map_shared(ip);
2164	error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
2165	if (error)
2166	goto out_unlock;
2167
2168	if (xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap)) {
2169	/*
2170	* If we found a data extent we are done.
2171	*/
2172	if (imap.br_startoff <= offset_fsb)
2173	goto done;
2174	data_fsb = imap.br_startoff;
2175	} else {
2176	/*
2177	* Fake a hole until the end of the file.
2178	*/
2179	data_fsb = xfs_iomap_end_fsb(mp, offset, length);
2180	}
2181
2182	/*
2183	* If a COW fork extent covers the hole, report it - capped to the next
2184	* data fork extent:
2185	*/
2186	if (xfs_inode_has_cow_data(ip) &&
2187	xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap))
2188	cow_fsb = cmap.br_startoff;
2189	if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
2190	if (data_fsb < cow_fsb + cmap.br_blockcount)
2191	end_fsb = min(end_fsb, data_fsb);
2192	xfs_trim_extent(&cmap, offset_fsb, end_fsb - offset_fsb);
2193	seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED);
2194	error = xfs_bmbt_to_iomap(ip, iomap, imap: &cmap, mapping_flags: flags,
2195	IOMAP_F_SHARED, sequence_cookie: seq);
2196	/*
2197	* This is a COW extent, so we must probe the page cache
2198	* because there could be dirty page cache being backed
2199	* by this extent.
2200	*/
2201	iomap->type = IOMAP_UNWRITTEN;
2202	goto out_unlock;
2203	}
2204
2205	/*
2206	* Else report a hole, capped to the next found data or COW extent.
2207	*/
2208	if (cow_fsb != NULLFILEOFF && cow_fsb < data_fsb)
2209	imap.br_blockcount = cow_fsb - offset_fsb;
2210	else
2211	imap.br_blockcount = data_fsb - offset_fsb;
2212	imap.br_startoff = offset_fsb;
2213	imap.br_startblock = HOLESTARTBLOCK;
2214	imap.br_state = XFS_EXT_NORM;
2215	done:
2216	seq = xfs_iomap_inode_sequence(ip, iomap_flags: 0);
2217	xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb);
2218	error = xfs_bmbt_to_iomap(ip, iomap, imap: &imap, mapping_flags: flags, iomap_flags: 0, sequence_cookie: seq);
2219	out_unlock:
2220	xfs_iunlock(ip, lockmode);
2221	return error;
2222	}
2223
2224	const struct iomap_ops xfs_seek_iomap_ops = {
2225	.iomap_begin = xfs_seek_iomap_begin,
2226	};
2227
2228	static int
2229	xfs_xattr_iomap_begin(
2230	struct inode *inode,
2231	loff_t offset,
2232	loff_t length,
2233	unsigned flags,
2234	struct iomap *iomap,
2235	struct iomap *srcmap)
2236	{
2237	struct xfs_inode *ip = XFS_I(inode);
2238	struct xfs_mount *mp = ip->i_mount;
2239	xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
2240	xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length);
2241	struct xfs_bmbt_irec imap;
2242	int nimaps = 1, error = 0;
2243	unsigned lockmode;
2244	int seq;
2245
2246	if (xfs_is_shutdown(mp))
2247	return -EIO;
2248
2249	lockmode = xfs_ilock_attr_map_shared(ip);
2250
2251	/* if there are no attribute fork or extents, return ENOENT */
2252	if (!xfs_inode_has_attr_fork(ip) || !ip->i_af.if_nextents) {
2253	error = -ENOENT;
2254	goto out_unlock;
2255	}
2256
2257	ASSERT(ip->i_af.if_format != XFS_DINODE_FMT_LOCAL);
2258	error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap,
2259	&nimaps, XFS_BMAPI_ATTRFORK);
2260	out_unlock:
2261
2262	seq = xfs_iomap_inode_sequence(ip, IOMAP_F_XATTR);
2263	xfs_iunlock(ip, lockmode);
2264
2265	if (error)
2266	return error;
2267	ASSERT(nimaps);
2268	return xfs_bmbt_to_iomap(ip, iomap, imap: &imap, mapping_flags: flags, IOMAP_F_XATTR, sequence_cookie: seq);
2269	}
2270
2271	const struct iomap_ops xfs_xattr_iomap_ops = {
2272	.iomap_begin = xfs_xattr_iomap_begin,
2273	};
2274
2275	int
2276	xfs_zero_range(
2277	struct xfs_inode *ip,
2278	loff_t pos,
2279	loff_t len,
2280	struct xfs_zone_alloc_ctx *ac,
2281	bool *did_zero)
2282	{
2283	struct inode *inode = VFS_I(ip);
2284
2285	xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL);
2286
2287	if (IS_DAX(inode))
2288	return dax_zero_range(inode, pos, len, did_zero,
2289	ops: &xfs_dax_write_iomap_ops);
2290	return iomap_zero_range(inode, pos, len, did_zero,
2291	ops: &xfs_buffered_write_iomap_ops, write_ops: &xfs_iomap_write_ops,
2292	private: ac);
2293	}
2294
2295	int
2296	xfs_truncate_page(
2297	struct xfs_inode *ip,
2298	loff_t pos,
2299	struct xfs_zone_alloc_ctx *ac,
2300	bool *did_zero)
2301	{
2302	struct inode *inode = VFS_I(ip);
2303
2304	if (IS_DAX(inode))
2305	return dax_truncate_page(inode, pos, did_zero,
2306	ops: &xfs_dax_write_iomap_ops);
2307	return iomap_truncate_page(inode, pos, did_zero,
2308	ops: &xfs_buffered_write_iomap_ops, write_ops: &xfs_iomap_write_ops,
2309	private: ac);
2310	}
2311