1	// SPDX-License-Identifier: GPL-2.0
2
3	#include <linux/bitops.h>
4	#include <linux/slab.h>
5	#include <linux/bio.h>
6	#include <linux/mm.h>
7	#include <linux/pagemap.h>
8	#include <linux/page-flags.h>
9	#include <linux/sched/mm.h>
10	#include <linux/spinlock.h>
11	#include <linux/blkdev.h>
12	#include <linux/swap.h>
13	#include <linux/writeback.h>
14	#include <linux/pagevec.h>
15	#include <linux/prefetch.h>
16	#include <linux/fsverity.h>
17	#include "extent_io.h"
18	#include "extent-io-tree.h"
19	#include "extent_map.h"
20	#include "ctree.h"
21	#include "btrfs_inode.h"
22	#include "bio.h"
23	#include "locking.h"
24	#include "backref.h"
25	#include "disk-io.h"
26	#include "subpage.h"
27	#include "zoned.h"
28	#include "block-group.h"
29	#include "compression.h"
30	#include "fs.h"
31	#include "accessors.h"
32	#include "file-item.h"
33	#include "file.h"
34	#include "dev-replace.h"
35	#include "super.h"
36	#include "transaction.h"
37
38	static struct kmem_cache *extent_buffer_cache;
39
40	#ifdef CONFIG_BTRFS_DEBUG
41	static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
42	{
43	struct btrfs_fs_info *fs_info = eb->fs_info;
44	unsigned long flags;
45
46	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
47	list_add(new: &eb->leak_list, head: &fs_info->allocated_ebs);
48	spin_unlock_irqrestore(lock: &fs_info->eb_leak_lock, flags);
49	}
50
51	static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
52	{
53	struct btrfs_fs_info *fs_info = eb->fs_info;
54	unsigned long flags;
55
56	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
57	list_del(entry: &eb->leak_list);
58	spin_unlock_irqrestore(lock: &fs_info->eb_leak_lock, flags);
59	}
60
61	void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
62	{
63	struct extent_buffer *eb;
64	unsigned long flags;
65
66	/*
67	* If we didn't get into open_ctree our allocated_ebs will not be
68	* initialized, so just skip this.
69	*/
70	if (!fs_info->allocated_ebs.next)
71	return;
72
73	WARN_ON(!list_empty(&fs_info->allocated_ebs));
74	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
75	while (!list_empty(head: &fs_info->allocated_ebs)) {
76	eb = list_first_entry(&fs_info->allocated_ebs,
77	struct extent_buffer, leak_list);
78	btrfs_err(fs_info,
79	"buffer leak start %llu len %u refs %d bflags %lu owner %llu",
80	eb->start, eb->len, refcount_read(&eb->refs), eb->bflags,
81	btrfs_header_owner(eb));
82	list_del(entry: &eb->leak_list);
83	WARN_ON_ONCE(1);
84	kmem_cache_free(s: extent_buffer_cache, objp: eb);
85	}
86	spin_unlock_irqrestore(lock: &fs_info->eb_leak_lock, flags);
87	}
88	#else
89	#define btrfs_leak_debug_add_eb(eb) do {} while (0)
90	#define btrfs_leak_debug_del_eb(eb) do {} while (0)
91	#endif
92
93	/*
94	* Structure to record info about the bio being assembled, and other info like
95	* how many bytes are there before stripe/ordered extent boundary.
96	*/
97	struct btrfs_bio_ctrl {
98	struct btrfs_bio *bbio;
99	/* Last byte contained in bbio + 1 . */
100	loff_t next_file_offset;
101	enum btrfs_compression_type compress_type;
102	u32 len_to_oe_boundary;
103	blk_opf_t opf;
104	/*
105	* For data read bios, we attempt to optimize csum lookups if the extent
106	* generation is older than the current one. To make this possible, we
107	* need to track the maximum generation of an extent in a bio_ctrl to
108	* make the decision when submitting the bio.
109	*
110	* The pattern between do_readpage(), submit_one_bio() and
111	* submit_extent_folio() is quite subtle, so tracking this is tricky.
112	*
113	* As we process extent E, we might submit a bio with existing built up
114	* extents before adding E to a new bio, or we might just add E to the
115	* bio. As a result, E's generation could apply to the current bio or
116	* to the next one, so we need to be careful to update the bio_ctrl's
117	* generation with E's only when we are sure E is added to bio_ctrl->bbio
118	* in submit_extent_folio().
119	*
120	* See the comment in btrfs_lookup_bio_sums() for more detail on the
121	* need for this optimization.
122	*/
123	u64 generation;
124	btrfs_bio_end_io_t end_io_func;
125	struct writeback_control *wbc;
126
127	/*
128	* The sectors of the page which are going to be submitted by
129	* extent_writepage_io().
130	* This is to avoid touching ranges covered by compression/inline.
131	*/
132	unsigned long submit_bitmap;
133	struct readahead_control *ractl;
134
135	/*
136	* The start offset of the last used extent map by a read operation.
137	*
138	* This is for proper compressed read merge.
139	* U64_MAX means we are starting the read and have made no progress yet.
140	*
141	* The current btrfs_bio_is_contig() only uses disk_bytenr as
142	* the condition to check if the read can be merged with previous
143	* bio, which is not correct. E.g. two file extents pointing to the
144	* same extent but with different offset.
145	*
146	* So here we need to do extra checks to only merge reads that are
147	* covered by the same extent map.
148	* Just extent_map::start will be enough, as they are unique
149	* inside the same inode.
150	*/
151	u64 last_em_start;
152	};
153
154	/*
155	* Helper to set the csum search commit root option for a bio_ctrl's bbio
156	* before submitting the bio.
157	*
158	* Only for use by submit_one_bio().
159	*/
160	static void bio_set_csum_search_commit_root(struct btrfs_bio_ctrl *bio_ctrl)
161	{
162	struct btrfs_bio *bbio = bio_ctrl->bbio;
163
164	ASSERT(bbio);
165
166	if (!(btrfs_op(bio: &bbio->bio) == BTRFS_MAP_READ && is_data_inode(inode: bbio->inode)))
167	return;
168
169	bio_ctrl->bbio->csum_search_commit_root =
170	(bio_ctrl->generation &&
171	bio_ctrl->generation < btrfs_get_fs_generation(fs_info: bbio->inode->root->fs_info));
172	}
173
174	static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
175	{
176	struct btrfs_bio *bbio = bio_ctrl->bbio;
177
178	if (!bbio)
179	return;
180
181	/* Caller should ensure the bio has at least some range added */
182	ASSERT(bbio->bio.bi_iter.bi_size);
183
184	bio_set_csum_search_commit_root(bio_ctrl);
185
186	if (btrfs_op(bio: &bbio->bio) == BTRFS_MAP_READ &&
187	bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
188	btrfs_submit_compressed_read(bbio);
189	else
190	btrfs_submit_bbio(bbio, mirror_num: 0);
191
192	/* The bbio is owned by the end_io handler now */
193	bio_ctrl->bbio = NULL;
194	/*
195	* We used the generation to decide whether to lookup csums in the
196	* commit_root or not when we called bio_set_csum_search_commit_root()
197	* above. Now, reset the generation for the next bio.
198	*/
199	bio_ctrl->generation = 0;
200	}
201
202	/*
203	* Submit or fail the current bio in the bio_ctrl structure.
204	*/
205	static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
206	{
207	struct btrfs_bio *bbio = bio_ctrl->bbio;
208
209	if (!bbio)
210	return;
211
212	if (ret) {
213	ASSERT(ret < 0);
214	btrfs_bio_end_io(bbio, status: errno_to_blk_status(errno: ret));
215	/* The bio is owned by the end_io handler now */
216	bio_ctrl->bbio = NULL;
217	} else {
218	submit_one_bio(bio_ctrl);
219	}
220	}
221
222	int __init extent_buffer_init_cachep(void)
223	{
224	extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
225	sizeof(struct extent_buffer), 0, 0,
226	NULL);
227	if (!extent_buffer_cache)
228	return -ENOMEM;
229
230	return 0;
231	}
232
233	void __cold extent_buffer_free_cachep(void)
234	{
235	/*
236	* Make sure all delayed rcu free are flushed before we
237	* destroy caches.
238	*/
239	rcu_barrier();
240	kmem_cache_destroy(s: extent_buffer_cache);
241	}
242
243	static void process_one_folio(struct btrfs_fs_info *fs_info,
244	struct folio *folio, const struct folio *locked_folio,
245	unsigned long page_ops, u64 start, u64 end)
246	{
247	u32 len;
248
249	ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
250	len = end + 1 - start;
251
252	if (page_ops & PAGE_SET_ORDERED)
253	btrfs_folio_clamp_set_ordered(fs_info, folio, start, len);
254	if (page_ops & PAGE_START_WRITEBACK) {
255	btrfs_folio_clamp_clear_dirty(fs_info, folio, start, len);
256	btrfs_folio_clamp_set_writeback(fs_info, folio, start, len);
257	}
258	if (page_ops & PAGE_END_WRITEBACK)
259	btrfs_folio_clamp_clear_writeback(fs_info, folio, start, len);
260
261	if (folio != locked_folio && (page_ops & PAGE_UNLOCK))
262	btrfs_folio_end_lock(fs_info, folio, start, len);
263	}
264
265	static void __process_folios_contig(struct address_space *mapping,
266	const struct folio *locked_folio, u64 start,
267	u64 end, unsigned long page_ops)
268	{
269	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
270	pgoff_t index = start >> PAGE_SHIFT;
271	pgoff_t end_index = end >> PAGE_SHIFT;
272	struct folio_batch fbatch;
273	int i;
274
275	folio_batch_init(fbatch: &fbatch);
276	while (index <= end_index) {
277	int found_folios;
278
279	found_folios = filemap_get_folios_contig(mapping, start: &index,
280	end: end_index, fbatch: &fbatch);
281	for (i = 0; i < found_folios; i++) {
282	struct folio *folio = fbatch.folios[i];
283
284	process_one_folio(fs_info, folio, locked_folio,
285	page_ops, start, end);
286	}
287	folio_batch_release(fbatch: &fbatch);
288	cond_resched();
289	}
290	}
291
292	static noinline void unlock_delalloc_folio(const struct inode *inode,
293	struct folio *locked_folio,
294	u64 start, u64 end)
295	{
296	ASSERT(locked_folio);
297
298	__process_folios_contig(mapping: inode->i_mapping, locked_folio, start, end,
299	page_ops: PAGE_UNLOCK);
300	}
301
302	static noinline int lock_delalloc_folios(struct inode *inode,
303	struct folio *locked_folio,
304	u64 start, u64 end)
305	{
306	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
307	struct address_space *mapping = inode->i_mapping;
308	pgoff_t index = start >> PAGE_SHIFT;
309	pgoff_t end_index = end >> PAGE_SHIFT;
310	u64 processed_end = start;
311	struct folio_batch fbatch;
312
313	folio_batch_init(fbatch: &fbatch);
314	while (index <= end_index) {
315	unsigned int found_folios, i;
316
317	found_folios = filemap_get_folios_contig(mapping, start: &index,
318	end: end_index, fbatch: &fbatch);
319	if (found_folios == 0)
320	goto out;
321
322	for (i = 0; i < found_folios; i++) {
323	struct folio *folio = fbatch.folios[i];
324	u64 range_start;
325	u32 range_len;
326
327	if (folio == locked_folio)
328	continue;
329
330	folio_lock(folio);
331	if (!folio_test_dirty(folio) || folio->mapping != mapping) {
332	folio_unlock(folio);
333	goto out;
334	}
335	range_start = max_t(u64, folio_pos(folio), start);
336	range_len = min_t(u64, folio_next_pos(folio), end + 1) - range_start;
337	btrfs_folio_set_lock(fs_info, folio, start: range_start, len: range_len);
338
339	processed_end = range_start + range_len - 1;
340	}
341	folio_batch_release(fbatch: &fbatch);
342	cond_resched();
343	}
344
345	return 0;
346	out:
347	folio_batch_release(fbatch: &fbatch);
348	if (processed_end > start)
349	unlock_delalloc_folio(inode, locked_folio, start, end: processed_end);
350	return -EAGAIN;
351	}
352
353	/*
354	* Find and lock a contiguous range of bytes in the file marked as delalloc, no
355	* more than @max_bytes.
356	*
357	* @start: The original start bytenr to search.
358	* Will store the extent range start bytenr.
359	* @end: The original end bytenr of the search range
360	* Will store the extent range end bytenr.
361	*
362	* Return true if we find a delalloc range which starts inside the original
363	* range, and @start/@end will store the delalloc range start/end.
364	*
365	* Return false if we can't find any delalloc range which starts inside the
366	* original range, and @start/@end will be the non-delalloc range start/end.
367	*/
368	EXPORT_FOR_TESTS
369	noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
370	struct folio *locked_folio,
371	u64 *start, u64 *end)
372	{
373	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
374	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
375	const u64 orig_start = *start;
376	const u64 orig_end = *end;
377	u64 max_bytes = fs_info->max_extent_size;
378	u64 delalloc_start;
379	u64 delalloc_end;
380	bool found;
381	struct extent_state *cached_state = NULL;
382	int ret;
383	int loops = 0;
384
385	/* Caller should pass a valid @end to indicate the search range end */
386	ASSERT(orig_end > orig_start);
387
388	/* The range should at least cover part of the folio */
389	ASSERT(!(orig_start >= folio_next_pos(locked_folio) ||
390	orig_end <= folio_pos(locked_folio)));
391	again:
392	/* step one, find a bunch of delalloc bytes starting at start */
393	delalloc_start = *start;
394	delalloc_end = 0;
395
396	/*
397	* If @max_bytes is smaller than a block, btrfs_find_delalloc_range() can
398	* return early without handling any dirty ranges.
399	*/
400	ASSERT(max_bytes >= fs_info->sectorsize);
401
402	found = btrfs_find_delalloc_range(tree, start: &delalloc_start, end: &delalloc_end,
403	max_bytes, cached_state: &cached_state);
404	if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
405	*start = delalloc_start;
406
407	/* @delalloc_end can be -1, never go beyond @orig_end */
408	*end = min(delalloc_end, orig_end);
409	btrfs_free_extent_state(state: cached_state);
410	return false;
411	}
412
413	/*
414	* start comes from the offset of locked_folio. We have to lock
415	* folios in order, so we can't process delalloc bytes before
416	* locked_folio
417	*/
418	if (delalloc_start < *start)
419	delalloc_start = *start;
420
421	/*
422	* make sure to limit the number of folios we try to lock down
423	*/
424	if (delalloc_end + 1 - delalloc_start > max_bytes)
425	delalloc_end = delalloc_start + max_bytes - 1;
426
427	/* step two, lock all the folios after the folios that has start */
428	ret = lock_delalloc_folios(inode, locked_folio, start: delalloc_start,
429	end: delalloc_end);
430	ASSERT(!ret || ret == -EAGAIN);
431	if (ret == -EAGAIN) {
432	/*
433	* Some of the folios are gone, lets avoid looping by
434	* shortening the size of the delalloc range we're searching.
435	*/
436	btrfs_free_extent_state(state: cached_state);
437	cached_state = NULL;
438	if (!loops) {
439	max_bytes = fs_info->sectorsize;
440	loops = 1;
441	goto again;
442	} else {
443	found = false;
444	goto out_failed;
445	}
446	}
447
448	/* step three, lock the state bits for the whole range */
449	btrfs_lock_extent(tree, start: delalloc_start, end: delalloc_end, cached: &cached_state);
450
451	/* then test to make sure it is all still delalloc */
452	ret = btrfs_test_range_bit(tree, start: delalloc_start, end: delalloc_end,
453	bit: EXTENT_DELALLOC, cached_state);
454
455	btrfs_unlock_extent(tree, start: delalloc_start, end: delalloc_end, cached: &cached_state);
456	if (!ret) {
457	unlock_delalloc_folio(inode, locked_folio, start: delalloc_start,
458	end: delalloc_end);
459	cond_resched();
460	goto again;
461	}
462	*start = delalloc_start;
463	*end = delalloc_end;
464	out_failed:
465	return found;
466	}
467
468	void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
469	const struct folio *locked_folio,
470	struct extent_state **cached,
471	u32 clear_bits, unsigned long page_ops)
472	{
473	btrfs_clear_extent_bit(tree: &inode->io_tree, start, end, bits: clear_bits, cached);
474
475	__process_folios_contig(mapping: inode->vfs_inode.i_mapping, locked_folio, start,
476	end, page_ops);
477	}
478
479	static bool btrfs_verify_folio(struct folio *folio, u64 start, u32 len)
480	{
481	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
482
483	if (!fsverity_active(inode: folio->mapping->host) ||
484	btrfs_folio_test_uptodate(fs_info, folio, start, len) ||
485	start >= i_size_read(inode: folio->mapping->host))
486	return true;
487	return fsverity_verify_folio(folio);
488	}
489
490	static void end_folio_read(struct folio *folio, bool uptodate, u64 start, u32 len)
491	{
492	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
493
494	ASSERT(folio_pos(folio) <= start &&
495	start + len <= folio_next_pos(folio));
496
497	if (uptodate && btrfs_verify_folio(folio, start, len))
498	btrfs_folio_set_uptodate(fs_info, folio, start, len);
499	else
500	btrfs_folio_clear_uptodate(fs_info, folio, start, len);
501
502	if (!btrfs_is_subpage(fs_info, folio))
503	folio_unlock(folio);
504	else
505	btrfs_folio_end_lock(fs_info, folio, start, len);
506	}
507
508	/*
509	* After a write IO is done, we need to:
510	*
511	* - clear the uptodate bits on error
512	* - clear the writeback bits in the extent tree for the range
513	* - filio_end_writeback() if there is no more pending io for the folio
514	*
515	* Scheduling is not allowed, so the extent state tree is expected
516	* to have one and only one object corresponding to this IO.
517	*/
518	static void end_bbio_data_write(struct btrfs_bio *bbio)
519	{
520	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
521	struct bio *bio = &bbio->bio;
522	int error = blk_status_to_errno(status: bio->bi_status);
523	struct folio_iter fi;
524	const u32 sectorsize = fs_info->sectorsize;
525
526	ASSERT(!bio_flagged(bio, BIO_CLONED));
527	bio_for_each_folio_all(fi, bio) {
528	struct folio *folio = fi.folio;
529	u64 start = folio_pos(folio) + fi.offset;
530	u32 len = fi.length;
531
532	/* Our read/write should always be sector aligned. */
533	if (!IS_ALIGNED(fi.offset, sectorsize))
534	btrfs_err(fs_info,
535	"partial page write in btrfs with offset %zu and length %zu",
536	fi.offset, fi.length);
537	else if (!IS_ALIGNED(fi.length, sectorsize))
538	btrfs_info(fs_info,
539	"incomplete page write with offset %zu and length %zu",
540	fi.offset, fi.length);
541
542	btrfs_finish_ordered_extent(ordered: bbio->ordered, folio, file_offset: start, len,
543	uptodate: !error);
544	if (error)
545	mapping_set_error(mapping: folio->mapping, error);
546	btrfs_folio_clear_writeback(fs_info, folio, start, len);
547	}
548
549	bio_put(bio);
550	}
551
552	static void begin_folio_read(struct btrfs_fs_info *fs_info, struct folio *folio)
553	{
554	ASSERT(folio_test_locked(folio));
555	if (!btrfs_is_subpage(fs_info, folio))
556	return;
557
558	ASSERT(folio_test_private(folio));
559	btrfs_folio_set_lock(fs_info, folio, start: folio_pos(folio), len: folio_size(folio));
560	}
561
562	/*
563	* After a data read IO is done, we need to:
564	*
565	* - clear the uptodate bits on error
566	* - set the uptodate bits if things worked
567	* - set the folio up to date if all extents in the tree are uptodate
568	* - clear the lock bit in the extent tree
569	* - unlock the folio if there are no other extents locked for it
570	*
571	* Scheduling is not allowed, so the extent state tree is expected
572	* to have one and only one object corresponding to this IO.
573	*/
574	static void end_bbio_data_read(struct btrfs_bio *bbio)
575	{
576	struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
577	struct bio *bio = &bbio->bio;
578	struct folio_iter fi;
579
580	ASSERT(!bio_flagged(bio, BIO_CLONED));
581	bio_for_each_folio_all(fi, &bbio->bio) {
582	bool uptodate = !bio->bi_status;
583	struct folio *folio = fi.folio;
584	struct inode *inode = folio->mapping->host;
585	u64 start = folio_pos(folio) + fi.offset;
586
587	btrfs_debug(fs_info,
588	"%s: bi_sector=%llu, err=%d, mirror=%u",
589	__func__, bio->bi_iter.bi_sector, bio->bi_status,
590	bbio->mirror_num);
591
592
593	if (likely(uptodate)) {
594	u64 end = start + fi.length - 1;
595	loff_t i_size = i_size_read(inode);
596
597	/*
598	* Zero out the remaining part if this range straddles
599	* i_size.
600	*
601	* Here we should only zero the range inside the folio,
602	* not touch anything else.
603	*
604	* NOTE: i_size is exclusive while end is inclusive and
605	* folio_contains() takes PAGE_SIZE units.
606	*/
607	if (folio_contains(folio, index: i_size >> PAGE_SHIFT) &&
608	i_size <= end) {
609	u32 zero_start = max(offset_in_folio(folio, i_size),
610	offset_in_folio(folio, start));
611	u32 zero_len = offset_in_folio(folio, end) + 1 -
612	zero_start;
613
614	folio_zero_range(folio, start: zero_start, length: zero_len);
615	}
616	}
617
618	/* Update page status and unlock. */
619	end_folio_read(folio, uptodate, start, len: fi.length);
620	}
621	bio_put(bio);
622	}
623
624	/*
625	* Populate every free slot in a provided array with folios using GFP_NOFS.
626	*
627	* @nr_folios: number of folios to allocate
628	* @order: the order of the folios to be allocated
629	* @folio_array: the array to fill with folios; any existing non-NULL entries in
630	* the array will be skipped
631	*
632	* Return: 0 if all folios were able to be allocated;
633	* -ENOMEM otherwise, the partially allocated folios would be freed and
634	* the array slots zeroed
635	*/
636	int btrfs_alloc_folio_array(unsigned int nr_folios, unsigned int order,
637	struct folio **folio_array)
638	{
639	for (int i = 0; i < nr_folios; i++) {
640	if (folio_array[i])
641	continue;
642	folio_array[i] = folio_alloc(GFP_NOFS, order);
643	if (!folio_array[i])
644	goto error;
645	}
646	return 0;
647	error:
648	for (int i = 0; i < nr_folios; i++) {
649	if (folio_array[i])
650	folio_put(folio: folio_array[i]);
651	folio_array[i] = NULL;
652	}
653	return -ENOMEM;
654	}
655
656	/*
657	* Populate every free slot in a provided array with pages, using GFP_NOFS.
658	*
659	* @nr_pages: number of pages to allocate
660	* @page_array: the array to fill with pages; any existing non-null entries in
661	* the array will be skipped
662	* @nofail: whether using __GFP_NOFAIL flag
663	*
664	* Return: 0 if all pages were able to be allocated;
665	* -ENOMEM otherwise, the partially allocated pages would be freed and
666	* the array slots zeroed
667	*/
668	int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array,
669	bool nofail)
670	{
671	const gfp_t gfp = nofail ? (GFP_NOFS | __GFP_NOFAIL) : GFP_NOFS;
672	unsigned int allocated;
673
674	for (allocated = 0; allocated < nr_pages;) {
675	unsigned int last = allocated;
676
677	allocated = alloc_pages_bulk(gfp, nr_pages, page_array);
678	if (unlikely(allocated == last)) {
679	/* No progress, fail and do cleanup. */
680	for (int i = 0; i < allocated; i++) {
681	__free_page(page_array[i]);
682	page_array[i] = NULL;
683	}
684	return -ENOMEM;
685	}
686	}
687	return 0;
688	}
689
690	/*
691	* Populate needed folios for the extent buffer.
692	*
693	* For now, the folios populated are always in order 0 (aka, single page).
694	*/
695	static int alloc_eb_folio_array(struct extent_buffer *eb, bool nofail)
696	{
697	struct page *page_array[INLINE_EXTENT_BUFFER_PAGES] = { 0 };
698	int num_pages = num_extent_pages(eb);
699	int ret;
700
701	ret = btrfs_alloc_page_array(nr_pages: num_pages, page_array, nofail);
702	if (ret < 0)
703	return ret;
704
705	for (int i = 0; i < num_pages; i++)
706	eb->folios[i] = page_folio(page_array[i]);
707	eb->folio_size = PAGE_SIZE;
708	eb->folio_shift = PAGE_SHIFT;
709	return 0;
710	}
711
712	static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
713	u64 disk_bytenr, loff_t file_offset)
714	{
715	struct bio *bio = &bio_ctrl->bbio->bio;
716	const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
717
718	if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
719	/*
720	* For compression, all IO should have its logical bytenr set
721	* to the starting bytenr of the compressed extent.
722	*/
723	return bio->bi_iter.bi_sector == sector;
724	}
725
726	/*
727	* To merge into a bio both the disk sector and the logical offset in
728	* the file need to be contiguous.
729	*/
730	return bio_ctrl->next_file_offset == file_offset &&
731	bio_end_sector(bio) == sector;
732	}
733
734	static void alloc_new_bio(struct btrfs_inode *inode,
735	struct btrfs_bio_ctrl *bio_ctrl,
736	u64 disk_bytenr, u64 file_offset)
737	{
738	struct btrfs_fs_info *fs_info = inode->root->fs_info;
739	struct btrfs_bio *bbio;
740
741	bbio = btrfs_bio_alloc(BIO_MAX_VECS, opf: bio_ctrl->opf, inode,
742	file_offset, end_io: bio_ctrl->end_io_func, NULL);
743	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
744	bbio->bio.bi_write_hint = inode->vfs_inode.i_write_hint;
745	bio_ctrl->bbio = bbio;
746	bio_ctrl->len_to_oe_boundary = U32_MAX;
747	bio_ctrl->next_file_offset = file_offset;
748
749	/* Limit data write bios to the ordered boundary. */
750	if (bio_ctrl->wbc) {
751	struct btrfs_ordered_extent *ordered;
752
753	ordered = btrfs_lookup_ordered_extent(inode, file_offset);
754	if (ordered) {
755	bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
756	ordered->file_offset +
757	ordered->disk_num_bytes - file_offset);
758	bbio->ordered = ordered;
759	}
760
761	/*
762	* Pick the last added device to support cgroup writeback. For
763	* multi-device file systems this means blk-cgroup policies have
764	* to always be set on the last added/replaced device.
765	* This is a bit odd but has been like that for a long time.
766	*/
767	bio_set_dev(bio: &bbio->bio, bdev: fs_info->fs_devices->latest_dev->bdev);
768	wbc_init_bio(wbc: bio_ctrl->wbc, bio: &bbio->bio);
769	}
770	}
771
772	/*
773	* @disk_bytenr: logical bytenr where the write will be
774	* @page: page to add to the bio
775	* @size: portion of page that we want to write to
776	* @pg_offset: offset of the new bio or to check whether we are adding
777	* a contiguous page to the previous one
778	* @read_em_generation: generation of the extent_map we are submitting
779	* (only used for read)
780	*
781	* The will either add the page into the existing @bio_ctrl->bbio, or allocate a
782	* new one in @bio_ctrl->bbio.
783	* The mirror number for this IO should already be initialized in
784	* @bio_ctrl->mirror_num.
785	*/
786	static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl,
787	u64 disk_bytenr, struct folio *folio,
788	size_t size, unsigned long pg_offset,
789	u64 read_em_generation)
790	{
791	struct btrfs_inode *inode = folio_to_inode(folio);
792	loff_t file_offset = folio_pos(folio) + pg_offset;
793
794	ASSERT(pg_offset + size <= folio_size(folio));
795	ASSERT(bio_ctrl->end_io_func);
796
797	if (bio_ctrl->bbio &&
798	!btrfs_bio_is_contig(bio_ctrl, disk_bytenr, file_offset))
799	submit_one_bio(bio_ctrl);
800
801	do {
802	u32 len = size;
803
804	/* Allocate new bio if needed */
805	if (!bio_ctrl->bbio)
806	alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset);
807
808	/* Cap to the current ordered extent boundary if there is one. */
809	if (len > bio_ctrl->len_to_oe_boundary) {
810	ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
811	ASSERT(is_data_inode(inode));
812	len = bio_ctrl->len_to_oe_boundary;
813	}
814
815	if (!bio_add_folio(bio: &bio_ctrl->bbio->bio, folio, len, off: pg_offset)) {
816	/* bio full: move on to a new one */
817	submit_one_bio(bio_ctrl);
818	continue;
819	}
820	/*
821	* Now that the folio is definitely added to the bio, include its
822	* generation in the max generation calculation.
823	*/
824	bio_ctrl->generation = max(bio_ctrl->generation, read_em_generation);
825	bio_ctrl->next_file_offset += len;
826
827	if (bio_ctrl->wbc)
828	wbc_account_cgroup_owner(wbc: bio_ctrl->wbc, folio, bytes: len);
829
830	size -= len;
831	pg_offset += len;
832	disk_bytenr += len;
833	file_offset += len;
834
835	/*
836	* len_to_oe_boundary defaults to U32_MAX, which isn't folio or
837	* sector aligned. alloc_new_bio() then sets it to the end of
838	* our ordered extent for writes into zoned devices.
839	*
840	* When len_to_oe_boundary is tracking an ordered extent, we
841	* trust the ordered extent code to align things properly, and
842	* the check above to cap our write to the ordered extent
843	* boundary is correct.
844	*
845	* When len_to_oe_boundary is U32_MAX, the cap above would
846	* result in a 4095 byte IO for the last folio right before
847	* we hit the bio limit of UINT_MAX. bio_add_folio() has all
848	* the checks required to make sure we don't overflow the bio,
849	* and we should just ignore len_to_oe_boundary completely
850	* unless we're using it to track an ordered extent.
851	*
852	* It's pretty hard to make a bio sized U32_MAX, but it can
853	* happen when the page cache is able to feed us contiguous
854	* folios for large extents.
855	*/
856	if (bio_ctrl->len_to_oe_boundary != U32_MAX)
857	bio_ctrl->len_to_oe_boundary -= len;
858
859	/* Ordered extent boundary: move on to a new bio. */
860	if (bio_ctrl->len_to_oe_boundary == 0)
861	submit_one_bio(bio_ctrl);
862	} while (size);
863	}
864
865	static int attach_extent_buffer_folio(struct extent_buffer *eb,
866	struct folio *folio,
867	struct btrfs_folio_state *prealloc)
868	{
869	struct btrfs_fs_info *fs_info = eb->fs_info;
870	int ret = 0;
871
872	/*
873	* If the page is mapped to btree inode, we should hold the private
874	* lock to prevent race.
875	* For cloned or dummy extent buffers, their pages are not mapped and
876	* will not race with any other ebs.
877	*/
878	if (folio->mapping)
879	lockdep_assert_held(&folio->mapping->i_private_lock);
880
881	if (!btrfs_meta_is_subpage(fs_info)) {
882	if (!folio_test_private(folio))
883	folio_attach_private(folio, data: eb);
884	else
885	WARN_ON(folio_get_private(folio) != eb);
886	return 0;
887	}
888
889	/* Already mapped, just free prealloc */
890	if (folio_test_private(folio)) {
891	btrfs_free_folio_state(bfs: prealloc);
892	return 0;
893	}
894
895	if (prealloc)
896	/* Has preallocated memory for subpage */
897	folio_attach_private(folio, data: prealloc);
898	else
899	/* Do new allocation to attach subpage */
900	ret = btrfs_attach_folio_state(fs_info, folio, type: BTRFS_SUBPAGE_METADATA);
901	return ret;
902	}
903
904	int set_folio_extent_mapped(struct folio *folio)
905	{
906	struct btrfs_fs_info *fs_info;
907
908	ASSERT(folio->mapping);
909
910	if (folio_test_private(folio))
911	return 0;
912
913	fs_info = folio_to_fs_info(folio);
914
915	if (btrfs_is_subpage(fs_info, folio))
916	return btrfs_attach_folio_state(fs_info, folio, type: BTRFS_SUBPAGE_DATA);
917
918	folio_attach_private(folio, data: (void *)EXTENT_FOLIO_PRIVATE);
919	return 0;
920	}
921
922	void clear_folio_extent_mapped(struct folio *folio)
923	{
924	struct btrfs_fs_info *fs_info;
925
926	ASSERT(folio->mapping);
927
928	if (!folio_test_private(folio))
929	return;
930
931	fs_info = folio_to_fs_info(folio);
932	if (btrfs_is_subpage(fs_info, folio))
933	return btrfs_detach_folio_state(fs_info, folio, type: BTRFS_SUBPAGE_DATA);
934
935	folio_detach_private(folio);
936	}
937
938	static struct extent_map *get_extent_map(struct btrfs_inode *inode,
939	struct folio *folio, u64 start,
940	u64 len, struct extent_map **em_cached)
941	{
942	struct extent_map *em;
943
944	ASSERT(em_cached);
945
946	if (*em_cached) {
947	em = *em_cached;
948	if (btrfs_extent_map_in_tree(em) && start >= em->start &&
949	start < btrfs_extent_map_end(em)) {
950	refcount_inc(r: &em->refs);
951	return em;
952	}
953
954	btrfs_free_extent_map(em);
955	*em_cached = NULL;
956	}
957
958	em = btrfs_get_extent(inode, folio, start, len);
959	if (!IS_ERR(ptr: em)) {
960	BUG_ON(*em_cached);
961	refcount_inc(r: &em->refs);
962	*em_cached = em;
963	}
964
965	return em;
966	}
967
968	static void btrfs_readahead_expand(struct readahead_control *ractl,
969	const struct extent_map *em)
970	{
971	const u64 ra_pos = readahead_pos(rac: ractl);
972	const u64 ra_end = ra_pos + readahead_length(rac: ractl);
973	const u64 em_end = em->start + em->len;
974
975	/* No expansion for holes and inline extents. */
976	if (em->disk_bytenr > EXTENT_MAP_LAST_BYTE)
977	return;
978
979	ASSERT(em_end >= ra_pos,
980	"extent_map %llu %llu ends before current readahead position %llu",
981	em->start, em->len, ra_pos);
982	if (em_end > ra_end)
983	readahead_expand(ractl, new_start: ra_pos, new_len: em_end - ra_pos);
984	}
985
986	/*
987	* basic readpage implementation. Locked extent state structs are inserted
988	* into the tree that are removed when the IO is done (by the end_io
989	* handlers)
990	* XXX JDM: This needs looking at to ensure proper page locking
991	* return 0 on success, otherwise return error
992	*/
993	static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
994	struct btrfs_bio_ctrl *bio_ctrl)
995	{
996	struct inode *inode = folio->mapping->host;
997	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
998	u64 start = folio_pos(folio);
999	const u64 end = start + folio_size(folio) - 1;
1000	u64 extent_offset;
1001	u64 last_byte = i_size_read(inode);
1002	struct extent_map *em;
1003	int ret = 0;
1004	const size_t blocksize = fs_info->sectorsize;
1005
1006	ret = set_folio_extent_mapped(folio);
1007	if (ret < 0) {
1008	folio_unlock(folio);
1009	return ret;
1010	}
1011
1012	if (folio_contains(folio, index: last_byte >> PAGE_SHIFT)) {
1013	size_t zero_offset = offset_in_folio(folio, last_byte);
1014
1015	if (zero_offset)
1016	folio_zero_range(folio, start: zero_offset,
1017	length: folio_size(folio) - zero_offset);
1018	}
1019	bio_ctrl->end_io_func = end_bbio_data_read;
1020	begin_folio_read(fs_info, folio);
1021	for (u64 cur = start; cur <= end; cur += blocksize) {
1022	enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
1023	unsigned long pg_offset = offset_in_folio(folio, cur);
1024	bool force_bio_submit = false;
1025	u64 disk_bytenr;
1026	u64 block_start;
1027	u64 em_gen;
1028
1029	ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
1030	if (cur >= last_byte) {
1031	folio_zero_range(folio, start: pg_offset, length: end - cur + 1);
1032	end_folio_read(folio, uptodate: true, start: cur, len: end - cur + 1);
1033	break;
1034	}
1035	if (btrfs_folio_test_uptodate(fs_info, folio, start: cur, len: blocksize)) {
1036	end_folio_read(folio, uptodate: true, start: cur, len: blocksize);
1037	continue;
1038	}
1039	em = get_extent_map(BTRFS_I(inode), folio, start: cur, len: end - cur + 1, em_cached);
1040	if (IS_ERR(ptr: em)) {
1041	end_folio_read(folio, uptodate: false, start: cur, len: end + 1 - cur);
1042	return PTR_ERR(ptr: em);
1043	}
1044	extent_offset = cur - em->start;
1045	BUG_ON(btrfs_extent_map_end(em) <= cur);
1046	BUG_ON(end < cur);
1047
1048	compress_type = btrfs_extent_map_compression(em);
1049
1050	/*
1051	* Only expand readahead for extents which are already creating
1052	* the pages anyway in add_ra_bio_pages, which is compressed
1053	* extents in the non subpage case.
1054	*/
1055	if (bio_ctrl->ractl &&
1056	!btrfs_is_subpage(fs_info, folio) &&
1057	compress_type != BTRFS_COMPRESS_NONE)
1058	btrfs_readahead_expand(ractl: bio_ctrl->ractl, em);
1059
1060	if (compress_type != BTRFS_COMPRESS_NONE)
1061	disk_bytenr = em->disk_bytenr;
1062	else
1063	disk_bytenr = btrfs_extent_map_block_start(em) + extent_offset;
1064
1065	if (em->flags & EXTENT_FLAG_PREALLOC)
1066	block_start = EXTENT_MAP_HOLE;
1067	else
1068	block_start = btrfs_extent_map_block_start(em);
1069
1070	/*
1071	* If we have a file range that points to a compressed extent
1072	* and it's followed by a consecutive file range that points
1073	* to the same compressed extent (possibly with a different
1074	* offset and/or length, so it either points to the whole extent
1075	* or only part of it), we must make sure we do not submit a
1076	* single bio to populate the folios for the 2 ranges because
1077	* this makes the compressed extent read zero out the folios
1078	* belonging to the 2nd range. Imagine the following scenario:
1079	*
1080	* File layout
1081	* [0 - 8K] [8K - 24K]
1082	* | |
1083	* | |
1084	* points to extent X, points to extent X,
1085	* offset 4K, length of 8K offset 0, length 16K
1086	*
1087	* [extent X, compressed length = 4K uncompressed length = 16K]
1088	*
1089	* If the bio to read the compressed extent covers both ranges,
1090	* it will decompress extent X into the folios belonging to the
1091	* first range and then it will stop, zeroing out the remaining
1092	* folios that belong to the other range that points to extent X.
1093	* So here we make sure we submit 2 bios, one for the first
1094	* range and another one for the third range. Both will target
1095	* the same physical extent from disk, but we can't currently
1096	* make the compressed bio endio callback populate the folios
1097	* for both ranges because each compressed bio is tightly
1098	* coupled with a single extent map, and each range can have
1099	* an extent map with a different offset value relative to the
1100	* uncompressed data of our extent and different lengths. This
1101	* is a corner case so we prioritize correctness over
1102	* non-optimal behavior (submitting 2 bios for the same extent).
1103	*/
1104	if (compress_type != BTRFS_COMPRESS_NONE &&
1105	bio_ctrl->last_em_start != U64_MAX &&
1106	bio_ctrl->last_em_start != em->start)
1107	force_bio_submit = true;
1108
1109	bio_ctrl->last_em_start = em->start;
1110
1111	em_gen = em->generation;
1112	btrfs_free_extent_map(em);
1113	em = NULL;
1114
1115	/* we've found a hole, just zero and go on */
1116	if (block_start == EXTENT_MAP_HOLE) {
1117	folio_zero_range(folio, start: pg_offset, length: blocksize);
1118	end_folio_read(folio, uptodate: true, start: cur, len: blocksize);
1119	continue;
1120	}
1121	/* the get_extent function already copied into the folio */
1122	if (block_start == EXTENT_MAP_INLINE) {
1123	end_folio_read(folio, uptodate: true, start: cur, len: blocksize);
1124	continue;
1125	}
1126
1127	if (bio_ctrl->compress_type != compress_type) {
1128	submit_one_bio(bio_ctrl);
1129	bio_ctrl->compress_type = compress_type;
1130	}
1131
1132	if (force_bio_submit)
1133	submit_one_bio(bio_ctrl);
1134	submit_extent_folio(bio_ctrl, disk_bytenr, folio, size: blocksize,
1135	pg_offset, read_em_generation: em_gen);
1136	}
1137	return 0;
1138	}
1139
1140	/*
1141	* Check if we can skip waiting the @ordered extent covering the block at @fileoff.
1142	*
1143	* @fileoff: Both input and output.
1144	* Input as the file offset where the check should start at.
1145	* Output as where the next check should start at,
1146	* if the function returns true.
1147	*
1148	* Return true if we can skip to @fileoff. The caller needs to check the new
1149	* @fileoff value to make sure it covers the full range, before skipping the
1150	* full OE.
1151	*
1152	* Return false if we must wait for the ordered extent.
1153	*/
1154	static bool can_skip_one_ordered_range(struct btrfs_inode *inode,
1155	struct btrfs_ordered_extent *ordered,
1156	u64 *fileoff)
1157	{
1158	const struct btrfs_fs_info *fs_info = inode->root->fs_info;
1159	struct folio *folio;
1160	const u32 blocksize = fs_info->sectorsize;
1161	u64 cur = *fileoff;
1162	bool ret;
1163
1164	folio = filemap_get_folio(mapping: inode->vfs_inode.i_mapping, index: cur >> PAGE_SHIFT);
1165
1166	/*
1167	* We should have locked the folio(s) for range [start, end], thus
1168	* there must be a folio and it must be locked.
1169	*/
1170	ASSERT(!IS_ERR(folio));
1171	ASSERT(folio_test_locked(folio));
1172
1173	/*
1174	* There are several cases for the folio and OE combination:
1175	*
1176	* 1) Folio has no private flag
1177	* The OE has all its IO done but not yet finished, and folio got
1178	* invalidated.
1179	*
1180	* Have we have to wait for the OE to finish, as it may contain the
1181	* to-be-inserted data checksum.
1182	* Without the data checksum inserted into the csum tree, read will
1183	* just fail with missing csum.
1184	*/
1185	if (!folio_test_private(folio)) {
1186	ret = false;
1187	goto out;
1188	}
1189
1190	/*
1191	* 2) The first block is DIRTY.
1192	*
1193	* This means the OE is created by some other folios whose file pos is
1194	* before this one. And since we are holding the folio lock, the writeback
1195	* of this folio cannot start.
1196	*
1197	* We must skip the whole OE, because it will never start until we
1198	* finished our folio read and unlocked the folio.
1199	*/
1200	if (btrfs_folio_test_dirty(fs_info, folio, start: cur, len: blocksize)) {
1201	u64 range_len = umin(folio_next_pos(folio),
1202	ordered->file_offset + ordered->num_bytes) - cur;
1203
1204	ret = true;
1205	/*
1206	* At least inside the folio, all the remaining blocks should
1207	* also be dirty.
1208	*/
1209	ASSERT(btrfs_folio_test_dirty(fs_info, folio, cur, range_len));
1210	*fileoff = ordered->file_offset + ordered->num_bytes;
1211	goto out;
1212	}
1213
1214	/*
1215	* 3) The first block is uptodate.
1216	*
1217	* At least the first block can be skipped, but we are still not fully
1218	* sure. E.g. if the OE has some other folios in the range that cannot
1219	* be skipped.
1220	* So we return true and update @next_ret to the OE/folio boundary.
1221	*/
1222	if (btrfs_folio_test_uptodate(fs_info, folio, start: cur, len: blocksize)) {
1223	u64 range_len = umin(folio_next_pos(folio),
1224	ordered->file_offset + ordered->num_bytes) - cur;
1225
1226	/*
1227	* The whole range to the OE end or folio boundary should also
1228	* be uptodate.
1229	*/
1230	ASSERT(btrfs_folio_test_uptodate(fs_info, folio, cur, range_len));
1231	ret = true;
1232	*fileoff = cur + range_len;
1233	goto out;
1234	}
1235
1236	/*
1237	* 4) The first block is not uptodate.
1238	*
1239	* This means the folio is invalidated after the writeback was finished,
1240	* but by some other operations (e.g. block aligned buffered write) the
1241	* folio is inserted into filemap.
1242	* Very much the same as case 1).
1243	*/
1244	ret = false;
1245	out:
1246	folio_put(folio);
1247	return ret;
1248	}
1249
1250	static bool can_skip_ordered_extent(struct btrfs_inode *inode,
1251	struct btrfs_ordered_extent *ordered,
1252	u64 start, u64 end)
1253	{
1254	const u64 range_end = min(end, ordered->file_offset + ordered->num_bytes - 1);
1255	u64 cur = max(start, ordered->file_offset);
1256
1257	while (cur < range_end) {
1258	bool can_skip;
1259
1260	can_skip = can_skip_one_ordered_range(inode, ordered, fileoff: &cur);
1261	if (!can_skip)
1262	return false;
1263	}
1264	return true;
1265	}
1266
1267	/*
1268	* Locking helper to make sure we get a stable view of extent maps for the
1269	* involved range.
1270	*
1271	* This is for folio read paths (read and readahead), thus the involved range
1272	* should have all the folios locked.
1273	*/
1274	static void lock_extents_for_read(struct btrfs_inode *inode, u64 start, u64 end,
1275	struct extent_state **cached_state)
1276	{
1277	u64 cur_pos;
1278
1279	/* Caller must provide a valid @cached_state. */
1280	ASSERT(cached_state);
1281
1282	/* The range must at least be page aligned, as all read paths are folio based. */
1283	ASSERT(IS_ALIGNED(start, PAGE_SIZE));
1284	ASSERT(IS_ALIGNED(end + 1, PAGE_SIZE));
1285
1286	again:
1287	btrfs_lock_extent(tree: &inode->io_tree, start, end, cached: cached_state);
1288	cur_pos = start;
1289	while (cur_pos < end) {
1290	struct btrfs_ordered_extent *ordered;
1291
1292	ordered = btrfs_lookup_ordered_range(inode, file_offset: cur_pos,
1293	len: end - cur_pos + 1);
1294	/*
1295	* No ordered extents in the range, and we hold the extent lock,
1296	* no one can modify the extent maps in the range, we're safe to return.
1297	*/
1298	if (!ordered)
1299	break;
1300
1301	/* Check if we can skip waiting for the whole OE. */
1302	if (can_skip_ordered_extent(inode, ordered, start, end)) {
1303	cur_pos = min(ordered->file_offset + ordered->num_bytes,
1304	end + 1);
1305	btrfs_put_ordered_extent(entry: ordered);
1306	continue;
1307	}
1308
1309	/* Now wait for the OE to finish. */
1310	btrfs_unlock_extent(tree: &inode->io_tree, start, end, cached: cached_state);
1311	btrfs_start_ordered_extent_nowriteback(entry: ordered, nowriteback_start: start, nowriteback_len: end + 1 - start);
1312	btrfs_put_ordered_extent(entry: ordered);
1313	/* We have unlocked the whole range, restart from the beginning. */
1314	goto again;
1315	}
1316	}
1317
1318	int btrfs_read_folio(struct file *file, struct folio *folio)
1319	{
1320	struct btrfs_inode *inode = folio_to_inode(folio);
1321	const u64 start = folio_pos(folio);
1322	const u64 end = start + folio_size(folio) - 1;
1323	struct extent_state *cached_state = NULL;
1324	struct btrfs_bio_ctrl bio_ctrl = {
1325	.opf = REQ_OP_READ,
1326	.last_em_start = U64_MAX,
1327	};
1328	struct extent_map *em_cached = NULL;
1329	int ret;
1330
1331	lock_extents_for_read(inode, start, end, cached_state: &cached_state);
1332	ret = btrfs_do_readpage(folio, em_cached: &em_cached, bio_ctrl: &bio_ctrl);
1333	btrfs_unlock_extent(tree: &inode->io_tree, start, end, cached: &cached_state);
1334
1335	btrfs_free_extent_map(em: em_cached);
1336
1337	/*
1338	* If btrfs_do_readpage() failed we will want to submit the assembled
1339	* bio to do the cleanup.
1340	*/
1341	submit_one_bio(bio_ctrl: &bio_ctrl);
1342	return ret;
1343	}
1344
1345	static void set_delalloc_bitmap(struct folio *folio, unsigned long *delalloc_bitmap,
1346	u64 start, u32 len)
1347	{
1348	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
1349	const u64 folio_start = folio_pos(folio);
1350	unsigned int start_bit;
1351	unsigned int nbits;
1352
1353	ASSERT(start >= folio_start && start + len <= folio_start + folio_size(folio));
1354	start_bit = (start - folio_start) >> fs_info->sectorsize_bits;
1355	nbits = len >> fs_info->sectorsize_bits;
1356	ASSERT(bitmap_test_range_all_zero(delalloc_bitmap, start_bit, nbits));
1357	bitmap_set(map: delalloc_bitmap, start: start_bit, nbits);
1358	}
1359
1360	static bool find_next_delalloc_bitmap(struct folio *folio,
1361	unsigned long *delalloc_bitmap, u64 start,
1362	u64 *found_start, u32 *found_len)
1363	{
1364	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
1365	const u64 folio_start = folio_pos(folio);
1366	const unsigned int bitmap_size = btrfs_blocks_per_folio(fs_info, folio);
1367	unsigned int start_bit;
1368	unsigned int first_zero;
1369	unsigned int first_set;
1370
1371	ASSERT(start >= folio_start && start < folio_start + folio_size(folio));
1372
1373	start_bit = (start - folio_start) >> fs_info->sectorsize_bits;
1374	first_set = find_next_bit(addr: delalloc_bitmap, size: bitmap_size, offset: start_bit);
1375	if (first_set >= bitmap_size)
1376	return false;
1377
1378	*found_start = folio_start + (first_set << fs_info->sectorsize_bits);
1379	first_zero = find_next_zero_bit(addr: delalloc_bitmap, size: bitmap_size, offset: first_set);
1380	*found_len = (first_zero - first_set) << fs_info->sectorsize_bits;
1381	return true;
1382	}
1383
1384	/*
1385	* Do all of the delayed allocation setup.
1386	*
1387	* Return >0 if all the dirty blocks are submitted async (compression) or inlined.
1388	* The @folio should no longer be touched (treat it as already unlocked).
1389	*
1390	* Return 0 if there is still dirty block that needs to be submitted through
1391	* extent_writepage_io().
1392	* bio_ctrl->submit_bitmap will indicate which blocks of the folio should be
1393	* submitted, and @folio is still kept locked.
1394	*
1395	* Return <0 if there is any error hit.
1396	* Any allocated ordered extent range covering this folio will be marked
1397	* finished (IOERR), and @folio is still kept locked.
1398	*/
1399	static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1400	struct folio *folio,
1401	struct btrfs_bio_ctrl *bio_ctrl)
1402	{
1403	struct btrfs_fs_info *fs_info = inode_to_fs_info(&inode->vfs_inode);
1404	struct writeback_control *wbc = bio_ctrl->wbc;
1405	const bool is_subpage = btrfs_is_subpage(fs_info, folio);
1406	const u64 page_start = folio_pos(folio);
1407	const u64 page_end = page_start + folio_size(folio) - 1;
1408	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1409	unsigned long delalloc_bitmap = 0;
1410	/*
1411	* Save the last found delalloc end. As the delalloc end can go beyond
1412	* page boundary, thus we cannot rely on subpage bitmap to locate the
1413	* last delalloc end.
1414	*/
1415	u64 last_delalloc_end = 0;
1416	/*
1417	* The range end (exclusive) of the last successfully finished delalloc
1418	* range.
1419	* Any range covered by ordered extent must either be manually marked
1420	* finished (error handling), or has IO submitted (and finish the
1421	* ordered extent normally).
1422	*
1423	* This records the end of ordered extent cleanup if we hit an error.
1424	*/
1425	u64 last_finished_delalloc_end = page_start;
1426	u64 delalloc_start = page_start;
1427	u64 delalloc_end = page_end;
1428	u64 delalloc_to_write = 0;
1429	int ret = 0;
1430	int bit;
1431
1432	/* Save the dirty bitmap as our submission bitmap will be a subset of it. */
1433	if (btrfs_is_subpage(fs_info, folio)) {
1434	ASSERT(blocks_per_folio > 1);
1435	btrfs_get_subpage_dirty_bitmap(fs_info, folio, ret_bitmap: &bio_ctrl->submit_bitmap);
1436	} else {
1437	bio_ctrl->submit_bitmap = 1;
1438	}
1439
1440	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, blocks_per_folio) {
1441	u64 start = page_start + (bit << fs_info->sectorsize_bits);
1442
1443	btrfs_folio_set_lock(fs_info, folio, start, len: fs_info->sectorsize);
1444	}
1445
1446	/* Lock all (subpage) delalloc ranges inside the folio first. */
1447	while (delalloc_start < page_end) {
1448	delalloc_end = page_end;
1449	if (!find_lock_delalloc_range(inode: &inode->vfs_inode, locked_folio: folio,
1450	start: &delalloc_start, end: &delalloc_end)) {
1451	delalloc_start = delalloc_end + 1;
1452	continue;
1453	}
1454	set_delalloc_bitmap(folio, delalloc_bitmap: &delalloc_bitmap, start: delalloc_start,
1455	min(delalloc_end, page_end) + 1 - delalloc_start);
1456	last_delalloc_end = delalloc_end;
1457	delalloc_start = delalloc_end + 1;
1458	}
1459	delalloc_start = page_start;
1460
1461	if (!last_delalloc_end)
1462	goto out;
1463
1464	/* Run the delalloc ranges for the above locked ranges. */
1465	while (delalloc_start < page_end) {
1466	u64 found_start;
1467	u32 found_len;
1468	bool found;
1469
1470	if (!is_subpage) {
1471	/*
1472	* For non-subpage case, the found delalloc range must
1473	* cover this folio and there must be only one locked
1474	* delalloc range.
1475	*/
1476	found_start = page_start;
1477	found_len = last_delalloc_end + 1 - found_start;
1478	found = true;
1479	} else {
1480	found = find_next_delalloc_bitmap(folio, delalloc_bitmap: &delalloc_bitmap,
1481	start: delalloc_start, found_start: &found_start, found_len: &found_len);
1482	}
1483	if (!found)
1484	break;
1485	/*
1486	* The subpage range covers the last sector, the delalloc range may
1487	* end beyond the folio boundary, use the saved delalloc_end
1488	* instead.
1489	*/
1490	if (found_start + found_len >= page_end)
1491	found_len = last_delalloc_end + 1 - found_start;
1492
1493	if (ret >= 0) {
1494	/*
1495	* Some delalloc range may be created by previous folios.
1496	* Thus we still need to clean up this range during error
1497	* handling.
1498	*/
1499	last_finished_delalloc_end = found_start;
1500	/* No errors hit so far, run the current delalloc range. */
1501	ret = btrfs_run_delalloc_range(inode, locked_folio: folio,
1502	start: found_start,
1503	end: found_start + found_len - 1,
1504	wbc);
1505	if (ret >= 0)
1506	last_finished_delalloc_end = found_start + found_len;
1507	if (unlikely(ret < 0))
1508	btrfs_err_rl(fs_info,
1509	"failed to run delalloc range, root=%lld ino=%llu folio=%llu submit_bitmap=%*pbl start=%llu len=%u: %d",
1510	btrfs_root_id(inode->root),
1511	btrfs_ino(inode),
1512	folio_pos(folio),
1513	blocks_per_folio,
1514	&bio_ctrl->submit_bitmap,
1515	found_start, found_len, ret);
1516	} else {
1517	/*
1518	* We've hit an error during previous delalloc range,
1519	* have to cleanup the remaining locked ranges.
1520	*/
1521	btrfs_unlock_extent(tree: &inode->io_tree, start: found_start,
1522	end: found_start + found_len - 1, NULL);
1523	unlock_delalloc_folio(inode: &inode->vfs_inode, locked_folio: folio,
1524	start: found_start,
1525	end: found_start + found_len - 1);
1526	}
1527
1528	/*
1529	* We have some ranges that's going to be submitted asynchronously
1530	* (compression or inline). These range have their own control
1531	* on when to unlock the pages. We should not touch them
1532	* anymore, so clear the range from the submission bitmap.
1533	*/
1534	if (ret > 0) {
1535	unsigned int start_bit = (found_start - page_start) >>
1536	fs_info->sectorsize_bits;
1537	unsigned int end_bit = (min(page_end + 1, found_start + found_len) -
1538	page_start) >> fs_info->sectorsize_bits;
1539	bitmap_clear(map: &bio_ctrl->submit_bitmap, start: start_bit, nbits: end_bit - start_bit);
1540	}
1541	/*
1542	* Above btrfs_run_delalloc_range() may have unlocked the folio,
1543	* thus for the last range, we cannot touch the folio anymore.
1544	*/
1545	if (found_start + found_len >= last_delalloc_end + 1)
1546	break;
1547
1548	delalloc_start = found_start + found_len;
1549	}
1550	/*
1551	* It's possible we had some ordered extents created before we hit
1552	* an error, cleanup non-async successfully created delalloc ranges.
1553	*/
1554	if (unlikely(ret < 0)) {
1555	unsigned int bitmap_size = min(
1556	(last_finished_delalloc_end - page_start) >>
1557	fs_info->sectorsize_bits,
1558	blocks_per_folio);
1559
1560	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, bitmap_size)
1561	btrfs_mark_ordered_io_finished(inode, folio,
1562	file_offset: page_start + (bit << fs_info->sectorsize_bits),
1563	num_bytes: fs_info->sectorsize, uptodate: false);
1564	return ret;
1565	}
1566	out:
1567	if (last_delalloc_end)
1568	delalloc_end = last_delalloc_end;
1569	else
1570	delalloc_end = page_end;
1571	/*
1572	* delalloc_end is already one less than the total length, so
1573	* we don't subtract one from PAGE_SIZE.
1574	*/
1575	delalloc_to_write +=
1576	DIV_ROUND_UP(delalloc_end + 1 - page_start, PAGE_SIZE);
1577
1578	/*
1579	* If all ranges are submitted asynchronously, we just need to account
1580	* for them here.
1581	*/
1582	if (bitmap_empty(src: &bio_ctrl->submit_bitmap, nbits: blocks_per_folio)) {
1583	wbc->nr_to_write -= delalloc_to_write;
1584	return 1;
1585	}
1586
1587	if (wbc->nr_to_write < delalloc_to_write) {
1588	int thresh = 8192;
1589
1590	if (delalloc_to_write < thresh * 2)
1591	thresh = delalloc_to_write;
1592	wbc->nr_to_write = min_t(u64, delalloc_to_write,
1593	thresh);
1594	}
1595
1596	return 0;
1597	}
1598
1599	/*
1600	* Return 0 if we have submitted or queued the sector for submission.
1601	* Return <0 for critical errors, and the sector will have its dirty flag cleared.
1602	*
1603	* Caller should make sure filepos < i_size and handle filepos >= i_size case.
1604	*/
1605	static int submit_one_sector(struct btrfs_inode *inode,
1606	struct folio *folio,
1607	u64 filepos, struct btrfs_bio_ctrl *bio_ctrl,
1608	loff_t i_size)
1609	{
1610	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1611	struct extent_map *em;
1612	u64 block_start;
1613	u64 disk_bytenr;
1614	u64 extent_offset;
1615	u64 em_end;
1616	const u32 sectorsize = fs_info->sectorsize;
1617
1618	ASSERT(IS_ALIGNED(filepos, sectorsize));
1619
1620	/* @filepos >= i_size case should be handled by the caller. */
1621	ASSERT(filepos < i_size);
1622
1623	em = btrfs_get_extent(inode, NULL, start: filepos, len: sectorsize);
1624	if (IS_ERR(ptr: em)) {
1625	/*
1626	* When submission failed, we should still clear the folio dirty.
1627	* Or the folio will be written back again but without any
1628	* ordered extent.
1629	*/
1630	btrfs_folio_clear_dirty(fs_info, folio, start: filepos, len: sectorsize);
1631	btrfs_folio_set_writeback(fs_info, folio, start: filepos, len: sectorsize);
1632	btrfs_folio_clear_writeback(fs_info, folio, start: filepos, len: sectorsize);
1633	return PTR_ERR(ptr: em);
1634	}
1635
1636	extent_offset = filepos - em->start;
1637	em_end = btrfs_extent_map_end(em);
1638	ASSERT(filepos <= em_end);
1639	ASSERT(IS_ALIGNED(em->start, sectorsize));
1640	ASSERT(IS_ALIGNED(em->len, sectorsize));
1641
1642	block_start = btrfs_extent_map_block_start(em);
1643	disk_bytenr = btrfs_extent_map_block_start(em) + extent_offset;
1644
1645	ASSERT(!btrfs_extent_map_is_compressed(em));
1646	ASSERT(block_start != EXTENT_MAP_HOLE);
1647	ASSERT(block_start != EXTENT_MAP_INLINE);
1648
1649	btrfs_free_extent_map(em);
1650	em = NULL;
1651
1652	/*
1653	* Although the PageDirty bit is cleared before entering this
1654	* function, subpage dirty bit is not cleared.
1655	* So clear subpage dirty bit here so next time we won't submit
1656	* a folio for a range already written to disk.
1657	*/
1658	btrfs_folio_clear_dirty(fs_info, folio, start: filepos, len: sectorsize);
1659	btrfs_folio_set_writeback(fs_info, folio, start: filepos, len: sectorsize);
1660	/*
1661	* Above call should set the whole folio with writeback flag, even
1662	* just for a single subpage sector.
1663	* As long as the folio is properly locked and the range is correct,
1664	* we should always get the folio with writeback flag.
1665	*/
1666	ASSERT(folio_test_writeback(folio));
1667
1668	submit_extent_folio(bio_ctrl, disk_bytenr, folio,
1669	size: sectorsize, pg_offset: filepos - folio_pos(folio), read_em_generation: 0);
1670	return 0;
1671	}
1672
1673	/*
1674	* Helper for extent_writepage(). This calls the writepage start hooks,
1675	* and does the loop to map the page into extents and bios.
1676	*
1677	* We return 1 if the IO is started and the page is unlocked,
1678	* 0 if all went well (page still locked)
1679	* < 0 if there were errors (page still locked)
1680	*/
1681	static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
1682	struct folio *folio,
1683	u64 start, u32 len,
1684	struct btrfs_bio_ctrl *bio_ctrl,
1685	loff_t i_size)
1686	{
1687	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1688	unsigned long range_bitmap = 0;
1689	bool submitted_io = false;
1690	int found_error = 0;
1691	const u64 end = start + len;
1692	const u64 folio_start = folio_pos(folio);
1693	const u64 folio_end = folio_start + folio_size(folio);
1694	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1695	u64 cur;
1696	int bit;
1697	int ret = 0;
1698
1699	ASSERT(start >= folio_start, "start=%llu folio_start=%llu", start, folio_start);
1700	ASSERT(end <= folio_end, "start=%llu len=%u folio_start=%llu folio_size=%zu",
1701	start, len, folio_start, folio_size(folio));
1702
1703	ret = btrfs_writepage_cow_fixup(folio);
1704	if (ret == -EAGAIN) {
1705	/* Fixup worker will requeue */
1706	folio_redirty_for_writepage(bio_ctrl->wbc, folio);
1707	folio_unlock(folio);
1708	return 1;
1709	}
1710	if (ret < 0) {
1711	btrfs_folio_clear_dirty(fs_info, folio, start, len);
1712	btrfs_folio_set_writeback(fs_info, folio, start, len);
1713	btrfs_folio_clear_writeback(fs_info, folio, start, len);
1714	return ret;
1715	}
1716
1717	for (cur = start; cur < end; cur += fs_info->sectorsize)
1718	set_bit(nr: (cur - folio_start) >> fs_info->sectorsize_bits, addr: &range_bitmap);
1719	bitmap_and(dst: &bio_ctrl->submit_bitmap, src1: &bio_ctrl->submit_bitmap, src2: &range_bitmap,
1720	nbits: blocks_per_folio);
1721
1722	bio_ctrl->end_io_func = end_bbio_data_write;
1723
1724	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, blocks_per_folio) {
1725	cur = folio_pos(folio) + (bit << fs_info->sectorsize_bits);
1726
1727	if (cur >= i_size) {
1728	struct btrfs_ordered_extent *ordered;
1729
1730	ordered = btrfs_lookup_first_ordered_range(inode, file_offset: cur,
1731	len: fs_info->sectorsize);
1732	/*
1733	* We have just run delalloc before getting here, so
1734	* there must be an ordered extent.
1735	*/
1736	ASSERT(ordered != NULL);
1737	spin_lock(lock: &inode->ordered_tree_lock);
1738	set_bit(nr: BTRFS_ORDERED_TRUNCATED, addr: &ordered->flags);
1739	ordered->truncated_len = min(ordered->truncated_len,
1740	cur - ordered->file_offset);
1741	spin_unlock(lock: &inode->ordered_tree_lock);
1742	btrfs_put_ordered_extent(entry: ordered);
1743
1744	btrfs_mark_ordered_io_finished(inode, folio, file_offset: cur,
1745	num_bytes: fs_info->sectorsize, uptodate: true);
1746	/*
1747	* This range is beyond i_size, thus we don't need to
1748	* bother writing back.
1749	* But we still need to clear the dirty subpage bit, or
1750	* the next time the folio gets dirtied, we will try to
1751	* writeback the sectors with subpage dirty bits,
1752	* causing writeback without ordered extent.
1753	*/
1754	btrfs_folio_clear_dirty(fs_info, folio, start: cur, len: fs_info->sectorsize);
1755	continue;
1756	}
1757	ret = submit_one_sector(inode, folio, filepos: cur, bio_ctrl, i_size);
1758	if (unlikely(ret < 0)) {
1759	/*
1760	* bio_ctrl may contain a bio crossing several folios.
1761	* Submit it immediately so that the bio has a chance
1762	* to finish normally, other than marked as error.
1763	*/
1764	submit_one_bio(bio_ctrl);
1765	/*
1766	* Failed to grab the extent map which should be very rare.
1767	* Since there is no bio submitted to finish the ordered
1768	* extent, we have to manually finish this sector.
1769	*/
1770	btrfs_mark_ordered_io_finished(inode, folio, file_offset: cur,
1771	num_bytes: fs_info->sectorsize, uptodate: false);
1772	if (!found_error)
1773	found_error = ret;
1774	continue;
1775	}
1776	submitted_io = true;
1777	}
1778
1779	/*
1780	* If we didn't submitted any sector (>= i_size), folio dirty get
1781	* cleared but PAGECACHE_TAG_DIRTY is not cleared (only cleared
1782	* by folio_start_writeback() if the folio is not dirty).
1783	*
1784	* Here we set writeback and clear for the range. If the full folio
1785	* is no longer dirty then we clear the PAGECACHE_TAG_DIRTY tag.
1786	*
1787	* If we hit any error, the corresponding sector will have its dirty
1788	* flag cleared and writeback finished, thus no need to handle the error case.
1789	*/
1790	if (!submitted_io && !found_error) {
1791	btrfs_folio_set_writeback(fs_info, folio, start, len);
1792	btrfs_folio_clear_writeback(fs_info, folio, start, len);
1793	}
1794	return found_error;
1795	}
1796
1797	/*
1798	* the writepage semantics are similar to regular writepage. extent
1799	* records are inserted to lock ranges in the tree, and as dirty areas
1800	* are found, they are marked writeback. Then the lock bits are removed
1801	* and the end_io handler clears the writeback ranges
1802	*
1803	* Return 0 if everything goes well.
1804	* Return <0 for error.
1805	*/
1806	static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl)
1807	{
1808	struct btrfs_inode *inode = BTRFS_I(folio->mapping->host);
1809	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1810	int ret;
1811	size_t pg_offset;
1812	loff_t i_size = i_size_read(inode: &inode->vfs_inode);
1813	const pgoff_t end_index = i_size >> PAGE_SHIFT;
1814	const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio);
1815
1816	trace_extent_writepage(folio, inode: &inode->vfs_inode, wbc: bio_ctrl->wbc);
1817
1818	WARN_ON(!folio_test_locked(folio));
1819
1820	pg_offset = offset_in_folio(folio, i_size);
1821	if (folio->index > end_index ||
1822	(folio->index == end_index && !pg_offset)) {
1823	folio_invalidate(folio, offset: 0, length: folio_size(folio));
1824	folio_unlock(folio);
1825	return 0;
1826	}
1827
1828	if (folio_contains(folio, index: end_index))
1829	folio_zero_range(folio, start: pg_offset, length: folio_size(folio) - pg_offset);
1830
1831	/*
1832	* Default to unlock the whole folio.
1833	* The proper bitmap can only be initialized until writepage_delalloc().
1834	*/
1835	bio_ctrl->submit_bitmap = (unsigned long)-1;
1836
1837	/*
1838	* If the page is dirty but without private set, it's marked dirty
1839	* without informing the fs.
1840	* Nowadays that is a bug, since the introduction of
1841	* pin_user_pages*().
1842	*
1843	* So here we check if the page has private set to rule out such
1844	* case.
1845	* But we also have a long history of relying on the COW fixup,
1846	* so here we only enable this check for experimental builds until
1847	* we're sure it's safe.
1848	*/
1849	if (IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL) &&
1850	unlikely(!folio_test_private(folio))) {
1851	WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
1852	btrfs_err_rl(fs_info,
1853	"root %lld ino %llu folio %llu is marked dirty without notifying the fs",
1854	btrfs_root_id(inode->root),
1855	btrfs_ino(inode), folio_pos(folio));
1856	ret = -EUCLEAN;
1857	goto done;
1858	}
1859
1860	ret = set_folio_extent_mapped(folio);
1861	if (ret < 0)
1862	goto done;
1863
1864	ret = writepage_delalloc(inode, folio, bio_ctrl);
1865	if (ret == 1)
1866	return 0;
1867	if (ret)
1868	goto done;
1869
1870	ret = extent_writepage_io(inode, folio, start: folio_pos(folio),
1871	len: folio_size(folio), bio_ctrl, i_size);
1872	if (ret == 1)
1873	return 0;
1874	if (unlikely(ret < 0))
1875	btrfs_err_rl(fs_info,
1876	"failed to submit blocks, root=%lld inode=%llu folio=%llu submit_bitmap=%*pbl: %d",
1877	btrfs_root_id(inode->root), btrfs_ino(inode),
1878	folio_pos(folio), blocks_per_folio,
1879	&bio_ctrl->submit_bitmap, ret);
1880
1881	bio_ctrl->wbc->nr_to_write--;
1882
1883	done:
1884	if (ret < 0)
1885	mapping_set_error(mapping: folio->mapping, error: ret);
1886	/*
1887	* Only unlock ranges that are submitted. As there can be some async
1888	* submitted ranges inside the folio.
1889	*/
1890	btrfs_folio_end_lock_bitmap(fs_info, folio, bitmap: bio_ctrl->submit_bitmap);
1891	ASSERT(ret <= 0);
1892	return ret;
1893	}
1894
1895	/*
1896	* Lock extent buffer status and pages for writeback.
1897	*
1898	* Return %false if the extent buffer doesn't need to be submitted (e.g. the
1899	* extent buffer is not dirty)
1900	* Return %true is the extent buffer is submitted to bio.
1901	*/
1902	static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
1903	struct writeback_control *wbc)
1904	{
1905	struct btrfs_fs_info *fs_info = eb->fs_info;
1906	bool ret = false;
1907
1908	btrfs_tree_lock(eb);
1909	while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
1910	btrfs_tree_unlock(eb);
1911	if (wbc->sync_mode != WB_SYNC_ALL)
1912	return false;
1913	wait_on_extent_buffer_writeback(eb);
1914	btrfs_tree_lock(eb);
1915	}
1916
1917	/*
1918	* We need to do this to prevent races in people who check if the eb is
1919	* under IO since we can end up having no IO bits set for a short period
1920	* of time.
1921	*/
1922	spin_lock(lock: &eb->refs_lock);
1923	if (test_and_clear_bit(nr: EXTENT_BUFFER_DIRTY, addr: &eb->bflags)) {
1924	XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
1925	unsigned long flags;
1926
1927	set_bit(nr: EXTENT_BUFFER_WRITEBACK, addr: &eb->bflags);
1928	spin_unlock(lock: &eb->refs_lock);
1929
1930	xas_lock_irqsave(&xas, flags);
1931	xas_load(&xas);
1932	xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
1933	xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
1934	xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
1935	xas_unlock_irqrestore(&xas, flags);
1936
1937	btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1938	percpu_counter_add_batch(fbc: &fs_info->dirty_metadata_bytes,
1939	amount: -eb->len,
1940	batch: fs_info->dirty_metadata_batch);
1941	ret = true;
1942	} else {
1943	spin_unlock(lock: &eb->refs_lock);
1944	}
1945	btrfs_tree_unlock(eb);
1946	return ret;
1947	}
1948
1949	static void set_btree_ioerr(struct extent_buffer *eb)
1950	{
1951	struct btrfs_fs_info *fs_info = eb->fs_info;
1952
1953	set_bit(nr: EXTENT_BUFFER_WRITE_ERR, addr: &eb->bflags);
1954
1955	/*
1956	* A read may stumble upon this buffer later, make sure that it gets an
1957	* error and knows there was an error.
1958	*/
1959	clear_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
1960
1961	/*
1962	* We need to set the mapping with the io error as well because a write
1963	* error will flip the file system readonly, and then syncfs() will
1964	* return a 0 because we are readonly if we don't modify the err seq for
1965	* the superblock.
1966	*/
1967	mapping_set_error(mapping: eb->fs_info->btree_inode->i_mapping, error: -EIO);
1968
1969	/*
1970	* If writeback for a btree extent that doesn't belong to a log tree
1971	* failed, increment the counter transaction->eb_write_errors.
1972	* We do this because while the transaction is running and before it's
1973	* committing (when we call filemap_fdata[write|wait]_range against
1974	* the btree inode), we might have
1975	* btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1976	* returns an error or an error happens during writeback, when we're
1977	* committing the transaction we wouldn't know about it, since the pages
1978	* can be no longer dirty nor marked anymore for writeback (if a
1979	* subsequent modification to the extent buffer didn't happen before the
1980	* transaction commit), which makes filemap_fdata[write|wait]_range not
1981	* able to find the pages which contain errors at transaction
1982	* commit time. So if this happens we must abort the transaction,
1983	* otherwise we commit a super block with btree roots that point to
1984	* btree nodes/leafs whose content on disk is invalid - either garbage
1985	* or the content of some node/leaf from a past generation that got
1986	* cowed or deleted and is no longer valid.
1987	*
1988	* Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1989	* not be enough - we need to distinguish between log tree extents vs
1990	* non-log tree extents, and the next filemap_fdatawait_range() call
1991	* will catch and clear such errors in the mapping - and that call might
1992	* be from a log sync and not from a transaction commit. Also, checking
1993	* for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1994	* not done and would not be reliable - the eb might have been released
1995	* from memory and reading it back again means that flag would not be
1996	* set (since it's a runtime flag, not persisted on disk).
1997	*
1998	* Using the flags below in the btree inode also makes us achieve the
1999	* goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
2000	* writeback for all dirty pages and before filemap_fdatawait_range()
2001	* is called, the writeback for all dirty pages had already finished
2002	* with errors - because we were not using AS_EIO/AS_ENOSPC,
2003	* filemap_fdatawait_range() would return success, as it could not know
2004	* that writeback errors happened (the pages were no longer tagged for
2005	* writeback).
2006	*/
2007	switch (eb->log_index) {
2008	case -1:
2009	set_bit(nr: BTRFS_FS_BTREE_ERR, addr: &fs_info->flags);
2010	break;
2011	case 0:
2012	set_bit(nr: BTRFS_FS_LOG1_ERR, addr: &fs_info->flags);
2013	break;
2014	case 1:
2015	set_bit(nr: BTRFS_FS_LOG2_ERR, addr: &fs_info->flags);
2016	break;
2017	default:
2018	BUG(); /* unexpected, logic error */
2019	}
2020	}
2021
2022	static void buffer_tree_set_mark(const struct extent_buffer *eb, xa_mark_t mark)
2023	{
2024	struct btrfs_fs_info *fs_info = eb->fs_info;
2025	XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
2026	unsigned long flags;
2027
2028	xas_lock_irqsave(&xas, flags);
2029	xas_load(&xas);
2030	xas_set_mark(&xas, mark);
2031	xas_unlock_irqrestore(&xas, flags);
2032	}
2033
2034	static void buffer_tree_clear_mark(const struct extent_buffer *eb, xa_mark_t mark)
2035	{
2036	struct btrfs_fs_info *fs_info = eb->fs_info;
2037	XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits);
2038	unsigned long flags;
2039
2040	xas_lock_irqsave(&xas, flags);
2041	xas_load(&xas);
2042	xas_clear_mark(&xas, mark);
2043	xas_unlock_irqrestore(&xas, flags);
2044	}
2045
2046	static void buffer_tree_tag_for_writeback(struct btrfs_fs_info *fs_info,
2047	unsigned long start, unsigned long end)
2048	{
2049	XA_STATE(xas, &fs_info->buffer_tree, start);
2050	unsigned int tagged = 0;
2051	void *eb;
2052
2053	xas_lock_irq(&xas);
2054	xas_for_each_marked(&xas, eb, end, PAGECACHE_TAG_DIRTY) {
2055	xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE);
2056	if (++tagged % XA_CHECK_SCHED)
2057	continue;
2058	xas_pause(&xas);
2059	xas_unlock_irq(&xas);
2060	cond_resched();
2061	xas_lock_irq(&xas);
2062	}
2063	xas_unlock_irq(&xas);
2064	}
2065
2066	struct eb_batch {
2067	unsigned int nr;
2068	unsigned int cur;
2069	struct extent_buffer *ebs[PAGEVEC_SIZE];
2070	};
2071
2072	static inline bool eb_batch_add(struct eb_batch *batch, struct extent_buffer *eb)
2073	{
2074	batch->ebs[batch->nr++] = eb;
2075	return (batch->nr < PAGEVEC_SIZE);
2076	}
2077
2078	static inline void eb_batch_init(struct eb_batch *batch)
2079	{
2080	batch->nr = 0;
2081	batch->cur = 0;
2082	}
2083
2084	static inline struct extent_buffer *eb_batch_next(struct eb_batch *batch)
2085	{
2086	if (batch->cur >= batch->nr)
2087	return NULL;
2088	return batch->ebs[batch->cur++];
2089	}
2090
2091	static inline void eb_batch_release(struct eb_batch *batch)
2092	{
2093	for (unsigned int i = 0; i < batch->nr; i++)
2094	free_extent_buffer(eb: batch->ebs[i]);
2095	eb_batch_init(batch);
2096	}
2097
2098	static inline struct extent_buffer *find_get_eb(struct xa_state *xas, unsigned long max,
2099	xa_mark_t mark)
2100	{
2101	struct extent_buffer *eb;
2102
2103	retry:
2104	eb = xas_find_marked(xas, max, mark);
2105
2106	if (xas_retry(xas, entry: eb))
2107	goto retry;
2108
2109	if (!eb)
2110	return NULL;
2111
2112	if (!refcount_inc_not_zero(r: &eb->refs)) {
2113	xas_reset(xas);
2114	goto retry;
2115	}
2116
2117	if (unlikely(eb != xas_reload(xas))) {
2118	free_extent_buffer(eb);
2119	xas_reset(xas);
2120	goto retry;
2121	}
2122
2123	return eb;
2124	}
2125
2126	static unsigned int buffer_tree_get_ebs_tag(struct btrfs_fs_info *fs_info,
2127	unsigned long *start,
2128	unsigned long end, xa_mark_t tag,
2129	struct eb_batch *batch)
2130	{
2131	XA_STATE(xas, &fs_info->buffer_tree, *start);
2132	struct extent_buffer *eb;
2133
2134	rcu_read_lock();
2135	while ((eb = find_get_eb(xas: &xas, max: end, mark: tag)) != NULL) {
2136	if (!eb_batch_add(batch, eb)) {
2137	*start = ((eb->start + eb->len) >> fs_info->nodesize_bits);
2138	goto out;
2139	}
2140	}
2141	if (end == ULONG_MAX)
2142	*start = ULONG_MAX;
2143	else
2144	*start = end + 1;
2145	out:
2146	rcu_read_unlock();
2147
2148	return batch->nr;
2149	}
2150
2151	/*
2152	* The endio specific version which won't touch any unsafe spinlock in endio
2153	* context.
2154	*/
2155	static struct extent_buffer *find_extent_buffer_nolock(
2156	struct btrfs_fs_info *fs_info, u64 start)
2157	{
2158	struct extent_buffer *eb;
2159	unsigned long index = (start >> fs_info->nodesize_bits);
2160
2161	rcu_read_lock();
2162	eb = xa_load(&fs_info->buffer_tree, index);
2163	if (eb && !refcount_inc_not_zero(r: &eb->refs))
2164	eb = NULL;
2165	rcu_read_unlock();
2166	return eb;
2167	}
2168
2169	static void end_bbio_meta_write(struct btrfs_bio *bbio)
2170	{
2171	struct extent_buffer *eb = bbio->private;
2172	struct folio_iter fi;
2173
2174	if (bbio->bio.bi_status != BLK_STS_OK)
2175	set_btree_ioerr(eb);
2176
2177	bio_for_each_folio_all(fi, &bbio->bio) {
2178	btrfs_meta_folio_clear_writeback(folio: fi.folio, eb);
2179	}
2180
2181	buffer_tree_clear_mark(eb, PAGECACHE_TAG_WRITEBACK);
2182	clear_and_wake_up_bit(bit: EXTENT_BUFFER_WRITEBACK, word: &eb->bflags);
2183	bio_put(&bbio->bio);
2184	}
2185
2186	static void prepare_eb_write(struct extent_buffer *eb)
2187	{
2188	u32 nritems;
2189	unsigned long start;
2190	unsigned long end;
2191
2192	clear_bit(nr: EXTENT_BUFFER_WRITE_ERR, addr: &eb->bflags);
2193
2194	/* Set btree blocks beyond nritems with 0 to avoid stale content */
2195	nritems = btrfs_header_nritems(eb);
2196	if (btrfs_header_level(eb) > 0) {
2197	end = btrfs_node_key_ptr_offset(eb, nr: nritems);
2198	memzero_extent_buffer(eb, start: end, len: eb->len - end);
2199	} else {
2200	/*
2201	* Leaf:
2202	* header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
2203	*/
2204	start = btrfs_item_nr_offset(eb, nr: nritems);
2205	end = btrfs_item_nr_offset(eb, nr: 0);
2206	if (nritems == 0)
2207	end += BTRFS_LEAF_DATA_SIZE(info: eb->fs_info);
2208	else
2209	end += btrfs_item_offset(eb, slot: nritems - 1);
2210	memzero_extent_buffer(eb, start, len: end - start);
2211	}
2212	}
2213
2214	static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
2215	struct writeback_control *wbc)
2216	{
2217	struct btrfs_fs_info *fs_info = eb->fs_info;
2218	struct btrfs_bio *bbio;
2219
2220	prepare_eb_write(eb);
2221
2222	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
2223	opf: REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
2224	BTRFS_I(fs_info->btree_inode), file_offset: eb->start,
2225	end_io: end_bbio_meta_write, private: eb);
2226	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
2227	bio_set_dev(bio: &bbio->bio, bdev: fs_info->fs_devices->latest_dev->bdev);
2228	wbc_init_bio(wbc, bio: &bbio->bio);
2229	for (int i = 0; i < num_extent_folios(eb); i++) {
2230	struct folio *folio = eb->folios[i];
2231	u64 range_start = max_t(u64, eb->start, folio_pos(folio));
2232	u32 range_len = min_t(u64, folio_next_pos(folio),
2233	eb->start + eb->len) - range_start;
2234
2235	folio_lock(folio);
2236	btrfs_meta_folio_clear_dirty(folio, eb);
2237	btrfs_meta_folio_set_writeback(folio, eb);
2238	if (!folio_test_dirty(folio))
2239	wbc->nr_to_write -= folio_nr_pages(folio);
2240	bio_add_folio_nofail(bio: &bbio->bio, folio, len: range_len,
2241	offset_in_folio(folio, range_start));
2242	wbc_account_cgroup_owner(wbc, folio, bytes: range_len);
2243	folio_unlock(folio);
2244	}
2245	/*
2246	* If the fs is already in error status, do not submit any writeback
2247	* but immediately finish it.
2248	*/
2249	if (unlikely(BTRFS_FS_ERROR(fs_info))) {
2250	btrfs_bio_end_io(bbio, status: errno_to_blk_status(BTRFS_FS_ERROR(fs_info)));
2251	return;
2252	}
2253	btrfs_submit_bbio(bbio, mirror_num: 0);
2254	}
2255
2256	/*
2257	* Wait for all eb writeback in the given range to finish.
2258	*
2259	* @fs_info: The fs_info for this file system.
2260	* @start: The offset of the range to start waiting on writeback.
2261	* @end: The end of the range, inclusive. This is meant to be used in
2262	* conjunction with wait_marked_extents, so this will usually be
2263	* the_next_eb->start - 1.
2264	*/
2265	void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start,
2266	u64 end)
2267	{
2268	struct eb_batch batch;
2269	unsigned long start_index = (start >> fs_info->nodesize_bits);
2270	unsigned long end_index = (end >> fs_info->nodesize_bits);
2271
2272	eb_batch_init(batch: &batch);
2273	while (start_index <= end_index) {
2274	struct extent_buffer *eb;
2275	unsigned int nr_ebs;
2276
2277	nr_ebs = buffer_tree_get_ebs_tag(fs_info, start: &start_index, end: end_index,
2278	PAGECACHE_TAG_WRITEBACK, batch: &batch);
2279	if (!nr_ebs)
2280	break;
2281
2282	while ((eb = eb_batch_next(batch: &batch)) != NULL)
2283	wait_on_extent_buffer_writeback(eb);
2284	eb_batch_release(batch: &batch);
2285	cond_resched();
2286	}
2287	}
2288
2289	int btree_writepages(struct address_space *mapping, struct writeback_control *wbc)
2290	{
2291	struct btrfs_eb_write_context ctx = { .wbc = wbc };
2292	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
2293	int ret = 0;
2294	int done = 0;
2295	int nr_to_write_done = 0;
2296	struct eb_batch batch;
2297	unsigned int nr_ebs;
2298	unsigned long index;
2299	unsigned long end;
2300	int scanned = 0;
2301	xa_mark_t tag;
2302
2303	eb_batch_init(batch: &batch);
2304	if (wbc->range_cyclic) {
2305	index = ((mapping->writeback_index << PAGE_SHIFT) >> fs_info->nodesize_bits);
2306	end = -1;
2307
2308	/*
2309	* Start from the beginning does not need to cycle over the
2310	* range, mark it as scanned.
2311	*/
2312	scanned = (index == 0);
2313	} else {
2314	index = (wbc->range_start >> fs_info->nodesize_bits);
2315	end = (wbc->range_end >> fs_info->nodesize_bits);
2316
2317	scanned = 1;
2318	}
2319	if (wbc->sync_mode == WB_SYNC_ALL)
2320	tag = PAGECACHE_TAG_TOWRITE;
2321	else
2322	tag = PAGECACHE_TAG_DIRTY;
2323	btrfs_zoned_meta_io_lock(fs_info);
2324	retry:
2325	if (wbc->sync_mode == WB_SYNC_ALL)
2326	buffer_tree_tag_for_writeback(fs_info, start: index, end);
2327	while (!done && !nr_to_write_done && (index <= end) &&
2328	(nr_ebs = buffer_tree_get_ebs_tag(fs_info, start: &index, end, tag, batch: &batch))) {
2329	struct extent_buffer *eb;
2330
2331	while ((eb = eb_batch_next(batch: &batch)) != NULL) {
2332	ctx.eb = eb;
2333
2334	ret = btrfs_check_meta_write_pointer(fs_info: eb->fs_info, ctx: &ctx);
2335	if (ret) {
2336	if (ret == -EBUSY)
2337	ret = 0;
2338
2339	if (ret) {
2340	done = 1;
2341	break;
2342	}
2343	continue;
2344	}
2345
2346	if (!lock_extent_buffer_for_io(eb, wbc))
2347	continue;
2348
2349	/* Implies write in zoned mode. */
2350	if (ctx.zoned_bg) {
2351	/* Mark the last eb in the block group. */
2352	btrfs_schedule_zone_finish_bg(bg: ctx.zoned_bg, eb);
2353	ctx.zoned_bg->meta_write_pointer += eb->len;
2354	}
2355	write_one_eb(eb, wbc);
2356	}
2357	nr_to_write_done = (wbc->nr_to_write <= 0);
2358	eb_batch_release(batch: &batch);
2359	cond_resched();
2360	}
2361	if (!scanned && !done) {
2362	/*
2363	* We hit the last page and there is more work to be done: wrap
2364	* back to the start of the file
2365	*/
2366	scanned = 1;
2367	index = 0;
2368	goto retry;
2369	}
2370	/*
2371	* If something went wrong, don't allow any metadata write bio to be
2372	* submitted.
2373	*
2374	* This would prevent use-after-free if we had dirty pages not
2375	* cleaned up, which can still happen by fuzzed images.
2376	*
2377	* - Bad extent tree
2378	* Allowing existing tree block to be allocated for other trees.
2379	*
2380	* - Log tree operations
2381	* Exiting tree blocks get allocated to log tree, bumps its
2382	* generation, then get cleaned in tree re-balance.
2383	* Such tree block will not be written back, since it's clean,
2384	* thus no WRITTEN flag set.
2385	* And after log writes back, this tree block is not traced by
2386	* any dirty extent_io_tree.
2387	*
2388	* - Offending tree block gets re-dirtied from its original owner
2389	* Since it has bumped generation, no WRITTEN flag, it can be
2390	* reused without COWing. This tree block will not be traced
2391	* by btrfs_transaction::dirty_pages.
2392	*
2393	* Now such dirty tree block will not be cleaned by any dirty
2394	* extent io tree. Thus we don't want to submit such wild eb
2395	* if the fs already has error.
2396	*
2397	* We can get ret > 0 from submit_extent_folio() indicating how many ebs
2398	* were submitted. Reset it to 0 to avoid false alerts for the caller.
2399	*/
2400	if (ret > 0)
2401	ret = 0;
2402	if (!ret && BTRFS_FS_ERROR(fs_info))
2403	ret = -EROFS;
2404
2405	if (ctx.zoned_bg)
2406	btrfs_put_block_group(cache: ctx.zoned_bg);
2407	btrfs_zoned_meta_io_unlock(fs_info);
2408	return ret;
2409	}
2410
2411	/*
2412	* Walk the list of dirty pages of the given address space and write all of them.
2413	*
2414	* @mapping: address space structure to write
2415	* @wbc: subtract the number of written pages from *@wbc->nr_to_write
2416	* @bio_ctrl: holds context for the write, namely the bio
2417	*
2418	* If a page is already under I/O, write_cache_pages() skips it, even
2419	* if it's dirty. This is desirable behaviour for memory-cleaning writeback,
2420	* but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
2421	* and msync() need to guarantee that all the data which was dirty at the time
2422	* the call was made get new I/O started against them. If wbc->sync_mode is
2423	* WB_SYNC_ALL then we were called for data integrity and we must wait for
2424	* existing IO to complete.
2425	*/
2426	static int extent_write_cache_pages(struct address_space *mapping,
2427	struct btrfs_bio_ctrl *bio_ctrl)
2428	{
2429	struct writeback_control *wbc = bio_ctrl->wbc;
2430	struct inode *inode = mapping->host;
2431	int ret = 0;
2432	int done = 0;
2433	int nr_to_write_done = 0;
2434	struct folio_batch fbatch;
2435	unsigned int nr_folios;
2436	pgoff_t index;
2437	pgoff_t end; /* Inclusive */
2438	pgoff_t done_index;
2439	int range_whole = 0;
2440	int scanned = 0;
2441	xa_mark_t tag;
2442
2443	/*
2444	* We have to hold onto the inode so that ordered extents can do their
2445	* work when the IO finishes. The alternative to this is failing to add
2446	* an ordered extent if the igrab() fails there and that is a huge pain
2447	* to deal with, so instead just hold onto the inode throughout the
2448	* writepages operation. If it fails here we are freeing up the inode
2449	* anyway and we'd rather not waste our time writing out stuff that is
2450	* going to be truncated anyway.
2451	*/
2452	if (!igrab(inode))
2453	return 0;
2454
2455	folio_batch_init(fbatch: &fbatch);
2456	if (wbc->range_cyclic) {
2457	index = mapping->writeback_index; /* Start from prev offset */
2458	end = -1;
2459	/*
2460	* Start from the beginning does not need to cycle over the
2461	* range, mark it as scanned.
2462	*/
2463	scanned = (index == 0);
2464	} else {
2465	index = wbc->range_start >> PAGE_SHIFT;
2466	end = wbc->range_end >> PAGE_SHIFT;
2467	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2468	range_whole = 1;
2469	scanned = 1;
2470	}
2471
2472	/*
2473	* We do the tagged writepage as long as the snapshot flush bit is set
2474	* and we are the first one who do the filemap_flush() on this inode.
2475	*
2476	* The nr_to_write == LONG_MAX is needed to make sure other flushers do
2477	* not race in and drop the bit.
2478	*/
2479	if (range_whole && wbc->nr_to_write == LONG_MAX &&
2480	test_and_clear_bit(nr: BTRFS_INODE_SNAPSHOT_FLUSH,
2481	addr: &BTRFS_I(inode)->runtime_flags))
2482	wbc->tagged_writepages = 1;
2483
2484	tag = wbc_to_tag(wbc);
2485	retry:
2486	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2487	tag_pages_for_writeback(mapping, start: index, end);
2488	done_index = index;
2489	while (!done && !nr_to_write_done && (index <= end) &&
2490	(nr_folios = filemap_get_folios_tag(mapping, start: &index,
2491	end, tag, fbatch: &fbatch))) {
2492	unsigned i;
2493
2494	for (i = 0; i < nr_folios; i++) {
2495	struct folio *folio = fbatch.folios[i];
2496
2497	done_index = folio_next_index(folio);
2498	/*
2499	* At this point we hold neither the i_pages lock nor
2500	* the folio lock: the folio may be truncated or
2501	* invalidated (changing folio->mapping to NULL).
2502	*/
2503	if (!folio_trylock(folio)) {
2504	submit_write_bio(bio_ctrl, ret: 0);
2505	folio_lock(folio);
2506	}
2507
2508	if (unlikely(folio->mapping != mapping)) {
2509	folio_unlock(folio);
2510	continue;
2511	}
2512
2513	if (!folio_test_dirty(folio)) {
2514	/* Someone wrote it for us. */
2515	folio_unlock(folio);
2516	continue;
2517	}
2518
2519	/*
2520	* For subpage case, compression can lead to mixed
2521	* writeback and dirty flags, e.g:
2522	* 0 32K 64K 96K 128K
2523	* | |//////||/////| |//|
2524	*
2525	* In above case, [32K, 96K) is asynchronously submitted
2526	* for compression, and [124K, 128K) needs to be written back.
2527	*
2528	* If we didn't wait writeback for page 64K, [128K, 128K)
2529	* won't be submitted as the page still has writeback flag
2530	* and will be skipped in the next check.
2531	*
2532	* This mixed writeback and dirty case is only possible for
2533	* subpage case.
2534	*
2535	* TODO: Remove this check after migrating compression to
2536	* regular submission.
2537	*/
2538	if (wbc->sync_mode != WB_SYNC_NONE ||
2539	btrfs_is_subpage(inode_to_fs_info(inode), folio)) {
2540	if (folio_test_writeback(folio))
2541	submit_write_bio(bio_ctrl, ret: 0);
2542	folio_wait_writeback(folio);
2543	}
2544
2545	if (folio_test_writeback(folio) ||
2546	!folio_clear_dirty_for_io(folio)) {
2547	folio_unlock(folio);
2548	continue;
2549	}
2550
2551	ret = extent_writepage(folio, bio_ctrl);
2552	if (ret < 0) {
2553	done = 1;
2554	break;
2555	}
2556
2557	/*
2558	* The filesystem may choose to bump up nr_to_write.
2559	* We have to make sure to honor the new nr_to_write
2560	* at any time.
2561	*/
2562	nr_to_write_done = (wbc->sync_mode == WB_SYNC_NONE &&
2563	wbc->nr_to_write <= 0);
2564	}
2565	folio_batch_release(fbatch: &fbatch);
2566	cond_resched();
2567	}
2568	if (!scanned && !done) {
2569	/*
2570	* We hit the last page and there is more work to be done: wrap
2571	* back to the start of the file
2572	*/
2573	scanned = 1;
2574	index = 0;
2575
2576	/*
2577	* If we're looping we could run into a page that is locked by a
2578	* writer and that writer could be waiting on writeback for a
2579	* page in our current bio, and thus deadlock, so flush the
2580	* write bio here.
2581	*/
2582	submit_write_bio(bio_ctrl, ret: 0);
2583	goto retry;
2584	}
2585
2586	if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2587	mapping->writeback_index = done_index;
2588
2589	btrfs_add_delayed_iput(BTRFS_I(inode));
2590	return ret;
2591	}
2592
2593	/*
2594	* Submit the pages in the range to bio for call sites which delalloc range has
2595	* already been ran (aka, ordered extent inserted) and all pages are still
2596	* locked.
2597	*/
2598	void extent_write_locked_range(struct inode *inode, const struct folio *locked_folio,
2599	u64 start, u64 end, struct writeback_control *wbc,
2600	bool pages_dirty)
2601	{
2602	bool found_error = false;
2603	int ret = 0;
2604	struct address_space *mapping = inode->i_mapping;
2605	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
2606	const u32 sectorsize = fs_info->sectorsize;
2607	loff_t i_size = i_size_read(inode);
2608	u64 cur = start;
2609	struct btrfs_bio_ctrl bio_ctrl = {
2610	.wbc = wbc,
2611	.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2612	};
2613
2614	if (wbc->no_cgroup_owner)
2615	bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
2616
2617	ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
2618
2619	while (cur <= end) {
2620	u64 cur_end;
2621	u32 cur_len;
2622	struct folio *folio;
2623
2624	folio = filemap_get_folio(mapping, index: cur >> PAGE_SHIFT);
2625
2626	/*
2627	* This shouldn't happen, the pages are pinned and locked, this
2628	* code is just in case, but shouldn't actually be run.
2629	*/
2630	if (IS_ERR(ptr: folio)) {
2631	cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
2632	cur_len = cur_end + 1 - cur;
2633	btrfs_mark_ordered_io_finished(BTRFS_I(inode), NULL,
2634	file_offset: cur, num_bytes: cur_len, uptodate: false);
2635	mapping_set_error(mapping, error: PTR_ERR(ptr: folio));
2636	cur = cur_end;
2637	continue;
2638	}
2639
2640	cur_end = min_t(u64, folio_next_pos(folio) - 1, end);
2641	cur_len = cur_end + 1 - cur;
2642
2643	ASSERT(folio_test_locked(folio));
2644	if (pages_dirty && folio != locked_folio)
2645	ASSERT(folio_test_dirty(folio));
2646
2647	/*
2648	* Set the submission bitmap to submit all sectors.
2649	* extent_writepage_io() will do the truncation correctly.
2650	*/
2651	bio_ctrl.submit_bitmap = (unsigned long)-1;
2652	ret = extent_writepage_io(BTRFS_I(inode), folio, start: cur, len: cur_len,
2653	bio_ctrl: &bio_ctrl, i_size);
2654	if (ret == 1)
2655	goto next_page;
2656
2657	if (ret)
2658	mapping_set_error(mapping, error: ret);
2659	btrfs_folio_end_lock(fs_info, folio, start: cur, len: cur_len);
2660	if (ret < 0)
2661	found_error = true;
2662	next_page:
2663	folio_put(folio);
2664	cur = cur_end + 1;
2665	}
2666
2667	submit_write_bio(bio_ctrl: &bio_ctrl, ret: found_error ? ret : 0);
2668	}
2669
2670	int btrfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
2671	{
2672	struct inode *inode = mapping->host;
2673	int ret = 0;
2674	struct btrfs_bio_ctrl bio_ctrl = {
2675	.wbc = wbc,
2676	.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2677	};
2678
2679	/*
2680	* Allow only a single thread to do the reloc work in zoned mode to
2681	* protect the write pointer updates.
2682	*/
2683	btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
2684	ret = extent_write_cache_pages(mapping, bio_ctrl: &bio_ctrl);
2685	submit_write_bio(bio_ctrl: &bio_ctrl, ret);
2686	btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
2687	return ret;
2688	}
2689
2690	void btrfs_readahead(struct readahead_control *rac)
2691	{
2692	struct btrfs_bio_ctrl bio_ctrl = {
2693	.opf = REQ_OP_READ | REQ_RAHEAD,
2694	.ractl = rac,
2695	.last_em_start = U64_MAX,
2696	};
2697	struct folio *folio;
2698	struct btrfs_inode *inode = BTRFS_I(rac->mapping->host);
2699	const u64 start = readahead_pos(rac);
2700	const u64 end = start + readahead_length(rac) - 1;
2701	struct extent_state *cached_state = NULL;
2702	struct extent_map *em_cached = NULL;
2703
2704	lock_extents_for_read(inode, start, end, cached_state: &cached_state);
2705
2706	while ((folio = readahead_folio(ractl: rac)) != NULL)
2707	btrfs_do_readpage(folio, em_cached: &em_cached, bio_ctrl: &bio_ctrl);
2708
2709	btrfs_unlock_extent(tree: &inode->io_tree, start, end, cached: &cached_state);
2710
2711	if (em_cached)
2712	btrfs_free_extent_map(em: em_cached);
2713	submit_one_bio(bio_ctrl: &bio_ctrl);
2714	}
2715
2716	/*
2717	* basic invalidate_folio code, this waits on any locked or writeback
2718	* ranges corresponding to the folio, and then deletes any extent state
2719	* records from the tree
2720	*/
2721	int extent_invalidate_folio(struct extent_io_tree *tree,
2722	struct folio *folio, size_t offset)
2723	{
2724	struct extent_state *cached_state = NULL;
2725	u64 start = folio_pos(folio);
2726	u64 end = start + folio_size(folio) - 1;
2727	size_t blocksize = folio_to_fs_info(folio)->sectorsize;
2728
2729	/* This function is only called for the btree inode */
2730	ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2731
2732	start += ALIGN(offset, blocksize);
2733	if (start > end)
2734	return 0;
2735
2736	btrfs_lock_extent(tree, start, end, cached: &cached_state);
2737	folio_wait_writeback(folio);
2738
2739	/*
2740	* Currently for btree io tree, only EXTENT_LOCKED is utilized,
2741	* so here we only need to unlock the extent range to free any
2742	* existing extent state.
2743	*/
2744	btrfs_unlock_extent(tree, start, end, cached: &cached_state);
2745	return 0;
2746	}
2747
2748	/*
2749	* A helper for struct address_space_operations::release_folio, this tests for
2750	* areas of the folio that are locked or under IO and drops the related state
2751	* bits if it is safe to drop the folio.
2752	*/
2753	static bool try_release_extent_state(struct extent_io_tree *tree,
2754	struct folio *folio)
2755	{
2756	struct extent_state *cached_state = NULL;
2757	u64 start = folio_pos(folio);
2758	u64 end = start + folio_size(folio) - 1;
2759	u32 range_bits;
2760	u32 clear_bits;
2761	bool ret = false;
2762	int ret2;
2763
2764	btrfs_get_range_bits(tree, start, end, bits: &range_bits, cached_state: &cached_state);
2765
2766	/*
2767	* We can release the folio if it's locked only for ordered extent
2768	* completion, since that doesn't require using the folio.
2769	*/
2770	if ((range_bits & EXTENT_LOCKED) &&
2771	!(range_bits & EXTENT_FINISHING_ORDERED))
2772	goto out;
2773
2774	clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM | EXTENT_DELALLOC_NEW |
2775	EXTENT_CTLBITS | EXTENT_QGROUP_RESERVED |
2776	EXTENT_FINISHING_ORDERED);
2777	/*
2778	* At this point we can safely clear everything except the locked,
2779	* nodatasum, delalloc new and finishing ordered bits. The delalloc new
2780	* bit will be cleared by ordered extent completion.
2781	*/
2782	ret2 = btrfs_clear_extent_bit(tree, start, end, bits: clear_bits, cached: &cached_state);
2783	/*
2784	* If clear_extent_bit failed for enomem reasons, we can't allow the
2785	* release to continue.
2786	*/
2787	if (ret2 == 0)
2788	ret = true;
2789	out:
2790	btrfs_free_extent_state(state: cached_state);
2791
2792	return ret;
2793	}
2794
2795	/*
2796	* a helper for release_folio. As long as there are no locked extents
2797	* in the range corresponding to the page, both state records and extent
2798	* map records are removed
2799	*/
2800	bool try_release_extent_mapping(struct folio *folio, gfp_t mask)
2801	{
2802	u64 start = folio_pos(folio);
2803	u64 end = start + folio_size(folio) - 1;
2804	struct btrfs_inode *inode = folio_to_inode(folio);
2805	struct extent_io_tree *io_tree = &inode->io_tree;
2806
2807	while (start <= end) {
2808	const u64 cur_gen = btrfs_get_fs_generation(fs_info: inode->root->fs_info);
2809	const u64 len = end - start + 1;
2810	struct extent_map_tree *extent_tree = &inode->extent_tree;
2811	struct extent_map *em;
2812
2813	write_lock(&extent_tree->lock);
2814	em = btrfs_lookup_extent_mapping(tree: extent_tree, start, len);
2815	if (!em) {
2816	write_unlock(&extent_tree->lock);
2817	break;
2818	}
2819	if ((em->flags & EXTENT_FLAG_PINNED) || em->start != start) {
2820	write_unlock(&extent_tree->lock);
2821	btrfs_free_extent_map(em);
2822	break;
2823	}
2824	if (btrfs_test_range_bit_exists(tree: io_tree, start: em->start,
2825	end: btrfs_extent_map_end(em) - 1,
2826	bit: EXTENT_LOCKED))
2827	goto next;
2828	/*
2829	* If it's not in the list of modified extents, used by a fast
2830	* fsync, we can remove it. If it's being logged we can safely
2831	* remove it since fsync took an extra reference on the em.
2832	*/
2833	if (list_empty(head: &em->list) || (em->flags & EXTENT_FLAG_LOGGING))
2834	goto remove_em;
2835	/*
2836	* If it's in the list of modified extents, remove it only if
2837	* its generation is older then the current one, in which case
2838	* we don't need it for a fast fsync. Otherwise don't remove it,
2839	* we could be racing with an ongoing fast fsync that could miss
2840	* the new extent.
2841	*/
2842	if (em->generation >= cur_gen)
2843	goto next;
2844	remove_em:
2845	/*
2846	* We only remove extent maps that are not in the list of
2847	* modified extents or that are in the list but with a
2848	* generation lower then the current generation, so there is no
2849	* need to set the full fsync flag on the inode (it hurts the
2850	* fsync performance for workloads with a data size that exceeds
2851	* or is close to the system's memory).
2852	*/
2853	btrfs_remove_extent_mapping(inode, em);
2854	/* Once for the inode's extent map tree. */
2855	btrfs_free_extent_map(em);
2856	next:
2857	start = btrfs_extent_map_end(em);
2858	write_unlock(&extent_tree->lock);
2859
2860	/* Once for us, for the lookup_extent_mapping() reference. */
2861	btrfs_free_extent_map(em);
2862
2863	if (need_resched()) {
2864	/*
2865	* If we need to resched but we can't block just exit
2866	* and leave any remaining extent maps.
2867	*/
2868	if (!gfpflags_allow_blocking(gfp_flags: mask))
2869	break;
2870
2871	cond_resched();
2872	}
2873	}
2874	return try_release_extent_state(tree: io_tree, folio);
2875	}
2876
2877	static int extent_buffer_under_io(const struct extent_buffer *eb)
2878	{
2879	return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
2880	test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
2881	}
2882
2883	static bool folio_range_has_eb(struct folio *folio)
2884	{
2885	struct btrfs_folio_state *bfs;
2886
2887	lockdep_assert_held(&folio->mapping->i_private_lock);
2888
2889	if (folio_test_private(folio)) {
2890	bfs = folio_get_private(folio);
2891	if (atomic_read(v: &bfs->eb_refs))
2892	return true;
2893	}
2894	return false;
2895	}
2896
2897	static void detach_extent_buffer_folio(const struct extent_buffer *eb, struct folio *folio)
2898	{
2899	struct btrfs_fs_info *fs_info = eb->fs_info;
2900	struct address_space *mapping = folio->mapping;
2901	const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
2902
2903	/*
2904	* For mapped eb, we're going to change the folio private, which should
2905	* be done under the i_private_lock.
2906	*/
2907	if (mapped)
2908	spin_lock(lock: &mapping->i_private_lock);
2909
2910	if (!folio_test_private(folio)) {
2911	if (mapped)
2912	spin_unlock(lock: &mapping->i_private_lock);
2913	return;
2914	}
2915
2916	if (!btrfs_meta_is_subpage(fs_info)) {
2917	/*
2918	* We do this since we'll remove the pages after we've removed
2919	* the eb from the xarray, so we could race and have this page
2920	* now attached to the new eb. So only clear folio if it's
2921	* still connected to this eb.
2922	*/
2923	if (folio_test_private(folio) && folio_get_private(folio) == eb) {
2924	BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
2925	BUG_ON(folio_test_dirty(folio));
2926	BUG_ON(folio_test_writeback(folio));
2927	/* We need to make sure we haven't be attached to a new eb. */
2928	folio_detach_private(folio);
2929	}
2930	if (mapped)
2931	spin_unlock(lock: &mapping->i_private_lock);
2932	return;
2933	}
2934
2935	/*
2936	* For subpage, we can have dummy eb with folio private attached. In
2937	* this case, we can directly detach the private as such folio is only
2938	* attached to one dummy eb, no sharing.
2939	*/
2940	if (!mapped) {
2941	btrfs_detach_folio_state(fs_info, folio, type: BTRFS_SUBPAGE_METADATA);
2942	return;
2943	}
2944
2945	btrfs_folio_dec_eb_refs(fs_info, folio);
2946
2947	/*
2948	* We can only detach the folio private if there are no other ebs in the
2949	* page range and no unfinished IO.
2950	*/
2951	if (!folio_range_has_eb(folio))
2952	btrfs_detach_folio_state(fs_info, folio, type: BTRFS_SUBPAGE_METADATA);
2953
2954	spin_unlock(lock: &mapping->i_private_lock);
2955	}
2956
2957	/* Release all folios attached to the extent buffer */
2958	static void btrfs_release_extent_buffer_folios(const struct extent_buffer *eb)
2959	{
2960	ASSERT(!extent_buffer_under_io(eb));
2961
2962	for (int i = 0; i < INLINE_EXTENT_BUFFER_PAGES; i++) {
2963	struct folio *folio = eb->folios[i];
2964
2965	if (!folio)
2966	continue;
2967
2968	detach_extent_buffer_folio(eb, folio);
2969	}
2970	}
2971
2972	/*
2973	* Helper for releasing the extent buffer.
2974	*/
2975	static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
2976	{
2977	btrfs_release_extent_buffer_folios(eb);
2978	btrfs_leak_debug_del_eb(eb);
2979	kmem_cache_free(s: extent_buffer_cache, objp: eb);
2980	}
2981
2982	static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info,
2983	u64 start)
2984	{
2985	struct extent_buffer *eb = NULL;
2986
2987	eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
2988	eb->start = start;
2989	eb->len = fs_info->nodesize;
2990	eb->fs_info = fs_info;
2991	init_rwsem(&eb->lock);
2992
2993	btrfs_leak_debug_add_eb(eb);
2994
2995	spin_lock_init(&eb->refs_lock);
2996	refcount_set(r: &eb->refs, n: 1);
2997
2998	ASSERT(eb->len <= BTRFS_MAX_METADATA_BLOCKSIZE);
2999
3000	return eb;
3001	}
3002
3003	/*
3004	* For use in eb allocation error cleanup paths, as btrfs_release_extent_buffer()
3005	* does not call folio_put(), and we need to set the folios to NULL so that
3006	* btrfs_release_extent_buffer() will not detach them a second time.
3007	*/
3008	static void cleanup_extent_buffer_folios(struct extent_buffer *eb)
3009	{
3010	const int num_folios = num_extent_folios(eb);
3011
3012	/* We cannot use num_extent_folios() as loop bound as eb->folios changes. */
3013	for (int i = 0; i < num_folios; i++) {
3014	ASSERT(eb->folios[i]);
3015	detach_extent_buffer_folio(eb, folio: eb->folios[i]);
3016	folio_put(folio: eb->folios[i]);
3017	eb->folios[i] = NULL;
3018	}
3019	}
3020
3021	struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
3022	{
3023	struct extent_buffer *new;
3024	int num_folios;
3025	int ret;
3026
3027	new = __alloc_extent_buffer(fs_info: src->fs_info, start: src->start);
3028	if (new == NULL)
3029	return NULL;
3030
3031	/*
3032	* Set UNMAPPED before calling btrfs_release_extent_buffer(), as
3033	* btrfs_release_extent_buffer() have different behavior for
3034	* UNMAPPED subpage extent buffer.
3035	*/
3036	set_bit(nr: EXTENT_BUFFER_UNMAPPED, addr: &new->bflags);
3037
3038	ret = alloc_eb_folio_array(eb: new, nofail: false);
3039	if (ret)
3040	goto release_eb;
3041
3042	ASSERT(num_extent_folios(src) == num_extent_folios(new),
3043	"%d != %d", num_extent_folios(src), num_extent_folios(new));
3044	/* Explicitly use the cached num_extent value from now on. */
3045	num_folios = num_extent_folios(eb: src);
3046	for (int i = 0; i < num_folios; i++) {
3047	struct folio *folio = new->folios[i];
3048
3049	ret = attach_extent_buffer_folio(eb: new, folio, NULL);
3050	if (ret < 0)
3051	goto cleanup_folios;
3052	WARN_ON(folio_test_dirty(folio));
3053	}
3054	for (int i = 0; i < num_folios; i++)
3055	folio_put(folio: new->folios[i]);
3056
3057	copy_extent_buffer_full(dst: new, src);
3058	set_extent_buffer_uptodate(new);
3059
3060	return new;
3061
3062	cleanup_folios:
3063	cleanup_extent_buffer_folios(eb: new);
3064	release_eb:
3065	btrfs_release_extent_buffer(eb: new);
3066	return NULL;
3067	}
3068
3069	struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
3070	u64 start)
3071	{
3072	struct extent_buffer *eb;
3073	int ret;
3074
3075	eb = __alloc_extent_buffer(fs_info, start);
3076	if (!eb)
3077	return NULL;
3078
3079	ret = alloc_eb_folio_array(eb, nofail: false);
3080	if (ret)
3081	goto release_eb;
3082
3083	for (int i = 0; i < num_extent_folios(eb); i++) {
3084	ret = attach_extent_buffer_folio(eb, folio: eb->folios[i], NULL);
3085	if (ret < 0)
3086	goto cleanup_folios;
3087	}
3088	for (int i = 0; i < num_extent_folios(eb); i++)
3089	folio_put(folio: eb->folios[i]);
3090
3091	set_extent_buffer_uptodate(eb);
3092	btrfs_set_header_nritems(eb, val: 0);
3093	set_bit(nr: EXTENT_BUFFER_UNMAPPED, addr: &eb->bflags);
3094
3095	return eb;
3096
3097	cleanup_folios:
3098	cleanup_extent_buffer_folios(eb);
3099	release_eb:
3100	btrfs_release_extent_buffer(eb);
3101	return NULL;
3102	}
3103
3104	static void check_buffer_tree_ref(struct extent_buffer *eb)
3105	{
3106	int refs;
3107	/*
3108	* The TREE_REF bit is first set when the extent_buffer is added to the
3109	* xarray. It is also reset, if unset, when a new reference is created
3110	* by find_extent_buffer.
3111	*
3112	* It is only cleared in two cases: freeing the last non-tree
3113	* reference to the extent_buffer when its STALE bit is set or
3114	* calling release_folio when the tree reference is the only reference.
3115	*
3116	* In both cases, care is taken to ensure that the extent_buffer's
3117	* pages are not under io. However, release_folio can be concurrently
3118	* called with creating new references, which is prone to race
3119	* conditions between the calls to check_buffer_tree_ref in those
3120	* codepaths and clearing TREE_REF in try_release_extent_buffer.
3121	*
3122	* The actual lifetime of the extent_buffer in the xarray is adequately
3123	* protected by the refcount, but the TREE_REF bit and its corresponding
3124	* reference are not. To protect against this class of races, we call
3125	* check_buffer_tree_ref() from the code paths which trigger io. Note that
3126	* once io is initiated, TREE_REF can no longer be cleared, so that is
3127	* the moment at which any such race is best fixed.
3128	*/
3129	refs = refcount_read(r: &eb->refs);
3130	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3131	return;
3132
3133	spin_lock(lock: &eb->refs_lock);
3134	if (!test_and_set_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags))
3135	refcount_inc(r: &eb->refs);
3136	spin_unlock(lock: &eb->refs_lock);
3137	}
3138
3139	static void mark_extent_buffer_accessed(struct extent_buffer *eb)
3140	{
3141	check_buffer_tree_ref(eb);
3142
3143	for (int i = 0; i < num_extent_folios(eb); i++)
3144	folio_mark_accessed(eb->folios[i]);
3145	}
3146
3147	struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
3148	u64 start)
3149	{
3150	struct extent_buffer *eb;
3151
3152	eb = find_extent_buffer_nolock(fs_info, start);
3153	if (!eb)
3154	return NULL;
3155	/*
3156	* Lock our eb's refs_lock to avoid races with free_extent_buffer().
3157	* When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
3158	* another task running free_extent_buffer() might have seen that flag
3159	* set, eb->refs == 2, that the buffer isn't under IO (dirty and
3160	* writeback flags not set) and it's still in the tree (flag
3161	* EXTENT_BUFFER_TREE_REF set), therefore being in the process of
3162	* decrementing the extent buffer's reference count twice. So here we
3163	* could race and increment the eb's reference count, clear its stale
3164	* flag, mark it as dirty and drop our reference before the other task
3165	* finishes executing free_extent_buffer, which would later result in
3166	* an attempt to free an extent buffer that is dirty.
3167	*/
3168	if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
3169	spin_lock(lock: &eb->refs_lock);
3170	spin_unlock(lock: &eb->refs_lock);
3171	}
3172	mark_extent_buffer_accessed(eb);
3173	return eb;
3174	}
3175
3176	struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
3177	u64 start)
3178	{
3179	#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3180	struct extent_buffer *eb, *exists = NULL;
3181	int ret;
3182
3183	eb = find_extent_buffer(fs_info, start);
3184	if (eb)
3185	return eb;
3186	eb = alloc_dummy_extent_buffer(fs_info, start);
3187	if (!eb)
3188	return ERR_PTR(error: -ENOMEM);
3189	eb->fs_info = fs_info;
3190	again:
3191	xa_lock_irq(&fs_info->buffer_tree);
3192	exists = __xa_cmpxchg(&fs_info->buffer_tree, index: start >> fs_info->nodesize_bits,
3193	NULL, entry: eb, GFP_NOFS);
3194	if (xa_is_err(entry: exists)) {
3195	ret = xa_err(entry: exists);
3196	xa_unlock_irq(&fs_info->buffer_tree);
3197	btrfs_release_extent_buffer(eb);
3198	return ERR_PTR(error: ret);
3199	}
3200	if (exists) {
3201	if (!refcount_inc_not_zero(r: &exists->refs)) {
3202	/* The extent buffer is being freed, retry. */
3203	xa_unlock_irq(&fs_info->buffer_tree);
3204	goto again;
3205	}
3206	xa_unlock_irq(&fs_info->buffer_tree);
3207	btrfs_release_extent_buffer(eb);
3208	return exists;
3209	}
3210	xa_unlock_irq(&fs_info->buffer_tree);
3211	check_buffer_tree_ref(eb);
3212
3213	return eb;
3214	#else
3215	/* Stub to avoid linker error when compiled with optimizations turned off. */
3216	return NULL;
3217	#endif
3218	}
3219
3220	static struct extent_buffer *grab_extent_buffer(struct btrfs_fs_info *fs_info,
3221	struct folio *folio)
3222	{
3223	struct extent_buffer *exists;
3224
3225	lockdep_assert_held(&folio->mapping->i_private_lock);
3226
3227	/*
3228	* For subpage case, we completely rely on xarray to ensure we don't try
3229	* to insert two ebs for the same bytenr. So here we always return NULL
3230	* and just continue.
3231	*/
3232	if (btrfs_meta_is_subpage(fs_info))
3233	return NULL;
3234
3235	/* Page not yet attached to an extent buffer */
3236	if (!folio_test_private(folio))
3237	return NULL;
3238
3239	/*
3240	* We could have already allocated an eb for this folio and attached one
3241	* so lets see if we can get a ref on the existing eb, and if we can we
3242	* know it's good and we can just return that one, else we know we can
3243	* just overwrite folio private.
3244	*/
3245	exists = folio_get_private(folio);
3246	if (refcount_inc_not_zero(r: &exists->refs))
3247	return exists;
3248
3249	WARN_ON(folio_test_dirty(folio));
3250	folio_detach_private(folio);
3251	return NULL;
3252	}
3253
3254	/*
3255	* Validate alignment constraints of eb at logical address @start.
3256	*/
3257	static bool check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
3258	{
3259	const u32 nodesize = fs_info->nodesize;
3260
3261	if (unlikely(!IS_ALIGNED(start, fs_info->sectorsize))) {
3262	btrfs_err(fs_info, "bad tree block start %llu", start);
3263	return true;
3264	}
3265
3266	if (unlikely(nodesize < PAGE_SIZE && !IS_ALIGNED(start, nodesize))) {
3267	btrfs_err(fs_info,
3268	"tree block is not nodesize aligned, start %llu nodesize %u",
3269	start, nodesize);
3270	return true;
3271	}
3272	if (unlikely(nodesize >= PAGE_SIZE && !PAGE_ALIGNED(start))) {
3273	btrfs_err(fs_info,
3274	"tree block is not page aligned, start %llu nodesize %u",
3275	start, nodesize);
3276	return true;
3277	}
3278	if (unlikely(!IS_ALIGNED(start, nodesize) &&
3279	!test_and_set_bit(BTRFS_FS_UNALIGNED_TREE_BLOCK, &fs_info->flags))) {
3280	btrfs_warn(fs_info,
3281	"tree block not nodesize aligned, start %llu nodesize %u, can be resolved by a full metadata balance",
3282	start, nodesize);
3283	}
3284	return false;
3285	}
3286
3287	/*
3288	* Return 0 if eb->folios[i] is attached to btree inode successfully.
3289	* Return >0 if there is already another extent buffer for the range,
3290	* and @found_eb_ret would be updated.
3291	* Return -EAGAIN if the filemap has an existing folio but with different size
3292	* than @eb.
3293	* The caller needs to free the existing folios and retry using the same order.
3294	*/
3295	static int attach_eb_folio_to_filemap(struct extent_buffer *eb, int i,
3296	struct btrfs_folio_state *prealloc,
3297	struct extent_buffer **found_eb_ret)
3298	{
3299
3300	struct btrfs_fs_info *fs_info = eb->fs_info;
3301	struct address_space *mapping = fs_info->btree_inode->i_mapping;
3302	const pgoff_t index = eb->start >> PAGE_SHIFT;
3303	struct folio *existing_folio;
3304	int ret;
3305
3306	ASSERT(found_eb_ret);
3307
3308	/* Caller should ensure the folio exists. */
3309	ASSERT(eb->folios[i]);
3310
3311	retry:
3312	existing_folio = NULL;
3313	ret = filemap_add_folio(mapping, folio: eb->folios[i], index: index + i,
3314	GFP_NOFS | __GFP_NOFAIL);
3315	if (!ret)
3316	goto finish;
3317
3318	existing_folio = filemap_lock_folio(mapping, index: index + i);
3319	/* The page cache only exists for a very short time, just retry. */
3320	if (IS_ERR(ptr: existing_folio))
3321	goto retry;
3322
3323	/* For now, we should only have single-page folios for btree inode. */
3324	ASSERT(folio_nr_pages(existing_folio) == 1);
3325
3326	if (folio_size(folio: existing_folio) != eb->folio_size) {
3327	folio_unlock(folio: existing_folio);
3328	folio_put(folio: existing_folio);
3329	return -EAGAIN;
3330	}
3331
3332	finish:
3333	spin_lock(lock: &mapping->i_private_lock);
3334	if (existing_folio && btrfs_meta_is_subpage(fs_info)) {
3335	/* We're going to reuse the existing page, can drop our folio now. */
3336	__free_page(folio_page(eb->folios[i], 0));
3337	eb->folios[i] = existing_folio;
3338	} else if (existing_folio) {
3339	struct extent_buffer *existing_eb;
3340
3341	existing_eb = grab_extent_buffer(fs_info, folio: existing_folio);
3342	if (existing_eb) {
3343	/* The extent buffer still exists, we can use it directly. */
3344	*found_eb_ret = existing_eb;
3345	spin_unlock(lock: &mapping->i_private_lock);
3346	folio_unlock(folio: existing_folio);
3347	folio_put(folio: existing_folio);
3348	return 1;
3349	}
3350	/* The extent buffer no longer exists, we can reuse the folio. */
3351	__free_page(folio_page(eb->folios[i], 0));
3352	eb->folios[i] = existing_folio;
3353	}
3354	eb->folio_size = folio_size(folio: eb->folios[i]);
3355	eb->folio_shift = folio_shift(folio: eb->folios[i]);
3356	/* Should not fail, as we have preallocated the memory. */
3357	ret = attach_extent_buffer_folio(eb, folio: eb->folios[i], prealloc);
3358	ASSERT(!ret);
3359	/*
3360	* To inform we have an extra eb under allocation, so that
3361	* detach_extent_buffer_page() won't release the folio private when the
3362	* eb hasn't been inserted into the xarray yet.
3363	*
3364	* The ref will be decreased when the eb releases the page, in
3365	* detach_extent_buffer_page(). Thus needs no special handling in the
3366	* error path.
3367	*/
3368	btrfs_folio_inc_eb_refs(fs_info, folio: eb->folios[i]);
3369	spin_unlock(lock: &mapping->i_private_lock);
3370	return 0;
3371	}
3372
3373	struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
3374	u64 start, u64 owner_root, int level)
3375	{
3376	int attached = 0;
3377	struct extent_buffer *eb;
3378	struct extent_buffer *existing_eb = NULL;
3379	struct btrfs_folio_state *prealloc = NULL;
3380	u64 lockdep_owner = owner_root;
3381	bool page_contig = true;
3382	int uptodate = 1;
3383	int ret;
3384
3385	if (check_eb_alignment(fs_info, start))
3386	return ERR_PTR(error: -EINVAL);
3387
3388	#if BITS_PER_LONG == 32
3389	if (start >= MAX_LFS_FILESIZE) {
3390	btrfs_err_rl(fs_info,
3391	"extent buffer %llu is beyond 32bit page cache limit", start);
3392	btrfs_err_32bit_limit(fs_info);
3393	return ERR_PTR(-EOVERFLOW);
3394	}
3395	if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3396	btrfs_warn_32bit_limit(fs_info);
3397	#endif
3398
3399	eb = find_extent_buffer(fs_info, start);
3400	if (eb)
3401	return eb;
3402
3403	eb = __alloc_extent_buffer(fs_info, start);
3404	if (!eb)
3405	return ERR_PTR(error: -ENOMEM);
3406
3407	/*
3408	* The reloc trees are just snapshots, so we need them to appear to be
3409	* just like any other fs tree WRT lockdep.
3410	*/
3411	if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3412	lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3413
3414	btrfs_set_buffer_lockdep_class(objectid: lockdep_owner, eb, level);
3415
3416	/*
3417	* Preallocate folio private for subpage case, so that we won't
3418	* allocate memory with i_private_lock nor page lock hold.
3419	*
3420	* The memory will be freed by attach_extent_buffer_page() or freed
3421	* manually if we exit earlier.
3422	*/
3423	if (btrfs_meta_is_subpage(fs_info)) {
3424	prealloc = btrfs_alloc_folio_state(fs_info, PAGE_SIZE, type: BTRFS_SUBPAGE_METADATA);
3425	if (IS_ERR(ptr: prealloc)) {
3426	ret = PTR_ERR(ptr: prealloc);
3427	goto out;
3428	}
3429	}
3430
3431	reallocate:
3432	/* Allocate all pages first. */
3433	ret = alloc_eb_folio_array(eb, nofail: true);
3434	if (ret < 0) {
3435	btrfs_free_folio_state(bfs: prealloc);
3436	goto out;
3437	}
3438
3439	/* Attach all pages to the filemap. */
3440	for (int i = 0; i < num_extent_folios(eb); i++) {
3441	struct folio *folio;
3442
3443	ret = attach_eb_folio_to_filemap(eb, i, prealloc, found_eb_ret: &existing_eb);
3444	if (ret > 0) {
3445	ASSERT(existing_eb);
3446	goto out;
3447	}
3448
3449	/*
3450	* TODO: Special handling for a corner case where the order of
3451	* folios mismatch between the new eb and filemap.
3452	*
3453	* This happens when:
3454	*
3455	* - the new eb is using higher order folio
3456	*
3457	* - the filemap is still using 0-order folios for the range
3458	* This can happen at the previous eb allocation, and we don't
3459	* have higher order folio for the call.
3460	*
3461	* - the existing eb has already been freed
3462	*
3463	* In this case, we have to free the existing folios first, and
3464	* re-allocate using the same order.
3465	* Thankfully this is not going to happen yet, as we're still
3466	* using 0-order folios.
3467	*/
3468	if (unlikely(ret == -EAGAIN)) {
3469	DEBUG_WARN("folio order mismatch between new eb and filemap");
3470	goto reallocate;
3471	}
3472	attached++;
3473
3474	/*
3475	* Only after attach_eb_folio_to_filemap(), eb->folios[] is
3476	* reliable, as we may choose to reuse the existing page cache
3477	* and free the allocated page.
3478	*/
3479	folio = eb->folios[i];
3480	WARN_ON(btrfs_meta_folio_test_dirty(folio, eb));
3481
3482	/*
3483	* Check if the current page is physically contiguous with previous eb
3484	* page.
3485	* At this stage, either we allocated a large folio, thus @i
3486	* would only be 0, or we fall back to per-page allocation.
3487	*/
3488	if (i && folio_page(eb->folios[i - 1], 0) + 1 != folio_page(folio, 0))
3489	page_contig = false;
3490
3491	if (!btrfs_meta_folio_test_uptodate(folio, eb))
3492	uptodate = 0;
3493
3494	/*
3495	* We can't unlock the pages just yet since the extent buffer
3496	* hasn't been properly inserted into the xarray, this opens a
3497	* race with btree_release_folio() which can free a page while we
3498	* are still filling in all pages for the buffer and we could crash.
3499	*/
3500	}
3501	if (uptodate)
3502	set_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
3503	/* All pages are physically contiguous, can skip cross page handling. */
3504	if (page_contig)
3505	eb->addr = folio_address(folio: eb->folios[0]) + offset_in_page(eb->start);
3506	again:
3507	xa_lock_irq(&fs_info->buffer_tree);
3508	existing_eb = __xa_cmpxchg(&fs_info->buffer_tree,
3509	index: start >> fs_info->nodesize_bits, NULL, entry: eb,
3510	GFP_NOFS);
3511	if (xa_is_err(entry: existing_eb)) {
3512	ret = xa_err(entry: existing_eb);
3513	xa_unlock_irq(&fs_info->buffer_tree);
3514	goto out;
3515	}
3516	if (existing_eb) {
3517	if (!refcount_inc_not_zero(r: &existing_eb->refs)) {
3518	xa_unlock_irq(&fs_info->buffer_tree);
3519	goto again;
3520	}
3521	xa_unlock_irq(&fs_info->buffer_tree);
3522	goto out;
3523	}
3524	xa_unlock_irq(&fs_info->buffer_tree);
3525
3526	/* add one reference for the tree */
3527	check_buffer_tree_ref(eb);
3528
3529	/*
3530	* Now it's safe to unlock the pages because any calls to
3531	* btree_release_folio will correctly detect that a page belongs to a
3532	* live buffer and won't free them prematurely.
3533	*/
3534	for (int i = 0; i < num_extent_folios(eb); i++) {
3535	folio_unlock(folio: eb->folios[i]);
3536	/*
3537	* A folio that has been added to an address_space mapping
3538	* should not continue holding the refcount from its original
3539	* allocation indefinitely.
3540	*/
3541	folio_put(folio: eb->folios[i]);
3542	}
3543	return eb;
3544
3545	out:
3546	WARN_ON(!refcount_dec_and_test(&eb->refs));
3547
3548	/*
3549	* Any attached folios need to be detached before we unlock them. This
3550	* is because when we're inserting our new folios into the mapping, and
3551	* then attaching our eb to that folio. If we fail to insert our folio
3552	* we'll lookup the folio for that index, and grab that EB. We do not
3553	* want that to grab this eb, as we're getting ready to free it. So we
3554	* have to detach it first and then unlock it.
3555	*
3556	* Note: the bounds is num_extent_pages() as we need to go through all slots.
3557	*/
3558	for (int i = 0; i < num_extent_pages(eb); i++) {
3559	struct folio *folio = eb->folios[i];
3560
3561	if (i < attached) {
3562	ASSERT(folio);
3563	detach_extent_buffer_folio(eb, folio);
3564	folio_unlock(folio);
3565	} else if (!folio) {
3566	continue;
3567	}
3568
3569	folio_put(folio);
3570	eb->folios[i] = NULL;
3571	}
3572	btrfs_release_extent_buffer(eb);
3573	if (ret < 0)
3574	return ERR_PTR(error: ret);
3575	ASSERT(existing_eb);
3576	return existing_eb;
3577	}
3578
3579	static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3580	{
3581	struct extent_buffer *eb =
3582	container_of(head, struct extent_buffer, rcu_head);
3583
3584	kmem_cache_free(s: extent_buffer_cache, objp: eb);
3585	}
3586
3587	static int release_extent_buffer(struct extent_buffer *eb)
3588	__releases(&eb->refs_lock)
3589	{
3590	lockdep_assert_held(&eb->refs_lock);
3591
3592	if (refcount_dec_and_test(r: &eb->refs)) {
3593	struct btrfs_fs_info *fs_info = eb->fs_info;
3594
3595	spin_unlock(lock: &eb->refs_lock);
3596
3597	/*
3598	* We're erasing, theoretically there will be no allocations, so
3599	* just use GFP_ATOMIC.
3600	*
3601	* We use cmpxchg instead of erase because we do not know if
3602	* this eb is actually in the tree or not, we could be cleaning
3603	* up an eb that we allocated but never inserted into the tree.
3604	* Thus use cmpxchg to remove it from the tree if it is there,
3605	* or leave the other entry if this isn't in the tree.
3606	*
3607	* The documentation says that putting a NULL value is the same
3608	* as erase as long as XA_FLAGS_ALLOC is not set, which it isn't
3609	* in this case.
3610	*/
3611	xa_cmpxchg_irq(xa: &fs_info->buffer_tree,
3612	index: eb->start >> fs_info->nodesize_bits, old: eb, NULL,
3613	GFP_ATOMIC);
3614
3615	btrfs_leak_debug_del_eb(eb);
3616	/* Should be safe to release folios at this point. */
3617	btrfs_release_extent_buffer_folios(eb);
3618	#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3619	if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
3620	kmem_cache_free(s: extent_buffer_cache, objp: eb);
3621	return 1;
3622	}
3623	#endif
3624	call_rcu(head: &eb->rcu_head, func: btrfs_release_extent_buffer_rcu);
3625	return 1;
3626	}
3627	spin_unlock(lock: &eb->refs_lock);
3628
3629	return 0;
3630	}
3631
3632	void free_extent_buffer(struct extent_buffer *eb)
3633	{
3634	int refs;
3635	if (!eb)
3636	return;
3637
3638	refs = refcount_read(r: &eb->refs);
3639	while (1) {
3640	if (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)) {
3641	if (refs == 1)
3642	break;
3643	} else if (refs <= 3) {
3644	break;
3645	}
3646
3647	/* Optimization to avoid locking eb->refs_lock. */
3648	if (atomic_try_cmpxchg(v: &eb->refs.refs, old: &refs, new: refs - 1))
3649	return;
3650	}
3651
3652	spin_lock(lock: &eb->refs_lock);
3653	if (refcount_read(r: &eb->refs) == 2 &&
3654	test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
3655	!extent_buffer_under_io(eb) &&
3656	test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags))
3657	refcount_dec(r: &eb->refs);
3658
3659	/*
3660	* I know this is terrible, but it's temporary until we stop tracking
3661	* the uptodate bits and such for the extent buffers.
3662	*/
3663	release_extent_buffer(eb);
3664	}
3665
3666	void free_extent_buffer_stale(struct extent_buffer *eb)
3667	{
3668	if (!eb)
3669	return;
3670
3671	spin_lock(lock: &eb->refs_lock);
3672	set_bit(nr: EXTENT_BUFFER_STALE, addr: &eb->bflags);
3673
3674	if (refcount_read(r: &eb->refs) == 2 && !extent_buffer_under_io(eb) &&
3675	test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags))
3676	refcount_dec(r: &eb->refs);
3677	release_extent_buffer(eb);
3678	}
3679
3680	static void btree_clear_folio_dirty_tag(struct folio *folio)
3681	{
3682	ASSERT(!folio_test_dirty(folio));
3683	ASSERT(folio_test_locked(folio));
3684	xa_lock_irq(&folio->mapping->i_pages);
3685	if (!folio_test_dirty(folio))
3686	__xa_clear_mark(&folio->mapping->i_pages, index: folio->index,
3687	PAGECACHE_TAG_DIRTY);
3688	xa_unlock_irq(&folio->mapping->i_pages);
3689	}
3690
3691	void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
3692	struct extent_buffer *eb)
3693	{
3694	struct btrfs_fs_info *fs_info = eb->fs_info;
3695
3696	btrfs_assert_tree_write_locked(eb);
3697
3698	if (trans && btrfs_header_generation(eb) != trans->transid)
3699	return;
3700
3701	/*
3702	* Instead of clearing the dirty flag off of the buffer, mark it as
3703	* EXTENT_BUFFER_ZONED_ZEROOUT. This allows us to preserve
3704	* write-ordering in zoned mode, without the need to later re-dirty
3705	* the extent_buffer.
3706	*
3707	* The actual zeroout of the buffer will happen later in
3708	* btree_csum_one_bio.
3709	*/
3710	if (btrfs_is_zoned(fs_info) && test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3711	set_bit(nr: EXTENT_BUFFER_ZONED_ZEROOUT, addr: &eb->bflags);
3712	return;
3713	}
3714
3715	if (!test_and_clear_bit(nr: EXTENT_BUFFER_DIRTY, addr: &eb->bflags))
3716	return;
3717
3718	buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY);
3719	percpu_counter_add_batch(fbc: &fs_info->dirty_metadata_bytes, amount: -eb->len,
3720	batch: fs_info->dirty_metadata_batch);
3721
3722	for (int i = 0; i < num_extent_folios(eb); i++) {
3723	struct folio *folio = eb->folios[i];
3724	bool last;
3725
3726	if (!folio_test_dirty(folio))
3727	continue;
3728	folio_lock(folio);
3729	last = btrfs_meta_folio_clear_and_test_dirty(folio, eb);
3730	if (last)
3731	btree_clear_folio_dirty_tag(folio);
3732	folio_unlock(folio);
3733	}
3734	WARN_ON(refcount_read(&eb->refs) == 0);
3735	}
3736
3737	void set_extent_buffer_dirty(struct extent_buffer *eb)
3738	{
3739	bool was_dirty;
3740
3741	check_buffer_tree_ref(eb);
3742
3743	was_dirty = test_and_set_bit(nr: EXTENT_BUFFER_DIRTY, addr: &eb->bflags);
3744
3745	WARN_ON(refcount_read(&eb->refs) == 0);
3746	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
3747	WARN_ON(test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags));
3748
3749	if (!was_dirty) {
3750	bool subpage = btrfs_meta_is_subpage(fs_info: eb->fs_info);
3751
3752	/*
3753	* For subpage case, we can have other extent buffers in the
3754	* same page, and in clear_extent_buffer_dirty() we
3755	* have to clear page dirty without subpage lock held.
3756	* This can cause race where our page gets dirty cleared after
3757	* we just set it.
3758	*
3759	* Thankfully, clear_extent_buffer_dirty() has locked
3760	* its page for other reasons, we can use page lock to prevent
3761	* the above race.
3762	*/
3763	if (subpage)
3764	folio_lock(folio: eb->folios[0]);
3765	for (int i = 0; i < num_extent_folios(eb); i++)
3766	btrfs_meta_folio_set_dirty(folio: eb->folios[i], eb);
3767	buffer_tree_set_mark(eb, PAGECACHE_TAG_DIRTY);
3768	if (subpage)
3769	folio_unlock(folio: eb->folios[0]);
3770	percpu_counter_add_batch(fbc: &eb->fs_info->dirty_metadata_bytes,
3771	amount: eb->len,
3772	batch: eb->fs_info->dirty_metadata_batch);
3773	}
3774	#ifdef CONFIG_BTRFS_DEBUG
3775	for (int i = 0; i < num_extent_folios(eb); i++)
3776	ASSERT(folio_test_dirty(eb->folios[i]));
3777	#endif
3778	}
3779
3780	void clear_extent_buffer_uptodate(struct extent_buffer *eb)
3781	{
3782
3783	clear_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
3784	for (int i = 0; i < num_extent_folios(eb); i++) {
3785	struct folio *folio = eb->folios[i];
3786
3787	if (!folio)
3788	continue;
3789
3790	btrfs_meta_folio_clear_uptodate(folio, eb);
3791	}
3792	}
3793
3794	void set_extent_buffer_uptodate(struct extent_buffer *eb)
3795	{
3796
3797	set_bit(nr: EXTENT_BUFFER_UPTODATE, addr: &eb->bflags);
3798	for (int i = 0; i < num_extent_folios(eb); i++)
3799	btrfs_meta_folio_set_uptodate(folio: eb->folios[i], eb);
3800	}
3801
3802	static void clear_extent_buffer_reading(struct extent_buffer *eb)
3803	{
3804	clear_and_wake_up_bit(bit: EXTENT_BUFFER_READING, word: &eb->bflags);
3805	}
3806
3807	static void end_bbio_meta_read(struct btrfs_bio *bbio)
3808	{
3809	struct extent_buffer *eb = bbio->private;
3810	bool uptodate = !bbio->bio.bi_status;
3811
3812	/*
3813	* If the extent buffer is marked UPTODATE before the read operation
3814	* completes, other calls to read_extent_buffer_pages() will return
3815	* early without waiting for the read to finish, causing data races.
3816	*/
3817	WARN_ON(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags));
3818
3819	eb->read_mirror = bbio->mirror_num;
3820
3821	if (uptodate &&
3822	btrfs_validate_extent_buffer(eb, check: &bbio->parent_check) < 0)
3823	uptodate = false;
3824
3825	if (uptodate)
3826	set_extent_buffer_uptodate(eb);
3827	else
3828	clear_extent_buffer_uptodate(eb);
3829
3830	clear_extent_buffer_reading(eb);
3831	free_extent_buffer(eb);
3832
3833	bio_put(&bbio->bio);
3834	}
3835
3836	int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num,
3837	const struct btrfs_tree_parent_check *check)
3838	{
3839	struct btrfs_fs_info *fs_info = eb->fs_info;
3840	struct btrfs_bio *bbio;
3841
3842	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3843	return 0;
3844
3845	/*
3846	* We could have had EXTENT_BUFFER_UPTODATE cleared by the write
3847	* operation, which could potentially still be in flight. In this case
3848	* we simply want to return an error.
3849	*/
3850	if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
3851	return -EIO;
3852
3853	/* Someone else is already reading the buffer, just wait for it. */
3854	if (test_and_set_bit(nr: EXTENT_BUFFER_READING, addr: &eb->bflags))
3855	return 0;
3856
3857	/*
3858	* Between the initial test_bit(EXTENT_BUFFER_UPTODATE) and the above
3859	* test_and_set_bit(EXTENT_BUFFER_READING), someone else could have
3860	* started and finished reading the same eb. In this case, UPTODATE
3861	* will now be set, and we shouldn't read it in again.
3862	*/
3863	if (unlikely(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))) {
3864	clear_extent_buffer_reading(eb);
3865	return 0;
3866	}
3867
3868	eb->read_mirror = 0;
3869	check_buffer_tree_ref(eb);
3870	refcount_inc(r: &eb->refs);
3871
3872	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
3873	opf: REQ_OP_READ | REQ_META, BTRFS_I(fs_info->btree_inode),
3874	file_offset: eb->start, end_io: end_bbio_meta_read, private: eb);
3875	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
3876	memcpy(&bbio->parent_check, check, sizeof(*check));
3877	for (int i = 0; i < num_extent_folios(eb); i++) {
3878	struct folio *folio = eb->folios[i];
3879	u64 range_start = max_t(u64, eb->start, folio_pos(folio));
3880	u32 range_len = min_t(u64, folio_next_pos(folio),
3881	eb->start + eb->len) - range_start;
3882
3883	bio_add_folio_nofail(bio: &bbio->bio, folio, len: range_len,
3884	offset_in_folio(folio, range_start));
3885	}
3886	btrfs_submit_bbio(bbio, mirror_num);
3887	return 0;
3888	}
3889
3890	int read_extent_buffer_pages(struct extent_buffer *eb, int mirror_num,
3891	const struct btrfs_tree_parent_check *check)
3892	{
3893	int ret;
3894
3895	ret = read_extent_buffer_pages_nowait(eb, mirror_num, check);
3896	if (ret < 0)
3897	return ret;
3898
3899	wait_on_bit_io(word: &eb->bflags, bit: EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
3900	if (unlikely(!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)))
3901	return -EIO;
3902	return 0;
3903	}
3904
3905	static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
3906	unsigned long len)
3907	{
3908	btrfs_warn(eb->fs_info,
3909	"access to eb bytenr %llu len %u out of range start %lu len %lu",
3910	eb->start, eb->len, start, len);
3911	DEBUG_WARN();
3912
3913	return true;
3914	}
3915
3916	/*
3917	* Check if the [start, start + len) range is valid before reading/writing
3918	* the eb.
3919	* NOTE: @start and @len are offset inside the eb, not logical address.
3920	*
3921	* Caller should not touch the dst/src memory if this function returns error.
3922	*/
3923	static inline int check_eb_range(const struct extent_buffer *eb,
3924	unsigned long start, unsigned long len)
3925	{
3926	unsigned long offset;
3927
3928	/* start, start + len should not go beyond eb->len nor overflow */
3929	if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
3930	return report_eb_range(eb, start, len);
3931
3932	return false;
3933	}
3934
3935	void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
3936	unsigned long start, unsigned long len)
3937	{
3938	const int unit_size = eb->folio_size;
3939	size_t cur;
3940	size_t offset;
3941	char *dst = (char *)dstv;
3942	unsigned long i = get_eb_folio_index(eb, offset: start);
3943
3944	if (check_eb_range(eb, start, len)) {
3945	/*
3946	* Invalid range hit, reset the memory, so callers won't get
3947	* some random garbage for their uninitialized memory.
3948	*/
3949	memset(dstv, 0, len);
3950	return;
3951	}
3952
3953	if (eb->addr) {
3954	memcpy(dstv, eb->addr + start, len);
3955	return;
3956	}
3957
3958	offset = get_eb_offset_in_folio(eb, offset: start);
3959
3960	while (len > 0) {
3961	char *kaddr;
3962
3963	cur = min(len, unit_size - offset);
3964	kaddr = folio_address(folio: eb->folios[i]);
3965	memcpy(dst, kaddr + offset, cur);
3966
3967	dst += cur;
3968	len -= cur;
3969	offset = 0;
3970	i++;
3971	}
3972	}
3973
3974	int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
3975	void __user *dstv,
3976	unsigned long start, unsigned long len)
3977	{
3978	const int unit_size = eb->folio_size;
3979	size_t cur;
3980	size_t offset;
3981	char __user *dst = (char __user *)dstv;
3982	unsigned long i = get_eb_folio_index(eb, offset: start);
3983	int ret = 0;
3984
3985	WARN_ON(start > eb->len);
3986	WARN_ON(start + len > eb->start + eb->len);
3987
3988	if (eb->addr) {
3989	if (copy_to_user_nofault(dst: dstv, src: eb->addr + start, size: len))
3990	ret = -EFAULT;
3991	return ret;
3992	}
3993
3994	offset = get_eb_offset_in_folio(eb, offset: start);
3995
3996	while (len > 0) {
3997	char *kaddr;
3998
3999	cur = min(len, unit_size - offset);
4000	kaddr = folio_address(folio: eb->folios[i]);
4001	if (copy_to_user_nofault(dst, src: kaddr + offset, size: cur)) {
4002	ret = -EFAULT;
4003	break;
4004	}
4005
4006	dst += cur;
4007	len -= cur;
4008	offset = 0;
4009	i++;
4010	}
4011
4012	return ret;
4013	}
4014
4015	int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
4016	unsigned long start, unsigned long len)
4017	{
4018	const int unit_size = eb->folio_size;
4019	size_t cur;
4020	size_t offset;
4021	char *kaddr;
4022	char *ptr = (char *)ptrv;
4023	unsigned long i = get_eb_folio_index(eb, offset: start);
4024	int ret = 0;
4025
4026	if (check_eb_range(eb, start, len))
4027	return -EINVAL;
4028
4029	if (eb->addr)
4030	return memcmp(p: ptrv, q: eb->addr + start, size: len);
4031
4032	offset = get_eb_offset_in_folio(eb, offset: start);
4033
4034	while (len > 0) {
4035	cur = min(len, unit_size - offset);
4036	kaddr = folio_address(folio: eb->folios[i]);
4037	ret = memcmp(p: ptr, q: kaddr + offset, size: cur);
4038	if (ret)
4039	break;
4040
4041	ptr += cur;
4042	len -= cur;
4043	offset = 0;
4044	i++;
4045	}
4046	return ret;
4047	}
4048
4049	/*
4050	* Check that the extent buffer is uptodate.
4051	*
4052	* For regular sector size == PAGE_SIZE case, check if @page is uptodate.
4053	* For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
4054	*/
4055	static void assert_eb_folio_uptodate(const struct extent_buffer *eb, int i)
4056	{
4057	struct btrfs_fs_info *fs_info = eb->fs_info;
4058	struct folio *folio = eb->folios[i];
4059
4060	ASSERT(folio);
4061
4062	/*
4063	* If we are using the commit root we could potentially clear a page
4064	* Uptodate while we're using the extent buffer that we've previously
4065	* looked up. We don't want to complain in this case, as the page was
4066	* valid before, we just didn't write it out. Instead we want to catch
4067	* the case where we didn't actually read the block properly, which
4068	* would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
4069	*/
4070	if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
4071	return;
4072
4073	if (btrfs_meta_is_subpage(fs_info)) {
4074	folio = eb->folios[0];
4075	ASSERT(i == 0);
4076	if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, folio,
4077	eb->start, eb->len)))
4078	btrfs_subpage_dump_bitmap(fs_info, folio, start: eb->start, len: eb->len);
4079	} else {
4080	WARN_ON(!folio_test_uptodate(folio));
4081	}
4082	}
4083
4084	static void __write_extent_buffer(const struct extent_buffer *eb,
4085	const void *srcv, unsigned long start,
4086	unsigned long len, bool use_memmove)
4087	{
4088	const int unit_size = eb->folio_size;
4089	size_t cur;
4090	size_t offset;
4091	char *kaddr;
4092	const char *src = (const char *)srcv;
4093	unsigned long i = get_eb_folio_index(eb, offset: start);
4094	/* For unmapped (dummy) ebs, no need to check their uptodate status. */
4095	const bool check_uptodate = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
4096
4097	if (check_eb_range(eb, start, len))
4098	return;
4099
4100	if (eb->addr) {
4101	if (use_memmove)
4102	memmove(eb->addr + start, srcv, len);
4103	else
4104	memcpy(eb->addr + start, srcv, len);
4105	return;
4106	}
4107
4108	offset = get_eb_offset_in_folio(eb, offset: start);
4109
4110	while (len > 0) {
4111	if (check_uptodate)
4112	assert_eb_folio_uptodate(eb, i);
4113
4114	cur = min(len, unit_size - offset);
4115	kaddr = folio_address(folio: eb->folios[i]);
4116	if (use_memmove)
4117	memmove(kaddr + offset, src, cur);
4118	else
4119	memcpy(kaddr + offset, src, cur);
4120
4121	src += cur;
4122	len -= cur;
4123	offset = 0;
4124	i++;
4125	}
4126	}
4127
4128	void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
4129	unsigned long start, unsigned long len)
4130	{
4131	return __write_extent_buffer(eb, srcv, start, len, use_memmove: false);
4132	}
4133
4134	static void memset_extent_buffer(const struct extent_buffer *eb, int c,
4135	unsigned long start, unsigned long len)
4136	{
4137	const int unit_size = eb->folio_size;
4138	unsigned long cur = start;
4139
4140	if (eb->addr) {
4141	memset(eb->addr + start, c, len);
4142	return;
4143	}
4144
4145	while (cur < start + len) {
4146	unsigned long index = get_eb_folio_index(eb, offset: cur);
4147	unsigned int offset = get_eb_offset_in_folio(eb, offset: cur);
4148	unsigned int cur_len = min(start + len - cur, unit_size - offset);
4149
4150	assert_eb_folio_uptodate(eb, i: index);
4151	memset(folio_address(eb->folios[index]) + offset, c, cur_len);
4152
4153	cur += cur_len;
4154	}
4155	}
4156
4157	void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
4158	unsigned long len)
4159	{
4160	if (check_eb_range(eb, start, len))
4161	return;
4162	return memset_extent_buffer(eb, c: 0, start, len);
4163	}
4164
4165	void copy_extent_buffer_full(const struct extent_buffer *dst,
4166	const struct extent_buffer *src)
4167	{
4168	const int unit_size = src->folio_size;
4169	unsigned long cur = 0;
4170
4171	ASSERT(dst->len == src->len);
4172
4173	while (cur < src->len) {
4174	unsigned long index = get_eb_folio_index(eb: src, offset: cur);
4175	unsigned long offset = get_eb_offset_in_folio(eb: src, offset: cur);
4176	unsigned long cur_len = min(src->len, unit_size - offset);
4177	void *addr = folio_address(folio: src->folios[index]) + offset;
4178
4179	write_extent_buffer(eb: dst, srcv: addr, start: cur, len: cur_len);
4180
4181	cur += cur_len;
4182	}
4183	}
4184
4185	void copy_extent_buffer(const struct extent_buffer *dst,
4186	const struct extent_buffer *src,
4187	unsigned long dst_offset, unsigned long src_offset,
4188	unsigned long len)
4189	{
4190	const int unit_size = dst->folio_size;
4191	u64 dst_len = dst->len;
4192	size_t cur;
4193	size_t offset;
4194	char *kaddr;
4195	unsigned long i = get_eb_folio_index(eb: dst, offset: dst_offset);
4196
4197	if (check_eb_range(eb: dst, start: dst_offset, len) ||
4198	check_eb_range(eb: src, start: src_offset, len))
4199	return;
4200
4201	WARN_ON(src->len != dst_len);
4202
4203	offset = get_eb_offset_in_folio(eb: dst, offset: dst_offset);
4204
4205	while (len > 0) {
4206	assert_eb_folio_uptodate(eb: dst, i);
4207
4208	cur = min(len, (unsigned long)(unit_size - offset));
4209
4210	kaddr = folio_address(folio: dst->folios[i]);
4211	read_extent_buffer(eb: src, dstv: kaddr + offset, start: src_offset, len: cur);
4212
4213	src_offset += cur;
4214	len -= cur;
4215	offset = 0;
4216	i++;
4217	}
4218	}
4219
4220	/*
4221	* Calculate the folio and offset of the byte containing the given bit number.
4222	*
4223	* @eb: the extent buffer
4224	* @start: offset of the bitmap item in the extent buffer
4225	* @nr: bit number
4226	* @folio_index: return index of the folio in the extent buffer that contains
4227	* the given bit number
4228	* @folio_offset: return offset into the folio given by folio_index
4229	*
4230	* This helper hides the ugliness of finding the byte in an extent buffer which
4231	* contains a given bit.
4232	*/
4233	static inline void eb_bitmap_offset(const struct extent_buffer *eb,
4234	unsigned long start, unsigned long nr,
4235	unsigned long *folio_index,
4236	size_t *folio_offset)
4237	{
4238	size_t byte_offset = BIT_BYTE(nr);
4239	size_t offset;
4240
4241	/*
4242	* The byte we want is the offset of the extent buffer + the offset of
4243	* the bitmap item in the extent buffer + the offset of the byte in the
4244	* bitmap item.
4245	*/
4246	offset = start + offset_in_eb_folio(eb, start: eb->start) + byte_offset;
4247
4248	*folio_index = offset >> eb->folio_shift;
4249	*folio_offset = offset_in_eb_folio(eb, start: offset);
4250	}
4251
4252	/*
4253	* Determine whether a bit in a bitmap item is set.
4254	*
4255	* @eb: the extent buffer
4256	* @start: offset of the bitmap item in the extent buffer
4257	* @nr: bit number to test
4258	*/
4259	bool extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
4260	unsigned long nr)
4261	{
4262	unsigned long i;
4263	size_t offset;
4264	u8 *kaddr;
4265
4266	eb_bitmap_offset(eb, start, nr, folio_index: &i, folio_offset: &offset);
4267	assert_eb_folio_uptodate(eb, i);
4268	kaddr = folio_address(folio: eb->folios[i]);
4269	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
4270	}
4271
4272	static u8 *extent_buffer_get_byte(const struct extent_buffer *eb, unsigned long bytenr)
4273	{
4274	unsigned long index = get_eb_folio_index(eb, offset: bytenr);
4275
4276	if (check_eb_range(eb, start: bytenr, len: 1))
4277	return NULL;
4278	return folio_address(folio: eb->folios[index]) + get_eb_offset_in_folio(eb, offset: bytenr);
4279	}
4280
4281	/*
4282	* Set an area of a bitmap to 1.
4283	*
4284	* @eb: the extent buffer
4285	* @start: offset of the bitmap item in the extent buffer
4286	* @pos: bit number of the first bit
4287	* @len: number of bits to set
4288	*/
4289	void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
4290	unsigned long pos, unsigned long len)
4291	{
4292	unsigned int first_byte = start + BIT_BYTE(pos);
4293	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4294	const bool same_byte = (first_byte == last_byte);
4295	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4296	u8 *kaddr;
4297
4298	if (same_byte)
4299	mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4300
4301	/* Handle the first byte. */
4302	kaddr = extent_buffer_get_byte(eb, bytenr: first_byte);
4303	*kaddr |= mask;
4304	if (same_byte)
4305	return;
4306
4307	/* Handle the byte aligned part. */
4308	ASSERT(first_byte + 1 <= last_byte);
4309	memset_extent_buffer(eb, c: 0xff, start: first_byte + 1, len: last_byte - first_byte - 1);
4310
4311	/* Handle the last byte. */
4312	kaddr = extent_buffer_get_byte(eb, bytenr: last_byte);
4313	*kaddr |= BITMAP_LAST_BYTE_MASK(pos + len);
4314	}
4315
4316
4317	/*
4318	* Clear an area of a bitmap.
4319	*
4320	* @eb: the extent buffer
4321	* @start: offset of the bitmap item in the extent buffer
4322	* @pos: bit number of the first bit
4323	* @len: number of bits to clear
4324	*/
4325	void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
4326	unsigned long start, unsigned long pos,
4327	unsigned long len)
4328	{
4329	unsigned int first_byte = start + BIT_BYTE(pos);
4330	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4331	const bool same_byte = (first_byte == last_byte);
4332	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4333	u8 *kaddr;
4334
4335	if (same_byte)
4336	mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4337
4338	/* Handle the first byte. */
4339	kaddr = extent_buffer_get_byte(eb, bytenr: first_byte);
4340	*kaddr &= ~mask;
4341	if (same_byte)
4342	return;
4343
4344	/* Handle the byte aligned part. */
4345	ASSERT(first_byte + 1 <= last_byte);
4346	memset_extent_buffer(eb, c: 0, start: first_byte + 1, len: last_byte - first_byte - 1);
4347
4348	/* Handle the last byte. */
4349	kaddr = extent_buffer_get_byte(eb, bytenr: last_byte);
4350	*kaddr &= ~BITMAP_LAST_BYTE_MASK(pos + len);
4351	}
4352
4353	static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4354	{
4355	unsigned long distance = (src > dst) ? src - dst : dst - src;
4356	return distance < len;
4357	}
4358
4359	void memcpy_extent_buffer(const struct extent_buffer *dst,
4360	unsigned long dst_offset, unsigned long src_offset,
4361	unsigned long len)
4362	{
4363	const int unit_size = dst->folio_size;
4364	unsigned long cur_off = 0;
4365
4366	if (check_eb_range(eb: dst, start: dst_offset, len) ||
4367	check_eb_range(eb: dst, start: src_offset, len))
4368	return;
4369
4370	if (dst->addr) {
4371	const bool use_memmove = areas_overlap(src: src_offset, dst: dst_offset, len);
4372
4373	if (use_memmove)
4374	memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4375	else
4376	memcpy(dst->addr + dst_offset, dst->addr + src_offset, len);
4377	return;
4378	}
4379
4380	while (cur_off < len) {
4381	unsigned long cur_src = cur_off + src_offset;
4382	unsigned long folio_index = get_eb_folio_index(eb: dst, offset: cur_src);
4383	unsigned long folio_off = get_eb_offset_in_folio(eb: dst, offset: cur_src);
4384	unsigned long cur_len = min(src_offset + len - cur_src,
4385	unit_size - folio_off);
4386	void *src_addr = folio_address(folio: dst->folios[folio_index]) + folio_off;
4387	const bool use_memmove = areas_overlap(src: src_offset + cur_off,
4388	dst: dst_offset + cur_off, len: cur_len);
4389
4390	__write_extent_buffer(eb: dst, srcv: src_addr, start: dst_offset + cur_off, len: cur_len,
4391	use_memmove);
4392	cur_off += cur_len;
4393	}
4394	}
4395
4396	void memmove_extent_buffer(const struct extent_buffer *dst,
4397	unsigned long dst_offset, unsigned long src_offset,
4398	unsigned long len)
4399	{
4400	unsigned long dst_end = dst_offset + len - 1;
4401	unsigned long src_end = src_offset + len - 1;
4402
4403	if (check_eb_range(eb: dst, start: dst_offset, len) ||
4404	check_eb_range(eb: dst, start: src_offset, len))
4405	return;
4406
4407	if (dst_offset < src_offset) {
4408	memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4409	return;
4410	}
4411
4412	if (dst->addr) {
4413	memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4414	return;
4415	}
4416
4417	while (len > 0) {
4418	unsigned long src_i;
4419	size_t cur;
4420	size_t dst_off_in_folio;
4421	size_t src_off_in_folio;
4422	void *src_addr;
4423	bool use_memmove;
4424
4425	src_i = get_eb_folio_index(eb: dst, offset: src_end);
4426
4427	dst_off_in_folio = get_eb_offset_in_folio(eb: dst, offset: dst_end);
4428	src_off_in_folio = get_eb_offset_in_folio(eb: dst, offset: src_end);
4429
4430	cur = min_t(unsigned long, len, src_off_in_folio + 1);
4431	cur = min(cur, dst_off_in_folio + 1);
4432
4433	src_addr = folio_address(folio: dst->folios[src_i]) + src_off_in_folio -
4434	cur + 1;
4435	use_memmove = areas_overlap(src: src_end - cur + 1, dst: dst_end - cur + 1,
4436	len: cur);
4437
4438	__write_extent_buffer(eb: dst, srcv: src_addr, start: dst_end - cur + 1, len: cur,
4439	use_memmove);
4440
4441	dst_end -= cur;
4442	src_end -= cur;
4443	len -= cur;
4444	}
4445	}
4446
4447	static int try_release_subpage_extent_buffer(struct folio *folio)
4448	{
4449	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
4450	struct extent_buffer *eb;
4451	unsigned long start = (folio_pos(folio) >> fs_info->nodesize_bits);
4452	unsigned long index = start;
4453	unsigned long end = index + (PAGE_SIZE >> fs_info->nodesize_bits) - 1;
4454	int ret;
4455
4456	rcu_read_lock();
4457	xa_for_each_range(&fs_info->buffer_tree, index, eb, start, end) {
4458	/*
4459	* The same as try_release_extent_buffer(), to ensure the eb
4460	* won't disappear out from under us.
4461	*/
4462	spin_lock(lock: &eb->refs_lock);
4463	rcu_read_unlock();
4464
4465	if (refcount_read(r: &eb->refs) != 1 || extent_buffer_under_io(eb)) {
4466	spin_unlock(lock: &eb->refs_lock);
4467	rcu_read_lock();
4468	continue;
4469	}
4470
4471	/*
4472	* If tree ref isn't set then we know the ref on this eb is a
4473	* real ref, so just return, this eb will likely be freed soon
4474	* anyway.
4475	*/
4476	if (!test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags)) {
4477	spin_unlock(lock: &eb->refs_lock);
4478	break;
4479	}
4480
4481	/*
4482	* Here we don't care about the return value, we will always
4483	* check the folio private at the end. And
4484	* release_extent_buffer() will release the refs_lock.
4485	*/
4486	release_extent_buffer(eb);
4487	rcu_read_lock();
4488	}
4489	rcu_read_unlock();
4490
4491	/*
4492	* Finally to check if we have cleared folio private, as if we have
4493	* released all ebs in the page, the folio private should be cleared now.
4494	*/
4495	spin_lock(lock: &folio->mapping->i_private_lock);
4496	if (!folio_test_private(folio))
4497	ret = 1;
4498	else
4499	ret = 0;
4500	spin_unlock(lock: &folio->mapping->i_private_lock);
4501	return ret;
4502	}
4503
4504	int try_release_extent_buffer(struct folio *folio)
4505	{
4506	struct extent_buffer *eb;
4507
4508	if (btrfs_meta_is_subpage(folio_to_fs_info(folio)))
4509	return try_release_subpage_extent_buffer(folio);
4510
4511	/*
4512	* We need to make sure nobody is changing folio private, as we rely on
4513	* folio private as the pointer to extent buffer.
4514	*/
4515	spin_lock(lock: &folio->mapping->i_private_lock);
4516	if (!folio_test_private(folio)) {
4517	spin_unlock(lock: &folio->mapping->i_private_lock);
4518	return 1;
4519	}
4520
4521	eb = folio_get_private(folio);
4522	BUG_ON(!eb);
4523
4524	/*
4525	* This is a little awful but should be ok, we need to make sure that
4526	* the eb doesn't disappear out from under us while we're looking at
4527	* this page.
4528	*/
4529	spin_lock(lock: &eb->refs_lock);
4530	if (refcount_read(r: &eb->refs) != 1 || extent_buffer_under_io(eb)) {
4531	spin_unlock(lock: &eb->refs_lock);
4532	spin_unlock(lock: &folio->mapping->i_private_lock);
4533	return 0;
4534	}
4535	spin_unlock(lock: &folio->mapping->i_private_lock);
4536
4537	/*
4538	* If tree ref isn't set then we know the ref on this eb is a real ref,
4539	* so just return, this page will likely be freed soon anyway.
4540	*/
4541	if (!test_and_clear_bit(nr: EXTENT_BUFFER_TREE_REF, addr: &eb->bflags)) {
4542	spin_unlock(lock: &eb->refs_lock);
4543	return 0;
4544	}
4545
4546	return release_extent_buffer(eb);
4547	}
4548
4549	/*
4550	* Attempt to readahead a child block.
4551	*
4552	* @fs_info: the fs_info
4553	* @bytenr: bytenr to read
4554	* @owner_root: objectid of the root that owns this eb
4555	* @gen: generation for the uptodate check, can be 0
4556	* @level: level for the eb
4557	*
4558	* Attempt to readahead a tree block at @bytenr. If @gen is 0 then we do a
4559	* normal uptodate check of the eb, without checking the generation. If we have
4560	* to read the block we will not block on anything.
4561	*/
4562	void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
4563	u64 bytenr, u64 owner_root, u64 gen, int level)
4564	{
4565	struct btrfs_tree_parent_check check = {
4566	.level = level,
4567	.transid = gen
4568	};
4569	struct extent_buffer *eb;
4570	int ret;
4571
4572	eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
4573	if (IS_ERR(ptr: eb))
4574	return;
4575
4576	if (btrfs_buffer_uptodate(buf: eb, parent_transid: gen, atomic: true)) {
4577	free_extent_buffer(eb);
4578	return;
4579	}
4580
4581	ret = read_extent_buffer_pages_nowait(eb, mirror_num: 0, check: &check);
4582	if (ret < 0)
4583	free_extent_buffer_stale(eb);
4584	else
4585	free_extent_buffer(eb);
4586	}
4587
4588	/*
4589	* Readahead a node's child block.
4590	*
4591	* @node: parent node we're reading from
4592	* @slot: slot in the parent node for the child we want to read
4593	*
4594	* A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
4595	* the slot in the node provided.
4596	*/
4597	void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
4598	{
4599	btrfs_readahead_tree_block(fs_info: node->fs_info,
4600	bytenr: btrfs_node_blockptr(eb: node, nr: slot),
4601	owner_root: btrfs_header_owner(eb: node),
4602	gen: btrfs_node_ptr_generation(eb: node, nr: slot),
4603	level: btrfs_header_level(eb: node) - 1);
4604	}
4605