1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (C) 2007 Oracle. All rights reserved.
4	*/
5
6	#include <linux/sched.h>
7	#include <linux/sched/mm.h>
8	#include <linux/slab.h>
9	#include <linux/ratelimit.h>
10	#include <linux/kthread.h>
11	#include <linux/semaphore.h>
12	#include <linux/uuid.h>
13	#include <linux/list_sort.h>
14	#include <linux/namei.h>
15	#include "misc.h"
16	#include "disk-io.h"
17	#include "extent-tree.h"
18	#include "transaction.h"
19	#include "volumes.h"
20	#include "raid56.h"
21	#include "dev-replace.h"
22	#include "sysfs.h"
23	#include "tree-checker.h"
24	#include "space-info.h"
25	#include "block-group.h"
26	#include "discard.h"
27	#include "zoned.h"
28	#include "fs.h"
29	#include "accessors.h"
30	#include "uuid-tree.h"
31	#include "ioctl.h"
32	#include "relocation.h"
33	#include "scrub.h"
34	#include "super.h"
35	#include "raid-stripe-tree.h"
36
37	#define BTRFS_BLOCK_GROUP_STRIPE_MASK (BTRFS_BLOCK_GROUP_RAID0 | \
38	BTRFS_BLOCK_GROUP_RAID10 | \
39	BTRFS_BLOCK_GROUP_RAID56_MASK)
40
41	struct btrfs_io_geometry {
42	u32 stripe_index;
43	u32 stripe_nr;
44	int mirror_num;
45	int num_stripes;
46	u64 stripe_offset;
47	u64 raid56_full_stripe_start;
48	int max_errors;
49	enum btrfs_map_op op;
50	bool use_rst;
51	};
52
53	const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
54	[BTRFS_RAID_RAID10] = {
55	.sub_stripes = 2,
56	.dev_stripes = 1,
57	.devs_max = 0, /* 0 == as many as possible */
58	.devs_min = 2,
59	.tolerated_failures = 1,
60	.devs_increment = 2,
61	.ncopies = 2,
62	.nparity = 0,
63	.raid_name = "raid10",
64	.bg_flag = BTRFS_BLOCK_GROUP_RAID10,
65	.mindev_error = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
66	},
67	[BTRFS_RAID_RAID1] = {
68	.sub_stripes = 1,
69	.dev_stripes = 1,
70	.devs_max = 2,
71	.devs_min = 2,
72	.tolerated_failures = 1,
73	.devs_increment = 2,
74	.ncopies = 2,
75	.nparity = 0,
76	.raid_name = "raid1",
77	.bg_flag = BTRFS_BLOCK_GROUP_RAID1,
78	.mindev_error = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
79	},
80	[BTRFS_RAID_RAID1C3] = {
81	.sub_stripes = 1,
82	.dev_stripes = 1,
83	.devs_max = 3,
84	.devs_min = 3,
85	.tolerated_failures = 2,
86	.devs_increment = 3,
87	.ncopies = 3,
88	.nparity = 0,
89	.raid_name = "raid1c3",
90	.bg_flag = BTRFS_BLOCK_GROUP_RAID1C3,
91	.mindev_error = BTRFS_ERROR_DEV_RAID1C3_MIN_NOT_MET,
92	},
93	[BTRFS_RAID_RAID1C4] = {
94	.sub_stripes = 1,
95	.dev_stripes = 1,
96	.devs_max = 4,
97	.devs_min = 4,
98	.tolerated_failures = 3,
99	.devs_increment = 4,
100	.ncopies = 4,
101	.nparity = 0,
102	.raid_name = "raid1c4",
103	.bg_flag = BTRFS_BLOCK_GROUP_RAID1C4,
104	.mindev_error = BTRFS_ERROR_DEV_RAID1C4_MIN_NOT_MET,
105	},
106	[BTRFS_RAID_DUP] = {
107	.sub_stripes = 1,
108	.dev_stripes = 2,
109	.devs_max = 1,
110	.devs_min = 1,
111	.tolerated_failures = 0,
112	.devs_increment = 1,
113	.ncopies = 2,
114	.nparity = 0,
115	.raid_name = "dup",
116	.bg_flag = BTRFS_BLOCK_GROUP_DUP,
117	.mindev_error = 0,
118	},
119	[BTRFS_RAID_RAID0] = {
120	.sub_stripes = 1,
121	.dev_stripes = 1,
122	.devs_max = 0,
123	.devs_min = 1,
124	.tolerated_failures = 0,
125	.devs_increment = 1,
126	.ncopies = 1,
127	.nparity = 0,
128	.raid_name = "raid0",
129	.bg_flag = BTRFS_BLOCK_GROUP_RAID0,
130	.mindev_error = 0,
131	},
132	[BTRFS_RAID_SINGLE] = {
133	.sub_stripes = 1,
134	.dev_stripes = 1,
135	.devs_max = 1,
136	.devs_min = 1,
137	.tolerated_failures = 0,
138	.devs_increment = 1,
139	.ncopies = 1,
140	.nparity = 0,
141	.raid_name = "single",
142	.bg_flag = 0,
143	.mindev_error = 0,
144	},
145	[BTRFS_RAID_RAID5] = {
146	.sub_stripes = 1,
147	.dev_stripes = 1,
148	.devs_max = 0,
149	.devs_min = 2,
150	.tolerated_failures = 1,
151	.devs_increment = 1,
152	.ncopies = 1,
153	.nparity = 1,
154	.raid_name = "raid5",
155	.bg_flag = BTRFS_BLOCK_GROUP_RAID5,
156	.mindev_error = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
157	},
158	[BTRFS_RAID_RAID6] = {
159	.sub_stripes = 1,
160	.dev_stripes = 1,
161	.devs_max = 0,
162	.devs_min = 3,
163	.tolerated_failures = 2,
164	.devs_increment = 1,
165	.ncopies = 1,
166	.nparity = 2,
167	.raid_name = "raid6",
168	.bg_flag = BTRFS_BLOCK_GROUP_RAID6,
169	.mindev_error = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
170	},
171	};
172
173	/*
174	* Convert block group flags (BTRFS_BLOCK_GROUP_*) to btrfs_raid_types, which
175	* can be used as index to access btrfs_raid_array[].
176	*/
177	enum btrfs_raid_types __attribute_const__ btrfs_bg_flags_to_raid_index(u64 flags)
178	{
179	const u64 profile = (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK);
180
181	if (!profile)
182	return BTRFS_RAID_SINGLE;
183
184	return BTRFS_BG_FLAG_TO_INDEX(profile);
185	}
186
187	const char *btrfs_bg_type_to_raid_name(u64 flags)
188	{
189	const int index = btrfs_bg_flags_to_raid_index(flags);
190
191	if (index >= BTRFS_NR_RAID_TYPES)
192	return NULL;
193
194	return btrfs_raid_array[index].raid_name;
195	}
196
197	int btrfs_nr_parity_stripes(u64 type)
198	{
199	enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(flags: type);
200
201	return btrfs_raid_array[index].nparity;
202	}
203
204	/*
205	* Fill @buf with textual description of @bg_flags, no more than @size_buf
206	* bytes including terminating null byte.
207	*/
208	void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf)
209	{
210	int i;
211	int ret;
212	char *bp = buf;
213	u64 flags = bg_flags;
214	u32 size_bp = size_buf;
215
216	if (!flags)
217	return;
218
219	#define DESCRIBE_FLAG(flag, desc) \
220	do { \
221	if (flags & (flag)) { \
222	ret = snprintf(bp, size_bp, "%s|", (desc)); \
223	if (ret < 0 || ret >= size_bp) \
224	goto out_overflow; \
225	size_bp -= ret; \
226	bp += ret; \
227	flags &= ~(flag); \
228	} \
229	} while (0)
230
231	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data");
232	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system");
233	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata");
234
235	DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single");
236	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
237	DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag,
238	btrfs_raid_array[i].raid_name);
239	#undef DESCRIBE_FLAG
240
241	if (flags) {
242	ret = snprintf(buf: bp, size: size_bp, fmt: "0x%llx|", flags);
243	size_bp -= ret;
244	}
245
246	if (size_bp < size_buf)
247	buf[size_buf - size_bp - 1] = '\0'; /* remove last | */
248
249	/*
250	* The text is trimmed, it's up to the caller to provide sufficiently
251	* large buffer
252	*/
253	out_overflow:;
254	}
255
256	static int init_first_rw_device(struct btrfs_trans_handle *trans);
257	static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
258	static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
259
260	/*
261	* Device locking
262	* ==============
263	*
264	* There are several mutexes that protect manipulation of devices and low-level
265	* structures like chunks but not block groups, extents or files
266	*
267	* uuid_mutex (global lock)
268	* ------------------------
269	* protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
270	* the SCAN_DEV ioctl registration or from mount either implicitly (the first
271	* device) or requested by the device= mount option
272	*
273	* the mutex can be very coarse and can cover long-running operations
274	*
275	* protects: updates to fs_devices counters like missing devices, rw devices,
276	* seeding, structure cloning, opening/closing devices at mount/umount time
277	*
278	* global::fs_devs - add, remove, updates to the global list
279	*
280	* does not protect: manipulation of the fs_devices::devices list in general
281	* but in mount context it could be used to exclude list modifications by eg.
282	* scan ioctl
283	*
284	* btrfs_device::name - renames (write side), read is RCU
285	*
286	* fs_devices::device_list_mutex (per-fs, with RCU)
287	* ------------------------------------------------
288	* protects updates to fs_devices::devices, ie. adding and deleting
289	*
290	* simple list traversal with read-only actions can be done with RCU protection
291	*
292	* may be used to exclude some operations from running concurrently without any
293	* modifications to the list (see write_all_supers)
294	*
295	* Is not required at mount and close times, because our device list is
296	* protected by the uuid_mutex at that point.
297	*
298	* balance_mutex
299	* -------------
300	* protects balance structures (status, state) and context accessed from
301	* several places (internally, ioctl)
302	*
303	* chunk_mutex
304	* -----------
305	* protects chunks, adding or removing during allocation, trim or when a new
306	* device is added/removed. Additionally it also protects post_commit_list of
307	* individual devices, since they can be added to the transaction's
308	* post_commit_list only with chunk_mutex held.
309	*
310	* cleaner_mutex
311	* -------------
312	* a big lock that is held by the cleaner thread and prevents running subvolume
313	* cleaning together with relocation or delayed iputs
314	*
315	*
316	* Lock nesting
317	* ============
318	*
319	* uuid_mutex
320	* device_list_mutex
321	* chunk_mutex
322	* balance_mutex
323	*
324	*
325	* Exclusive operations
326	* ====================
327	*
328	* Maintains the exclusivity of the following operations that apply to the
329	* whole filesystem and cannot run in parallel.
330	*
331	* - Balance (*)
332	* - Device add
333	* - Device remove
334	* - Device replace (*)
335	* - Resize
336	*
337	* The device operations (as above) can be in one of the following states:
338	*
339	* - Running state
340	* - Paused state
341	* - Completed state
342	*
343	* Only device operations marked with (*) can go into the Paused state for the
344	* following reasons:
345	*
346	* - ioctl (only Balance can be Paused through ioctl)
347	* - filesystem remounted as read-only
348	* - filesystem unmounted and mounted as read-only
349	* - system power-cycle and filesystem mounted as read-only
350	* - filesystem or device errors leading to forced read-only
351	*
352	* The status of exclusive operation is set and cleared atomically.
353	* During the course of Paused state, fs_info::exclusive_operation remains set.
354	* A device operation in Paused or Running state can be canceled or resumed
355	* either by ioctl (Balance only) or when remounted as read-write.
356	* The exclusive status is cleared when the device operation is canceled or
357	* completed.
358	*/
359
360	DEFINE_MUTEX(uuid_mutex);
361	static LIST_HEAD(fs_uuids);
362	struct list_head * __attribute_const__ btrfs_get_fs_uuids(void)
363	{
364	return &fs_uuids;
365	}
366
367	/*
368	* Allocate new btrfs_fs_devices structure identified by a fsid.
369	*
370	* @fsid: if not NULL, copy the UUID to fs_devices::fsid and to
371	* fs_devices::metadata_fsid
372	*
373	* Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR().
374	* The returned struct is not linked onto any lists and can be destroyed with
375	* kfree() right away.
376	*/
377	static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid)
378	{
379	struct btrfs_fs_devices *fs_devs;
380
381	fs_devs = kzalloc(sizeof(*fs_devs), GFP_KERNEL);
382	if (!fs_devs)
383	return ERR_PTR(error: -ENOMEM);
384
385	mutex_init(&fs_devs->device_list_mutex);
386
387	INIT_LIST_HEAD(list: &fs_devs->devices);
388	INIT_LIST_HEAD(list: &fs_devs->alloc_list);
389	INIT_LIST_HEAD(list: &fs_devs->fs_list);
390	INIT_LIST_HEAD(list: &fs_devs->seed_list);
391
392	if (fsid) {
393	memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
394	memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE);
395	}
396
397	return fs_devs;
398	}
399
400	static void btrfs_free_device(struct btrfs_device *device)
401	{
402	WARN_ON(!list_empty(&device->post_commit_list));
403	/*
404	* No need to call kfree_rcu() nor do RCU lock/unlock, nothing is
405	* reading the device name.
406	*/
407	kfree(rcu_dereference_raw(device->name));
408	btrfs_extent_io_tree_release(tree: &device->alloc_state);
409	btrfs_destroy_dev_zone_info(device);
410	kfree(objp: device);
411	}
412
413	static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
414	{
415	struct btrfs_device *device;
416
417	WARN_ON(fs_devices->opened);
418	WARN_ON(fs_devices->holding);
419	while (!list_empty(head: &fs_devices->devices)) {
420	device = list_first_entry(&fs_devices->devices,
421	struct btrfs_device, dev_list);
422	list_del(entry: &device->dev_list);
423	btrfs_free_device(device);
424	}
425	kfree(objp: fs_devices);
426	}
427
428	void __exit btrfs_cleanup_fs_uuids(void)
429	{
430	struct btrfs_fs_devices *fs_devices;
431
432	while (!list_empty(head: &fs_uuids)) {
433	fs_devices = list_first_entry(&fs_uuids, struct btrfs_fs_devices,
434	fs_list);
435	list_del(entry: &fs_devices->fs_list);
436	free_fs_devices(fs_devices);
437	}
438	}
439
440	static bool match_fsid_fs_devices(const struct btrfs_fs_devices *fs_devices,
441	const u8 *fsid, const u8 *metadata_fsid)
442	{
443	if (memcmp(p: fsid, q: fs_devices->fsid, BTRFS_FSID_SIZE) != 0)
444	return false;
445
446	if (!metadata_fsid)
447	return true;
448
449	if (memcmp(p: metadata_fsid, q: fs_devices->metadata_uuid, BTRFS_FSID_SIZE) != 0)
450	return false;
451
452	return true;
453	}
454
455	static noinline struct btrfs_fs_devices *find_fsid(
456	const u8 *fsid, const u8 *metadata_fsid)
457	{
458	struct btrfs_fs_devices *fs_devices;
459
460	ASSERT(fsid);
461
462	/* Handle non-split brain cases */
463	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
464	if (match_fsid_fs_devices(fs_devices, fsid, metadata_fsid))
465	return fs_devices;
466	}
467	return NULL;
468	}
469
470	static int
471	btrfs_get_bdev_and_sb(const char *device_path, blk_mode_t flags, void *holder,
472	int flush, struct file **bdev_file,
473	struct btrfs_super_block **disk_super)
474	{
475	struct block_device *bdev;
476	int ret;
477
478	*bdev_file = bdev_file_open_by_path(path: device_path, mode: flags, holder, hops: &fs_holder_ops);
479
480	if (IS_ERR(ptr: *bdev_file)) {
481	ret = PTR_ERR(ptr: *bdev_file);
482	btrfs_err(NULL, "failed to open device for path %s with flags 0x%x: %d",
483	device_path, flags, ret);
484	goto error;
485	}
486	bdev = file_bdev(bdev_file: *bdev_file);
487
488	if (flush)
489	sync_blockdev(bdev);
490	if (holder) {
491	ret = set_blocksize(file: *bdev_file, BTRFS_BDEV_BLOCKSIZE);
492	if (ret) {
493	bdev_fput(bdev_file: *bdev_file);
494	goto error;
495	}
496	}
497	invalidate_bdev(bdev);
498	*disk_super = btrfs_read_disk_super(bdev, copy_num: 0, drop_cache: false);
499	if (IS_ERR(ptr: *disk_super)) {
500	ret = PTR_ERR(ptr: *disk_super);
501	bdev_fput(bdev_file: *bdev_file);
502	goto error;
503	}
504
505	return 0;
506
507	error:
508	*disk_super = NULL;
509	*bdev_file = NULL;
510	return ret;
511	}
512
513	/*
514	* Search and remove all stale devices (which are not mounted). When both
515	* inputs are NULL, it will search and release all stale devices.
516	*
517	* @devt: Optional. When provided will it release all unmounted devices
518	* matching this devt only.
519	* @skip_device: Optional. Will skip this device when searching for the stale
520	* devices.
521	*
522	* Return: 0 for success or if @devt is 0.
523	* -EBUSY if @devt is a mounted device.
524	* -ENOENT if @devt does not match any device in the list.
525	*/
526	static int btrfs_free_stale_devices(dev_t devt, struct btrfs_device *skip_device)
527	{
528	struct btrfs_fs_devices *fs_devices, *tmp_fs_devices;
529	struct btrfs_device *device, *tmp_device;
530	int ret;
531	bool freed = false;
532
533	lockdep_assert_held(&uuid_mutex);
534
535	/* Return good status if there is no instance of devt. */
536	ret = 0;
537	list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) {
538
539	mutex_lock(&fs_devices->device_list_mutex);
540	list_for_each_entry_safe(device, tmp_device,
541	&fs_devices->devices, dev_list) {
542	if (skip_device && skip_device == device)
543	continue;
544	if (devt && devt != device->devt)
545	continue;
546	if (fs_devices->opened || fs_devices->holding) {
547	if (devt)
548	ret = -EBUSY;
549	break;
550	}
551
552	/* delete the stale device */
553	fs_devices->num_devices--;
554	list_del(entry: &device->dev_list);
555	btrfs_free_device(device);
556
557	freed = true;
558	}
559	mutex_unlock(lock: &fs_devices->device_list_mutex);
560
561	if (fs_devices->num_devices == 0) {
562	btrfs_sysfs_remove_fsid(fs_devs: fs_devices);
563	list_del(entry: &fs_devices->fs_list);
564	free_fs_devices(fs_devices);
565	}
566	}
567
568	/* If there is at least one freed device return 0. */
569	if (freed)
570	return 0;
571
572	return ret;
573	}
574
575	static struct btrfs_fs_devices *find_fsid_by_device(
576	struct btrfs_super_block *disk_super,
577	dev_t devt, bool *same_fsid_diff_dev)
578	{
579	struct btrfs_fs_devices *fsid_fs_devices;
580	struct btrfs_fs_devices *devt_fs_devices;
581	const bool has_metadata_uuid = (btrfs_super_incompat_flags(s: disk_super) &
582	BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
583	bool found_by_devt = false;
584
585	/* Find the fs_device by the usual method, if found use it. */
586	fsid_fs_devices = find_fsid(fsid: disk_super->fsid,
587	metadata_fsid: has_metadata_uuid ? disk_super->metadata_uuid : NULL);
588
589	/* The temp_fsid feature is supported only with single device filesystem. */
590	if (btrfs_super_num_devices(s: disk_super) != 1)
591	return fsid_fs_devices;
592
593	/*
594	* A seed device is an integral component of the sprout device, which
595	* functions as a multi-device filesystem. So, temp-fsid feature is
596	* not supported.
597	*/
598	if (btrfs_super_flags(s: disk_super) & BTRFS_SUPER_FLAG_SEEDING)
599	return fsid_fs_devices;
600
601	/* Try to find a fs_devices by matching devt. */
602	list_for_each_entry(devt_fs_devices, &fs_uuids, fs_list) {
603	struct btrfs_device *device;
604
605	list_for_each_entry(device, &devt_fs_devices->devices, dev_list) {
606	if (device->devt == devt) {
607	found_by_devt = true;
608	break;
609	}
610	}
611	if (found_by_devt)
612	break;
613	}
614
615	if (found_by_devt) {
616	/* Existing device. */
617	if (fsid_fs_devices == NULL) {
618	if (devt_fs_devices->opened == 0) {
619	/* Stale device. */
620	return NULL;
621	} else {
622	/* temp_fsid is mounting a subvol. */
623	return devt_fs_devices;
624	}
625	} else {
626	/* Regular or temp_fsid device mounting a subvol. */
627	return devt_fs_devices;
628	}
629	} else {
630	/* New device. */
631	if (fsid_fs_devices == NULL) {
632	return NULL;
633	} else {
634	/* sb::fsid is already used create a new temp_fsid. */
635	*same_fsid_diff_dev = true;
636	return NULL;
637	}
638	}
639
640	/* Not reached. */
641	}
642
643	/*
644	* This is only used on mount, and we are protected from competing things
645	* messing with our fs_devices by the uuid_mutex, thus we do not need the
646	* fs_devices->device_list_mutex here.
647	*/
648	static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
649	struct btrfs_device *device, blk_mode_t flags,
650	void *holder)
651	{
652	struct file *bdev_file;
653	struct btrfs_super_block *disk_super;
654	u64 devid;
655	int ret;
656
657	if (device->bdev)
658	return -EINVAL;
659	if (!device->name)
660	return -EINVAL;
661
662	ret = btrfs_get_bdev_and_sb(rcu_dereference_raw(device->name), flags, holder, flush: 1,
663	bdev_file: &bdev_file, disk_super: &disk_super);
664	if (ret)
665	return ret;
666
667	devid = btrfs_stack_device_id(s: &disk_super->dev_item);
668	if (devid != device->devid)
669	goto error_free_page;
670
671	if (memcmp(p: device->uuid, q: disk_super->dev_item.uuid, BTRFS_UUID_SIZE))
672	goto error_free_page;
673
674	device->generation = btrfs_super_generation(s: disk_super);
675
676	if (btrfs_super_flags(s: disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
677	if (btrfs_super_incompat_flags(s: disk_super) &
678	BTRFS_FEATURE_INCOMPAT_METADATA_UUID) {
679	btrfs_err(NULL,
680	"invalid seeding and uuid-changed device detected");
681	goto error_free_page;
682	}
683
684	clear_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
685	fs_devices->seeding = true;
686	} else {
687	if (bdev_read_only(bdev: file_bdev(bdev_file)))
688	clear_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
689	else
690	set_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
691	}
692
693	if (!bdev_nonrot(bdev: file_bdev(bdev_file)))
694	fs_devices->rotating = true;
695
696	if (bdev_max_discard_sectors(bdev: file_bdev(bdev_file)))
697	fs_devices->discardable = true;
698
699	device->bdev_file = bdev_file;
700	device->bdev = file_bdev(bdev_file);
701	clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, addr: &device->dev_state);
702
703	if (device->devt != device->bdev->bd_dev) {
704	btrfs_warn(NULL,
705	"device %s maj:min changed from %d:%d to %d:%d",
706	rcu_dereference_raw(device->name), MAJOR(device->devt),
707	MINOR(device->devt), MAJOR(device->bdev->bd_dev),
708	MINOR(device->bdev->bd_dev));
709
710	device->devt = device->bdev->bd_dev;
711	}
712
713	fs_devices->open_devices++;
714	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
715	device->devid != BTRFS_DEV_REPLACE_DEVID) {
716	fs_devices->rw_devices++;
717	list_add_tail(new: &device->dev_alloc_list, head: &fs_devices->alloc_list);
718	}
719	btrfs_release_disk_super(super: disk_super);
720
721	return 0;
722
723	error_free_page:
724	btrfs_release_disk_super(super: disk_super);
725	bdev_fput(bdev_file);
726
727	return -EINVAL;
728	}
729
730	const u8 *btrfs_sb_fsid_ptr(const struct btrfs_super_block *sb)
731	{
732	bool has_metadata_uuid = (btrfs_super_incompat_flags(s: sb) &
733	BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
734
735	return has_metadata_uuid ? sb->metadata_uuid : sb->fsid;
736	}
737
738	static bool is_same_device(struct btrfs_device *device, const char *new_path)
739	{
740	struct path old = { .mnt = NULL, .dentry = NULL };
741	struct path new = { .mnt = NULL, .dentry = NULL };
742	char AUTO_KFREE(old_path);
743	bool is_same = false;
744	int ret;
745
746	if (!device->name)
747	goto out;
748
749	old_path = kzalloc(PATH_MAX, GFP_NOFS);
750	if (!old_path)
751	goto out;
752
753	rcu_read_lock();
754	ret = strscpy(old_path, rcu_dereference(device->name), PATH_MAX);
755	rcu_read_unlock();
756	if (ret < 0)
757	goto out;
758
759	ret = kern_path(old_path, LOOKUP_FOLLOW, &old);
760	if (ret)
761	goto out;
762	ret = kern_path(new_path, LOOKUP_FOLLOW, &new);
763	if (ret)
764	goto out;
765	if (path_equal(path1: &old, path2: &new))
766	is_same = true;
767	out:
768	path_put(&old);
769	path_put(&new);
770	return is_same;
771	}
772
773	/*
774	* Add new device to list of registered devices
775	*
776	* Returns:
777	* device pointer which was just added or updated when successful
778	* error pointer when failed
779	*/
780	static noinline struct btrfs_device *device_list_add(const char *path,
781	struct btrfs_super_block *disk_super,
782	bool *new_device_added)
783	{
784	struct btrfs_device *device;
785	struct btrfs_fs_devices *fs_devices = NULL;
786	const char *name;
787	u64 found_transid = btrfs_super_generation(s: disk_super);
788	u64 devid = btrfs_stack_device_id(s: &disk_super->dev_item);
789	dev_t path_devt;
790	int ret;
791	bool same_fsid_diff_dev = false;
792	bool has_metadata_uuid = (btrfs_super_incompat_flags(s: disk_super) &
793	BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
794
795	if (btrfs_super_flags(s: disk_super) & BTRFS_SUPER_FLAG_CHANGING_FSID_V2) {
796	btrfs_err(NULL,
797	"device %s has incomplete metadata_uuid change, please use btrfstune to complete",
798	path);
799	return ERR_PTR(error: -EAGAIN);
800	}
801
802	ret = lookup_bdev(pathname: path, dev: &path_devt);
803	if (ret) {
804	btrfs_err(NULL, "failed to lookup block device for path %s: %d",
805	path, ret);
806	return ERR_PTR(error: ret);
807	}
808
809	fs_devices = find_fsid_by_device(disk_super, devt: path_devt, same_fsid_diff_dev: &same_fsid_diff_dev);
810
811	if (!fs_devices) {
812	fs_devices = alloc_fs_devices(fsid: disk_super->fsid);
813	if (IS_ERR(ptr: fs_devices))
814	return ERR_CAST(ptr: fs_devices);
815
816	if (has_metadata_uuid)
817	memcpy(fs_devices->metadata_uuid,
818	disk_super->metadata_uuid, BTRFS_FSID_SIZE);
819
820	if (same_fsid_diff_dev) {
821	generate_random_uuid(uuid: fs_devices->fsid);
822	fs_devices->temp_fsid = true;
823	btrfs_info(NULL, "device %s (%d:%d) using temp-fsid %pU",
824	path, MAJOR(path_devt), MINOR(path_devt),
825	fs_devices->fsid);
826	}
827
828	mutex_lock(&fs_devices->device_list_mutex);
829	list_add(new: &fs_devices->fs_list, head: &fs_uuids);
830
831	device = NULL;
832	} else {
833	struct btrfs_dev_lookup_args args = {
834	.devid = devid,
835	.uuid = disk_super->dev_item.uuid,
836	};
837
838	mutex_lock(&fs_devices->device_list_mutex);
839	device = btrfs_find_device(fs_devices, args: &args);
840
841	if (found_transid > fs_devices->latest_generation) {
842	memcpy(fs_devices->fsid, disk_super->fsid,
843	BTRFS_FSID_SIZE);
844	memcpy(fs_devices->metadata_uuid,
845	btrfs_sb_fsid_ptr(disk_super), BTRFS_FSID_SIZE);
846	}
847	}
848
849	if (!device) {
850	unsigned int nofs_flag;
851
852	if (fs_devices->opened) {
853	btrfs_err(NULL,
854	"device %s (%d:%d) belongs to fsid %pU, and the fs is already mounted, scanned by %s (%d)",
855	path, MAJOR(path_devt), MINOR(path_devt),
856	fs_devices->fsid, current->comm,
857	task_pid_nr(current));
858	mutex_unlock(lock: &fs_devices->device_list_mutex);
859	return ERR_PTR(error: -EBUSY);
860	}
861
862	nofs_flag = memalloc_nofs_save();
863	device = btrfs_alloc_device(NULL, devid: &devid,
864	uuid: disk_super->dev_item.uuid, path);
865	memalloc_nofs_restore(flags: nofs_flag);
866	if (IS_ERR(ptr: device)) {
867	mutex_unlock(lock: &fs_devices->device_list_mutex);
868	/* we can safely leave the fs_devices entry around */
869	return device;
870	}
871
872	device->devt = path_devt;
873
874	list_add_rcu(new: &device->dev_list, head: &fs_devices->devices);
875	fs_devices->num_devices++;
876
877	device->fs_devices = fs_devices;
878	*new_device_added = true;
879
880	if (disk_super->label[0])
881	pr_info(
882	"BTRFS: device label %s devid %llu transid %llu %s (%d:%d) scanned by %s (%d)\n",
883	disk_super->label, devid, found_transid, path,
884	MAJOR(path_devt), MINOR(path_devt),
885	current->comm, task_pid_nr(current));
886	else
887	pr_info(
888	"BTRFS: device fsid %pU devid %llu transid %llu %s (%d:%d) scanned by %s (%d)\n",
889	disk_super->fsid, devid, found_transid, path,
890	MAJOR(path_devt), MINOR(path_devt),
891	current->comm, task_pid_nr(current));
892
893	} else if (!device->name || !is_same_device(device, new_path: path)) {
894	const char *old_name;
895
896	/*
897	* When FS is already mounted.
898	* 1. If you are here and if the device->name is NULL that
899	* means this device was missing at time of FS mount.
900	* 2. If you are here and if the device->name is different
901	* from 'path' that means either
902	* a. The same device disappeared and reappeared with
903	* different name. or
904	* b. The missing-disk-which-was-replaced, has
905	* reappeared now.
906	*
907	* We must allow 1 and 2a above. But 2b would be a spurious
908	* and unintentional.
909	*
910	* Further in case of 1 and 2a above, the disk at 'path'
911	* would have missed some transaction when it was away and
912	* in case of 2a the stale bdev has to be updated as well.
913	* 2b must not be allowed at all time.
914	*/
915
916	/*
917	* For now, we do allow update to btrfs_fs_device through the
918	* btrfs dev scan cli after FS has been mounted. We're still
919	* tracking a problem where systems fail mount by subvolume id
920	* when we reject replacement on a mounted FS.
921	*/
922	if (!fs_devices->opened && found_transid < device->generation) {
923	/*
924	* That is if the FS is _not_ mounted and if you
925	* are here, that means there is more than one
926	* disk with same uuid and devid.We keep the one
927	* with larger generation number or the last-in if
928	* generation are equal.
929	*/
930	mutex_unlock(lock: &fs_devices->device_list_mutex);
931	btrfs_err(NULL,
932	"device %s already registered with a higher generation, found %llu expect %llu",
933	path, found_transid, device->generation);
934	return ERR_PTR(error: -EEXIST);
935	}
936
937	/*
938	* We are going to replace the device path for a given devid,
939	* make sure it's the same device if the device is mounted
940	*
941	* NOTE: the device->fs_info may not be reliable here so pass
942	* in a NULL to message helpers instead. This avoids a possible
943	* use-after-free when the fs_info and fs_info->sb are already
944	* torn down.
945	*/
946	if (device->bdev) {
947	if (device->devt != path_devt) {
948	mutex_unlock(lock: &fs_devices->device_list_mutex);
949	btrfs_warn(NULL,
950	"duplicate device %s devid %llu generation %llu scanned by %s (%d)",
951	path, devid, found_transid,
952	current->comm,
953	task_pid_nr(current));
954	return ERR_PTR(error: -EEXIST);
955	}
956	btrfs_info(NULL,
957	"devid %llu device path %s changed to %s scanned by %s (%d)",
958	devid, btrfs_dev_name(device),
959	path, current->comm,
960	task_pid_nr(current));
961	}
962
963	name = kstrdup(s: path, GFP_NOFS);
964	if (!name) {
965	mutex_unlock(lock: &fs_devices->device_list_mutex);
966	return ERR_PTR(error: -ENOMEM);
967	}
968	rcu_read_lock();
969	old_name = rcu_dereference(device->name);
970	rcu_read_unlock();
971	rcu_assign_pointer(device->name, name);
972	kfree_rcu_mightsleep(old_name);
973
974	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
975	fs_devices->missing_devices--;
976	clear_bit(BTRFS_DEV_STATE_MISSING, addr: &device->dev_state);
977	}
978	device->devt = path_devt;
979	}
980
981	/*
982	* Unmount does not free the btrfs_device struct but would zero
983	* generation along with most of the other members. So just update
984	* it back. We need it to pick the disk with largest generation
985	* (as above).
986	*/
987	if (!fs_devices->opened) {
988	device->generation = found_transid;
989	fs_devices->latest_generation = max_t(u64, found_transid,
990	fs_devices->latest_generation);
991	}
992
993	fs_devices->total_devices = btrfs_super_num_devices(s: disk_super);
994
995	mutex_unlock(lock: &fs_devices->device_list_mutex);
996	return device;
997	}
998
999	static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
1000	{
1001	struct btrfs_fs_devices *fs_devices;
1002	struct btrfs_device *device;
1003	struct btrfs_device *orig_dev;
1004	int ret = 0;
1005
1006	lockdep_assert_held(&uuid_mutex);
1007
1008	fs_devices = alloc_fs_devices(fsid: orig->fsid);
1009	if (IS_ERR(ptr: fs_devices))
1010	return fs_devices;
1011
1012	fs_devices->total_devices = orig->total_devices;
1013
1014	list_for_each_entry(orig_dev, &orig->devices, dev_list) {
1015	const char *dev_path = NULL;
1016
1017	/*
1018	* This is ok to do without RCU read locked because we hold the
1019	* uuid mutex so nothing we touch in here is going to disappear.
1020	*/
1021	if (orig_dev->name)
1022	dev_path = rcu_dereference_raw(orig_dev->name);
1023
1024	device = btrfs_alloc_device(NULL, devid: &orig_dev->devid,
1025	uuid: orig_dev->uuid, path: dev_path);
1026	if (IS_ERR(ptr: device)) {
1027	ret = PTR_ERR(ptr: device);
1028	goto error;
1029	}
1030
1031	if (orig_dev->zone_info) {
1032	struct btrfs_zoned_device_info *zone_info;
1033
1034	zone_info = btrfs_clone_dev_zone_info(orig_dev);
1035	if (!zone_info) {
1036	btrfs_free_device(device);
1037	ret = -ENOMEM;
1038	goto error;
1039	}
1040	device->zone_info = zone_info;
1041	}
1042
1043	list_add(new: &device->dev_list, head: &fs_devices->devices);
1044	device->fs_devices = fs_devices;
1045	fs_devices->num_devices++;
1046	}
1047	return fs_devices;
1048	error:
1049	free_fs_devices(fs_devices);
1050	return ERR_PTR(error: ret);
1051	}
1052
1053	static void __btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices,
1054	struct btrfs_device **latest_dev)
1055	{
1056	struct btrfs_device *device, *next;
1057
1058	/* This is the initialized path, it is safe to release the devices. */
1059	list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
1060	if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) {
1061	if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1062	&device->dev_state) &&
1063	!test_bit(BTRFS_DEV_STATE_MISSING,
1064	&device->dev_state) &&
1065	(!*latest_dev ||
1066	device->generation > (*latest_dev)->generation)) {
1067	*latest_dev = device;
1068	}
1069	continue;
1070	}
1071
1072	/*
1073	* We have already validated the presence of BTRFS_DEV_REPLACE_DEVID,
1074	* in btrfs_init_dev_replace() so just continue.
1075	*/
1076	if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1077	continue;
1078
1079	if (device->bdev_file) {
1080	bdev_fput(bdev_file: device->bdev_file);
1081	device->bdev = NULL;
1082	device->bdev_file = NULL;
1083	fs_devices->open_devices--;
1084	}
1085	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1086	list_del_init(entry: &device->dev_alloc_list);
1087	clear_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
1088	fs_devices->rw_devices--;
1089	}
1090	list_del_init(entry: &device->dev_list);
1091	fs_devices->num_devices--;
1092	btrfs_free_device(device);
1093	}
1094
1095	}
1096
1097	/*
1098	* After we have read the system tree and know devids belonging to this
1099	* filesystem, remove the device which does not belong there.
1100	*/
1101	void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices)
1102	{
1103	struct btrfs_device *latest_dev = NULL;
1104	struct btrfs_fs_devices *seed_dev;
1105
1106	mutex_lock(&uuid_mutex);
1107	__btrfs_free_extra_devids(fs_devices, latest_dev: &latest_dev);
1108
1109	list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list)
1110	__btrfs_free_extra_devids(fs_devices: seed_dev, latest_dev: &latest_dev);
1111
1112	fs_devices->latest_dev = latest_dev;
1113
1114	mutex_unlock(lock: &uuid_mutex);
1115	}
1116
1117	static void btrfs_close_bdev(struct btrfs_device *device)
1118	{
1119	if (!device->bdev)
1120	return;
1121
1122	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1123	sync_blockdev(bdev: device->bdev);
1124	invalidate_bdev(bdev: device->bdev);
1125	}
1126
1127	bdev_fput(bdev_file: device->bdev_file);
1128	}
1129
1130	static void btrfs_close_one_device(struct btrfs_device *device)
1131	{
1132	struct btrfs_fs_devices *fs_devices = device->fs_devices;
1133
1134	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
1135	device->devid != BTRFS_DEV_REPLACE_DEVID) {
1136	list_del_init(entry: &device->dev_alloc_list);
1137	fs_devices->rw_devices--;
1138	}
1139
1140	if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1141	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, addr: &device->dev_state);
1142
1143	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
1144	clear_bit(BTRFS_DEV_STATE_MISSING, addr: &device->dev_state);
1145	fs_devices->missing_devices--;
1146	}
1147
1148	btrfs_close_bdev(device);
1149	if (device->bdev) {
1150	fs_devices->open_devices--;
1151	device->bdev = NULL;
1152	device->bdev_file = NULL;
1153	}
1154	clear_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
1155	btrfs_destroy_dev_zone_info(device);
1156
1157	device->fs_info = NULL;
1158	atomic_set(v: &device->dev_stats_ccnt, i: 0);
1159	btrfs_extent_io_tree_release(tree: &device->alloc_state);
1160
1161	/*
1162	* Reset the flush error record. We might have a transient flush error
1163	* in this mount, and if so we aborted the current transaction and set
1164	* the fs to an error state, guaranteeing no super blocks can be further
1165	* committed. However that error might be transient and if we unmount the
1166	* filesystem and mount it again, we should allow the mount to succeed
1167	* (btrfs_check_rw_degradable() should not fail) - if after mounting the
1168	* filesystem again we still get flush errors, then we will again abort
1169	* any transaction and set the error state, guaranteeing no commits of
1170	* unsafe super blocks.
1171	*/
1172	device->last_flush_error = 0;
1173
1174	/* Verify the device is back in a pristine state */
1175	WARN_ON(test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state));
1176	WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1177	WARN_ON(!list_empty(&device->dev_alloc_list));
1178	WARN_ON(!list_empty(&device->post_commit_list));
1179	}
1180
1181	static void close_fs_devices(struct btrfs_fs_devices *fs_devices)
1182	{
1183	struct btrfs_device *device, *tmp;
1184
1185	lockdep_assert_held(&uuid_mutex);
1186
1187	if (--fs_devices->opened > 0)
1188	return;
1189
1190	list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list)
1191	btrfs_close_one_device(device);
1192
1193	WARN_ON(fs_devices->open_devices);
1194	WARN_ON(fs_devices->rw_devices);
1195	fs_devices->opened = 0;
1196	fs_devices->seeding = false;
1197	fs_devices->fs_info = NULL;
1198	}
1199
1200	void btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
1201	{
1202	LIST_HEAD(list);
1203	struct btrfs_fs_devices *tmp;
1204
1205	mutex_lock(&uuid_mutex);
1206	close_fs_devices(fs_devices);
1207	if (!fs_devices->opened && !fs_devices->holding) {
1208	list_splice_init(list: &fs_devices->seed_list, head: &list);
1209
1210	/*
1211	* If the struct btrfs_fs_devices is not assembled with any
1212	* other device, it can be re-initialized during the next mount
1213	* without the needing device-scan step. Therefore, it can be
1214	* fully freed.
1215	*/
1216	if (fs_devices->num_devices == 1) {
1217	list_del(entry: &fs_devices->fs_list);
1218	free_fs_devices(fs_devices);
1219	}
1220	}
1221
1222
1223	list_for_each_entry_safe(fs_devices, tmp, &list, seed_list) {
1224	close_fs_devices(fs_devices);
1225	list_del(entry: &fs_devices->seed_list);
1226	free_fs_devices(fs_devices);
1227	}
1228	mutex_unlock(lock: &uuid_mutex);
1229	}
1230
1231	static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
1232	blk_mode_t flags, void *holder)
1233	{
1234	struct btrfs_device *device;
1235	struct btrfs_device *latest_dev = NULL;
1236	struct btrfs_device *tmp_device;
1237	s64 __maybe_unused value = 0;
1238	int ret = 0;
1239
1240	list_for_each_entry_safe(device, tmp_device, &fs_devices->devices,
1241	dev_list) {
1242	int ret2;
1243
1244	ret2 = btrfs_open_one_device(fs_devices, device, flags, holder);
1245	if (ret2 == 0 &&
1246	(!latest_dev || device->generation > latest_dev->generation)) {
1247	latest_dev = device;
1248	} else if (ret2 == -ENODATA) {
1249	fs_devices->num_devices--;
1250	list_del(entry: &device->dev_list);
1251	btrfs_free_device(device);
1252	}
1253	if (ret == 0 && ret2 != 0)
1254	ret = ret2;
1255	}
1256
1257	if (fs_devices->open_devices == 0) {
1258	if (ret)
1259	return ret;
1260	return -EINVAL;
1261	}
1262
1263	fs_devices->opened = 1;
1264	fs_devices->latest_dev = latest_dev;
1265	fs_devices->total_rw_bytes = 0;
1266	fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR;
1267	#ifdef CONFIG_BTRFS_EXPERIMENTAL
1268	fs_devices->rr_min_contig_read = BTRFS_DEFAULT_RR_MIN_CONTIG_READ;
1269	fs_devices->read_devid = latest_dev->devid;
1270	fs_devices->read_policy = btrfs_read_policy_to_enum(str: btrfs_get_mod_read_policy(),
1271	value: &value);
1272	if (fs_devices->read_policy == BTRFS_READ_POLICY_RR)
1273	fs_devices->collect_fs_stats = true;
1274
1275	if (value) {
1276	if (fs_devices->read_policy == BTRFS_READ_POLICY_RR)
1277	fs_devices->rr_min_contig_read = value;
1278	if (fs_devices->read_policy == BTRFS_READ_POLICY_DEVID)
1279	fs_devices->read_devid = value;
1280	}
1281	#else
1282	fs_devices->read_policy = BTRFS_READ_POLICY_PID;
1283	#endif
1284
1285	return 0;
1286	}
1287
1288	static int devid_cmp(void *priv, const struct list_head *a,
1289	const struct list_head *b)
1290	{
1291	const struct btrfs_device *dev1, *dev2;
1292
1293	dev1 = list_entry(a, struct btrfs_device, dev_list);
1294	dev2 = list_entry(b, struct btrfs_device, dev_list);
1295
1296	if (dev1->devid < dev2->devid)
1297	return -1;
1298	else if (dev1->devid > dev2->devid)
1299	return 1;
1300	return 0;
1301	}
1302
1303	int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1304	blk_mode_t flags, void *holder)
1305	{
1306	int ret;
1307
1308	lockdep_assert_held(&uuid_mutex);
1309	/*
1310	* The device_list_mutex cannot be taken here in case opening the
1311	* underlying device takes further locks like open_mutex.
1312	*
1313	* We also don't need the lock here as this is called during mount and
1314	* exclusion is provided by uuid_mutex
1315	*/
1316
1317	if (fs_devices->opened) {
1318	fs_devices->opened++;
1319	ret = 0;
1320	} else {
1321	list_sort(NULL, head: &fs_devices->devices, cmp: devid_cmp);
1322	ret = open_fs_devices(fs_devices, flags, holder);
1323	}
1324
1325	return ret;
1326	}
1327
1328	void btrfs_release_disk_super(struct btrfs_super_block *super)
1329	{
1330	struct page *page = virt_to_page(super);
1331
1332	put_page(page);
1333	}
1334
1335	struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
1336	int copy_num, bool drop_cache)
1337	{
1338	struct btrfs_super_block *super;
1339	struct page *page;
1340	u64 bytenr, bytenr_orig;
1341	struct address_space *mapping = bdev->bd_mapping;
1342	int ret;
1343
1344	bytenr_orig = btrfs_sb_offset(mirror: copy_num);
1345	ret = btrfs_sb_log_location_bdev(bdev, mirror: copy_num, READ, bytenr_ret: &bytenr);
1346	if (ret < 0) {
1347	if (ret == -ENOENT)
1348	ret = -EINVAL;
1349	return ERR_PTR(error: ret);
1350	}
1351
1352	if (bytenr + BTRFS_SUPER_INFO_SIZE >= bdev_nr_bytes(bdev))
1353	return ERR_PTR(error: -EINVAL);
1354
1355	if (drop_cache) {
1356	/* This should only be called with the primary sb. */
1357	ASSERT(copy_num == 0);
1358
1359	/*
1360	* Drop the page of the primary superblock, so later read will
1361	* always read from the device.
1362	*/
1363	invalidate_inode_pages2_range(mapping, start: bytenr >> PAGE_SHIFT,
1364	end: (bytenr + BTRFS_SUPER_INFO_SIZE) >> PAGE_SHIFT);
1365	}
1366
1367	filemap_invalidate_lock(mapping);
1368	page = read_cache_page_gfp(mapping, index: bytenr >> PAGE_SHIFT, GFP_NOFS);
1369	filemap_invalidate_unlock(mapping);
1370	if (IS_ERR(ptr: page))
1371	return ERR_CAST(ptr: page);
1372
1373	super = page_address(page);
1374	if (btrfs_super_magic(s: super) != BTRFS_MAGIC ||
1375	btrfs_super_bytenr(s: super) != bytenr_orig) {
1376	btrfs_release_disk_super(super);
1377	return ERR_PTR(error: -EINVAL);
1378	}
1379
1380	/*
1381	* Make sure the last byte of label is properly NUL terminated. We use
1382	* '%s' to print the label, if not properly NUL terminated we can access
1383	* beyond the label.
1384	*/
1385	if (super->label[0] && super->label[BTRFS_LABEL_SIZE - 1])
1386	super->label[BTRFS_LABEL_SIZE - 1] = 0;
1387
1388	return super;
1389	}
1390
1391	int btrfs_forget_devices(dev_t devt)
1392	{
1393	int ret;
1394
1395	mutex_lock(&uuid_mutex);
1396	ret = btrfs_free_stale_devices(devt, NULL);
1397	mutex_unlock(lock: &uuid_mutex);
1398
1399	return ret;
1400	}
1401
1402	static bool btrfs_skip_registration(struct btrfs_super_block *disk_super,
1403	const char *path, dev_t devt,
1404	bool mount_arg_dev)
1405	{
1406	struct btrfs_fs_devices *fs_devices;
1407
1408	/*
1409	* Do not skip device registration for mounted devices with matching
1410	* maj:min but different paths. Booting without initrd relies on
1411	* /dev/root initially, later replaced with the actual root device.
1412	* A successful scan ensures grub2-probe selects the correct device.
1413	*/
1414	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
1415	struct btrfs_device *device;
1416
1417	mutex_lock(&fs_devices->device_list_mutex);
1418
1419	if (!fs_devices->opened) {
1420	mutex_unlock(lock: &fs_devices->device_list_mutex);
1421	continue;
1422	}
1423
1424	list_for_each_entry(device, &fs_devices->devices, dev_list) {
1425	if (device->bdev && (device->bdev->bd_dev == devt) &&
1426	strcmp(rcu_dereference_raw(device->name), path) != 0) {
1427	mutex_unlock(lock: &fs_devices->device_list_mutex);
1428
1429	/* Do not skip registration. */
1430	return false;
1431	}
1432	}
1433	mutex_unlock(lock: &fs_devices->device_list_mutex);
1434	}
1435
1436	if (!mount_arg_dev && btrfs_super_num_devices(s: disk_super) == 1 &&
1437	!(btrfs_super_flags(s: disk_super) & BTRFS_SUPER_FLAG_SEEDING))
1438	return true;
1439
1440	return false;
1441	}
1442
1443	/*
1444	* Look for a btrfs signature on a device. This may be called out of the mount path
1445	* and we are not allowed to call set_blocksize during the scan. The superblock
1446	* is read via pagecache.
1447	*
1448	* With @mount_arg_dev it's a scan during mount time that will always register
1449	* the device or return an error. Multi-device and seeding devices are registered
1450	* in both cases.
1451	*/
1452	struct btrfs_device *btrfs_scan_one_device(const char *path,
1453	bool mount_arg_dev)
1454	{
1455	struct btrfs_super_block *disk_super;
1456	bool new_device_added = false;
1457	struct btrfs_device *device = NULL;
1458	struct file *bdev_file;
1459	dev_t devt;
1460
1461	lockdep_assert_held(&uuid_mutex);
1462
1463	/*
1464	* Avoid an exclusive open here, as the systemd-udev may initiate the
1465	* device scan which may race with the user's mount or mkfs command,
1466	* resulting in failure.
1467	* Since the device scan is solely for reading purposes, there is no
1468	* need for an exclusive open. Additionally, the devices are read again
1469	* during the mount process. It is ok to get some inconsistent
1470	* values temporarily, as the device paths of the fsid are the only
1471	* required information for assembling the volume.
1472	*/
1473	bdev_file = bdev_file_open_by_path(path, BLK_OPEN_READ, NULL, NULL);
1474	if (IS_ERR(ptr: bdev_file))
1475	return ERR_CAST(ptr: bdev_file);
1476
1477	disk_super = btrfs_read_disk_super(bdev: file_bdev(bdev_file), copy_num: 0, drop_cache: false);
1478	if (IS_ERR(ptr: disk_super)) {
1479	device = ERR_CAST(ptr: disk_super);
1480	goto error_bdev_put;
1481	}
1482
1483	devt = file_bdev(bdev_file)->bd_dev;
1484	if (btrfs_skip_registration(disk_super, path, devt, mount_arg_dev)) {
1485	btrfs_debug(NULL, "skip registering single non-seed device %s (%d:%d)",
1486	path, MAJOR(devt), MINOR(devt));
1487
1488	btrfs_free_stale_devices(devt, NULL);
1489
1490	device = NULL;
1491	goto free_disk_super;
1492	}
1493
1494	device = device_list_add(path, disk_super, new_device_added: &new_device_added);
1495	if (!IS_ERR(ptr: device) && new_device_added)
1496	btrfs_free_stale_devices(devt: device->devt, skip_device: device);
1497
1498	free_disk_super:
1499	btrfs_release_disk_super(super: disk_super);
1500
1501	error_bdev_put:
1502	bdev_fput(bdev_file);
1503
1504	return device;
1505	}
1506
1507	/*
1508	* Try to find a chunk that intersects [start, start + len] range and when one
1509	* such is found, record the end of it in *start
1510	*/
1511	static bool contains_pending_extent(struct btrfs_device *device, u64 *start,
1512	u64 len)
1513	{
1514	u64 physical_start, physical_end;
1515
1516	lockdep_assert_held(&device->fs_info->chunk_mutex);
1517
1518	if (btrfs_find_first_extent_bit(tree: &device->alloc_state, start: *start,
1519	start_ret: &physical_start, end_ret: &physical_end,
1520	CHUNK_ALLOCATED, NULL)) {
1521
1522	if (in_range(physical_start, *start, len) ||
1523	in_range(*start, physical_start,
1524	physical_end + 1 - physical_start)) {
1525	*start = physical_end + 1;
1526	return true;
1527	}
1528	}
1529	return false;
1530	}
1531
1532	static u64 dev_extent_search_start(struct btrfs_device *device)
1533	{
1534	switch (device->fs_devices->chunk_alloc_policy) {
1535	default:
1536	btrfs_warn_unknown_chunk_allocation(pol: device->fs_devices->chunk_alloc_policy);
1537	fallthrough;
1538	case BTRFS_CHUNK_ALLOC_REGULAR:
1539	return BTRFS_DEVICE_RANGE_RESERVED;
1540	case BTRFS_CHUNK_ALLOC_ZONED:
1541	/*
1542	* We don't care about the starting region like regular
1543	* allocator, because we anyway use/reserve the first two zones
1544	* for superblock logging.
1545	*/
1546	return 0;
1547	}
1548	}
1549
1550	static bool dev_extent_hole_check_zoned(struct btrfs_device *device,
1551	u64 *hole_start, u64 *hole_size,
1552	u64 num_bytes)
1553	{
1554	u64 zone_size = device->zone_info->zone_size;
1555	u64 pos;
1556	int ret;
1557	bool changed = false;
1558
1559	ASSERT(IS_ALIGNED(*hole_start, zone_size),
1560	"hole_start=%llu zone_size=%llu", *hole_start, zone_size);
1561
1562	while (*hole_size > 0) {
1563	pos = btrfs_find_allocatable_zones(device, hole_start: *hole_start,
1564	hole_end: *hole_start + *hole_size,
1565	num_bytes);
1566	if (pos != *hole_start) {
1567	*hole_size = *hole_start + *hole_size - pos;
1568	*hole_start = pos;
1569	changed = true;
1570	if (*hole_size < num_bytes)
1571	break;
1572	}
1573
1574	ret = btrfs_ensure_empty_zones(device, start: pos, size: num_bytes);
1575
1576	/* Range is ensured to be empty */
1577	if (!ret)
1578	return changed;
1579
1580	/* Given hole range was invalid (outside of device) */
1581	if (ret == -ERANGE) {
1582	*hole_start += *hole_size;
1583	*hole_size = 0;
1584	return true;
1585	}
1586
1587	*hole_start += zone_size;
1588	*hole_size -= zone_size;
1589	changed = true;
1590	}
1591
1592	return changed;
1593	}
1594
1595	/*
1596	* Check if specified hole is suitable for allocation.
1597	*
1598	* @device: the device which we have the hole
1599	* @hole_start: starting position of the hole
1600	* @hole_size: the size of the hole
1601	* @num_bytes: the size of the free space that we need
1602	*
1603	* This function may modify @hole_start and @hole_size to reflect the suitable
1604	* position for allocation. Returns 1 if hole position is updated, 0 otherwise.
1605	*/
1606	static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start,
1607	u64 *hole_size, u64 num_bytes)
1608	{
1609	bool changed = false;
1610	u64 hole_end = *hole_start + *hole_size;
1611
1612	for (;;) {
1613	/*
1614	* Check before we set max_hole_start, otherwise we could end up
1615	* sending back this offset anyway.
1616	*/
1617	if (contains_pending_extent(device, start: hole_start, len: *hole_size)) {
1618	if (hole_end >= *hole_start)
1619	*hole_size = hole_end - *hole_start;
1620	else
1621	*hole_size = 0;
1622	changed = true;
1623	}
1624
1625	switch (device->fs_devices->chunk_alloc_policy) {
1626	default:
1627	btrfs_warn_unknown_chunk_allocation(pol: device->fs_devices->chunk_alloc_policy);
1628	fallthrough;
1629	case BTRFS_CHUNK_ALLOC_REGULAR:
1630	/* No extra check */
1631	break;
1632	case BTRFS_CHUNK_ALLOC_ZONED:
1633	if (dev_extent_hole_check_zoned(device, hole_start,
1634	hole_size, num_bytes)) {
1635	changed = true;
1636	/*
1637	* The changed hole can contain pending extent.
1638	* Loop again to check that.
1639	*/
1640	continue;
1641	}
1642	break;
1643	}
1644
1645	break;
1646	}
1647
1648	return changed;
1649	}
1650
1651	/*
1652	* Find free space in the specified device.
1653	*
1654	* @device: the device which we search the free space in
1655	* @num_bytes: the size of the free space that we need
1656	* @search_start: the position from which to begin the search
1657	* @start: store the start of the free space.
1658	* @len: the size of the free space. that we find, or the size
1659	* of the max free space if we don't find suitable free space
1660	*
1661	* This does a pretty simple search, the expectation is that it is called very
1662	* infrequently and that a given device has a small number of extents.
1663	*
1664	* @start is used to store the start of the free space if we find. But if we
1665	* don't find suitable free space, it will be used to store the start position
1666	* of the max free space.
1667	*
1668	* @len is used to store the size of the free space that we find.
1669	* But if we don't find suitable free space, it is used to store the size of
1670	* the max free space.
1671	*
1672	* NOTE: This function will search *commit* root of device tree, and does extra
1673	* check to ensure dev extents are not double allocated.
1674	* This makes the function safe to allocate dev extents but may not report
1675	* correct usable device space, as device extent freed in current transaction
1676	* is not reported as available.
1677	*/
1678	static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes,
1679	u64 *start, u64 *len)
1680	{
1681	struct btrfs_fs_info *fs_info = device->fs_info;
1682	struct btrfs_root *root = fs_info->dev_root;
1683	struct btrfs_key key;
1684	struct btrfs_dev_extent *dev_extent;
1685	BTRFS_PATH_AUTO_FREE(path);
1686	u64 search_start;
1687	u64 hole_size;
1688	u64 max_hole_start;
1689	u64 max_hole_size = 0;
1690	u64 extent_end;
1691	u64 search_end = device->total_bytes;
1692	int ret;
1693	int slot;
1694	struct extent_buffer *l;
1695
1696	search_start = dev_extent_search_start(device);
1697	max_hole_start = search_start;
1698
1699	WARN_ON(device->zone_info &&
1700	!IS_ALIGNED(num_bytes, device->zone_info->zone_size));
1701
1702	path = btrfs_alloc_path();
1703	if (!path) {
1704	ret = -ENOMEM;
1705	goto out;
1706	}
1707	again:
1708	if (search_start >= search_end ||
1709	test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
1710	ret = -ENOSPC;
1711	goto out;
1712	}
1713
1714	path->reada = READA_FORWARD;
1715	path->search_commit_root = true;
1716	path->skip_locking = true;
1717
1718	key.objectid = device->devid;
1719	key.type = BTRFS_DEV_EXTENT_KEY;
1720	key.offset = search_start;
1721
1722	ret = btrfs_search_backwards(root, key: &key, path);
1723	if (ret < 0)
1724	goto out;
1725
1726	while (search_start < search_end) {
1727	l = path->nodes[0];
1728	slot = path->slots[0];
1729	if (slot >= btrfs_header_nritems(eb: l)) {
1730	ret = btrfs_next_leaf(root, path);
1731	if (ret == 0)
1732	continue;
1733	if (ret < 0)
1734	goto out;
1735
1736	break;
1737	}
1738	btrfs_item_key_to_cpu(eb: l, cpu_key: &key, nr: slot);
1739
1740	if (key.objectid < device->devid)
1741	goto next;
1742
1743	if (key.objectid > device->devid)
1744	break;
1745
1746	if (key.type != BTRFS_DEV_EXTENT_KEY)
1747	goto next;
1748
1749	if (key.offset > search_end)
1750	break;
1751
1752	if (key.offset > search_start) {
1753	hole_size = key.offset - search_start;
1754	dev_extent_hole_check(device, hole_start: &search_start, hole_size: &hole_size,
1755	num_bytes);
1756
1757	if (hole_size > max_hole_size) {
1758	max_hole_start = search_start;
1759	max_hole_size = hole_size;
1760	}
1761
1762	/*
1763	* If this free space is greater than which we need,
1764	* it must be the max free space that we have found
1765	* until now, so max_hole_start must point to the start
1766	* of this free space and the length of this free space
1767	* is stored in max_hole_size. Thus, we return
1768	* max_hole_start and max_hole_size and go back to the
1769	* caller.
1770	*/
1771	if (hole_size >= num_bytes) {
1772	ret = 0;
1773	goto out;
1774	}
1775	}
1776
1777	dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1778	extent_end = key.offset + btrfs_dev_extent_length(eb: l,
1779	s: dev_extent);
1780	if (extent_end > search_start)
1781	search_start = extent_end;
1782	next:
1783	path->slots[0]++;
1784	cond_resched();
1785	}
1786
1787	/*
1788	* At this point, search_start should be the end of
1789	* allocated dev extents, and when shrinking the device,
1790	* search_end may be smaller than search_start.
1791	*/
1792	if (search_end > search_start) {
1793	hole_size = search_end - search_start;
1794	if (dev_extent_hole_check(device, hole_start: &search_start, hole_size: &hole_size,
1795	num_bytes)) {
1796	btrfs_release_path(p: path);
1797	goto again;
1798	}
1799
1800	if (hole_size > max_hole_size) {
1801	max_hole_start = search_start;
1802	max_hole_size = hole_size;
1803	}
1804	}
1805
1806	/* See above. */
1807	if (max_hole_size < num_bytes)
1808	ret = -ENOSPC;
1809	else
1810	ret = 0;
1811
1812	ASSERT(max_hole_start + max_hole_size <= search_end,
1813	"max_hole_start=%llu max_hole_size=%llu search_end=%llu",
1814	max_hole_start, max_hole_size, search_end);
1815	out:
1816	*start = max_hole_start;
1817	if (len)
1818	*len = max_hole_size;
1819	return ret;
1820	}
1821
1822	static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1823	struct btrfs_device *device,
1824	u64 start, u64 *dev_extent_len)
1825	{
1826	struct btrfs_fs_info *fs_info = device->fs_info;
1827	struct btrfs_root *root = fs_info->dev_root;
1828	int ret;
1829	BTRFS_PATH_AUTO_FREE(path);
1830	struct btrfs_key key;
1831	struct btrfs_key found_key;
1832	struct extent_buffer *leaf = NULL;
1833	struct btrfs_dev_extent *extent = NULL;
1834
1835	path = btrfs_alloc_path();
1836	if (!path)
1837	return -ENOMEM;
1838
1839	key.objectid = device->devid;
1840	key.type = BTRFS_DEV_EXTENT_KEY;
1841	key.offset = start;
1842	again:
1843	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: -1, cow: 1);
1844	if (ret > 0) {
1845	ret = btrfs_previous_item(root, path, min_objectid: key.objectid,
1846	BTRFS_DEV_EXTENT_KEY);
1847	if (ret)
1848	return ret;
1849	leaf = path->nodes[0];
1850	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
1851	extent = btrfs_item_ptr(leaf, path->slots[0],
1852	struct btrfs_dev_extent);
1853	BUG_ON(found_key.offset > start || found_key.offset +
1854	btrfs_dev_extent_length(leaf, extent) < start);
1855	key = found_key;
1856	btrfs_release_path(p: path);
1857	goto again;
1858	} else if (ret == 0) {
1859	leaf = path->nodes[0];
1860	extent = btrfs_item_ptr(leaf, path->slots[0],
1861	struct btrfs_dev_extent);
1862	} else {
1863	return ret;
1864	}
1865
1866	*dev_extent_len = btrfs_dev_extent_length(eb: leaf, s: extent);
1867
1868	ret = btrfs_del_item(trans, root, path);
1869	if (ret == 0)
1870	set_bit(BTRFS_TRANS_HAVE_FREE_BGS, addr: &trans->transaction->flags);
1871	return ret;
1872	}
1873
1874	static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1875	{
1876	struct rb_node *n;
1877	u64 ret = 0;
1878
1879	read_lock(&fs_info->mapping_tree_lock);
1880	n = rb_last(root: &fs_info->mapping_tree.rb_root);
1881	if (n) {
1882	struct btrfs_chunk_map *map;
1883
1884	map = rb_entry(n, struct btrfs_chunk_map, rb_node);
1885	ret = map->start + map->chunk_len;
1886	}
1887	read_unlock(&fs_info->mapping_tree_lock);
1888
1889	return ret;
1890	}
1891
1892	static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1893	u64 *devid_ret)
1894	{
1895	int ret;
1896	struct btrfs_key key;
1897	struct btrfs_key found_key;
1898	BTRFS_PATH_AUTO_FREE(path);
1899
1900	path = btrfs_alloc_path();
1901	if (!path)
1902	return -ENOMEM;
1903
1904	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1905	key.type = BTRFS_DEV_ITEM_KEY;
1906	key.offset = (u64)-1;
1907
1908	ret = btrfs_search_slot(NULL, root: fs_info->chunk_root, key: &key, p: path, ins_len: 0, cow: 0);
1909	if (ret < 0)
1910	return ret;
1911
1912	if (unlikely(ret == 0)) {
1913	/* Corruption */
1914	btrfs_err(fs_info, "corrupted chunk tree devid -1 matched");
1915	return -EUCLEAN;
1916	}
1917
1918	ret = btrfs_previous_item(root: fs_info->chunk_root, path,
1919	BTRFS_DEV_ITEMS_OBJECTID,
1920	BTRFS_DEV_ITEM_KEY);
1921	if (ret) {
1922	*devid_ret = 1;
1923	} else {
1924	btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &found_key,
1925	nr: path->slots[0]);
1926	*devid_ret = found_key.offset + 1;
1927	}
1928	return 0;
1929	}
1930
1931	/*
1932	* the device information is stored in the chunk root
1933	* the btrfs_device struct should be fully filled in
1934	*/
1935	static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
1936	struct btrfs_device *device)
1937	{
1938	int ret;
1939	BTRFS_PATH_AUTO_FREE(path);
1940	struct btrfs_dev_item *dev_item;
1941	struct extent_buffer *leaf;
1942	struct btrfs_key key;
1943	unsigned long ptr;
1944
1945	path = btrfs_alloc_path();
1946	if (!path)
1947	return -ENOMEM;
1948
1949	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1950	key.type = BTRFS_DEV_ITEM_KEY;
1951	key.offset = device->devid;
1952
1953	btrfs_reserve_chunk_metadata(trans, is_item_insertion: true);
1954	ret = btrfs_insert_empty_item(trans, root: trans->fs_info->chunk_root, path,
1955	key: &key, data_size: sizeof(*dev_item));
1956	btrfs_trans_release_chunk_metadata(trans);
1957	if (ret)
1958	return ret;
1959
1960	leaf = path->nodes[0];
1961	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1962
1963	btrfs_set_device_id(eb: leaf, s: dev_item, val: device->devid);
1964	btrfs_set_device_generation(eb: leaf, s: dev_item, val: 0);
1965	btrfs_set_device_type(eb: leaf, s: dev_item, val: device->type);
1966	btrfs_set_device_io_align(eb: leaf, s: dev_item, val: device->io_align);
1967	btrfs_set_device_io_width(eb: leaf, s: dev_item, val: device->io_width);
1968	btrfs_set_device_sector_size(eb: leaf, s: dev_item, val: device->sector_size);
1969	btrfs_set_device_total_bytes(eb: leaf, s: dev_item,
1970	val: btrfs_device_get_disk_total_bytes(dev: device));
1971	btrfs_set_device_bytes_used(eb: leaf, s: dev_item,
1972	val: btrfs_device_get_bytes_used(dev: device));
1973	btrfs_set_device_group(eb: leaf, s: dev_item, val: 0);
1974	btrfs_set_device_seek_speed(eb: leaf, s: dev_item, val: 0);
1975	btrfs_set_device_bandwidth(eb: leaf, s: dev_item, val: 0);
1976	btrfs_set_device_start_offset(eb: leaf, s: dev_item, val: 0);
1977
1978	ptr = btrfs_device_uuid(d: dev_item);
1979	write_extent_buffer(eb: leaf, src: device->uuid, start: ptr, BTRFS_UUID_SIZE);
1980	ptr = btrfs_device_fsid(d: dev_item);
1981	write_extent_buffer(eb: leaf, src: trans->fs_info->fs_devices->metadata_uuid,
1982	start: ptr, BTRFS_FSID_SIZE);
1983
1984	return 0;
1985	}
1986
1987	/*
1988	* Function to update ctime/mtime for a given device path.
1989	* Mainly used for ctime/mtime based probe like libblkid.
1990	*
1991	* We don't care about errors here, this is just to be kind to userspace.
1992	*/
1993	static void update_dev_time(const char *device_path)
1994	{
1995	struct path path;
1996
1997	if (!kern_path(device_path, LOOKUP_FOLLOW, &path)) {
1998	vfs_utimes(path: &path, NULL);
1999	path_put(&path);
2000	}
2001	}
2002
2003	static int btrfs_rm_dev_item(struct btrfs_trans_handle *trans,
2004	struct btrfs_device *device)
2005	{
2006	struct btrfs_root *root = device->fs_info->chunk_root;
2007	int ret;
2008	BTRFS_PATH_AUTO_FREE(path);
2009	struct btrfs_key key;
2010
2011	path = btrfs_alloc_path();
2012	if (!path)
2013	return -ENOMEM;
2014
2015	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2016	key.type = BTRFS_DEV_ITEM_KEY;
2017	key.offset = device->devid;
2018
2019	btrfs_reserve_chunk_metadata(trans, is_item_insertion: false);
2020	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: -1, cow: 1);
2021	btrfs_trans_release_chunk_metadata(trans);
2022	if (ret > 0)
2023	return -ENOENT;
2024	if (ret < 0)
2025	return ret;
2026
2027	return btrfs_del_item(trans, root, path);
2028	}
2029
2030	/*
2031	* Verify that @num_devices satisfies the RAID profile constraints in the whole
2032	* filesystem. It's up to the caller to adjust that number regarding eg. device
2033	* replace.
2034	*/
2035	static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
2036	u64 num_devices)
2037	{
2038	u64 all_avail;
2039	unsigned seq;
2040	int i;
2041
2042	do {
2043	seq = read_seqbegin(sl: &fs_info->profiles_lock);
2044
2045	all_avail = fs_info->avail_data_alloc_bits |
2046	fs_info->avail_system_alloc_bits |
2047	fs_info->avail_metadata_alloc_bits;
2048	} while (read_seqretry(sl: &fs_info->profiles_lock, start: seq));
2049
2050	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
2051	if (!(all_avail & btrfs_raid_array[i].bg_flag))
2052	continue;
2053
2054	if (num_devices < btrfs_raid_array[i].devs_min)
2055	return btrfs_raid_array[i].mindev_error;
2056	}
2057
2058	return 0;
2059	}
2060
2061	static struct btrfs_device * btrfs_find_next_active_device(
2062	struct btrfs_fs_devices *fs_devs, struct btrfs_device *device)
2063	{
2064	struct btrfs_device *next_device;
2065
2066	list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
2067	if (next_device != device &&
2068	!test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state)
2069	&& next_device->bdev)
2070	return next_device;
2071	}
2072
2073	return NULL;
2074	}
2075
2076	/*
2077	* Helper function to check if the given device is part of s_bdev / latest_dev
2078	* and replace it with the provided or the next active device, in the context
2079	* where this function called, there should be always be another device (or
2080	* this_dev) which is active.
2081	*/
2082	void __cold btrfs_assign_next_active_device(struct btrfs_device *device,
2083	struct btrfs_device *next_device)
2084	{
2085	struct btrfs_fs_info *fs_info = device->fs_info;
2086
2087	if (!next_device)
2088	next_device = btrfs_find_next_active_device(fs_devs: fs_info->fs_devices,
2089	device);
2090	ASSERT(next_device);
2091
2092	if (fs_info->sb->s_bdev &&
2093	(fs_info->sb->s_bdev == device->bdev))
2094	fs_info->sb->s_bdev = next_device->bdev;
2095
2096	if (fs_info->fs_devices->latest_dev->bdev == device->bdev)
2097	fs_info->fs_devices->latest_dev = next_device;
2098	}
2099
2100	/*
2101	* Return btrfs_fs_devices::num_devices excluding the device that's being
2102	* currently replaced.
2103	*/
2104	static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info)
2105	{
2106	u64 num_devices = fs_info->fs_devices->num_devices;
2107
2108	down_read(sem: &fs_info->dev_replace.rwsem);
2109	if (btrfs_dev_replace_is_ongoing(dev_replace: &fs_info->dev_replace)) {
2110	ASSERT(num_devices > 1, "num_devices=%llu", num_devices);
2111	num_devices--;
2112	}
2113	up_read(sem: &fs_info->dev_replace.rwsem);
2114
2115	return num_devices;
2116	}
2117
2118	static void btrfs_scratch_superblock(struct btrfs_fs_info *fs_info,
2119	struct block_device *bdev, int copy_num)
2120	{
2121	struct btrfs_super_block *disk_super;
2122	const size_t len = sizeof(disk_super->magic);
2123	const u64 bytenr = btrfs_sb_offset(mirror: copy_num);
2124	int ret;
2125
2126	disk_super = btrfs_read_disk_super(bdev, copy_num, drop_cache: false);
2127	if (IS_ERR(ptr: disk_super))
2128	return;
2129
2130	memset(&disk_super->magic, 0, len);
2131	folio_mark_dirty(folio: virt_to_folio(x: disk_super));
2132	btrfs_release_disk_super(super: disk_super);
2133
2134	ret = sync_blockdev_range(bdev, lstart: bytenr, lend: bytenr + len - 1);
2135	if (ret)
2136	btrfs_warn(fs_info, "error clearing superblock number %d (%d)",
2137	copy_num, ret);
2138	}
2139
2140	void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info, struct btrfs_device *device)
2141	{
2142	int copy_num;
2143	struct block_device *bdev = device->bdev;
2144
2145	if (!bdev)
2146	return;
2147
2148	for (copy_num = 0; copy_num < BTRFS_SUPER_MIRROR_MAX; copy_num++) {
2149	if (bdev_is_zoned(bdev))
2150	btrfs_reset_sb_log_zones(bdev, mirror: copy_num);
2151	else
2152	btrfs_scratch_superblock(fs_info, bdev, copy_num);
2153	}
2154
2155	/* Notify udev that device has changed */
2156	btrfs_kobject_uevent(bdev, action: KOBJ_CHANGE);
2157
2158	/* Update ctime/mtime for device path for libblkid */
2159	update_dev_time(rcu_dereference_raw(device->name));
2160	}
2161
2162	int btrfs_rm_device(struct btrfs_fs_info *fs_info,
2163	struct btrfs_dev_lookup_args *args,
2164	struct file **bdev_file)
2165	{
2166	struct btrfs_trans_handle *trans;
2167	struct btrfs_device *device;
2168	struct btrfs_fs_devices *cur_devices;
2169	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2170	u64 num_devices;
2171	int ret = 0;
2172
2173	if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
2174	btrfs_err(fs_info, "device remove not supported on extent tree v2 yet");
2175	return -EINVAL;
2176	}
2177
2178	/*
2179	* The device list in fs_devices is accessed without locks (neither
2180	* uuid_mutex nor device_list_mutex) as it won't change on a mounted
2181	* filesystem and another device rm cannot run.
2182	*/
2183	num_devices = btrfs_num_devices(fs_info);
2184
2185	ret = btrfs_check_raid_min_devices(fs_info, num_devices: num_devices - 1);
2186	if (ret)
2187	return ret;
2188
2189	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args);
2190	if (!device) {
2191	if (args->missing)
2192	ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2193	else
2194	ret = -ENOENT;
2195	return ret;
2196	}
2197
2198	if (btrfs_pinned_by_swapfile(fs_info, ptr: device)) {
2199	btrfs_warn(fs_info,
2200	"cannot remove device %s (devid %llu) due to active swapfile",
2201	btrfs_dev_name(device), device->devid);
2202	return -ETXTBSY;
2203	}
2204
2205	if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
2206	return BTRFS_ERROR_DEV_TGT_REPLACE;
2207
2208	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
2209	fs_info->fs_devices->rw_devices == 1)
2210	return BTRFS_ERROR_DEV_ONLY_WRITABLE;
2211
2212	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2213	mutex_lock(&fs_info->chunk_mutex);
2214	list_del_init(entry: &device->dev_alloc_list);
2215	device->fs_devices->rw_devices--;
2216	mutex_unlock(lock: &fs_info->chunk_mutex);
2217	}
2218
2219	ret = btrfs_shrink_device(device, new_size: 0);
2220	if (ret)
2221	goto error_undo;
2222
2223	trans = btrfs_start_transaction(root: fs_info->chunk_root, num_items: 0);
2224	if (IS_ERR(ptr: trans)) {
2225	ret = PTR_ERR(ptr: trans);
2226	goto error_undo;
2227	}
2228
2229	ret = btrfs_rm_dev_item(trans, device);
2230	if (unlikely(ret)) {
2231	/* Any error in dev item removal is critical */
2232	btrfs_crit(fs_info,
2233	"failed to remove device item for devid %llu: %d",
2234	device->devid, ret);
2235	btrfs_abort_transaction(trans, ret);
2236	btrfs_end_transaction(trans);
2237	return ret;
2238	}
2239
2240	clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, addr: &device->dev_state);
2241	btrfs_scrub_cancel_dev(dev: device);
2242
2243	/*
2244	* the device list mutex makes sure that we don't change
2245	* the device list while someone else is writing out all
2246	* the device supers. Whoever is writing all supers, should
2247	* lock the device list mutex before getting the number of
2248	* devices in the super block (super_copy). Conversely,
2249	* whoever updates the number of devices in the super block
2250	* (super_copy) should hold the device list mutex.
2251	*/
2252
2253	/*
2254	* In normal cases the cur_devices == fs_devices. But in case
2255	* of deleting a seed device, the cur_devices should point to
2256	* its own fs_devices listed under the fs_devices->seed_list.
2257	*/
2258	cur_devices = device->fs_devices;
2259	mutex_lock(&fs_devices->device_list_mutex);
2260	list_del_rcu(entry: &device->dev_list);
2261
2262	cur_devices->num_devices--;
2263	cur_devices->total_devices--;
2264	/* Update total_devices of the parent fs_devices if it's seed */
2265	if (cur_devices != fs_devices)
2266	fs_devices->total_devices--;
2267
2268	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
2269	cur_devices->missing_devices--;
2270
2271	btrfs_assign_next_active_device(device, NULL);
2272
2273	if (device->bdev_file) {
2274	cur_devices->open_devices--;
2275	/* remove sysfs entry */
2276	btrfs_sysfs_remove_device(device);
2277	}
2278
2279	num_devices = btrfs_super_num_devices(s: fs_info->super_copy) - 1;
2280	btrfs_set_super_num_devices(s: fs_info->super_copy, val: num_devices);
2281	mutex_unlock(lock: &fs_devices->device_list_mutex);
2282
2283	/*
2284	* At this point, the device is zero sized and detached from the
2285	* devices list. All that's left is to zero out the old supers and
2286	* free the device.
2287	*
2288	* We cannot call btrfs_close_bdev() here because we're holding the sb
2289	* write lock, and bdev_fput() on the block device will pull in the
2290	* ->open_mutex on the block device and it's dependencies. Instead
2291	* just flush the device and let the caller do the final bdev_release.
2292	*/
2293	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2294	btrfs_scratch_superblocks(fs_info, device);
2295	if (device->bdev) {
2296	sync_blockdev(bdev: device->bdev);
2297	invalidate_bdev(bdev: device->bdev);
2298	}
2299	}
2300
2301	*bdev_file = device->bdev_file;
2302	synchronize_rcu();
2303	btrfs_free_device(device);
2304
2305	/*
2306	* This can happen if cur_devices is the private seed devices list. We
2307	* cannot call close_fs_devices() here because it expects the uuid_mutex
2308	* to be held, but in fact we don't need that for the private
2309	* seed_devices, we can simply decrement cur_devices->opened and then
2310	* remove it from our list and free the fs_devices.
2311	*/
2312	if (cur_devices->num_devices == 0) {
2313	list_del_init(entry: &cur_devices->seed_list);
2314	ASSERT(cur_devices->opened == 1, "opened=%d", cur_devices->opened);
2315	cur_devices->opened--;
2316	free_fs_devices(fs_devices: cur_devices);
2317	}
2318
2319	ret = btrfs_commit_transaction(trans);
2320
2321	return ret;
2322
2323	error_undo:
2324	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2325	mutex_lock(&fs_info->chunk_mutex);
2326	list_add(new: &device->dev_alloc_list,
2327	head: &fs_devices->alloc_list);
2328	device->fs_devices->rw_devices++;
2329	mutex_unlock(lock: &fs_info->chunk_mutex);
2330	}
2331	return ret;
2332	}
2333
2334	void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev)
2335	{
2336	struct btrfs_fs_devices *fs_devices;
2337
2338	lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex);
2339
2340	/*
2341	* in case of fs with no seed, srcdev->fs_devices will point
2342	* to fs_devices of fs_info. However when the dev being replaced is
2343	* a seed dev it will point to the seed's local fs_devices. In short
2344	* srcdev will have its correct fs_devices in both the cases.
2345	*/
2346	fs_devices = srcdev->fs_devices;
2347
2348	list_del_rcu(entry: &srcdev->dev_list);
2349	list_del(entry: &srcdev->dev_alloc_list);
2350	fs_devices->num_devices--;
2351	if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state))
2352	fs_devices->missing_devices--;
2353
2354	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state))
2355	fs_devices->rw_devices--;
2356
2357	if (srcdev->bdev)
2358	fs_devices->open_devices--;
2359	}
2360
2361	void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
2362	{
2363	struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2364
2365	mutex_lock(&uuid_mutex);
2366
2367	btrfs_close_bdev(device: srcdev);
2368	synchronize_rcu();
2369	btrfs_free_device(device: srcdev);
2370
2371	/* if this is no devs we rather delete the fs_devices */
2372	if (!fs_devices->num_devices) {
2373	/*
2374	* On a mounted FS, num_devices can't be zero unless it's a
2375	* seed. In case of a seed device being replaced, the replace
2376	* target added to the sprout FS, so there will be no more
2377	* device left under the seed FS.
2378	*/
2379	ASSERT(fs_devices->seeding);
2380
2381	list_del_init(entry: &fs_devices->seed_list);
2382	close_fs_devices(fs_devices);
2383	free_fs_devices(fs_devices);
2384	}
2385	mutex_unlock(lock: &uuid_mutex);
2386	}
2387
2388	void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
2389	{
2390	struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
2391
2392	mutex_lock(&fs_devices->device_list_mutex);
2393
2394	btrfs_sysfs_remove_device(device: tgtdev);
2395
2396	if (tgtdev->bdev)
2397	fs_devices->open_devices--;
2398
2399	fs_devices->num_devices--;
2400
2401	btrfs_assign_next_active_device(device: tgtdev, NULL);
2402
2403	list_del_rcu(entry: &tgtdev->dev_list);
2404
2405	mutex_unlock(lock: &fs_devices->device_list_mutex);
2406
2407	btrfs_scratch_superblocks(fs_info: tgtdev->fs_info, device: tgtdev);
2408
2409	btrfs_close_bdev(device: tgtdev);
2410	synchronize_rcu();
2411	btrfs_free_device(device: tgtdev);
2412	}
2413
2414	/*
2415	* Populate args from device at path.
2416	*
2417	* @fs_info: the filesystem
2418	* @args: the args to populate
2419	* @path: the path to the device
2420	*
2421	* This will read the super block of the device at @path and populate @args with
2422	* the devid, fsid, and uuid. This is meant to be used for ioctls that need to
2423	* lookup a device to operate on, but need to do it before we take any locks.
2424	* This properly handles the special case of "missing" that a user may pass in,
2425	* and does some basic sanity checks. The caller must make sure that @path is
2426	* properly NUL terminated before calling in, and must call
2427	* btrfs_put_dev_args_from_path() in order to free up the temporary fsid and
2428	* uuid buffers.
2429	*
2430	* Return: 0 for success, -errno for failure
2431	*/
2432	int btrfs_get_dev_args_from_path(struct btrfs_fs_info *fs_info,
2433	struct btrfs_dev_lookup_args *args,
2434	const char *path)
2435	{
2436	struct btrfs_super_block *disk_super;
2437	struct file *bdev_file;
2438	int ret;
2439
2440	if (!path || !path[0])
2441	return -EINVAL;
2442	if (!strcmp(path, "missing")) {
2443	args->missing = true;
2444	return 0;
2445	}
2446
2447	args->uuid = kzalloc(BTRFS_UUID_SIZE, GFP_KERNEL);
2448	args->fsid = kzalloc(BTRFS_FSID_SIZE, GFP_KERNEL);
2449	if (!args->uuid || !args->fsid) {
2450	btrfs_put_dev_args_from_path(args);
2451	return -ENOMEM;
2452	}
2453
2454	ret = btrfs_get_bdev_and_sb(device_path: path, BLK_OPEN_READ, NULL, flush: 0,
2455	bdev_file: &bdev_file, disk_super: &disk_super);
2456	if (ret) {
2457	btrfs_put_dev_args_from_path(args);
2458	return ret;
2459	}
2460
2461	args->devid = btrfs_stack_device_id(s: &disk_super->dev_item);
2462	memcpy(args->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE);
2463	if (btrfs_fs_incompat(fs_info, METADATA_UUID))
2464	memcpy(args->fsid, disk_super->metadata_uuid, BTRFS_FSID_SIZE);
2465	else
2466	memcpy(args->fsid, disk_super->fsid, BTRFS_FSID_SIZE);
2467	btrfs_release_disk_super(super: disk_super);
2468	bdev_fput(bdev_file);
2469	return 0;
2470	}
2471
2472	/*
2473	* Only use this jointly with btrfs_get_dev_args_from_path() because we will
2474	* allocate our ->uuid and ->fsid pointers, everybody else uses local variables
2475	* that don't need to be freed.
2476	*/
2477	void btrfs_put_dev_args_from_path(struct btrfs_dev_lookup_args *args)
2478	{
2479	kfree(objp: args->uuid);
2480	kfree(objp: args->fsid);
2481	args->uuid = NULL;
2482	args->fsid = NULL;
2483	}
2484
2485	struct btrfs_device *btrfs_find_device_by_devspec(
2486	struct btrfs_fs_info *fs_info, u64 devid,
2487	const char *device_path)
2488	{
2489	BTRFS_DEV_LOOKUP_ARGS(args);
2490	struct btrfs_device *device;
2491	int ret;
2492
2493	if (devid) {
2494	args.devid = devid;
2495	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
2496	if (!device)
2497	return ERR_PTR(error: -ENOENT);
2498	return device;
2499	}
2500
2501	ret = btrfs_get_dev_args_from_path(fs_info, args: &args, path: device_path);
2502	if (ret)
2503	return ERR_PTR(error: ret);
2504	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
2505	btrfs_put_dev_args_from_path(args: &args);
2506	if (!device)
2507	return ERR_PTR(error: -ENOENT);
2508	return device;
2509	}
2510
2511	static struct btrfs_fs_devices *btrfs_init_sprout(struct btrfs_fs_info *fs_info)
2512	{
2513	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2514	struct btrfs_fs_devices *old_devices;
2515	struct btrfs_fs_devices *seed_devices;
2516
2517	lockdep_assert_held(&uuid_mutex);
2518	if (!fs_devices->seeding)
2519	return ERR_PTR(error: -EINVAL);
2520
2521	/*
2522	* Private copy of the seed devices, anchored at
2523	* fs_info->fs_devices->seed_list
2524	*/
2525	seed_devices = alloc_fs_devices(NULL);
2526	if (IS_ERR(ptr: seed_devices))
2527	return seed_devices;
2528
2529	/*
2530	* It's necessary to retain a copy of the original seed fs_devices in
2531	* fs_uuids so that filesystems which have been seeded can successfully
2532	* reference the seed device from open_seed_devices. This also supports
2533	* multiple fs seed.
2534	*/
2535	old_devices = clone_fs_devices(orig: fs_devices);
2536	if (IS_ERR(ptr: old_devices)) {
2537	kfree(objp: seed_devices);
2538	return old_devices;
2539	}
2540
2541	list_add(new: &old_devices->fs_list, head: &fs_uuids);
2542
2543	memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
2544	seed_devices->opened = 1;
2545	INIT_LIST_HEAD(list: &seed_devices->devices);
2546	INIT_LIST_HEAD(list: &seed_devices->alloc_list);
2547	mutex_init(&seed_devices->device_list_mutex);
2548
2549	return seed_devices;
2550	}
2551
2552	/*
2553	* Splice seed devices into the sprout fs_devices.
2554	* Generate a new fsid for the sprouted read-write filesystem.
2555	*/
2556	static void btrfs_setup_sprout(struct btrfs_fs_info *fs_info,
2557	struct btrfs_fs_devices *seed_devices)
2558	{
2559	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2560	struct btrfs_super_block *disk_super = fs_info->super_copy;
2561	struct btrfs_device *device;
2562	u64 super_flags;
2563
2564	/*
2565	* We are updating the fsid, the thread leading to device_list_add()
2566	* could race, so uuid_mutex is needed.
2567	*/
2568	lockdep_assert_held(&uuid_mutex);
2569
2570	/*
2571	* The threads listed below may traverse dev_list but can do that without
2572	* device_list_mutex:
2573	* - All device ops and balance - as we are in btrfs_exclop_start.
2574	* - Various dev_list readers - are using RCU.
2575	* - btrfs_ioctl_fitrim() - is using RCU.
2576	*
2577	* For-read threads as below are using device_list_mutex:
2578	* - Readonly scrub btrfs_scrub_dev()
2579	* - Readonly scrub btrfs_scrub_progress()
2580	* - btrfs_get_dev_stats()
2581	*/
2582	lockdep_assert_held(&fs_devices->device_list_mutex);
2583
2584	list_splice_init_rcu(list: &fs_devices->devices, head: &seed_devices->devices,
2585	sync: synchronize_rcu);
2586	list_for_each_entry(device, &seed_devices->devices, dev_list)
2587	device->fs_devices = seed_devices;
2588
2589	fs_devices->seeding = false;
2590	fs_devices->num_devices = 0;
2591	fs_devices->open_devices = 0;
2592	fs_devices->missing_devices = 0;
2593	fs_devices->rotating = false;
2594	list_add(new: &seed_devices->seed_list, head: &fs_devices->seed_list);
2595
2596	generate_random_uuid(uuid: fs_devices->fsid);
2597	memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
2598	memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2599
2600	super_flags = btrfs_super_flags(s: disk_super) &
2601	~BTRFS_SUPER_FLAG_SEEDING;
2602	btrfs_set_super_flags(s: disk_super, val: super_flags);
2603	}
2604
2605	/*
2606	* Store the expected generation for seed devices in device items.
2607	*/
2608	static int btrfs_finish_sprout(struct btrfs_trans_handle *trans)
2609	{
2610	BTRFS_DEV_LOOKUP_ARGS(args);
2611	struct btrfs_fs_info *fs_info = trans->fs_info;
2612	struct btrfs_root *root = fs_info->chunk_root;
2613	BTRFS_PATH_AUTO_FREE(path);
2614	struct extent_buffer *leaf;
2615	struct btrfs_dev_item *dev_item;
2616	struct btrfs_device *device;
2617	struct btrfs_key key;
2618	u8 fs_uuid[BTRFS_FSID_SIZE];
2619	u8 dev_uuid[BTRFS_UUID_SIZE];
2620	int ret;
2621
2622	path = btrfs_alloc_path();
2623	if (!path)
2624	return -ENOMEM;
2625
2626	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2627	key.type = BTRFS_DEV_ITEM_KEY;
2628	key.offset = 0;
2629
2630	while (1) {
2631	btrfs_reserve_chunk_metadata(trans, is_item_insertion: false);
2632	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: 0, cow: 1);
2633	btrfs_trans_release_chunk_metadata(trans);
2634	if (ret < 0)
2635	return ret;
2636
2637	leaf = path->nodes[0];
2638	next_slot:
2639	if (path->slots[0] >= btrfs_header_nritems(eb: leaf)) {
2640	ret = btrfs_next_leaf(root, path);
2641	if (ret > 0)
2642	break;
2643	if (ret < 0)
2644	return ret;
2645	leaf = path->nodes[0];
2646	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
2647	btrfs_release_path(p: path);
2648	continue;
2649	}
2650
2651	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
2652	if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2653	key.type != BTRFS_DEV_ITEM_KEY)
2654	break;
2655
2656	dev_item = btrfs_item_ptr(leaf, path->slots[0],
2657	struct btrfs_dev_item);
2658	args.devid = btrfs_device_id(eb: leaf, s: dev_item);
2659	read_extent_buffer(eb: leaf, dst: dev_uuid, start: btrfs_device_uuid(d: dev_item),
2660	BTRFS_UUID_SIZE);
2661	read_extent_buffer(eb: leaf, dst: fs_uuid, start: btrfs_device_fsid(d: dev_item),
2662	BTRFS_FSID_SIZE);
2663	args.uuid = dev_uuid;
2664	args.fsid = fs_uuid;
2665	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
2666	BUG_ON(!device); /* Logic error */
2667
2668	if (device->fs_devices->seeding)
2669	btrfs_set_device_generation(eb: leaf, s: dev_item,
2670	val: device->generation);
2671
2672	path->slots[0]++;
2673	goto next_slot;
2674	}
2675	return 0;
2676	}
2677
2678	int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
2679	{
2680	struct btrfs_root *root = fs_info->dev_root;
2681	struct btrfs_trans_handle *trans;
2682	struct btrfs_device *device;
2683	struct file *bdev_file;
2684	struct super_block *sb = fs_info->sb;
2685	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2686	struct btrfs_fs_devices *seed_devices = NULL;
2687	u64 orig_super_total_bytes;
2688	u64 orig_super_num_devices;
2689	int ret = 0;
2690	bool seeding_dev = false;
2691	bool locked = false;
2692
2693	if (sb_rdonly(sb) && !fs_devices->seeding)
2694	return -EROFS;
2695
2696	bdev_file = bdev_file_open_by_path(path: device_path, BLK_OPEN_WRITE,
2697	holder: fs_info->sb, hops: &fs_holder_ops);
2698	if (IS_ERR(ptr: bdev_file))
2699	return PTR_ERR(ptr: bdev_file);
2700
2701	if (!btrfs_check_device_zone_type(fs_info, bdev: file_bdev(bdev_file))) {
2702	ret = -EINVAL;
2703	goto error;
2704	}
2705
2706	if (bdev_nr_bytes(bdev: file_bdev(bdev_file)) <= BTRFS_DEVICE_RANGE_RESERVED) {
2707	ret = -EINVAL;
2708	goto error;
2709	}
2710
2711	if (fs_devices->seeding) {
2712	seeding_dev = true;
2713	down_write(sem: &sb->s_umount);
2714	mutex_lock(&uuid_mutex);
2715	locked = true;
2716	}
2717
2718	sync_blockdev(bdev: file_bdev(bdev_file));
2719
2720	rcu_read_lock();
2721	list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) {
2722	if (device->bdev == file_bdev(bdev_file)) {
2723	ret = -EEXIST;
2724	rcu_read_unlock();
2725	goto error;
2726	}
2727	}
2728	rcu_read_unlock();
2729
2730	device = btrfs_alloc_device(fs_info, NULL, NULL, path: device_path);
2731	if (IS_ERR(ptr: device)) {
2732	/* we can safely leave the fs_devices entry around */
2733	ret = PTR_ERR(ptr: device);
2734	goto error;
2735	}
2736
2737	device->fs_info = fs_info;
2738	device->bdev_file = bdev_file;
2739	device->bdev = file_bdev(bdev_file);
2740	ret = lookup_bdev(pathname: device_path, dev: &device->devt);
2741	if (ret)
2742	goto error_free_device;
2743
2744	ret = btrfs_get_dev_zone_info(device, populate_cache: false);
2745	if (ret)
2746	goto error_free_device;
2747
2748	trans = btrfs_start_transaction(root, num_items: 0);
2749	if (IS_ERR(ptr: trans)) {
2750	ret = PTR_ERR(ptr: trans);
2751	goto error_free_zone;
2752	}
2753
2754	set_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
2755	device->generation = trans->transid;
2756	device->io_width = fs_info->sectorsize;
2757	device->io_align = fs_info->sectorsize;
2758	device->sector_size = fs_info->sectorsize;
2759	device->total_bytes =
2760	round_down(bdev_nr_bytes(device->bdev), fs_info->sectorsize);
2761	device->disk_total_bytes = device->total_bytes;
2762	device->commit_total_bytes = device->total_bytes;
2763	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, addr: &device->dev_state);
2764	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, addr: &device->dev_state);
2765	device->dev_stats_valid = 1;
2766	set_blocksize(file: device->bdev_file, BTRFS_BDEV_BLOCKSIZE);
2767
2768	if (seeding_dev) {
2769	/* GFP_KERNEL allocation must not be under device_list_mutex */
2770	seed_devices = btrfs_init_sprout(fs_info);
2771	if (IS_ERR(ptr: seed_devices)) {
2772	ret = PTR_ERR(ptr: seed_devices);
2773	btrfs_abort_transaction(trans, ret);
2774	goto error_trans;
2775	}
2776	}
2777
2778	mutex_lock(&fs_devices->device_list_mutex);
2779	if (seeding_dev) {
2780	btrfs_setup_sprout(fs_info, seed_devices);
2781	btrfs_assign_next_active_device(device: fs_info->fs_devices->latest_dev,
2782	next_device: device);
2783	}
2784
2785	device->fs_devices = fs_devices;
2786
2787	mutex_lock(&fs_info->chunk_mutex);
2788	list_add_rcu(new: &device->dev_list, head: &fs_devices->devices);
2789	list_add(new: &device->dev_alloc_list, head: &fs_devices->alloc_list);
2790	fs_devices->num_devices++;
2791	fs_devices->open_devices++;
2792	fs_devices->rw_devices++;
2793	fs_devices->total_devices++;
2794	fs_devices->total_rw_bytes += device->total_bytes;
2795
2796	atomic64_add(i: device->total_bytes, v: &fs_info->free_chunk_space);
2797
2798	if (!bdev_nonrot(bdev: device->bdev))
2799	fs_devices->rotating = true;
2800
2801	orig_super_total_bytes = btrfs_super_total_bytes(s: fs_info->super_copy);
2802	btrfs_set_super_total_bytes(s: fs_info->super_copy,
2803	round_down(orig_super_total_bytes + device->total_bytes,
2804	fs_info->sectorsize));
2805
2806	orig_super_num_devices = btrfs_super_num_devices(s: fs_info->super_copy);
2807	btrfs_set_super_num_devices(s: fs_info->super_copy,
2808	val: orig_super_num_devices + 1);
2809
2810	/*
2811	* we've got more storage, clear any full flags on the space
2812	* infos
2813	*/
2814	btrfs_clear_space_info_full(info: fs_info);
2815
2816	mutex_unlock(lock: &fs_info->chunk_mutex);
2817
2818	/* Add sysfs device entry */
2819	btrfs_sysfs_add_device(device);
2820
2821	mutex_unlock(lock: &fs_devices->device_list_mutex);
2822
2823	if (seeding_dev) {
2824	mutex_lock(&fs_info->chunk_mutex);
2825	ret = init_first_rw_device(trans);
2826	mutex_unlock(lock: &fs_info->chunk_mutex);
2827	if (unlikely(ret)) {
2828	btrfs_abort_transaction(trans, ret);
2829	goto error_sysfs;
2830	}
2831	}
2832
2833	ret = btrfs_add_dev_item(trans, device);
2834	if (unlikely(ret)) {
2835	btrfs_abort_transaction(trans, ret);
2836	goto error_sysfs;
2837	}
2838
2839	if (seeding_dev) {
2840	ret = btrfs_finish_sprout(trans);
2841	if (unlikely(ret)) {
2842	btrfs_abort_transaction(trans, ret);
2843	goto error_sysfs;
2844	}
2845
2846	/*
2847	* fs_devices now represents the newly sprouted filesystem and
2848	* its fsid has been changed by btrfs_sprout_splice().
2849	*/
2850	btrfs_sysfs_update_sprout_fsid(fs_devices);
2851	}
2852
2853	ret = btrfs_commit_transaction(trans);
2854
2855	if (seeding_dev) {
2856	mutex_unlock(lock: &uuid_mutex);
2857	up_write(sem: &sb->s_umount);
2858	locked = false;
2859
2860	if (ret) /* transaction commit */
2861	return ret;
2862
2863	ret = btrfs_relocate_sys_chunks(fs_info);
2864	if (ret < 0)
2865	btrfs_handle_fs_error(fs_info, ret,
2866	"Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2867	trans = btrfs_attach_transaction(root);
2868	if (IS_ERR(ptr: trans)) {
2869	if (PTR_ERR(ptr: trans) == -ENOENT)
2870	return 0;
2871	ret = PTR_ERR(ptr: trans);
2872	trans = NULL;
2873	goto error_sysfs;
2874	}
2875	ret = btrfs_commit_transaction(trans);
2876	}
2877
2878	/*
2879	* Now that we have written a new super block to this device, check all
2880	* other fs_devices list if device_path alienates any other scanned
2881	* device.
2882	* We can ignore the return value as it typically returns -EINVAL and
2883	* only succeeds if the device was an alien.
2884	*/
2885	btrfs_forget_devices(devt: device->devt);
2886
2887	/* Update ctime/mtime for blkid or udev */
2888	update_dev_time(device_path);
2889
2890	return ret;
2891
2892	error_sysfs:
2893	btrfs_sysfs_remove_device(device);
2894	mutex_lock(&fs_info->fs_devices->device_list_mutex);
2895	mutex_lock(&fs_info->chunk_mutex);
2896	list_del_rcu(entry: &device->dev_list);
2897	list_del(entry: &device->dev_alloc_list);
2898	fs_info->fs_devices->num_devices--;
2899	fs_info->fs_devices->open_devices--;
2900	fs_info->fs_devices->rw_devices--;
2901	fs_info->fs_devices->total_devices--;
2902	fs_info->fs_devices->total_rw_bytes -= device->total_bytes;
2903	atomic64_sub(i: device->total_bytes, v: &fs_info->free_chunk_space);
2904	btrfs_set_super_total_bytes(s: fs_info->super_copy,
2905	val: orig_super_total_bytes);
2906	btrfs_set_super_num_devices(s: fs_info->super_copy,
2907	val: orig_super_num_devices);
2908	mutex_unlock(lock: &fs_info->chunk_mutex);
2909	mutex_unlock(lock: &fs_info->fs_devices->device_list_mutex);
2910	error_trans:
2911	if (trans)
2912	btrfs_end_transaction(trans);
2913	error_free_zone:
2914	btrfs_destroy_dev_zone_info(device);
2915	error_free_device:
2916	btrfs_free_device(device);
2917	error:
2918	bdev_fput(bdev_file);
2919	if (locked) {
2920	mutex_unlock(lock: &uuid_mutex);
2921	up_write(sem: &sb->s_umount);
2922	}
2923	return ret;
2924	}
2925
2926	static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
2927	struct btrfs_device *device)
2928	{
2929	int ret;
2930	BTRFS_PATH_AUTO_FREE(path);
2931	struct btrfs_root *root = device->fs_info->chunk_root;
2932	struct btrfs_dev_item *dev_item;
2933	struct extent_buffer *leaf;
2934	struct btrfs_key key;
2935
2936	path = btrfs_alloc_path();
2937	if (!path)
2938	return -ENOMEM;
2939
2940	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2941	key.type = BTRFS_DEV_ITEM_KEY;
2942	key.offset = device->devid;
2943
2944	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: 0, cow: 1);
2945	if (ret < 0)
2946	return ret;
2947
2948	if (ret > 0)
2949	return -ENOENT;
2950
2951	leaf = path->nodes[0];
2952	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2953
2954	btrfs_set_device_id(eb: leaf, s: dev_item, val: device->devid);
2955	btrfs_set_device_type(eb: leaf, s: dev_item, val: device->type);
2956	btrfs_set_device_io_align(eb: leaf, s: dev_item, val: device->io_align);
2957	btrfs_set_device_io_width(eb: leaf, s: dev_item, val: device->io_width);
2958	btrfs_set_device_sector_size(eb: leaf, s: dev_item, val: device->sector_size);
2959	btrfs_set_device_total_bytes(eb: leaf, s: dev_item,
2960	val: btrfs_device_get_disk_total_bytes(dev: device));
2961	btrfs_set_device_bytes_used(eb: leaf, s: dev_item,
2962	val: btrfs_device_get_bytes_used(dev: device));
2963	return ret;
2964	}
2965
2966	int btrfs_grow_device(struct btrfs_trans_handle *trans,
2967	struct btrfs_device *device, u64 new_size)
2968	{
2969	struct btrfs_fs_info *fs_info = device->fs_info;
2970	struct btrfs_super_block *super_copy = fs_info->super_copy;
2971	u64 old_total;
2972	u64 diff;
2973	int ret;
2974
2975	if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2976	return -EACCES;
2977
2978	new_size = round_down(new_size, fs_info->sectorsize);
2979
2980	mutex_lock(&fs_info->chunk_mutex);
2981	old_total = btrfs_super_total_bytes(s: super_copy);
2982	diff = round_down(new_size - device->total_bytes, fs_info->sectorsize);
2983
2984	if (new_size <= device->total_bytes ||
2985	test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2986	mutex_unlock(lock: &fs_info->chunk_mutex);
2987	return -EINVAL;
2988	}
2989
2990	btrfs_set_super_total_bytes(s: super_copy,
2991	round_down(old_total + diff, fs_info->sectorsize));
2992	device->fs_devices->total_rw_bytes += diff;
2993	atomic64_add(i: diff, v: &fs_info->free_chunk_space);
2994
2995	btrfs_device_set_total_bytes(dev: device, size: new_size);
2996	btrfs_device_set_disk_total_bytes(dev: device, size: new_size);
2997	btrfs_clear_space_info_full(info: device->fs_info);
2998	if (list_empty(head: &device->post_commit_list))
2999	list_add_tail(new: &device->post_commit_list,
3000	head: &trans->transaction->dev_update_list);
3001	mutex_unlock(lock: &fs_info->chunk_mutex);
3002
3003	btrfs_reserve_chunk_metadata(trans, is_item_insertion: false);
3004	ret = btrfs_update_device(trans, device);
3005	btrfs_trans_release_chunk_metadata(trans);
3006
3007	return ret;
3008	}
3009
3010	static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
3011	{
3012	struct btrfs_fs_info *fs_info = trans->fs_info;
3013	struct btrfs_root *root = fs_info->chunk_root;
3014	int ret;
3015	BTRFS_PATH_AUTO_FREE(path);
3016	struct btrfs_key key;
3017
3018	path = btrfs_alloc_path();
3019	if (!path)
3020	return -ENOMEM;
3021
3022	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3023	key.type = BTRFS_CHUNK_ITEM_KEY;
3024	key.offset = chunk_offset;
3025
3026	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: -1, cow: 1);
3027	if (ret < 0)
3028	return ret;
3029	if (unlikely(ret > 0)) {
3030	/* Logic error or corruption */
3031	btrfs_err(fs_info, "failed to lookup chunk %llu when freeing",
3032	chunk_offset);
3033	btrfs_abort_transaction(trans, -ENOENT);
3034	return -EUCLEAN;
3035	}
3036
3037	ret = btrfs_del_item(trans, root, path);
3038	if (unlikely(ret < 0)) {
3039	btrfs_err(fs_info, "failed to delete chunk %llu item", chunk_offset);
3040	btrfs_abort_transaction(trans, ret);
3041	return ret;
3042	}
3043	return ret;
3044	}
3045
3046	static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3047	{
3048	struct btrfs_super_block *super_copy = fs_info->super_copy;
3049	struct btrfs_disk_key *disk_key;
3050	struct btrfs_chunk *chunk;
3051	u8 *ptr;
3052	int ret = 0;
3053	u32 num_stripes;
3054	u32 array_size;
3055	u32 len = 0;
3056	u32 cur;
3057	struct btrfs_key key;
3058
3059	lockdep_assert_held(&fs_info->chunk_mutex);
3060	array_size = btrfs_super_sys_array_size(s: super_copy);
3061
3062	ptr = super_copy->sys_chunk_array;
3063	cur = 0;
3064
3065	while (cur < array_size) {
3066	disk_key = (struct btrfs_disk_key *)ptr;
3067	btrfs_disk_key_to_cpu(cpu_key: &key, disk_key);
3068
3069	len = sizeof(*disk_key);
3070
3071	if (key.type == BTRFS_CHUNK_ITEM_KEY) {
3072	chunk = (struct btrfs_chunk *)(ptr + len);
3073	num_stripes = btrfs_stack_chunk_num_stripes(s: chunk);
3074	len += btrfs_chunk_item_size(num_stripes);
3075	} else {
3076	ret = -EIO;
3077	break;
3078	}
3079	if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID &&
3080	key.offset == chunk_offset) {
3081	memmove(ptr, ptr + len, array_size - (cur + len));
3082	array_size -= len;
3083	btrfs_set_super_sys_array_size(s: super_copy, val: array_size);
3084	} else {
3085	ptr += len;
3086	cur += len;
3087	}
3088	}
3089	return ret;
3090	}
3091
3092	struct btrfs_chunk_map *btrfs_find_chunk_map_nolock(struct btrfs_fs_info *fs_info,
3093	u64 logical, u64 length)
3094	{
3095	struct rb_node *node = fs_info->mapping_tree.rb_root.rb_node;
3096	struct rb_node *prev = NULL;
3097	struct rb_node *orig_prev;
3098	struct btrfs_chunk_map *map;
3099	struct btrfs_chunk_map *prev_map = NULL;
3100
3101	while (node) {
3102	map = rb_entry(node, struct btrfs_chunk_map, rb_node);
3103	prev = node;
3104	prev_map = map;
3105
3106	if (logical < map->start) {
3107	node = node->rb_left;
3108	} else if (logical >= map->start + map->chunk_len) {
3109	node = node->rb_right;
3110	} else {
3111	refcount_inc(r: &map->refs);
3112	return map;
3113	}
3114	}
3115
3116	if (!prev)
3117	return NULL;
3118
3119	orig_prev = prev;
3120	while (prev && logical >= prev_map->start + prev_map->chunk_len) {
3121	prev = rb_next(prev);
3122	prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node);
3123	}
3124
3125	if (!prev) {
3126	prev = orig_prev;
3127	prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node);
3128	while (prev && logical < prev_map->start) {
3129	prev = rb_prev(prev);
3130	prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node);
3131	}
3132	}
3133
3134	if (prev) {
3135	u64 end = logical + length;
3136
3137	/*
3138	* Caller can pass a U64_MAX length when it wants to get any
3139	* chunk starting at an offset of 'logical' or higher, so deal
3140	* with underflow by resetting the end offset to U64_MAX.
3141	*/
3142	if (end < logical)
3143	end = U64_MAX;
3144
3145	if (end > prev_map->start &&
3146	logical < prev_map->start + prev_map->chunk_len) {
3147	refcount_inc(r: &prev_map->refs);
3148	return prev_map;
3149	}
3150	}
3151
3152	return NULL;
3153	}
3154
3155	struct btrfs_chunk_map *btrfs_find_chunk_map(struct btrfs_fs_info *fs_info,
3156	u64 logical, u64 length)
3157	{
3158	struct btrfs_chunk_map *map;
3159
3160	read_lock(&fs_info->mapping_tree_lock);
3161	map = btrfs_find_chunk_map_nolock(fs_info, logical, length);
3162	read_unlock(&fs_info->mapping_tree_lock);
3163
3164	return map;
3165	}
3166
3167	/*
3168	* Find the mapping containing the given logical extent.
3169	*
3170	* @logical: Logical block offset in bytes.
3171	* @length: Length of extent in bytes.
3172	*
3173	* Return: Chunk mapping or ERR_PTR.
3174	*/
3175	struct btrfs_chunk_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info,
3176	u64 logical, u64 length)
3177	{
3178	struct btrfs_chunk_map *map;
3179
3180	map = btrfs_find_chunk_map(fs_info, logical, length);
3181
3182	if (unlikely(!map)) {
3183	btrfs_crit(fs_info,
3184	"unable to find chunk map for logical %llu length %llu",
3185	logical, length);
3186	return ERR_PTR(error: -EINVAL);
3187	}
3188
3189	if (unlikely(map->start > logical || map->start + map->chunk_len <= logical)) {
3190	btrfs_crit(fs_info,
3191	"found a bad chunk map, wanted %llu-%llu, found %llu-%llu",
3192	logical, logical + length, map->start,
3193	map->start + map->chunk_len);
3194	btrfs_free_chunk_map(map);
3195	return ERR_PTR(error: -EINVAL);
3196	}
3197
3198	/* Callers are responsible for dropping the reference. */
3199	return map;
3200	}
3201
3202	static int remove_chunk_item(struct btrfs_trans_handle *trans,
3203	struct btrfs_chunk_map *map, u64 chunk_offset)
3204	{
3205	int i;
3206
3207	/*
3208	* Removing chunk items and updating the device items in the chunks btree
3209	* requires holding the chunk_mutex.
3210	* See the comment at btrfs_chunk_alloc() for the details.
3211	*/
3212	lockdep_assert_held(&trans->fs_info->chunk_mutex);
3213
3214	for (i = 0; i < map->num_stripes; i++) {
3215	int ret;
3216
3217	ret = btrfs_update_device(trans, device: map->stripes[i].dev);
3218	if (ret)
3219	return ret;
3220	}
3221
3222	return btrfs_free_chunk(trans, chunk_offset);
3223	}
3224
3225	int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
3226	{
3227	struct btrfs_fs_info *fs_info = trans->fs_info;
3228	struct btrfs_chunk_map *map;
3229	u64 dev_extent_len = 0;
3230	int i, ret = 0;
3231	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
3232
3233	map = btrfs_get_chunk_map(fs_info, logical: chunk_offset, length: 1);
3234	if (IS_ERR(ptr: map)) {
3235	/*
3236	* This is a logic error, but we don't want to just rely on the
3237	* user having built with ASSERT enabled, so if ASSERT doesn't
3238	* do anything we still error out.
3239	*/
3240	DEBUG_WARN("errr %ld reading chunk map at offset %llu",
3241	PTR_ERR(map), chunk_offset);
3242	return PTR_ERR(ptr: map);
3243	}
3244
3245	/*
3246	* First delete the device extent items from the devices btree.
3247	* We take the device_list_mutex to avoid racing with the finishing phase
3248	* of a device replace operation. See the comment below before acquiring
3249	* fs_info->chunk_mutex. Note that here we do not acquire the chunk_mutex
3250	* because that can result in a deadlock when deleting the device extent
3251	* items from the devices btree - COWing an extent buffer from the btree
3252	* may result in allocating a new metadata chunk, which would attempt to
3253	* lock again fs_info->chunk_mutex.
3254	*/
3255	mutex_lock(&fs_devices->device_list_mutex);
3256	for (i = 0; i < map->num_stripes; i++) {
3257	struct btrfs_device *device = map->stripes[i].dev;
3258	ret = btrfs_free_dev_extent(trans, device,
3259	start: map->stripes[i].physical,
3260	dev_extent_len: &dev_extent_len);
3261	if (unlikely(ret)) {
3262	mutex_unlock(lock: &fs_devices->device_list_mutex);
3263	btrfs_abort_transaction(trans, ret);
3264	goto out;
3265	}
3266
3267	if (device->bytes_used > 0) {
3268	mutex_lock(&fs_info->chunk_mutex);
3269	btrfs_device_set_bytes_used(dev: device,
3270	size: device->bytes_used - dev_extent_len);
3271	atomic64_add(i: dev_extent_len, v: &fs_info->free_chunk_space);
3272	btrfs_clear_space_info_full(info: fs_info);
3273
3274	if (list_empty(head: &device->post_commit_list)) {
3275	list_add_tail(new: &device->post_commit_list,
3276	head: &trans->transaction->dev_update_list);
3277	}
3278
3279	mutex_unlock(lock: &fs_info->chunk_mutex);
3280	}
3281	}
3282	mutex_unlock(lock: &fs_devices->device_list_mutex);
3283
3284	/*
3285	* We acquire fs_info->chunk_mutex for 2 reasons:
3286	*
3287	* 1) Just like with the first phase of the chunk allocation, we must
3288	* reserve system space, do all chunk btree updates and deletions, and
3289	* update the system chunk array in the superblock while holding this
3290	* mutex. This is for similar reasons as explained on the comment at
3291	* the top of btrfs_chunk_alloc();
3292	*
3293	* 2) Prevent races with the final phase of a device replace operation
3294	* that replaces the device object associated with the map's stripes,
3295	* because the device object's id can change at any time during that
3296	* final phase of the device replace operation
3297	* (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the
3298	* replaced device and then see it with an ID of
3299	* BTRFS_DEV_REPLACE_DEVID, which would cause a failure when updating
3300	* the device item, which does not exists on the chunk btree.
3301	* The finishing phase of device replace acquires both the
3302	* device_list_mutex and the chunk_mutex, in that order, so we are
3303	* safe by just acquiring the chunk_mutex.
3304	*/
3305	trans->removing_chunk = true;
3306	mutex_lock(&fs_info->chunk_mutex);
3307
3308	check_system_chunk(trans, type: map->type);
3309
3310	ret = remove_chunk_item(trans, map, chunk_offset);
3311	/*
3312	* Normally we should not get -ENOSPC since we reserved space before
3313	* through the call to check_system_chunk().
3314	*
3315	* Despite our system space_info having enough free space, we may not
3316	* be able to allocate extents from its block groups, because all have
3317	* an incompatible profile, which will force us to allocate a new system
3318	* block group with the right profile, or right after we called
3319	* check_system_space() above, a scrub turned the only system block group
3320	* with enough free space into RO mode.
3321	* This is explained with more detail at do_chunk_alloc().
3322	*
3323	* So if we get -ENOSPC, allocate a new system chunk and retry once.
3324	*/
3325	if (ret == -ENOSPC) {
3326	const u64 sys_flags = btrfs_system_alloc_profile(fs_info);
3327	struct btrfs_block_group *sys_bg;
3328	struct btrfs_space_info *space_info;
3329
3330	space_info = btrfs_find_space_info(info: fs_info, flags: sys_flags);
3331	if (unlikely(!space_info)) {
3332	ret = -EINVAL;
3333	btrfs_abort_transaction(trans, ret);
3334	goto out;
3335	}
3336
3337	sys_bg = btrfs_create_chunk(trans, space_info, type: sys_flags);
3338	if (IS_ERR(ptr: sys_bg)) {
3339	ret = PTR_ERR(ptr: sys_bg);
3340	btrfs_abort_transaction(trans, ret);
3341	goto out;
3342	}
3343
3344	ret = btrfs_chunk_alloc_add_chunk_item(trans, bg: sys_bg);
3345	if (unlikely(ret)) {
3346	btrfs_abort_transaction(trans, ret);
3347	goto out;
3348	}
3349
3350	ret = remove_chunk_item(trans, map, chunk_offset);
3351	if (unlikely(ret)) {
3352	btrfs_abort_transaction(trans, ret);
3353	goto out;
3354	}
3355	} else if (unlikely(ret)) {
3356	btrfs_abort_transaction(trans, ret);
3357	goto out;
3358	}
3359
3360	trace_btrfs_chunk_free(fs_info, map, offset: chunk_offset, size: map->chunk_len);
3361
3362	if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
3363	ret = btrfs_del_sys_chunk(fs_info, chunk_offset);
3364	if (unlikely(ret)) {
3365	btrfs_abort_transaction(trans, ret);
3366	goto out;
3367	}
3368	}
3369
3370	mutex_unlock(lock: &fs_info->chunk_mutex);
3371	trans->removing_chunk = false;
3372
3373	/*
3374	* We are done with chunk btree updates and deletions, so release the
3375	* system space we previously reserved (with check_system_chunk()).
3376	*/
3377	btrfs_trans_release_chunk_metadata(trans);
3378
3379	ret = btrfs_remove_block_group(trans, map);
3380	if (unlikely(ret)) {
3381	btrfs_abort_transaction(trans, ret);
3382	goto out;
3383	}
3384
3385	out:
3386	if (trans->removing_chunk) {
3387	mutex_unlock(lock: &fs_info->chunk_mutex);
3388	trans->removing_chunk = false;
3389	}
3390	/* once for us */
3391	btrfs_free_chunk_map(map);
3392	return ret;
3393	}
3394
3395	int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset,
3396	bool verbose)
3397	{
3398	struct btrfs_root *root = fs_info->chunk_root;
3399	struct btrfs_trans_handle *trans;
3400	struct btrfs_block_group *block_group;
3401	u64 length;
3402	int ret;
3403
3404	if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
3405	btrfs_err(fs_info,
3406	"relocate: not supported on extent tree v2 yet");
3407	return -EINVAL;
3408	}
3409
3410	/*
3411	* Prevent races with automatic removal of unused block groups.
3412	* After we relocate and before we remove the chunk with offset
3413	* chunk_offset, automatic removal of the block group can kick in,
3414	* resulting in a failure when calling btrfs_remove_chunk() below.
3415	*
3416	* Make sure to acquire this mutex before doing a tree search (dev
3417	* or chunk trees) to find chunks. Otherwise the cleaner kthread might
3418	* call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
3419	* we release the path used to search the chunk/dev tree and before
3420	* the current task acquires this mutex and calls us.
3421	*/
3422	lockdep_assert_held(&fs_info->reclaim_bgs_lock);
3423
3424	/* step one, relocate all the extents inside this chunk */
3425	btrfs_scrub_pause(fs_info);
3426	ret = btrfs_relocate_block_group(fs_info, group_start: chunk_offset, verbose: true);
3427	btrfs_scrub_continue(fs_info);
3428	if (ret) {
3429	/*
3430	* If we had a transaction abort, stop all running scrubs.
3431	* See transaction.c:cleanup_transaction() why we do it here.
3432	*/
3433	if (BTRFS_FS_ERROR(fs_info))
3434	btrfs_scrub_cancel(info: fs_info);
3435	return ret;
3436	}
3437
3438	block_group = btrfs_lookup_block_group(info: fs_info, bytenr: chunk_offset);
3439	if (!block_group)
3440	return -ENOENT;
3441	btrfs_discard_cancel_work(discard_ctl: &fs_info->discard_ctl, block_group);
3442	length = block_group->length;
3443	btrfs_put_block_group(cache: block_group);
3444
3445	/*
3446	* On a zoned file system, discard the whole block group, this will
3447	* trigger a REQ_OP_ZONE_RESET operation on the device zone. If
3448	* resetting the zone fails, don't treat it as a fatal problem from the
3449	* filesystem's point of view.
3450	*/
3451	if (btrfs_is_zoned(fs_info)) {
3452	ret = btrfs_discard_extent(fs_info, bytenr: chunk_offset, num_bytes: length, NULL);
3453	if (ret)
3454	btrfs_info(fs_info,
3455	"failed to reset zone %llu after relocation",
3456	chunk_offset);
3457	}
3458
3459	trans = btrfs_start_trans_remove_block_group(fs_info: root->fs_info,
3460	chunk_offset);
3461	if (IS_ERR(ptr: trans)) {
3462	ret = PTR_ERR(ptr: trans);
3463	btrfs_handle_fs_error(root->fs_info, ret, NULL);
3464	return ret;
3465	}
3466
3467	/*
3468	* step two, delete the device extents and the
3469	* chunk tree entries
3470	*/
3471	ret = btrfs_remove_chunk(trans, chunk_offset);
3472	btrfs_end_transaction(trans);
3473	return ret;
3474	}
3475
3476	static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
3477	{
3478	struct btrfs_root *chunk_root = fs_info->chunk_root;
3479	BTRFS_PATH_AUTO_FREE(path);
3480	struct extent_buffer *leaf;
3481	struct btrfs_chunk *chunk;
3482	struct btrfs_key key;
3483	struct btrfs_key found_key;
3484	u64 chunk_type;
3485	bool retried = false;
3486	int failed = 0;
3487	int ret;
3488
3489	path = btrfs_alloc_path();
3490	if (!path)
3491	return -ENOMEM;
3492
3493	again:
3494	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3495	key.type = BTRFS_CHUNK_ITEM_KEY;
3496	key.offset = (u64)-1;
3497
3498	while (1) {
3499	mutex_lock(&fs_info->reclaim_bgs_lock);
3500	ret = btrfs_search_slot(NULL, root: chunk_root, key: &key, p: path, ins_len: 0, cow: 0);
3501	if (ret < 0) {
3502	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
3503	return ret;
3504	}
3505	if (unlikely(ret == 0)) {
3506	/*
3507	* On the first search we would find chunk tree with
3508	* offset -1, which is not possible. On subsequent
3509	* loops this would find an existing item on an invalid
3510	* offset (one less than the previous one, wrong
3511	* alignment and size).
3512	*/
3513	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
3514	return -EUCLEAN;
3515	}
3516
3517	ret = btrfs_previous_item(root: chunk_root, path, min_objectid: key.objectid,
3518	type: key.type);
3519	if (ret)
3520	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
3521	if (ret < 0)
3522	return ret;
3523	if (ret > 0)
3524	break;
3525
3526	leaf = path->nodes[0];
3527	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
3528
3529	chunk = btrfs_item_ptr(leaf, path->slots[0],
3530	struct btrfs_chunk);
3531	chunk_type = btrfs_chunk_type(eb: leaf, s: chunk);
3532	btrfs_release_path(p: path);
3533
3534	if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
3535	ret = btrfs_relocate_chunk(fs_info, chunk_offset: found_key.offset,
3536	verbose: true);
3537	if (ret == -ENOSPC)
3538	failed++;
3539	else
3540	BUG_ON(ret);
3541	}
3542	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
3543
3544	if (found_key.offset == 0)
3545	break;
3546	key.offset = found_key.offset - 1;
3547	}
3548	ret = 0;
3549	if (failed && !retried) {
3550	failed = 0;
3551	retried = true;
3552	goto again;
3553	} else if (WARN_ON(failed && retried)) {
3554	ret = -ENOSPC;
3555	}
3556	return ret;
3557	}
3558
3559	/*
3560	* return 1 : allocate a data chunk successfully,
3561	* return <0: errors during allocating a data chunk,
3562	* return 0 : no need to allocate a data chunk.
3563	*/
3564	static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info,
3565	u64 chunk_offset)
3566	{
3567	struct btrfs_block_group *cache;
3568	u64 bytes_used;
3569	u64 chunk_type;
3570
3571	cache = btrfs_lookup_block_group(info: fs_info, bytenr: chunk_offset);
3572	ASSERT(cache);
3573	chunk_type = cache->flags;
3574	btrfs_put_block_group(cache);
3575
3576	if (!(chunk_type & BTRFS_BLOCK_GROUP_DATA))
3577	return 0;
3578
3579	spin_lock(lock: &fs_info->data_sinfo->lock);
3580	bytes_used = fs_info->data_sinfo->bytes_used;
3581	spin_unlock(lock: &fs_info->data_sinfo->lock);
3582
3583	if (!bytes_used) {
3584	struct btrfs_trans_handle *trans;
3585	int ret;
3586
3587	trans = btrfs_join_transaction(root: fs_info->tree_root);
3588	if (IS_ERR(ptr: trans))
3589	return PTR_ERR(ptr: trans);
3590
3591	ret = btrfs_force_chunk_alloc(trans, BTRFS_BLOCK_GROUP_DATA);
3592	btrfs_end_transaction(trans);
3593	if (ret < 0)
3594	return ret;
3595	return 1;
3596	}
3597
3598	return 0;
3599	}
3600
3601	static void btrfs_disk_balance_args_to_cpu(struct btrfs_balance_args *cpu,
3602	const struct btrfs_disk_balance_args *disk)
3603	{
3604	memset(cpu, 0, sizeof(*cpu));
3605
3606	cpu->profiles = le64_to_cpu(disk->profiles);
3607	cpu->usage = le64_to_cpu(disk->usage);
3608	cpu->devid = le64_to_cpu(disk->devid);
3609	cpu->pstart = le64_to_cpu(disk->pstart);
3610	cpu->pend = le64_to_cpu(disk->pend);
3611	cpu->vstart = le64_to_cpu(disk->vstart);
3612	cpu->vend = le64_to_cpu(disk->vend);
3613	cpu->target = le64_to_cpu(disk->target);
3614	cpu->flags = le64_to_cpu(disk->flags);
3615	cpu->limit = le64_to_cpu(disk->limit);
3616	cpu->stripes_min = le32_to_cpu(disk->stripes_min);
3617	cpu->stripes_max = le32_to_cpu(disk->stripes_max);
3618	}
3619
3620	static void btrfs_cpu_balance_args_to_disk(struct btrfs_disk_balance_args *disk,
3621	const struct btrfs_balance_args *cpu)
3622	{
3623	memset(disk, 0, sizeof(*disk));
3624
3625	disk->profiles = cpu_to_le64(cpu->profiles);
3626	disk->usage = cpu_to_le64(cpu->usage);
3627	disk->devid = cpu_to_le64(cpu->devid);
3628	disk->pstart = cpu_to_le64(cpu->pstart);
3629	disk->pend = cpu_to_le64(cpu->pend);
3630	disk->vstart = cpu_to_le64(cpu->vstart);
3631	disk->vend = cpu_to_le64(cpu->vend);
3632	disk->target = cpu_to_le64(cpu->target);
3633	disk->flags = cpu_to_le64(cpu->flags);
3634	disk->limit = cpu_to_le64(cpu->limit);
3635	disk->stripes_min = cpu_to_le32(cpu->stripes_min);
3636	disk->stripes_max = cpu_to_le32(cpu->stripes_max);
3637	}
3638
3639	static int insert_balance_item(struct btrfs_fs_info *fs_info,
3640	struct btrfs_balance_control *bctl)
3641	{
3642	struct btrfs_root *root = fs_info->tree_root;
3643	struct btrfs_trans_handle *trans;
3644	struct btrfs_balance_item *item;
3645	struct btrfs_disk_balance_args disk_bargs;
3646	struct btrfs_path *path;
3647	struct extent_buffer *leaf;
3648	struct btrfs_key key;
3649	int ret, err;
3650
3651	path = btrfs_alloc_path();
3652	if (!path)
3653	return -ENOMEM;
3654
3655	trans = btrfs_start_transaction(root, num_items: 0);
3656	if (IS_ERR(ptr: trans)) {
3657	btrfs_free_path(p: path);
3658	return PTR_ERR(ptr: trans);
3659	}
3660
3661	key.objectid = BTRFS_BALANCE_OBJECTID;
3662	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3663	key.offset = 0;
3664
3665	ret = btrfs_insert_empty_item(trans, root, path, key: &key,
3666	data_size: sizeof(*item));
3667	if (ret)
3668	goto out;
3669
3670	leaf = path->nodes[0];
3671	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
3672
3673	memzero_extent_buffer(eb: leaf, start: (unsigned long)item, len: sizeof(*item));
3674
3675	btrfs_cpu_balance_args_to_disk(disk: &disk_bargs, cpu: &bctl->data);
3676	btrfs_set_balance_data(eb: leaf, bi: item, ba: &disk_bargs);
3677	btrfs_cpu_balance_args_to_disk(disk: &disk_bargs, cpu: &bctl->meta);
3678	btrfs_set_balance_meta(eb: leaf, bi: item, ba: &disk_bargs);
3679	btrfs_cpu_balance_args_to_disk(disk: &disk_bargs, cpu: &bctl->sys);
3680	btrfs_set_balance_sys(eb: leaf, bi: item, ba: &disk_bargs);
3681	btrfs_set_balance_flags(eb: leaf, s: item, val: bctl->flags);
3682	out:
3683	btrfs_free_path(p: path);
3684	err = btrfs_commit_transaction(trans);
3685	if (err && !ret)
3686	ret = err;
3687	return ret;
3688	}
3689
3690	static int del_balance_item(struct btrfs_fs_info *fs_info)
3691	{
3692	struct btrfs_root *root = fs_info->tree_root;
3693	struct btrfs_trans_handle *trans;
3694	struct btrfs_path *path;
3695	struct btrfs_key key;
3696	int ret, err;
3697
3698	path = btrfs_alloc_path();
3699	if (!path)
3700	return -ENOMEM;
3701
3702	trans = btrfs_start_transaction_fallback_global_rsv(root, num_items: 0);
3703	if (IS_ERR(ptr: trans)) {
3704	btrfs_free_path(p: path);
3705	return PTR_ERR(ptr: trans);
3706	}
3707
3708	key.objectid = BTRFS_BALANCE_OBJECTID;
3709	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3710	key.offset = 0;
3711
3712	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: -1, cow: 1);
3713	if (ret < 0)
3714	goto out;
3715	if (ret > 0) {
3716	ret = -ENOENT;
3717	goto out;
3718	}
3719
3720	ret = btrfs_del_item(trans, root, path);
3721	out:
3722	btrfs_free_path(p: path);
3723	err = btrfs_commit_transaction(trans);
3724	if (err && !ret)
3725	ret = err;
3726	return ret;
3727	}
3728
3729	/*
3730	* This is a heuristic used to reduce the number of chunks balanced on
3731	* resume after balance was interrupted.
3732	*/
3733	static void update_balance_args(struct btrfs_balance_control *bctl)
3734	{
3735	/*
3736	* Turn on soft mode for chunk types that were being converted.
3737	*/
3738	if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)
3739	bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT;
3740	if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)
3741	bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT;
3742	if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)
3743	bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT;
3744
3745	/*
3746	* Turn on usage filter if is not already used. The idea is
3747	* that chunks that we have already balanced should be
3748	* reasonably full. Don't do it for chunks that are being
3749	* converted - that will keep us from relocating unconverted
3750	* (albeit full) chunks.
3751	*/
3752	if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3753	!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3754	!(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3755	bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE;
3756	bctl->data.usage = 90;
3757	}
3758	if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3759	!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3760	!(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3761	bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE;
3762	bctl->sys.usage = 90;
3763	}
3764	if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3765	!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3766	!(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3767	bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE;
3768	bctl->meta.usage = 90;
3769	}
3770	}
3771
3772	/*
3773	* Clear the balance status in fs_info and delete the balance item from disk.
3774	*/
3775	static void reset_balance_state(struct btrfs_fs_info *fs_info)
3776	{
3777	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3778	int ret;
3779
3780	ASSERT(fs_info->balance_ctl);
3781
3782	spin_lock(lock: &fs_info->balance_lock);
3783	fs_info->balance_ctl = NULL;
3784	spin_unlock(lock: &fs_info->balance_lock);
3785
3786	kfree(objp: bctl);
3787	ret = del_balance_item(fs_info);
3788	if (ret)
3789	btrfs_handle_fs_error(fs_info, ret, NULL);
3790	}
3791
3792	/*
3793	* Balance filters. Return 1 if chunk should be filtered out
3794	* (should not be balanced).
3795	*/
3796	static bool chunk_profiles_filter(u64 chunk_type, struct btrfs_balance_args *bargs)
3797	{
3798	chunk_type = chunk_to_extended(flags: chunk_type) &
3799	BTRFS_EXTENDED_PROFILE_MASK;
3800
3801	if (bargs->profiles & chunk_type)
3802	return false;
3803
3804	return true;
3805	}
3806
3807	static bool chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset,
3808	struct btrfs_balance_args *bargs)
3809	{
3810	struct btrfs_block_group *cache;
3811	u64 chunk_used;
3812	u64 user_thresh_min;
3813	u64 user_thresh_max;
3814	bool ret = true;
3815
3816	cache = btrfs_lookup_block_group(info: fs_info, bytenr: chunk_offset);
3817	chunk_used = cache->used;
3818
3819	if (bargs->usage_min == 0)
3820	user_thresh_min = 0;
3821	else
3822	user_thresh_min = mult_perc(num: cache->length, percent: bargs->usage_min);
3823
3824	if (bargs->usage_max == 0)
3825	user_thresh_max = 1;
3826	else if (bargs->usage_max > 100)
3827	user_thresh_max = cache->length;
3828	else
3829	user_thresh_max = mult_perc(num: cache->length, percent: bargs->usage_max);
3830
3831	if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max)
3832	ret = false;
3833
3834	btrfs_put_block_group(cache);
3835	return ret;
3836	}
3837
3838	static bool chunk_usage_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset,
3839	struct btrfs_balance_args *bargs)
3840	{
3841	struct btrfs_block_group *cache;
3842	u64 chunk_used, user_thresh;
3843	bool ret = true;
3844
3845	cache = btrfs_lookup_block_group(info: fs_info, bytenr: chunk_offset);
3846	chunk_used = cache->used;
3847
3848	if (bargs->usage_min == 0)
3849	user_thresh = 1;
3850	else if (bargs->usage > 100)
3851	user_thresh = cache->length;
3852	else
3853	user_thresh = mult_perc(num: cache->length, percent: bargs->usage);
3854
3855	if (chunk_used < user_thresh)
3856	ret = false;
3857
3858	btrfs_put_block_group(cache);
3859	return ret;
3860	}
3861
3862	static bool chunk_devid_filter(struct extent_buffer *leaf, struct btrfs_chunk *chunk,
3863	struct btrfs_balance_args *bargs)
3864	{
3865	struct btrfs_stripe *stripe;
3866	int num_stripes = btrfs_chunk_num_stripes(eb: leaf, s: chunk);
3867	int i;
3868
3869	for (i = 0; i < num_stripes; i++) {
3870	stripe = btrfs_stripe_nr(c: chunk, nr: i);
3871	if (btrfs_stripe_devid(eb: leaf, s: stripe) == bargs->devid)
3872	return false;
3873	}
3874
3875	return true;
3876	}
3877
3878	static u64 calc_data_stripes(u64 type, int num_stripes)
3879	{
3880	const int index = btrfs_bg_flags_to_raid_index(flags: type);
3881	const int ncopies = btrfs_raid_array[index].ncopies;
3882	const int nparity = btrfs_raid_array[index].nparity;
3883
3884	return (num_stripes - nparity) / ncopies;
3885	}
3886
3887	/* [pstart, pend) */
3888	static bool chunk_drange_filter(struct extent_buffer *leaf, struct btrfs_chunk *chunk,
3889	struct btrfs_balance_args *bargs)
3890	{
3891	struct btrfs_stripe *stripe;
3892	int num_stripes = btrfs_chunk_num_stripes(eb: leaf, s: chunk);
3893	u64 stripe_offset;
3894	u64 stripe_length;
3895	u64 type;
3896	int factor;
3897	int i;
3898
3899	if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID))
3900	return false;
3901
3902	type = btrfs_chunk_type(eb: leaf, s: chunk);
3903	factor = calc_data_stripes(type, num_stripes);
3904
3905	for (i = 0; i < num_stripes; i++) {
3906	stripe = btrfs_stripe_nr(c: chunk, nr: i);
3907	if (btrfs_stripe_devid(eb: leaf, s: stripe) != bargs->devid)
3908	continue;
3909
3910	stripe_offset = btrfs_stripe_offset(eb: leaf, s: stripe);
3911	stripe_length = btrfs_chunk_length(eb: leaf, s: chunk);
3912	stripe_length = div_u64(dividend: stripe_length, divisor: factor);
3913
3914	if (stripe_offset < bargs->pend &&
3915	stripe_offset + stripe_length > bargs->pstart)
3916	return false;
3917	}
3918
3919	return true;
3920	}
3921
3922	/* [vstart, vend) */
3923	static bool chunk_vrange_filter(struct extent_buffer *leaf, struct btrfs_chunk *chunk,
3924	u64 chunk_offset, struct btrfs_balance_args *bargs)
3925	{
3926	if (chunk_offset < bargs->vend &&
3927	chunk_offset + btrfs_chunk_length(eb: leaf, s: chunk) > bargs->vstart)
3928	/* at least part of the chunk is inside this vrange */
3929	return false;
3930
3931	return true;
3932	}
3933
3934	static bool chunk_stripes_range_filter(struct extent_buffer *leaf,
3935	struct btrfs_chunk *chunk,
3936	struct btrfs_balance_args *bargs)
3937	{
3938	int num_stripes = btrfs_chunk_num_stripes(eb: leaf, s: chunk);
3939
3940	if (bargs->stripes_min <= num_stripes
3941	&& num_stripes <= bargs->stripes_max)
3942	return false;
3943
3944	return true;
3945	}
3946
3947	static bool chunk_soft_convert_filter(u64 chunk_type, struct btrfs_balance_args *bargs)
3948	{
3949	if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
3950	return false;
3951
3952	chunk_type = chunk_to_extended(flags: chunk_type) &
3953	BTRFS_EXTENDED_PROFILE_MASK;
3954
3955	if (bargs->target == chunk_type)
3956	return true;
3957
3958	return false;
3959	}
3960
3961	static bool should_balance_chunk(struct extent_buffer *leaf, struct btrfs_chunk *chunk,
3962	u64 chunk_offset)
3963	{
3964	struct btrfs_fs_info *fs_info = leaf->fs_info;
3965	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3966	struct btrfs_balance_args *bargs = NULL;
3967	u64 chunk_type = btrfs_chunk_type(eb: leaf, s: chunk);
3968
3969	/* type filter */
3970	if (!((chunk_type & BTRFS_BLOCK_GROUP_TYPE_MASK) &
3971	(bctl->flags & BTRFS_BALANCE_TYPE_MASK))) {
3972	return false;
3973	}
3974
3975	if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
3976	bargs = &bctl->data;
3977	else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
3978	bargs = &bctl->sys;
3979	else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
3980	bargs = &bctl->meta;
3981
3982	/* profiles filter */
3983	if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) &&
3984	chunk_profiles_filter(chunk_type, bargs)) {
3985	return false;
3986	}
3987
3988	/* usage filter */
3989	if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) &&
3990	chunk_usage_filter(fs_info, chunk_offset, bargs)) {
3991	return false;
3992	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3993	chunk_usage_range_filter(fs_info, chunk_offset, bargs)) {
3994	return false;
3995	}
3996
3997	/* devid filter */
3998	if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) &&
3999	chunk_devid_filter(leaf, chunk, bargs)) {
4000	return false;
4001	}
4002
4003	/* drange filter, makes sense only with devid filter */
4004	if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) &&
4005	chunk_drange_filter(leaf, chunk, bargs)) {
4006	return false;
4007	}
4008
4009	/* vrange filter */
4010	if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) &&
4011	chunk_vrange_filter(leaf, chunk, chunk_offset, bargs)) {
4012	return false;
4013	}
4014
4015	/* stripes filter */
4016	if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) &&
4017	chunk_stripes_range_filter(leaf, chunk, bargs)) {
4018	return false;
4019	}
4020
4021	/* soft profile changing mode */
4022	if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) &&
4023	chunk_soft_convert_filter(chunk_type, bargs)) {
4024	return false;
4025	}
4026
4027	/*
4028	* limited by count, must be the last filter
4029	*/
4030	if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) {
4031	if (bargs->limit == 0)
4032	return false;
4033	else
4034	bargs->limit--;
4035	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) {
4036	/*
4037	* Same logic as the 'limit' filter; the minimum cannot be
4038	* determined here because we do not have the global information
4039	* about the count of all chunks that satisfy the filters.
4040	*/
4041	if (bargs->limit_max == 0)
4042	return false;
4043	else
4044	bargs->limit_max--;
4045	}
4046
4047	return true;
4048	}
4049
4050	static int __btrfs_balance(struct btrfs_fs_info *fs_info)
4051	{
4052	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
4053	struct btrfs_root *chunk_root = fs_info->chunk_root;
4054	u64 chunk_type;
4055	struct btrfs_chunk *chunk;
4056	BTRFS_PATH_AUTO_FREE(path);
4057	struct btrfs_key key;
4058	struct btrfs_key found_key;
4059	struct extent_buffer *leaf;
4060	int slot;
4061	int ret;
4062	int enospc_errors = 0;
4063	bool counting = true;
4064	/* The single value limit and min/max limits use the same bytes in the */
4065	u64 limit_data = bctl->data.limit;
4066	u64 limit_meta = bctl->meta.limit;
4067	u64 limit_sys = bctl->sys.limit;
4068	u32 count_data = 0;
4069	u32 count_meta = 0;
4070	u32 count_sys = 0;
4071	int chunk_reserved = 0;
4072
4073	path = btrfs_alloc_path();
4074	if (!path) {
4075	ret = -ENOMEM;
4076	goto error;
4077	}
4078
4079	/* zero out stat counters */
4080	spin_lock(lock: &fs_info->balance_lock);
4081	memset(&bctl->stat, 0, sizeof(bctl->stat));
4082	spin_unlock(lock: &fs_info->balance_lock);
4083	again:
4084	if (!counting) {
4085	/*
4086	* The single value limit and min/max limits use the same bytes
4087	* in the
4088	*/
4089	bctl->data.limit = limit_data;
4090	bctl->meta.limit = limit_meta;
4091	bctl->sys.limit = limit_sys;
4092	}
4093	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
4094	key.type = BTRFS_CHUNK_ITEM_KEY;
4095	key.offset = (u64)-1;
4096
4097	while (1) {
4098	if ((!counting && atomic_read(v: &fs_info->balance_pause_req)) ||
4099	atomic_read(v: &fs_info->balance_cancel_req)) {
4100	ret = -ECANCELED;
4101	goto error;
4102	}
4103
4104	mutex_lock(&fs_info->reclaim_bgs_lock);
4105	ret = btrfs_search_slot(NULL, root: chunk_root, key: &key, p: path, ins_len: 0, cow: 0);
4106	if (ret < 0) {
4107	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4108	goto error;
4109	}
4110
4111	/*
4112	* this shouldn't happen, it means the last relocate
4113	* failed
4114	*/
4115	if (ret == 0)
4116	BUG(); /* FIXME break ? */
4117
4118	ret = btrfs_previous_item(root: chunk_root, path, min_objectid: 0,
4119	BTRFS_CHUNK_ITEM_KEY);
4120	if (ret) {
4121	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4122	ret = 0;
4123	break;
4124	}
4125
4126	leaf = path->nodes[0];
4127	slot = path->slots[0];
4128	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: slot);
4129
4130	if (found_key.objectid != key.objectid) {
4131	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4132	break;
4133	}
4134
4135	chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
4136	chunk_type = btrfs_chunk_type(eb: leaf, s: chunk);
4137
4138	if (!counting) {
4139	spin_lock(lock: &fs_info->balance_lock);
4140	bctl->stat.considered++;
4141	spin_unlock(lock: &fs_info->balance_lock);
4142	}
4143
4144	ret = should_balance_chunk(leaf, chunk, chunk_offset: found_key.offset);
4145
4146	btrfs_release_path(p: path);
4147	if (!ret) {
4148	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4149	goto loop;
4150	}
4151
4152	if (counting) {
4153	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4154	spin_lock(lock: &fs_info->balance_lock);
4155	bctl->stat.expected++;
4156	spin_unlock(lock: &fs_info->balance_lock);
4157
4158	if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
4159	count_data++;
4160	else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
4161	count_sys++;
4162	else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
4163	count_meta++;
4164
4165	goto loop;
4166	}
4167
4168	/*
4169	* Apply limit_min filter, no need to check if the LIMITS
4170	* filter is used, limit_min is 0 by default
4171	*/
4172	if (((chunk_type & BTRFS_BLOCK_GROUP_DATA) &&
4173	count_data < bctl->data.limit_min)
4174	|| ((chunk_type & BTRFS_BLOCK_GROUP_METADATA) &&
4175	count_meta < bctl->meta.limit_min)
4176	|| ((chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) &&
4177	count_sys < bctl->sys.limit_min)) {
4178	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4179	goto loop;
4180	}
4181
4182	if (!chunk_reserved) {
4183	/*
4184	* We may be relocating the only data chunk we have,
4185	* which could potentially end up with losing data's
4186	* raid profile, so lets allocate an empty one in
4187	* advance.
4188	*/
4189	ret = btrfs_may_alloc_data_chunk(fs_info,
4190	chunk_offset: found_key.offset);
4191	if (ret < 0) {
4192	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4193	goto error;
4194	} else if (ret == 1) {
4195	chunk_reserved = 1;
4196	}
4197	}
4198
4199	ret = btrfs_relocate_chunk(fs_info, chunk_offset: found_key.offset, verbose: true);
4200	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4201	if (ret == -ENOSPC) {
4202	enospc_errors++;
4203	} else if (ret == -ETXTBSY) {
4204	btrfs_info(fs_info,
4205	"skipping relocation of block group %llu due to active swapfile",
4206	found_key.offset);
4207	ret = 0;
4208	} else if (ret) {
4209	goto error;
4210	} else {
4211	spin_lock(lock: &fs_info->balance_lock);
4212	bctl->stat.completed++;
4213	spin_unlock(lock: &fs_info->balance_lock);
4214	}
4215	loop:
4216	if (found_key.offset == 0)
4217	break;
4218	key.offset = found_key.offset - 1;
4219	}
4220
4221	if (counting) {
4222	btrfs_release_path(p: path);
4223	counting = false;
4224	goto again;
4225	}
4226	error:
4227	if (enospc_errors) {
4228	btrfs_info(fs_info, "%d enospc errors during balance",
4229	enospc_errors);
4230	if (!ret)
4231	ret = -ENOSPC;
4232	}
4233
4234	return ret;
4235	}
4236
4237	/*
4238	* See if a given profile is valid and reduced.
4239	*
4240	* @flags: profile to validate
4241	* @extended: if true @flags is treated as an extended profile
4242	*/
4243	static int alloc_profile_is_valid(u64 flags, bool extended)
4244	{
4245	u64 mask = (extended ? BTRFS_EXTENDED_PROFILE_MASK :
4246	BTRFS_BLOCK_GROUP_PROFILE_MASK);
4247
4248	flags &= ~BTRFS_BLOCK_GROUP_TYPE_MASK;
4249
4250	/* 1) check that all other bits are zeroed */
4251	if (flags & ~mask)
4252	return 0;
4253
4254	/* 2) see if profile is reduced */
4255	if (flags == 0)
4256	return !extended; /* "0" is valid for usual profiles */
4257
4258	return has_single_bit_set(n: flags);
4259	}
4260
4261	/*
4262	* Validate target profile against allowed profiles and return true if it's OK.
4263	* Otherwise print the error message and return false.
4264	*/
4265	static inline int validate_convert_profile(struct btrfs_fs_info *fs_info,
4266	const struct btrfs_balance_args *bargs,
4267	u64 allowed, const char *type)
4268	{
4269	if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
4270	return true;
4271
4272	/* Profile is valid and does not have bits outside of the allowed set */
4273	if (alloc_profile_is_valid(flags: bargs->target, extended: 1) &&
4274	(bargs->target & ~allowed) == 0)
4275	return true;
4276
4277	btrfs_err(fs_info, "balance: invalid convert %s profile %s",
4278	type, btrfs_bg_type_to_raid_name(bargs->target));
4279	return false;
4280	}
4281
4282	/*
4283	* Fill @buf with textual description of balance filter flags @bargs, up to
4284	* @size_buf including the terminating null. The output may be trimmed if it
4285	* does not fit into the provided buffer.
4286	*/
4287	static void describe_balance_args(struct btrfs_balance_args *bargs, char *buf,
4288	u32 size_buf)
4289	{
4290	int ret;
4291	u32 size_bp = size_buf;
4292	char *bp = buf;
4293	u64 flags = bargs->flags;
4294	char tmp_buf[128] = {'\0'};
4295
4296	if (!flags)
4297	return;
4298
4299	#define CHECK_APPEND_NOARG(a) \
4300	do { \
4301	ret = snprintf(bp, size_bp, (a)); \
4302	if (ret < 0 || ret >= size_bp) \
4303	goto out_overflow; \
4304	size_bp -= ret; \
4305	bp += ret; \
4306	} while (0)
4307
4308	#define CHECK_APPEND_1ARG(a, v1) \
4309	do { \
4310	ret = snprintf(bp, size_bp, (a), (v1)); \
4311	if (ret < 0 || ret >= size_bp) \
4312	goto out_overflow; \
4313	size_bp -= ret; \
4314	bp += ret; \
4315	} while (0)
4316
4317	#define CHECK_APPEND_2ARG(a, v1, v2) \
4318	do { \
4319	ret = snprintf(bp, size_bp, (a), (v1), (v2)); \
4320	if (ret < 0 || ret >= size_bp) \
4321	goto out_overflow; \
4322	size_bp -= ret; \
4323	bp += ret; \
4324	} while (0)
4325
4326	if (flags & BTRFS_BALANCE_ARGS_CONVERT)
4327	CHECK_APPEND_1ARG("convert=%s,",
4328	btrfs_bg_type_to_raid_name(bargs->target));
4329
4330	if (flags & BTRFS_BALANCE_ARGS_SOFT)
4331	CHECK_APPEND_NOARG("soft,");
4332
4333	if (flags & BTRFS_BALANCE_ARGS_PROFILES) {
4334	btrfs_describe_block_groups(bg_flags: bargs->profiles, buf: tmp_buf,
4335	size_buf: sizeof(tmp_buf));
4336	CHECK_APPEND_1ARG("profiles=%s,", tmp_buf);
4337	}
4338
4339	if (flags & BTRFS_BALANCE_ARGS_USAGE)
4340	CHECK_APPEND_1ARG("usage=%llu,", bargs->usage);
4341
4342	if (flags & BTRFS_BALANCE_ARGS_USAGE_RANGE)
4343	CHECK_APPEND_2ARG("usage=%u..%u,",
4344	bargs->usage_min, bargs->usage_max);
4345
4346	if (flags & BTRFS_BALANCE_ARGS_DEVID)
4347	CHECK_APPEND_1ARG("devid=%llu,", bargs->devid);
4348
4349	if (flags & BTRFS_BALANCE_ARGS_DRANGE)
4350	CHECK_APPEND_2ARG("drange=%llu..%llu,",
4351	bargs->pstart, bargs->pend);
4352
4353	if (flags & BTRFS_BALANCE_ARGS_VRANGE)
4354	CHECK_APPEND_2ARG("vrange=%llu..%llu,",
4355	bargs->vstart, bargs->vend);
4356
4357	if (flags & BTRFS_BALANCE_ARGS_LIMIT)
4358	CHECK_APPEND_1ARG("limit=%llu,", bargs->limit);
4359
4360	if (flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)
4361	CHECK_APPEND_2ARG("limit=%u..%u,",
4362	bargs->limit_min, bargs->limit_max);
4363
4364	if (flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE)
4365	CHECK_APPEND_2ARG("stripes=%u..%u,",
4366	bargs->stripes_min, bargs->stripes_max);
4367
4368	#undef CHECK_APPEND_2ARG
4369	#undef CHECK_APPEND_1ARG
4370	#undef CHECK_APPEND_NOARG
4371
4372	out_overflow:
4373
4374	if (size_bp < size_buf)
4375	buf[size_buf - size_bp - 1] = '\0'; /* remove last , */
4376	else
4377	buf[0] = '\0';
4378	}
4379
4380	static void describe_balance_start_or_resume(struct btrfs_fs_info *fs_info)
4381	{
4382	u32 size_buf = 1024;
4383	char tmp_buf[192] = {'\0'};
4384	char AUTO_KFREE(buf);
4385	char *bp;
4386	u32 size_bp = size_buf;
4387	int ret;
4388	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
4389
4390	buf = kzalloc(size_buf, GFP_KERNEL);
4391	if (!buf)
4392	return;
4393
4394	bp = buf;
4395
4396	#define CHECK_APPEND_1ARG(a, v1) \
4397	do { \
4398	ret = snprintf(bp, size_bp, (a), (v1)); \
4399	if (ret < 0 || ret >= size_bp) \
4400	goto out_overflow; \
4401	size_bp -= ret; \
4402	bp += ret; \
4403	} while (0)
4404
4405	if (bctl->flags & BTRFS_BALANCE_FORCE)
4406	CHECK_APPEND_1ARG("%s", "-f ");
4407
4408	if (bctl->flags & BTRFS_BALANCE_DATA) {
4409	describe_balance_args(bargs: &bctl->data, buf: tmp_buf, size_buf: sizeof(tmp_buf));
4410	CHECK_APPEND_1ARG("-d%s ", tmp_buf);
4411	}
4412
4413	if (bctl->flags & BTRFS_BALANCE_METADATA) {
4414	describe_balance_args(bargs: &bctl->meta, buf: tmp_buf, size_buf: sizeof(tmp_buf));
4415	CHECK_APPEND_1ARG("-m%s ", tmp_buf);
4416	}
4417
4418	if (bctl->flags & BTRFS_BALANCE_SYSTEM) {
4419	describe_balance_args(bargs: &bctl->sys, buf: tmp_buf, size_buf: sizeof(tmp_buf));
4420	CHECK_APPEND_1ARG("-s%s ", tmp_buf);
4421	}
4422
4423	#undef CHECK_APPEND_1ARG
4424
4425	out_overflow:
4426
4427	if (size_bp < size_buf)
4428	buf[size_buf - size_bp - 1] = '\0'; /* remove last " " */
4429	btrfs_info(fs_info, "balance: %s %s",
4430	(bctl->flags & BTRFS_BALANCE_RESUME) ?
4431	"resume" : "start", buf);
4432	}
4433
4434	/*
4435	* Should be called with balance mutex held
4436	*/
4437	int btrfs_balance(struct btrfs_fs_info *fs_info,
4438	struct btrfs_balance_control *bctl,
4439	struct btrfs_ioctl_balance_args *bargs)
4440	{
4441	u64 meta_target, data_target;
4442	u64 allowed;
4443	int mixed = 0;
4444	int ret;
4445	u64 num_devices;
4446	unsigned seq;
4447	bool reducing_redundancy;
4448	bool paused = false;
4449	int i;
4450
4451	if (btrfs_fs_closing(fs_info) ||
4452	atomic_read(v: &fs_info->balance_pause_req) ||
4453	btrfs_should_cancel_balance(fs_info)) {
4454	ret = -EINVAL;
4455	goto out;
4456	}
4457
4458	allowed = btrfs_super_incompat_flags(s: fs_info->super_copy);
4459	if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
4460	mixed = 1;
4461
4462	/*
4463	* In case of mixed groups both data and meta should be picked,
4464	* and identical options should be given for both of them.
4465	*/
4466	allowed = BTRFS_BALANCE_DATA | BTRFS_BALANCE_METADATA;
4467	if (mixed && (bctl->flags & allowed)) {
4468	if (!(bctl->flags & BTRFS_BALANCE_DATA) ||
4469	!(bctl->flags & BTRFS_BALANCE_METADATA) ||
4470	memcmp(p: &bctl->data, q: &bctl->meta, size: sizeof(bctl->data))) {
4471	btrfs_err(fs_info,
4472	"balance: mixed groups data and metadata options must be the same");
4473	ret = -EINVAL;
4474	goto out;
4475	}
4476	}
4477
4478	/*
4479	* rw_devices will not change at the moment, device add/delete/replace
4480	* are exclusive
4481	*/
4482	num_devices = fs_info->fs_devices->rw_devices;
4483
4484	/*
4485	* SINGLE profile on-disk has no profile bit, but in-memory we have a
4486	* special bit for it, to make it easier to distinguish. Thus we need
4487	* to set it manually, or balance would refuse the profile.
4488	*/
4489	allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE;
4490	for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++)
4491	if (num_devices >= btrfs_raid_array[i].devs_min)
4492	allowed |= btrfs_raid_array[i].bg_flag;
4493
4494	if (!validate_convert_profile(fs_info, bargs: &bctl->data, allowed, type: "data") ||
4495	!validate_convert_profile(fs_info, bargs: &bctl->meta, allowed, type: "metadata") ||
4496	!validate_convert_profile(fs_info, bargs: &bctl->sys, allowed, type: "system")) {
4497	ret = -EINVAL;
4498	goto out;
4499	}
4500
4501	/*
4502	* Allow to reduce metadata or system integrity only if force set for
4503	* profiles with redundancy (copies, parity)
4504	*/
4505	allowed = 0;
4506	for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) {
4507	if (btrfs_raid_array[i].ncopies >= 2 ||
4508	btrfs_raid_array[i].tolerated_failures >= 1)
4509	allowed |= btrfs_raid_array[i].bg_flag;
4510	}
4511	do {
4512	seq = read_seqbegin(sl: &fs_info->profiles_lock);
4513
4514	if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4515	(fs_info->avail_system_alloc_bits & allowed) &&
4516	!(bctl->sys.target & allowed)) ||
4517	((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4518	(fs_info->avail_metadata_alloc_bits & allowed) &&
4519	!(bctl->meta.target & allowed)))
4520	reducing_redundancy = true;
4521	else
4522	reducing_redundancy = false;
4523
4524	/* if we're not converting, the target field is uninitialized */
4525	meta_target = (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4526	bctl->meta.target : fs_info->avail_metadata_alloc_bits;
4527	data_target = (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4528	bctl->data.target : fs_info->avail_data_alloc_bits;
4529	} while (read_seqretry(sl: &fs_info->profiles_lock, start: seq));
4530
4531	if (reducing_redundancy) {
4532	if (bctl->flags & BTRFS_BALANCE_FORCE) {
4533	btrfs_info(fs_info,
4534	"balance: force reducing metadata redundancy");
4535	} else {
4536	btrfs_err(fs_info,
4537	"balance: reduces metadata redundancy, use --force if you want this");
4538	ret = -EINVAL;
4539	goto out;
4540	}
4541	}
4542
4543	if (btrfs_get_num_tolerated_disk_barrier_failures(flags: meta_target) <
4544	btrfs_get_num_tolerated_disk_barrier_failures(flags: data_target)) {
4545	btrfs_warn(fs_info,
4546	"balance: metadata profile %s has lower redundancy than data profile %s",
4547	btrfs_bg_type_to_raid_name(meta_target),
4548	btrfs_bg_type_to_raid_name(data_target));
4549	}
4550
4551	ret = insert_balance_item(fs_info, bctl);
4552	if (ret && ret != -EEXIST)
4553	goto out;
4554
4555	if (!(bctl->flags & BTRFS_BALANCE_RESUME)) {
4556	BUG_ON(ret == -EEXIST);
4557	BUG_ON(fs_info->balance_ctl);
4558	spin_lock(lock: &fs_info->balance_lock);
4559	fs_info->balance_ctl = bctl;
4560	spin_unlock(lock: &fs_info->balance_lock);
4561	} else {
4562	BUG_ON(ret != -EEXIST);
4563	spin_lock(lock: &fs_info->balance_lock);
4564	update_balance_args(bctl);
4565	spin_unlock(lock: &fs_info->balance_lock);
4566	}
4567
4568	ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4569	set_bit(nr: BTRFS_FS_BALANCE_RUNNING, addr: &fs_info->flags);
4570	describe_balance_start_or_resume(fs_info);
4571	mutex_unlock(lock: &fs_info->balance_mutex);
4572
4573	ret = __btrfs_balance(fs_info);
4574
4575	mutex_lock(&fs_info->balance_mutex);
4576	if (ret == -ECANCELED && atomic_read(v: &fs_info->balance_pause_req)) {
4577	btrfs_info(fs_info, "balance: paused");
4578	btrfs_exclop_balance(fs_info, op: BTRFS_EXCLOP_BALANCE_PAUSED);
4579	paused = true;
4580	}
4581	/*
4582	* Balance can be canceled by:
4583	*
4584	* - Regular cancel request
4585	* Then ret == -ECANCELED and balance_cancel_req > 0
4586	*
4587	* - Fatal signal to "btrfs" process
4588	* Either the signal caught by wait_reserve_ticket() and callers
4589	* got -EINTR, or caught by btrfs_should_cancel_balance() and
4590	* got -ECANCELED.
4591	* Either way, in this case balance_cancel_req = 0, and
4592	* ret == -EINTR or ret == -ECANCELED.
4593	*
4594	* So here we only check the return value to catch canceled balance.
4595	*/
4596	else if (ret == -ECANCELED || ret == -EINTR)
4597	btrfs_info(fs_info, "balance: canceled");
4598	else
4599	btrfs_info(fs_info, "balance: ended with status: %d", ret);
4600
4601	clear_bit(nr: BTRFS_FS_BALANCE_RUNNING, addr: &fs_info->flags);
4602
4603	if (bargs) {
4604	memset(bargs, 0, sizeof(*bargs));
4605	btrfs_update_ioctl_balance_args(fs_info, bargs);
4606	}
4607
4608	/* We didn't pause, we can clean everything up. */
4609	if (!paused) {
4610	reset_balance_state(fs_info);
4611	btrfs_exclop_finish(fs_info);
4612	}
4613
4614	wake_up(&fs_info->balance_wait_q);
4615
4616	return ret;
4617	out:
4618	if (bctl->flags & BTRFS_BALANCE_RESUME)
4619	reset_balance_state(fs_info);
4620	else
4621	kfree(objp: bctl);
4622	btrfs_exclop_finish(fs_info);
4623
4624	return ret;
4625	}
4626
4627	static int balance_kthread(void *data)
4628	{
4629	struct btrfs_fs_info *fs_info = data;
4630	int ret = 0;
4631
4632	guard(super_write)(T: fs_info->sb);
4633
4634	mutex_lock(&fs_info->balance_mutex);
4635	if (fs_info->balance_ctl)
4636	ret = btrfs_balance(fs_info, bctl: fs_info->balance_ctl, NULL);
4637	mutex_unlock(lock: &fs_info->balance_mutex);
4638
4639	return ret;
4640	}
4641
4642	int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info)
4643	{
4644	struct task_struct *tsk;
4645
4646	mutex_lock(&fs_info->balance_mutex);
4647	if (!fs_info->balance_ctl) {
4648	mutex_unlock(lock: &fs_info->balance_mutex);
4649	return 0;
4650	}
4651	mutex_unlock(lock: &fs_info->balance_mutex);
4652
4653	if (btrfs_test_opt(fs_info, SKIP_BALANCE)) {
4654	btrfs_info(fs_info, "balance: resume skipped");
4655	return 0;
4656	}
4657
4658	spin_lock(lock: &fs_info->super_lock);
4659	ASSERT(fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED,
4660	"exclusive_operation=%d", fs_info->exclusive_operation);
4661	fs_info->exclusive_operation = BTRFS_EXCLOP_BALANCE;
4662	spin_unlock(lock: &fs_info->super_lock);
4663	/*
4664	* A ro->rw remount sequence should continue with the paused balance
4665	* regardless of who pauses it, system or the user as of now, so set
4666	* the resume flag.
4667	*/
4668	spin_lock(lock: &fs_info->balance_lock);
4669	fs_info->balance_ctl->flags |= BTRFS_BALANCE_RESUME;
4670	spin_unlock(lock: &fs_info->balance_lock);
4671
4672	tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance");
4673	return PTR_ERR_OR_ZERO(ptr: tsk);
4674	}
4675
4676	int btrfs_recover_balance(struct btrfs_fs_info *fs_info)
4677	{
4678	struct btrfs_balance_control *bctl;
4679	struct btrfs_balance_item *item;
4680	struct btrfs_disk_balance_args disk_bargs;
4681	BTRFS_PATH_AUTO_FREE(path);
4682	struct extent_buffer *leaf;
4683	struct btrfs_key key;
4684	int ret;
4685
4686	path = btrfs_alloc_path();
4687	if (!path)
4688	return -ENOMEM;
4689
4690	key.objectid = BTRFS_BALANCE_OBJECTID;
4691	key.type = BTRFS_TEMPORARY_ITEM_KEY;
4692	key.offset = 0;
4693
4694	ret = btrfs_search_slot(NULL, root: fs_info->tree_root, key: &key, p: path, ins_len: 0, cow: 0);
4695	if (ret < 0)
4696	return ret;
4697	if (ret > 0) { /* ret = -ENOENT; */
4698	return 0;
4699	}
4700
4701	bctl = kzalloc(sizeof(*bctl), GFP_NOFS);
4702	if (!bctl)
4703	return -ENOMEM;
4704
4705	leaf = path->nodes[0];
4706	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
4707
4708	bctl->flags = btrfs_balance_flags(eb: leaf, s: item);
4709	bctl->flags |= BTRFS_BALANCE_RESUME;
4710
4711	btrfs_balance_data(eb: leaf, bi: item, ba: &disk_bargs);
4712	btrfs_disk_balance_args_to_cpu(cpu: &bctl->data, disk: &disk_bargs);
4713	btrfs_balance_meta(eb: leaf, bi: item, ba: &disk_bargs);
4714	btrfs_disk_balance_args_to_cpu(cpu: &bctl->meta, disk: &disk_bargs);
4715	btrfs_balance_sys(eb: leaf, bi: item, ba: &disk_bargs);
4716	btrfs_disk_balance_args_to_cpu(cpu: &bctl->sys, disk: &disk_bargs);
4717
4718	/*
4719	* This should never happen, as the paused balance state is recovered
4720	* during mount without any chance of other exclusive ops to collide.
4721	*
4722	* This gives the exclusive op status to balance and keeps in paused
4723	* state until user intervention (cancel or umount). If the ownership
4724	* cannot be assigned, show a message but do not fail. The balance
4725	* is in a paused state and must have fs_info::balance_ctl properly
4726	* set up.
4727	*/
4728	if (!btrfs_exclop_start(fs_info, type: BTRFS_EXCLOP_BALANCE_PAUSED))
4729	btrfs_warn(fs_info,
4730	"balance: cannot set exclusive op status, resume manually");
4731
4732	btrfs_release_path(p: path);
4733
4734	mutex_lock(&fs_info->balance_mutex);
4735	BUG_ON(fs_info->balance_ctl);
4736	spin_lock(lock: &fs_info->balance_lock);
4737	fs_info->balance_ctl = bctl;
4738	spin_unlock(lock: &fs_info->balance_lock);
4739	mutex_unlock(lock: &fs_info->balance_mutex);
4740	return ret;
4741	}
4742
4743	int btrfs_pause_balance(struct btrfs_fs_info *fs_info)
4744	{
4745	int ret = 0;
4746
4747	mutex_lock(&fs_info->balance_mutex);
4748	if (!fs_info->balance_ctl) {
4749	mutex_unlock(lock: &fs_info->balance_mutex);
4750	return -ENOTCONN;
4751	}
4752
4753	if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4754	atomic_inc(v: &fs_info->balance_pause_req);
4755	mutex_unlock(lock: &fs_info->balance_mutex);
4756
4757	wait_event(fs_info->balance_wait_q,
4758	!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4759
4760	mutex_lock(&fs_info->balance_mutex);
4761	/* we are good with balance_ctl ripped off from under us */
4762	BUG_ON(test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4763	atomic_dec(v: &fs_info->balance_pause_req);
4764	} else {
4765	ret = -ENOTCONN;
4766	}
4767
4768	mutex_unlock(lock: &fs_info->balance_mutex);
4769	return ret;
4770	}
4771
4772	int btrfs_cancel_balance(struct btrfs_fs_info *fs_info)
4773	{
4774	mutex_lock(&fs_info->balance_mutex);
4775	if (!fs_info->balance_ctl) {
4776	mutex_unlock(lock: &fs_info->balance_mutex);
4777	return -ENOTCONN;
4778	}
4779
4780	/*
4781	* A paused balance with the item stored on disk can be resumed at
4782	* mount time if the mount is read-write. Otherwise it's still paused
4783	* and we must not allow cancelling as it deletes the item.
4784	*/
4785	if (sb_rdonly(sb: fs_info->sb)) {
4786	mutex_unlock(lock: &fs_info->balance_mutex);
4787	return -EROFS;
4788	}
4789
4790	atomic_inc(v: &fs_info->balance_cancel_req);
4791	/*
4792	* if we are running just wait and return, balance item is
4793	* deleted in btrfs_balance in this case
4794	*/
4795	if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4796	mutex_unlock(lock: &fs_info->balance_mutex);
4797	wait_event(fs_info->balance_wait_q,
4798	!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4799	mutex_lock(&fs_info->balance_mutex);
4800	} else {
4801	mutex_unlock(lock: &fs_info->balance_mutex);
4802	/*
4803	* Lock released to allow other waiters to continue, we'll
4804	* reexamine the status again.
4805	*/
4806	mutex_lock(&fs_info->balance_mutex);
4807
4808	if (fs_info->balance_ctl) {
4809	reset_balance_state(fs_info);
4810	btrfs_exclop_finish(fs_info);
4811	btrfs_info(fs_info, "balance: canceled");
4812	}
4813	}
4814
4815	ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4816	atomic_dec(v: &fs_info->balance_cancel_req);
4817	mutex_unlock(lock: &fs_info->balance_mutex);
4818	return 0;
4819	}
4820
4821	/*
4822	* shrinking a device means finding all of the device extents past
4823	* the new size, and then following the back refs to the chunks.
4824	* The chunk relocation code actually frees the device extent
4825	*/
4826	int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
4827	{
4828	struct btrfs_fs_info *fs_info = device->fs_info;
4829	struct btrfs_root *root = fs_info->dev_root;
4830	struct btrfs_trans_handle *trans;
4831	struct btrfs_dev_extent *dev_extent = NULL;
4832	struct btrfs_path *path;
4833	u64 length;
4834	u64 chunk_offset;
4835	int ret;
4836	int slot;
4837	int failed = 0;
4838	bool retried = false;
4839	struct extent_buffer *l;
4840	struct btrfs_key key;
4841	struct btrfs_super_block *super_copy = fs_info->super_copy;
4842	u64 old_total = btrfs_super_total_bytes(s: super_copy);
4843	u64 old_size = btrfs_device_get_total_bytes(dev: device);
4844	u64 diff;
4845	u64 start;
4846	u64 free_diff = 0;
4847
4848	new_size = round_down(new_size, fs_info->sectorsize);
4849	start = new_size;
4850	diff = round_down(old_size - new_size, fs_info->sectorsize);
4851
4852	if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
4853	return -EINVAL;
4854
4855	path = btrfs_alloc_path();
4856	if (!path)
4857	return -ENOMEM;
4858
4859	path->reada = READA_BACK;
4860
4861	trans = btrfs_start_transaction(root, num_items: 0);
4862	if (IS_ERR(ptr: trans)) {
4863	btrfs_free_path(p: path);
4864	return PTR_ERR(ptr: trans);
4865	}
4866
4867	mutex_lock(&fs_info->chunk_mutex);
4868
4869	btrfs_device_set_total_bytes(dev: device, size: new_size);
4870	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
4871	device->fs_devices->total_rw_bytes -= diff;
4872
4873	/*
4874	* The new free_chunk_space is new_size - used, so we have to
4875	* subtract the delta of the old free_chunk_space which included
4876	* old_size - used. If used > new_size then just subtract this
4877	* entire device's free space.
4878	*/
4879	if (device->bytes_used < new_size)
4880	free_diff = (old_size - device->bytes_used) -
4881	(new_size - device->bytes_used);
4882	else
4883	free_diff = old_size - device->bytes_used;
4884	atomic64_sub(i: free_diff, v: &fs_info->free_chunk_space);
4885	}
4886
4887	/*
4888	* Once the device's size has been set to the new size, ensure all
4889	* in-memory chunks are synced to disk so that the loop below sees them
4890	* and relocates them accordingly.
4891	*/
4892	if (contains_pending_extent(device, start: &start, len: diff)) {
4893	mutex_unlock(lock: &fs_info->chunk_mutex);
4894	ret = btrfs_commit_transaction(trans);
4895	if (ret)
4896	goto done;
4897	} else {
4898	mutex_unlock(lock: &fs_info->chunk_mutex);
4899	btrfs_end_transaction(trans);
4900	}
4901
4902	again:
4903	key.objectid = device->devid;
4904	key.type = BTRFS_DEV_EXTENT_KEY;
4905	key.offset = (u64)-1;
4906
4907	do {
4908	mutex_lock(&fs_info->reclaim_bgs_lock);
4909	ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
4910	if (ret < 0) {
4911	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4912	goto done;
4913	}
4914
4915	ret = btrfs_previous_item(root, path, min_objectid: 0, type: key.type);
4916	if (ret) {
4917	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4918	if (ret < 0)
4919	goto done;
4920	ret = 0;
4921	btrfs_release_path(p: path);
4922	break;
4923	}
4924
4925	l = path->nodes[0];
4926	slot = path->slots[0];
4927	btrfs_item_key_to_cpu(eb: l, cpu_key: &key, nr: path->slots[0]);
4928
4929	if (key.objectid != device->devid) {
4930	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4931	btrfs_release_path(p: path);
4932	break;
4933	}
4934
4935	dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
4936	length = btrfs_dev_extent_length(eb: l, s: dev_extent);
4937
4938	if (key.offset + length <= new_size) {
4939	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4940	btrfs_release_path(p: path);
4941	break;
4942	}
4943
4944	chunk_offset = btrfs_dev_extent_chunk_offset(eb: l, s: dev_extent);
4945	btrfs_release_path(p: path);
4946
4947	/*
4948	* We may be relocating the only data chunk we have,
4949	* which could potentially end up with losing data's
4950	* raid profile, so lets allocate an empty one in
4951	* advance.
4952	*/
4953	ret = btrfs_may_alloc_data_chunk(fs_info, chunk_offset);
4954	if (ret < 0) {
4955	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4956	goto done;
4957	}
4958
4959	ret = btrfs_relocate_chunk(fs_info, chunk_offset, verbose: true);
4960	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4961	if (ret == -ENOSPC) {
4962	failed++;
4963	} else if (ret) {
4964	if (ret == -ETXTBSY) {
4965	btrfs_warn(fs_info,
4966	"could not shrink block group %llu due to active swapfile",
4967	chunk_offset);
4968	}
4969	goto done;
4970	}
4971	} while (key.offset-- > 0);
4972
4973	if (failed && !retried) {
4974	failed = 0;
4975	retried = true;
4976	goto again;
4977	} else if (failed && retried) {
4978	ret = -ENOSPC;
4979	goto done;
4980	}
4981
4982	/* Shrinking succeeded, else we would be at "done". */
4983	trans = btrfs_start_transaction(root, num_items: 0);
4984	if (IS_ERR(ptr: trans)) {
4985	ret = PTR_ERR(ptr: trans);
4986	goto done;
4987	}
4988
4989	mutex_lock(&fs_info->chunk_mutex);
4990	/* Clear all state bits beyond the shrunk device size */
4991	btrfs_clear_extent_bit(tree: &device->alloc_state, start: new_size, end: (u64)-1,
4992	CHUNK_STATE_MASK, NULL);
4993
4994	btrfs_device_set_disk_total_bytes(dev: device, size: new_size);
4995	if (list_empty(head: &device->post_commit_list))
4996	list_add_tail(new: &device->post_commit_list,
4997	head: &trans->transaction->dev_update_list);
4998
4999	WARN_ON(diff > old_total);
5000	btrfs_set_super_total_bytes(s: super_copy,
5001	round_down(old_total - diff, fs_info->sectorsize));
5002	mutex_unlock(lock: &fs_info->chunk_mutex);
5003
5004	btrfs_reserve_chunk_metadata(trans, is_item_insertion: false);
5005	/* Now btrfs_update_device() will change the on-disk size. */
5006	ret = btrfs_update_device(trans, device);
5007	btrfs_trans_release_chunk_metadata(trans);
5008	if (unlikely(ret < 0)) {
5009	btrfs_abort_transaction(trans, ret);
5010	btrfs_end_transaction(trans);
5011	} else {
5012	ret = btrfs_commit_transaction(trans);
5013	}
5014	done:
5015	btrfs_free_path(p: path);
5016	if (ret) {
5017	mutex_lock(&fs_info->chunk_mutex);
5018	btrfs_device_set_total_bytes(dev: device, size: old_size);
5019	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
5020	device->fs_devices->total_rw_bytes += diff;
5021	atomic64_add(i: free_diff, v: &fs_info->free_chunk_space);
5022	}
5023	mutex_unlock(lock: &fs_info->chunk_mutex);
5024	}
5025	return ret;
5026	}
5027
5028	static int btrfs_add_system_chunk(struct btrfs_fs_info *fs_info,
5029	struct btrfs_key *key,
5030	struct btrfs_chunk *chunk, int item_size)
5031	{
5032	struct btrfs_super_block *super_copy = fs_info->super_copy;
5033	struct btrfs_disk_key disk_key;
5034	u32 array_size;
5035	u8 *ptr;
5036
5037	lockdep_assert_held(&fs_info->chunk_mutex);
5038
5039	array_size = btrfs_super_sys_array_size(s: super_copy);
5040	if (array_size + item_size + sizeof(disk_key)
5041	> BTRFS_SYSTEM_CHUNK_ARRAY_SIZE)
5042	return -EFBIG;
5043
5044	ptr = super_copy->sys_chunk_array + array_size;
5045	btrfs_cpu_key_to_disk(disk_key: &disk_key, cpu_key: key);
5046	memcpy(ptr, &disk_key, sizeof(disk_key));
5047	ptr += sizeof(disk_key);
5048	memcpy(ptr, chunk, item_size);
5049	item_size += sizeof(disk_key);
5050	btrfs_set_super_sys_array_size(s: super_copy, val: array_size + item_size);
5051
5052	return 0;
5053	}
5054
5055	/*
5056	* sort the devices in descending order by max_avail, total_avail
5057	*/
5058	static int btrfs_cmp_device_info(const void *a, const void *b)
5059	{
5060	const struct btrfs_device_info *di_a = a;
5061	const struct btrfs_device_info *di_b = b;
5062
5063	if (di_a->max_avail > di_b->max_avail)
5064	return -1;
5065	if (di_a->max_avail < di_b->max_avail)
5066	return 1;
5067	if (di_a->total_avail > di_b->total_avail)
5068	return -1;
5069	if (di_a->total_avail < di_b->total_avail)
5070	return 1;
5071	return 0;
5072	}
5073
5074	static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type)
5075	{
5076	if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK))
5077	return;
5078
5079	btrfs_set_fs_incompat(info, RAID56);
5080	}
5081
5082	static void check_raid1c34_incompat_flag(struct btrfs_fs_info *info, u64 type)
5083	{
5084	if (!(type & (BTRFS_BLOCK_GROUP_RAID1C3 | BTRFS_BLOCK_GROUP_RAID1C4)))
5085	return;
5086
5087	btrfs_set_fs_incompat(info, RAID1C34);
5088	}
5089
5090	/*
5091	* Structure used internally for btrfs_create_chunk() function.
5092	* Wraps needed parameters.
5093	*/
5094	struct alloc_chunk_ctl {
5095	u64 start;
5096	u64 type;
5097	/* Total number of stripes to allocate */
5098	int num_stripes;
5099	/* sub_stripes info for map */
5100	int sub_stripes;
5101	/* Stripes per device */
5102	int dev_stripes;
5103	/* Maximum number of devices to use */
5104	int devs_max;
5105	/* Minimum number of devices to use */
5106	int devs_min;
5107	/* ndevs has to be a multiple of this */
5108	int devs_increment;
5109	/* Number of copies */
5110	int ncopies;
5111	/* Number of stripes worth of bytes to store parity information */
5112	int nparity;
5113	u64 max_stripe_size;
5114	u64 max_chunk_size;
5115	u64 dev_extent_min;
5116	u64 stripe_size;
5117	u64 chunk_size;
5118	int ndevs;
5119	/* Space_info the block group is going to belong. */
5120	struct btrfs_space_info *space_info;
5121	};
5122
5123	static void init_alloc_chunk_ctl_policy_regular(
5124	struct btrfs_fs_devices *fs_devices,
5125	struct alloc_chunk_ctl *ctl)
5126	{
5127	struct btrfs_space_info *space_info;
5128
5129	space_info = btrfs_find_space_info(info: fs_devices->fs_info, flags: ctl->type);
5130	ASSERT(space_info);
5131
5132	ctl->max_chunk_size = READ_ONCE(space_info->chunk_size);
5133	ctl->max_stripe_size = min_t(u64, ctl->max_chunk_size, SZ_1G);
5134
5135	if (ctl->type & BTRFS_BLOCK_GROUP_SYSTEM)
5136	ctl->devs_max = min_t(int, ctl->devs_max, BTRFS_MAX_DEVS_SYS_CHUNK);
5137
5138	/* We don't want a chunk larger than 10% of writable space */
5139	ctl->max_chunk_size = min(mult_perc(fs_devices->total_rw_bytes, 10),
5140	ctl->max_chunk_size);
5141	ctl->dev_extent_min = btrfs_stripe_nr_to_offset(stripe_nr: ctl->dev_stripes);
5142	}
5143
5144	static void init_alloc_chunk_ctl_policy_zoned(
5145	struct btrfs_fs_devices *fs_devices,
5146	struct alloc_chunk_ctl *ctl)
5147	{
5148	u64 zone_size = fs_devices->fs_info->zone_size;
5149	u64 limit;
5150	int min_num_stripes = ctl->devs_min * ctl->dev_stripes;
5151	int min_data_stripes = (min_num_stripes - ctl->nparity) / ctl->ncopies;
5152	u64 min_chunk_size = min_data_stripes * zone_size;
5153	u64 type = ctl->type;
5154
5155	ctl->max_stripe_size = zone_size;
5156	if (type & BTRFS_BLOCK_GROUP_DATA) {
5157	ctl->max_chunk_size = round_down(BTRFS_MAX_DATA_CHUNK_SIZE,
5158	zone_size);
5159	} else if (type & BTRFS_BLOCK_GROUP_METADATA) {
5160	ctl->max_chunk_size = ctl->max_stripe_size;
5161	} else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
5162	ctl->max_chunk_size = 2 * ctl->max_stripe_size;
5163	ctl->devs_max = min_t(int, ctl->devs_max,
5164	BTRFS_MAX_DEVS_SYS_CHUNK);
5165	} else {
5166	BUG();
5167	}
5168
5169	/* We don't want a chunk larger than 10% of writable space */
5170	limit = max(round_down(mult_perc(fs_devices->total_rw_bytes, 10),
5171	zone_size),
5172	min_chunk_size);
5173	ctl->max_chunk_size = min(limit, ctl->max_chunk_size);
5174	ctl->dev_extent_min = zone_size * ctl->dev_stripes;
5175	}
5176
5177	static void init_alloc_chunk_ctl(struct btrfs_fs_devices *fs_devices,
5178	struct alloc_chunk_ctl *ctl)
5179	{
5180	int index = btrfs_bg_flags_to_raid_index(flags: ctl->type);
5181
5182	ctl->sub_stripes = btrfs_raid_array[index].sub_stripes;
5183	ctl->dev_stripes = btrfs_raid_array[index].dev_stripes;
5184	ctl->devs_max = btrfs_raid_array[index].devs_max;
5185	if (!ctl->devs_max)
5186	ctl->devs_max = BTRFS_MAX_DEVS(fs_devices->fs_info);
5187	ctl->devs_min = btrfs_raid_array[index].devs_min;
5188	ctl->devs_increment = btrfs_raid_array[index].devs_increment;
5189	ctl->ncopies = btrfs_raid_array[index].ncopies;
5190	ctl->nparity = btrfs_raid_array[index].nparity;
5191	ctl->ndevs = 0;
5192
5193	switch (fs_devices->chunk_alloc_policy) {
5194	default:
5195	btrfs_warn_unknown_chunk_allocation(pol: fs_devices->chunk_alloc_policy);
5196	fallthrough;
5197	case BTRFS_CHUNK_ALLOC_REGULAR:
5198	init_alloc_chunk_ctl_policy_regular(fs_devices, ctl);
5199	break;
5200	case BTRFS_CHUNK_ALLOC_ZONED:
5201	init_alloc_chunk_ctl_policy_zoned(fs_devices, ctl);
5202	break;
5203	}
5204	}
5205
5206	static int gather_device_info(struct btrfs_fs_devices *fs_devices,
5207	struct alloc_chunk_ctl *ctl,
5208	struct btrfs_device_info *devices_info)
5209	{
5210	struct btrfs_fs_info *info = fs_devices->fs_info;
5211	struct btrfs_device *device;
5212	u64 total_avail;
5213	u64 dev_extent_want = ctl->max_stripe_size * ctl->dev_stripes;
5214	int ret;
5215	int ndevs = 0;
5216	u64 max_avail;
5217	u64 dev_offset;
5218
5219	/*
5220	* in the first pass through the devices list, we gather information
5221	* about the available holes on each device.
5222	*/
5223	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
5224	if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
5225	WARN(1, KERN_ERR
5226	"BTRFS: read-only device in alloc_list\n");
5227	continue;
5228	}
5229
5230	if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
5231	&device->dev_state) ||
5232	test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
5233	continue;
5234
5235	if (device->total_bytes > device->bytes_used)
5236	total_avail = device->total_bytes - device->bytes_used;
5237	else
5238	total_avail = 0;
5239
5240	/* If there is no space on this device, skip it. */
5241	if (total_avail < ctl->dev_extent_min)
5242	continue;
5243
5244	ret = find_free_dev_extent(device, num_bytes: dev_extent_want, start: &dev_offset,
5245	len: &max_avail);
5246	if (ret && ret != -ENOSPC)
5247	return ret;
5248
5249	if (ret == 0)
5250	max_avail = dev_extent_want;
5251
5252	if (max_avail < ctl->dev_extent_min) {
5253	if (btrfs_test_opt(info, ENOSPC_DEBUG))
5254	btrfs_debug(info,
5255	"%s: devid %llu has no free space, have=%llu want=%llu",
5256	__func__, device->devid, max_avail,
5257	ctl->dev_extent_min);
5258	continue;
5259	}
5260
5261	if (ndevs == fs_devices->rw_devices) {
5262	WARN(1, "%s: found more than %llu devices\n",
5263	__func__, fs_devices->rw_devices);
5264	break;
5265	}
5266	devices_info[ndevs].dev_offset = dev_offset;
5267	devices_info[ndevs].max_avail = max_avail;
5268	devices_info[ndevs].total_avail = total_avail;
5269	devices_info[ndevs].dev = device;
5270	++ndevs;
5271	}
5272	ctl->ndevs = ndevs;
5273
5274	/*
5275	* now sort the devices by hole size / available space
5276	*/
5277	sort(base: devices_info, num: ndevs, size: sizeof(struct btrfs_device_info),
5278	cmp_func: btrfs_cmp_device_info, NULL);
5279
5280	return 0;
5281	}
5282
5283	static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl,
5284	struct btrfs_device_info *devices_info)
5285	{
5286	/* Number of stripes that count for block group size */
5287	int data_stripes;
5288
5289	/*
5290	* The primary goal is to maximize the number of stripes, so use as
5291	* many devices as possible, even if the stripes are not maximum sized.
5292	*
5293	* The DUP profile stores more than one stripe per device, the
5294	* max_avail is the total size so we have to adjust.
5295	*/
5296	ctl->stripe_size = div_u64(dividend: devices_info[ctl->ndevs - 1].max_avail,
5297	divisor: ctl->dev_stripes);
5298	ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5299
5300	/* This will have to be fixed for RAID1 and RAID10 over more drives */
5301	data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5302
5303	/*
5304	* Use the number of data stripes to figure out how big this chunk is
5305	* really going to be in terms of logical address space, and compare
5306	* that answer with the max chunk size. If it's higher, we try to
5307	* reduce stripe_size.
5308	*/
5309	if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
5310	/*
5311	* Reduce stripe_size, round it up to a 16MB boundary again and
5312	* then use it, unless it ends up being even bigger than the
5313	* previous value we had already.
5314	*/
5315	ctl->stripe_size = min(round_up(div_u64(ctl->max_chunk_size,
5316	data_stripes), SZ_16M),
5317	ctl->stripe_size);
5318	}
5319
5320	/* Stripe size should not go beyond 1G. */
5321	ctl->stripe_size = min_t(u64, ctl->stripe_size, SZ_1G);
5322
5323	/* Align to BTRFS_STRIPE_LEN */
5324	ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN);
5325	ctl->chunk_size = ctl->stripe_size * data_stripes;
5326
5327	return 0;
5328	}
5329
5330	static int decide_stripe_size_zoned(struct alloc_chunk_ctl *ctl,
5331	struct btrfs_device_info *devices_info)
5332	{
5333	u64 zone_size = devices_info[0].dev->zone_info->zone_size;
5334	/* Number of stripes that count for block group size */
5335	int data_stripes;
5336
5337	/*
5338	* It should hold because:
5339	* dev_extent_min == dev_extent_want == zone_size * dev_stripes
5340	*/
5341	ASSERT(devices_info[ctl->ndevs - 1].max_avail == ctl->dev_extent_min,
5342	"ndevs=%d max_avail=%llu dev_extent_min=%llu", ctl->ndevs,
5343	devices_info[ctl->ndevs - 1].max_avail, ctl->dev_extent_min);
5344
5345	ctl->stripe_size = zone_size;
5346	ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5347	data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5348
5349	/* stripe_size is fixed in zoned filesystem. Reduce ndevs instead. */
5350	if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
5351	ctl->ndevs = div_u64(dividend: div_u64(dividend: ctl->max_chunk_size * ctl->ncopies,
5352	divisor: ctl->stripe_size) + ctl->nparity,
5353	divisor: ctl->dev_stripes);
5354	ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5355	data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5356	ASSERT(ctl->stripe_size * data_stripes <= ctl->max_chunk_size,
5357	"stripe_size=%llu data_stripes=%d max_chunk_size=%llu",
5358	ctl->stripe_size, data_stripes, ctl->max_chunk_size);
5359	}
5360
5361	ctl->chunk_size = ctl->stripe_size * data_stripes;
5362
5363	return 0;
5364	}
5365
5366	static int decide_stripe_size(struct btrfs_fs_devices *fs_devices,
5367	struct alloc_chunk_ctl *ctl,
5368	struct btrfs_device_info *devices_info)
5369	{
5370	struct btrfs_fs_info *info = fs_devices->fs_info;
5371
5372	/*
5373	* Round down to number of usable stripes, devs_increment can be any
5374	* number so we can't use round_down() that requires power of 2, while
5375	* rounddown is safe.
5376	*/
5377	ctl->ndevs = rounddown(ctl->ndevs, ctl->devs_increment);
5378
5379	if (ctl->ndevs < ctl->devs_min) {
5380	if (btrfs_test_opt(info, ENOSPC_DEBUG)) {
5381	btrfs_debug(info,
5382	"%s: not enough devices with free space: have=%d minimum required=%d",
5383	__func__, ctl->ndevs, ctl->devs_min);
5384	}
5385	return -ENOSPC;
5386	}
5387
5388	ctl->ndevs = min(ctl->ndevs, ctl->devs_max);
5389
5390	switch (fs_devices->chunk_alloc_policy) {
5391	default:
5392	btrfs_warn_unknown_chunk_allocation(pol: fs_devices->chunk_alloc_policy);
5393	fallthrough;
5394	case BTRFS_CHUNK_ALLOC_REGULAR:
5395	return decide_stripe_size_regular(ctl, devices_info);
5396	case BTRFS_CHUNK_ALLOC_ZONED:
5397	return decide_stripe_size_zoned(ctl, devices_info);
5398	}
5399	}
5400
5401	static void chunk_map_device_set_bits(struct btrfs_chunk_map *map, unsigned int bits)
5402	{
5403	for (int i = 0; i < map->num_stripes; i++) {
5404	struct btrfs_io_stripe *stripe = &map->stripes[i];
5405	struct btrfs_device *device = stripe->dev;
5406
5407	btrfs_set_extent_bit(tree: &device->alloc_state, start: stripe->physical,
5408	end: stripe->physical + map->stripe_size - 1,
5409	bits: bits | EXTENT_NOWAIT, NULL);
5410	}
5411	}
5412
5413	static void chunk_map_device_clear_bits(struct btrfs_chunk_map *map, unsigned int bits)
5414	{
5415	for (int i = 0; i < map->num_stripes; i++) {
5416	struct btrfs_io_stripe *stripe = &map->stripes[i];
5417	struct btrfs_device *device = stripe->dev;
5418
5419	btrfs_clear_extent_bit(tree: &device->alloc_state, start: stripe->physical,
5420	end: stripe->physical + map->stripe_size - 1,
5421	bits: bits | EXTENT_NOWAIT, NULL);
5422	}
5423	}
5424
5425	void btrfs_remove_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map)
5426	{
5427	write_lock(&fs_info->mapping_tree_lock);
5428	rb_erase_cached(node: &map->rb_node, root: &fs_info->mapping_tree);
5429	RB_CLEAR_NODE(&map->rb_node);
5430	chunk_map_device_clear_bits(map, CHUNK_ALLOCATED);
5431	write_unlock(&fs_info->mapping_tree_lock);
5432
5433	/* Once for the tree reference. */
5434	btrfs_free_chunk_map(map);
5435	}
5436
5437	static int btrfs_chunk_map_cmp(const struct rb_node *new,
5438	const struct rb_node *exist)
5439	{
5440	const struct btrfs_chunk_map *new_map =
5441	rb_entry(new, struct btrfs_chunk_map, rb_node);
5442	const struct btrfs_chunk_map *exist_map =
5443	rb_entry(exist, struct btrfs_chunk_map, rb_node);
5444
5445	if (new_map->start == exist_map->start)
5446	return 0;
5447	if (new_map->start < exist_map->start)
5448	return -1;
5449	return 1;
5450	}
5451
5452	EXPORT_FOR_TESTS
5453	int btrfs_add_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map)
5454	{
5455	struct rb_node *exist;
5456
5457	write_lock(&fs_info->mapping_tree_lock);
5458	exist = rb_find_add_cached(node: &map->rb_node, tree: &fs_info->mapping_tree,
5459	cmp: btrfs_chunk_map_cmp);
5460
5461	if (exist) {
5462	write_unlock(&fs_info->mapping_tree_lock);
5463	return -EEXIST;
5464	}
5465	chunk_map_device_set_bits(map, CHUNK_ALLOCATED);
5466	chunk_map_device_clear_bits(map, CHUNK_TRIMMED);
5467	write_unlock(&fs_info->mapping_tree_lock);
5468
5469	return 0;
5470	}
5471
5472	EXPORT_FOR_TESTS
5473	struct btrfs_chunk_map *btrfs_alloc_chunk_map(int num_stripes, gfp_t gfp)
5474	{
5475	struct btrfs_chunk_map *map;
5476
5477	map = kmalloc(btrfs_chunk_map_size(num_stripes), gfp);
5478	if (!map)
5479	return NULL;
5480
5481	refcount_set(r: &map->refs, n: 1);
5482	RB_CLEAR_NODE(&map->rb_node);
5483
5484	return map;
5485	}
5486
5487	static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans,
5488	struct alloc_chunk_ctl *ctl,
5489	struct btrfs_device_info *devices_info)
5490	{
5491	struct btrfs_fs_info *info = trans->fs_info;
5492	struct btrfs_chunk_map *map;
5493	struct btrfs_block_group *block_group;
5494	u64 start = ctl->start;
5495	u64 type = ctl->type;
5496	int ret;
5497
5498	map = btrfs_alloc_chunk_map(num_stripes: ctl->num_stripes, GFP_NOFS);
5499	if (!map)
5500	return ERR_PTR(error: -ENOMEM);
5501
5502	map->start = start;
5503	map->chunk_len = ctl->chunk_size;
5504	map->stripe_size = ctl->stripe_size;
5505	map->type = type;
5506	map->io_align = BTRFS_STRIPE_LEN;
5507	map->io_width = BTRFS_STRIPE_LEN;
5508	map->sub_stripes = ctl->sub_stripes;
5509	map->num_stripes = ctl->num_stripes;
5510
5511	for (int i = 0; i < ctl->ndevs; i++) {
5512	for (int j = 0; j < ctl->dev_stripes; j++) {
5513	int s = i * ctl->dev_stripes + j;
5514	map->stripes[s].dev = devices_info[i].dev;
5515	map->stripes[s].physical = devices_info[i].dev_offset +
5516	j * ctl->stripe_size;
5517	}
5518	}
5519
5520	trace_btrfs_chunk_alloc(fs_info: info, map, offset: start, size: ctl->chunk_size);
5521
5522	ret = btrfs_add_chunk_map(fs_info: info, map);
5523	if (ret) {
5524	btrfs_free_chunk_map(map);
5525	return ERR_PTR(error: ret);
5526	}
5527
5528	block_group = btrfs_make_block_group(trans, space_info: ctl->space_info, type, chunk_offset: start,
5529	size: ctl->chunk_size);
5530	if (IS_ERR(ptr: block_group)) {
5531	btrfs_remove_chunk_map(fs_info: info, map);
5532	return block_group;
5533	}
5534
5535	for (int i = 0; i < map->num_stripes; i++) {
5536	struct btrfs_device *dev = map->stripes[i].dev;
5537
5538	btrfs_device_set_bytes_used(dev,
5539	size: dev->bytes_used + ctl->stripe_size);
5540	if (list_empty(head: &dev->post_commit_list))
5541	list_add_tail(new: &dev->post_commit_list,
5542	head: &trans->transaction->dev_update_list);
5543	}
5544
5545	atomic64_sub(i: ctl->stripe_size * map->num_stripes,
5546	v: &info->free_chunk_space);
5547
5548	check_raid56_incompat_flag(info, type);
5549	check_raid1c34_incompat_flag(info, type);
5550
5551	return block_group;
5552	}
5553
5554	struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans,
5555	struct btrfs_space_info *space_info,
5556	u64 type)
5557	{
5558	struct btrfs_fs_info *info = trans->fs_info;
5559	struct btrfs_fs_devices *fs_devices = info->fs_devices;
5560	struct btrfs_device_info AUTO_KFREE(devices_info);
5561	struct alloc_chunk_ctl ctl;
5562	int ret;
5563
5564	lockdep_assert_held(&info->chunk_mutex);
5565
5566	if (!alloc_profile_is_valid(flags: type, extended: 0)) {
5567	DEBUG_WARN("invalid alloc profile for type %llu", type);
5568	return ERR_PTR(error: -EINVAL);
5569	}
5570
5571	if (list_empty(head: &fs_devices->alloc_list)) {
5572	if (btrfs_test_opt(info, ENOSPC_DEBUG))
5573	btrfs_debug(info, "%s: no writable device", __func__);
5574	return ERR_PTR(error: -ENOSPC);
5575	}
5576
5577	if (!(type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
5578	btrfs_err(info, "invalid chunk type 0x%llx requested", type);
5579	DEBUG_WARN();
5580	return ERR_PTR(error: -EINVAL);
5581	}
5582
5583	ctl.start = find_next_chunk(fs_info: info);
5584	ctl.type = type;
5585	ctl.space_info = space_info;
5586	init_alloc_chunk_ctl(fs_devices, ctl: &ctl);
5587
5588	devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info),
5589	GFP_NOFS);
5590	if (!devices_info)
5591	return ERR_PTR(error: -ENOMEM);
5592
5593	ret = gather_device_info(fs_devices, ctl: &ctl, devices_info);
5594	if (ret < 0)
5595	return ERR_PTR(error: ret);
5596
5597	ret = decide_stripe_size(fs_devices, ctl: &ctl, devices_info);
5598	if (ret < 0)
5599	return ERR_PTR(error: ret);
5600
5601	return create_chunk(trans, ctl: &ctl, devices_info);
5602	}
5603
5604	/*
5605	* This function, btrfs_chunk_alloc_add_chunk_item(), typically belongs to the
5606	* phase 1 of chunk allocation. It belongs to phase 2 only when allocating system
5607	* chunks.
5608	*
5609	* See the comment at btrfs_chunk_alloc() for details about the chunk allocation
5610	* phases.
5611	*/
5612	int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans,
5613	struct btrfs_block_group *bg)
5614	{
5615	struct btrfs_fs_info *fs_info = trans->fs_info;
5616	struct btrfs_root *chunk_root = fs_info->chunk_root;
5617	struct btrfs_key key;
5618	struct btrfs_chunk *chunk;
5619	struct btrfs_stripe *stripe;
5620	struct btrfs_chunk_map *map;
5621	size_t item_size;
5622	int i;
5623	int ret;
5624
5625	/*
5626	* We take the chunk_mutex for 2 reasons:
5627	*
5628	* 1) Updates and insertions in the chunk btree must be done while holding
5629	* the chunk_mutex, as well as updating the system chunk array in the
5630	* superblock. See the comment on top of btrfs_chunk_alloc() for the
5631	* details;
5632	*
5633	* 2) To prevent races with the final phase of a device replace operation
5634	* that replaces the device object associated with the map's stripes,
5635	* because the device object's id can change at any time during that
5636	* final phase of the device replace operation
5637	* (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the
5638	* replaced device and then see it with an ID of BTRFS_DEV_REPLACE_DEVID,
5639	* which would cause a failure when updating the device item, which does
5640	* not exists, or persisting a stripe of the chunk item with such ID.
5641	* Here we can't use the device_list_mutex because our caller already
5642	* has locked the chunk_mutex, and the final phase of device replace
5643	* acquires both mutexes - first the device_list_mutex and then the
5644	* chunk_mutex. Using any of those two mutexes protects us from a
5645	* concurrent device replace.
5646	*/
5647	lockdep_assert_held(&fs_info->chunk_mutex);
5648
5649	map = btrfs_get_chunk_map(fs_info, logical: bg->start, length: bg->length);
5650	if (IS_ERR(ptr: map)) {
5651	ret = PTR_ERR(ptr: map);
5652	btrfs_abort_transaction(trans, ret);
5653	return ret;
5654	}
5655
5656	item_size = btrfs_chunk_item_size(num_stripes: map->num_stripes);
5657
5658	chunk = kzalloc(item_size, GFP_NOFS);
5659	if (unlikely(!chunk)) {
5660	ret = -ENOMEM;
5661	btrfs_abort_transaction(trans, ret);
5662	goto out;
5663	}
5664
5665	for (i = 0; i < map->num_stripes; i++) {
5666	struct btrfs_device *device = map->stripes[i].dev;
5667
5668	ret = btrfs_update_device(trans, device);
5669	if (ret)
5670	goto out;
5671	}
5672
5673	stripe = &chunk->stripe;
5674	for (i = 0; i < map->num_stripes; i++) {
5675	struct btrfs_device *device = map->stripes[i].dev;
5676	const u64 dev_offset = map->stripes[i].physical;
5677
5678	btrfs_set_stack_stripe_devid(s: stripe, val: device->devid);
5679	btrfs_set_stack_stripe_offset(s: stripe, val: dev_offset);
5680	memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE);
5681	stripe++;
5682	}
5683
5684	btrfs_set_stack_chunk_length(s: chunk, val: bg->length);
5685	btrfs_set_stack_chunk_owner(s: chunk, BTRFS_EXTENT_TREE_OBJECTID);
5686	btrfs_set_stack_chunk_stripe_len(s: chunk, BTRFS_STRIPE_LEN);
5687	btrfs_set_stack_chunk_type(s: chunk, val: map->type);
5688	btrfs_set_stack_chunk_num_stripes(s: chunk, val: map->num_stripes);
5689	btrfs_set_stack_chunk_io_align(s: chunk, BTRFS_STRIPE_LEN);
5690	btrfs_set_stack_chunk_io_width(s: chunk, BTRFS_STRIPE_LEN);
5691	btrfs_set_stack_chunk_sector_size(s: chunk, val: fs_info->sectorsize);
5692	btrfs_set_stack_chunk_sub_stripes(s: chunk, val: map->sub_stripes);
5693
5694	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
5695	key.type = BTRFS_CHUNK_ITEM_KEY;
5696	key.offset = bg->start;
5697
5698	ret = btrfs_insert_item(trans, root: chunk_root, key: &key, data: chunk, data_size: item_size);
5699	if (ret)
5700	goto out;
5701
5702	set_bit(nr: BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, addr: &bg->runtime_flags);
5703
5704	if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
5705	ret = btrfs_add_system_chunk(fs_info, key: &key, chunk, item_size);
5706	if (ret)
5707	goto out;
5708	}
5709
5710	out:
5711	kfree(objp: chunk);
5712	btrfs_free_chunk_map(map);
5713	return ret;
5714	}
5715
5716	static noinline int init_first_rw_device(struct btrfs_trans_handle *trans)
5717	{
5718	struct btrfs_fs_info *fs_info = trans->fs_info;
5719	u64 alloc_profile;
5720	struct btrfs_block_group *meta_bg;
5721	struct btrfs_space_info *meta_space_info;
5722	struct btrfs_block_group *sys_bg;
5723	struct btrfs_space_info *sys_space_info;
5724
5725	/*
5726	* When adding a new device for sprouting, the seed device is read-only
5727	* so we must first allocate a metadata and a system chunk. But before
5728	* adding the block group items to the extent, device and chunk btrees,
5729	* we must first:
5730	*
5731	* 1) Create both chunks without doing any changes to the btrees, as
5732	* otherwise we would get -ENOSPC since the block groups from the
5733	* seed device are read-only;
5734	*
5735	* 2) Add the device item for the new sprout device - finishing the setup
5736	* of a new block group requires updating the device item in the chunk
5737	* btree, so it must exist when we attempt to do it. The previous step
5738	* ensures this does not fail with -ENOSPC.
5739	*
5740	* After that we can add the block group items to their btrees:
5741	* update existing device item in the chunk btree, add a new block group
5742	* item to the extent btree, add a new chunk item to the chunk btree and
5743	* finally add the new device extent items to the devices btree.
5744	*/
5745
5746	alloc_profile = btrfs_metadata_alloc_profile(fs_info);
5747	meta_space_info = btrfs_find_space_info(info: fs_info, flags: alloc_profile);
5748	if (!meta_space_info) {
5749	DEBUG_WARN();
5750	return -EINVAL;
5751	}
5752	meta_bg = btrfs_create_chunk(trans, space_info: meta_space_info, type: alloc_profile);
5753	if (IS_ERR(ptr: meta_bg))
5754	return PTR_ERR(ptr: meta_bg);
5755
5756	alloc_profile = btrfs_system_alloc_profile(fs_info);
5757	sys_space_info = btrfs_find_space_info(info: fs_info, flags: alloc_profile);
5758	if (!sys_space_info) {
5759	DEBUG_WARN();
5760	return -EINVAL;
5761	}
5762	sys_bg = btrfs_create_chunk(trans, space_info: sys_space_info, type: alloc_profile);
5763	if (IS_ERR(ptr: sys_bg))
5764	return PTR_ERR(ptr: sys_bg);
5765
5766	return 0;
5767	}
5768
5769	static inline int btrfs_chunk_max_errors(struct btrfs_chunk_map *map)
5770	{
5771	const int index = btrfs_bg_flags_to_raid_index(flags: map->type);
5772
5773	return btrfs_raid_array[index].tolerated_failures;
5774	}
5775
5776	bool btrfs_chunk_writeable(struct btrfs_fs_info *fs_info, u64 chunk_offset)
5777	{
5778	struct btrfs_chunk_map *map;
5779	int miss_ndevs = 0;
5780	int i;
5781	bool ret = true;
5782
5783	map = btrfs_get_chunk_map(fs_info, logical: chunk_offset, length: 1);
5784	if (IS_ERR(ptr: map))
5785	return false;
5786
5787	for (i = 0; i < map->num_stripes; i++) {
5788	if (test_bit(BTRFS_DEV_STATE_MISSING,
5789	&map->stripes[i].dev->dev_state)) {
5790	miss_ndevs++;
5791	continue;
5792	}
5793	if (!test_bit(BTRFS_DEV_STATE_WRITEABLE,
5794	&map->stripes[i].dev->dev_state)) {
5795	ret = false;
5796	goto end;
5797	}
5798	}
5799
5800	/*
5801	* If the number of missing devices is larger than max errors, we can
5802	* not write the data into that chunk successfully.
5803	*/
5804	if (miss_ndevs > btrfs_chunk_max_errors(map))
5805	ret = false;
5806	end:
5807	btrfs_free_chunk_map(map);
5808	return ret;
5809	}
5810
5811	void btrfs_mapping_tree_free(struct btrfs_fs_info *fs_info)
5812	{
5813	write_lock(&fs_info->mapping_tree_lock);
5814	while (!RB_EMPTY_ROOT(&fs_info->mapping_tree.rb_root)) {
5815	struct btrfs_chunk_map *map;
5816	struct rb_node *node;
5817
5818	node = rb_first_cached(&fs_info->mapping_tree);
5819	map = rb_entry(node, struct btrfs_chunk_map, rb_node);
5820	rb_erase_cached(node: &map->rb_node, root: &fs_info->mapping_tree);
5821	RB_CLEAR_NODE(&map->rb_node);
5822	chunk_map_device_clear_bits(map, CHUNK_ALLOCATED);
5823	/* Once for the tree ref. */
5824	btrfs_free_chunk_map(map);
5825	cond_resched_rwlock_write(&fs_info->mapping_tree_lock);
5826	}
5827	write_unlock(&fs_info->mapping_tree_lock);
5828	}
5829
5830	static int btrfs_chunk_map_num_copies(const struct btrfs_chunk_map *map)
5831	{
5832	enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(flags: map->type);
5833
5834	if (map->type & BTRFS_BLOCK_GROUP_RAID5)
5835	return 2;
5836
5837	/*
5838	* There could be two corrupted data stripes, we need to loop retry in
5839	* order to rebuild the correct data.
5840	*
5841	* Fail a stripe at a time on every retry except the stripe under
5842	* reconstruction.
5843	*/
5844	if (map->type & BTRFS_BLOCK_GROUP_RAID6)
5845	return map->num_stripes;
5846
5847	/* Non-RAID56, use their ncopies from btrfs_raid_array. */
5848	return btrfs_raid_array[index].ncopies;
5849	}
5850
5851	int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5852	{
5853	struct btrfs_chunk_map *map;
5854	int ret;
5855
5856	map = btrfs_get_chunk_map(fs_info, logical, length: len);
5857	if (IS_ERR(ptr: map))
5858	/*
5859	* We could return errors for these cases, but that could get
5860	* ugly and we'd probably do the same thing which is just not do
5861	* anything else and exit, so return 1 so the callers don't try
5862	* to use other copies.
5863	*/
5864	return 1;
5865
5866	ret = btrfs_chunk_map_num_copies(map);
5867	btrfs_free_chunk_map(map);
5868	return ret;
5869	}
5870
5871	unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
5872	u64 logical)
5873	{
5874	struct btrfs_chunk_map *map;
5875	unsigned long len = fs_info->sectorsize;
5876
5877	if (!btrfs_fs_incompat(fs_info, RAID56))
5878	return len;
5879
5880	map = btrfs_get_chunk_map(fs_info, logical, length: len);
5881
5882	if (!WARN_ON(IS_ERR(map))) {
5883	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
5884	len = btrfs_stripe_nr_to_offset(stripe_nr: nr_data_stripes(map));
5885	btrfs_free_chunk_map(map);
5886	}
5887	return len;
5888	}
5889
5890	#ifdef CONFIG_BTRFS_EXPERIMENTAL
5891	static int btrfs_read_preferred(struct btrfs_chunk_map *map, int first, int num_stripes)
5892	{
5893	for (int index = first; index < first + num_stripes; index++) {
5894	const struct btrfs_device *device = map->stripes[index].dev;
5895
5896	if (device->devid == READ_ONCE(device->fs_devices->read_devid))
5897	return index;
5898	}
5899
5900	/* If no read-preferred device is set use the first stripe. */
5901	return first;
5902	}
5903
5904	struct stripe_mirror {
5905	u64 devid;
5906	int num;
5907	};
5908
5909	static int btrfs_cmp_devid(const void *a, const void *b)
5910	{
5911	const struct stripe_mirror *s1 = (const struct stripe_mirror *)a;
5912	const struct stripe_mirror *s2 = (const struct stripe_mirror *)b;
5913
5914	if (s1->devid < s2->devid)
5915	return -1;
5916	if (s1->devid > s2->devid)
5917	return 1;
5918	return 0;
5919	}
5920
5921	/*
5922	* Select a stripe for reading using the round-robin algorithm.
5923	*
5924	* 1. Compute the read cycle as the total sectors read divided by the minimum
5925	* sectors per device.
5926	* 2. Determine the stripe number for the current read by taking the modulus
5927	* of the read cycle with the total number of stripes:
5928	*
5929	* stripe index = (total sectors / min sectors per dev) % num stripes
5930	*
5931	* The calculated stripe index is then used to select the corresponding device
5932	* from the list of devices, which is ordered by devid.
5933	*/
5934	static int btrfs_read_rr(const struct btrfs_chunk_map *map, int first, int num_stripes)
5935	{
5936	struct stripe_mirror stripes[BTRFS_RAID1_MAX_MIRRORS] = { 0 };
5937	struct btrfs_device *device = map->stripes[first].dev;
5938	struct btrfs_fs_info *fs_info = device->fs_devices->fs_info;
5939	unsigned int read_cycle;
5940	unsigned int total_reads;
5941	unsigned int min_reads_per_dev;
5942
5943	total_reads = percpu_counter_sum(fbc: &fs_info->stats_read_blocks);
5944	min_reads_per_dev = READ_ONCE(fs_info->fs_devices->rr_min_contig_read) >>
5945	fs_info->sectorsize_bits;
5946
5947	for (int index = 0, i = first; i < first + num_stripes; i++) {
5948	stripes[index].devid = map->stripes[i].dev->devid;
5949	stripes[index].num = i;
5950	index++;
5951	}
5952	sort(base: stripes, num: num_stripes, size: sizeof(struct stripe_mirror),
5953	cmp_func: btrfs_cmp_devid, NULL);
5954
5955	read_cycle = total_reads / min_reads_per_dev;
5956	return stripes[read_cycle % num_stripes].num;
5957	}
5958	#endif
5959
5960	static int find_live_mirror(struct btrfs_fs_info *fs_info,
5961	struct btrfs_chunk_map *map, int first,
5962	bool dev_replace_is_ongoing)
5963	{
5964	const enum btrfs_read_policy policy = READ_ONCE(fs_info->fs_devices->read_policy);
5965	int i;
5966	int num_stripes;
5967	int preferred_mirror;
5968	int tolerance;
5969	struct btrfs_device *srcdev;
5970
5971	ASSERT((map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10)),
5972	"type=%llu", map->type);
5973
5974	if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5975	num_stripes = map->sub_stripes;
5976	else
5977	num_stripes = map->num_stripes;
5978
5979	switch (policy) {
5980	default:
5981	/* Shouldn't happen, just warn and use pid instead of failing */
5982	btrfs_warn_rl(fs_info, "unknown read_policy type %u, reset to pid",
5983	policy);
5984	WRITE_ONCE(fs_info->fs_devices->read_policy, BTRFS_READ_POLICY_PID);
5985	fallthrough;
5986	case BTRFS_READ_POLICY_PID:
5987	preferred_mirror = first + (current->pid % num_stripes);
5988	break;
5989	#ifdef CONFIG_BTRFS_EXPERIMENTAL
5990	case BTRFS_READ_POLICY_RR:
5991	preferred_mirror = btrfs_read_rr(map, first, num_stripes);
5992	break;
5993	case BTRFS_READ_POLICY_DEVID:
5994	preferred_mirror = btrfs_read_preferred(map, first, num_stripes);
5995	break;
5996	#endif
5997	}
5998
5999	if (dev_replace_is_ongoing &&
6000	fs_info->dev_replace.cont_reading_from_srcdev_mode ==
6001	BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID)
6002	srcdev = fs_info->dev_replace.srcdev;
6003	else
6004	srcdev = NULL;
6005
6006	/*
6007	* try to avoid the drive that is the source drive for a
6008	* dev-replace procedure, only choose it if no other non-missing
6009	* mirror is available
6010	*/
6011	for (tolerance = 0; tolerance < 2; tolerance++) {
6012	if (map->stripes[preferred_mirror].dev->bdev &&
6013	(tolerance || map->stripes[preferred_mirror].dev != srcdev))
6014	return preferred_mirror;
6015	for (i = first; i < first + num_stripes; i++) {
6016	if (map->stripes[i].dev->bdev &&
6017	(tolerance || map->stripes[i].dev != srcdev))
6018	return i;
6019	}
6020	}
6021
6022	/* we couldn't find one that doesn't fail. Just return something
6023	* and the io error handling code will clean up eventually
6024	*/
6025	return preferred_mirror;
6026	}
6027
6028	EXPORT_FOR_TESTS
6029	struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_info,
6030	u64 logical, u16 total_stripes)
6031	{
6032	struct btrfs_io_context *bioc;
6033
6034	bioc = kzalloc(struct_size(bioc, stripes, total_stripes), GFP_NOFS);
6035
6036	if (!bioc)
6037	return NULL;
6038
6039	refcount_set(r: &bioc->refs, n: 1);
6040
6041	bioc->fs_info = fs_info;
6042	bioc->replace_stripe_src = -1;
6043	bioc->full_stripe_logical = (u64)-1;
6044	bioc->logical = logical;
6045
6046	return bioc;
6047	}
6048
6049	void btrfs_get_bioc(struct btrfs_io_context *bioc)
6050	{
6051	WARN_ON(!refcount_read(&bioc->refs));
6052	refcount_inc(r: &bioc->refs);
6053	}
6054
6055	void btrfs_put_bioc(struct btrfs_io_context *bioc)
6056	{
6057	if (!bioc)
6058	return;
6059	if (refcount_dec_and_test(r: &bioc->refs))
6060	kfree(objp: bioc);
6061	}
6062
6063	/*
6064	* Please note that, discard won't be sent to target device of device
6065	* replace.
6066	*/
6067	struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
6068	u64 logical, u64 *length_ret,
6069	u32 *num_stripes)
6070	{
6071	struct btrfs_chunk_map *map;
6072	struct btrfs_discard_stripe *stripes;
6073	u64 length = *length_ret;
6074	u64 offset;
6075	u32 stripe_nr;
6076	u32 stripe_nr_end;
6077	u32 stripe_cnt;
6078	u64 stripe_end_offset;
6079	u64 stripe_offset;
6080	u32 stripe_index;
6081	u32 factor = 0;
6082	u32 sub_stripes = 0;
6083	u32 stripes_per_dev = 0;
6084	u32 remaining_stripes = 0;
6085	u32 last_stripe = 0;
6086	int ret;
6087	int i;
6088
6089	map = btrfs_get_chunk_map(fs_info, logical, length);
6090	if (IS_ERR(ptr: map))
6091	return ERR_CAST(ptr: map);
6092
6093	/* we don't discard raid56 yet */
6094	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6095	ret = -EOPNOTSUPP;
6096	goto out_free_map;
6097	}
6098
6099	offset = logical - map->start;
6100	length = min_t(u64, map->start + map->chunk_len - logical, length);
6101	*length_ret = length;
6102
6103	/*
6104	* stripe_nr counts the total number of stripes we have to stride
6105	* to get to this block
6106	*/
6107	stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT;
6108
6109	/* stripe_offset is the offset of this block in its stripe */
6110	stripe_offset = offset - btrfs_stripe_nr_to_offset(stripe_nr);
6111
6112	stripe_nr_end = round_up(offset + length, BTRFS_STRIPE_LEN) >>
6113	BTRFS_STRIPE_LEN_SHIFT;
6114	stripe_cnt = stripe_nr_end - stripe_nr;
6115	stripe_end_offset = btrfs_stripe_nr_to_offset(stripe_nr: stripe_nr_end) -
6116	(offset + length);
6117	/*
6118	* after this, stripe_nr is the number of stripes on this
6119	* device we have to walk to find the data, and stripe_index is
6120	* the number of our device in the stripe array
6121	*/
6122	*num_stripes = 1;
6123	stripe_index = 0;
6124	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
6125	BTRFS_BLOCK_GROUP_RAID10)) {
6126	if (map->type & BTRFS_BLOCK_GROUP_RAID0)
6127	sub_stripes = 1;
6128	else
6129	sub_stripes = map->sub_stripes;
6130
6131	factor = map->num_stripes / sub_stripes;
6132	*num_stripes = min_t(u64, map->num_stripes,
6133	sub_stripes * stripe_cnt);
6134	stripe_index = stripe_nr % factor;
6135	stripe_nr /= factor;
6136	stripe_index *= sub_stripes;
6137
6138	remaining_stripes = stripe_cnt % factor;
6139	stripes_per_dev = stripe_cnt / factor;
6140	last_stripe = ((stripe_nr_end - 1) % factor) * sub_stripes;
6141	} else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK |
6142	BTRFS_BLOCK_GROUP_DUP)) {
6143	*num_stripes = map->num_stripes;
6144	} else {
6145	stripe_index = stripe_nr % map->num_stripes;
6146	stripe_nr /= map->num_stripes;
6147	}
6148
6149	stripes = kcalloc(*num_stripes, sizeof(*stripes), GFP_NOFS);
6150	if (!stripes) {
6151	ret = -ENOMEM;
6152	goto out_free_map;
6153	}
6154
6155	for (i = 0; i < *num_stripes; i++) {
6156	stripes[i].physical =
6157	map->stripes[stripe_index].physical +
6158	stripe_offset + btrfs_stripe_nr_to_offset(stripe_nr);
6159	stripes[i].dev = map->stripes[stripe_index].dev;
6160
6161	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
6162	BTRFS_BLOCK_GROUP_RAID10)) {
6163	stripes[i].length = btrfs_stripe_nr_to_offset(stripe_nr: stripes_per_dev);
6164
6165	if (i / sub_stripes < remaining_stripes)
6166	stripes[i].length += BTRFS_STRIPE_LEN;
6167
6168	/*
6169	* Special for the first stripe and
6170	* the last stripe:
6171	*
6172	* |-------|...|-------|
6173	* |----------|
6174	* off end_off
6175	*/
6176	if (i < sub_stripes)
6177	stripes[i].length -= stripe_offset;
6178
6179	if (stripe_index >= last_stripe &&
6180	stripe_index <= (last_stripe +
6181	sub_stripes - 1))
6182	stripes[i].length -= stripe_end_offset;
6183
6184	if (i == sub_stripes - 1)
6185	stripe_offset = 0;
6186	} else {
6187	stripes[i].length = length;
6188	}
6189
6190	stripe_index++;
6191	if (stripe_index == map->num_stripes) {
6192	stripe_index = 0;
6193	stripe_nr++;
6194	}
6195	}
6196
6197	btrfs_free_chunk_map(map);
6198	return stripes;
6199	out_free_map:
6200	btrfs_free_chunk_map(map);
6201	return ERR_PTR(error: ret);
6202	}
6203
6204	static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical)
6205	{
6206	struct btrfs_block_group *cache;
6207	bool ret;
6208
6209	/* Non zoned filesystem does not use "to_copy" flag */
6210	if (!btrfs_is_zoned(fs_info))
6211	return false;
6212
6213	cache = btrfs_lookup_block_group(info: fs_info, bytenr: logical);
6214
6215	ret = test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags);
6216
6217	btrfs_put_block_group(cache);
6218	return ret;
6219	}
6220
6221	static void handle_ops_on_dev_replace(struct btrfs_io_context *bioc,
6222	struct btrfs_dev_replace *dev_replace,
6223	u64 logical,
6224	struct btrfs_io_geometry *io_geom)
6225	{
6226	u64 srcdev_devid = dev_replace->srcdev->devid;
6227	/*
6228	* At this stage, num_stripes is still the real number of stripes,
6229	* excluding the duplicated stripes.
6230	*/
6231	int num_stripes = io_geom->num_stripes;
6232	int max_errors = io_geom->max_errors;
6233	int nr_extra_stripes = 0;
6234	int i;
6235
6236	/*
6237	* A block group which has "to_copy" set will eventually be copied by
6238	* the dev-replace process. We can avoid cloning IO here.
6239	*/
6240	if (is_block_group_to_copy(fs_info: dev_replace->srcdev->fs_info, logical))
6241	return;
6242
6243	/*
6244	* Duplicate the write operations while the dev-replace procedure is
6245	* running. Since the copying of the old disk to the new disk takes
6246	* place at run time while the filesystem is mounted writable, the
6247	* regular write operations to the old disk have to be duplicated to go
6248	* to the new disk as well.
6249	*
6250	* Note that device->missing is handled by the caller, and that the
6251	* write to the old disk is already set up in the stripes array.
6252	*/
6253	for (i = 0; i < num_stripes; i++) {
6254	struct btrfs_io_stripe *old = &bioc->stripes[i];
6255	struct btrfs_io_stripe *new = &bioc->stripes[num_stripes + nr_extra_stripes];
6256
6257	if (old->dev->devid != srcdev_devid)
6258	continue;
6259
6260	new->physical = old->physical;
6261	new->dev = dev_replace->tgtdev;
6262	if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK)
6263	bioc->replace_stripe_src = i;
6264	nr_extra_stripes++;
6265	}
6266
6267	/* We can only have at most 2 extra nr_stripes (for DUP). */
6268	ASSERT(nr_extra_stripes <= 2, "nr_extra_stripes=%d", nr_extra_stripes);
6269	/*
6270	* For GET_READ_MIRRORS, we can only return at most 1 extra stripe for
6271	* replace.
6272	* If we have 2 extra stripes, only choose the one with smaller physical.
6273	*/
6274	if (io_geom->op == BTRFS_MAP_GET_READ_MIRRORS && nr_extra_stripes == 2) {
6275	struct btrfs_io_stripe *first = &bioc->stripes[num_stripes];
6276	struct btrfs_io_stripe *second = &bioc->stripes[num_stripes + 1];
6277
6278	/* Only DUP can have two extra stripes. */
6279	ASSERT(bioc->map_type & BTRFS_BLOCK_GROUP_DUP,
6280	"map_type=%llu", bioc->map_type);
6281
6282	/*
6283	* Swap the last stripe stripes and reduce @nr_extra_stripes.
6284	* The extra stripe would still be there, but won't be accessed.
6285	*/
6286	if (first->physical > second->physical) {
6287	swap(second->physical, first->physical);
6288	swap(second->dev, first->dev);
6289	nr_extra_stripes--;
6290	}
6291	}
6292
6293	io_geom->num_stripes = num_stripes + nr_extra_stripes;
6294	io_geom->max_errors = max_errors + nr_extra_stripes;
6295	bioc->replace_nr_stripes = nr_extra_stripes;
6296	}
6297
6298	static u64 btrfs_max_io_len(struct btrfs_chunk_map *map, u64 offset,
6299	struct btrfs_io_geometry *io_geom)
6300	{
6301	/*
6302	* Stripe_nr is the stripe where this block falls. stripe_offset is
6303	* the offset of this block in its stripe.
6304	*/
6305	io_geom->stripe_offset = offset & BTRFS_STRIPE_LEN_MASK;
6306	io_geom->stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT;
6307	ASSERT(io_geom->stripe_offset < U32_MAX,
6308	"stripe_offset=%llu", io_geom->stripe_offset);
6309
6310	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6311	unsigned long full_stripe_len =
6312	btrfs_stripe_nr_to_offset(stripe_nr: nr_data_stripes(map));
6313
6314	/*
6315	* For full stripe start, we use previously calculated
6316	* @stripe_nr. Align it to nr_data_stripes, then multiply with
6317	* STRIPE_LEN.
6318	*
6319	* By this we can avoid u64 division completely. And we have
6320	* to go rounddown(), not round_down(), as nr_data_stripes is
6321	* not ensured to be power of 2.
6322	*/
6323	io_geom->raid56_full_stripe_start = btrfs_stripe_nr_to_offset(
6324	rounddown(io_geom->stripe_nr, nr_data_stripes(map)));
6325
6326	ASSERT(io_geom->raid56_full_stripe_start + full_stripe_len > offset,
6327	"raid56_full_stripe_start=%llu full_stripe_len=%lu offset=%llu",
6328	io_geom->raid56_full_stripe_start, full_stripe_len, offset);
6329	ASSERT(io_geom->raid56_full_stripe_start <= offset,
6330	"raid56_full_stripe_start=%llu offset=%llu",
6331	io_geom->raid56_full_stripe_start, offset);
6332	/*
6333	* For writes to RAID56, allow to write a full stripe set, but
6334	* no straddling of stripe sets.
6335	*/
6336	if (io_geom->op == BTRFS_MAP_WRITE)
6337	return full_stripe_len - (offset - io_geom->raid56_full_stripe_start);
6338	}
6339
6340	/*
6341	* For other RAID types and for RAID56 reads, allow a single stripe (on
6342	* a single disk).
6343	*/
6344	if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK)
6345	return BTRFS_STRIPE_LEN - io_geom->stripe_offset;
6346	return U64_MAX;
6347	}
6348
6349	static int set_io_stripe(struct btrfs_fs_info *fs_info, u64 logical,
6350	u64 *length, struct btrfs_io_stripe *dst,
6351	struct btrfs_chunk_map *map,
6352	struct btrfs_io_geometry *io_geom)
6353	{
6354	dst->dev = map->stripes[io_geom->stripe_index].dev;
6355
6356	if (io_geom->op == BTRFS_MAP_READ && io_geom->use_rst)
6357	return btrfs_get_raid_extent_offset(fs_info, logical, length,
6358	map_type: map->type,
6359	stripe_index: io_geom->stripe_index, stripe: dst);
6360
6361	dst->physical = map->stripes[io_geom->stripe_index].physical +
6362	io_geom->stripe_offset +
6363	btrfs_stripe_nr_to_offset(stripe_nr: io_geom->stripe_nr);
6364	return 0;
6365	}
6366
6367	static bool is_single_device_io(struct btrfs_fs_info *fs_info,
6368	const struct btrfs_io_stripe *smap,
6369	const struct btrfs_chunk_map *map,
6370	int num_alloc_stripes,
6371	struct btrfs_io_geometry *io_geom)
6372	{
6373	if (!smap)
6374	return false;
6375
6376	if (num_alloc_stripes != 1)
6377	return false;
6378
6379	if (io_geom->use_rst && io_geom->op != BTRFS_MAP_READ)
6380	return false;
6381
6382	if ((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) && io_geom->mirror_num > 1)
6383	return false;
6384
6385	return true;
6386	}
6387
6388	static void map_blocks_raid0(const struct btrfs_chunk_map *map,
6389	struct btrfs_io_geometry *io_geom)
6390	{
6391	io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes;
6392	io_geom->stripe_nr /= map->num_stripes;
6393	if (io_geom->op == BTRFS_MAP_READ)
6394	io_geom->mirror_num = 1;
6395	}
6396
6397	static void map_blocks_raid1(struct btrfs_fs_info *fs_info,
6398	struct btrfs_chunk_map *map,
6399	struct btrfs_io_geometry *io_geom,
6400	bool dev_replace_is_ongoing)
6401	{
6402	if (io_geom->op != BTRFS_MAP_READ) {
6403	io_geom->num_stripes = map->num_stripes;
6404	return;
6405	}
6406
6407	if (io_geom->mirror_num) {
6408	io_geom->stripe_index = io_geom->mirror_num - 1;
6409	return;
6410	}
6411
6412	io_geom->stripe_index = find_live_mirror(fs_info, map, first: 0,
6413	dev_replace_is_ongoing);
6414	io_geom->mirror_num = io_geom->stripe_index + 1;
6415	}
6416
6417	static void map_blocks_dup(const struct btrfs_chunk_map *map,
6418	struct btrfs_io_geometry *io_geom)
6419	{
6420	if (io_geom->op != BTRFS_MAP_READ) {
6421	io_geom->num_stripes = map->num_stripes;
6422	return;
6423	}
6424
6425	if (io_geom->mirror_num) {
6426	io_geom->stripe_index = io_geom->mirror_num - 1;
6427	return;
6428	}
6429
6430	io_geom->mirror_num = 1;
6431	}
6432
6433	static void map_blocks_raid10(struct btrfs_fs_info *fs_info,
6434	struct btrfs_chunk_map *map,
6435	struct btrfs_io_geometry *io_geom,
6436	bool dev_replace_is_ongoing)
6437	{
6438	u32 factor = map->num_stripes / map->sub_stripes;
6439	int old_stripe_index;
6440
6441	io_geom->stripe_index = (io_geom->stripe_nr % factor) * map->sub_stripes;
6442	io_geom->stripe_nr /= factor;
6443
6444	if (io_geom->op != BTRFS_MAP_READ) {
6445	io_geom->num_stripes = map->sub_stripes;
6446	return;
6447	}
6448
6449	if (io_geom->mirror_num) {
6450	io_geom->stripe_index += io_geom->mirror_num - 1;
6451	return;
6452	}
6453
6454	old_stripe_index = io_geom->stripe_index;
6455	io_geom->stripe_index = find_live_mirror(fs_info, map,
6456	first: io_geom->stripe_index,
6457	dev_replace_is_ongoing);
6458	io_geom->mirror_num = io_geom->stripe_index - old_stripe_index + 1;
6459	}
6460
6461	static void map_blocks_raid56_write(struct btrfs_chunk_map *map,
6462	struct btrfs_io_geometry *io_geom,
6463	u64 logical, u64 *length)
6464	{
6465	int data_stripes = nr_data_stripes(map);
6466
6467	/*
6468	* Needs full stripe mapping.
6469	*
6470	* Push stripe_nr back to the start of the full stripe For those cases
6471	* needing a full stripe, @stripe_nr is the full stripe number.
6472	*
6473	* Originally we go raid56_full_stripe_start / full_stripe_len, but
6474	* that can be expensive. Here we just divide @stripe_nr with
6475	* @data_stripes.
6476	*/
6477	io_geom->stripe_nr /= data_stripes;
6478
6479	/* RAID[56] write or recovery. Return all stripes */
6480	io_geom->num_stripes = map->num_stripes;
6481	io_geom->max_errors = btrfs_chunk_max_errors(map);
6482
6483	/* Return the length to the full stripe end. */
6484	*length = min(logical + *length,
6485	io_geom->raid56_full_stripe_start + map->start +
6486	btrfs_stripe_nr_to_offset(data_stripes)) -
6487	logical;
6488	io_geom->stripe_index = 0;
6489	io_geom->stripe_offset = 0;
6490	}
6491
6492	static void map_blocks_raid56_read(struct btrfs_chunk_map *map,
6493	struct btrfs_io_geometry *io_geom)
6494	{
6495	int data_stripes = nr_data_stripes(map);
6496
6497	ASSERT(io_geom->mirror_num <= 1, "mirror_num=%d", io_geom->mirror_num);
6498	/* Just grab the data stripe directly. */
6499	io_geom->stripe_index = io_geom->stripe_nr % data_stripes;
6500	io_geom->stripe_nr /= data_stripes;
6501
6502	/* We distribute the parity blocks across stripes. */
6503	io_geom->stripe_index =
6504	(io_geom->stripe_nr + io_geom->stripe_index) % map->num_stripes;
6505
6506	if (io_geom->op == BTRFS_MAP_READ && io_geom->mirror_num < 1)
6507	io_geom->mirror_num = 1;
6508	}
6509
6510	static void map_blocks_single(const struct btrfs_chunk_map *map,
6511	struct btrfs_io_geometry *io_geom)
6512	{
6513	io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes;
6514	io_geom->stripe_nr /= map->num_stripes;
6515	io_geom->mirror_num = io_geom->stripe_index + 1;
6516	}
6517
6518	/*
6519	* Map one logical range to one or more physical ranges.
6520	*
6521	* @length: (Mandatory) mapped length of this run.
6522	* One logical range can be split into different segments
6523	* due to factors like zones and RAID0/5/6/10 stripe
6524	* boundaries.
6525	*
6526	* @bioc_ret: (Mandatory) returned btrfs_io_context structure.
6527	* which has one or more physical ranges (btrfs_io_stripe)
6528	* recorded inside.
6529	* Caller should call btrfs_put_bioc() to free it after use.
6530	*
6531	* @smap: (Optional) single physical range optimization.
6532	* If the map request can be fulfilled by one single
6533	* physical range, and this is parameter is not NULL,
6534	* then @bioc_ret would be NULL, and @smap would be
6535	* updated.
6536	*
6537	* @mirror_num_ret: (Mandatory) returned mirror number if the original
6538	* value is 0.
6539	*
6540	* Mirror number 0 means to choose any live mirrors.
6541	*
6542	* For non-RAID56 profiles, non-zero mirror_num means
6543	* the Nth mirror. (e.g. mirror_num 1 means the first
6544	* copy).
6545	*
6546	* For RAID56 profile, mirror 1 means rebuild from P and
6547	* the remaining data stripes.
6548	*
6549	* For RAID6 profile, mirror > 2 means mark another
6550	* data/P stripe error and rebuild from the remaining
6551	* stripes..
6552	*/
6553	int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
6554	u64 logical, u64 *length,
6555	struct btrfs_io_context **bioc_ret,
6556	struct btrfs_io_stripe *smap, int *mirror_num_ret)
6557	{
6558	struct btrfs_chunk_map *map;
6559	struct btrfs_io_geometry io_geom = { 0 };
6560	u64 map_offset;
6561	int ret = 0;
6562	int num_copies;
6563	struct btrfs_io_context *bioc = NULL;
6564	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
6565	bool dev_replace_is_ongoing = false;
6566	u16 num_alloc_stripes;
6567	u64 max_len;
6568
6569	ASSERT(bioc_ret);
6570
6571	io_geom.mirror_num = (mirror_num_ret ? *mirror_num_ret : 0);
6572	io_geom.num_stripes = 1;
6573	io_geom.stripe_index = 0;
6574	io_geom.op = op;
6575
6576	map = btrfs_get_chunk_map(fs_info, logical, length: *length);
6577	if (IS_ERR(ptr: map))
6578	return PTR_ERR(ptr: map);
6579
6580	num_copies = btrfs_chunk_map_num_copies(map);
6581	if (io_geom.mirror_num > num_copies)
6582	return -EINVAL;
6583
6584	map_offset = logical - map->start;
6585	io_geom.raid56_full_stripe_start = (u64)-1;
6586	max_len = btrfs_max_io_len(map, offset: map_offset, io_geom: &io_geom);
6587	*length = min_t(u64, map->chunk_len - map_offset, max_len);
6588	io_geom.use_rst = btrfs_need_stripe_tree_update(fs_info, map_type: map->type);
6589
6590	if (dev_replace->replace_task != current)
6591	down_read(sem: &dev_replace->rwsem);
6592
6593	dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace);
6594	/*
6595	* Hold the semaphore for read during the whole operation, write is
6596	* requested at commit time but must wait.
6597	*/
6598	if (!dev_replace_is_ongoing && dev_replace->replace_task != current)
6599	up_read(sem: &dev_replace->rwsem);
6600
6601	switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
6602	case BTRFS_BLOCK_GROUP_RAID0:
6603	map_blocks_raid0(map, io_geom: &io_geom);
6604	break;
6605	case BTRFS_BLOCK_GROUP_RAID1:
6606	case BTRFS_BLOCK_GROUP_RAID1C3:
6607	case BTRFS_BLOCK_GROUP_RAID1C4:
6608	map_blocks_raid1(fs_info, map, io_geom: &io_geom, dev_replace_is_ongoing);
6609	break;
6610	case BTRFS_BLOCK_GROUP_DUP:
6611	map_blocks_dup(map, io_geom: &io_geom);
6612	break;
6613	case BTRFS_BLOCK_GROUP_RAID10:
6614	map_blocks_raid10(fs_info, map, io_geom: &io_geom, dev_replace_is_ongoing);
6615	break;
6616	case BTRFS_BLOCK_GROUP_RAID5:
6617	case BTRFS_BLOCK_GROUP_RAID6:
6618	if (op != BTRFS_MAP_READ || io_geom.mirror_num > 1)
6619	map_blocks_raid56_write(map, io_geom: &io_geom, logical, length);
6620	else
6621	map_blocks_raid56_read(map, io_geom: &io_geom);
6622	break;
6623	default:
6624	/*
6625	* After this, stripe_nr is the number of stripes on this
6626	* device we have to walk to find the data, and stripe_index is
6627	* the number of our device in the stripe array
6628	*/
6629	map_blocks_single(map, io_geom: &io_geom);
6630	break;
6631	}
6632	if (io_geom.stripe_index >= map->num_stripes) {
6633	btrfs_crit(fs_info,
6634	"stripe index math went horribly wrong, got stripe_index=%u, num_stripes=%u",
6635	io_geom.stripe_index, map->num_stripes);
6636	ret = -EINVAL;
6637	goto out;
6638	}
6639
6640	num_alloc_stripes = io_geom.num_stripes;
6641	if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
6642	op != BTRFS_MAP_READ)
6643	/*
6644	* For replace case, we need to add extra stripes for extra
6645	* duplicated stripes.
6646	*
6647	* For both WRITE and GET_READ_MIRRORS, we may have at most
6648	* 2 more stripes (DUP types, otherwise 1).
6649	*/
6650	num_alloc_stripes += 2;
6651
6652	/*
6653	* If this I/O maps to a single device, try to return the device and
6654	* physical block information on the stack instead of allocating an
6655	* I/O context structure.
6656	*/
6657	if (is_single_device_io(fs_info, smap, map, num_alloc_stripes, io_geom: &io_geom)) {
6658	ret = set_io_stripe(fs_info, logical, length, dst: smap, map, io_geom: &io_geom);
6659	if (mirror_num_ret)
6660	*mirror_num_ret = io_geom.mirror_num;
6661	*bioc_ret = NULL;
6662	goto out;
6663	}
6664
6665	bioc = alloc_btrfs_io_context(fs_info, logical, total_stripes: num_alloc_stripes);
6666	if (!bioc) {
6667	ret = -ENOMEM;
6668	goto out;
6669	}
6670	bioc->map_type = map->type;
6671	bioc->use_rst = io_geom.use_rst;
6672
6673	/*
6674	* For RAID56 full map, we need to make sure the stripes[] follows the
6675	* rule that data stripes are all ordered, then followed with P and Q
6676	* (if we have).
6677	*
6678	* It's still mostly the same as other profiles, just with extra rotation.
6679	*/
6680	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK &&
6681	(op != BTRFS_MAP_READ || io_geom.mirror_num > 1)) {
6682	/*
6683	* For RAID56 @stripe_nr is already the number of full stripes
6684	* before us, which is also the rotation value (needs to modulo
6685	* with num_stripes).
6686	*
6687	* In this case, we just add @stripe_nr with @i, then do the
6688	* modulo, to reduce one modulo call.
6689	*/
6690	bioc->full_stripe_logical = map->start +
6691	btrfs_stripe_nr_to_offset(stripe_nr: io_geom.stripe_nr *
6692	nr_data_stripes(map));
6693	for (int i = 0; i < io_geom.num_stripes; i++) {
6694	struct btrfs_io_stripe *dst = &bioc->stripes[i];
6695	u32 stripe_index;
6696
6697	stripe_index = (i + io_geom.stripe_nr) % io_geom.num_stripes;
6698	dst->dev = map->stripes[stripe_index].dev;
6699	dst->physical =
6700	map->stripes[stripe_index].physical +
6701	io_geom.stripe_offset +
6702	btrfs_stripe_nr_to_offset(stripe_nr: io_geom.stripe_nr);
6703	}
6704	} else {
6705	/*
6706	* For all other non-RAID56 profiles, just copy the target
6707	* stripe into the bioc.
6708	*/
6709	for (int i = 0; i < io_geom.num_stripes; i++) {
6710	ret = set_io_stripe(fs_info, logical, length,
6711	dst: &bioc->stripes[i], map, io_geom: &io_geom);
6712	if (ret < 0)
6713	break;
6714	io_geom.stripe_index++;
6715	}
6716	}
6717
6718	if (ret) {
6719	*bioc_ret = NULL;
6720	btrfs_put_bioc(bioc);
6721	goto out;
6722	}
6723
6724	if (op != BTRFS_MAP_READ)
6725	io_geom.max_errors = btrfs_chunk_max_errors(map);
6726
6727	if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
6728	op != BTRFS_MAP_READ) {
6729	handle_ops_on_dev_replace(bioc, dev_replace, logical, io_geom: &io_geom);
6730	}
6731
6732	*bioc_ret = bioc;
6733	bioc->num_stripes = io_geom.num_stripes;
6734	bioc->max_errors = io_geom.max_errors;
6735	bioc->mirror_num = io_geom.mirror_num;
6736
6737	out:
6738	if (dev_replace_is_ongoing && dev_replace->replace_task != current) {
6739	lockdep_assert_held(&dev_replace->rwsem);
6740	/* Unlock and let waiting writers proceed */
6741	up_read(sem: &dev_replace->rwsem);
6742	}
6743	btrfs_free_chunk_map(map);
6744	return ret;
6745	}
6746
6747	static bool dev_args_match_fs_devices(const struct btrfs_dev_lookup_args *args,
6748	const struct btrfs_fs_devices *fs_devices)
6749	{
6750	if (args->fsid == NULL)
6751	return true;
6752	if (memcmp(p: fs_devices->metadata_uuid, q: args->fsid, BTRFS_FSID_SIZE) == 0)
6753	return true;
6754	return false;
6755	}
6756
6757	static bool dev_args_match_device(const struct btrfs_dev_lookup_args *args,
6758	const struct btrfs_device *device)
6759	{
6760	if (args->devt)
6761	return device->devt == args->devt;
6762	if (args->missing) {
6763	if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state) &&
6764	!device->bdev)
6765	return true;
6766	return false;
6767	}
6768
6769	if (device->devid != args->devid)
6770	return false;
6771	if (args->uuid && memcmp(p: device->uuid, q: args->uuid, BTRFS_UUID_SIZE) != 0)
6772	return false;
6773	return true;
6774	}
6775
6776	/*
6777	* Find a device specified by @devid or @uuid in the list of @fs_devices, or
6778	* return NULL.
6779	*
6780	* If devid and uuid are both specified, the match must be exact, otherwise
6781	* only devid is used.
6782	*/
6783	struct btrfs_device *btrfs_find_device(const struct btrfs_fs_devices *fs_devices,
6784	const struct btrfs_dev_lookup_args *args)
6785	{
6786	struct btrfs_device *device;
6787	struct btrfs_fs_devices *seed_devs;
6788
6789	if (dev_args_match_fs_devices(args, fs_devices)) {
6790	list_for_each_entry(device, &fs_devices->devices, dev_list) {
6791	if (dev_args_match_device(args, device))
6792	return device;
6793	}
6794	}
6795
6796	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
6797	if (!dev_args_match_fs_devices(args, fs_devices: seed_devs))
6798	continue;
6799	list_for_each_entry(device, &seed_devs->devices, dev_list) {
6800	if (dev_args_match_device(args, device))
6801	return device;
6802	}
6803	}
6804
6805	return NULL;
6806	}
6807
6808	static struct btrfs_device *add_missing_dev(struct btrfs_fs_devices *fs_devices,
6809	u64 devid, u8 *dev_uuid)
6810	{
6811	struct btrfs_device *device;
6812	unsigned int nofs_flag;
6813
6814	/*
6815	* We call this under the chunk_mutex, so we want to use NOFS for this
6816	* allocation, however we don't want to change btrfs_alloc_device() to
6817	* always do NOFS because we use it in a lot of other GFP_KERNEL safe
6818	* places.
6819	*/
6820
6821	nofs_flag = memalloc_nofs_save();
6822	device = btrfs_alloc_device(NULL, devid: &devid, uuid: dev_uuid, NULL);
6823	memalloc_nofs_restore(flags: nofs_flag);
6824	if (IS_ERR(ptr: device))
6825	return device;
6826
6827	list_add(new: &device->dev_list, head: &fs_devices->devices);
6828	device->fs_devices = fs_devices;
6829	fs_devices->num_devices++;
6830
6831	set_bit(BTRFS_DEV_STATE_MISSING, addr: &device->dev_state);
6832	fs_devices->missing_devices++;
6833
6834	return device;
6835	}
6836
6837	/*
6838	* Allocate new device struct, set up devid and UUID.
6839	*
6840	* @fs_info: used only for generating a new devid, can be NULL if
6841	* devid is provided (i.e. @devid != NULL).
6842	* @devid: a pointer to devid for this device. If NULL a new devid
6843	* is generated.
6844	* @uuid: a pointer to UUID for this device. If NULL a new UUID
6845	* is generated.
6846	* @path: a pointer to device path if available, NULL otherwise.
6847	*
6848	* Return: a pointer to a new &struct btrfs_device on success; ERR_PTR()
6849	* on error. Returned struct is not linked onto any lists and must be
6850	* destroyed with btrfs_free_device.
6851	*/
6852	struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info,
6853	const u64 *devid, const u8 *uuid,
6854	const char *path)
6855	{
6856	struct btrfs_device *dev;
6857	u64 tmp;
6858
6859	if (WARN_ON(!devid && !fs_info))
6860	return ERR_PTR(error: -EINVAL);
6861
6862	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
6863	if (!dev)
6864	return ERR_PTR(error: -ENOMEM);
6865
6866	INIT_LIST_HEAD(list: &dev->dev_list);
6867	INIT_LIST_HEAD(list: &dev->dev_alloc_list);
6868	INIT_LIST_HEAD(list: &dev->post_commit_list);
6869
6870	atomic_set(v: &dev->dev_stats_ccnt, i: 0);
6871	btrfs_device_data_ordered_init(dev);
6872	btrfs_extent_io_tree_init(fs_info, tree: &dev->alloc_state, owner: IO_TREE_DEVICE_ALLOC_STATE);
6873
6874	if (devid)
6875	tmp = *devid;
6876	else {
6877	int ret;
6878
6879	ret = find_next_devid(fs_info, devid_ret: &tmp);
6880	if (ret) {
6881	btrfs_free_device(device: dev);
6882	return ERR_PTR(error: ret);
6883	}
6884	}
6885	dev->devid = tmp;
6886
6887	if (uuid)
6888	memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE);
6889	else
6890	generate_random_uuid(uuid: dev->uuid);
6891
6892	if (path) {
6893	const char *name;
6894
6895	name = kstrdup(s: path, GFP_KERNEL);
6896	if (!name) {
6897	btrfs_free_device(device: dev);
6898	return ERR_PTR(error: -ENOMEM);
6899	}
6900	rcu_assign_pointer(dev->name, name);
6901	}
6902
6903	return dev;
6904	}
6905
6906	static void btrfs_report_missing_device(struct btrfs_fs_info *fs_info,
6907	u64 devid, u8 *uuid, bool error)
6908	{
6909	if (error)
6910	btrfs_err_rl(fs_info, "devid %llu uuid %pU is missing",
6911	devid, uuid);
6912	else
6913	btrfs_warn_rl(fs_info, "devid %llu uuid %pU is missing",
6914	devid, uuid);
6915	}
6916
6917	u64 btrfs_calc_stripe_length(const struct btrfs_chunk_map *map)
6918	{
6919	const int data_stripes = calc_data_stripes(type: map->type, num_stripes: map->num_stripes);
6920
6921	return div_u64(dividend: map->chunk_len, divisor: data_stripes);
6922	}
6923
6924	#if BITS_PER_LONG == 32
6925	/*
6926	* Due to page cache limit, metadata beyond BTRFS_32BIT_MAX_FILE_SIZE
6927	* can't be accessed on 32bit systems.
6928	*
6929	* This function do mount time check to reject the fs if it already has
6930	* metadata chunk beyond that limit.
6931	*/
6932	static int check_32bit_meta_chunk(struct btrfs_fs_info *fs_info,
6933	u64 logical, u64 length, u64 type)
6934	{
6935	if (!(type & BTRFS_BLOCK_GROUP_METADATA))
6936	return 0;
6937
6938	if (logical + length < MAX_LFS_FILESIZE)
6939	return 0;
6940
6941	btrfs_err_32bit_limit(fs_info);
6942	return -EOVERFLOW;
6943	}
6944
6945	/*
6946	* This is to give early warning for any metadata chunk reaching
6947	* BTRFS_32BIT_EARLY_WARN_THRESHOLD.
6948	* Although we can still access the metadata, it's not going to be possible
6949	* once the limit is reached.
6950	*/
6951	static void warn_32bit_meta_chunk(struct btrfs_fs_info *fs_info,
6952	u64 logical, u64 length, u64 type)
6953	{
6954	if (!(type & BTRFS_BLOCK_GROUP_METADATA))
6955	return;
6956
6957	if (logical + length < BTRFS_32BIT_EARLY_WARN_THRESHOLD)
6958	return;
6959
6960	btrfs_warn_32bit_limit(fs_info);
6961	}
6962	#endif
6963
6964	static struct btrfs_device *handle_missing_device(struct btrfs_fs_info *fs_info,
6965	u64 devid, u8 *uuid)
6966	{
6967	struct btrfs_device *dev;
6968
6969	if (!btrfs_test_opt(fs_info, DEGRADED)) {
6970	btrfs_report_missing_device(fs_info, devid, uuid, error: true);
6971	return ERR_PTR(error: -ENOENT);
6972	}
6973
6974	dev = add_missing_dev(fs_devices: fs_info->fs_devices, devid, dev_uuid: uuid);
6975	if (IS_ERR(ptr: dev)) {
6976	btrfs_err(fs_info, "failed to init missing device %llu: %ld",
6977	devid, PTR_ERR(dev));
6978	return dev;
6979	}
6980	btrfs_report_missing_device(fs_info, devid, uuid, error: false);
6981
6982	return dev;
6983	}
6984
6985	static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf,
6986	struct btrfs_chunk *chunk)
6987	{
6988	BTRFS_DEV_LOOKUP_ARGS(args);
6989	struct btrfs_fs_info *fs_info = leaf->fs_info;
6990	struct btrfs_chunk_map *map;
6991	u64 logical;
6992	u64 length;
6993	u64 devid;
6994	u64 type;
6995	u8 uuid[BTRFS_UUID_SIZE];
6996	int index;
6997	int num_stripes;
6998	int ret;
6999	int i;
7000
7001	logical = key->offset;
7002	length = btrfs_chunk_length(eb: leaf, s: chunk);
7003	type = btrfs_chunk_type(eb: leaf, s: chunk);
7004	index = btrfs_bg_flags_to_raid_index(flags: type);
7005	num_stripes = btrfs_chunk_num_stripes(eb: leaf, s: chunk);
7006
7007	#if BITS_PER_LONG == 32
7008	ret = check_32bit_meta_chunk(fs_info, logical, length, type);
7009	if (ret < 0)
7010	return ret;
7011	warn_32bit_meta_chunk(fs_info, logical, length, type);
7012	#endif
7013
7014	map = btrfs_find_chunk_map(fs_info, logical, length: 1);
7015
7016	/* already mapped? */
7017	if (map && map->start <= logical && map->start + map->chunk_len > logical) {
7018	btrfs_free_chunk_map(map);
7019	return 0;
7020	} else if (map) {
7021	btrfs_free_chunk_map(map);
7022	}
7023
7024	map = btrfs_alloc_chunk_map(num_stripes, GFP_NOFS);
7025	if (!map)
7026	return -ENOMEM;
7027
7028	map->start = logical;
7029	map->chunk_len = length;
7030	map->num_stripes = num_stripes;
7031	map->io_width = btrfs_chunk_io_width(eb: leaf, s: chunk);
7032	map->io_align = btrfs_chunk_io_align(eb: leaf, s: chunk);
7033	map->type = type;
7034	/*
7035	* We can't use the sub_stripes value, as for profiles other than
7036	* RAID10, they may have 0 as sub_stripes for filesystems created by
7037	* older mkfs (<v5.4).
7038	* In that case, it can cause divide-by-zero errors later.
7039	* Since currently sub_stripes is fixed for each profile, let's
7040	* use the trusted value instead.
7041	*/
7042	map->sub_stripes = btrfs_raid_array[index].sub_stripes;
7043	map->verified_stripes = 0;
7044	map->stripe_size = btrfs_calc_stripe_length(map);
7045	for (i = 0; i < num_stripes; i++) {
7046	map->stripes[i].physical =
7047	btrfs_stripe_offset_nr(eb: leaf, c: chunk, nr: i);
7048	devid = btrfs_stripe_devid_nr(eb: leaf, c: chunk, nr: i);
7049	args.devid = devid;
7050	read_extent_buffer(eb: leaf, dst: uuid, start: (unsigned long)
7051	btrfs_stripe_dev_uuid_nr(c: chunk, nr: i),
7052	BTRFS_UUID_SIZE);
7053	args.uuid = uuid;
7054	map->stripes[i].dev = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
7055	if (!map->stripes[i].dev) {
7056	map->stripes[i].dev = handle_missing_device(fs_info,
7057	devid, uuid);
7058	if (IS_ERR(ptr: map->stripes[i].dev)) {
7059	ret = PTR_ERR(ptr: map->stripes[i].dev);
7060	btrfs_free_chunk_map(map);
7061	return ret;
7062	}
7063	}
7064
7065	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
7066	addr: &(map->stripes[i].dev->dev_state));
7067	}
7068
7069	ret = btrfs_add_chunk_map(fs_info, map);
7070	if (ret < 0) {
7071	btrfs_err(fs_info,
7072	"failed to add chunk map, start=%llu len=%llu: %d",
7073	map->start, map->chunk_len, ret);
7074	btrfs_free_chunk_map(map);
7075	}
7076
7077	return ret;
7078	}
7079
7080	static void fill_device_from_item(struct extent_buffer *leaf,
7081	struct btrfs_dev_item *dev_item,
7082	struct btrfs_device *device)
7083	{
7084	unsigned long ptr;
7085
7086	device->devid = btrfs_device_id(eb: leaf, s: dev_item);
7087	device->disk_total_bytes = btrfs_device_total_bytes(eb: leaf, s: dev_item);
7088	device->total_bytes = device->disk_total_bytes;
7089	device->commit_total_bytes = device->disk_total_bytes;
7090	device->bytes_used = btrfs_device_bytes_used(eb: leaf, s: dev_item);
7091	device->commit_bytes_used = device->bytes_used;
7092	device->type = btrfs_device_type(eb: leaf, s: dev_item);
7093	device->io_align = btrfs_device_io_align(eb: leaf, s: dev_item);
7094	device->io_width = btrfs_device_io_width(eb: leaf, s: dev_item);
7095	device->sector_size = btrfs_device_sector_size(eb: leaf, s: dev_item);
7096	WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID);
7097	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, addr: &device->dev_state);
7098
7099	ptr = btrfs_device_uuid(d: dev_item);
7100	read_extent_buffer(eb: leaf, dst: device->uuid, start: ptr, BTRFS_UUID_SIZE);
7101	}
7102
7103	static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info,
7104	u8 *fsid)
7105	{
7106	struct btrfs_fs_devices *fs_devices;
7107	int ret;
7108
7109	lockdep_assert_held(&uuid_mutex);
7110	ASSERT(fsid);
7111
7112	/* This will match only for multi-device seed fs */
7113	list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list)
7114	if (!memcmp(p: fs_devices->fsid, q: fsid, BTRFS_FSID_SIZE))
7115	return fs_devices;
7116
7117
7118	fs_devices = find_fsid(fsid, NULL);
7119	if (!fs_devices) {
7120	if (!btrfs_test_opt(fs_info, DEGRADED)) {
7121	btrfs_err(fs_info,
7122	"failed to find fsid %pU when attempting to open seed devices",
7123	fsid);
7124	return ERR_PTR(error: -ENOENT);
7125	}
7126
7127	fs_devices = alloc_fs_devices(fsid);
7128	if (IS_ERR(ptr: fs_devices))
7129	return fs_devices;
7130
7131	fs_devices->seeding = true;
7132	fs_devices->opened = 1;
7133	list_add(new: &fs_devices->seed_list, head: &fs_info->fs_devices->seed_list);
7134	return fs_devices;
7135	}
7136
7137	/*
7138	* Upon first call for a seed fs fsid, just create a private copy of the
7139	* respective fs_devices and anchor it at fs_info->fs_devices->seed_list
7140	*/
7141	fs_devices = clone_fs_devices(orig: fs_devices);
7142	if (IS_ERR(ptr: fs_devices))
7143	return fs_devices;
7144
7145	ret = open_fs_devices(fs_devices, BLK_OPEN_READ, holder: fs_info->sb);
7146	if (ret) {
7147	free_fs_devices(fs_devices);
7148	return ERR_PTR(error: ret);
7149	}
7150
7151	if (!fs_devices->seeding) {
7152	close_fs_devices(fs_devices);
7153	free_fs_devices(fs_devices);
7154	return ERR_PTR(error: -EINVAL);
7155	}
7156
7157	list_add(new: &fs_devices->seed_list, head: &fs_info->fs_devices->seed_list);
7158
7159	return fs_devices;
7160	}
7161
7162	static int read_one_dev(struct extent_buffer *leaf,
7163	struct btrfs_dev_item *dev_item)
7164	{
7165	BTRFS_DEV_LOOKUP_ARGS(args);
7166	struct btrfs_fs_info *fs_info = leaf->fs_info;
7167	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7168	struct btrfs_device *device;
7169	u64 devid;
7170	int ret;
7171	u8 fs_uuid[BTRFS_FSID_SIZE];
7172	u8 dev_uuid[BTRFS_UUID_SIZE];
7173
7174	devid = btrfs_device_id(eb: leaf, s: dev_item);
7175	args.devid = devid;
7176	read_extent_buffer(eb: leaf, dst: dev_uuid, start: btrfs_device_uuid(d: dev_item),
7177	BTRFS_UUID_SIZE);
7178	read_extent_buffer(eb: leaf, dst: fs_uuid, start: btrfs_device_fsid(d: dev_item),
7179	BTRFS_FSID_SIZE);
7180	args.uuid = dev_uuid;
7181	args.fsid = fs_uuid;
7182
7183	if (memcmp(p: fs_uuid, q: fs_devices->metadata_uuid, BTRFS_FSID_SIZE)) {
7184	fs_devices = open_seed_devices(fs_info, fsid: fs_uuid);
7185	if (IS_ERR(ptr: fs_devices))
7186	return PTR_ERR(ptr: fs_devices);
7187	}
7188
7189	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
7190	if (!device) {
7191	if (!btrfs_test_opt(fs_info, DEGRADED)) {
7192	btrfs_report_missing_device(fs_info, devid,
7193	uuid: dev_uuid, error: true);
7194	return -ENOENT;
7195	}
7196
7197	device = add_missing_dev(fs_devices, devid, dev_uuid);
7198	if (IS_ERR(ptr: device)) {
7199	btrfs_err(fs_info,
7200	"failed to add missing dev %llu: %ld",
7201	devid, PTR_ERR(device));
7202	return PTR_ERR(ptr: device);
7203	}
7204	btrfs_report_missing_device(fs_info, devid, uuid: dev_uuid, error: false);
7205	} else {
7206	if (!device->bdev) {
7207	if (!btrfs_test_opt(fs_info, DEGRADED)) {
7208	btrfs_report_missing_device(fs_info,
7209	devid, uuid: dev_uuid, error: true);
7210	return -ENOENT;
7211	}
7212	btrfs_report_missing_device(fs_info, devid,
7213	uuid: dev_uuid, error: false);
7214	}
7215
7216	if (!device->bdev &&
7217	!test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
7218	/*
7219	* this happens when a device that was properly setup
7220	* in the device info lists suddenly goes bad.
7221	* device->bdev is NULL, and so we have to set
7222	* device->missing to one here
7223	*/
7224	device->fs_devices->missing_devices++;
7225	set_bit(BTRFS_DEV_STATE_MISSING, addr: &device->dev_state);
7226	}
7227
7228	/* Move the device to its own fs_devices */
7229	if (device->fs_devices != fs_devices) {
7230	ASSERT(test_bit(BTRFS_DEV_STATE_MISSING,
7231	&device->dev_state));
7232
7233	list_move(list: &device->dev_list, head: &fs_devices->devices);
7234	device->fs_devices->num_devices--;
7235	fs_devices->num_devices++;
7236
7237	device->fs_devices->missing_devices--;
7238	fs_devices->missing_devices++;
7239
7240	device->fs_devices = fs_devices;
7241	}
7242	}
7243
7244	if (device->fs_devices != fs_info->fs_devices) {
7245	BUG_ON(test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state));
7246	if (device->generation !=
7247	btrfs_device_generation(eb: leaf, s: dev_item))
7248	return -EINVAL;
7249	}
7250
7251	fill_device_from_item(leaf, dev_item, device);
7252	if (device->bdev) {
7253	u64 max_total_bytes = bdev_nr_bytes(bdev: device->bdev);
7254
7255	if (device->total_bytes > max_total_bytes) {
7256	btrfs_err(fs_info,
7257	"device total_bytes should be at most %llu but found %llu",
7258	max_total_bytes, device->total_bytes);
7259	return -EINVAL;
7260	}
7261	}
7262	set_bit(BTRFS_DEV_STATE_ITEM_FOUND, addr: &device->dev_state);
7263	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, addr: &device->dev_state);
7264	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
7265	!test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
7266	device->fs_devices->total_rw_bytes += device->total_bytes;
7267	atomic64_add(i: device->total_bytes - device->bytes_used,
7268	v: &fs_info->free_chunk_space);
7269	}
7270	ret = 0;
7271	return ret;
7272	}
7273
7274	int btrfs_read_sys_array(struct btrfs_fs_info *fs_info)
7275	{
7276	struct btrfs_super_block *super_copy = fs_info->super_copy;
7277	struct extent_buffer *sb;
7278	u8 *array_ptr;
7279	unsigned long sb_array_offset;
7280	int ret = 0;
7281	u32 array_size;
7282	u32 cur_offset;
7283	struct btrfs_key key;
7284
7285	ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize);
7286
7287	/*
7288	* We allocated a dummy extent, just to use extent buffer accessors.
7289	* There will be unused space after BTRFS_SUPER_INFO_SIZE, but
7290	* that's fine, we will not go beyond system chunk array anyway.
7291	*/
7292	sb = alloc_dummy_extent_buffer(fs_info, BTRFS_SUPER_INFO_OFFSET);
7293	if (!sb)
7294	return -ENOMEM;
7295	set_extent_buffer_uptodate(sb);
7296
7297	write_extent_buffer(eb: sb, src: super_copy, start: 0, BTRFS_SUPER_INFO_SIZE);
7298	array_size = btrfs_super_sys_array_size(s: super_copy);
7299
7300	array_ptr = super_copy->sys_chunk_array;
7301	sb_array_offset = offsetof(struct btrfs_super_block, sys_chunk_array);
7302	cur_offset = 0;
7303
7304	while (cur_offset < array_size) {
7305	struct btrfs_chunk *chunk;
7306	struct btrfs_disk_key *disk_key = (struct btrfs_disk_key *)array_ptr;
7307	u32 len = sizeof(*disk_key);
7308
7309	/*
7310	* The sys_chunk_array has been already verified at super block
7311	* read time. Only do ASSERT()s for basic checks.
7312	*/
7313	ASSERT(cur_offset + len <= array_size);
7314
7315	btrfs_disk_key_to_cpu(cpu_key: &key, disk_key);
7316
7317	array_ptr += len;
7318	sb_array_offset += len;
7319	cur_offset += len;
7320
7321	ASSERT(key.type == BTRFS_CHUNK_ITEM_KEY);
7322
7323	chunk = (struct btrfs_chunk *)sb_array_offset;
7324	ASSERT(btrfs_chunk_type(sb, chunk) & BTRFS_BLOCK_GROUP_SYSTEM);
7325
7326	len = btrfs_chunk_item_size(num_stripes: btrfs_chunk_num_stripes(eb: sb, s: chunk));
7327
7328	ASSERT(cur_offset + len <= array_size);
7329
7330	ret = read_one_chunk(key: &key, leaf: sb, chunk);
7331	if (ret)
7332	break;
7333
7334	array_ptr += len;
7335	sb_array_offset += len;
7336	cur_offset += len;
7337	}
7338	clear_extent_buffer_uptodate(eb: sb);
7339	free_extent_buffer_stale(eb: sb);
7340	return ret;
7341	}
7342
7343	/*
7344	* Check if all chunks in the fs are OK for read-write degraded mount
7345	*
7346	* If the @failing_dev is specified, it's accounted as missing.
7347	*
7348	* Return true if all chunks meet the minimal RW mount requirements.
7349	* Return false if any chunk doesn't meet the minimal RW mount requirements.
7350	*/
7351	bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info,
7352	struct btrfs_device *failing_dev)
7353	{
7354	struct btrfs_chunk_map *map;
7355	u64 next_start;
7356	bool ret = true;
7357
7358	map = btrfs_find_chunk_map(fs_info, logical: 0, U64_MAX);
7359	/* No chunk at all? Return false anyway */
7360	if (!map) {
7361	ret = false;
7362	goto out;
7363	}
7364	while (map) {
7365	int missing = 0;
7366	int max_tolerated;
7367	int i;
7368
7369	max_tolerated =
7370	btrfs_get_num_tolerated_disk_barrier_failures(
7371	flags: map->type);
7372	for (i = 0; i < map->num_stripes; i++) {
7373	struct btrfs_device *dev = map->stripes[i].dev;
7374
7375	if (!dev || !dev->bdev ||
7376	test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) ||
7377	dev->last_flush_error)
7378	missing++;
7379	else if (failing_dev && failing_dev == dev)
7380	missing++;
7381	}
7382	if (missing > max_tolerated) {
7383	if (!failing_dev)
7384	btrfs_warn(fs_info,
7385	"chunk %llu missing %d devices, max tolerance is %d for writable mount",
7386	map->start, missing, max_tolerated);
7387	btrfs_free_chunk_map(map);
7388	ret = false;
7389	goto out;
7390	}
7391	next_start = map->start + map->chunk_len;
7392	btrfs_free_chunk_map(map);
7393
7394	map = btrfs_find_chunk_map(fs_info, logical: next_start, U64_MAX - next_start);
7395	}
7396	out:
7397	return ret;
7398	}
7399
7400	static void readahead_tree_node_children(struct extent_buffer *node)
7401	{
7402	int i;
7403	const int nr_items = btrfs_header_nritems(eb: node);
7404
7405	for (i = 0; i < nr_items; i++)
7406	btrfs_readahead_node_child(node, slot: i);
7407	}
7408
7409	int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
7410	{
7411	struct btrfs_root *root = fs_info->chunk_root;
7412	BTRFS_PATH_AUTO_FREE(path);
7413	struct extent_buffer *leaf;
7414	struct btrfs_key key;
7415	struct btrfs_key found_key;
7416	int ret;
7417	int slot;
7418	int iter_ret = 0;
7419	u64 total_dev = 0;
7420	u64 last_ra_node = 0;
7421
7422	path = btrfs_alloc_path();
7423	if (!path)
7424	return -ENOMEM;
7425
7426	/*
7427	* uuid_mutex is needed only if we are mounting a sprout FS
7428	* otherwise we don't need it.
7429	*/
7430	mutex_lock(&uuid_mutex);
7431
7432	/*
7433	* It is possible for mount and umount to race in such a way that
7434	* we execute this code path, but open_fs_devices failed to clear
7435	* total_rw_bytes. We certainly want it cleared before reading the
7436	* device items, so clear it here.
7437	*/
7438	fs_info->fs_devices->total_rw_bytes = 0;
7439
7440	/*
7441	* Lockdep complains about possible circular locking dependency between
7442	* a disk's open_mutex (struct gendisk.open_mutex), the rw semaphores
7443	* used for freeze protection of a fs (struct super_block.s_writers),
7444	* which we take when starting a transaction, and extent buffers of the
7445	* chunk tree if we call read_one_dev() while holding a lock on an
7446	* extent buffer of the chunk tree. Since we are mounting the filesystem
7447	* and at this point there can't be any concurrent task modifying the
7448	* chunk tree, to keep it simple, just skip locking on the chunk tree.
7449	*/
7450	ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags));
7451	path->skip_locking = true;
7452
7453	/*
7454	* Read all device items, and then all the chunk items. All
7455	* device items are found before any chunk item (their object id
7456	* is smaller than the lowest possible object id for a chunk
7457	* item - BTRFS_FIRST_CHUNK_TREE_OBJECTID).
7458	*/
7459	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
7460	key.type = 0;
7461	key.offset = 0;
7462	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
7463	struct extent_buffer *node = path->nodes[1];
7464
7465	leaf = path->nodes[0];
7466	slot = path->slots[0];
7467
7468	if (node) {
7469	if (last_ra_node != node->start) {
7470	readahead_tree_node_children(node);
7471	last_ra_node = node->start;
7472	}
7473	}
7474	if (found_key.type == BTRFS_DEV_ITEM_KEY) {
7475	struct btrfs_dev_item *dev_item;
7476	dev_item = btrfs_item_ptr(leaf, slot,
7477	struct btrfs_dev_item);
7478	ret = read_one_dev(leaf, dev_item);
7479	if (ret)
7480	goto error;
7481	total_dev++;
7482	} else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) {
7483	struct btrfs_chunk *chunk;
7484
7485	/*
7486	* We are only called at mount time, so no need to take
7487	* fs_info->chunk_mutex. Plus, to avoid lockdep warnings,
7488	* we always lock first fs_info->chunk_mutex before
7489	* acquiring any locks on the chunk tree. This is a
7490	* requirement for chunk allocation, see the comment on
7491	* top of btrfs_chunk_alloc() for details.
7492	*/
7493	chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
7494	ret = read_one_chunk(key: &found_key, leaf, chunk);
7495	if (ret)
7496	goto error;
7497	}
7498	}
7499	/* Catch error found during iteration */
7500	if (iter_ret < 0) {
7501	ret = iter_ret;
7502	goto error;
7503	}
7504
7505	/*
7506	* After loading chunk tree, we've got all device information,
7507	* do another round of validation checks.
7508	*/
7509	if (total_dev != fs_info->fs_devices->total_devices) {
7510	btrfs_warn(fs_info,
7511	"super block num_devices %llu mismatch with DEV_ITEM count %llu, will be repaired on next transaction commit",
7512	btrfs_super_num_devices(fs_info->super_copy),
7513	total_dev);
7514	fs_info->fs_devices->total_devices = total_dev;
7515	btrfs_set_super_num_devices(s: fs_info->super_copy, val: total_dev);
7516	}
7517	if (btrfs_super_total_bytes(s: fs_info->super_copy) <
7518	fs_info->fs_devices->total_rw_bytes) {
7519	btrfs_err(fs_info,
7520	"super_total_bytes %llu mismatch with fs_devices total_rw_bytes %llu",
7521	btrfs_super_total_bytes(fs_info->super_copy),
7522	fs_info->fs_devices->total_rw_bytes);
7523	ret = -EINVAL;
7524	goto error;
7525	}
7526	ret = 0;
7527	error:
7528	mutex_unlock(lock: &uuid_mutex);
7529	return ret;
7530	}
7531
7532	int btrfs_init_devices_late(struct btrfs_fs_info *fs_info)
7533	{
7534	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7535	struct btrfs_device *device;
7536	int ret = 0;
7537
7538	mutex_lock(&fs_devices->device_list_mutex);
7539	list_for_each_entry(device, &fs_devices->devices, dev_list)
7540	device->fs_info = fs_info;
7541
7542	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7543	list_for_each_entry(device, &seed_devs->devices, dev_list) {
7544	device->fs_info = fs_info;
7545	ret = btrfs_get_dev_zone_info(device, populate_cache: false);
7546	if (ret)
7547	break;
7548	}
7549
7550	seed_devs->fs_info = fs_info;
7551	}
7552	mutex_unlock(lock: &fs_devices->device_list_mutex);
7553
7554	return ret;
7555	}
7556
7557	static u64 btrfs_dev_stats_value(const struct extent_buffer *eb,
7558	const struct btrfs_dev_stats_item *ptr,
7559	int index)
7560	{
7561	u64 val;
7562
7563	read_extent_buffer(eb, dst: &val,
7564	offsetof(struct btrfs_dev_stats_item, values) +
7565	((unsigned long)ptr) + (index * sizeof(u64)),
7566	len: sizeof(val));
7567	return val;
7568	}
7569
7570	static void btrfs_set_dev_stats_value(struct extent_buffer *eb,
7571	struct btrfs_dev_stats_item *ptr,
7572	int index, u64 val)
7573	{
7574	write_extent_buffer(eb, src: &val,
7575	offsetof(struct btrfs_dev_stats_item, values) +
7576	((unsigned long)ptr) + (index * sizeof(u64)),
7577	len: sizeof(val));
7578	}
7579
7580	static int btrfs_device_init_dev_stats(struct btrfs_device *device,
7581	struct btrfs_path *path)
7582	{
7583	struct btrfs_dev_stats_item *ptr;
7584	struct extent_buffer *eb;
7585	struct btrfs_key key;
7586	int item_size;
7587	int i, ret, slot;
7588
7589	if (!device->fs_info->dev_root)
7590	return 0;
7591
7592	key.objectid = BTRFS_DEV_STATS_OBJECTID;
7593	key.type = BTRFS_PERSISTENT_ITEM_KEY;
7594	key.offset = device->devid;
7595	ret = btrfs_search_slot(NULL, root: device->fs_info->dev_root, key: &key, p: path, ins_len: 0, cow: 0);
7596	if (ret) {
7597	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7598	btrfs_dev_stat_set(dev: device, index: i, val: 0);
7599	device->dev_stats_valid = 1;
7600	btrfs_release_path(p: path);
7601	return ret < 0 ? ret : 0;
7602	}
7603	slot = path->slots[0];
7604	eb = path->nodes[0];
7605	item_size = btrfs_item_size(eb, slot);
7606
7607	ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_stats_item);
7608
7609	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7610	if (item_size >= (1 + i) * sizeof(__le64))
7611	btrfs_dev_stat_set(dev: device, index: i,
7612	val: btrfs_dev_stats_value(eb, ptr, index: i));
7613	else
7614	btrfs_dev_stat_set(dev: device, index: i, val: 0);
7615	}
7616
7617	device->dev_stats_valid = 1;
7618	btrfs_dev_stat_print_on_load(device);
7619	btrfs_release_path(p: path);
7620
7621	return 0;
7622	}
7623
7624	int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
7625	{
7626	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7627	struct btrfs_device *device;
7628	BTRFS_PATH_AUTO_FREE(path);
7629	int ret = 0;
7630
7631	path = btrfs_alloc_path();
7632	if (!path)
7633	return -ENOMEM;
7634
7635	mutex_lock(&fs_devices->device_list_mutex);
7636	list_for_each_entry(device, &fs_devices->devices, dev_list) {
7637	ret = btrfs_device_init_dev_stats(device, path);
7638	if (ret)
7639	goto out;
7640	}
7641	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7642	list_for_each_entry(device, &seed_devs->devices, dev_list) {
7643	ret = btrfs_device_init_dev_stats(device, path);
7644	if (ret)
7645	goto out;
7646	}
7647	}
7648	out:
7649	mutex_unlock(lock: &fs_devices->device_list_mutex);
7650	return ret;
7651	}
7652
7653	static int update_dev_stat_item(struct btrfs_trans_handle *trans,
7654	struct btrfs_device *device)
7655	{
7656	struct btrfs_fs_info *fs_info = trans->fs_info;
7657	struct btrfs_root *dev_root = fs_info->dev_root;
7658	BTRFS_PATH_AUTO_FREE(path);
7659	struct btrfs_key key;
7660	struct extent_buffer *eb;
7661	struct btrfs_dev_stats_item *ptr;
7662	int ret;
7663	int i;
7664
7665	key.objectid = BTRFS_DEV_STATS_OBJECTID;
7666	key.type = BTRFS_PERSISTENT_ITEM_KEY;
7667	key.offset = device->devid;
7668
7669	path = btrfs_alloc_path();
7670	if (!path)
7671	return -ENOMEM;
7672	ret = btrfs_search_slot(trans, root: dev_root, key: &key, p: path, ins_len: -1, cow: 1);
7673	if (ret < 0) {
7674	btrfs_warn(fs_info,
7675	"error %d while searching for dev_stats item for device %s",
7676	ret, btrfs_dev_name(device));
7677	return ret;
7678	}
7679
7680	if (ret == 0 &&
7681	btrfs_item_size(eb: path->nodes[0], slot: path->slots[0]) < sizeof(*ptr)) {
7682	/* need to delete old one and insert a new one */
7683	ret = btrfs_del_item(trans, root: dev_root, path);
7684	if (ret != 0) {
7685	btrfs_warn(fs_info,
7686	"delete too small dev_stats item for device %s failed %d",
7687	btrfs_dev_name(device), ret);
7688	return ret;
7689	}
7690	ret = 1;
7691	}
7692
7693	if (ret == 1) {
7694	/* need to insert a new item */
7695	btrfs_release_path(p: path);
7696	ret = btrfs_insert_empty_item(trans, root: dev_root, path,
7697	key: &key, data_size: sizeof(*ptr));
7698	if (ret < 0) {
7699	btrfs_warn(fs_info,
7700	"insert dev_stats item for device %s failed %d",
7701	btrfs_dev_name(device), ret);
7702	return ret;
7703	}
7704	}
7705
7706	eb = path->nodes[0];
7707	ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item);
7708	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7709	btrfs_set_dev_stats_value(eb, ptr, index: i,
7710	val: btrfs_dev_stat_read(dev: device, index: i));
7711	return ret;
7712	}
7713
7714	/*
7715	* called from commit_transaction. Writes all changed device stats to disk.
7716	*/
7717	int btrfs_run_dev_stats(struct btrfs_trans_handle *trans)
7718	{
7719	struct btrfs_fs_info *fs_info = trans->fs_info;
7720	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7721	struct btrfs_device *device;
7722	int stats_cnt;
7723	int ret = 0;
7724
7725	mutex_lock(&fs_devices->device_list_mutex);
7726	list_for_each_entry(device, &fs_devices->devices, dev_list) {
7727	stats_cnt = atomic_read(v: &device->dev_stats_ccnt);
7728	if (!device->dev_stats_valid || stats_cnt == 0)
7729	continue;
7730
7731
7732	/*
7733	* There is a LOAD-LOAD control dependency between the value of
7734	* dev_stats_ccnt and updating the on-disk values which requires
7735	* reading the in-memory counters. Such control dependencies
7736	* require explicit read memory barriers.
7737	*
7738	* This memory barriers pairs with smp_mb__before_atomic in
7739	* btrfs_dev_stat_inc/btrfs_dev_stat_set and with the full
7740	* barrier implied by atomic_xchg in
7741	* btrfs_dev_stats_read_and_reset
7742	*/
7743	smp_rmb();
7744
7745	ret = update_dev_stat_item(trans, device);
7746	if (!ret)
7747	atomic_sub(i: stats_cnt, v: &device->dev_stats_ccnt);
7748	}
7749	mutex_unlock(lock: &fs_devices->device_list_mutex);
7750
7751	return ret;
7752	}
7753
7754	void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)
7755	{
7756	btrfs_dev_stat_inc(dev, index);
7757
7758	if (!dev->dev_stats_valid)
7759	return;
7760	btrfs_err_rl(dev->fs_info,
7761	"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7762	btrfs_dev_name(dev),
7763	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7764	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7765	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7766	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7767	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7768	}
7769
7770	static void btrfs_dev_stat_print_on_load(struct btrfs_device *dev)
7771	{
7772	int i;
7773
7774	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7775	if (btrfs_dev_stat_read(dev, index: i) != 0)
7776	break;
7777	if (i == BTRFS_DEV_STAT_VALUES_MAX)
7778	return; /* all values == 0, suppress message */
7779
7780	btrfs_info(dev->fs_info,
7781	"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7782	btrfs_dev_name(dev),
7783	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7784	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7785	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7786	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7787	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7788	}
7789
7790	int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info,
7791	struct btrfs_ioctl_get_dev_stats *stats)
7792	{
7793	BTRFS_DEV_LOOKUP_ARGS(args);
7794	struct btrfs_device *dev;
7795	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7796	int i;
7797
7798	mutex_lock(&fs_devices->device_list_mutex);
7799	args.devid = stats->devid;
7800	dev = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
7801	mutex_unlock(lock: &fs_devices->device_list_mutex);
7802
7803	if (!dev) {
7804	btrfs_warn(fs_info, "get dev_stats failed, device not found");
7805	return -ENODEV;
7806	} else if (!dev->dev_stats_valid) {
7807	btrfs_warn(fs_info, "get dev_stats failed, not yet valid");
7808	return -ENODEV;
7809	} else if (stats->flags & BTRFS_DEV_STATS_RESET) {
7810	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7811	if (stats->nr_items > i)
7812	stats->values[i] =
7813	btrfs_dev_stat_read_and_reset(dev, index: i);
7814	else
7815	btrfs_dev_stat_set(dev, index: i, val: 0);
7816	}
7817	btrfs_info(fs_info, "device stats zeroed by %s (%d)",
7818	current->comm, task_pid_nr(current));
7819	} else {
7820	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7821	if (stats->nr_items > i)
7822	stats->values[i] = btrfs_dev_stat_read(dev, index: i);
7823	}
7824	if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX)
7825	stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX;
7826	return 0;
7827	}
7828
7829	/*
7830	* Update the size and bytes used for each device where it changed. This is
7831	* delayed since we would otherwise get errors while writing out the
7832	* superblocks.
7833	*
7834	* Must be invoked during transaction commit.
7835	*/
7836	void btrfs_commit_device_sizes(struct btrfs_transaction *trans)
7837	{
7838	struct btrfs_device *curr, *next;
7839
7840	ASSERT(trans->state == TRANS_STATE_COMMIT_DOING, "state=%d" , trans->state);
7841
7842	if (list_empty(head: &trans->dev_update_list))
7843	return;
7844
7845	/*
7846	* We don't need the device_list_mutex here. This list is owned by the
7847	* transaction and the transaction must complete before the device is
7848	* released.
7849	*/
7850	mutex_lock(&trans->fs_info->chunk_mutex);
7851	list_for_each_entry_safe(curr, next, &trans->dev_update_list,
7852	post_commit_list) {
7853	list_del_init(entry: &curr->post_commit_list);
7854	curr->commit_total_bytes = curr->disk_total_bytes;
7855	curr->commit_bytes_used = curr->bytes_used;
7856	}
7857	mutex_unlock(lock: &trans->fs_info->chunk_mutex);
7858	}
7859
7860	/*
7861	* Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10.
7862	*/
7863	int btrfs_bg_type_to_factor(u64 flags)
7864	{
7865	const int index = btrfs_bg_flags_to_raid_index(flags);
7866
7867	return btrfs_raid_array[index].ncopies;
7868	}
7869
7870	static int verify_one_dev_extent(struct btrfs_fs_info *fs_info,
7871	u64 chunk_offset, u64 devid,
7872	u64 physical_offset, u64 physical_len)
7873	{
7874	struct btrfs_dev_lookup_args args = { .devid = devid };
7875	struct btrfs_chunk_map *map;
7876	struct btrfs_device *dev;
7877	u64 stripe_len;
7878	bool found = false;
7879	int ret = 0;
7880	int i;
7881
7882	map = btrfs_find_chunk_map(fs_info, logical: chunk_offset, length: 1);
7883	if (unlikely(!map)) {
7884	btrfs_err(fs_info,
7885	"dev extent physical offset %llu on devid %llu doesn't have corresponding chunk",
7886	physical_offset, devid);
7887	ret = -EUCLEAN;
7888	goto out;
7889	}
7890
7891	stripe_len = btrfs_calc_stripe_length(map);
7892	if (unlikely(physical_len != stripe_len)) {
7893	btrfs_err(fs_info,
7894	"dev extent physical offset %llu on devid %llu length doesn't match chunk %llu, have %llu expect %llu",
7895	physical_offset, devid, map->start, physical_len,
7896	stripe_len);
7897	ret = -EUCLEAN;
7898	goto out;
7899	}
7900
7901	/*
7902	* Very old mkfs.btrfs (before v4.15) will not respect the reserved
7903	* space. Although kernel can handle it without problem, better to warn
7904	* the users.
7905	*/
7906	if (physical_offset < BTRFS_DEVICE_RANGE_RESERVED)
7907	btrfs_warn(fs_info,
7908	"devid %llu physical %llu len %llu inside the reserved space",
7909	devid, physical_offset, physical_len);
7910
7911	for (i = 0; i < map->num_stripes; i++) {
7912	if (unlikely(map->stripes[i].dev->devid == devid &&
7913	map->stripes[i].physical == physical_offset)) {
7914	found = true;
7915	if (map->verified_stripes >= map->num_stripes) {
7916	btrfs_err(fs_info,
7917	"too many dev extents for chunk %llu found",
7918	map->start);
7919	ret = -EUCLEAN;
7920	goto out;
7921	}
7922	map->verified_stripes++;
7923	break;
7924	}
7925	}
7926	if (unlikely(!found)) {
7927	btrfs_err(fs_info,
7928	"dev extent physical offset %llu devid %llu has no corresponding chunk",
7929	physical_offset, devid);
7930	ret = -EUCLEAN;
7931	}
7932
7933	/* Make sure no dev extent is beyond device boundary */
7934	dev = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
7935	if (unlikely(!dev)) {
7936	btrfs_err(fs_info, "failed to find devid %llu", devid);
7937	ret = -EUCLEAN;
7938	goto out;
7939	}
7940
7941	if (unlikely(physical_offset + physical_len > dev->disk_total_bytes)) {
7942	btrfs_err(fs_info,
7943	"dev extent devid %llu physical offset %llu len %llu is beyond device boundary %llu",
7944	devid, physical_offset, physical_len,
7945	dev->disk_total_bytes);
7946	ret = -EUCLEAN;
7947	goto out;
7948	}
7949
7950	if (dev->zone_info) {
7951	u64 zone_size = dev->zone_info->zone_size;
7952
7953	if (unlikely(!IS_ALIGNED(physical_offset, zone_size) ||
7954	!IS_ALIGNED(physical_len, zone_size))) {
7955	btrfs_err(fs_info,
7956	"zoned: dev extent devid %llu physical offset %llu len %llu is not aligned to device zone",
7957	devid, physical_offset, physical_len);
7958	ret = -EUCLEAN;
7959	goto out;
7960	}
7961	}
7962
7963	out:
7964	btrfs_free_chunk_map(map);
7965	return ret;
7966	}
7967
7968	static int verify_chunk_dev_extent_mapping(struct btrfs_fs_info *fs_info)
7969	{
7970	struct rb_node *node;
7971	int ret = 0;
7972
7973	read_lock(&fs_info->mapping_tree_lock);
7974	for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) {
7975	struct btrfs_chunk_map *map;
7976
7977	map = rb_entry(node, struct btrfs_chunk_map, rb_node);
7978	if (unlikely(map->num_stripes != map->verified_stripes)) {
7979	btrfs_err(fs_info,
7980	"chunk %llu has missing dev extent, have %d expect %d",
7981	map->start, map->verified_stripes, map->num_stripes);
7982	ret = -EUCLEAN;
7983	goto out;
7984	}
7985	}
7986	out:
7987	read_unlock(&fs_info->mapping_tree_lock);
7988	return ret;
7989	}
7990
7991	/*
7992	* Ensure that all dev extents are mapped to correct chunk, otherwise
7993	* later chunk allocation/free would cause unexpected behavior.
7994	*
7995	* NOTE: This will iterate through the whole device tree, which should be of
7996	* the same size level as the chunk tree. This slightly increases mount time.
7997	*/
7998	int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info)
7999	{
8000	BTRFS_PATH_AUTO_FREE(path);
8001	struct btrfs_root *root = fs_info->dev_root;
8002	struct btrfs_key key;
8003	u64 prev_devid = 0;
8004	u64 prev_dev_ext_end = 0;
8005	int ret = 0;
8006
8007	/*
8008	* We don't have a dev_root because we mounted with ignorebadroots and
8009	* failed to load the root, so we want to skip the verification in this
8010	* case for sure.
8011	*
8012	* However if the dev root is fine, but the tree itself is corrupted
8013	* we'd still fail to mount. This verification is only to make sure
8014	* writes can happen safely, so instead just bypass this check
8015	* completely in the case of IGNOREBADROOTS.
8016	*/
8017	if (btrfs_test_opt(fs_info, IGNOREBADROOTS))
8018	return 0;
8019
8020	key.objectid = 1;
8021	key.type = BTRFS_DEV_EXTENT_KEY;
8022	key.offset = 0;
8023
8024	path = btrfs_alloc_path();
8025	if (!path)
8026	return -ENOMEM;
8027
8028	path->reada = READA_FORWARD;
8029	ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
8030	if (ret < 0)
8031	return ret;
8032
8033	if (path->slots[0] >= btrfs_header_nritems(eb: path->nodes[0])) {
8034	ret = btrfs_next_leaf(root, path);
8035	if (ret < 0)
8036	return ret;
8037	/* No dev extents at all? Not good */
8038	if (unlikely(ret > 0))
8039	return -EUCLEAN;
8040	}
8041	while (1) {
8042	struct extent_buffer *leaf = path->nodes[0];
8043	struct btrfs_dev_extent *dext;
8044	int slot = path->slots[0];
8045	u64 chunk_offset;
8046	u64 physical_offset;
8047	u64 physical_len;
8048	u64 devid;
8049
8050	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: slot);
8051	if (key.type != BTRFS_DEV_EXTENT_KEY)
8052	break;
8053	devid = key.objectid;
8054	physical_offset = key.offset;
8055
8056	dext = btrfs_item_ptr(leaf, slot, struct btrfs_dev_extent);
8057	chunk_offset = btrfs_dev_extent_chunk_offset(eb: leaf, s: dext);
8058	physical_len = btrfs_dev_extent_length(eb: leaf, s: dext);
8059
8060	/* Check if this dev extent overlaps with the previous one */
8061	if (unlikely(devid == prev_devid && physical_offset < prev_dev_ext_end)) {
8062	btrfs_err(fs_info,
8063	"dev extent devid %llu physical offset %llu overlap with previous dev extent end %llu",
8064	devid, physical_offset, prev_dev_ext_end);
8065	return -EUCLEAN;
8066	}
8067
8068	ret = verify_one_dev_extent(fs_info, chunk_offset, devid,
8069	physical_offset, physical_len);
8070	if (ret < 0)
8071	return ret;
8072	prev_devid = devid;
8073	prev_dev_ext_end = physical_offset + physical_len;
8074
8075	ret = btrfs_next_item(root, p: path);
8076	if (ret < 0)
8077	return ret;
8078	if (ret > 0) {
8079	ret = 0;
8080	break;
8081	}
8082	}
8083
8084	/* Ensure all chunks have corresponding dev extents */
8085	return verify_chunk_dev_extent_mapping(fs_info);
8086	}
8087
8088	/*
8089	* Ensure that all devices registered in the fs have their device items in the
8090	* chunk tree.
8091	*
8092	* Return true if unexpected device is found.
8093	* Return false otherwise.
8094	*/
8095	bool btrfs_verify_dev_items(const struct btrfs_fs_info *fs_info)
8096	{
8097	struct btrfs_fs_devices *seed_devs;
8098	struct btrfs_device *dev;
8099	bool ret = false;
8100
8101	mutex_lock(&uuid_mutex);
8102	list_for_each_entry(dev, &fs_info->fs_devices->devices, dev_list) {
8103	if (!test_bit(BTRFS_DEV_STATE_ITEM_FOUND, &dev->dev_state)) {
8104	btrfs_err(fs_info,
8105	"devid %llu path %s is registered but not found in chunk tree",
8106	dev->devid, btrfs_dev_name(dev));
8107	ret = true;
8108	}
8109	}
8110	list_for_each_entry(seed_devs, &fs_info->fs_devices->seed_list, seed_list) {
8111	list_for_each_entry(dev, &seed_devs->devices, dev_list) {
8112	if (!test_bit(BTRFS_DEV_STATE_ITEM_FOUND, &dev->dev_state)) {
8113	btrfs_err(fs_info,
8114	"devid %llu path %s is registered but not found in chunk tree",
8115	dev->devid, btrfs_dev_name(dev));
8116	ret = true;
8117	}
8118	}
8119	}
8120	mutex_unlock(lock: &uuid_mutex);
8121	if (ret)
8122	btrfs_err(fs_info,
8123	"remove the above devices or use 'btrfs device scan --forget <dev>' to unregister them before mount");
8124	return ret;
8125	}
8126
8127	/*
8128	* Check whether the given block group or device is pinned by any inode being
8129	* used as a swapfile.
8130	*/
8131	bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr)
8132	{
8133	struct btrfs_swapfile_pin *sp;
8134	struct rb_node *node;
8135
8136	spin_lock(lock: &fs_info->swapfile_pins_lock);
8137	node = fs_info->swapfile_pins.rb_node;
8138	while (node) {
8139	sp = rb_entry(node, struct btrfs_swapfile_pin, node);
8140	if (ptr < sp->ptr)
8141	node = node->rb_left;
8142	else if (ptr > sp->ptr)
8143	node = node->rb_right;
8144	else
8145	break;
8146	}
8147	spin_unlock(lock: &fs_info->swapfile_pins_lock);
8148	return node != NULL;
8149	}
8150
8151	static int relocating_repair_kthread(void *data)
8152	{
8153	struct btrfs_block_group *cache = data;
8154	struct btrfs_fs_info *fs_info = cache->fs_info;
8155	u64 target;
8156	int ret = 0;
8157
8158	target = cache->start;
8159	btrfs_put_block_group(cache);
8160
8161	guard(super_write)(T: fs_info->sb);
8162
8163	if (!btrfs_exclop_start(fs_info, type: BTRFS_EXCLOP_BALANCE)) {
8164	btrfs_info(fs_info,
8165	"zoned: skip relocating block group %llu to repair: EBUSY",
8166	target);
8167	return -EBUSY;
8168	}
8169
8170	mutex_lock(&fs_info->reclaim_bgs_lock);
8171
8172	/* Ensure block group still exists */
8173	cache = btrfs_lookup_block_group(info: fs_info, bytenr: target);
8174	if (!cache)
8175	goto out;
8176
8177	if (!test_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags))
8178	goto out;
8179
8180	ret = btrfs_may_alloc_data_chunk(fs_info, chunk_offset: target);
8181	if (ret < 0)
8182	goto out;
8183
8184	btrfs_info(fs_info,
8185	"zoned: relocating block group %llu to repair IO failure",
8186	target);
8187	ret = btrfs_relocate_chunk(fs_info, chunk_offset: target, verbose: true);
8188
8189	out:
8190	if (cache)
8191	btrfs_put_block_group(cache);
8192	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
8193	btrfs_exclop_finish(fs_info);
8194
8195	return ret;
8196	}
8197
8198	bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical)
8199	{
8200	struct btrfs_block_group *cache;
8201
8202	if (!btrfs_is_zoned(fs_info))
8203	return false;
8204
8205	/* Do not attempt to repair in degraded state */
8206	if (btrfs_test_opt(fs_info, DEGRADED))
8207	return true;
8208
8209	cache = btrfs_lookup_block_group(info: fs_info, bytenr: logical);
8210	if (!cache)
8211	return true;
8212
8213	if (test_and_set_bit(nr: BLOCK_GROUP_FLAG_RELOCATING_REPAIR, addr: &cache->runtime_flags)) {
8214	btrfs_put_block_group(cache);
8215	return true;
8216	}
8217
8218	kthread_run(relocating_repair_kthread, cache,
8219	"btrfs-relocating-repair");
8220
8221	return true;
8222	}
8223
8224	static void map_raid56_repair_block(struct btrfs_io_context *bioc,
8225	struct btrfs_io_stripe *smap,
8226	u64 logical)
8227	{
8228	int data_stripes = nr_bioc_data_stripes(bioc);
8229	int i;
8230
8231	for (i = 0; i < data_stripes; i++) {
8232	u64 stripe_start = bioc->full_stripe_logical +
8233	btrfs_stripe_nr_to_offset(stripe_nr: i);
8234
8235	if (logical >= stripe_start &&
8236	logical < stripe_start + BTRFS_STRIPE_LEN)
8237	break;
8238	}
8239	ASSERT(i < data_stripes, "i=%d data_stripes=%d", i, data_stripes);
8240	smap->dev = bioc->stripes[i].dev;
8241	smap->physical = bioc->stripes[i].physical +
8242	((logical - bioc->full_stripe_logical) &
8243	BTRFS_STRIPE_LEN_MASK);
8244	}
8245
8246	/*
8247	* Map a repair write into a single device.
8248	*
8249	* A repair write is triggered by read time repair or scrub, which would only
8250	* update the contents of a single device.
8251	* Not update any other mirrors nor go through RMW path.
8252	*
8253	* Callers should ensure:
8254	*
8255	* - Call btrfs_bio_counter_inc_blocked() first
8256	* - The range does not cross stripe boundary
8257	* - Has a valid @mirror_num passed in.
8258	*/
8259	int btrfs_map_repair_block(struct btrfs_fs_info *fs_info,
8260	struct btrfs_io_stripe *smap, u64 logical,
8261	u32 length, int mirror_num)
8262	{
8263	struct btrfs_io_context *bioc = NULL;
8264	u64 map_length = length;
8265	int mirror_ret = mirror_num;
8266	int ret;
8267
8268	ASSERT(mirror_num > 0, "mirror_num=%d", mirror_num);
8269
8270	ret = btrfs_map_block(fs_info, op: BTRFS_MAP_WRITE, logical, length: &map_length,
8271	bioc_ret: &bioc, smap, mirror_num_ret: &mirror_ret);
8272	if (ret < 0)
8273	return ret;
8274
8275	/* The map range should not cross stripe boundary. */
8276	ASSERT(map_length >= length, "map_length=%llu length=%u", map_length, length);
8277
8278	/* Already mapped to single stripe. */
8279	if (!bioc)
8280	goto out;
8281
8282	/* Map the RAID56 multi-stripe writes to a single one. */
8283	if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
8284	map_raid56_repair_block(bioc, smap, logical);
8285	goto out;
8286	}
8287
8288	ASSERT(mirror_num <= bioc->num_stripes,
8289	"mirror_num=%d num_stripes=%d", mirror_num, bioc->num_stripes);
8290	smap->dev = bioc->stripes[mirror_num - 1].dev;
8291	smap->physical = bioc->stripes[mirror_num - 1].physical;
8292	out:
8293	btrfs_put_bioc(bioc);
8294	ASSERT(smap->dev);
8295	return 0;
8296	}
8297