1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (C) 2007 Oracle. All rights reserved.
4	*/
5
6	#include <linux/sched.h>
7	#include <linux/sched/mm.h>
8	#include <linux/slab.h>
9	#include <linux/ratelimit.h>
10	#include <linux/kthread.h>
11	#include <linux/semaphore.h>
12	#include <linux/uuid.h>
13	#include <linux/list_sort.h>
14	#include <linux/namei.h>
15	#include "misc.h"
16	#include "ctree.h"
17	#include "disk-io.h"
18	#include "transaction.h"
19	#include "volumes.h"
20	#include "raid56.h"
21	#include "rcu-string.h"
22	#include "dev-replace.h"
23	#include "sysfs.h"
24	#include "tree-checker.h"
25	#include "space-info.h"
26	#include "block-group.h"
27	#include "discard.h"
28	#include "zoned.h"
29	#include "fs.h"
30	#include "accessors.h"
31	#include "uuid-tree.h"
32	#include "ioctl.h"
33	#include "relocation.h"
34	#include "scrub.h"
35	#include "super.h"
36	#include "raid-stripe-tree.h"
37
38	#define BTRFS_BLOCK_GROUP_STRIPE_MASK (BTRFS_BLOCK_GROUP_RAID0 | \
39	BTRFS_BLOCK_GROUP_RAID10 | \
40	BTRFS_BLOCK_GROUP_RAID56_MASK)
41
42	struct btrfs_io_geometry {
43	u32 stripe_index;
44	u32 stripe_nr;
45	int mirror_num;
46	int num_stripes;
47	u64 stripe_offset;
48	u64 raid56_full_stripe_start;
49	int max_errors;
50	enum btrfs_map_op op;
51	};
52
53	const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
54	[BTRFS_RAID_RAID10] = {
55	.sub_stripes = 2,
56	.dev_stripes = 1,
57	.devs_max = 0, /* 0 == as many as possible */
58	.devs_min = 2,
59	.tolerated_failures = 1,
60	.devs_increment = 2,
61	.ncopies = 2,
62	.nparity = 0,
63	.raid_name = "raid10",
64	.bg_flag = BTRFS_BLOCK_GROUP_RAID10,
65	.mindev_error = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
66	},
67	[BTRFS_RAID_RAID1] = {
68	.sub_stripes = 1,
69	.dev_stripes = 1,
70	.devs_max = 2,
71	.devs_min = 2,
72	.tolerated_failures = 1,
73	.devs_increment = 2,
74	.ncopies = 2,
75	.nparity = 0,
76	.raid_name = "raid1",
77	.bg_flag = BTRFS_BLOCK_GROUP_RAID1,
78	.mindev_error = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
79	},
80	[BTRFS_RAID_RAID1C3] = {
81	.sub_stripes = 1,
82	.dev_stripes = 1,
83	.devs_max = 3,
84	.devs_min = 3,
85	.tolerated_failures = 2,
86	.devs_increment = 3,
87	.ncopies = 3,
88	.nparity = 0,
89	.raid_name = "raid1c3",
90	.bg_flag = BTRFS_BLOCK_GROUP_RAID1C3,
91	.mindev_error = BTRFS_ERROR_DEV_RAID1C3_MIN_NOT_MET,
92	},
93	[BTRFS_RAID_RAID1C4] = {
94	.sub_stripes = 1,
95	.dev_stripes = 1,
96	.devs_max = 4,
97	.devs_min = 4,
98	.tolerated_failures = 3,
99	.devs_increment = 4,
100	.ncopies = 4,
101	.nparity = 0,
102	.raid_name = "raid1c4",
103	.bg_flag = BTRFS_BLOCK_GROUP_RAID1C4,
104	.mindev_error = BTRFS_ERROR_DEV_RAID1C4_MIN_NOT_MET,
105	},
106	[BTRFS_RAID_DUP] = {
107	.sub_stripes = 1,
108	.dev_stripes = 2,
109	.devs_max = 1,
110	.devs_min = 1,
111	.tolerated_failures = 0,
112	.devs_increment = 1,
113	.ncopies = 2,
114	.nparity = 0,
115	.raid_name = "dup",
116	.bg_flag = BTRFS_BLOCK_GROUP_DUP,
117	.mindev_error = 0,
118	},
119	[BTRFS_RAID_RAID0] = {
120	.sub_stripes = 1,
121	.dev_stripes = 1,
122	.devs_max = 0,
123	.devs_min = 1,
124	.tolerated_failures = 0,
125	.devs_increment = 1,
126	.ncopies = 1,
127	.nparity = 0,
128	.raid_name = "raid0",
129	.bg_flag = BTRFS_BLOCK_GROUP_RAID0,
130	.mindev_error = 0,
131	},
132	[BTRFS_RAID_SINGLE] = {
133	.sub_stripes = 1,
134	.dev_stripes = 1,
135	.devs_max = 1,
136	.devs_min = 1,
137	.tolerated_failures = 0,
138	.devs_increment = 1,
139	.ncopies = 1,
140	.nparity = 0,
141	.raid_name = "single",
142	.bg_flag = 0,
143	.mindev_error = 0,
144	},
145	[BTRFS_RAID_RAID5] = {
146	.sub_stripes = 1,
147	.dev_stripes = 1,
148	.devs_max = 0,
149	.devs_min = 2,
150	.tolerated_failures = 1,
151	.devs_increment = 1,
152	.ncopies = 1,
153	.nparity = 1,
154	.raid_name = "raid5",
155	.bg_flag = BTRFS_BLOCK_GROUP_RAID5,
156	.mindev_error = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
157	},
158	[BTRFS_RAID_RAID6] = {
159	.sub_stripes = 1,
160	.dev_stripes = 1,
161	.devs_max = 0,
162	.devs_min = 3,
163	.tolerated_failures = 2,
164	.devs_increment = 1,
165	.ncopies = 1,
166	.nparity = 2,
167	.raid_name = "raid6",
168	.bg_flag = BTRFS_BLOCK_GROUP_RAID6,
169	.mindev_error = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
170	},
171	};
172
173	/*
174	* Convert block group flags (BTRFS_BLOCK_GROUP_*) to btrfs_raid_types, which
175	* can be used as index to access btrfs_raid_array[].
176	*/
177	enum btrfs_raid_types __attribute_const__ btrfs_bg_flags_to_raid_index(u64 flags)
178	{
179	const u64 profile = (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK);
180
181	if (!profile)
182	return BTRFS_RAID_SINGLE;
183
184	return BTRFS_BG_FLAG_TO_INDEX(profile);
185	}
186
187	const char *btrfs_bg_type_to_raid_name(u64 flags)
188	{
189	const int index = btrfs_bg_flags_to_raid_index(flags);
190
191	if (index >= BTRFS_NR_RAID_TYPES)
192	return NULL;
193
194	return btrfs_raid_array[index].raid_name;
195	}
196
197	int btrfs_nr_parity_stripes(u64 type)
198	{
199	enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(flags: type);
200
201	return btrfs_raid_array[index].nparity;
202	}
203
204	/*
205	* Fill @buf with textual description of @bg_flags, no more than @size_buf
206	* bytes including terminating null byte.
207	*/
208	void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf)
209	{
210	int i;
211	int ret;
212	char *bp = buf;
213	u64 flags = bg_flags;
214	u32 size_bp = size_buf;
215
216	if (!flags) {
217	strcpy(p: bp, q: "NONE");
218	return;
219	}
220
221	#define DESCRIBE_FLAG(flag, desc) \
222	do { \
223	if (flags & (flag)) { \
224	ret = snprintf(bp, size_bp, "%s|", (desc)); \
225	if (ret < 0 || ret >= size_bp) \
226	goto out_overflow; \
227	size_bp -= ret; \
228	bp += ret; \
229	flags &= ~(flag); \
230	} \
231	} while (0)
232
233	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data");
234	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system");
235	DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata");
236
237	DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single");
238	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
239	DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag,
240	btrfs_raid_array[i].raid_name);
241	#undef DESCRIBE_FLAG
242
243	if (flags) {
244	ret = snprintf(buf: bp, size: size_bp, fmt: "0x%llx|", flags);
245	size_bp -= ret;
246	}
247
248	if (size_bp < size_buf)
249	buf[size_buf - size_bp - 1] = '\0'; /* remove last | */
250
251	/*
252	* The text is trimmed, it's up to the caller to provide sufficiently
253	* large buffer
254	*/
255	out_overflow:;
256	}
257
258	static int init_first_rw_device(struct btrfs_trans_handle *trans);
259	static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
260	static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
261
262	/*
263	* Device locking
264	* ==============
265	*
266	* There are several mutexes that protect manipulation of devices and low-level
267	* structures like chunks but not block groups, extents or files
268	*
269	* uuid_mutex (global lock)
270	* ------------------------
271	* protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
272	* the SCAN_DEV ioctl registration or from mount either implicitly (the first
273	* device) or requested by the device= mount option
274	*
275	* the mutex can be very coarse and can cover long-running operations
276	*
277	* protects: updates to fs_devices counters like missing devices, rw devices,
278	* seeding, structure cloning, opening/closing devices at mount/umount time
279	*
280	* global::fs_devs - add, remove, updates to the global list
281	*
282	* does not protect: manipulation of the fs_devices::devices list in general
283	* but in mount context it could be used to exclude list modifications by eg.
284	* scan ioctl
285	*
286	* btrfs_device::name - renames (write side), read is RCU
287	*
288	* fs_devices::device_list_mutex (per-fs, with RCU)
289	* ------------------------------------------------
290	* protects updates to fs_devices::devices, ie. adding and deleting
291	*
292	* simple list traversal with read-only actions can be done with RCU protection
293	*
294	* may be used to exclude some operations from running concurrently without any
295	* modifications to the list (see write_all_supers)
296	*
297	* Is not required at mount and close times, because our device list is
298	* protected by the uuid_mutex at that point.
299	*
300	* balance_mutex
301	* -------------
302	* protects balance structures (status, state) and context accessed from
303	* several places (internally, ioctl)
304	*
305	* chunk_mutex
306	* -----------
307	* protects chunks, adding or removing during allocation, trim or when a new
308	* device is added/removed. Additionally it also protects post_commit_list of
309	* individual devices, since they can be added to the transaction's
310	* post_commit_list only with chunk_mutex held.
311	*
312	* cleaner_mutex
313	* -------------
314	* a big lock that is held by the cleaner thread and prevents running subvolume
315	* cleaning together with relocation or delayed iputs
316	*
317	*
318	* Lock nesting
319	* ============
320	*
321	* uuid_mutex
322	* device_list_mutex
323	* chunk_mutex
324	* balance_mutex
325	*
326	*
327	* Exclusive operations
328	* ====================
329	*
330	* Maintains the exclusivity of the following operations that apply to the
331	* whole filesystem and cannot run in parallel.
332	*
333	* - Balance (*)
334	* - Device add
335	* - Device remove
336	* - Device replace (*)
337	* - Resize
338	*
339	* The device operations (as above) can be in one of the following states:
340	*
341	* - Running state
342	* - Paused state
343	* - Completed state
344	*
345	* Only device operations marked with (*) can go into the Paused state for the
346	* following reasons:
347	*
348	* - ioctl (only Balance can be Paused through ioctl)
349	* - filesystem remounted as read-only
350	* - filesystem unmounted and mounted as read-only
351	* - system power-cycle and filesystem mounted as read-only
352	* - filesystem or device errors leading to forced read-only
353	*
354	* The status of exclusive operation is set and cleared atomically.
355	* During the course of Paused state, fs_info::exclusive_operation remains set.
356	* A device operation in Paused or Running state can be canceled or resumed
357	* either by ioctl (Balance only) or when remounted as read-write.
358	* The exclusive status is cleared when the device operation is canceled or
359	* completed.
360	*/
361
362	DEFINE_MUTEX(uuid_mutex);
363	static LIST_HEAD(fs_uuids);
364	struct list_head * __attribute_const__ btrfs_get_fs_uuids(void)
365	{
366	return &fs_uuids;
367	}
368
369	/*
370	* Allocate new btrfs_fs_devices structure identified by a fsid.
371	*
372	* @fsid: if not NULL, copy the UUID to fs_devices::fsid and to
373	* fs_devices::metadata_fsid
374	*
375	* Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR().
376	* The returned struct is not linked onto any lists and can be destroyed with
377	* kfree() right away.
378	*/
379	static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid)
380	{
381	struct btrfs_fs_devices *fs_devs;
382
383	fs_devs = kzalloc(size: sizeof(*fs_devs), GFP_KERNEL);
384	if (!fs_devs)
385	return ERR_PTR(error: -ENOMEM);
386
387	mutex_init(&fs_devs->device_list_mutex);
388
389	INIT_LIST_HEAD(list: &fs_devs->devices);
390	INIT_LIST_HEAD(list: &fs_devs->alloc_list);
391	INIT_LIST_HEAD(list: &fs_devs->fs_list);
392	INIT_LIST_HEAD(list: &fs_devs->seed_list);
393
394	if (fsid) {
395	memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
396	memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE);
397	}
398
399	return fs_devs;
400	}
401
402	static void btrfs_free_device(struct btrfs_device *device)
403	{
404	WARN_ON(!list_empty(&device->post_commit_list));
405	rcu_string_free(str: device->name);
406	extent_io_tree_release(tree: &device->alloc_state);
407	btrfs_destroy_dev_zone_info(device);
408	kfree(objp: device);
409	}
410
411	static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
412	{
413	struct btrfs_device *device;
414
415	WARN_ON(fs_devices->opened);
416	while (!list_empty(head: &fs_devices->devices)) {
417	device = list_entry(fs_devices->devices.next,
418	struct btrfs_device, dev_list);
419	list_del(entry: &device->dev_list);
420	btrfs_free_device(device);
421	}
422	kfree(objp: fs_devices);
423	}
424
425	void __exit btrfs_cleanup_fs_uuids(void)
426	{
427	struct btrfs_fs_devices *fs_devices;
428
429	while (!list_empty(head: &fs_uuids)) {
430	fs_devices = list_entry(fs_uuids.next,
431	struct btrfs_fs_devices, fs_list);
432	list_del(entry: &fs_devices->fs_list);
433	free_fs_devices(fs_devices);
434	}
435	}
436
437	static bool match_fsid_fs_devices(const struct btrfs_fs_devices *fs_devices,
438	const u8 *fsid, const u8 *metadata_fsid)
439	{
440	if (memcmp(p: fsid, q: fs_devices->fsid, BTRFS_FSID_SIZE) != 0)
441	return false;
442
443	if (!metadata_fsid)
444	return true;
445
446	if (memcmp(p: metadata_fsid, q: fs_devices->metadata_uuid, BTRFS_FSID_SIZE) != 0)
447	return false;
448
449	return true;
450	}
451
452	static noinline struct btrfs_fs_devices *find_fsid(
453	const u8 *fsid, const u8 *metadata_fsid)
454	{
455	struct btrfs_fs_devices *fs_devices;
456
457	ASSERT(fsid);
458
459	/* Handle non-split brain cases */
460	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
461	if (match_fsid_fs_devices(fs_devices, fsid, metadata_fsid))
462	return fs_devices;
463	}
464	return NULL;
465	}
466
467	static int
468	btrfs_get_bdev_and_sb(const char *device_path, blk_mode_t flags, void *holder,
469	int flush, struct file **bdev_file,
470	struct btrfs_super_block **disk_super)
471	{
472	struct block_device *bdev;
473	int ret;
474
475	*bdev_file = bdev_file_open_by_path(path: device_path, mode: flags, holder, NULL);
476
477	if (IS_ERR(ptr: *bdev_file)) {
478	ret = PTR_ERR(ptr: *bdev_file);
479	goto error;
480	}
481	bdev = file_bdev(bdev_file: *bdev_file);
482
483	if (flush)
484	sync_blockdev(bdev);
485	ret = set_blocksize(bdev, BTRFS_BDEV_BLOCKSIZE);
486	if (ret) {
487	fput(*bdev_file);
488	goto error;
489	}
490	invalidate_bdev(bdev);
491	*disk_super = btrfs_read_dev_super(bdev);
492	if (IS_ERR(ptr: *disk_super)) {
493	ret = PTR_ERR(ptr: *disk_super);
494	fput(*bdev_file);
495	goto error;
496	}
497
498	return 0;
499
500	error:
501	*bdev_file = NULL;
502	return ret;
503	}
504
505	/*
506	* Search and remove all stale devices (which are not mounted). When both
507	* inputs are NULL, it will search and release all stale devices.
508	*
509	* @devt: Optional. When provided will it release all unmounted devices
510	* matching this devt only.
511	* @skip_device: Optional. Will skip this device when searching for the stale
512	* devices.
513	*
514	* Return: 0 for success or if @devt is 0.
515	* -EBUSY if @devt is a mounted device.
516	* -ENOENT if @devt does not match any device in the list.
517	*/
518	static int btrfs_free_stale_devices(dev_t devt, struct btrfs_device *skip_device)
519	{
520	struct btrfs_fs_devices *fs_devices, *tmp_fs_devices;
521	struct btrfs_device *device, *tmp_device;
522	int ret;
523	bool freed = false;
524
525	lockdep_assert_held(&uuid_mutex);
526
527	/* Return good status if there is no instance of devt. */
528	ret = 0;
529	list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) {
530
531	mutex_lock(&fs_devices->device_list_mutex);
532	list_for_each_entry_safe(device, tmp_device,
533	&fs_devices->devices, dev_list) {
534	if (skip_device && skip_device == device)
535	continue;
536	if (devt && devt != device->devt)
537	continue;
538	if (fs_devices->opened) {
539	if (devt)
540	ret = -EBUSY;
541	break;
542	}
543
544	/* delete the stale device */
545	fs_devices->num_devices--;
546	list_del(entry: &device->dev_list);
547	btrfs_free_device(device);
548
549	freed = true;
550	}
551	mutex_unlock(lock: &fs_devices->device_list_mutex);
552
553	if (fs_devices->num_devices == 0) {
554	btrfs_sysfs_remove_fsid(fs_devs: fs_devices);
555	list_del(entry: &fs_devices->fs_list);
556	free_fs_devices(fs_devices);
557	}
558	}
559
560	/* If there is at least one freed device return 0. */
561	if (freed)
562	return 0;
563
564	return ret;
565	}
566
567	static struct btrfs_fs_devices *find_fsid_by_device(
568	struct btrfs_super_block *disk_super,
569	dev_t devt, bool *same_fsid_diff_dev)
570	{
571	struct btrfs_fs_devices *fsid_fs_devices;
572	struct btrfs_fs_devices *devt_fs_devices;
573	const bool has_metadata_uuid = (btrfs_super_incompat_flags(s: disk_super) &
574	BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
575	bool found_by_devt = false;
576
577	/* Find the fs_device by the usual method, if found use it. */
578	fsid_fs_devices = find_fsid(fsid: disk_super->fsid,
579	metadata_fsid: has_metadata_uuid ? disk_super->metadata_uuid : NULL);
580
581	/* The temp_fsid feature is supported only with single device filesystem. */
582	if (btrfs_super_num_devices(s: disk_super) != 1)
583	return fsid_fs_devices;
584
585	/*
586	* A seed device is an integral component of the sprout device, which
587	* functions as a multi-device filesystem. So, temp-fsid feature is
588	* not supported.
589	*/
590	if (btrfs_super_flags(s: disk_super) & BTRFS_SUPER_FLAG_SEEDING)
591	return fsid_fs_devices;
592
593	/* Try to find a fs_devices by matching devt. */
594	list_for_each_entry(devt_fs_devices, &fs_uuids, fs_list) {
595	struct btrfs_device *device;
596
597	list_for_each_entry(device, &devt_fs_devices->devices, dev_list) {
598	if (device->devt == devt) {
599	found_by_devt = true;
600	break;
601	}
602	}
603	if (found_by_devt)
604	break;
605	}
606
607	if (found_by_devt) {
608	/* Existing device. */
609	if (fsid_fs_devices == NULL) {
610	if (devt_fs_devices->opened == 0) {
611	/* Stale device. */
612	return NULL;
613	} else {
614	/* temp_fsid is mounting a subvol. */
615	return devt_fs_devices;
616	}
617	} else {
618	/* Regular or temp_fsid device mounting a subvol. */
619	return devt_fs_devices;
620	}
621	} else {
622	/* New device. */
623	if (fsid_fs_devices == NULL) {
624	return NULL;
625	} else {
626	/* sb::fsid is already used create a new temp_fsid. */
627	*same_fsid_diff_dev = true;
628	return NULL;
629	}
630	}
631
632	/* Not reached. */
633	}
634
635	/*
636	* This is only used on mount, and we are protected from competing things
637	* messing with our fs_devices by the uuid_mutex, thus we do not need the
638	* fs_devices->device_list_mutex here.
639	*/
640	static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
641	struct btrfs_device *device, blk_mode_t flags,
642	void *holder)
643	{
644	struct file *bdev_file;
645	struct btrfs_super_block *disk_super;
646	u64 devid;
647	int ret;
648
649	if (device->bdev)
650	return -EINVAL;
651	if (!device->name)
652	return -EINVAL;
653
654	ret = btrfs_get_bdev_and_sb(device_path: device->name->str, flags, holder, flush: 1,
655	bdev_file: &bdev_file, disk_super: &disk_super);
656	if (ret)
657	return ret;
658
659	devid = btrfs_stack_device_id(s: &disk_super->dev_item);
660	if (devid != device->devid)
661	goto error_free_page;
662
663	if (memcmp(p: device->uuid, q: disk_super->dev_item.uuid, BTRFS_UUID_SIZE))
664	goto error_free_page;
665
666	device->generation = btrfs_super_generation(s: disk_super);
667
668	if (btrfs_super_flags(s: disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
669	if (btrfs_super_incompat_flags(s: disk_super) &
670	BTRFS_FEATURE_INCOMPAT_METADATA_UUID) {
671	pr_err(
672	"BTRFS: Invalid seeding and uuid-changed device detected\n");
673	goto error_free_page;
674	}
675
676	clear_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
677	fs_devices->seeding = true;
678	} else {
679	if (bdev_read_only(bdev: file_bdev(bdev_file)))
680	clear_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
681	else
682	set_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
683	}
684
685	if (!bdev_nonrot(bdev: file_bdev(bdev_file)))
686	fs_devices->rotating = true;
687
688	if (bdev_max_discard_sectors(bdev: file_bdev(bdev_file)))
689	fs_devices->discardable = true;
690
691	device->bdev_file = bdev_file;
692	device->bdev = file_bdev(bdev_file);
693	clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, addr: &device->dev_state);
694
695	if (device->devt != device->bdev->bd_dev) {
696	btrfs_warn(NULL,
697	"device %s maj:min changed from %d:%d to %d:%d",
698	device->name->str, MAJOR(device->devt),
699	MINOR(device->devt), MAJOR(device->bdev->bd_dev),
700	MINOR(device->bdev->bd_dev));
701
702	device->devt = device->bdev->bd_dev;
703	}
704
705	fs_devices->open_devices++;
706	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
707	device->devid != BTRFS_DEV_REPLACE_DEVID) {
708	fs_devices->rw_devices++;
709	list_add_tail(new: &device->dev_alloc_list, head: &fs_devices->alloc_list);
710	}
711	btrfs_release_disk_super(super: disk_super);
712
713	return 0;
714
715	error_free_page:
716	btrfs_release_disk_super(super: disk_super);
717	fput(bdev_file);
718
719	return -EINVAL;
720	}
721
722	u8 *btrfs_sb_fsid_ptr(struct btrfs_super_block *sb)
723	{
724	bool has_metadata_uuid = (btrfs_super_incompat_flags(s: sb) &
725	BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
726
727	return has_metadata_uuid ? sb->metadata_uuid : sb->fsid;
728	}
729
730	/*
731	* Add new device to list of registered devices
732	*
733	* Returns:
734	* device pointer which was just added or updated when successful
735	* error pointer when failed
736	*/
737	static noinline struct btrfs_device *device_list_add(const char *path,
738	struct btrfs_super_block *disk_super,
739	bool *new_device_added)
740	{
741	struct btrfs_device *device;
742	struct btrfs_fs_devices *fs_devices = NULL;
743	struct rcu_string *name;
744	u64 found_transid = btrfs_super_generation(s: disk_super);
745	u64 devid = btrfs_stack_device_id(s: &disk_super->dev_item);
746	dev_t path_devt;
747	int error;
748	bool same_fsid_diff_dev = false;
749	bool has_metadata_uuid = (btrfs_super_incompat_flags(s: disk_super) &
750	BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
751
752	if (btrfs_super_flags(s: disk_super) & BTRFS_SUPER_FLAG_CHANGING_FSID_V2) {
753	btrfs_err(NULL,
754	"device %s has incomplete metadata_uuid change, please use btrfstune to complete",
755	path);
756	return ERR_PTR(error: -EAGAIN);
757	}
758
759	error = lookup_bdev(pathname: path, dev: &path_devt);
760	if (error) {
761	btrfs_err(NULL, "failed to lookup block device for path %s: %d",
762	path, error);
763	return ERR_PTR(error);
764	}
765
766	fs_devices = find_fsid_by_device(disk_super, devt: path_devt, same_fsid_diff_dev: &same_fsid_diff_dev);
767
768	if (!fs_devices) {
769	fs_devices = alloc_fs_devices(fsid: disk_super->fsid);
770	if (IS_ERR(ptr: fs_devices))
771	return ERR_CAST(ptr: fs_devices);
772
773	if (has_metadata_uuid)
774	memcpy(fs_devices->metadata_uuid,
775	disk_super->metadata_uuid, BTRFS_FSID_SIZE);
776
777	if (same_fsid_diff_dev) {
778	generate_random_uuid(uuid: fs_devices->fsid);
779	fs_devices->temp_fsid = true;
780	pr_info("BTRFS: device %s (%d:%d) using temp-fsid %pU\n",
781	path, MAJOR(path_devt), MINOR(path_devt),
782	fs_devices->fsid);
783	}
784
785	mutex_lock(&fs_devices->device_list_mutex);
786	list_add(new: &fs_devices->fs_list, head: &fs_uuids);
787
788	device = NULL;
789	} else {
790	struct btrfs_dev_lookup_args args = {
791	.devid = devid,
792	.uuid = disk_super->dev_item.uuid,
793	};
794
795	mutex_lock(&fs_devices->device_list_mutex);
796	device = btrfs_find_device(fs_devices, args: &args);
797
798	if (found_transid > fs_devices->latest_generation) {
799	memcpy(fs_devices->fsid, disk_super->fsid,
800	BTRFS_FSID_SIZE);
801	memcpy(fs_devices->metadata_uuid,
802	btrfs_sb_fsid_ptr(disk_super), BTRFS_FSID_SIZE);
803	}
804	}
805
806	if (!device) {
807	unsigned int nofs_flag;
808
809	if (fs_devices->opened) {
810	btrfs_err(NULL,
811	"device %s (%d:%d) belongs to fsid %pU, and the fs is already mounted, scanned by %s (%d)",
812	path, MAJOR(path_devt), MINOR(path_devt),
813	fs_devices->fsid, current->comm,
814	task_pid_nr(current));
815	mutex_unlock(lock: &fs_devices->device_list_mutex);
816	return ERR_PTR(error: -EBUSY);
817	}
818
819	nofs_flag = memalloc_nofs_save();
820	device = btrfs_alloc_device(NULL, devid: &devid,
821	uuid: disk_super->dev_item.uuid, path);
822	memalloc_nofs_restore(flags: nofs_flag);
823	if (IS_ERR(ptr: device)) {
824	mutex_unlock(lock: &fs_devices->device_list_mutex);
825	/* we can safely leave the fs_devices entry around */
826	return device;
827	}
828
829	device->devt = path_devt;
830
831	list_add_rcu(new: &device->dev_list, head: &fs_devices->devices);
832	fs_devices->num_devices++;
833
834	device->fs_devices = fs_devices;
835	*new_device_added = true;
836
837	if (disk_super->label[0])
838	pr_info(
839	"BTRFS: device label %s devid %llu transid %llu %s (%d:%d) scanned by %s (%d)\n",
840	disk_super->label, devid, found_transid, path,
841	MAJOR(path_devt), MINOR(path_devt),
842	current->comm, task_pid_nr(current));
843	else
844	pr_info(
845	"BTRFS: device fsid %pU devid %llu transid %llu %s (%d:%d) scanned by %s (%d)\n",
846	disk_super->fsid, devid, found_transid, path,
847	MAJOR(path_devt), MINOR(path_devt),
848	current->comm, task_pid_nr(current));
849
850	} else if (!device->name || strcmp(device->name->str, path)) {
851	/*
852	* When FS is already mounted.
853	* 1. If you are here and if the device->name is NULL that
854	* means this device was missing at time of FS mount.
855	* 2. If you are here and if the device->name is different
856	* from 'path' that means either
857	* a. The same device disappeared and reappeared with
858	* different name. or
859	* b. The missing-disk-which-was-replaced, has
860	* reappeared now.
861	*
862	* We must allow 1 and 2a above. But 2b would be a spurious
863	* and unintentional.
864	*
865	* Further in case of 1 and 2a above, the disk at 'path'
866	* would have missed some transaction when it was away and
867	* in case of 2a the stale bdev has to be updated as well.
868	* 2b must not be allowed at all time.
869	*/
870
871	/*
872	* For now, we do allow update to btrfs_fs_device through the
873	* btrfs dev scan cli after FS has been mounted. We're still
874	* tracking a problem where systems fail mount by subvolume id
875	* when we reject replacement on a mounted FS.
876	*/
877	if (!fs_devices->opened && found_transid < device->generation) {
878	/*
879	* That is if the FS is _not_ mounted and if you
880	* are here, that means there is more than one
881	* disk with same uuid and devid.We keep the one
882	* with larger generation number or the last-in if
883	* generation are equal.
884	*/
885	mutex_unlock(lock: &fs_devices->device_list_mutex);
886	btrfs_err(NULL,
887	"device %s already registered with a higher generation, found %llu expect %llu",
888	path, found_transid, device->generation);
889	return ERR_PTR(error: -EEXIST);
890	}
891
892	/*
893	* We are going to replace the device path for a given devid,
894	* make sure it's the same device if the device is mounted
895	*
896	* NOTE: the device->fs_info may not be reliable here so pass
897	* in a NULL to message helpers instead. This avoids a possible
898	* use-after-free when the fs_info and fs_info->sb are already
899	* torn down.
900	*/
901	if (device->bdev) {
902	if (device->devt != path_devt) {
903	mutex_unlock(lock: &fs_devices->device_list_mutex);
904	btrfs_warn_in_rcu(NULL,
905	"duplicate device %s devid %llu generation %llu scanned by %s (%d)",
906	path, devid, found_transid,
907	current->comm,
908	task_pid_nr(current));
909	return ERR_PTR(error: -EEXIST);
910	}
911	btrfs_info_in_rcu(NULL,
912	"devid %llu device path %s changed to %s scanned by %s (%d)",
913	devid, btrfs_dev_name(device),
914	path, current->comm,
915	task_pid_nr(current));
916	}
917
918	name = rcu_string_strdup(src: path, GFP_NOFS);
919	if (!name) {
920	mutex_unlock(lock: &fs_devices->device_list_mutex);
921	return ERR_PTR(error: -ENOMEM);
922	}
923	rcu_string_free(str: device->name);
924	rcu_assign_pointer(device->name, name);
925	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
926	fs_devices->missing_devices--;
927	clear_bit(BTRFS_DEV_STATE_MISSING, addr: &device->dev_state);
928	}
929	device->devt = path_devt;
930	}
931
932	/*
933	* Unmount does not free the btrfs_device struct but would zero
934	* generation along with most of the other members. So just update
935	* it back. We need it to pick the disk with largest generation
936	* (as above).
937	*/
938	if (!fs_devices->opened) {
939	device->generation = found_transid;
940	fs_devices->latest_generation = max_t(u64, found_transid,
941	fs_devices->latest_generation);
942	}
943
944	fs_devices->total_devices = btrfs_super_num_devices(s: disk_super);
945
946	mutex_unlock(lock: &fs_devices->device_list_mutex);
947	return device;
948	}
949
950	static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
951	{
952	struct btrfs_fs_devices *fs_devices;
953	struct btrfs_device *device;
954	struct btrfs_device *orig_dev;
955	int ret = 0;
956
957	lockdep_assert_held(&uuid_mutex);
958
959	fs_devices = alloc_fs_devices(fsid: orig->fsid);
960	if (IS_ERR(ptr: fs_devices))
961	return fs_devices;
962
963	fs_devices->total_devices = orig->total_devices;
964
965	list_for_each_entry(orig_dev, &orig->devices, dev_list) {
966	const char *dev_path = NULL;
967
968	/*
969	* This is ok to do without RCU read locked because we hold the
970	* uuid mutex so nothing we touch in here is going to disappear.
971	*/
972	if (orig_dev->name)
973	dev_path = orig_dev->name->str;
974
975	device = btrfs_alloc_device(NULL, devid: &orig_dev->devid,
976	uuid: orig_dev->uuid, path: dev_path);
977	if (IS_ERR(ptr: device)) {
978	ret = PTR_ERR(ptr: device);
979	goto error;
980	}
981
982	if (orig_dev->zone_info) {
983	struct btrfs_zoned_device_info *zone_info;
984
985	zone_info = btrfs_clone_dev_zone_info(orig_dev);
986	if (!zone_info) {
987	btrfs_free_device(device);
988	ret = -ENOMEM;
989	goto error;
990	}
991	device->zone_info = zone_info;
992	}
993
994	list_add(new: &device->dev_list, head: &fs_devices->devices);
995	device->fs_devices = fs_devices;
996	fs_devices->num_devices++;
997	}
998	return fs_devices;
999	error:
1000	free_fs_devices(fs_devices);
1001	return ERR_PTR(error: ret);
1002	}
1003
1004	static void __btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices,
1005	struct btrfs_device **latest_dev)
1006	{
1007	struct btrfs_device *device, *next;
1008
1009	/* This is the initialized path, it is safe to release the devices. */
1010	list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
1011	if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) {
1012	if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1013	&device->dev_state) &&
1014	!test_bit(BTRFS_DEV_STATE_MISSING,
1015	&device->dev_state) &&
1016	(!*latest_dev ||
1017	device->generation > (*latest_dev)->generation)) {
1018	*latest_dev = device;
1019	}
1020	continue;
1021	}
1022
1023	/*
1024	* We have already validated the presence of BTRFS_DEV_REPLACE_DEVID,
1025	* in btrfs_init_dev_replace() so just continue.
1026	*/
1027	if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1028	continue;
1029
1030	if (device->bdev_file) {
1031	fput(device->bdev_file);
1032	device->bdev = NULL;
1033	device->bdev_file = NULL;
1034	fs_devices->open_devices--;
1035	}
1036	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1037	list_del_init(entry: &device->dev_alloc_list);
1038	clear_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
1039	fs_devices->rw_devices--;
1040	}
1041	list_del_init(entry: &device->dev_list);
1042	fs_devices->num_devices--;
1043	btrfs_free_device(device);
1044	}
1045
1046	}
1047
1048	/*
1049	* After we have read the system tree and know devids belonging to this
1050	* filesystem, remove the device which does not belong there.
1051	*/
1052	void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices)
1053	{
1054	struct btrfs_device *latest_dev = NULL;
1055	struct btrfs_fs_devices *seed_dev;
1056
1057	mutex_lock(&uuid_mutex);
1058	__btrfs_free_extra_devids(fs_devices, latest_dev: &latest_dev);
1059
1060	list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list)
1061	__btrfs_free_extra_devids(fs_devices: seed_dev, latest_dev: &latest_dev);
1062
1063	fs_devices->latest_dev = latest_dev;
1064
1065	mutex_unlock(lock: &uuid_mutex);
1066	}
1067
1068	static void btrfs_close_bdev(struct btrfs_device *device)
1069	{
1070	if (!device->bdev)
1071	return;
1072
1073	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1074	sync_blockdev(bdev: device->bdev);
1075	invalidate_bdev(bdev: device->bdev);
1076	}
1077
1078	fput(device->bdev_file);
1079	}
1080
1081	static void btrfs_close_one_device(struct btrfs_device *device)
1082	{
1083	struct btrfs_fs_devices *fs_devices = device->fs_devices;
1084
1085	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
1086	device->devid != BTRFS_DEV_REPLACE_DEVID) {
1087	list_del_init(entry: &device->dev_alloc_list);
1088	fs_devices->rw_devices--;
1089	}
1090
1091	if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1092	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, addr: &device->dev_state);
1093
1094	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
1095	clear_bit(BTRFS_DEV_STATE_MISSING, addr: &device->dev_state);
1096	fs_devices->missing_devices--;
1097	}
1098
1099	btrfs_close_bdev(device);
1100	if (device->bdev) {
1101	fs_devices->open_devices--;
1102	device->bdev = NULL;
1103	}
1104	clear_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
1105	btrfs_destroy_dev_zone_info(device);
1106
1107	device->fs_info = NULL;
1108	atomic_set(v: &device->dev_stats_ccnt, i: 0);
1109	extent_io_tree_release(tree: &device->alloc_state);
1110
1111	/*
1112	* Reset the flush error record. We might have a transient flush error
1113	* in this mount, and if so we aborted the current transaction and set
1114	* the fs to an error state, guaranteeing no super blocks can be further
1115	* committed. However that error might be transient and if we unmount the
1116	* filesystem and mount it again, we should allow the mount to succeed
1117	* (btrfs_check_rw_degradable() should not fail) - if after mounting the
1118	* filesystem again we still get flush errors, then we will again abort
1119	* any transaction and set the error state, guaranteeing no commits of
1120	* unsafe super blocks.
1121	*/
1122	device->last_flush_error = 0;
1123
1124	/* Verify the device is back in a pristine state */
1125	WARN_ON(test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state));
1126	WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1127	WARN_ON(!list_empty(&device->dev_alloc_list));
1128	WARN_ON(!list_empty(&device->post_commit_list));
1129	}
1130
1131	static void close_fs_devices(struct btrfs_fs_devices *fs_devices)
1132	{
1133	struct btrfs_device *device, *tmp;
1134
1135	lockdep_assert_held(&uuid_mutex);
1136
1137	if (--fs_devices->opened > 0)
1138	return;
1139
1140	list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list)
1141	btrfs_close_one_device(device);
1142
1143	WARN_ON(fs_devices->open_devices);
1144	WARN_ON(fs_devices->rw_devices);
1145	fs_devices->opened = 0;
1146	fs_devices->seeding = false;
1147	fs_devices->fs_info = NULL;
1148	}
1149
1150	void btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
1151	{
1152	LIST_HEAD(list);
1153	struct btrfs_fs_devices *tmp;
1154
1155	mutex_lock(&uuid_mutex);
1156	close_fs_devices(fs_devices);
1157	if (!fs_devices->opened) {
1158	list_splice_init(list: &fs_devices->seed_list, head: &list);
1159
1160	/*
1161	* If the struct btrfs_fs_devices is not assembled with any
1162	* other device, it can be re-initialized during the next mount
1163	* without the needing device-scan step. Therefore, it can be
1164	* fully freed.
1165	*/
1166	if (fs_devices->num_devices == 1) {
1167	list_del(entry: &fs_devices->fs_list);
1168	free_fs_devices(fs_devices);
1169	}
1170	}
1171
1172
1173	list_for_each_entry_safe(fs_devices, tmp, &list, seed_list) {
1174	close_fs_devices(fs_devices);
1175	list_del(entry: &fs_devices->seed_list);
1176	free_fs_devices(fs_devices);
1177	}
1178	mutex_unlock(lock: &uuid_mutex);
1179	}
1180
1181	static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
1182	blk_mode_t flags, void *holder)
1183	{
1184	struct btrfs_device *device;
1185	struct btrfs_device *latest_dev = NULL;
1186	struct btrfs_device *tmp_device;
1187	int ret = 0;
1188
1189	list_for_each_entry_safe(device, tmp_device, &fs_devices->devices,
1190	dev_list) {
1191	int ret2;
1192
1193	ret2 = btrfs_open_one_device(fs_devices, device, flags, holder);
1194	if (ret2 == 0 &&
1195	(!latest_dev || device->generation > latest_dev->generation)) {
1196	latest_dev = device;
1197	} else if (ret2 == -ENODATA) {
1198	fs_devices->num_devices--;
1199	list_del(entry: &device->dev_list);
1200	btrfs_free_device(device);
1201	}
1202	if (ret == 0 && ret2 != 0)
1203	ret = ret2;
1204	}
1205
1206	if (fs_devices->open_devices == 0) {
1207	if (ret)
1208	return ret;
1209	return -EINVAL;
1210	}
1211
1212	fs_devices->opened = 1;
1213	fs_devices->latest_dev = latest_dev;
1214	fs_devices->total_rw_bytes = 0;
1215	fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR;
1216	fs_devices->read_policy = BTRFS_READ_POLICY_PID;
1217
1218	return 0;
1219	}
1220
1221	static int devid_cmp(void *priv, const struct list_head *a,
1222	const struct list_head *b)
1223	{
1224	const struct btrfs_device *dev1, *dev2;
1225
1226	dev1 = list_entry(a, struct btrfs_device, dev_list);
1227	dev2 = list_entry(b, struct btrfs_device, dev_list);
1228
1229	if (dev1->devid < dev2->devid)
1230	return -1;
1231	else if (dev1->devid > dev2->devid)
1232	return 1;
1233	return 0;
1234	}
1235
1236	int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1237	blk_mode_t flags, void *holder)
1238	{
1239	int ret;
1240
1241	lockdep_assert_held(&uuid_mutex);
1242	/*
1243	* The device_list_mutex cannot be taken here in case opening the
1244	* underlying device takes further locks like open_mutex.
1245	*
1246	* We also don't need the lock here as this is called during mount and
1247	* exclusion is provided by uuid_mutex
1248	*/
1249
1250	if (fs_devices->opened) {
1251	fs_devices->opened++;
1252	ret = 0;
1253	} else {
1254	list_sort(NULL, head: &fs_devices->devices, cmp: devid_cmp);
1255	ret = open_fs_devices(fs_devices, flags, holder);
1256	}
1257
1258	return ret;
1259	}
1260
1261	void btrfs_release_disk_super(struct btrfs_super_block *super)
1262	{
1263	struct page *page = virt_to_page(super);
1264
1265	put_page(page);
1266	}
1267
1268	static struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
1269	u64 bytenr, u64 bytenr_orig)
1270	{
1271	struct btrfs_super_block *disk_super;
1272	struct page *page;
1273	void *p;
1274	pgoff_t index;
1275
1276	/* make sure our super fits in the device */
1277	if (bytenr + PAGE_SIZE >= bdev_nr_bytes(bdev))
1278	return ERR_PTR(error: -EINVAL);
1279
1280	/* make sure our super fits in the page */
1281	if (sizeof(*disk_super) > PAGE_SIZE)
1282	return ERR_PTR(error: -EINVAL);
1283
1284	/* make sure our super doesn't straddle pages on disk */
1285	index = bytenr >> PAGE_SHIFT;
1286	if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_SHIFT != index)
1287	return ERR_PTR(error: -EINVAL);
1288
1289	/* pull in the page with our super */
1290	page = read_cache_page_gfp(mapping: bdev->bd_inode->i_mapping, index, GFP_KERNEL);
1291
1292	if (IS_ERR(ptr: page))
1293	return ERR_CAST(ptr: page);
1294
1295	p = page_address(page);
1296
1297	/* align our pointer to the offset of the super block */
1298	disk_super = p + offset_in_page(bytenr);
1299
1300	if (btrfs_super_bytenr(s: disk_super) != bytenr_orig ||
1301	btrfs_super_magic(s: disk_super) != BTRFS_MAGIC) {
1302	btrfs_release_disk_super(super: p);
1303	return ERR_PTR(error: -EINVAL);
1304	}
1305
1306	if (disk_super->label[0] && disk_super->label[BTRFS_LABEL_SIZE - 1])
1307	disk_super->label[BTRFS_LABEL_SIZE - 1] = 0;
1308
1309	return disk_super;
1310	}
1311
1312	int btrfs_forget_devices(dev_t devt)
1313	{
1314	int ret;
1315
1316	mutex_lock(&uuid_mutex);
1317	ret = btrfs_free_stale_devices(devt, NULL);
1318	mutex_unlock(lock: &uuid_mutex);
1319
1320	return ret;
1321	}
1322
1323	static bool btrfs_skip_registration(struct btrfs_super_block *disk_super,
1324	const char *path, dev_t devt,
1325	bool mount_arg_dev)
1326	{
1327	struct btrfs_fs_devices *fs_devices;
1328
1329	/*
1330	* Do not skip device registration for mounted devices with matching
1331	* maj:min but different paths. Booting without initrd relies on
1332	* /dev/root initially, later replaced with the actual root device.
1333	* A successful scan ensures grub2-probe selects the correct device.
1334	*/
1335	list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
1336	struct btrfs_device *device;
1337
1338	mutex_lock(&fs_devices->device_list_mutex);
1339
1340	if (!fs_devices->opened) {
1341	mutex_unlock(lock: &fs_devices->device_list_mutex);
1342	continue;
1343	}
1344
1345	list_for_each_entry(device, &fs_devices->devices, dev_list) {
1346	if (device->bdev && (device->bdev->bd_dev == devt) &&
1347	strcmp(device->name->str, path) != 0) {
1348	mutex_unlock(lock: &fs_devices->device_list_mutex);
1349
1350	/* Do not skip registration. */
1351	return false;
1352	}
1353	}
1354	mutex_unlock(lock: &fs_devices->device_list_mutex);
1355	}
1356
1357	if (!mount_arg_dev && btrfs_super_num_devices(s: disk_super) == 1 &&
1358	!(btrfs_super_flags(s: disk_super) & BTRFS_SUPER_FLAG_SEEDING))
1359	return true;
1360
1361	return false;
1362	}
1363
1364	/*
1365	* Look for a btrfs signature on a device. This may be called out of the mount path
1366	* and we are not allowed to call set_blocksize during the scan. The superblock
1367	* is read via pagecache.
1368	*
1369	* With @mount_arg_dev it's a scan during mount time that will always register
1370	* the device or return an error. Multi-device and seeding devices are registered
1371	* in both cases.
1372	*/
1373	struct btrfs_device *btrfs_scan_one_device(const char *path, blk_mode_t flags,
1374	bool mount_arg_dev)
1375	{
1376	struct btrfs_super_block *disk_super;
1377	bool new_device_added = false;
1378	struct btrfs_device *device = NULL;
1379	struct file *bdev_file;
1380	u64 bytenr, bytenr_orig;
1381	dev_t devt;
1382	int ret;
1383
1384	lockdep_assert_held(&uuid_mutex);
1385
1386	/*
1387	* we would like to check all the supers, but that would make
1388	* a btrfs mount succeed after a mkfs from a different FS.
1389	* So, we need to add a special mount option to scan for
1390	* later supers, using BTRFS_SUPER_MIRROR_MAX instead
1391	*/
1392
1393	/*
1394	* Avoid an exclusive open here, as the systemd-udev may initiate the
1395	* device scan which may race with the user's mount or mkfs command,
1396	* resulting in failure.
1397	* Since the device scan is solely for reading purposes, there is no
1398	* need for an exclusive open. Additionally, the devices are read again
1399	* during the mount process. It is ok to get some inconsistent
1400	* values temporarily, as the device paths of the fsid are the only
1401	* required information for assembling the volume.
1402	*/
1403	bdev_file = bdev_file_open_by_path(path, mode: flags, NULL, NULL);
1404	if (IS_ERR(ptr: bdev_file))
1405	return ERR_CAST(ptr: bdev_file);
1406
1407	bytenr_orig = btrfs_sb_offset(mirror: 0);
1408	ret = btrfs_sb_log_location_bdev(bdev: file_bdev(bdev_file), mirror: 0, READ, bytenr_ret: &bytenr);
1409	if (ret) {
1410	device = ERR_PTR(error: ret);
1411	goto error_bdev_put;
1412	}
1413
1414	disk_super = btrfs_read_disk_super(bdev: file_bdev(bdev_file), bytenr,
1415	bytenr_orig);
1416	if (IS_ERR(ptr: disk_super)) {
1417	device = ERR_CAST(ptr: disk_super);
1418	goto error_bdev_put;
1419	}
1420
1421	devt = file_bdev(bdev_file)->bd_dev;
1422	if (btrfs_skip_registration(disk_super, path, devt, mount_arg_dev)) {
1423	pr_debug("BTRFS: skip registering single non-seed device %s (%d:%d)\n",
1424	path, MAJOR(devt), MINOR(devt));
1425
1426	btrfs_free_stale_devices(devt, NULL);
1427
1428	device = NULL;
1429	goto free_disk_super;
1430	}
1431
1432	device = device_list_add(path, disk_super, new_device_added: &new_device_added);
1433	if (!IS_ERR(ptr: device) && new_device_added)
1434	btrfs_free_stale_devices(devt: device->devt, skip_device: device);
1435
1436	free_disk_super:
1437	btrfs_release_disk_super(super: disk_super);
1438
1439	error_bdev_put:
1440	fput(bdev_file);
1441
1442	return device;
1443	}
1444
1445	/*
1446	* Try to find a chunk that intersects [start, start + len] range and when one
1447	* such is found, record the end of it in *start
1448	*/
1449	static bool contains_pending_extent(struct btrfs_device *device, u64 *start,
1450	u64 len)
1451	{
1452	u64 physical_start, physical_end;
1453
1454	lockdep_assert_held(&device->fs_info->chunk_mutex);
1455
1456	if (find_first_extent_bit(tree: &device->alloc_state, start: *start,
1457	start_ret: &physical_start, end_ret: &physical_end,
1458	CHUNK_ALLOCATED, NULL)) {
1459
1460	if (in_range(physical_start, *start, len) ||
1461	in_range(*start, physical_start,
1462	physical_end + 1 - physical_start)) {
1463	*start = physical_end + 1;
1464	return true;
1465	}
1466	}
1467	return false;
1468	}
1469
1470	static u64 dev_extent_search_start(struct btrfs_device *device)
1471	{
1472	switch (device->fs_devices->chunk_alloc_policy) {
1473	case BTRFS_CHUNK_ALLOC_REGULAR:
1474	return BTRFS_DEVICE_RANGE_RESERVED;
1475	case BTRFS_CHUNK_ALLOC_ZONED:
1476	/*
1477	* We don't care about the starting region like regular
1478	* allocator, because we anyway use/reserve the first two zones
1479	* for superblock logging.
1480	*/
1481	return 0;
1482	default:
1483	BUG();
1484	}
1485	}
1486
1487	static bool dev_extent_hole_check_zoned(struct btrfs_device *device,
1488	u64 *hole_start, u64 *hole_size,
1489	u64 num_bytes)
1490	{
1491	u64 zone_size = device->zone_info->zone_size;
1492	u64 pos;
1493	int ret;
1494	bool changed = false;
1495
1496	ASSERT(IS_ALIGNED(*hole_start, zone_size));
1497
1498	while (*hole_size > 0) {
1499	pos = btrfs_find_allocatable_zones(device, hole_start: *hole_start,
1500	hole_end: *hole_start + *hole_size,
1501	num_bytes);
1502	if (pos != *hole_start) {
1503	*hole_size = *hole_start + *hole_size - pos;
1504	*hole_start = pos;
1505	changed = true;
1506	if (*hole_size < num_bytes)
1507	break;
1508	}
1509
1510	ret = btrfs_ensure_empty_zones(device, start: pos, size: num_bytes);
1511
1512	/* Range is ensured to be empty */
1513	if (!ret)
1514	return changed;
1515
1516	/* Given hole range was invalid (outside of device) */
1517	if (ret == -ERANGE) {
1518	*hole_start += *hole_size;
1519	*hole_size = 0;
1520	return true;
1521	}
1522
1523	*hole_start += zone_size;
1524	*hole_size -= zone_size;
1525	changed = true;
1526	}
1527
1528	return changed;
1529	}
1530
1531	/*
1532	* Check if specified hole is suitable for allocation.
1533	*
1534	* @device: the device which we have the hole
1535	* @hole_start: starting position of the hole
1536	* @hole_size: the size of the hole
1537	* @num_bytes: the size of the free space that we need
1538	*
1539	* This function may modify @hole_start and @hole_size to reflect the suitable
1540	* position for allocation. Returns 1 if hole position is updated, 0 otherwise.
1541	*/
1542	static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start,
1543	u64 *hole_size, u64 num_bytes)
1544	{
1545	bool changed = false;
1546	u64 hole_end = *hole_start + *hole_size;
1547
1548	for (;;) {
1549	/*
1550	* Check before we set max_hole_start, otherwise we could end up
1551	* sending back this offset anyway.
1552	*/
1553	if (contains_pending_extent(device, start: hole_start, len: *hole_size)) {
1554	if (hole_end >= *hole_start)
1555	*hole_size = hole_end - *hole_start;
1556	else
1557	*hole_size = 0;
1558	changed = true;
1559	}
1560
1561	switch (device->fs_devices->chunk_alloc_policy) {
1562	case BTRFS_CHUNK_ALLOC_REGULAR:
1563	/* No extra check */
1564	break;
1565	case BTRFS_CHUNK_ALLOC_ZONED:
1566	if (dev_extent_hole_check_zoned(device, hole_start,
1567	hole_size, num_bytes)) {
1568	changed = true;
1569	/*
1570	* The changed hole can contain pending extent.
1571	* Loop again to check that.
1572	*/
1573	continue;
1574	}
1575	break;
1576	default:
1577	BUG();
1578	}
1579
1580	break;
1581	}
1582
1583	return changed;
1584	}
1585
1586	/*
1587	* Find free space in the specified device.
1588	*
1589	* @device: the device which we search the free space in
1590	* @num_bytes: the size of the free space that we need
1591	* @search_start: the position from which to begin the search
1592	* @start: store the start of the free space.
1593	* @len: the size of the free space. that we find, or the size
1594	* of the max free space if we don't find suitable free space
1595	*
1596	* This does a pretty simple search, the expectation is that it is called very
1597	* infrequently and that a given device has a small number of extents.
1598	*
1599	* @start is used to store the start of the free space if we find. But if we
1600	* don't find suitable free space, it will be used to store the start position
1601	* of the max free space.
1602	*
1603	* @len is used to store the size of the free space that we find.
1604	* But if we don't find suitable free space, it is used to store the size of
1605	* the max free space.
1606	*
1607	* NOTE: This function will search *commit* root of device tree, and does extra
1608	* check to ensure dev extents are not double allocated.
1609	* This makes the function safe to allocate dev extents but may not report
1610	* correct usable device space, as device extent freed in current transaction
1611	* is not reported as available.
1612	*/
1613	static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes,
1614	u64 *start, u64 *len)
1615	{
1616	struct btrfs_fs_info *fs_info = device->fs_info;
1617	struct btrfs_root *root = fs_info->dev_root;
1618	struct btrfs_key key;
1619	struct btrfs_dev_extent *dev_extent;
1620	struct btrfs_path *path;
1621	u64 search_start;
1622	u64 hole_size;
1623	u64 max_hole_start;
1624	u64 max_hole_size = 0;
1625	u64 extent_end;
1626	u64 search_end = device->total_bytes;
1627	int ret;
1628	int slot;
1629	struct extent_buffer *l;
1630
1631	search_start = dev_extent_search_start(device);
1632	max_hole_start = search_start;
1633
1634	WARN_ON(device->zone_info &&
1635	!IS_ALIGNED(num_bytes, device->zone_info->zone_size));
1636
1637	path = btrfs_alloc_path();
1638	if (!path) {
1639	ret = -ENOMEM;
1640	goto out;
1641	}
1642	again:
1643	if (search_start >= search_end ||
1644	test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
1645	ret = -ENOSPC;
1646	goto out;
1647	}
1648
1649	path->reada = READA_FORWARD;
1650	path->search_commit_root = 1;
1651	path->skip_locking = 1;
1652
1653	key.objectid = device->devid;
1654	key.offset = search_start;
1655	key.type = BTRFS_DEV_EXTENT_KEY;
1656
1657	ret = btrfs_search_backwards(root, key: &key, path);
1658	if (ret < 0)
1659	goto out;
1660
1661	while (search_start < search_end) {
1662	l = path->nodes[0];
1663	slot = path->slots[0];
1664	if (slot >= btrfs_header_nritems(eb: l)) {
1665	ret = btrfs_next_leaf(root, path);
1666	if (ret == 0)
1667	continue;
1668	if (ret < 0)
1669	goto out;
1670
1671	break;
1672	}
1673	btrfs_item_key_to_cpu(eb: l, cpu_key: &key, nr: slot);
1674
1675	if (key.objectid < device->devid)
1676	goto next;
1677
1678	if (key.objectid > device->devid)
1679	break;
1680
1681	if (key.type != BTRFS_DEV_EXTENT_KEY)
1682	goto next;
1683
1684	if (key.offset > search_end)
1685	break;
1686
1687	if (key.offset > search_start) {
1688	hole_size = key.offset - search_start;
1689	dev_extent_hole_check(device, hole_start: &search_start, hole_size: &hole_size,
1690	num_bytes);
1691
1692	if (hole_size > max_hole_size) {
1693	max_hole_start = search_start;
1694	max_hole_size = hole_size;
1695	}
1696
1697	/*
1698	* If this free space is greater than which we need,
1699	* it must be the max free space that we have found
1700	* until now, so max_hole_start must point to the start
1701	* of this free space and the length of this free space
1702	* is stored in max_hole_size. Thus, we return
1703	* max_hole_start and max_hole_size and go back to the
1704	* caller.
1705	*/
1706	if (hole_size >= num_bytes) {
1707	ret = 0;
1708	goto out;
1709	}
1710	}
1711
1712	dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1713	extent_end = key.offset + btrfs_dev_extent_length(eb: l,
1714	s: dev_extent);
1715	if (extent_end > search_start)
1716	search_start = extent_end;
1717	next:
1718	path->slots[0]++;
1719	cond_resched();
1720	}
1721
1722	/*
1723	* At this point, search_start should be the end of
1724	* allocated dev extents, and when shrinking the device,
1725	* search_end may be smaller than search_start.
1726	*/
1727	if (search_end > search_start) {
1728	hole_size = search_end - search_start;
1729	if (dev_extent_hole_check(device, hole_start: &search_start, hole_size: &hole_size,
1730	num_bytes)) {
1731	btrfs_release_path(p: path);
1732	goto again;
1733	}
1734
1735	if (hole_size > max_hole_size) {
1736	max_hole_start = search_start;
1737	max_hole_size = hole_size;
1738	}
1739	}
1740
1741	/* See above. */
1742	if (max_hole_size < num_bytes)
1743	ret = -ENOSPC;
1744	else
1745	ret = 0;
1746
1747	ASSERT(max_hole_start + max_hole_size <= search_end);
1748	out:
1749	btrfs_free_path(p: path);
1750	*start = max_hole_start;
1751	if (len)
1752	*len = max_hole_size;
1753	return ret;
1754	}
1755
1756	static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1757	struct btrfs_device *device,
1758	u64 start, u64 *dev_extent_len)
1759	{
1760	struct btrfs_fs_info *fs_info = device->fs_info;
1761	struct btrfs_root *root = fs_info->dev_root;
1762	int ret;
1763	struct btrfs_path *path;
1764	struct btrfs_key key;
1765	struct btrfs_key found_key;
1766	struct extent_buffer *leaf = NULL;
1767	struct btrfs_dev_extent *extent = NULL;
1768
1769	path = btrfs_alloc_path();
1770	if (!path)
1771	return -ENOMEM;
1772
1773	key.objectid = device->devid;
1774	key.offset = start;
1775	key.type = BTRFS_DEV_EXTENT_KEY;
1776	again:
1777	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: -1, cow: 1);
1778	if (ret > 0) {
1779	ret = btrfs_previous_item(root, path, min_objectid: key.objectid,
1780	BTRFS_DEV_EXTENT_KEY);
1781	if (ret)
1782	goto out;
1783	leaf = path->nodes[0];
1784	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
1785	extent = btrfs_item_ptr(leaf, path->slots[0],
1786	struct btrfs_dev_extent);
1787	BUG_ON(found_key.offset > start || found_key.offset +
1788	btrfs_dev_extent_length(leaf, extent) < start);
1789	key = found_key;
1790	btrfs_release_path(p: path);
1791	goto again;
1792	} else if (ret == 0) {
1793	leaf = path->nodes[0];
1794	extent = btrfs_item_ptr(leaf, path->slots[0],
1795	struct btrfs_dev_extent);
1796	} else {
1797	goto out;
1798	}
1799
1800	*dev_extent_len = btrfs_dev_extent_length(eb: leaf, s: extent);
1801
1802	ret = btrfs_del_item(trans, root, path);
1803	if (ret == 0)
1804	set_bit(BTRFS_TRANS_HAVE_FREE_BGS, addr: &trans->transaction->flags);
1805	out:
1806	btrfs_free_path(p: path);
1807	return ret;
1808	}
1809
1810	static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1811	{
1812	struct rb_node *n;
1813	u64 ret = 0;
1814
1815	read_lock(&fs_info->mapping_tree_lock);
1816	n = rb_last(&fs_info->mapping_tree.rb_root);
1817	if (n) {
1818	struct btrfs_chunk_map *map;
1819
1820	map = rb_entry(n, struct btrfs_chunk_map, rb_node);
1821	ret = map->start + map->chunk_len;
1822	}
1823	read_unlock(&fs_info->mapping_tree_lock);
1824
1825	return ret;
1826	}
1827
1828	static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1829	u64 *devid_ret)
1830	{
1831	int ret;
1832	struct btrfs_key key;
1833	struct btrfs_key found_key;
1834	struct btrfs_path *path;
1835
1836	path = btrfs_alloc_path();
1837	if (!path)
1838	return -ENOMEM;
1839
1840	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1841	key.type = BTRFS_DEV_ITEM_KEY;
1842	key.offset = (u64)-1;
1843
1844	ret = btrfs_search_slot(NULL, root: fs_info->chunk_root, key: &key, p: path, ins_len: 0, cow: 0);
1845	if (ret < 0)
1846	goto error;
1847
1848	if (ret == 0) {
1849	/* Corruption */
1850	btrfs_err(fs_info, "corrupted chunk tree devid -1 matched");
1851	ret = -EUCLEAN;
1852	goto error;
1853	}
1854
1855	ret = btrfs_previous_item(root: fs_info->chunk_root, path,
1856	BTRFS_DEV_ITEMS_OBJECTID,
1857	BTRFS_DEV_ITEM_KEY);
1858	if (ret) {
1859	*devid_ret = 1;
1860	} else {
1861	btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &found_key,
1862	nr: path->slots[0]);
1863	*devid_ret = found_key.offset + 1;
1864	}
1865	ret = 0;
1866	error:
1867	btrfs_free_path(p: path);
1868	return ret;
1869	}
1870
1871	/*
1872	* the device information is stored in the chunk root
1873	* the btrfs_device struct should be fully filled in
1874	*/
1875	static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
1876	struct btrfs_device *device)
1877	{
1878	int ret;
1879	struct btrfs_path *path;
1880	struct btrfs_dev_item *dev_item;
1881	struct extent_buffer *leaf;
1882	struct btrfs_key key;
1883	unsigned long ptr;
1884
1885	path = btrfs_alloc_path();
1886	if (!path)
1887	return -ENOMEM;
1888
1889	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1890	key.type = BTRFS_DEV_ITEM_KEY;
1891	key.offset = device->devid;
1892
1893	btrfs_reserve_chunk_metadata(trans, is_item_insertion: true);
1894	ret = btrfs_insert_empty_item(trans, root: trans->fs_info->chunk_root, path,
1895	key: &key, data_size: sizeof(*dev_item));
1896	btrfs_trans_release_chunk_metadata(trans);
1897	if (ret)
1898	goto out;
1899
1900	leaf = path->nodes[0];
1901	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1902
1903	btrfs_set_device_id(eb: leaf, s: dev_item, val: device->devid);
1904	btrfs_set_device_generation(eb: leaf, s: dev_item, val: 0);
1905	btrfs_set_device_type(eb: leaf, s: dev_item, val: device->type);
1906	btrfs_set_device_io_align(eb: leaf, s: dev_item, val: device->io_align);
1907	btrfs_set_device_io_width(eb: leaf, s: dev_item, val: device->io_width);
1908	btrfs_set_device_sector_size(eb: leaf, s: dev_item, val: device->sector_size);
1909	btrfs_set_device_total_bytes(eb: leaf, s: dev_item,
1910	val: btrfs_device_get_disk_total_bytes(dev: device));
1911	btrfs_set_device_bytes_used(eb: leaf, s: dev_item,
1912	val: btrfs_device_get_bytes_used(dev: device));
1913	btrfs_set_device_group(eb: leaf, s: dev_item, val: 0);
1914	btrfs_set_device_seek_speed(eb: leaf, s: dev_item, val: 0);
1915	btrfs_set_device_bandwidth(eb: leaf, s: dev_item, val: 0);
1916	btrfs_set_device_start_offset(eb: leaf, s: dev_item, val: 0);
1917
1918	ptr = btrfs_device_uuid(d: dev_item);
1919	write_extent_buffer(eb: leaf, src: device->uuid, start: ptr, BTRFS_UUID_SIZE);
1920	ptr = btrfs_device_fsid(d: dev_item);
1921	write_extent_buffer(eb: leaf, src: trans->fs_info->fs_devices->metadata_uuid,
1922	start: ptr, BTRFS_FSID_SIZE);
1923	btrfs_mark_buffer_dirty(trans, buf: leaf);
1924
1925	ret = 0;
1926	out:
1927	btrfs_free_path(p: path);
1928	return ret;
1929	}
1930
1931	/*
1932	* Function to update ctime/mtime for a given device path.
1933	* Mainly used for ctime/mtime based probe like libblkid.
1934	*
1935	* We don't care about errors here, this is just to be kind to userspace.
1936	*/
1937	static void update_dev_time(const char *device_path)
1938	{
1939	struct path path;
1940	int ret;
1941
1942	ret = kern_path(device_path, LOOKUP_FOLLOW, &path);
1943	if (ret)
1944	return;
1945
1946	inode_update_time(inode: d_inode(dentry: path.dentry), flags: S_MTIME | S_CTIME | S_VERSION);
1947	path_put(&path);
1948	}
1949
1950	static int btrfs_rm_dev_item(struct btrfs_trans_handle *trans,
1951	struct btrfs_device *device)
1952	{
1953	struct btrfs_root *root = device->fs_info->chunk_root;
1954	int ret;
1955	struct btrfs_path *path;
1956	struct btrfs_key key;
1957
1958	path = btrfs_alloc_path();
1959	if (!path)
1960	return -ENOMEM;
1961
1962	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1963	key.type = BTRFS_DEV_ITEM_KEY;
1964	key.offset = device->devid;
1965
1966	btrfs_reserve_chunk_metadata(trans, is_item_insertion: false);
1967	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: -1, cow: 1);
1968	btrfs_trans_release_chunk_metadata(trans);
1969	if (ret) {
1970	if (ret > 0)
1971	ret = -ENOENT;
1972	goto out;
1973	}
1974
1975	ret = btrfs_del_item(trans, root, path);
1976	out:
1977	btrfs_free_path(p: path);
1978	return ret;
1979	}
1980
1981	/*
1982	* Verify that @num_devices satisfies the RAID profile constraints in the whole
1983	* filesystem. It's up to the caller to adjust that number regarding eg. device
1984	* replace.
1985	*/
1986	static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
1987	u64 num_devices)
1988	{
1989	u64 all_avail;
1990	unsigned seq;
1991	int i;
1992
1993	do {
1994	seq = read_seqbegin(sl: &fs_info->profiles_lock);
1995
1996	all_avail = fs_info->avail_data_alloc_bits |
1997	fs_info->avail_system_alloc_bits |
1998	fs_info->avail_metadata_alloc_bits;
1999	} while (read_seqretry(sl: &fs_info->profiles_lock, start: seq));
2000
2001	for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
2002	if (!(all_avail & btrfs_raid_array[i].bg_flag))
2003	continue;
2004
2005	if (num_devices < btrfs_raid_array[i].devs_min)
2006	return btrfs_raid_array[i].mindev_error;
2007	}
2008
2009	return 0;
2010	}
2011
2012	static struct btrfs_device * btrfs_find_next_active_device(
2013	struct btrfs_fs_devices *fs_devs, struct btrfs_device *device)
2014	{
2015	struct btrfs_device *next_device;
2016
2017	list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
2018	if (next_device != device &&
2019	!test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state)
2020	&& next_device->bdev)
2021	return next_device;
2022	}
2023
2024	return NULL;
2025	}
2026
2027	/*
2028	* Helper function to check if the given device is part of s_bdev / latest_dev
2029	* and replace it with the provided or the next active device, in the context
2030	* where this function called, there should be always be another device (or
2031	* this_dev) which is active.
2032	*/
2033	void __cold btrfs_assign_next_active_device(struct btrfs_device *device,
2034	struct btrfs_device *next_device)
2035	{
2036	struct btrfs_fs_info *fs_info = device->fs_info;
2037
2038	if (!next_device)
2039	next_device = btrfs_find_next_active_device(fs_devs: fs_info->fs_devices,
2040	device);
2041	ASSERT(next_device);
2042
2043	if (fs_info->sb->s_bdev &&
2044	(fs_info->sb->s_bdev == device->bdev))
2045	fs_info->sb->s_bdev = next_device->bdev;
2046
2047	if (fs_info->fs_devices->latest_dev->bdev == device->bdev)
2048	fs_info->fs_devices->latest_dev = next_device;
2049	}
2050
2051	/*
2052	* Return btrfs_fs_devices::num_devices excluding the device that's being
2053	* currently replaced.
2054	*/
2055	static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info)
2056	{
2057	u64 num_devices = fs_info->fs_devices->num_devices;
2058
2059	down_read(sem: &fs_info->dev_replace.rwsem);
2060	if (btrfs_dev_replace_is_ongoing(dev_replace: &fs_info->dev_replace)) {
2061	ASSERT(num_devices > 1);
2062	num_devices--;
2063	}
2064	up_read(sem: &fs_info->dev_replace.rwsem);
2065
2066	return num_devices;
2067	}
2068
2069	static void btrfs_scratch_superblock(struct btrfs_fs_info *fs_info,
2070	struct block_device *bdev, int copy_num)
2071	{
2072	struct btrfs_super_block *disk_super;
2073	const size_t len = sizeof(disk_super->magic);
2074	const u64 bytenr = btrfs_sb_offset(mirror: copy_num);
2075	int ret;
2076
2077	disk_super = btrfs_read_disk_super(bdev, bytenr, bytenr_orig: bytenr);
2078	if (IS_ERR(ptr: disk_super))
2079	return;
2080
2081	memset(&disk_super->magic, 0, len);
2082	folio_mark_dirty(folio: virt_to_folio(x: disk_super));
2083	btrfs_release_disk_super(super: disk_super);
2084
2085	ret = sync_blockdev_range(bdev, lstart: bytenr, lend: bytenr + len - 1);
2086	if (ret)
2087	btrfs_warn(fs_info, "error clearing superblock number %d (%d)",
2088	copy_num, ret);
2089	}
2090
2091	void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info, struct btrfs_device *device)
2092	{
2093	int copy_num;
2094	struct block_device *bdev = device->bdev;
2095
2096	if (!bdev)
2097	return;
2098
2099	for (copy_num = 0; copy_num < BTRFS_SUPER_MIRROR_MAX; copy_num++) {
2100	if (bdev_is_zoned(bdev))
2101	btrfs_reset_sb_log_zones(bdev, mirror: copy_num);
2102	else
2103	btrfs_scratch_superblock(fs_info, bdev, copy_num);
2104	}
2105
2106	/* Notify udev that device has changed */
2107	btrfs_kobject_uevent(bdev, action: KOBJ_CHANGE);
2108
2109	/* Update ctime/mtime for device path for libblkid */
2110	update_dev_time(device_path: device->name->str);
2111	}
2112
2113	int btrfs_rm_device(struct btrfs_fs_info *fs_info,
2114	struct btrfs_dev_lookup_args *args,
2115	struct file **bdev_file)
2116	{
2117	struct btrfs_trans_handle *trans;
2118	struct btrfs_device *device;
2119	struct btrfs_fs_devices *cur_devices;
2120	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2121	u64 num_devices;
2122	int ret = 0;
2123
2124	if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
2125	btrfs_err(fs_info, "device remove not supported on extent tree v2 yet");
2126	return -EINVAL;
2127	}
2128
2129	/*
2130	* The device list in fs_devices is accessed without locks (neither
2131	* uuid_mutex nor device_list_mutex) as it won't change on a mounted
2132	* filesystem and another device rm cannot run.
2133	*/
2134	num_devices = btrfs_num_devices(fs_info);
2135
2136	ret = btrfs_check_raid_min_devices(fs_info, num_devices: num_devices - 1);
2137	if (ret)
2138	return ret;
2139
2140	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args);
2141	if (!device) {
2142	if (args->missing)
2143	ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2144	else
2145	ret = -ENOENT;
2146	return ret;
2147	}
2148
2149	if (btrfs_pinned_by_swapfile(fs_info, ptr: device)) {
2150	btrfs_warn_in_rcu(fs_info,
2151	"cannot remove device %s (devid %llu) due to active swapfile",
2152	btrfs_dev_name(device), device->devid);
2153	return -ETXTBSY;
2154	}
2155
2156	if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
2157	return BTRFS_ERROR_DEV_TGT_REPLACE;
2158
2159	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
2160	fs_info->fs_devices->rw_devices == 1)
2161	return BTRFS_ERROR_DEV_ONLY_WRITABLE;
2162
2163	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2164	mutex_lock(&fs_info->chunk_mutex);
2165	list_del_init(entry: &device->dev_alloc_list);
2166	device->fs_devices->rw_devices--;
2167	mutex_unlock(lock: &fs_info->chunk_mutex);
2168	}
2169
2170	ret = btrfs_shrink_device(device, new_size: 0);
2171	if (ret)
2172	goto error_undo;
2173
2174	trans = btrfs_start_transaction(root: fs_info->chunk_root, num_items: 0);
2175	if (IS_ERR(ptr: trans)) {
2176	ret = PTR_ERR(ptr: trans);
2177	goto error_undo;
2178	}
2179
2180	ret = btrfs_rm_dev_item(trans, device);
2181	if (ret) {
2182	/* Any error in dev item removal is critical */
2183	btrfs_crit(fs_info,
2184	"failed to remove device item for devid %llu: %d",
2185	device->devid, ret);
2186	btrfs_abort_transaction(trans, ret);
2187	btrfs_end_transaction(trans);
2188	return ret;
2189	}
2190
2191	clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, addr: &device->dev_state);
2192	btrfs_scrub_cancel_dev(dev: device);
2193
2194	/*
2195	* the device list mutex makes sure that we don't change
2196	* the device list while someone else is writing out all
2197	* the device supers. Whoever is writing all supers, should
2198	* lock the device list mutex before getting the number of
2199	* devices in the super block (super_copy). Conversely,
2200	* whoever updates the number of devices in the super block
2201	* (super_copy) should hold the device list mutex.
2202	*/
2203
2204	/*
2205	* In normal cases the cur_devices == fs_devices. But in case
2206	* of deleting a seed device, the cur_devices should point to
2207	* its own fs_devices listed under the fs_devices->seed_list.
2208	*/
2209	cur_devices = device->fs_devices;
2210	mutex_lock(&fs_devices->device_list_mutex);
2211	list_del_rcu(entry: &device->dev_list);
2212
2213	cur_devices->num_devices--;
2214	cur_devices->total_devices--;
2215	/* Update total_devices of the parent fs_devices if it's seed */
2216	if (cur_devices != fs_devices)
2217	fs_devices->total_devices--;
2218
2219	if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
2220	cur_devices->missing_devices--;
2221
2222	btrfs_assign_next_active_device(device, NULL);
2223
2224	if (device->bdev_file) {
2225	cur_devices->open_devices--;
2226	/* remove sysfs entry */
2227	btrfs_sysfs_remove_device(device);
2228	}
2229
2230	num_devices = btrfs_super_num_devices(s: fs_info->super_copy) - 1;
2231	btrfs_set_super_num_devices(s: fs_info->super_copy, val: num_devices);
2232	mutex_unlock(lock: &fs_devices->device_list_mutex);
2233
2234	/*
2235	* At this point, the device is zero sized and detached from the
2236	* devices list. All that's left is to zero out the old supers and
2237	* free the device.
2238	*
2239	* We cannot call btrfs_close_bdev() here because we're holding the sb
2240	* write lock, and fput() on the block device will pull in the
2241	* ->open_mutex on the block device and it's dependencies. Instead
2242	* just flush the device and let the caller do the final bdev_release.
2243	*/
2244	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2245	btrfs_scratch_superblocks(fs_info, device);
2246	if (device->bdev) {
2247	sync_blockdev(bdev: device->bdev);
2248	invalidate_bdev(bdev: device->bdev);
2249	}
2250	}
2251
2252	*bdev_file = device->bdev_file;
2253	synchronize_rcu();
2254	btrfs_free_device(device);
2255
2256	/*
2257	* This can happen if cur_devices is the private seed devices list. We
2258	* cannot call close_fs_devices() here because it expects the uuid_mutex
2259	* to be held, but in fact we don't need that for the private
2260	* seed_devices, we can simply decrement cur_devices->opened and then
2261	* remove it from our list and free the fs_devices.
2262	*/
2263	if (cur_devices->num_devices == 0) {
2264	list_del_init(entry: &cur_devices->seed_list);
2265	ASSERT(cur_devices->opened == 1);
2266	cur_devices->opened--;
2267	free_fs_devices(fs_devices: cur_devices);
2268	}
2269
2270	ret = btrfs_commit_transaction(trans);
2271
2272	return ret;
2273
2274	error_undo:
2275	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2276	mutex_lock(&fs_info->chunk_mutex);
2277	list_add(new: &device->dev_alloc_list,
2278	head: &fs_devices->alloc_list);
2279	device->fs_devices->rw_devices++;
2280	mutex_unlock(lock: &fs_info->chunk_mutex);
2281	}
2282	return ret;
2283	}
2284
2285	void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev)
2286	{
2287	struct btrfs_fs_devices *fs_devices;
2288
2289	lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex);
2290
2291	/*
2292	* in case of fs with no seed, srcdev->fs_devices will point
2293	* to fs_devices of fs_info. However when the dev being replaced is
2294	* a seed dev it will point to the seed's local fs_devices. In short
2295	* srcdev will have its correct fs_devices in both the cases.
2296	*/
2297	fs_devices = srcdev->fs_devices;
2298
2299	list_del_rcu(entry: &srcdev->dev_list);
2300	list_del(entry: &srcdev->dev_alloc_list);
2301	fs_devices->num_devices--;
2302	if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state))
2303	fs_devices->missing_devices--;
2304
2305	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state))
2306	fs_devices->rw_devices--;
2307
2308	if (srcdev->bdev)
2309	fs_devices->open_devices--;
2310	}
2311
2312	void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
2313	{
2314	struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2315
2316	mutex_lock(&uuid_mutex);
2317
2318	btrfs_close_bdev(device: srcdev);
2319	synchronize_rcu();
2320	btrfs_free_device(device: srcdev);
2321
2322	/* if this is no devs we rather delete the fs_devices */
2323	if (!fs_devices->num_devices) {
2324	/*
2325	* On a mounted FS, num_devices can't be zero unless it's a
2326	* seed. In case of a seed device being replaced, the replace
2327	* target added to the sprout FS, so there will be no more
2328	* device left under the seed FS.
2329	*/
2330	ASSERT(fs_devices->seeding);
2331
2332	list_del_init(entry: &fs_devices->seed_list);
2333	close_fs_devices(fs_devices);
2334	free_fs_devices(fs_devices);
2335	}
2336	mutex_unlock(lock: &uuid_mutex);
2337	}
2338
2339	void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
2340	{
2341	struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
2342
2343	mutex_lock(&fs_devices->device_list_mutex);
2344
2345	btrfs_sysfs_remove_device(device: tgtdev);
2346
2347	if (tgtdev->bdev)
2348	fs_devices->open_devices--;
2349
2350	fs_devices->num_devices--;
2351
2352	btrfs_assign_next_active_device(device: tgtdev, NULL);
2353
2354	list_del_rcu(entry: &tgtdev->dev_list);
2355
2356	mutex_unlock(lock: &fs_devices->device_list_mutex);
2357
2358	btrfs_scratch_superblocks(fs_info: tgtdev->fs_info, device: tgtdev);
2359
2360	btrfs_close_bdev(device: tgtdev);
2361	synchronize_rcu();
2362	btrfs_free_device(device: tgtdev);
2363	}
2364
2365	/*
2366	* Populate args from device at path.
2367	*
2368	* @fs_info: the filesystem
2369	* @args: the args to populate
2370	* @path: the path to the device
2371	*
2372	* This will read the super block of the device at @path and populate @args with
2373	* the devid, fsid, and uuid. This is meant to be used for ioctls that need to
2374	* lookup a device to operate on, but need to do it before we take any locks.
2375	* This properly handles the special case of "missing" that a user may pass in,
2376	* and does some basic sanity checks. The caller must make sure that @path is
2377	* properly NUL terminated before calling in, and must call
2378	* btrfs_put_dev_args_from_path() in order to free up the temporary fsid and
2379	* uuid buffers.
2380	*
2381	* Return: 0 for success, -errno for failure
2382	*/
2383	int btrfs_get_dev_args_from_path(struct btrfs_fs_info *fs_info,
2384	struct btrfs_dev_lookup_args *args,
2385	const char *path)
2386	{
2387	struct btrfs_super_block *disk_super;
2388	struct file *bdev_file;
2389	int ret;
2390
2391	if (!path || !path[0])
2392	return -EINVAL;
2393	if (!strcmp(path, "missing")) {
2394	args->missing = true;
2395	return 0;
2396	}
2397
2398	args->uuid = kzalloc(BTRFS_UUID_SIZE, GFP_KERNEL);
2399	args->fsid = kzalloc(BTRFS_FSID_SIZE, GFP_KERNEL);
2400	if (!args->uuid || !args->fsid) {
2401	btrfs_put_dev_args_from_path(args);
2402	return -ENOMEM;
2403	}
2404
2405	ret = btrfs_get_bdev_and_sb(device_path: path, BLK_OPEN_READ, NULL, flush: 0,
2406	bdev_file: &bdev_file, disk_super: &disk_super);
2407	if (ret) {
2408	btrfs_put_dev_args_from_path(args);
2409	return ret;
2410	}
2411
2412	args->devid = btrfs_stack_device_id(s: &disk_super->dev_item);
2413	memcpy(args->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE);
2414	if (btrfs_fs_incompat(fs_info, METADATA_UUID))
2415	memcpy(args->fsid, disk_super->metadata_uuid, BTRFS_FSID_SIZE);
2416	else
2417	memcpy(args->fsid, disk_super->fsid, BTRFS_FSID_SIZE);
2418	btrfs_release_disk_super(super: disk_super);
2419	fput(bdev_file);
2420	return 0;
2421	}
2422
2423	/*
2424	* Only use this jointly with btrfs_get_dev_args_from_path() because we will
2425	* allocate our ->uuid and ->fsid pointers, everybody else uses local variables
2426	* that don't need to be freed.
2427	*/
2428	void btrfs_put_dev_args_from_path(struct btrfs_dev_lookup_args *args)
2429	{
2430	kfree(objp: args->uuid);
2431	kfree(objp: args->fsid);
2432	args->uuid = NULL;
2433	args->fsid = NULL;
2434	}
2435
2436	struct btrfs_device *btrfs_find_device_by_devspec(
2437	struct btrfs_fs_info *fs_info, u64 devid,
2438	const char *device_path)
2439	{
2440	BTRFS_DEV_LOOKUP_ARGS(args);
2441	struct btrfs_device *device;
2442	int ret;
2443
2444	if (devid) {
2445	args.devid = devid;
2446	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
2447	if (!device)
2448	return ERR_PTR(error: -ENOENT);
2449	return device;
2450	}
2451
2452	ret = btrfs_get_dev_args_from_path(fs_info, args: &args, path: device_path);
2453	if (ret)
2454	return ERR_PTR(error: ret);
2455	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
2456	btrfs_put_dev_args_from_path(args: &args);
2457	if (!device)
2458	return ERR_PTR(error: -ENOENT);
2459	return device;
2460	}
2461
2462	static struct btrfs_fs_devices *btrfs_init_sprout(struct btrfs_fs_info *fs_info)
2463	{
2464	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2465	struct btrfs_fs_devices *old_devices;
2466	struct btrfs_fs_devices *seed_devices;
2467
2468	lockdep_assert_held(&uuid_mutex);
2469	if (!fs_devices->seeding)
2470	return ERR_PTR(error: -EINVAL);
2471
2472	/*
2473	* Private copy of the seed devices, anchored at
2474	* fs_info->fs_devices->seed_list
2475	*/
2476	seed_devices = alloc_fs_devices(NULL);
2477	if (IS_ERR(ptr: seed_devices))
2478	return seed_devices;
2479
2480	/*
2481	* It's necessary to retain a copy of the original seed fs_devices in
2482	* fs_uuids so that filesystems which have been seeded can successfully
2483	* reference the seed device from open_seed_devices. This also supports
2484	* multiple fs seed.
2485	*/
2486	old_devices = clone_fs_devices(orig: fs_devices);
2487	if (IS_ERR(ptr: old_devices)) {
2488	kfree(objp: seed_devices);
2489	return old_devices;
2490	}
2491
2492	list_add(new: &old_devices->fs_list, head: &fs_uuids);
2493
2494	memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
2495	seed_devices->opened = 1;
2496	INIT_LIST_HEAD(list: &seed_devices->devices);
2497	INIT_LIST_HEAD(list: &seed_devices->alloc_list);
2498	mutex_init(&seed_devices->device_list_mutex);
2499
2500	return seed_devices;
2501	}
2502
2503	/*
2504	* Splice seed devices into the sprout fs_devices.
2505	* Generate a new fsid for the sprouted read-write filesystem.
2506	*/
2507	static void btrfs_setup_sprout(struct btrfs_fs_info *fs_info,
2508	struct btrfs_fs_devices *seed_devices)
2509	{
2510	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2511	struct btrfs_super_block *disk_super = fs_info->super_copy;
2512	struct btrfs_device *device;
2513	u64 super_flags;
2514
2515	/*
2516	* We are updating the fsid, the thread leading to device_list_add()
2517	* could race, so uuid_mutex is needed.
2518	*/
2519	lockdep_assert_held(&uuid_mutex);
2520
2521	/*
2522	* The threads listed below may traverse dev_list but can do that without
2523	* device_list_mutex:
2524	* - All device ops and balance - as we are in btrfs_exclop_start.
2525	* - Various dev_list readers - are using RCU.
2526	* - btrfs_ioctl_fitrim() - is using RCU.
2527	*
2528	* For-read threads as below are using device_list_mutex:
2529	* - Readonly scrub btrfs_scrub_dev()
2530	* - Readonly scrub btrfs_scrub_progress()
2531	* - btrfs_get_dev_stats()
2532	*/
2533	lockdep_assert_held(&fs_devices->device_list_mutex);
2534
2535	list_splice_init_rcu(list: &fs_devices->devices, head: &seed_devices->devices,
2536	sync: synchronize_rcu);
2537	list_for_each_entry(device, &seed_devices->devices, dev_list)
2538	device->fs_devices = seed_devices;
2539
2540	fs_devices->seeding = false;
2541	fs_devices->num_devices = 0;
2542	fs_devices->open_devices = 0;
2543	fs_devices->missing_devices = 0;
2544	fs_devices->rotating = false;
2545	list_add(new: &seed_devices->seed_list, head: &fs_devices->seed_list);
2546
2547	generate_random_uuid(uuid: fs_devices->fsid);
2548	memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
2549	memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2550
2551	super_flags = btrfs_super_flags(s: disk_super) &
2552	~BTRFS_SUPER_FLAG_SEEDING;
2553	btrfs_set_super_flags(s: disk_super, val: super_flags);
2554	}
2555
2556	/*
2557	* Store the expected generation for seed devices in device items.
2558	*/
2559	static int btrfs_finish_sprout(struct btrfs_trans_handle *trans)
2560	{
2561	BTRFS_DEV_LOOKUP_ARGS(args);
2562	struct btrfs_fs_info *fs_info = trans->fs_info;
2563	struct btrfs_root *root = fs_info->chunk_root;
2564	struct btrfs_path *path;
2565	struct extent_buffer *leaf;
2566	struct btrfs_dev_item *dev_item;
2567	struct btrfs_device *device;
2568	struct btrfs_key key;
2569	u8 fs_uuid[BTRFS_FSID_SIZE];
2570	u8 dev_uuid[BTRFS_UUID_SIZE];
2571	int ret;
2572
2573	path = btrfs_alloc_path();
2574	if (!path)
2575	return -ENOMEM;
2576
2577	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2578	key.offset = 0;
2579	key.type = BTRFS_DEV_ITEM_KEY;
2580
2581	while (1) {
2582	btrfs_reserve_chunk_metadata(trans, is_item_insertion: false);
2583	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: 0, cow: 1);
2584	btrfs_trans_release_chunk_metadata(trans);
2585	if (ret < 0)
2586	goto error;
2587
2588	leaf = path->nodes[0];
2589	next_slot:
2590	if (path->slots[0] >= btrfs_header_nritems(eb: leaf)) {
2591	ret = btrfs_next_leaf(root, path);
2592	if (ret > 0)
2593	break;
2594	if (ret < 0)
2595	goto error;
2596	leaf = path->nodes[0];
2597	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
2598	btrfs_release_path(p: path);
2599	continue;
2600	}
2601
2602	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
2603	if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2604	key.type != BTRFS_DEV_ITEM_KEY)
2605	break;
2606
2607	dev_item = btrfs_item_ptr(leaf, path->slots[0],
2608	struct btrfs_dev_item);
2609	args.devid = btrfs_device_id(eb: leaf, s: dev_item);
2610	read_extent_buffer(eb: leaf, dst: dev_uuid, start: btrfs_device_uuid(d: dev_item),
2611	BTRFS_UUID_SIZE);
2612	read_extent_buffer(eb: leaf, dst: fs_uuid, start: btrfs_device_fsid(d: dev_item),
2613	BTRFS_FSID_SIZE);
2614	args.uuid = dev_uuid;
2615	args.fsid = fs_uuid;
2616	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
2617	BUG_ON(!device); /* Logic error */
2618
2619	if (device->fs_devices->seeding) {
2620	btrfs_set_device_generation(eb: leaf, s: dev_item,
2621	val: device->generation);
2622	btrfs_mark_buffer_dirty(trans, buf: leaf);
2623	}
2624
2625	path->slots[0]++;
2626	goto next_slot;
2627	}
2628	ret = 0;
2629	error:
2630	btrfs_free_path(p: path);
2631	return ret;
2632	}
2633
2634	int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
2635	{
2636	struct btrfs_root *root = fs_info->dev_root;
2637	struct btrfs_trans_handle *trans;
2638	struct btrfs_device *device;
2639	struct file *bdev_file;
2640	struct super_block *sb = fs_info->sb;
2641	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2642	struct btrfs_fs_devices *seed_devices = NULL;
2643	u64 orig_super_total_bytes;
2644	u64 orig_super_num_devices;
2645	int ret = 0;
2646	bool seeding_dev = false;
2647	bool locked = false;
2648
2649	if (sb_rdonly(sb) && !fs_devices->seeding)
2650	return -EROFS;
2651
2652	bdev_file = bdev_file_open_by_path(path: device_path, BLK_OPEN_WRITE,
2653	holder: fs_info->bdev_holder, NULL);
2654	if (IS_ERR(ptr: bdev_file))
2655	return PTR_ERR(ptr: bdev_file);
2656
2657	if (!btrfs_check_device_zone_type(fs_info, bdev: file_bdev(bdev_file))) {
2658	ret = -EINVAL;
2659	goto error;
2660	}
2661
2662	if (fs_devices->seeding) {
2663	seeding_dev = true;
2664	down_write(sem: &sb->s_umount);
2665	mutex_lock(&uuid_mutex);
2666	locked = true;
2667	}
2668
2669	sync_blockdev(bdev: file_bdev(bdev_file));
2670
2671	rcu_read_lock();
2672	list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) {
2673	if (device->bdev == file_bdev(bdev_file)) {
2674	ret = -EEXIST;
2675	rcu_read_unlock();
2676	goto error;
2677	}
2678	}
2679	rcu_read_unlock();
2680
2681	device = btrfs_alloc_device(fs_info, NULL, NULL, path: device_path);
2682	if (IS_ERR(ptr: device)) {
2683	/* we can safely leave the fs_devices entry around */
2684	ret = PTR_ERR(ptr: device);
2685	goto error;
2686	}
2687
2688	device->fs_info = fs_info;
2689	device->bdev_file = bdev_file;
2690	device->bdev = file_bdev(bdev_file);
2691	ret = lookup_bdev(pathname: device_path, dev: &device->devt);
2692	if (ret)
2693	goto error_free_device;
2694
2695	ret = btrfs_get_dev_zone_info(device, populate_cache: false);
2696	if (ret)
2697	goto error_free_device;
2698
2699	trans = btrfs_start_transaction(root, num_items: 0);
2700	if (IS_ERR(ptr: trans)) {
2701	ret = PTR_ERR(ptr: trans);
2702	goto error_free_zone;
2703	}
2704
2705	set_bit(BTRFS_DEV_STATE_WRITEABLE, addr: &device->dev_state);
2706	device->generation = trans->transid;
2707	device->io_width = fs_info->sectorsize;
2708	device->io_align = fs_info->sectorsize;
2709	device->sector_size = fs_info->sectorsize;
2710	device->total_bytes =
2711	round_down(bdev_nr_bytes(device->bdev), fs_info->sectorsize);
2712	device->disk_total_bytes = device->total_bytes;
2713	device->commit_total_bytes = device->total_bytes;
2714	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, addr: &device->dev_state);
2715	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, addr: &device->dev_state);
2716	device->dev_stats_valid = 1;
2717	set_blocksize(bdev: device->bdev, BTRFS_BDEV_BLOCKSIZE);
2718
2719	if (seeding_dev) {
2720	btrfs_clear_sb_rdonly(sb);
2721
2722	/* GFP_KERNEL allocation must not be under device_list_mutex */
2723	seed_devices = btrfs_init_sprout(fs_info);
2724	if (IS_ERR(ptr: seed_devices)) {
2725	ret = PTR_ERR(ptr: seed_devices);
2726	btrfs_abort_transaction(trans, ret);
2727	goto error_trans;
2728	}
2729	}
2730
2731	mutex_lock(&fs_devices->device_list_mutex);
2732	if (seeding_dev) {
2733	btrfs_setup_sprout(fs_info, seed_devices);
2734	btrfs_assign_next_active_device(device: fs_info->fs_devices->latest_dev,
2735	next_device: device);
2736	}
2737
2738	device->fs_devices = fs_devices;
2739
2740	mutex_lock(&fs_info->chunk_mutex);
2741	list_add_rcu(new: &device->dev_list, head: &fs_devices->devices);
2742	list_add(new: &device->dev_alloc_list, head: &fs_devices->alloc_list);
2743	fs_devices->num_devices++;
2744	fs_devices->open_devices++;
2745	fs_devices->rw_devices++;
2746	fs_devices->total_devices++;
2747	fs_devices->total_rw_bytes += device->total_bytes;
2748
2749	atomic64_add(i: device->total_bytes, v: &fs_info->free_chunk_space);
2750
2751	if (!bdev_nonrot(bdev: device->bdev))
2752	fs_devices->rotating = true;
2753
2754	orig_super_total_bytes = btrfs_super_total_bytes(s: fs_info->super_copy);
2755	btrfs_set_super_total_bytes(s: fs_info->super_copy,
2756	round_down(orig_super_total_bytes + device->total_bytes,
2757	fs_info->sectorsize));
2758
2759	orig_super_num_devices = btrfs_super_num_devices(s: fs_info->super_copy);
2760	btrfs_set_super_num_devices(s: fs_info->super_copy,
2761	val: orig_super_num_devices + 1);
2762
2763	/*
2764	* we've got more storage, clear any full flags on the space
2765	* infos
2766	*/
2767	btrfs_clear_space_info_full(info: fs_info);
2768
2769	mutex_unlock(lock: &fs_info->chunk_mutex);
2770
2771	/* Add sysfs device entry */
2772	btrfs_sysfs_add_device(device);
2773
2774	mutex_unlock(lock: &fs_devices->device_list_mutex);
2775
2776	if (seeding_dev) {
2777	mutex_lock(&fs_info->chunk_mutex);
2778	ret = init_first_rw_device(trans);
2779	mutex_unlock(lock: &fs_info->chunk_mutex);
2780	if (ret) {
2781	btrfs_abort_transaction(trans, ret);
2782	goto error_sysfs;
2783	}
2784	}
2785
2786	ret = btrfs_add_dev_item(trans, device);
2787	if (ret) {
2788	btrfs_abort_transaction(trans, ret);
2789	goto error_sysfs;
2790	}
2791
2792	if (seeding_dev) {
2793	ret = btrfs_finish_sprout(trans);
2794	if (ret) {
2795	btrfs_abort_transaction(trans, ret);
2796	goto error_sysfs;
2797	}
2798
2799	/*
2800	* fs_devices now represents the newly sprouted filesystem and
2801	* its fsid has been changed by btrfs_sprout_splice().
2802	*/
2803	btrfs_sysfs_update_sprout_fsid(fs_devices);
2804	}
2805
2806	ret = btrfs_commit_transaction(trans);
2807
2808	if (seeding_dev) {
2809	mutex_unlock(lock: &uuid_mutex);
2810	up_write(sem: &sb->s_umount);
2811	locked = false;
2812
2813	if (ret) /* transaction commit */
2814	return ret;
2815
2816	ret = btrfs_relocate_sys_chunks(fs_info);
2817	if (ret < 0)
2818	btrfs_handle_fs_error(fs_info, ret,
2819	"Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2820	trans = btrfs_attach_transaction(root);
2821	if (IS_ERR(ptr: trans)) {
2822	if (PTR_ERR(ptr: trans) == -ENOENT)
2823	return 0;
2824	ret = PTR_ERR(ptr: trans);
2825	trans = NULL;
2826	goto error_sysfs;
2827	}
2828	ret = btrfs_commit_transaction(trans);
2829	}
2830
2831	/*
2832	* Now that we have written a new super block to this device, check all
2833	* other fs_devices list if device_path alienates any other scanned
2834	* device.
2835	* We can ignore the return value as it typically returns -EINVAL and
2836	* only succeeds if the device was an alien.
2837	*/
2838	btrfs_forget_devices(devt: device->devt);
2839
2840	/* Update ctime/mtime for blkid or udev */
2841	update_dev_time(device_path);
2842
2843	return ret;
2844
2845	error_sysfs:
2846	btrfs_sysfs_remove_device(device);
2847	mutex_lock(&fs_info->fs_devices->device_list_mutex);
2848	mutex_lock(&fs_info->chunk_mutex);
2849	list_del_rcu(entry: &device->dev_list);
2850	list_del(entry: &device->dev_alloc_list);
2851	fs_info->fs_devices->num_devices--;
2852	fs_info->fs_devices->open_devices--;
2853	fs_info->fs_devices->rw_devices--;
2854	fs_info->fs_devices->total_devices--;
2855	fs_info->fs_devices->total_rw_bytes -= device->total_bytes;
2856	atomic64_sub(i: device->total_bytes, v: &fs_info->free_chunk_space);
2857	btrfs_set_super_total_bytes(s: fs_info->super_copy,
2858	val: orig_super_total_bytes);
2859	btrfs_set_super_num_devices(s: fs_info->super_copy,
2860	val: orig_super_num_devices);
2861	mutex_unlock(lock: &fs_info->chunk_mutex);
2862	mutex_unlock(lock: &fs_info->fs_devices->device_list_mutex);
2863	error_trans:
2864	if (seeding_dev)
2865	btrfs_set_sb_rdonly(sb);
2866	if (trans)
2867	btrfs_end_transaction(trans);
2868	error_free_zone:
2869	btrfs_destroy_dev_zone_info(device);
2870	error_free_device:
2871	btrfs_free_device(device);
2872	error:
2873	fput(bdev_file);
2874	if (locked) {
2875	mutex_unlock(lock: &uuid_mutex);
2876	up_write(sem: &sb->s_umount);
2877	}
2878	return ret;
2879	}
2880
2881	static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
2882	struct btrfs_device *device)
2883	{
2884	int ret;
2885	struct btrfs_path *path;
2886	struct btrfs_root *root = device->fs_info->chunk_root;
2887	struct btrfs_dev_item *dev_item;
2888	struct extent_buffer *leaf;
2889	struct btrfs_key key;
2890
2891	path = btrfs_alloc_path();
2892	if (!path)
2893	return -ENOMEM;
2894
2895	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2896	key.type = BTRFS_DEV_ITEM_KEY;
2897	key.offset = device->devid;
2898
2899	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: 0, cow: 1);
2900	if (ret < 0)
2901	goto out;
2902
2903	if (ret > 0) {
2904	ret = -ENOENT;
2905	goto out;
2906	}
2907
2908	leaf = path->nodes[0];
2909	dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2910
2911	btrfs_set_device_id(eb: leaf, s: dev_item, val: device->devid);
2912	btrfs_set_device_type(eb: leaf, s: dev_item, val: device->type);
2913	btrfs_set_device_io_align(eb: leaf, s: dev_item, val: device->io_align);
2914	btrfs_set_device_io_width(eb: leaf, s: dev_item, val: device->io_width);
2915	btrfs_set_device_sector_size(eb: leaf, s: dev_item, val: device->sector_size);
2916	btrfs_set_device_total_bytes(eb: leaf, s: dev_item,
2917	val: btrfs_device_get_disk_total_bytes(dev: device));
2918	btrfs_set_device_bytes_used(eb: leaf, s: dev_item,
2919	val: btrfs_device_get_bytes_used(dev: device));
2920	btrfs_mark_buffer_dirty(trans, buf: leaf);
2921
2922	out:
2923	btrfs_free_path(p: path);
2924	return ret;
2925	}
2926
2927	int btrfs_grow_device(struct btrfs_trans_handle *trans,
2928	struct btrfs_device *device, u64 new_size)
2929	{
2930	struct btrfs_fs_info *fs_info = device->fs_info;
2931	struct btrfs_super_block *super_copy = fs_info->super_copy;
2932	u64 old_total;
2933	u64 diff;
2934	int ret;
2935
2936	if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2937	return -EACCES;
2938
2939	new_size = round_down(new_size, fs_info->sectorsize);
2940
2941	mutex_lock(&fs_info->chunk_mutex);
2942	old_total = btrfs_super_total_bytes(s: super_copy);
2943	diff = round_down(new_size - device->total_bytes, fs_info->sectorsize);
2944
2945	if (new_size <= device->total_bytes ||
2946	test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2947	mutex_unlock(lock: &fs_info->chunk_mutex);
2948	return -EINVAL;
2949	}
2950
2951	btrfs_set_super_total_bytes(s: super_copy,
2952	round_down(old_total + diff, fs_info->sectorsize));
2953	device->fs_devices->total_rw_bytes += diff;
2954	atomic64_add(i: diff, v: &fs_info->free_chunk_space);
2955
2956	btrfs_device_set_total_bytes(dev: device, size: new_size);
2957	btrfs_device_set_disk_total_bytes(dev: device, size: new_size);
2958	btrfs_clear_space_info_full(info: device->fs_info);
2959	if (list_empty(head: &device->post_commit_list))
2960	list_add_tail(new: &device->post_commit_list,
2961	head: &trans->transaction->dev_update_list);
2962	mutex_unlock(lock: &fs_info->chunk_mutex);
2963
2964	btrfs_reserve_chunk_metadata(trans, is_item_insertion: false);
2965	ret = btrfs_update_device(trans, device);
2966	btrfs_trans_release_chunk_metadata(trans);
2967
2968	return ret;
2969	}
2970
2971	static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2972	{
2973	struct btrfs_fs_info *fs_info = trans->fs_info;
2974	struct btrfs_root *root = fs_info->chunk_root;
2975	int ret;
2976	struct btrfs_path *path;
2977	struct btrfs_key key;
2978
2979	path = btrfs_alloc_path();
2980	if (!path)
2981	return -ENOMEM;
2982
2983	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2984	key.offset = chunk_offset;
2985	key.type = BTRFS_CHUNK_ITEM_KEY;
2986
2987	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: -1, cow: 1);
2988	if (ret < 0)
2989	goto out;
2990	else if (ret > 0) { /* Logic error or corruption */
2991	btrfs_handle_fs_error(fs_info, -ENOENT,
2992	"Failed lookup while freeing chunk.");
2993	ret = -ENOENT;
2994	goto out;
2995	}
2996
2997	ret = btrfs_del_item(trans, root, path);
2998	if (ret < 0)
2999	btrfs_handle_fs_error(fs_info, ret,
3000	"Failed to delete chunk item.");
3001	out:
3002	btrfs_free_path(p: path);
3003	return ret;
3004	}
3005
3006	static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3007	{
3008	struct btrfs_super_block *super_copy = fs_info->super_copy;
3009	struct btrfs_disk_key *disk_key;
3010	struct btrfs_chunk *chunk;
3011	u8 *ptr;
3012	int ret = 0;
3013	u32 num_stripes;
3014	u32 array_size;
3015	u32 len = 0;
3016	u32 cur;
3017	struct btrfs_key key;
3018
3019	lockdep_assert_held(&fs_info->chunk_mutex);
3020	array_size = btrfs_super_sys_array_size(s: super_copy);
3021
3022	ptr = super_copy->sys_chunk_array;
3023	cur = 0;
3024
3025	while (cur < array_size) {
3026	disk_key = (struct btrfs_disk_key *)ptr;
3027	btrfs_disk_key_to_cpu(cpu_key: &key, disk_key);
3028
3029	len = sizeof(*disk_key);
3030
3031	if (key.type == BTRFS_CHUNK_ITEM_KEY) {
3032	chunk = (struct btrfs_chunk *)(ptr + len);
3033	num_stripes = btrfs_stack_chunk_num_stripes(s: chunk);
3034	len += btrfs_chunk_item_size(num_stripes);
3035	} else {
3036	ret = -EIO;
3037	break;
3038	}
3039	if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID &&
3040	key.offset == chunk_offset) {
3041	memmove(ptr, ptr + len, array_size - (cur + len));
3042	array_size -= len;
3043	btrfs_set_super_sys_array_size(s: super_copy, val: array_size);
3044	} else {
3045	ptr += len;
3046	cur += len;
3047	}
3048	}
3049	return ret;
3050	}
3051
3052	struct btrfs_chunk_map *btrfs_find_chunk_map_nolock(struct btrfs_fs_info *fs_info,
3053	u64 logical, u64 length)
3054	{
3055	struct rb_node *node = fs_info->mapping_tree.rb_root.rb_node;
3056	struct rb_node *prev = NULL;
3057	struct rb_node *orig_prev;
3058	struct btrfs_chunk_map *map;
3059	struct btrfs_chunk_map *prev_map = NULL;
3060
3061	while (node) {
3062	map = rb_entry(node, struct btrfs_chunk_map, rb_node);
3063	prev = node;
3064	prev_map = map;
3065
3066	if (logical < map->start) {
3067	node = node->rb_left;
3068	} else if (logical >= map->start + map->chunk_len) {
3069	node = node->rb_right;
3070	} else {
3071	refcount_inc(r: &map->refs);
3072	return map;
3073	}
3074	}
3075
3076	if (!prev)
3077	return NULL;
3078
3079	orig_prev = prev;
3080	while (prev && logical >= prev_map->start + prev_map->chunk_len) {
3081	prev = rb_next(prev);
3082	prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node);
3083	}
3084
3085	if (!prev) {
3086	prev = orig_prev;
3087	prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node);
3088	while (prev && logical < prev_map->start) {
3089	prev = rb_prev(prev);
3090	prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node);
3091	}
3092	}
3093
3094	if (prev) {
3095	u64 end = logical + length;
3096
3097	/*
3098	* Caller can pass a U64_MAX length when it wants to get any
3099	* chunk starting at an offset of 'logical' or higher, so deal
3100	* with underflow by resetting the end offset to U64_MAX.
3101	*/
3102	if (end < logical)
3103	end = U64_MAX;
3104
3105	if (end > prev_map->start &&
3106	logical < prev_map->start + prev_map->chunk_len) {
3107	refcount_inc(r: &prev_map->refs);
3108	return prev_map;
3109	}
3110	}
3111
3112	return NULL;
3113	}
3114
3115	struct btrfs_chunk_map *btrfs_find_chunk_map(struct btrfs_fs_info *fs_info,
3116	u64 logical, u64 length)
3117	{
3118	struct btrfs_chunk_map *map;
3119
3120	read_lock(&fs_info->mapping_tree_lock);
3121	map = btrfs_find_chunk_map_nolock(fs_info, logical, length);
3122	read_unlock(&fs_info->mapping_tree_lock);
3123
3124	return map;
3125	}
3126
3127	/*
3128	* Find the mapping containing the given logical extent.
3129	*
3130	* @logical: Logical block offset in bytes.
3131	* @length: Length of extent in bytes.
3132	*
3133	* Return: Chunk mapping or ERR_PTR.
3134	*/
3135	struct btrfs_chunk_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info,
3136	u64 logical, u64 length)
3137	{
3138	struct btrfs_chunk_map *map;
3139
3140	map = btrfs_find_chunk_map(fs_info, logical, length);
3141
3142	if (unlikely(!map)) {
3143	btrfs_crit(fs_info,
3144	"unable to find chunk map for logical %llu length %llu",
3145	logical, length);
3146	return ERR_PTR(error: -EINVAL);
3147	}
3148
3149	if (unlikely(map->start > logical || map->start + map->chunk_len <= logical)) {
3150	btrfs_crit(fs_info,
3151	"found a bad chunk map, wanted %llu-%llu, found %llu-%llu",
3152	logical, logical + length, map->start,
3153	map->start + map->chunk_len);
3154	btrfs_free_chunk_map(map);
3155	return ERR_PTR(error: -EINVAL);
3156	}
3157
3158	/* Callers are responsible for dropping the reference. */
3159	return map;
3160	}
3161
3162	static int remove_chunk_item(struct btrfs_trans_handle *trans,
3163	struct btrfs_chunk_map *map, u64 chunk_offset)
3164	{
3165	int i;
3166
3167	/*
3168	* Removing chunk items and updating the device items in the chunks btree
3169	* requires holding the chunk_mutex.
3170	* See the comment at btrfs_chunk_alloc() for the details.
3171	*/
3172	lockdep_assert_held(&trans->fs_info->chunk_mutex);
3173
3174	for (i = 0; i < map->num_stripes; i++) {
3175	int ret;
3176
3177	ret = btrfs_update_device(trans, device: map->stripes[i].dev);
3178	if (ret)
3179	return ret;
3180	}
3181
3182	return btrfs_free_chunk(trans, chunk_offset);
3183	}
3184
3185	int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
3186	{
3187	struct btrfs_fs_info *fs_info = trans->fs_info;
3188	struct btrfs_chunk_map *map;
3189	u64 dev_extent_len = 0;
3190	int i, ret = 0;
3191	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
3192
3193	map = btrfs_get_chunk_map(fs_info, logical: chunk_offset, length: 1);
3194	if (IS_ERR(ptr: map)) {
3195	/*
3196	* This is a logic error, but we don't want to just rely on the
3197	* user having built with ASSERT enabled, so if ASSERT doesn't
3198	* do anything we still error out.
3199	*/
3200	ASSERT(0);
3201	return PTR_ERR(ptr: map);
3202	}
3203
3204	/*
3205	* First delete the device extent items from the devices btree.
3206	* We take the device_list_mutex to avoid racing with the finishing phase
3207	* of a device replace operation. See the comment below before acquiring
3208	* fs_info->chunk_mutex. Note that here we do not acquire the chunk_mutex
3209	* because that can result in a deadlock when deleting the device extent
3210	* items from the devices btree - COWing an extent buffer from the btree
3211	* may result in allocating a new metadata chunk, which would attempt to
3212	* lock again fs_info->chunk_mutex.
3213	*/
3214	mutex_lock(&fs_devices->device_list_mutex);
3215	for (i = 0; i < map->num_stripes; i++) {
3216	struct btrfs_device *device = map->stripes[i].dev;
3217	ret = btrfs_free_dev_extent(trans, device,
3218	start: map->stripes[i].physical,
3219	dev_extent_len: &dev_extent_len);
3220	if (ret) {
3221	mutex_unlock(lock: &fs_devices->device_list_mutex);
3222	btrfs_abort_transaction(trans, ret);
3223	goto out;
3224	}
3225
3226	if (device->bytes_used > 0) {
3227	mutex_lock(&fs_info->chunk_mutex);
3228	btrfs_device_set_bytes_used(dev: device,
3229	size: device->bytes_used - dev_extent_len);
3230	atomic64_add(i: dev_extent_len, v: &fs_info->free_chunk_space);
3231	btrfs_clear_space_info_full(info: fs_info);
3232	mutex_unlock(lock: &fs_info->chunk_mutex);
3233	}
3234	}
3235	mutex_unlock(lock: &fs_devices->device_list_mutex);
3236
3237	/*
3238	* We acquire fs_info->chunk_mutex for 2 reasons:
3239	*
3240	* 1) Just like with the first phase of the chunk allocation, we must
3241	* reserve system space, do all chunk btree updates and deletions, and
3242	* update the system chunk array in the superblock while holding this
3243	* mutex. This is for similar reasons as explained on the comment at
3244	* the top of btrfs_chunk_alloc();
3245	*
3246	* 2) Prevent races with the final phase of a device replace operation
3247	* that replaces the device object associated with the map's stripes,
3248	* because the device object's id can change at any time during that
3249	* final phase of the device replace operation
3250	* (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the
3251	* replaced device and then see it with an ID of
3252	* BTRFS_DEV_REPLACE_DEVID, which would cause a failure when updating
3253	* the device item, which does not exists on the chunk btree.
3254	* The finishing phase of device replace acquires both the
3255	* device_list_mutex and the chunk_mutex, in that order, so we are
3256	* safe by just acquiring the chunk_mutex.
3257	*/
3258	trans->removing_chunk = true;
3259	mutex_lock(&fs_info->chunk_mutex);
3260
3261	check_system_chunk(trans, type: map->type);
3262
3263	ret = remove_chunk_item(trans, map, chunk_offset);
3264	/*
3265	* Normally we should not get -ENOSPC since we reserved space before
3266	* through the call to check_system_chunk().
3267	*
3268	* Despite our system space_info having enough free space, we may not
3269	* be able to allocate extents from its block groups, because all have
3270	* an incompatible profile, which will force us to allocate a new system
3271	* block group with the right profile, or right after we called
3272	* check_system_space() above, a scrub turned the only system block group
3273	* with enough free space into RO mode.
3274	* This is explained with more detail at do_chunk_alloc().
3275	*
3276	* So if we get -ENOSPC, allocate a new system chunk and retry once.
3277	*/
3278	if (ret == -ENOSPC) {
3279	const u64 sys_flags = btrfs_system_alloc_profile(fs_info);
3280	struct btrfs_block_group *sys_bg;
3281
3282	sys_bg = btrfs_create_chunk(trans, type: sys_flags);
3283	if (IS_ERR(ptr: sys_bg)) {
3284	ret = PTR_ERR(ptr: sys_bg);
3285	btrfs_abort_transaction(trans, ret);
3286	goto out;
3287	}
3288
3289	ret = btrfs_chunk_alloc_add_chunk_item(trans, bg: sys_bg);
3290	if (ret) {
3291	btrfs_abort_transaction(trans, ret);
3292	goto out;
3293	}
3294
3295	ret = remove_chunk_item(trans, map, chunk_offset);
3296	if (ret) {
3297	btrfs_abort_transaction(trans, ret);
3298	goto out;
3299	}
3300	} else if (ret) {
3301	btrfs_abort_transaction(trans, ret);
3302	goto out;
3303	}
3304
3305	trace_btrfs_chunk_free(fs_info, map, offset: chunk_offset, size: map->chunk_len);
3306
3307	if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
3308	ret = btrfs_del_sys_chunk(fs_info, chunk_offset);
3309	if (ret) {
3310	btrfs_abort_transaction(trans, ret);
3311	goto out;
3312	}
3313	}
3314
3315	mutex_unlock(lock: &fs_info->chunk_mutex);
3316	trans->removing_chunk = false;
3317
3318	/*
3319	* We are done with chunk btree updates and deletions, so release the
3320	* system space we previously reserved (with check_system_chunk()).
3321	*/
3322	btrfs_trans_release_chunk_metadata(trans);
3323
3324	ret = btrfs_remove_block_group(trans, map);
3325	if (ret) {
3326	btrfs_abort_transaction(trans, ret);
3327	goto out;
3328	}
3329
3330	out:
3331	if (trans->removing_chunk) {
3332	mutex_unlock(lock: &fs_info->chunk_mutex);
3333	trans->removing_chunk = false;
3334	}
3335	/* once for us */
3336	btrfs_free_chunk_map(map);
3337	return ret;
3338	}
3339
3340	int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3341	{
3342	struct btrfs_root *root = fs_info->chunk_root;
3343	struct btrfs_trans_handle *trans;
3344	struct btrfs_block_group *block_group;
3345	u64 length;
3346	int ret;
3347
3348	if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
3349	btrfs_err(fs_info,
3350	"relocate: not supported on extent tree v2 yet");
3351	return -EINVAL;
3352	}
3353
3354	/*
3355	* Prevent races with automatic removal of unused block groups.
3356	* After we relocate and before we remove the chunk with offset
3357	* chunk_offset, automatic removal of the block group can kick in,
3358	* resulting in a failure when calling btrfs_remove_chunk() below.
3359	*
3360	* Make sure to acquire this mutex before doing a tree search (dev
3361	* or chunk trees) to find chunks. Otherwise the cleaner kthread might
3362	* call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
3363	* we release the path used to search the chunk/dev tree and before
3364	* the current task acquires this mutex and calls us.
3365	*/
3366	lockdep_assert_held(&fs_info->reclaim_bgs_lock);
3367
3368	/* step one, relocate all the extents inside this chunk */
3369	btrfs_scrub_pause(fs_info);
3370	ret = btrfs_relocate_block_group(fs_info, group_start: chunk_offset);
3371	btrfs_scrub_continue(fs_info);
3372	if (ret) {
3373	/*
3374	* If we had a transaction abort, stop all running scrubs.
3375	* See transaction.c:cleanup_transaction() why we do it here.
3376	*/
3377	if (BTRFS_FS_ERROR(fs_info))
3378	btrfs_scrub_cancel(info: fs_info);
3379	return ret;
3380	}
3381
3382	block_group = btrfs_lookup_block_group(info: fs_info, bytenr: chunk_offset);
3383	if (!block_group)
3384	return -ENOENT;
3385	btrfs_discard_cancel_work(discard_ctl: &fs_info->discard_ctl, block_group);
3386	length = block_group->length;
3387	btrfs_put_block_group(cache: block_group);
3388
3389	/*
3390	* On a zoned file system, discard the whole block group, this will
3391	* trigger a REQ_OP_ZONE_RESET operation on the device zone. If
3392	* resetting the zone fails, don't treat it as a fatal problem from the
3393	* filesystem's point of view.
3394	*/
3395	if (btrfs_is_zoned(fs_info)) {
3396	ret = btrfs_discard_extent(fs_info, bytenr: chunk_offset, num_bytes: length, NULL);
3397	if (ret)
3398	btrfs_info(fs_info,
3399	"failed to reset zone %llu after relocation",
3400	chunk_offset);
3401	}
3402
3403	trans = btrfs_start_trans_remove_block_group(fs_info: root->fs_info,
3404	chunk_offset);
3405	if (IS_ERR(ptr: trans)) {
3406	ret = PTR_ERR(ptr: trans);
3407	btrfs_handle_fs_error(root->fs_info, ret, NULL);
3408	return ret;
3409	}
3410
3411	/*
3412	* step two, delete the device extents and the
3413	* chunk tree entries
3414	*/
3415	ret = btrfs_remove_chunk(trans, chunk_offset);
3416	btrfs_end_transaction(trans);
3417	return ret;
3418	}
3419
3420	static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
3421	{
3422	struct btrfs_root *chunk_root = fs_info->chunk_root;
3423	struct btrfs_path *path;
3424	struct extent_buffer *leaf;
3425	struct btrfs_chunk *chunk;
3426	struct btrfs_key key;
3427	struct btrfs_key found_key;
3428	u64 chunk_type;
3429	bool retried = false;
3430	int failed = 0;
3431	int ret;
3432
3433	path = btrfs_alloc_path();
3434	if (!path)
3435	return -ENOMEM;
3436
3437	again:
3438	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3439	key.offset = (u64)-1;
3440	key.type = BTRFS_CHUNK_ITEM_KEY;
3441
3442	while (1) {
3443	mutex_lock(&fs_info->reclaim_bgs_lock);
3444	ret = btrfs_search_slot(NULL, root: chunk_root, key: &key, p: path, ins_len: 0, cow: 0);
3445	if (ret < 0) {
3446	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
3447	goto error;
3448	}
3449	if (ret == 0) {
3450	/*
3451	* On the first search we would find chunk tree with
3452	* offset -1, which is not possible. On subsequent
3453	* loops this would find an existing item on an invalid
3454	* offset (one less than the previous one, wrong
3455	* alignment and size).
3456	*/
3457	ret = -EUCLEAN;
3458	goto error;
3459	}
3460
3461	ret = btrfs_previous_item(root: chunk_root, path, min_objectid: key.objectid,
3462	type: key.type);
3463	if (ret)
3464	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
3465	if (ret < 0)
3466	goto error;
3467	if (ret > 0)
3468	break;
3469
3470	leaf = path->nodes[0];
3471	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
3472
3473	chunk = btrfs_item_ptr(leaf, path->slots[0],
3474	struct btrfs_chunk);
3475	chunk_type = btrfs_chunk_type(eb: leaf, s: chunk);
3476	btrfs_release_path(p: path);
3477
3478	if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
3479	ret = btrfs_relocate_chunk(fs_info, chunk_offset: found_key.offset);
3480	if (ret == -ENOSPC)
3481	failed++;
3482	else
3483	BUG_ON(ret);
3484	}
3485	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
3486
3487	if (found_key.offset == 0)
3488	break;
3489	key.offset = found_key.offset - 1;
3490	}
3491	ret = 0;
3492	if (failed && !retried) {
3493	failed = 0;
3494	retried = true;
3495	goto again;
3496	} else if (WARN_ON(failed && retried)) {
3497	ret = -ENOSPC;
3498	}
3499	error:
3500	btrfs_free_path(p: path);
3501	return ret;
3502	}
3503
3504	/*
3505	* return 1 : allocate a data chunk successfully,
3506	* return <0: errors during allocating a data chunk,
3507	* return 0 : no need to allocate a data chunk.
3508	*/
3509	static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info,
3510	u64 chunk_offset)
3511	{
3512	struct btrfs_block_group *cache;
3513	u64 bytes_used;
3514	u64 chunk_type;
3515
3516	cache = btrfs_lookup_block_group(info: fs_info, bytenr: chunk_offset);
3517	ASSERT(cache);
3518	chunk_type = cache->flags;
3519	btrfs_put_block_group(cache);
3520
3521	if (!(chunk_type & BTRFS_BLOCK_GROUP_DATA))
3522	return 0;
3523
3524	spin_lock(lock: &fs_info->data_sinfo->lock);
3525	bytes_used = fs_info->data_sinfo->bytes_used;
3526	spin_unlock(lock: &fs_info->data_sinfo->lock);
3527
3528	if (!bytes_used) {
3529	struct btrfs_trans_handle *trans;
3530	int ret;
3531
3532	trans = btrfs_join_transaction(root: fs_info->tree_root);
3533	if (IS_ERR(ptr: trans))
3534	return PTR_ERR(ptr: trans);
3535
3536	ret = btrfs_force_chunk_alloc(trans, BTRFS_BLOCK_GROUP_DATA);
3537	btrfs_end_transaction(trans);
3538	if (ret < 0)
3539	return ret;
3540	return 1;
3541	}
3542
3543	return 0;
3544	}
3545
3546	static void btrfs_disk_balance_args_to_cpu(struct btrfs_balance_args *cpu,
3547	const struct btrfs_disk_balance_args *disk)
3548	{
3549	memset(cpu, 0, sizeof(*cpu));
3550
3551	cpu->profiles = le64_to_cpu(disk->profiles);
3552	cpu->usage = le64_to_cpu(disk->usage);
3553	cpu->devid = le64_to_cpu(disk->devid);
3554	cpu->pstart = le64_to_cpu(disk->pstart);
3555	cpu->pend = le64_to_cpu(disk->pend);
3556	cpu->vstart = le64_to_cpu(disk->vstart);
3557	cpu->vend = le64_to_cpu(disk->vend);
3558	cpu->target = le64_to_cpu(disk->target);
3559	cpu->flags = le64_to_cpu(disk->flags);
3560	cpu->limit = le64_to_cpu(disk->limit);
3561	cpu->stripes_min = le32_to_cpu(disk->stripes_min);
3562	cpu->stripes_max = le32_to_cpu(disk->stripes_max);
3563	}
3564
3565	static void btrfs_cpu_balance_args_to_disk(struct btrfs_disk_balance_args *disk,
3566	const struct btrfs_balance_args *cpu)
3567	{
3568	memset(disk, 0, sizeof(*disk));
3569
3570	disk->profiles = cpu_to_le64(cpu->profiles);
3571	disk->usage = cpu_to_le64(cpu->usage);
3572	disk->devid = cpu_to_le64(cpu->devid);
3573	disk->pstart = cpu_to_le64(cpu->pstart);
3574	disk->pend = cpu_to_le64(cpu->pend);
3575	disk->vstart = cpu_to_le64(cpu->vstart);
3576	disk->vend = cpu_to_le64(cpu->vend);
3577	disk->target = cpu_to_le64(cpu->target);
3578	disk->flags = cpu_to_le64(cpu->flags);
3579	disk->limit = cpu_to_le64(cpu->limit);
3580	disk->stripes_min = cpu_to_le32(cpu->stripes_min);
3581	disk->stripes_max = cpu_to_le32(cpu->stripes_max);
3582	}
3583
3584	static int insert_balance_item(struct btrfs_fs_info *fs_info,
3585	struct btrfs_balance_control *bctl)
3586	{
3587	struct btrfs_root *root = fs_info->tree_root;
3588	struct btrfs_trans_handle *trans;
3589	struct btrfs_balance_item *item;
3590	struct btrfs_disk_balance_args disk_bargs;
3591	struct btrfs_path *path;
3592	struct extent_buffer *leaf;
3593	struct btrfs_key key;
3594	int ret, err;
3595
3596	path = btrfs_alloc_path();
3597	if (!path)
3598	return -ENOMEM;
3599
3600	trans = btrfs_start_transaction(root, num_items: 0);
3601	if (IS_ERR(ptr: trans)) {
3602	btrfs_free_path(p: path);
3603	return PTR_ERR(ptr: trans);
3604	}
3605
3606	key.objectid = BTRFS_BALANCE_OBJECTID;
3607	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3608	key.offset = 0;
3609
3610	ret = btrfs_insert_empty_item(trans, root, path, key: &key,
3611	data_size: sizeof(*item));
3612	if (ret)
3613	goto out;
3614
3615	leaf = path->nodes[0];
3616	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
3617
3618	memzero_extent_buffer(eb: leaf, start: (unsigned long)item, len: sizeof(*item));
3619
3620	btrfs_cpu_balance_args_to_disk(disk: &disk_bargs, cpu: &bctl->data);
3621	btrfs_set_balance_data(eb: leaf, bi: item, ba: &disk_bargs);
3622	btrfs_cpu_balance_args_to_disk(disk: &disk_bargs, cpu: &bctl->meta);
3623	btrfs_set_balance_meta(eb: leaf, bi: item, ba: &disk_bargs);
3624	btrfs_cpu_balance_args_to_disk(disk: &disk_bargs, cpu: &bctl->sys);
3625	btrfs_set_balance_sys(eb: leaf, bi: item, ba: &disk_bargs);
3626
3627	btrfs_set_balance_flags(eb: leaf, s: item, val: bctl->flags);
3628
3629	btrfs_mark_buffer_dirty(trans, buf: leaf);
3630	out:
3631	btrfs_free_path(p: path);
3632	err = btrfs_commit_transaction(trans);
3633	if (err && !ret)
3634	ret = err;
3635	return ret;
3636	}
3637
3638	static int del_balance_item(struct btrfs_fs_info *fs_info)
3639	{
3640	struct btrfs_root *root = fs_info->tree_root;
3641	struct btrfs_trans_handle *trans;
3642	struct btrfs_path *path;
3643	struct btrfs_key key;
3644	int ret, err;
3645
3646	path = btrfs_alloc_path();
3647	if (!path)
3648	return -ENOMEM;
3649
3650	trans = btrfs_start_transaction_fallback_global_rsv(root, num_items: 0);
3651	if (IS_ERR(ptr: trans)) {
3652	btrfs_free_path(p: path);
3653	return PTR_ERR(ptr: trans);
3654	}
3655
3656	key.objectid = BTRFS_BALANCE_OBJECTID;
3657	key.type = BTRFS_TEMPORARY_ITEM_KEY;
3658	key.offset = 0;
3659
3660	ret = btrfs_search_slot(trans, root, key: &key, p: path, ins_len: -1, cow: 1);
3661	if (ret < 0)
3662	goto out;
3663	if (ret > 0) {
3664	ret = -ENOENT;
3665	goto out;
3666	}
3667
3668	ret = btrfs_del_item(trans, root, path);
3669	out:
3670	btrfs_free_path(p: path);
3671	err = btrfs_commit_transaction(trans);
3672	if (err && !ret)
3673	ret = err;
3674	return ret;
3675	}
3676
3677	/*
3678	* This is a heuristic used to reduce the number of chunks balanced on
3679	* resume after balance was interrupted.
3680	*/
3681	static void update_balance_args(struct btrfs_balance_control *bctl)
3682	{
3683	/*
3684	* Turn on soft mode for chunk types that were being converted.
3685	*/
3686	if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)
3687	bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT;
3688	if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)
3689	bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT;
3690	if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)
3691	bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT;
3692
3693	/*
3694	* Turn on usage filter if is not already used. The idea is
3695	* that chunks that we have already balanced should be
3696	* reasonably full. Don't do it for chunks that are being
3697	* converted - that will keep us from relocating unconverted
3698	* (albeit full) chunks.
3699	*/
3700	if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3701	!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3702	!(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3703	bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE;
3704	bctl->data.usage = 90;
3705	}
3706	if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3707	!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3708	!(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3709	bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE;
3710	bctl->sys.usage = 90;
3711	}
3712	if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3713	!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3714	!(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3715	bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE;
3716	bctl->meta.usage = 90;
3717	}
3718	}
3719
3720	/*
3721	* Clear the balance status in fs_info and delete the balance item from disk.
3722	*/
3723	static void reset_balance_state(struct btrfs_fs_info *fs_info)
3724	{
3725	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3726	int ret;
3727
3728	ASSERT(fs_info->balance_ctl);
3729
3730	spin_lock(lock: &fs_info->balance_lock);
3731	fs_info->balance_ctl = NULL;
3732	spin_unlock(lock: &fs_info->balance_lock);
3733
3734	kfree(objp: bctl);
3735	ret = del_balance_item(fs_info);
3736	if (ret)
3737	btrfs_handle_fs_error(fs_info, ret, NULL);
3738	}
3739
3740	/*
3741	* Balance filters. Return 1 if chunk should be filtered out
3742	* (should not be balanced).
3743	*/
3744	static int chunk_profiles_filter(u64 chunk_type,
3745	struct btrfs_balance_args *bargs)
3746	{
3747	chunk_type = chunk_to_extended(flags: chunk_type) &
3748	BTRFS_EXTENDED_PROFILE_MASK;
3749
3750	if (bargs->profiles & chunk_type)
3751	return 0;
3752
3753	return 1;
3754	}
3755
3756	static int chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset,
3757	struct btrfs_balance_args *bargs)
3758	{
3759	struct btrfs_block_group *cache;
3760	u64 chunk_used;
3761	u64 user_thresh_min;
3762	u64 user_thresh_max;
3763	int ret = 1;
3764
3765	cache = btrfs_lookup_block_group(info: fs_info, bytenr: chunk_offset);
3766	chunk_used = cache->used;
3767
3768	if (bargs->usage_min == 0)
3769	user_thresh_min = 0;
3770	else
3771	user_thresh_min = mult_perc(num: cache->length, percent: bargs->usage_min);
3772
3773	if (bargs->usage_max == 0)
3774	user_thresh_max = 1;
3775	else if (bargs->usage_max > 100)
3776	user_thresh_max = cache->length;
3777	else
3778	user_thresh_max = mult_perc(num: cache->length, percent: bargs->usage_max);
3779
3780	if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max)
3781	ret = 0;
3782
3783	btrfs_put_block_group(cache);
3784	return ret;
3785	}
3786
3787	static int chunk_usage_filter(struct btrfs_fs_info *fs_info,
3788	u64 chunk_offset, struct btrfs_balance_args *bargs)
3789	{
3790	struct btrfs_block_group *cache;
3791	u64 chunk_used, user_thresh;
3792	int ret = 1;
3793
3794	cache = btrfs_lookup_block_group(info: fs_info, bytenr: chunk_offset);
3795	chunk_used = cache->used;
3796
3797	if (bargs->usage_min == 0)
3798	user_thresh = 1;
3799	else if (bargs->usage > 100)
3800	user_thresh = cache->length;
3801	else
3802	user_thresh = mult_perc(num: cache->length, percent: bargs->usage);
3803
3804	if (chunk_used < user_thresh)
3805	ret = 0;
3806
3807	btrfs_put_block_group(cache);
3808	return ret;
3809	}
3810
3811	static int chunk_devid_filter(struct extent_buffer *leaf,
3812	struct btrfs_chunk *chunk,
3813	struct btrfs_balance_args *bargs)
3814	{
3815	struct btrfs_stripe *stripe;
3816	int num_stripes = btrfs_chunk_num_stripes(eb: leaf, s: chunk);
3817	int i;
3818
3819	for (i = 0; i < num_stripes; i++) {
3820	stripe = btrfs_stripe_nr(c: chunk, nr: i);
3821	if (btrfs_stripe_devid(eb: leaf, s: stripe) == bargs->devid)
3822	return 0;
3823	}
3824
3825	return 1;
3826	}
3827
3828	static u64 calc_data_stripes(u64 type, int num_stripes)
3829	{
3830	const int index = btrfs_bg_flags_to_raid_index(flags: type);
3831	const int ncopies = btrfs_raid_array[index].ncopies;
3832	const int nparity = btrfs_raid_array[index].nparity;
3833
3834	return (num_stripes - nparity) / ncopies;
3835	}
3836
3837	/* [pstart, pend) */
3838	static int chunk_drange_filter(struct extent_buffer *leaf,
3839	struct btrfs_chunk *chunk,
3840	struct btrfs_balance_args *bargs)
3841	{
3842	struct btrfs_stripe *stripe;
3843	int num_stripes = btrfs_chunk_num_stripes(eb: leaf, s: chunk);
3844	u64 stripe_offset;
3845	u64 stripe_length;
3846	u64 type;
3847	int factor;
3848	int i;
3849
3850	if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID))
3851	return 0;
3852
3853	type = btrfs_chunk_type(eb: leaf, s: chunk);
3854	factor = calc_data_stripes(type, num_stripes);
3855
3856	for (i = 0; i < num_stripes; i++) {
3857	stripe = btrfs_stripe_nr(c: chunk, nr: i);
3858	if (btrfs_stripe_devid(eb: leaf, s: stripe) != bargs->devid)
3859	continue;
3860
3861	stripe_offset = btrfs_stripe_offset(eb: leaf, s: stripe);
3862	stripe_length = btrfs_chunk_length(eb: leaf, s: chunk);
3863	stripe_length = div_u64(dividend: stripe_length, divisor: factor);
3864
3865	if (stripe_offset < bargs->pend &&
3866	stripe_offset + stripe_length > bargs->pstart)
3867	return 0;
3868	}
3869
3870	return 1;
3871	}
3872
3873	/* [vstart, vend) */
3874	static int chunk_vrange_filter(struct extent_buffer *leaf,
3875	struct btrfs_chunk *chunk,
3876	u64 chunk_offset,
3877	struct btrfs_balance_args *bargs)
3878	{
3879	if (chunk_offset < bargs->vend &&
3880	chunk_offset + btrfs_chunk_length(eb: leaf, s: chunk) > bargs->vstart)
3881	/* at least part of the chunk is inside this vrange */
3882	return 0;
3883
3884	return 1;
3885	}
3886
3887	static int chunk_stripes_range_filter(struct extent_buffer *leaf,
3888	struct btrfs_chunk *chunk,
3889	struct btrfs_balance_args *bargs)
3890	{
3891	int num_stripes = btrfs_chunk_num_stripes(eb: leaf, s: chunk);
3892
3893	if (bargs->stripes_min <= num_stripes
3894	&& num_stripes <= bargs->stripes_max)
3895	return 0;
3896
3897	return 1;
3898	}
3899
3900	static int chunk_soft_convert_filter(u64 chunk_type,
3901	struct btrfs_balance_args *bargs)
3902	{
3903	if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
3904	return 0;
3905
3906	chunk_type = chunk_to_extended(flags: chunk_type) &
3907	BTRFS_EXTENDED_PROFILE_MASK;
3908
3909	if (bargs->target == chunk_type)
3910	return 1;
3911
3912	return 0;
3913	}
3914
3915	static int should_balance_chunk(struct extent_buffer *leaf,
3916	struct btrfs_chunk *chunk, u64 chunk_offset)
3917	{
3918	struct btrfs_fs_info *fs_info = leaf->fs_info;
3919	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3920	struct btrfs_balance_args *bargs = NULL;
3921	u64 chunk_type = btrfs_chunk_type(eb: leaf, s: chunk);
3922
3923	/* type filter */
3924	if (!((chunk_type & BTRFS_BLOCK_GROUP_TYPE_MASK) &
3925	(bctl->flags & BTRFS_BALANCE_TYPE_MASK))) {
3926	return 0;
3927	}
3928
3929	if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
3930	bargs = &bctl->data;
3931	else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
3932	bargs = &bctl->sys;
3933	else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
3934	bargs = &bctl->meta;
3935
3936	/* profiles filter */
3937	if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) &&
3938	chunk_profiles_filter(chunk_type, bargs)) {
3939	return 0;
3940	}
3941
3942	/* usage filter */
3943	if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) &&
3944	chunk_usage_filter(fs_info, chunk_offset, bargs)) {
3945	return 0;
3946	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3947	chunk_usage_range_filter(fs_info, chunk_offset, bargs)) {
3948	return 0;
3949	}
3950
3951	/* devid filter */
3952	if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) &&
3953	chunk_devid_filter(leaf, chunk, bargs)) {
3954	return 0;
3955	}
3956
3957	/* drange filter, makes sense only with devid filter */
3958	if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) &&
3959	chunk_drange_filter(leaf, chunk, bargs)) {
3960	return 0;
3961	}
3962
3963	/* vrange filter */
3964	if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) &&
3965	chunk_vrange_filter(leaf, chunk, chunk_offset, bargs)) {
3966	return 0;
3967	}
3968
3969	/* stripes filter */
3970	if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) &&
3971	chunk_stripes_range_filter(leaf, chunk, bargs)) {
3972	return 0;
3973	}
3974
3975	/* soft profile changing mode */
3976	if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) &&
3977	chunk_soft_convert_filter(chunk_type, bargs)) {
3978	return 0;
3979	}
3980
3981	/*
3982	* limited by count, must be the last filter
3983	*/
3984	if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) {
3985	if (bargs->limit == 0)
3986	return 0;
3987	else
3988	bargs->limit--;
3989	} else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) {
3990	/*
3991	* Same logic as the 'limit' filter; the minimum cannot be
3992	* determined here because we do not have the global information
3993	* about the count of all chunks that satisfy the filters.
3994	*/
3995	if (bargs->limit_max == 0)
3996	return 0;
3997	else
3998	bargs->limit_max--;
3999	}
4000
4001	return 1;
4002	}
4003
4004	static int __btrfs_balance(struct btrfs_fs_info *fs_info)
4005	{
4006	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
4007	struct btrfs_root *chunk_root = fs_info->chunk_root;
4008	u64 chunk_type;
4009	struct btrfs_chunk *chunk;
4010	struct btrfs_path *path = NULL;
4011	struct btrfs_key key;
4012	struct btrfs_key found_key;
4013	struct extent_buffer *leaf;
4014	int slot;
4015	int ret;
4016	int enospc_errors = 0;
4017	bool counting = true;
4018	/* The single value limit and min/max limits use the same bytes in the */
4019	u64 limit_data = bctl->data.limit;
4020	u64 limit_meta = bctl->meta.limit;
4021	u64 limit_sys = bctl->sys.limit;
4022	u32 count_data = 0;
4023	u32 count_meta = 0;
4024	u32 count_sys = 0;
4025	int chunk_reserved = 0;
4026
4027	path = btrfs_alloc_path();
4028	if (!path) {
4029	ret = -ENOMEM;
4030	goto error;
4031	}
4032
4033	/* zero out stat counters */
4034	spin_lock(lock: &fs_info->balance_lock);
4035	memset(&bctl->stat, 0, sizeof(bctl->stat));
4036	spin_unlock(lock: &fs_info->balance_lock);
4037	again:
4038	if (!counting) {
4039	/*
4040	* The single value limit and min/max limits use the same bytes
4041	* in the
4042	*/
4043	bctl->data.limit = limit_data;
4044	bctl->meta.limit = limit_meta;
4045	bctl->sys.limit = limit_sys;
4046	}
4047	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
4048	key.offset = (u64)-1;
4049	key.type = BTRFS_CHUNK_ITEM_KEY;
4050
4051	while (1) {
4052	if ((!counting && atomic_read(v: &fs_info->balance_pause_req)) ||
4053	atomic_read(v: &fs_info->balance_cancel_req)) {
4054	ret = -ECANCELED;
4055	goto error;
4056	}
4057
4058	mutex_lock(&fs_info->reclaim_bgs_lock);
4059	ret = btrfs_search_slot(NULL, root: chunk_root, key: &key, p: path, ins_len: 0, cow: 0);
4060	if (ret < 0) {
4061	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4062	goto error;
4063	}
4064
4065	/*
4066	* this shouldn't happen, it means the last relocate
4067	* failed
4068	*/
4069	if (ret == 0)
4070	BUG(); /* FIXME break ? */
4071
4072	ret = btrfs_previous_item(root: chunk_root, path, min_objectid: 0,
4073	BTRFS_CHUNK_ITEM_KEY);
4074	if (ret) {
4075	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4076	ret = 0;
4077	break;
4078	}
4079
4080	leaf = path->nodes[0];
4081	slot = path->slots[0];
4082	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: slot);
4083
4084	if (found_key.objectid != key.objectid) {
4085	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4086	break;
4087	}
4088
4089	chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
4090	chunk_type = btrfs_chunk_type(eb: leaf, s: chunk);
4091
4092	if (!counting) {
4093	spin_lock(lock: &fs_info->balance_lock);
4094	bctl->stat.considered++;
4095	spin_unlock(lock: &fs_info->balance_lock);
4096	}
4097
4098	ret = should_balance_chunk(leaf, chunk, chunk_offset: found_key.offset);
4099
4100	btrfs_release_path(p: path);
4101	if (!ret) {
4102	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4103	goto loop;
4104	}
4105
4106	if (counting) {
4107	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4108	spin_lock(lock: &fs_info->balance_lock);
4109	bctl->stat.expected++;
4110	spin_unlock(lock: &fs_info->balance_lock);
4111
4112	if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
4113	count_data++;
4114	else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
4115	count_sys++;
4116	else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
4117	count_meta++;
4118
4119	goto loop;
4120	}
4121
4122	/*
4123	* Apply limit_min filter, no need to check if the LIMITS
4124	* filter is used, limit_min is 0 by default
4125	*/
4126	if (((chunk_type & BTRFS_BLOCK_GROUP_DATA) &&
4127	count_data < bctl->data.limit_min)
4128	|| ((chunk_type & BTRFS_BLOCK_GROUP_METADATA) &&
4129	count_meta < bctl->meta.limit_min)
4130	|| ((chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) &&
4131	count_sys < bctl->sys.limit_min)) {
4132	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4133	goto loop;
4134	}
4135
4136	if (!chunk_reserved) {
4137	/*
4138	* We may be relocating the only data chunk we have,
4139	* which could potentially end up with losing data's
4140	* raid profile, so lets allocate an empty one in
4141	* advance.
4142	*/
4143	ret = btrfs_may_alloc_data_chunk(fs_info,
4144	chunk_offset: found_key.offset);
4145	if (ret < 0) {
4146	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4147	goto error;
4148	} else if (ret == 1) {
4149	chunk_reserved = 1;
4150	}
4151	}
4152
4153	ret = btrfs_relocate_chunk(fs_info, chunk_offset: found_key.offset);
4154	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
4155	if (ret == -ENOSPC) {
4156	enospc_errors++;
4157	} else if (ret == -ETXTBSY) {
4158	btrfs_info(fs_info,
4159	"skipping relocation of block group %llu due to active swapfile",
4160	found_key.offset);
4161	ret = 0;
4162	} else if (ret) {
4163	goto error;
4164	} else {
4165	spin_lock(lock: &fs_info->balance_lock);
4166	bctl->stat.completed++;
4167	spin_unlock(lock: &fs_info->balance_lock);
4168	}
4169	loop:
4170	if (found_key.offset == 0)
4171	break;
4172	key.offset = found_key.offset - 1;
4173	}
4174
4175	if (counting) {
4176	btrfs_release_path(p: path);
4177	counting = false;
4178	goto again;
4179	}
4180	error:
4181	btrfs_free_path(p: path);
4182	if (enospc_errors) {
4183	btrfs_info(fs_info, "%d enospc errors during balance",
4184	enospc_errors);
4185	if (!ret)
4186	ret = -ENOSPC;
4187	}
4188
4189	return ret;
4190	}
4191
4192	/*
4193	* See if a given profile is valid and reduced.
4194	*
4195	* @flags: profile to validate
4196	* @extended: if true @flags is treated as an extended profile
4197	*/
4198	static int alloc_profile_is_valid(u64 flags, int extended)
4199	{
4200	u64 mask = (extended ? BTRFS_EXTENDED_PROFILE_MASK :
4201	BTRFS_BLOCK_GROUP_PROFILE_MASK);
4202
4203	flags &= ~BTRFS_BLOCK_GROUP_TYPE_MASK;
4204
4205	/* 1) check that all other bits are zeroed */
4206	if (flags & ~mask)
4207	return 0;
4208
4209	/* 2) see if profile is reduced */
4210	if (flags == 0)
4211	return !extended; /* "0" is valid for usual profiles */
4212
4213	return has_single_bit_set(n: flags);
4214	}
4215
4216	/*
4217	* Validate target profile against allowed profiles and return true if it's OK.
4218	* Otherwise print the error message and return false.
4219	*/
4220	static inline int validate_convert_profile(struct btrfs_fs_info *fs_info,
4221	const struct btrfs_balance_args *bargs,
4222	u64 allowed, const char *type)
4223	{
4224	if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
4225	return true;
4226
4227	/* Profile is valid and does not have bits outside of the allowed set */
4228	if (alloc_profile_is_valid(flags: bargs->target, extended: 1) &&
4229	(bargs->target & ~allowed) == 0)
4230	return true;
4231
4232	btrfs_err(fs_info, "balance: invalid convert %s profile %s",
4233	type, btrfs_bg_type_to_raid_name(bargs->target));
4234	return false;
4235	}
4236
4237	/*
4238	* Fill @buf with textual description of balance filter flags @bargs, up to
4239	* @size_buf including the terminating null. The output may be trimmed if it
4240	* does not fit into the provided buffer.
4241	*/
4242	static void describe_balance_args(struct btrfs_balance_args *bargs, char *buf,
4243	u32 size_buf)
4244	{
4245	int ret;
4246	u32 size_bp = size_buf;
4247	char *bp = buf;
4248	u64 flags = bargs->flags;
4249	char tmp_buf[128] = {'\0'};
4250
4251	if (!flags)
4252	return;
4253
4254	#define CHECK_APPEND_NOARG(a) \
4255	do { \
4256	ret = snprintf(bp, size_bp, (a)); \
4257	if (ret < 0 || ret >= size_bp) \
4258	goto out_overflow; \
4259	size_bp -= ret; \
4260	bp += ret; \
4261	} while (0)
4262
4263	#define CHECK_APPEND_1ARG(a, v1) \
4264	do { \
4265	ret = snprintf(bp, size_bp, (a), (v1)); \
4266	if (ret < 0 || ret >= size_bp) \
4267	goto out_overflow; \
4268	size_bp -= ret; \
4269	bp += ret; \
4270	} while (0)
4271
4272	#define CHECK_APPEND_2ARG(a, v1, v2) \
4273	do { \
4274	ret = snprintf(bp, size_bp, (a), (v1), (v2)); \
4275	if (ret < 0 || ret >= size_bp) \
4276	goto out_overflow; \
4277	size_bp -= ret; \
4278	bp += ret; \
4279	} while (0)
4280
4281	if (flags & BTRFS_BALANCE_ARGS_CONVERT)
4282	CHECK_APPEND_1ARG("convert=%s,",
4283	btrfs_bg_type_to_raid_name(bargs->target));
4284
4285	if (flags & BTRFS_BALANCE_ARGS_SOFT)
4286	CHECK_APPEND_NOARG("soft,");
4287
4288	if (flags & BTRFS_BALANCE_ARGS_PROFILES) {
4289	btrfs_describe_block_groups(bg_flags: bargs->profiles, buf: tmp_buf,
4290	size_buf: sizeof(tmp_buf));
4291	CHECK_APPEND_1ARG("profiles=%s,", tmp_buf);
4292	}
4293
4294	if (flags & BTRFS_BALANCE_ARGS_USAGE)
4295	CHECK_APPEND_1ARG("usage=%llu,", bargs->usage);
4296
4297	if (flags & BTRFS_BALANCE_ARGS_USAGE_RANGE)
4298	CHECK_APPEND_2ARG("usage=%u..%u,",
4299	bargs->usage_min, bargs->usage_max);
4300
4301	if (flags & BTRFS_BALANCE_ARGS_DEVID)
4302	CHECK_APPEND_1ARG("devid=%llu,", bargs->devid);
4303
4304	if (flags & BTRFS_BALANCE_ARGS_DRANGE)
4305	CHECK_APPEND_2ARG("drange=%llu..%llu,",
4306	bargs->pstart, bargs->pend);
4307
4308	if (flags & BTRFS_BALANCE_ARGS_VRANGE)
4309	CHECK_APPEND_2ARG("vrange=%llu..%llu,",
4310	bargs->vstart, bargs->vend);
4311
4312	if (flags & BTRFS_BALANCE_ARGS_LIMIT)
4313	CHECK_APPEND_1ARG("limit=%llu,", bargs->limit);
4314
4315	if (flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)
4316	CHECK_APPEND_2ARG("limit=%u..%u,",
4317	bargs->limit_min, bargs->limit_max);
4318
4319	if (flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE)
4320	CHECK_APPEND_2ARG("stripes=%u..%u,",
4321	bargs->stripes_min, bargs->stripes_max);
4322
4323	#undef CHECK_APPEND_2ARG
4324	#undef CHECK_APPEND_1ARG
4325	#undef CHECK_APPEND_NOARG
4326
4327	out_overflow:
4328
4329	if (size_bp < size_buf)
4330	buf[size_buf - size_bp - 1] = '\0'; /* remove last , */
4331	else
4332	buf[0] = '\0';
4333	}
4334
4335	static void describe_balance_start_or_resume(struct btrfs_fs_info *fs_info)
4336	{
4337	u32 size_buf = 1024;
4338	char tmp_buf[192] = {'\0'};
4339	char *buf;
4340	char *bp;
4341	u32 size_bp = size_buf;
4342	int ret;
4343	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
4344
4345	buf = kzalloc(size: size_buf, GFP_KERNEL);
4346	if (!buf)
4347	return;
4348
4349	bp = buf;
4350
4351	#define CHECK_APPEND_1ARG(a, v1) \
4352	do { \
4353	ret = snprintf(bp, size_bp, (a), (v1)); \
4354	if (ret < 0 || ret >= size_bp) \
4355	goto out_overflow; \
4356	size_bp -= ret; \
4357	bp += ret; \
4358	} while (0)
4359
4360	if (bctl->flags & BTRFS_BALANCE_FORCE)
4361	CHECK_APPEND_1ARG("%s", "-f ");
4362
4363	if (bctl->flags & BTRFS_BALANCE_DATA) {
4364	describe_balance_args(bargs: &bctl->data, buf: tmp_buf, size_buf: sizeof(tmp_buf));
4365	CHECK_APPEND_1ARG("-d%s ", tmp_buf);
4366	}
4367
4368	if (bctl->flags & BTRFS_BALANCE_METADATA) {
4369	describe_balance_args(bargs: &bctl->meta, buf: tmp_buf, size_buf: sizeof(tmp_buf));
4370	CHECK_APPEND_1ARG("-m%s ", tmp_buf);
4371	}
4372
4373	if (bctl->flags & BTRFS_BALANCE_SYSTEM) {
4374	describe_balance_args(bargs: &bctl->sys, buf: tmp_buf, size_buf: sizeof(tmp_buf));
4375	CHECK_APPEND_1ARG("-s%s ", tmp_buf);
4376	}
4377
4378	#undef CHECK_APPEND_1ARG
4379
4380	out_overflow:
4381
4382	if (size_bp < size_buf)
4383	buf[size_buf - size_bp - 1] = '\0'; /* remove last " " */
4384	btrfs_info(fs_info, "balance: %s %s",
4385	(bctl->flags & BTRFS_BALANCE_RESUME) ?
4386	"resume" : "start", buf);
4387
4388	kfree(objp: buf);
4389	}
4390
4391	/*
4392	* Should be called with balance mutexe held
4393	*/
4394	int btrfs_balance(struct btrfs_fs_info *fs_info,
4395	struct btrfs_balance_control *bctl,
4396	struct btrfs_ioctl_balance_args *bargs)
4397	{
4398	u64 meta_target, data_target;
4399	u64 allowed;
4400	int mixed = 0;
4401	int ret;
4402	u64 num_devices;
4403	unsigned seq;
4404	bool reducing_redundancy;
4405	bool paused = false;
4406	int i;
4407
4408	if (btrfs_fs_closing(fs_info) ||
4409	atomic_read(v: &fs_info->balance_pause_req) ||
4410	btrfs_should_cancel_balance(fs_info)) {
4411	ret = -EINVAL;
4412	goto out;
4413	}
4414
4415	allowed = btrfs_super_incompat_flags(s: fs_info->super_copy);
4416	if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
4417	mixed = 1;
4418
4419	/*
4420	* In case of mixed groups both data and meta should be picked,
4421	* and identical options should be given for both of them.
4422	*/
4423	allowed = BTRFS_BALANCE_DATA | BTRFS_BALANCE_METADATA;
4424	if (mixed && (bctl->flags & allowed)) {
4425	if (!(bctl->flags & BTRFS_BALANCE_DATA) ||
4426	!(bctl->flags & BTRFS_BALANCE_METADATA) ||
4427	memcmp(p: &bctl->data, q: &bctl->meta, size: sizeof(bctl->data))) {
4428	btrfs_err(fs_info,
4429	"balance: mixed groups data and metadata options must be the same");
4430	ret = -EINVAL;
4431	goto out;
4432	}
4433	}
4434
4435	/*
4436	* rw_devices will not change at the moment, device add/delete/replace
4437	* are exclusive
4438	*/
4439	num_devices = fs_info->fs_devices->rw_devices;
4440
4441	/*
4442	* SINGLE profile on-disk has no profile bit, but in-memory we have a
4443	* special bit for it, to make it easier to distinguish. Thus we need
4444	* to set it manually, or balance would refuse the profile.
4445	*/
4446	allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE;
4447	for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++)
4448	if (num_devices >= btrfs_raid_array[i].devs_min)
4449	allowed |= btrfs_raid_array[i].bg_flag;
4450
4451	if (!validate_convert_profile(fs_info, bargs: &bctl->data, allowed, type: "data") ||
4452	!validate_convert_profile(fs_info, bargs: &bctl->meta, allowed, type: "metadata") ||
4453	!validate_convert_profile(fs_info, bargs: &bctl->sys, allowed, type: "system")) {
4454	ret = -EINVAL;
4455	goto out;
4456	}
4457
4458	/*
4459	* Allow to reduce metadata or system integrity only if force set for
4460	* profiles with redundancy (copies, parity)
4461	*/
4462	allowed = 0;
4463	for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) {
4464	if (btrfs_raid_array[i].ncopies >= 2 ||
4465	btrfs_raid_array[i].tolerated_failures >= 1)
4466	allowed |= btrfs_raid_array[i].bg_flag;
4467	}
4468	do {
4469	seq = read_seqbegin(sl: &fs_info->profiles_lock);
4470
4471	if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4472	(fs_info->avail_system_alloc_bits & allowed) &&
4473	!(bctl->sys.target & allowed)) ||
4474	((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4475	(fs_info->avail_metadata_alloc_bits & allowed) &&
4476	!(bctl->meta.target & allowed)))
4477	reducing_redundancy = true;
4478	else
4479	reducing_redundancy = false;
4480
4481	/* if we're not converting, the target field is uninitialized */
4482	meta_target = (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4483	bctl->meta.target : fs_info->avail_metadata_alloc_bits;
4484	data_target = (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4485	bctl->data.target : fs_info->avail_data_alloc_bits;
4486	} while (read_seqretry(sl: &fs_info->profiles_lock, start: seq));
4487
4488	if (reducing_redundancy) {
4489	if (bctl->flags & BTRFS_BALANCE_FORCE) {
4490	btrfs_info(fs_info,
4491	"balance: force reducing metadata redundancy");
4492	} else {
4493	btrfs_err(fs_info,
4494	"balance: reduces metadata redundancy, use --force if you want this");
4495	ret = -EINVAL;
4496	goto out;
4497	}
4498	}
4499
4500	if (btrfs_get_num_tolerated_disk_barrier_failures(flags: meta_target) <
4501	btrfs_get_num_tolerated_disk_barrier_failures(flags: data_target)) {
4502	btrfs_warn(fs_info,
4503	"balance: metadata profile %s has lower redundancy than data profile %s",
4504	btrfs_bg_type_to_raid_name(meta_target),
4505	btrfs_bg_type_to_raid_name(data_target));
4506	}
4507
4508	ret = insert_balance_item(fs_info, bctl);
4509	if (ret && ret != -EEXIST)
4510	goto out;
4511
4512	if (!(bctl->flags & BTRFS_BALANCE_RESUME)) {
4513	BUG_ON(ret == -EEXIST);
4514	BUG_ON(fs_info->balance_ctl);
4515	spin_lock(lock: &fs_info->balance_lock);
4516	fs_info->balance_ctl = bctl;
4517	spin_unlock(lock: &fs_info->balance_lock);
4518	} else {
4519	BUG_ON(ret != -EEXIST);
4520	spin_lock(lock: &fs_info->balance_lock);
4521	update_balance_args(bctl);
4522	spin_unlock(lock: &fs_info->balance_lock);
4523	}
4524
4525	ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4526	set_bit(nr: BTRFS_FS_BALANCE_RUNNING, addr: &fs_info->flags);
4527	describe_balance_start_or_resume(fs_info);
4528	mutex_unlock(lock: &fs_info->balance_mutex);
4529
4530	ret = __btrfs_balance(fs_info);
4531
4532	mutex_lock(&fs_info->balance_mutex);
4533	if (ret == -ECANCELED && atomic_read(v: &fs_info->balance_pause_req)) {
4534	btrfs_info(fs_info, "balance: paused");
4535	btrfs_exclop_balance(fs_info, op: BTRFS_EXCLOP_BALANCE_PAUSED);
4536	paused = true;
4537	}
4538	/*
4539	* Balance can be canceled by:
4540	*
4541	* - Regular cancel request
4542	* Then ret == -ECANCELED and balance_cancel_req > 0
4543	*
4544	* - Fatal signal to "btrfs" process
4545	* Either the signal caught by wait_reserve_ticket() and callers
4546	* got -EINTR, or caught by btrfs_should_cancel_balance() and
4547	* got -ECANCELED.
4548	* Either way, in this case balance_cancel_req = 0, and
4549	* ret == -EINTR or ret == -ECANCELED.
4550	*
4551	* So here we only check the return value to catch canceled balance.
4552	*/
4553	else if (ret == -ECANCELED || ret == -EINTR)
4554	btrfs_info(fs_info, "balance: canceled");
4555	else
4556	btrfs_info(fs_info, "balance: ended with status: %d", ret);
4557
4558	clear_bit(nr: BTRFS_FS_BALANCE_RUNNING, addr: &fs_info->flags);
4559
4560	if (bargs) {
4561	memset(bargs, 0, sizeof(*bargs));
4562	btrfs_update_ioctl_balance_args(fs_info, bargs);
4563	}
4564
4565	/* We didn't pause, we can clean everything up. */
4566	if (!paused) {
4567	reset_balance_state(fs_info);
4568	btrfs_exclop_finish(fs_info);
4569	}
4570
4571	wake_up(&fs_info->balance_wait_q);
4572
4573	return ret;
4574	out:
4575	if (bctl->flags & BTRFS_BALANCE_RESUME)
4576	reset_balance_state(fs_info);
4577	else
4578	kfree(objp: bctl);
4579	btrfs_exclop_finish(fs_info);
4580
4581	return ret;
4582	}
4583
4584	static int balance_kthread(void *data)
4585	{
4586	struct btrfs_fs_info *fs_info = data;
4587	int ret = 0;
4588
4589	sb_start_write(sb: fs_info->sb);
4590	mutex_lock(&fs_info->balance_mutex);
4591	if (fs_info->balance_ctl)
4592	ret = btrfs_balance(fs_info, bctl: fs_info->balance_ctl, NULL);
4593	mutex_unlock(lock: &fs_info->balance_mutex);
4594	sb_end_write(sb: fs_info->sb);
4595
4596	return ret;
4597	}
4598
4599	int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info)
4600	{
4601	struct task_struct *tsk;
4602
4603	mutex_lock(&fs_info->balance_mutex);
4604	if (!fs_info->balance_ctl) {
4605	mutex_unlock(lock: &fs_info->balance_mutex);
4606	return 0;
4607	}
4608	mutex_unlock(lock: &fs_info->balance_mutex);
4609
4610	if (btrfs_test_opt(fs_info, SKIP_BALANCE)) {
4611	btrfs_info(fs_info, "balance: resume skipped");
4612	return 0;
4613	}
4614
4615	spin_lock(lock: &fs_info->super_lock);
4616	ASSERT(fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED);
4617	fs_info->exclusive_operation = BTRFS_EXCLOP_BALANCE;
4618	spin_unlock(lock: &fs_info->super_lock);
4619	/*
4620	* A ro->rw remount sequence should continue with the paused balance
4621	* regardless of who pauses it, system or the user as of now, so set
4622	* the resume flag.
4623	*/
4624	spin_lock(lock: &fs_info->balance_lock);
4625	fs_info->balance_ctl->flags |= BTRFS_BALANCE_RESUME;
4626	spin_unlock(lock: &fs_info->balance_lock);
4627
4628	tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance");
4629	return PTR_ERR_OR_ZERO(ptr: tsk);
4630	}
4631
4632	int btrfs_recover_balance(struct btrfs_fs_info *fs_info)
4633	{
4634	struct btrfs_balance_control *bctl;
4635	struct btrfs_balance_item *item;
4636	struct btrfs_disk_balance_args disk_bargs;
4637	struct btrfs_path *path;
4638	struct extent_buffer *leaf;
4639	struct btrfs_key key;
4640	int ret;
4641
4642	path = btrfs_alloc_path();
4643	if (!path)
4644	return -ENOMEM;
4645
4646	key.objectid = BTRFS_BALANCE_OBJECTID;
4647	key.type = BTRFS_TEMPORARY_ITEM_KEY;
4648	key.offset = 0;
4649
4650	ret = btrfs_search_slot(NULL, root: fs_info->tree_root, key: &key, p: path, ins_len: 0, cow: 0);
4651	if (ret < 0)
4652	goto out;
4653	if (ret > 0) { /* ret = -ENOENT; */
4654	ret = 0;
4655	goto out;
4656	}
4657
4658	bctl = kzalloc(size: sizeof(*bctl), GFP_NOFS);
4659	if (!bctl) {
4660	ret = -ENOMEM;
4661	goto out;
4662	}
4663
4664	leaf = path->nodes[0];
4665	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
4666
4667	bctl->flags = btrfs_balance_flags(eb: leaf, s: item);
4668	bctl->flags |= BTRFS_BALANCE_RESUME;
4669
4670	btrfs_balance_data(eb: leaf, bi: item, ba: &disk_bargs);
4671	btrfs_disk_balance_args_to_cpu(cpu: &bctl->data, disk: &disk_bargs);
4672	btrfs_balance_meta(eb: leaf, bi: item, ba: &disk_bargs);
4673	btrfs_disk_balance_args_to_cpu(cpu: &bctl->meta, disk: &disk_bargs);
4674	btrfs_balance_sys(eb: leaf, bi: item, ba: &disk_bargs);
4675	btrfs_disk_balance_args_to_cpu(cpu: &bctl->sys, disk: &disk_bargs);
4676
4677	/*
4678	* This should never happen, as the paused balance state is recovered
4679	* during mount without any chance of other exclusive ops to collide.
4680	*
4681	* This gives the exclusive op status to balance and keeps in paused
4682	* state until user intervention (cancel or umount). If the ownership
4683	* cannot be assigned, show a message but do not fail. The balance
4684	* is in a paused state and must have fs_info::balance_ctl properly
4685	* set up.
4686	*/
4687	if (!btrfs_exclop_start(fs_info, type: BTRFS_EXCLOP_BALANCE_PAUSED))
4688	btrfs_warn(fs_info,
4689	"balance: cannot set exclusive op status, resume manually");
4690
4691	btrfs_release_path(p: path);
4692
4693	mutex_lock(&fs_info->balance_mutex);
4694	BUG_ON(fs_info->balance_ctl);
4695	spin_lock(lock: &fs_info->balance_lock);
4696	fs_info->balance_ctl = bctl;
4697	spin_unlock(lock: &fs_info->balance_lock);
4698	mutex_unlock(lock: &fs_info->balance_mutex);
4699	out:
4700	btrfs_free_path(p: path);
4701	return ret;
4702	}
4703
4704	int btrfs_pause_balance(struct btrfs_fs_info *fs_info)
4705	{
4706	int ret = 0;
4707
4708	mutex_lock(&fs_info->balance_mutex);
4709	if (!fs_info->balance_ctl) {
4710	mutex_unlock(lock: &fs_info->balance_mutex);
4711	return -ENOTCONN;
4712	}
4713
4714	if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4715	atomic_inc(v: &fs_info->balance_pause_req);
4716	mutex_unlock(lock: &fs_info->balance_mutex);
4717
4718	wait_event(fs_info->balance_wait_q,
4719	!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4720
4721	mutex_lock(&fs_info->balance_mutex);
4722	/* we are good with balance_ctl ripped off from under us */
4723	BUG_ON(test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4724	atomic_dec(v: &fs_info->balance_pause_req);
4725	} else {
4726	ret = -ENOTCONN;
4727	}
4728
4729	mutex_unlock(lock: &fs_info->balance_mutex);
4730	return ret;
4731	}
4732
4733	int btrfs_cancel_balance(struct btrfs_fs_info *fs_info)
4734	{
4735	mutex_lock(&fs_info->balance_mutex);
4736	if (!fs_info->balance_ctl) {
4737	mutex_unlock(lock: &fs_info->balance_mutex);
4738	return -ENOTCONN;
4739	}
4740
4741	/*
4742	* A paused balance with the item stored on disk can be resumed at
4743	* mount time if the mount is read-write. Otherwise it's still paused
4744	* and we must not allow cancelling as it deletes the item.
4745	*/
4746	if (sb_rdonly(sb: fs_info->sb)) {
4747	mutex_unlock(lock: &fs_info->balance_mutex);
4748	return -EROFS;
4749	}
4750
4751	atomic_inc(v: &fs_info->balance_cancel_req);
4752	/*
4753	* if we are running just wait and return, balance item is
4754	* deleted in btrfs_balance in this case
4755	*/
4756	if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4757	mutex_unlock(lock: &fs_info->balance_mutex);
4758	wait_event(fs_info->balance_wait_q,
4759	!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4760	mutex_lock(&fs_info->balance_mutex);
4761	} else {
4762	mutex_unlock(lock: &fs_info->balance_mutex);
4763	/*
4764	* Lock released to allow other waiters to continue, we'll
4765	* reexamine the status again.
4766	*/
4767	mutex_lock(&fs_info->balance_mutex);
4768
4769	if (fs_info->balance_ctl) {
4770	reset_balance_state(fs_info);
4771	btrfs_exclop_finish(fs_info);
4772	btrfs_info(fs_info, "balance: canceled");
4773	}
4774	}
4775
4776	ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4777	atomic_dec(v: &fs_info->balance_cancel_req);
4778	mutex_unlock(lock: &fs_info->balance_mutex);
4779	return 0;
4780	}
4781
4782	int btrfs_uuid_scan_kthread(void *data)
4783	{
4784	struct btrfs_fs_info *fs_info = data;
4785	struct btrfs_root *root = fs_info->tree_root;
4786	struct btrfs_key key;
4787	struct btrfs_path *path = NULL;
4788	int ret = 0;
4789	struct extent_buffer *eb;
4790	int slot;
4791	struct btrfs_root_item root_item;
4792	u32 item_size;
4793	struct btrfs_trans_handle *trans = NULL;
4794	bool closing = false;
4795
4796	path = btrfs_alloc_path();
4797	if (!path) {
4798	ret = -ENOMEM;
4799	goto out;
4800	}
4801
4802	key.objectid = 0;
4803	key.type = BTRFS_ROOT_ITEM_KEY;
4804	key.offset = 0;
4805
4806	while (1) {
4807	if (btrfs_fs_closing(fs_info)) {
4808	closing = true;
4809	break;
4810	}
4811	ret = btrfs_search_forward(root, min_key: &key, path,
4812	BTRFS_OLDEST_GENERATION);
4813	if (ret) {
4814	if (ret > 0)
4815	ret = 0;
4816	break;
4817	}
4818
4819	if (key.type != BTRFS_ROOT_ITEM_KEY ||
4820	(key.objectid < BTRFS_FIRST_FREE_OBJECTID &&
4821	key.objectid != BTRFS_FS_TREE_OBJECTID) ||
4822	key.objectid > BTRFS_LAST_FREE_OBJECTID)
4823	goto skip;
4824
4825	eb = path->nodes[0];
4826	slot = path->slots[0];
4827	item_size = btrfs_item_size(eb, slot);
4828	if (item_size < sizeof(root_item))
4829	goto skip;
4830
4831	read_extent_buffer(eb, dst: &root_item,
4832	btrfs_item_ptr_offset(eb, slot),
4833	len: (int)sizeof(root_item));
4834	if (btrfs_root_refs(s: &root_item) == 0)
4835	goto skip;
4836
4837	if (!btrfs_is_empty_uuid(uuid: root_item.uuid) ||
4838	!btrfs_is_empty_uuid(uuid: root_item.received_uuid)) {
4839	if (trans)
4840	goto update_tree;
4841
4842	btrfs_release_path(p: path);
4843	/*
4844	* 1 - subvol uuid item
4845	* 1 - received_subvol uuid item
4846	*/
4847	trans = btrfs_start_transaction(root: fs_info->uuid_root, num_items: 2);
4848	if (IS_ERR(ptr: trans)) {
4849	ret = PTR_ERR(ptr: trans);
4850	break;
4851	}
4852	continue;
4853	} else {
4854	goto skip;
4855	}
4856	update_tree:
4857	btrfs_release_path(p: path);
4858	if (!btrfs_is_empty_uuid(uuid: root_item.uuid)) {
4859	ret = btrfs_uuid_tree_add(trans, uuid: root_item.uuid,
4860	BTRFS_UUID_KEY_SUBVOL,
4861	subid: key.objectid);
4862	if (ret < 0) {
4863	btrfs_warn(fs_info, "uuid_tree_add failed %d",
4864	ret);
4865	break;
4866	}
4867	}
4868
4869	if (!btrfs_is_empty_uuid(uuid: root_item.received_uuid)) {
4870	ret = btrfs_uuid_tree_add(trans,
4871	uuid: root_item.received_uuid,
4872	BTRFS_UUID_KEY_RECEIVED_SUBVOL,
4873	subid: key.objectid);
4874	if (ret < 0) {
4875	btrfs_warn(fs_info, "uuid_tree_add failed %d",
4876	ret);
4877	break;
4878	}
4879	}
4880
4881	skip:
4882	btrfs_release_path(p: path);
4883	if (trans) {
4884	ret = btrfs_end_transaction(trans);
4885	trans = NULL;
4886	if (ret)
4887	break;
4888	}
4889
4890	if (key.offset < (u64)-1) {
4891	key.offset++;
4892	} else if (key.type < BTRFS_ROOT_ITEM_KEY) {
4893	key.offset = 0;
4894	key.type = BTRFS_ROOT_ITEM_KEY;
4895	} else if (key.objectid < (u64)-1) {
4896	key.offset = 0;
4897	key.type = BTRFS_ROOT_ITEM_KEY;
4898	key.objectid++;
4899	} else {
4900	break;
4901	}
4902	cond_resched();
4903	}
4904
4905	out:
4906	btrfs_free_path(p: path);
4907	if (trans && !IS_ERR(ptr: trans))
4908	btrfs_end_transaction(trans);
4909	if (ret)
4910	btrfs_warn(fs_info, "btrfs_uuid_scan_kthread failed %d", ret);
4911	else if (!closing)
4912	set_bit(nr: BTRFS_FS_UPDATE_UUID_TREE_GEN, addr: &fs_info->flags);
4913	up(sem: &fs_info->uuid_tree_rescan_sem);
4914	return 0;
4915	}
4916
4917	int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info)
4918	{
4919	struct btrfs_trans_handle *trans;
4920	struct btrfs_root *tree_root = fs_info->tree_root;
4921	struct btrfs_root *uuid_root;
4922	struct task_struct *task;
4923	int ret;
4924
4925	/*
4926	* 1 - root node
4927	* 1 - root item
4928	*/
4929	trans = btrfs_start_transaction(root: tree_root, num_items: 2);
4930	if (IS_ERR(ptr: trans))
4931	return PTR_ERR(ptr: trans);
4932
4933	uuid_root = btrfs_create_tree(trans, BTRFS_UUID_TREE_OBJECTID);
4934	if (IS_ERR(ptr: uuid_root)) {
4935	ret = PTR_ERR(ptr: uuid_root);
4936	btrfs_abort_transaction(trans, ret);
4937	btrfs_end_transaction(trans);
4938	return ret;
4939	}
4940
4941	fs_info->uuid_root = uuid_root;
4942
4943	ret = btrfs_commit_transaction(trans);
4944	if (ret)
4945	return ret;
4946
4947	down(sem: &fs_info->uuid_tree_rescan_sem);
4948	task = kthread_run(btrfs_uuid_scan_kthread, fs_info, "btrfs-uuid");
4949	if (IS_ERR(ptr: task)) {
4950	/* fs_info->update_uuid_tree_gen remains 0 in all error case */
4951	btrfs_warn(fs_info, "failed to start uuid_scan task");
4952	up(sem: &fs_info->uuid_tree_rescan_sem);
4953	return PTR_ERR(ptr: task);
4954	}
4955
4956	return 0;
4957	}
4958
4959	/*
4960	* shrinking a device means finding all of the device extents past
4961	* the new size, and then following the back refs to the chunks.
4962	* The chunk relocation code actually frees the device extent
4963	*/
4964	int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
4965	{
4966	struct btrfs_fs_info *fs_info = device->fs_info;
4967	struct btrfs_root *root = fs_info->dev_root;
4968	struct btrfs_trans_handle *trans;
4969	struct btrfs_dev_extent *dev_extent = NULL;
4970	struct btrfs_path *path;
4971	u64 length;
4972	u64 chunk_offset;
4973	int ret;
4974	int slot;
4975	int failed = 0;
4976	bool retried = false;
4977	struct extent_buffer *l;
4978	struct btrfs_key key;
4979	struct btrfs_super_block *super_copy = fs_info->super_copy;
4980	u64 old_total = btrfs_super_total_bytes(s: super_copy);
4981	u64 old_size = btrfs_device_get_total_bytes(dev: device);
4982	u64 diff;
4983	u64 start;
4984	u64 free_diff = 0;
4985
4986	new_size = round_down(new_size, fs_info->sectorsize);
4987	start = new_size;
4988	diff = round_down(old_size - new_size, fs_info->sectorsize);
4989
4990	if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
4991	return -EINVAL;
4992
4993	path = btrfs_alloc_path();
4994	if (!path)
4995	return -ENOMEM;
4996
4997	path->reada = READA_BACK;
4998
4999	trans = btrfs_start_transaction(root, num_items: 0);
5000	if (IS_ERR(ptr: trans)) {
5001	btrfs_free_path(p: path);
5002	return PTR_ERR(ptr: trans);
5003	}
5004
5005	mutex_lock(&fs_info->chunk_mutex);
5006
5007	btrfs_device_set_total_bytes(dev: device, size: new_size);
5008	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
5009	device->fs_devices->total_rw_bytes -= diff;
5010
5011	/*
5012	* The new free_chunk_space is new_size - used, so we have to
5013	* subtract the delta of the old free_chunk_space which included
5014	* old_size - used. If used > new_size then just subtract this
5015	* entire device's free space.
5016	*/
5017	if (device->bytes_used < new_size)
5018	free_diff = (old_size - device->bytes_used) -
5019	(new_size - device->bytes_used);
5020	else
5021	free_diff = old_size - device->bytes_used;
5022	atomic64_sub(i: free_diff, v: &fs_info->free_chunk_space);
5023	}
5024
5025	/*
5026	* Once the device's size has been set to the new size, ensure all
5027	* in-memory chunks are synced to disk so that the loop below sees them
5028	* and relocates them accordingly.
5029	*/
5030	if (contains_pending_extent(device, start: &start, len: diff)) {
5031	mutex_unlock(lock: &fs_info->chunk_mutex);
5032	ret = btrfs_commit_transaction(trans);
5033	if (ret)
5034	goto done;
5035	} else {
5036	mutex_unlock(lock: &fs_info->chunk_mutex);
5037	btrfs_end_transaction(trans);
5038	}
5039
5040	again:
5041	key.objectid = device->devid;
5042	key.offset = (u64)-1;
5043	key.type = BTRFS_DEV_EXTENT_KEY;
5044
5045	do {
5046	mutex_lock(&fs_info->reclaim_bgs_lock);
5047	ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
5048	if (ret < 0) {
5049	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
5050	goto done;
5051	}
5052
5053	ret = btrfs_previous_item(root, path, min_objectid: 0, type: key.type);
5054	if (ret) {
5055	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
5056	if (ret < 0)
5057	goto done;
5058	ret = 0;
5059	btrfs_release_path(p: path);
5060	break;
5061	}
5062
5063	l = path->nodes[0];
5064	slot = path->slots[0];
5065	btrfs_item_key_to_cpu(eb: l, cpu_key: &key, nr: path->slots[0]);
5066
5067	if (key.objectid != device->devid) {
5068	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
5069	btrfs_release_path(p: path);
5070	break;
5071	}
5072
5073	dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
5074	length = btrfs_dev_extent_length(eb: l, s: dev_extent);
5075
5076	if (key.offset + length <= new_size) {
5077	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
5078	btrfs_release_path(p: path);
5079	break;
5080	}
5081
5082	chunk_offset = btrfs_dev_extent_chunk_offset(eb: l, s: dev_extent);
5083	btrfs_release_path(p: path);
5084
5085	/*
5086	* We may be relocating the only data chunk we have,
5087	* which could potentially end up with losing data's
5088	* raid profile, so lets allocate an empty one in
5089	* advance.
5090	*/
5091	ret = btrfs_may_alloc_data_chunk(fs_info, chunk_offset);
5092	if (ret < 0) {
5093	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
5094	goto done;
5095	}
5096
5097	ret = btrfs_relocate_chunk(fs_info, chunk_offset);
5098	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
5099	if (ret == -ENOSPC) {
5100	failed++;
5101	} else if (ret) {
5102	if (ret == -ETXTBSY) {
5103	btrfs_warn(fs_info,
5104	"could not shrink block group %llu due to active swapfile",
5105	chunk_offset);
5106	}
5107	goto done;
5108	}
5109	} while (key.offset-- > 0);
5110
5111	if (failed && !retried) {
5112	failed = 0;
5113	retried = true;
5114	goto again;
5115	} else if (failed && retried) {
5116	ret = -ENOSPC;
5117	goto done;
5118	}
5119
5120	/* Shrinking succeeded, else we would be at "done". */
5121	trans = btrfs_start_transaction(root, num_items: 0);
5122	if (IS_ERR(ptr: trans)) {
5123	ret = PTR_ERR(ptr: trans);
5124	goto done;
5125	}
5126
5127	mutex_lock(&fs_info->chunk_mutex);
5128	/* Clear all state bits beyond the shrunk device size */
5129	clear_extent_bits(tree: &device->alloc_state, start: new_size, end: (u64)-1,
5130	CHUNK_STATE_MASK);
5131
5132	btrfs_device_set_disk_total_bytes(dev: device, size: new_size);
5133	if (list_empty(head: &device->post_commit_list))
5134	list_add_tail(new: &device->post_commit_list,
5135	head: &trans->transaction->dev_update_list);
5136
5137	WARN_ON(diff > old_total);
5138	btrfs_set_super_total_bytes(s: super_copy,
5139	round_down(old_total - diff, fs_info->sectorsize));
5140	mutex_unlock(lock: &fs_info->chunk_mutex);
5141
5142	btrfs_reserve_chunk_metadata(trans, is_item_insertion: false);
5143	/* Now btrfs_update_device() will change the on-disk size. */
5144	ret = btrfs_update_device(trans, device);
5145	btrfs_trans_release_chunk_metadata(trans);
5146	if (ret < 0) {
5147	btrfs_abort_transaction(trans, ret);
5148	btrfs_end_transaction(trans);
5149	} else {
5150	ret = btrfs_commit_transaction(trans);
5151	}
5152	done:
5153	btrfs_free_path(p: path);
5154	if (ret) {
5155	mutex_lock(&fs_info->chunk_mutex);
5156	btrfs_device_set_total_bytes(dev: device, size: old_size);
5157	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
5158	device->fs_devices->total_rw_bytes += diff;
5159	atomic64_add(i: free_diff, v: &fs_info->free_chunk_space);
5160	}
5161	mutex_unlock(lock: &fs_info->chunk_mutex);
5162	}
5163	return ret;
5164	}
5165
5166	static int btrfs_add_system_chunk(struct btrfs_fs_info *fs_info,
5167	struct btrfs_key *key,
5168	struct btrfs_chunk *chunk, int item_size)
5169	{
5170	struct btrfs_super_block *super_copy = fs_info->super_copy;
5171	struct btrfs_disk_key disk_key;
5172	u32 array_size;
5173	u8 *ptr;
5174
5175	lockdep_assert_held(&fs_info->chunk_mutex);
5176
5177	array_size = btrfs_super_sys_array_size(s: super_copy);
5178	if (array_size + item_size + sizeof(disk_key)
5179	> BTRFS_SYSTEM_CHUNK_ARRAY_SIZE)
5180	return -EFBIG;
5181
5182	ptr = super_copy->sys_chunk_array + array_size;
5183	btrfs_cpu_key_to_disk(disk_key: &disk_key, cpu_key: key);
5184	memcpy(ptr, &disk_key, sizeof(disk_key));
5185	ptr += sizeof(disk_key);
5186	memcpy(ptr, chunk, item_size);
5187	item_size += sizeof(disk_key);
5188	btrfs_set_super_sys_array_size(s: super_copy, val: array_size + item_size);
5189
5190	return 0;
5191	}
5192
5193	/*
5194	* sort the devices in descending order by max_avail, total_avail
5195	*/
5196	static int btrfs_cmp_device_info(const void *a, const void *b)
5197	{
5198	const struct btrfs_device_info *di_a = a;
5199	const struct btrfs_device_info *di_b = b;
5200
5201	if (di_a->max_avail > di_b->max_avail)
5202	return -1;
5203	if (di_a->max_avail < di_b->max_avail)
5204	return 1;
5205	if (di_a->total_avail > di_b->total_avail)
5206	return -1;
5207	if (di_a->total_avail < di_b->total_avail)
5208	return 1;
5209	return 0;
5210	}
5211
5212	static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type)
5213	{
5214	if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK))
5215	return;
5216
5217	btrfs_set_fs_incompat(info, RAID56);
5218	}
5219
5220	static void check_raid1c34_incompat_flag(struct btrfs_fs_info *info, u64 type)
5221	{
5222	if (!(type & (BTRFS_BLOCK_GROUP_RAID1C3 | BTRFS_BLOCK_GROUP_RAID1C4)))
5223	return;
5224
5225	btrfs_set_fs_incompat(info, RAID1C34);
5226	}
5227
5228	/*
5229	* Structure used internally for btrfs_create_chunk() function.
5230	* Wraps needed parameters.
5231	*/
5232	struct alloc_chunk_ctl {
5233	u64 start;
5234	u64 type;
5235	/* Total number of stripes to allocate */
5236	int num_stripes;
5237	/* sub_stripes info for map */
5238	int sub_stripes;
5239	/* Stripes per device */
5240	int dev_stripes;
5241	/* Maximum number of devices to use */
5242	int devs_max;
5243	/* Minimum number of devices to use */
5244	int devs_min;
5245	/* ndevs has to be a multiple of this */
5246	int devs_increment;
5247	/* Number of copies */
5248	int ncopies;
5249	/* Number of stripes worth of bytes to store parity information */
5250	int nparity;
5251	u64 max_stripe_size;
5252	u64 max_chunk_size;
5253	u64 dev_extent_min;
5254	u64 stripe_size;
5255	u64 chunk_size;
5256	int ndevs;
5257	};
5258
5259	static void init_alloc_chunk_ctl_policy_regular(
5260	struct btrfs_fs_devices *fs_devices,
5261	struct alloc_chunk_ctl *ctl)
5262	{
5263	struct btrfs_space_info *space_info;
5264
5265	space_info = btrfs_find_space_info(info: fs_devices->fs_info, flags: ctl->type);
5266	ASSERT(space_info);
5267
5268	ctl->max_chunk_size = READ_ONCE(space_info->chunk_size);
5269	ctl->max_stripe_size = min_t(u64, ctl->max_chunk_size, SZ_1G);
5270
5271	if (ctl->type & BTRFS_BLOCK_GROUP_SYSTEM)
5272	ctl->devs_max = min_t(int, ctl->devs_max, BTRFS_MAX_DEVS_SYS_CHUNK);
5273
5274	/* We don't want a chunk larger than 10% of writable space */
5275	ctl->max_chunk_size = min(mult_perc(fs_devices->total_rw_bytes, 10),
5276	ctl->max_chunk_size);
5277	ctl->dev_extent_min = btrfs_stripe_nr_to_offset(stripe_nr: ctl->dev_stripes);
5278	}
5279
5280	static void init_alloc_chunk_ctl_policy_zoned(
5281	struct btrfs_fs_devices *fs_devices,
5282	struct alloc_chunk_ctl *ctl)
5283	{
5284	u64 zone_size = fs_devices->fs_info->zone_size;
5285	u64 limit;
5286	int min_num_stripes = ctl->devs_min * ctl->dev_stripes;
5287	int min_data_stripes = (min_num_stripes - ctl->nparity) / ctl->ncopies;
5288	u64 min_chunk_size = min_data_stripes * zone_size;
5289	u64 type = ctl->type;
5290
5291	ctl->max_stripe_size = zone_size;
5292	if (type & BTRFS_BLOCK_GROUP_DATA) {
5293	ctl->max_chunk_size = round_down(BTRFS_MAX_DATA_CHUNK_SIZE,
5294	zone_size);
5295	} else if (type & BTRFS_BLOCK_GROUP_METADATA) {
5296	ctl->max_chunk_size = ctl->max_stripe_size;
5297	} else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
5298	ctl->max_chunk_size = 2 * ctl->max_stripe_size;
5299	ctl->devs_max = min_t(int, ctl->devs_max,
5300	BTRFS_MAX_DEVS_SYS_CHUNK);
5301	} else {
5302	BUG();
5303	}
5304
5305	/* We don't want a chunk larger than 10% of writable space */
5306	limit = max(round_down(mult_perc(fs_devices->total_rw_bytes, 10),
5307	zone_size),
5308	min_chunk_size);
5309	ctl->max_chunk_size = min(limit, ctl->max_chunk_size);
5310	ctl->dev_extent_min = zone_size * ctl->dev_stripes;
5311	}
5312
5313	static void init_alloc_chunk_ctl(struct btrfs_fs_devices *fs_devices,
5314	struct alloc_chunk_ctl *ctl)
5315	{
5316	int index = btrfs_bg_flags_to_raid_index(flags: ctl->type);
5317
5318	ctl->sub_stripes = btrfs_raid_array[index].sub_stripes;
5319	ctl->dev_stripes = btrfs_raid_array[index].dev_stripes;
5320	ctl->devs_max = btrfs_raid_array[index].devs_max;
5321	if (!ctl->devs_max)
5322	ctl->devs_max = BTRFS_MAX_DEVS(fs_devices->fs_info);
5323	ctl->devs_min = btrfs_raid_array[index].devs_min;
5324	ctl->devs_increment = btrfs_raid_array[index].devs_increment;
5325	ctl->ncopies = btrfs_raid_array[index].ncopies;
5326	ctl->nparity = btrfs_raid_array[index].nparity;
5327	ctl->ndevs = 0;
5328
5329	switch (fs_devices->chunk_alloc_policy) {
5330	case BTRFS_CHUNK_ALLOC_REGULAR:
5331	init_alloc_chunk_ctl_policy_regular(fs_devices, ctl);
5332	break;
5333	case BTRFS_CHUNK_ALLOC_ZONED:
5334	init_alloc_chunk_ctl_policy_zoned(fs_devices, ctl);
5335	break;
5336	default:
5337	BUG();
5338	}
5339	}
5340
5341	static int gather_device_info(struct btrfs_fs_devices *fs_devices,
5342	struct alloc_chunk_ctl *ctl,
5343	struct btrfs_device_info *devices_info)
5344	{
5345	struct btrfs_fs_info *info = fs_devices->fs_info;
5346	struct btrfs_device *device;
5347	u64 total_avail;
5348	u64 dev_extent_want = ctl->max_stripe_size * ctl->dev_stripes;
5349	int ret;
5350	int ndevs = 0;
5351	u64 max_avail;
5352	u64 dev_offset;
5353
5354	/*
5355	* in the first pass through the devices list, we gather information
5356	* about the available holes on each device.
5357	*/
5358	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
5359	if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
5360	WARN(1, KERN_ERR
5361	"BTRFS: read-only device in alloc_list\n");
5362	continue;
5363	}
5364
5365	if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
5366	&device->dev_state) ||
5367	test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
5368	continue;
5369
5370	if (device->total_bytes > device->bytes_used)
5371	total_avail = device->total_bytes - device->bytes_used;
5372	else
5373	total_avail = 0;
5374
5375	/* If there is no space on this device, skip it. */
5376	if (total_avail < ctl->dev_extent_min)
5377	continue;
5378
5379	ret = find_free_dev_extent(device, num_bytes: dev_extent_want, start: &dev_offset,
5380	len: &max_avail);
5381	if (ret && ret != -ENOSPC)
5382	return ret;
5383
5384	if (ret == 0)
5385	max_avail = dev_extent_want;
5386
5387	if (max_avail < ctl->dev_extent_min) {
5388	if (btrfs_test_opt(info, ENOSPC_DEBUG))
5389	btrfs_debug(info,
5390	"%s: devid %llu has no free space, have=%llu want=%llu",
5391	__func__, device->devid, max_avail,
5392	ctl->dev_extent_min);
5393	continue;
5394	}
5395
5396	if (ndevs == fs_devices->rw_devices) {
5397	WARN(1, "%s: found more than %llu devices\n",
5398	__func__, fs_devices->rw_devices);
5399	break;
5400	}
5401	devices_info[ndevs].dev_offset = dev_offset;
5402	devices_info[ndevs].max_avail = max_avail;
5403	devices_info[ndevs].total_avail = total_avail;
5404	devices_info[ndevs].dev = device;
5405	++ndevs;
5406	}
5407	ctl->ndevs = ndevs;
5408
5409	/*
5410	* now sort the devices by hole size / available space
5411	*/
5412	sort(base: devices_info, num: ndevs, size: sizeof(struct btrfs_device_info),
5413	cmp_func: btrfs_cmp_device_info, NULL);
5414
5415	return 0;
5416	}
5417
5418	static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl,
5419	struct btrfs_device_info *devices_info)
5420	{
5421	/* Number of stripes that count for block group size */
5422	int data_stripes;
5423
5424	/*
5425	* The primary goal is to maximize the number of stripes, so use as
5426	* many devices as possible, even if the stripes are not maximum sized.
5427	*
5428	* The DUP profile stores more than one stripe per device, the
5429	* max_avail is the total size so we have to adjust.
5430	*/
5431	ctl->stripe_size = div_u64(dividend: devices_info[ctl->ndevs - 1].max_avail,
5432	divisor: ctl->dev_stripes);
5433	ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5434
5435	/* This will have to be fixed for RAID1 and RAID10 over more drives */
5436	data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5437
5438	/*
5439	* Use the number of data stripes to figure out how big this chunk is
5440	* really going to be in terms of logical address space, and compare
5441	* that answer with the max chunk size. If it's higher, we try to
5442	* reduce stripe_size.
5443	*/
5444	if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
5445	/*
5446	* Reduce stripe_size, round it up to a 16MB boundary again and
5447	* then use it, unless it ends up being even bigger than the
5448	* previous value we had already.
5449	*/
5450	ctl->stripe_size = min(round_up(div_u64(ctl->max_chunk_size,
5451	data_stripes), SZ_16M),
5452	ctl->stripe_size);
5453	}
5454
5455	/* Stripe size should not go beyond 1G. */
5456	ctl->stripe_size = min_t(u64, ctl->stripe_size, SZ_1G);
5457
5458	/* Align to BTRFS_STRIPE_LEN */
5459	ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN);
5460	ctl->chunk_size = ctl->stripe_size * data_stripes;
5461
5462	return 0;
5463	}
5464
5465	static int decide_stripe_size_zoned(struct alloc_chunk_ctl *ctl,
5466	struct btrfs_device_info *devices_info)
5467	{
5468	u64 zone_size = devices_info[0].dev->zone_info->zone_size;
5469	/* Number of stripes that count for block group size */
5470	int data_stripes;
5471
5472	/*
5473	* It should hold because:
5474	* dev_extent_min == dev_extent_want == zone_size * dev_stripes
5475	*/
5476	ASSERT(devices_info[ctl->ndevs - 1].max_avail == ctl->dev_extent_min);
5477
5478	ctl->stripe_size = zone_size;
5479	ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5480	data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5481
5482	/* stripe_size is fixed in zoned filesysmte. Reduce ndevs instead. */
5483	if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
5484	ctl->ndevs = div_u64(dividend: div_u64(dividend: ctl->max_chunk_size * ctl->ncopies,
5485	divisor: ctl->stripe_size) + ctl->nparity,
5486	divisor: ctl->dev_stripes);
5487	ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
5488	data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
5489	ASSERT(ctl->stripe_size * data_stripes <= ctl->max_chunk_size);
5490	}
5491
5492	ctl->chunk_size = ctl->stripe_size * data_stripes;
5493
5494	return 0;
5495	}
5496
5497	static int decide_stripe_size(struct btrfs_fs_devices *fs_devices,
5498	struct alloc_chunk_ctl *ctl,
5499	struct btrfs_device_info *devices_info)
5500	{
5501	struct btrfs_fs_info *info = fs_devices->fs_info;
5502
5503	/*
5504	* Round down to number of usable stripes, devs_increment can be any
5505	* number so we can't use round_down() that requires power of 2, while
5506	* rounddown is safe.
5507	*/
5508	ctl->ndevs = rounddown(ctl->ndevs, ctl->devs_increment);
5509
5510	if (ctl->ndevs < ctl->devs_min) {
5511	if (btrfs_test_opt(info, ENOSPC_DEBUG)) {
5512	btrfs_debug(info,
5513	"%s: not enough devices with free space: have=%d minimum required=%d",
5514	__func__, ctl->ndevs, ctl->devs_min);
5515	}
5516	return -ENOSPC;
5517	}
5518
5519	ctl->ndevs = min(ctl->ndevs, ctl->devs_max);
5520
5521	switch (fs_devices->chunk_alloc_policy) {
5522	case BTRFS_CHUNK_ALLOC_REGULAR:
5523	return decide_stripe_size_regular(ctl, devices_info);
5524	case BTRFS_CHUNK_ALLOC_ZONED:
5525	return decide_stripe_size_zoned(ctl, devices_info);
5526	default:
5527	BUG();
5528	}
5529	}
5530
5531	static void chunk_map_device_set_bits(struct btrfs_chunk_map *map, unsigned int bits)
5532	{
5533	for (int i = 0; i < map->num_stripes; i++) {
5534	struct btrfs_io_stripe *stripe = &map->stripes[i];
5535	struct btrfs_device *device = stripe->dev;
5536
5537	set_extent_bit(tree: &device->alloc_state, start: stripe->physical,
5538	end: stripe->physical + map->stripe_size - 1,
5539	bits: bits | EXTENT_NOWAIT, NULL);
5540	}
5541	}
5542
5543	static void chunk_map_device_clear_bits(struct btrfs_chunk_map *map, unsigned int bits)
5544	{
5545	for (int i = 0; i < map->num_stripes; i++) {
5546	struct btrfs_io_stripe *stripe = &map->stripes[i];
5547	struct btrfs_device *device = stripe->dev;
5548
5549	__clear_extent_bit(tree: &device->alloc_state, start: stripe->physical,
5550	end: stripe->physical + map->stripe_size - 1,
5551	bits: bits | EXTENT_NOWAIT,
5552	NULL, NULL);
5553	}
5554	}
5555
5556	void btrfs_remove_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map)
5557	{
5558	write_lock(&fs_info->mapping_tree_lock);
5559	rb_erase_cached(node: &map->rb_node, root: &fs_info->mapping_tree);
5560	RB_CLEAR_NODE(&map->rb_node);
5561	chunk_map_device_clear_bits(map, CHUNK_ALLOCATED);
5562	write_unlock(&fs_info->mapping_tree_lock);
5563
5564	/* Once for the tree reference. */
5565	btrfs_free_chunk_map(map);
5566	}
5567
5568	EXPORT_FOR_TESTS
5569	int btrfs_add_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map)
5570	{
5571	struct rb_node **p;
5572	struct rb_node *parent = NULL;
5573	bool leftmost = true;
5574
5575	write_lock(&fs_info->mapping_tree_lock);
5576	p = &fs_info->mapping_tree.rb_root.rb_node;
5577	while (*p) {
5578	struct btrfs_chunk_map *entry;
5579
5580	parent = *p;
5581	entry = rb_entry(parent, struct btrfs_chunk_map, rb_node);
5582
5583	if (map->start < entry->start) {
5584	p = &(*p)->rb_left;
5585	} else if (map->start > entry->start) {
5586	p = &(*p)->rb_right;
5587	leftmost = false;
5588	} else {
5589	write_unlock(&fs_info->mapping_tree_lock);
5590	return -EEXIST;
5591	}
5592	}
5593	rb_link_node(node: &map->rb_node, parent, rb_link: p);
5594	rb_insert_color_cached(node: &map->rb_node, root: &fs_info->mapping_tree, leftmost);
5595	chunk_map_device_set_bits(map, CHUNK_ALLOCATED);
5596	chunk_map_device_clear_bits(map, CHUNK_TRIMMED);
5597	write_unlock(&fs_info->mapping_tree_lock);
5598
5599	return 0;
5600	}
5601
5602	EXPORT_FOR_TESTS
5603	struct btrfs_chunk_map *btrfs_alloc_chunk_map(int num_stripes, gfp_t gfp)
5604	{
5605	struct btrfs_chunk_map *map;
5606
5607	map = kmalloc(btrfs_chunk_map_size(num_stripes), flags: gfp);
5608	if (!map)
5609	return NULL;
5610
5611	refcount_set(r: &map->refs, n: 1);
5612	RB_CLEAR_NODE(&map->rb_node);
5613
5614	return map;
5615	}
5616
5617	struct btrfs_chunk_map *btrfs_clone_chunk_map(struct btrfs_chunk_map *map, gfp_t gfp)
5618	{
5619	const int size = btrfs_chunk_map_size(map->num_stripes);
5620	struct btrfs_chunk_map *clone;
5621
5622	clone = kmemdup(p: map, size, gfp);
5623	if (!clone)
5624	return NULL;
5625
5626	refcount_set(r: &clone->refs, n: 1);
5627	RB_CLEAR_NODE(&clone->rb_node);
5628
5629	return clone;
5630	}
5631
5632	static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans,
5633	struct alloc_chunk_ctl *ctl,
5634	struct btrfs_device_info *devices_info)
5635	{
5636	struct btrfs_fs_info *info = trans->fs_info;
5637	struct btrfs_chunk_map *map;
5638	struct btrfs_block_group *block_group;
5639	u64 start = ctl->start;
5640	u64 type = ctl->type;
5641	int ret;
5642	int i;
5643	int j;
5644
5645	map = btrfs_alloc_chunk_map(num_stripes: ctl->num_stripes, GFP_NOFS);
5646	if (!map)
5647	return ERR_PTR(error: -ENOMEM);
5648
5649	map->start = start;
5650	map->chunk_len = ctl->chunk_size;
5651	map->stripe_size = ctl->stripe_size;
5652	map->type = type;
5653	map->io_align = BTRFS_STRIPE_LEN;
5654	map->io_width = BTRFS_STRIPE_LEN;
5655	map->sub_stripes = ctl->sub_stripes;
5656	map->num_stripes = ctl->num_stripes;
5657
5658	for (i = 0; i < ctl->ndevs; ++i) {
5659	for (j = 0; j < ctl->dev_stripes; ++j) {
5660	int s = i * ctl->dev_stripes + j;
5661	map->stripes[s].dev = devices_info[i].dev;
5662	map->stripes[s].physical = devices_info[i].dev_offset +
5663	j * ctl->stripe_size;
5664	}
5665	}
5666
5667	trace_btrfs_chunk_alloc(fs_info: info, map, offset: start, size: ctl->chunk_size);
5668
5669	ret = btrfs_add_chunk_map(fs_info: info, map);
5670	if (ret) {
5671	btrfs_free_chunk_map(map);
5672	return ERR_PTR(error: ret);
5673	}
5674
5675	block_group = btrfs_make_block_group(trans, type, chunk_offset: start, size: ctl->chunk_size);
5676	if (IS_ERR(ptr: block_group)) {
5677	btrfs_remove_chunk_map(fs_info: info, map);
5678	return block_group;
5679	}
5680
5681	for (int i = 0; i < map->num_stripes; i++) {
5682	struct btrfs_device *dev = map->stripes[i].dev;
5683
5684	btrfs_device_set_bytes_used(dev,
5685	size: dev->bytes_used + ctl->stripe_size);
5686	if (list_empty(head: &dev->post_commit_list))
5687	list_add_tail(new: &dev->post_commit_list,
5688	head: &trans->transaction->dev_update_list);
5689	}
5690
5691	atomic64_sub(i: ctl->stripe_size * map->num_stripes,
5692	v: &info->free_chunk_space);
5693
5694	check_raid56_incompat_flag(info, type);
5695	check_raid1c34_incompat_flag(info, type);
5696
5697	return block_group;
5698	}
5699
5700	struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans,
5701	u64 type)
5702	{
5703	struct btrfs_fs_info *info = trans->fs_info;
5704	struct btrfs_fs_devices *fs_devices = info->fs_devices;
5705	struct btrfs_device_info *devices_info = NULL;
5706	struct alloc_chunk_ctl ctl;
5707	struct btrfs_block_group *block_group;
5708	int ret;
5709
5710	lockdep_assert_held(&info->chunk_mutex);
5711
5712	if (!alloc_profile_is_valid(flags: type, extended: 0)) {
5713	ASSERT(0);
5714	return ERR_PTR(error: -EINVAL);
5715	}
5716
5717	if (list_empty(head: &fs_devices->alloc_list)) {
5718	if (btrfs_test_opt(info, ENOSPC_DEBUG))
5719	btrfs_debug(info, "%s: no writable device", __func__);
5720	return ERR_PTR(error: -ENOSPC);
5721	}
5722
5723	if (!(type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
5724	btrfs_err(info, "invalid chunk type 0x%llx requested", type);
5725	ASSERT(0);
5726	return ERR_PTR(error: -EINVAL);
5727	}
5728
5729	ctl.start = find_next_chunk(fs_info: info);
5730	ctl.type = type;
5731	init_alloc_chunk_ctl(fs_devices, ctl: &ctl);
5732
5733	devices_info = kcalloc(n: fs_devices->rw_devices, size: sizeof(*devices_info),
5734	GFP_NOFS);
5735	if (!devices_info)
5736	return ERR_PTR(error: -ENOMEM);
5737
5738	ret = gather_device_info(fs_devices, ctl: &ctl, devices_info);
5739	if (ret < 0) {
5740	block_group = ERR_PTR(error: ret);
5741	goto out;
5742	}
5743
5744	ret = decide_stripe_size(fs_devices, ctl: &ctl, devices_info);
5745	if (ret < 0) {
5746	block_group = ERR_PTR(error: ret);
5747	goto out;
5748	}
5749
5750	block_group = create_chunk(trans, ctl: &ctl, devices_info);
5751
5752	out:
5753	kfree(objp: devices_info);
5754	return block_group;
5755	}
5756
5757	/*
5758	* This function, btrfs_chunk_alloc_add_chunk_item(), typically belongs to the
5759	* phase 1 of chunk allocation. It belongs to phase 2 only when allocating system
5760	* chunks.
5761	*
5762	* See the comment at btrfs_chunk_alloc() for details about the chunk allocation
5763	* phases.
5764	*/
5765	int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans,
5766	struct btrfs_block_group *bg)
5767	{
5768	struct btrfs_fs_info *fs_info = trans->fs_info;
5769	struct btrfs_root *chunk_root = fs_info->chunk_root;
5770	struct btrfs_key key;
5771	struct btrfs_chunk *chunk;
5772	struct btrfs_stripe *stripe;
5773	struct btrfs_chunk_map *map;
5774	size_t item_size;
5775	int i;
5776	int ret;
5777
5778	/*
5779	* We take the chunk_mutex for 2 reasons:
5780	*
5781	* 1) Updates and insertions in the chunk btree must be done while holding
5782	* the chunk_mutex, as well as updating the system chunk array in the
5783	* superblock. See the comment on top of btrfs_chunk_alloc() for the
5784	* details;
5785	*
5786	* 2) To prevent races with the final phase of a device replace operation
5787	* that replaces the device object associated with the map's stripes,
5788	* because the device object's id can change at any time during that
5789	* final phase of the device replace operation
5790	* (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the
5791	* replaced device and then see it with an ID of BTRFS_DEV_REPLACE_DEVID,
5792	* which would cause a failure when updating the device item, which does
5793	* not exists, or persisting a stripe of the chunk item with such ID.
5794	* Here we can't use the device_list_mutex because our caller already
5795	* has locked the chunk_mutex, and the final phase of device replace
5796	* acquires both mutexes - first the device_list_mutex and then the
5797	* chunk_mutex. Using any of those two mutexes protects us from a
5798	* concurrent device replace.
5799	*/
5800	lockdep_assert_held(&fs_info->chunk_mutex);
5801
5802	map = btrfs_get_chunk_map(fs_info, logical: bg->start, length: bg->length);
5803	if (IS_ERR(ptr: map)) {
5804	ret = PTR_ERR(ptr: map);
5805	btrfs_abort_transaction(trans, ret);
5806	return ret;
5807	}
5808
5809	item_size = btrfs_chunk_item_size(num_stripes: map->num_stripes);
5810
5811	chunk = kzalloc(size: item_size, GFP_NOFS);
5812	if (!chunk) {
5813	ret = -ENOMEM;
5814	btrfs_abort_transaction(trans, ret);
5815	goto out;
5816	}
5817
5818	for (i = 0; i < map->num_stripes; i++) {
5819	struct btrfs_device *device = map->stripes[i].dev;
5820
5821	ret = btrfs_update_device(trans, device);
5822	if (ret)
5823	goto out;
5824	}
5825
5826	stripe = &chunk->stripe;
5827	for (i = 0; i < map->num_stripes; i++) {
5828	struct btrfs_device *device = map->stripes[i].dev;
5829	const u64 dev_offset = map->stripes[i].physical;
5830
5831	btrfs_set_stack_stripe_devid(s: stripe, val: device->devid);
5832	btrfs_set_stack_stripe_offset(s: stripe, val: dev_offset);
5833	memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE);
5834	stripe++;
5835	}
5836
5837	btrfs_set_stack_chunk_length(s: chunk, val: bg->length);
5838	btrfs_set_stack_chunk_owner(s: chunk, BTRFS_EXTENT_TREE_OBJECTID);
5839	btrfs_set_stack_chunk_stripe_len(s: chunk, BTRFS_STRIPE_LEN);
5840	btrfs_set_stack_chunk_type(s: chunk, val: map->type);
5841	btrfs_set_stack_chunk_num_stripes(s: chunk, val: map->num_stripes);
5842	btrfs_set_stack_chunk_io_align(s: chunk, BTRFS_STRIPE_LEN);
5843	btrfs_set_stack_chunk_io_width(s: chunk, BTRFS_STRIPE_LEN);
5844	btrfs_set_stack_chunk_sector_size(s: chunk, val: fs_info->sectorsize);
5845	btrfs_set_stack_chunk_sub_stripes(s: chunk, val: map->sub_stripes);
5846
5847	key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
5848	key.type = BTRFS_CHUNK_ITEM_KEY;
5849	key.offset = bg->start;
5850
5851	ret = btrfs_insert_item(trans, root: chunk_root, key: &key, data: chunk, data_size: item_size);
5852	if (ret)
5853	goto out;
5854
5855	set_bit(nr: BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, addr: &bg->runtime_flags);
5856
5857	if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
5858	ret = btrfs_add_system_chunk(fs_info, key: &key, chunk, item_size);
5859	if (ret)
5860	goto out;
5861	}
5862
5863	out:
5864	kfree(objp: chunk);
5865	btrfs_free_chunk_map(map);
5866	return ret;
5867	}
5868
5869	static noinline int init_first_rw_device(struct btrfs_trans_handle *trans)
5870	{
5871	struct btrfs_fs_info *fs_info = trans->fs_info;
5872	u64 alloc_profile;
5873	struct btrfs_block_group *meta_bg;
5874	struct btrfs_block_group *sys_bg;
5875
5876	/*
5877	* When adding a new device for sprouting, the seed device is read-only
5878	* so we must first allocate a metadata and a system chunk. But before
5879	* adding the block group items to the extent, device and chunk btrees,
5880	* we must first:
5881	*
5882	* 1) Create both chunks without doing any changes to the btrees, as
5883	* otherwise we would get -ENOSPC since the block groups from the
5884	* seed device are read-only;
5885	*
5886	* 2) Add the device item for the new sprout device - finishing the setup
5887	* of a new block group requires updating the device item in the chunk
5888	* btree, so it must exist when we attempt to do it. The previous step
5889	* ensures this does not fail with -ENOSPC.
5890	*
5891	* After that we can add the block group items to their btrees:
5892	* update existing device item in the chunk btree, add a new block group
5893	* item to the extent btree, add a new chunk item to the chunk btree and
5894	* finally add the new device extent items to the devices btree.
5895	*/
5896
5897	alloc_profile = btrfs_metadata_alloc_profile(fs_info);
5898	meta_bg = btrfs_create_chunk(trans, type: alloc_profile);
5899	if (IS_ERR(ptr: meta_bg))
5900	return PTR_ERR(ptr: meta_bg);
5901
5902	alloc_profile = btrfs_system_alloc_profile(fs_info);
5903	sys_bg = btrfs_create_chunk(trans, type: alloc_profile);
5904	if (IS_ERR(ptr: sys_bg))
5905	return PTR_ERR(ptr: sys_bg);
5906
5907	return 0;
5908	}
5909
5910	static inline int btrfs_chunk_max_errors(struct btrfs_chunk_map *map)
5911	{
5912	const int index = btrfs_bg_flags_to_raid_index(flags: map->type);
5913
5914	return btrfs_raid_array[index].tolerated_failures;
5915	}
5916
5917	bool btrfs_chunk_writeable(struct btrfs_fs_info *fs_info, u64 chunk_offset)
5918	{
5919	struct btrfs_chunk_map *map;
5920	int miss_ndevs = 0;
5921	int i;
5922	bool ret = true;
5923
5924	map = btrfs_get_chunk_map(fs_info, logical: chunk_offset, length: 1);
5925	if (IS_ERR(ptr: map))
5926	return false;
5927
5928	for (i = 0; i < map->num_stripes; i++) {
5929	if (test_bit(BTRFS_DEV_STATE_MISSING,
5930	&map->stripes[i].dev->dev_state)) {
5931	miss_ndevs++;
5932	continue;
5933	}
5934	if (!test_bit(BTRFS_DEV_STATE_WRITEABLE,
5935	&map->stripes[i].dev->dev_state)) {
5936	ret = false;
5937	goto end;
5938	}
5939	}
5940
5941	/*
5942	* If the number of missing devices is larger than max errors, we can
5943	* not write the data into that chunk successfully.
5944	*/
5945	if (miss_ndevs > btrfs_chunk_max_errors(map))
5946	ret = false;
5947	end:
5948	btrfs_free_chunk_map(map);
5949	return ret;
5950	}
5951
5952	void btrfs_mapping_tree_free(struct btrfs_fs_info *fs_info)
5953	{
5954	write_lock(&fs_info->mapping_tree_lock);
5955	while (!RB_EMPTY_ROOT(&fs_info->mapping_tree.rb_root)) {
5956	struct btrfs_chunk_map *map;
5957	struct rb_node *node;
5958
5959	node = rb_first_cached(&fs_info->mapping_tree);
5960	map = rb_entry(node, struct btrfs_chunk_map, rb_node);
5961	rb_erase_cached(node: &map->rb_node, root: &fs_info->mapping_tree);
5962	RB_CLEAR_NODE(&map->rb_node);
5963	chunk_map_device_clear_bits(map, CHUNK_ALLOCATED);
5964	/* Once for the tree ref. */
5965	btrfs_free_chunk_map(map);
5966	cond_resched_rwlock_write(&fs_info->mapping_tree_lock);
5967	}
5968	write_unlock(&fs_info->mapping_tree_lock);
5969	}
5970
5971	int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5972	{
5973	struct btrfs_chunk_map *map;
5974	enum btrfs_raid_types index;
5975	int ret = 1;
5976
5977	map = btrfs_get_chunk_map(fs_info, logical, length: len);
5978	if (IS_ERR(ptr: map))
5979	/*
5980	* We could return errors for these cases, but that could get
5981	* ugly and we'd probably do the same thing which is just not do
5982	* anything else and exit, so return 1 so the callers don't try
5983	* to use other copies.
5984	*/
5985	return 1;
5986
5987	index = btrfs_bg_flags_to_raid_index(flags: map->type);
5988
5989	/* Non-RAID56, use their ncopies from btrfs_raid_array. */
5990	if (!(map->type & BTRFS_BLOCK_GROUP_RAID56_MASK))
5991	ret = btrfs_raid_array[index].ncopies;
5992	else if (map->type & BTRFS_BLOCK_GROUP_RAID5)
5993	ret = 2;
5994	else if (map->type & BTRFS_BLOCK_GROUP_RAID6)
5995	/*
5996	* There could be two corrupted data stripes, we need
5997	* to loop retry in order to rebuild the correct data.
5998	*
5999	* Fail a stripe at a time on every retry except the
6000	* stripe under reconstruction.
6001	*/
6002	ret = map->num_stripes;
6003	btrfs_free_chunk_map(map);
6004	return ret;
6005	}
6006
6007	unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
6008	u64 logical)
6009	{
6010	struct btrfs_chunk_map *map;
6011	unsigned long len = fs_info->sectorsize;
6012
6013	if (!btrfs_fs_incompat(fs_info, RAID56))
6014	return len;
6015
6016	map = btrfs_get_chunk_map(fs_info, logical, length: len);
6017
6018	if (!WARN_ON(IS_ERR(map))) {
6019	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
6020	len = btrfs_stripe_nr_to_offset(stripe_nr: nr_data_stripes(map));
6021	btrfs_free_chunk_map(map);
6022	}
6023	return len;
6024	}
6025
6026	int btrfs_is_parity_mirror(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
6027	{
6028	struct btrfs_chunk_map *map;
6029	int ret = 0;
6030
6031	if (!btrfs_fs_incompat(fs_info, RAID56))
6032	return 0;
6033
6034	map = btrfs_get_chunk_map(fs_info, logical, length: len);
6035
6036	if (!WARN_ON(IS_ERR(map))) {
6037	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
6038	ret = 1;
6039	btrfs_free_chunk_map(map);
6040	}
6041	return ret;
6042	}
6043
6044	static int find_live_mirror(struct btrfs_fs_info *fs_info,
6045	struct btrfs_chunk_map *map, int first,
6046	int dev_replace_is_ongoing)
6047	{
6048	const enum btrfs_read_policy policy = READ_ONCE(fs_info->fs_devices->read_policy);
6049	int i;
6050	int num_stripes;
6051	int preferred_mirror;
6052	int tolerance;
6053	struct btrfs_device *srcdev;
6054
6055	ASSERT((map->type &
6056	(BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10)));
6057
6058	if (map->type & BTRFS_BLOCK_GROUP_RAID10)
6059	num_stripes = map->sub_stripes;
6060	else
6061	num_stripes = map->num_stripes;
6062
6063	switch (policy) {
6064	default:
6065	/* Shouldn't happen, just warn and use pid instead of failing */
6066	btrfs_warn_rl(fs_info, "unknown read_policy type %u, reset to pid",
6067	policy);
6068	WRITE_ONCE(fs_info->fs_devices->read_policy, BTRFS_READ_POLICY_PID);
6069	fallthrough;
6070	case BTRFS_READ_POLICY_PID:
6071	preferred_mirror = first + (current->pid % num_stripes);
6072	break;
6073	}
6074
6075	if (dev_replace_is_ongoing &&
6076	fs_info->dev_replace.cont_reading_from_srcdev_mode ==
6077	BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID)
6078	srcdev = fs_info->dev_replace.srcdev;
6079	else
6080	srcdev = NULL;
6081
6082	/*
6083	* try to avoid the drive that is the source drive for a
6084	* dev-replace procedure, only choose it if no other non-missing
6085	* mirror is available
6086	*/
6087	for (tolerance = 0; tolerance < 2; tolerance++) {
6088	if (map->stripes[preferred_mirror].dev->bdev &&
6089	(tolerance || map->stripes[preferred_mirror].dev != srcdev))
6090	return preferred_mirror;
6091	for (i = first; i < first + num_stripes; i++) {
6092	if (map->stripes[i].dev->bdev &&
6093	(tolerance || map->stripes[i].dev != srcdev))
6094	return i;
6095	}
6096	}
6097
6098	/* we couldn't find one that doesn't fail. Just return something
6099	* and the io error handling code will clean up eventually
6100	*/
6101	return preferred_mirror;
6102	}
6103
6104	static struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_info,
6105	u64 logical,
6106	u16 total_stripes)
6107	{
6108	struct btrfs_io_context *bioc;
6109
6110	bioc = kzalloc(
6111	/* The size of btrfs_io_context */
6112	size: sizeof(struct btrfs_io_context) +
6113	/* Plus the variable array for the stripes */
6114	sizeof(struct btrfs_io_stripe) * (total_stripes),
6115	GFP_NOFS);
6116
6117	if (!bioc)
6118	return NULL;
6119
6120	refcount_set(r: &bioc->refs, n: 1);
6121
6122	bioc->fs_info = fs_info;
6123	bioc->replace_stripe_src = -1;
6124	bioc->full_stripe_logical = (u64)-1;
6125	bioc->logical = logical;
6126
6127	return bioc;
6128	}
6129
6130	void btrfs_get_bioc(struct btrfs_io_context *bioc)
6131	{
6132	WARN_ON(!refcount_read(&bioc->refs));
6133	refcount_inc(r: &bioc->refs);
6134	}
6135
6136	void btrfs_put_bioc(struct btrfs_io_context *bioc)
6137	{
6138	if (!bioc)
6139	return;
6140	if (refcount_dec_and_test(r: &bioc->refs))
6141	kfree(objp: bioc);
6142	}
6143
6144	/*
6145	* Please note that, discard won't be sent to target device of device
6146	* replace.
6147	*/
6148	struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info,
6149	u64 logical, u64 *length_ret,
6150	u32 *num_stripes)
6151	{
6152	struct btrfs_chunk_map *map;
6153	struct btrfs_discard_stripe *stripes;
6154	u64 length = *length_ret;
6155	u64 offset;
6156	u32 stripe_nr;
6157	u32 stripe_nr_end;
6158	u32 stripe_cnt;
6159	u64 stripe_end_offset;
6160	u64 stripe_offset;
6161	u32 stripe_index;
6162	u32 factor = 0;
6163	u32 sub_stripes = 0;
6164	u32 stripes_per_dev = 0;
6165	u32 remaining_stripes = 0;
6166	u32 last_stripe = 0;
6167	int ret;
6168	int i;
6169
6170	map = btrfs_get_chunk_map(fs_info, logical, length);
6171	if (IS_ERR(ptr: map))
6172	return ERR_CAST(ptr: map);
6173
6174	/* we don't discard raid56 yet */
6175	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6176	ret = -EOPNOTSUPP;
6177	goto out_free_map;
6178	}
6179
6180	offset = logical - map->start;
6181	length = min_t(u64, map->start + map->chunk_len - logical, length);
6182	*length_ret = length;
6183
6184	/*
6185	* stripe_nr counts the total number of stripes we have to stride
6186	* to get to this block
6187	*/
6188	stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT;
6189
6190	/* stripe_offset is the offset of this block in its stripe */
6191	stripe_offset = offset - btrfs_stripe_nr_to_offset(stripe_nr);
6192
6193	stripe_nr_end = round_up(offset + length, BTRFS_STRIPE_LEN) >>
6194	BTRFS_STRIPE_LEN_SHIFT;
6195	stripe_cnt = stripe_nr_end - stripe_nr;
6196	stripe_end_offset = btrfs_stripe_nr_to_offset(stripe_nr: stripe_nr_end) -
6197	(offset + length);
6198	/*
6199	* after this, stripe_nr is the number of stripes on this
6200	* device we have to walk to find the data, and stripe_index is
6201	* the number of our device in the stripe array
6202	*/
6203	*num_stripes = 1;
6204	stripe_index = 0;
6205	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
6206	BTRFS_BLOCK_GROUP_RAID10)) {
6207	if (map->type & BTRFS_BLOCK_GROUP_RAID0)
6208	sub_stripes = 1;
6209	else
6210	sub_stripes = map->sub_stripes;
6211
6212	factor = map->num_stripes / sub_stripes;
6213	*num_stripes = min_t(u64, map->num_stripes,
6214	sub_stripes * stripe_cnt);
6215	stripe_index = stripe_nr % factor;
6216	stripe_nr /= factor;
6217	stripe_index *= sub_stripes;
6218
6219	remaining_stripes = stripe_cnt % factor;
6220	stripes_per_dev = stripe_cnt / factor;
6221	last_stripe = ((stripe_nr_end - 1) % factor) * sub_stripes;
6222	} else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK |
6223	BTRFS_BLOCK_GROUP_DUP)) {
6224	*num_stripes = map->num_stripes;
6225	} else {
6226	stripe_index = stripe_nr % map->num_stripes;
6227	stripe_nr /= map->num_stripes;
6228	}
6229
6230	stripes = kcalloc(n: *num_stripes, size: sizeof(*stripes), GFP_NOFS);
6231	if (!stripes) {
6232	ret = -ENOMEM;
6233	goto out_free_map;
6234	}
6235
6236	for (i = 0; i < *num_stripes; i++) {
6237	stripes[i].physical =
6238	map->stripes[stripe_index].physical +
6239	stripe_offset + btrfs_stripe_nr_to_offset(stripe_nr);
6240	stripes[i].dev = map->stripes[stripe_index].dev;
6241
6242	if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
6243	BTRFS_BLOCK_GROUP_RAID10)) {
6244	stripes[i].length = btrfs_stripe_nr_to_offset(stripe_nr: stripes_per_dev);
6245
6246	if (i / sub_stripes < remaining_stripes)
6247	stripes[i].length += BTRFS_STRIPE_LEN;
6248
6249	/*
6250	* Special for the first stripe and
6251	* the last stripe:
6252	*
6253	* |-------|...|-------|
6254	* |----------|
6255	* off end_off
6256	*/
6257	if (i < sub_stripes)
6258	stripes[i].length -= stripe_offset;
6259
6260	if (stripe_index >= last_stripe &&
6261	stripe_index <= (last_stripe +
6262	sub_stripes - 1))
6263	stripes[i].length -= stripe_end_offset;
6264
6265	if (i == sub_stripes - 1)
6266	stripe_offset = 0;
6267	} else {
6268	stripes[i].length = length;
6269	}
6270
6271	stripe_index++;
6272	if (stripe_index == map->num_stripes) {
6273	stripe_index = 0;
6274	stripe_nr++;
6275	}
6276	}
6277
6278	btrfs_free_chunk_map(map);
6279	return stripes;
6280	out_free_map:
6281	btrfs_free_chunk_map(map);
6282	return ERR_PTR(error: ret);
6283	}
6284
6285	static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical)
6286	{
6287	struct btrfs_block_group *cache;
6288	bool ret;
6289
6290	/* Non zoned filesystem does not use "to_copy" flag */
6291	if (!btrfs_is_zoned(fs_info))
6292	return false;
6293
6294	cache = btrfs_lookup_block_group(info: fs_info, bytenr: logical);
6295
6296	ret = test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags);
6297
6298	btrfs_put_block_group(cache);
6299	return ret;
6300	}
6301
6302	static void handle_ops_on_dev_replace(enum btrfs_map_op op,
6303	struct btrfs_io_context *bioc,
6304	struct btrfs_dev_replace *dev_replace,
6305	u64 logical,
6306	int *num_stripes_ret, int *max_errors_ret)
6307	{
6308	u64 srcdev_devid = dev_replace->srcdev->devid;
6309	/*
6310	* At this stage, num_stripes is still the real number of stripes,
6311	* excluding the duplicated stripes.
6312	*/
6313	int num_stripes = *num_stripes_ret;
6314	int nr_extra_stripes = 0;
6315	int max_errors = *max_errors_ret;
6316	int i;
6317
6318	/*
6319	* A block group which has "to_copy" set will eventually be copied by
6320	* the dev-replace process. We can avoid cloning IO here.
6321	*/
6322	if (is_block_group_to_copy(fs_info: dev_replace->srcdev->fs_info, logical))
6323	return;
6324
6325	/*
6326	* Duplicate the write operations while the dev-replace procedure is
6327	* running. Since the copying of the old disk to the new disk takes
6328	* place at run time while the filesystem is mounted writable, the
6329	* regular write operations to the old disk have to be duplicated to go
6330	* to the new disk as well.
6331	*
6332	* Note that device->missing is handled by the caller, and that the
6333	* write to the old disk is already set up in the stripes array.
6334	*/
6335	for (i = 0; i < num_stripes; i++) {
6336	struct btrfs_io_stripe *old = &bioc->stripes[i];
6337	struct btrfs_io_stripe *new = &bioc->stripes[num_stripes + nr_extra_stripes];
6338
6339	if (old->dev->devid != srcdev_devid)
6340	continue;
6341
6342	new->physical = old->physical;
6343	new->dev = dev_replace->tgtdev;
6344	if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK)
6345	bioc->replace_stripe_src = i;
6346	nr_extra_stripes++;
6347	}
6348
6349	/* We can only have at most 2 extra nr_stripes (for DUP). */
6350	ASSERT(nr_extra_stripes <= 2);
6351	/*
6352	* For GET_READ_MIRRORS, we can only return at most 1 extra stripe for
6353	* replace.
6354	* If we have 2 extra stripes, only choose the one with smaller physical.
6355	*/
6356	if (op == BTRFS_MAP_GET_READ_MIRRORS && nr_extra_stripes == 2) {
6357	struct btrfs_io_stripe *first = &bioc->stripes[num_stripes];
6358	struct btrfs_io_stripe *second = &bioc->stripes[num_stripes + 1];
6359
6360	/* Only DUP can have two extra stripes. */
6361	ASSERT(bioc->map_type & BTRFS_BLOCK_GROUP_DUP);
6362
6363	/*
6364	* Swap the last stripe stripes and reduce @nr_extra_stripes.
6365	* The extra stripe would still be there, but won't be accessed.
6366	*/
6367	if (first->physical > second->physical) {
6368	swap(second->physical, first->physical);
6369	swap(second->dev, first->dev);
6370	nr_extra_stripes--;
6371	}
6372	}
6373
6374	*num_stripes_ret = num_stripes + nr_extra_stripes;
6375	*max_errors_ret = max_errors + nr_extra_stripes;
6376	bioc->replace_nr_stripes = nr_extra_stripes;
6377	}
6378
6379	static u64 btrfs_max_io_len(struct btrfs_chunk_map *map, u64 offset,
6380	struct btrfs_io_geometry *io_geom)
6381	{
6382	/*
6383	* Stripe_nr is the stripe where this block falls. stripe_offset is
6384	* the offset of this block in its stripe.
6385	*/
6386	io_geom->stripe_offset = offset & BTRFS_STRIPE_LEN_MASK;
6387	io_geom->stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT;
6388	ASSERT(io_geom->stripe_offset < U32_MAX);
6389
6390	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6391	unsigned long full_stripe_len =
6392	btrfs_stripe_nr_to_offset(stripe_nr: nr_data_stripes(map));
6393
6394	/*
6395	* For full stripe start, we use previously calculated
6396	* @stripe_nr. Align it to nr_data_stripes, then multiply with
6397	* STRIPE_LEN.
6398	*
6399	* By this we can avoid u64 division completely. And we have
6400	* to go rounddown(), not round_down(), as nr_data_stripes is
6401	* not ensured to be power of 2.
6402	*/
6403	io_geom->raid56_full_stripe_start = btrfs_stripe_nr_to_offset(
6404	rounddown(io_geom->stripe_nr, nr_data_stripes(map)));
6405
6406	ASSERT(io_geom->raid56_full_stripe_start + full_stripe_len > offset);
6407	ASSERT(io_geom->raid56_full_stripe_start <= offset);
6408	/*
6409	* For writes to RAID56, allow to write a full stripe set, but
6410	* no straddling of stripe sets.
6411	*/
6412	if (io_geom->op == BTRFS_MAP_WRITE)
6413	return full_stripe_len - (offset - io_geom->raid56_full_stripe_start);
6414	}
6415
6416	/*
6417	* For other RAID types and for RAID56 reads, allow a single stripe (on
6418	* a single disk).
6419	*/
6420	if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK)
6421	return BTRFS_STRIPE_LEN - io_geom->stripe_offset;
6422	return U64_MAX;
6423	}
6424
6425	static int set_io_stripe(struct btrfs_fs_info *fs_info, u64 logical,
6426	u64 *length, struct btrfs_io_stripe *dst,
6427	struct btrfs_chunk_map *map,
6428	struct btrfs_io_geometry *io_geom)
6429	{
6430	dst->dev = map->stripes[io_geom->stripe_index].dev;
6431
6432	if (io_geom->op == BTRFS_MAP_READ &&
6433	btrfs_need_stripe_tree_update(fs_info, map_type: map->type))
6434	return btrfs_get_raid_extent_offset(fs_info, logical, length,
6435	map_type: map->type,
6436	stripe_index: io_geom->stripe_index, stripe: dst);
6437
6438	dst->physical = map->stripes[io_geom->stripe_index].physical +
6439	io_geom->stripe_offset +
6440	btrfs_stripe_nr_to_offset(stripe_nr: io_geom->stripe_nr);
6441	return 0;
6442	}
6443
6444	static bool is_single_device_io(struct btrfs_fs_info *fs_info,
6445	const struct btrfs_io_stripe *smap,
6446	const struct btrfs_chunk_map *map,
6447	int num_alloc_stripes,
6448	enum btrfs_map_op op, int mirror_num)
6449	{
6450	if (!smap)
6451	return false;
6452
6453	if (num_alloc_stripes != 1)
6454	return false;
6455
6456	if (btrfs_need_stripe_tree_update(fs_info, map_type: map->type) && op != BTRFS_MAP_READ)
6457	return false;
6458
6459	if ((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) && mirror_num > 1)
6460	return false;
6461
6462	return true;
6463	}
6464
6465	static void map_blocks_raid0(const struct btrfs_chunk_map *map,
6466	struct btrfs_io_geometry *io_geom)
6467	{
6468	io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes;
6469	io_geom->stripe_nr /= map->num_stripes;
6470	if (io_geom->op == BTRFS_MAP_READ)
6471	io_geom->mirror_num = 1;
6472	}
6473
6474	static void map_blocks_raid1(struct btrfs_fs_info *fs_info,
6475	struct btrfs_chunk_map *map,
6476	struct btrfs_io_geometry *io_geom,
6477	bool dev_replace_is_ongoing)
6478	{
6479	if (io_geom->op != BTRFS_MAP_READ) {
6480	io_geom->num_stripes = map->num_stripes;
6481	return;
6482	}
6483
6484	if (io_geom->mirror_num) {
6485	io_geom->stripe_index = io_geom->mirror_num - 1;
6486	return;
6487	}
6488
6489	io_geom->stripe_index = find_live_mirror(fs_info, map, first: 0,
6490	dev_replace_is_ongoing);
6491	io_geom->mirror_num = io_geom->stripe_index + 1;
6492	}
6493
6494	static void map_blocks_dup(const struct btrfs_chunk_map *map,
6495	struct btrfs_io_geometry *io_geom)
6496	{
6497	if (io_geom->op != BTRFS_MAP_READ) {
6498	io_geom->num_stripes = map->num_stripes;
6499	return;
6500	}
6501
6502	if (io_geom->mirror_num) {
6503	io_geom->stripe_index = io_geom->mirror_num - 1;
6504	return;
6505	}
6506
6507	io_geom->mirror_num = 1;
6508	}
6509
6510	static void map_blocks_raid10(struct btrfs_fs_info *fs_info,
6511	struct btrfs_chunk_map *map,
6512	struct btrfs_io_geometry *io_geom,
6513	bool dev_replace_is_ongoing)
6514	{
6515	u32 factor = map->num_stripes / map->sub_stripes;
6516	int old_stripe_index;
6517
6518	io_geom->stripe_index = (io_geom->stripe_nr % factor) * map->sub_stripes;
6519	io_geom->stripe_nr /= factor;
6520
6521	if (io_geom->op != BTRFS_MAP_READ) {
6522	io_geom->num_stripes = map->sub_stripes;
6523	return;
6524	}
6525
6526	if (io_geom->mirror_num) {
6527	io_geom->stripe_index += io_geom->mirror_num - 1;
6528	return;
6529	}
6530
6531	old_stripe_index = io_geom->stripe_index;
6532	io_geom->stripe_index = find_live_mirror(fs_info, map,
6533	first: io_geom->stripe_index,
6534	dev_replace_is_ongoing);
6535	io_geom->mirror_num = io_geom->stripe_index - old_stripe_index + 1;
6536	}
6537
6538	static void map_blocks_raid56_write(struct btrfs_chunk_map *map,
6539	struct btrfs_io_geometry *io_geom,
6540	u64 logical, u64 *length)
6541	{
6542	int data_stripes = nr_data_stripes(map);
6543
6544	/*
6545	* Needs full stripe mapping.
6546	*
6547	* Push stripe_nr back to the start of the full stripe For those cases
6548	* needing a full stripe, @stripe_nr is the full stripe number.
6549	*
6550	* Originally we go raid56_full_stripe_start / full_stripe_len, but
6551	* that can be expensive. Here we just divide @stripe_nr with
6552	* @data_stripes.
6553	*/
6554	io_geom->stripe_nr /= data_stripes;
6555
6556	/* RAID[56] write or recovery. Return all stripes */
6557	io_geom->num_stripes = map->num_stripes;
6558	io_geom->max_errors = btrfs_chunk_max_errors(map);
6559
6560	/* Return the length to the full stripe end. */
6561	*length = min(logical + *length,
6562	io_geom->raid56_full_stripe_start + map->start +
6563	btrfs_stripe_nr_to_offset(data_stripes)) -
6564	logical;
6565	io_geom->stripe_index = 0;
6566	io_geom->stripe_offset = 0;
6567	}
6568
6569	static void map_blocks_raid56_read(struct btrfs_chunk_map *map,
6570	struct btrfs_io_geometry *io_geom)
6571	{
6572	int data_stripes = nr_data_stripes(map);
6573
6574	ASSERT(io_geom->mirror_num <= 1);
6575	/* Just grab the data stripe directly. */
6576	io_geom->stripe_index = io_geom->stripe_nr % data_stripes;
6577	io_geom->stripe_nr /= data_stripes;
6578
6579	/* We distribute the parity blocks across stripes. */
6580	io_geom->stripe_index =
6581	(io_geom->stripe_nr + io_geom->stripe_index) % map->num_stripes;
6582
6583	if (io_geom->op == BTRFS_MAP_READ && io_geom->mirror_num < 1)
6584	io_geom->mirror_num = 1;
6585	}
6586
6587	static void map_blocks_single(const struct btrfs_chunk_map *map,
6588	struct btrfs_io_geometry *io_geom)
6589	{
6590	io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes;
6591	io_geom->stripe_nr /= map->num_stripes;
6592	io_geom->mirror_num = io_geom->stripe_index + 1;
6593	}
6594
6595	/*
6596	* Map one logical range to one or more physical ranges.
6597	*
6598	* @length: (Mandatory) mapped length of this run.
6599	* One logical range can be split into different segments
6600	* due to factors like zones and RAID0/5/6/10 stripe
6601	* boundaries.
6602	*
6603	* @bioc_ret: (Mandatory) returned btrfs_io_context structure.
6604	* which has one or more physical ranges (btrfs_io_stripe)
6605	* recorded inside.
6606	* Caller should call btrfs_put_bioc() to free it after use.
6607	*
6608	* @smap: (Optional) single physical range optimization.
6609	* If the map request can be fulfilled by one single
6610	* physical range, and this is parameter is not NULL,
6611	* then @bioc_ret would be NULL, and @smap would be
6612	* updated.
6613	*
6614	* @mirror_num_ret: (Mandatory) returned mirror number if the original
6615	* value is 0.
6616	*
6617	* Mirror number 0 means to choose any live mirrors.
6618	*
6619	* For non-RAID56 profiles, non-zero mirror_num means
6620	* the Nth mirror. (e.g. mirror_num 1 means the first
6621	* copy).
6622	*
6623	* For RAID56 profile, mirror 1 means rebuild from P and
6624	* the remaining data stripes.
6625	*
6626	* For RAID6 profile, mirror > 2 means mark another
6627	* data/P stripe error and rebuild from the remaining
6628	* stripes..
6629	*/
6630	int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
6631	u64 logical, u64 *length,
6632	struct btrfs_io_context **bioc_ret,
6633	struct btrfs_io_stripe *smap, int *mirror_num_ret)
6634	{
6635	struct btrfs_chunk_map *map;
6636	struct btrfs_io_geometry io_geom = { 0 };
6637	u64 map_offset;
6638	int i;
6639	int ret = 0;
6640	int num_copies;
6641	struct btrfs_io_context *bioc = NULL;
6642	struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
6643	int dev_replace_is_ongoing = 0;
6644	u16 num_alloc_stripes;
6645	u64 max_len;
6646
6647	ASSERT(bioc_ret);
6648
6649	io_geom.mirror_num = (mirror_num_ret ? *mirror_num_ret : 0);
6650	io_geom.num_stripes = 1;
6651	io_geom.stripe_index = 0;
6652	io_geom.op = op;
6653
6654	num_copies = btrfs_num_copies(fs_info, logical, len: fs_info->sectorsize);
6655	if (io_geom.mirror_num > num_copies)
6656	return -EINVAL;
6657
6658	map = btrfs_get_chunk_map(fs_info, logical, length: *length);
6659	if (IS_ERR(ptr: map))
6660	return PTR_ERR(ptr: map);
6661
6662	map_offset = logical - map->start;
6663	io_geom.raid56_full_stripe_start = (u64)-1;
6664	max_len = btrfs_max_io_len(map, offset: map_offset, io_geom: &io_geom);
6665	*length = min_t(u64, map->chunk_len - map_offset, max_len);
6666
6667	down_read(sem: &dev_replace->rwsem);
6668	dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace);
6669	/*
6670	* Hold the semaphore for read during the whole operation, write is
6671	* requested at commit time but must wait.
6672	*/
6673	if (!dev_replace_is_ongoing)
6674	up_read(sem: &dev_replace->rwsem);
6675
6676	switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
6677	case BTRFS_BLOCK_GROUP_RAID0:
6678	map_blocks_raid0(map, io_geom: &io_geom);
6679	break;
6680	case BTRFS_BLOCK_GROUP_RAID1:
6681	case BTRFS_BLOCK_GROUP_RAID1C3:
6682	case BTRFS_BLOCK_GROUP_RAID1C4:
6683	map_blocks_raid1(fs_info, map, io_geom: &io_geom, dev_replace_is_ongoing);
6684	break;
6685	case BTRFS_BLOCK_GROUP_DUP:
6686	map_blocks_dup(map, io_geom: &io_geom);
6687	break;
6688	case BTRFS_BLOCK_GROUP_RAID10:
6689	map_blocks_raid10(fs_info, map, io_geom: &io_geom, dev_replace_is_ongoing);
6690	break;
6691	case BTRFS_BLOCK_GROUP_RAID5:
6692	case BTRFS_BLOCK_GROUP_RAID6:
6693	if (op != BTRFS_MAP_READ || io_geom.mirror_num > 1)
6694	map_blocks_raid56_write(map, io_geom: &io_geom, logical, length);
6695	else
6696	map_blocks_raid56_read(map, io_geom: &io_geom);
6697	break;
6698	default:
6699	/*
6700	* After this, stripe_nr is the number of stripes on this
6701	* device we have to walk to find the data, and stripe_index is
6702	* the number of our device in the stripe array
6703	*/
6704	map_blocks_single(map, io_geom: &io_geom);
6705	break;
6706	}
6707	if (io_geom.stripe_index >= map->num_stripes) {
6708	btrfs_crit(fs_info,
6709	"stripe index math went horribly wrong, got stripe_index=%u, num_stripes=%u",
6710	io_geom.stripe_index, map->num_stripes);
6711	ret = -EINVAL;
6712	goto out;
6713	}
6714
6715	num_alloc_stripes = io_geom.num_stripes;
6716	if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
6717	op != BTRFS_MAP_READ)
6718	/*
6719	* For replace case, we need to add extra stripes for extra
6720	* duplicated stripes.
6721	*
6722	* For both WRITE and GET_READ_MIRRORS, we may have at most
6723	* 2 more stripes (DUP types, otherwise 1).
6724	*/
6725	num_alloc_stripes += 2;
6726
6727	/*
6728	* If this I/O maps to a single device, try to return the device and
6729	* physical block information on the stack instead of allocating an
6730	* I/O context structure.
6731	*/
6732	if (is_single_device_io(fs_info, smap, map, num_alloc_stripes, op,
6733	mirror_num: io_geom.mirror_num)) {
6734	ret = set_io_stripe(fs_info, logical, length, dst: smap, map, io_geom: &io_geom);
6735	if (mirror_num_ret)
6736	*mirror_num_ret = io_geom.mirror_num;
6737	*bioc_ret = NULL;
6738	goto out;
6739	}
6740
6741	bioc = alloc_btrfs_io_context(fs_info, logical, total_stripes: num_alloc_stripes);
6742	if (!bioc) {
6743	ret = -ENOMEM;
6744	goto out;
6745	}
6746	bioc->map_type = map->type;
6747
6748	/*
6749	* For RAID56 full map, we need to make sure the stripes[] follows the
6750	* rule that data stripes are all ordered, then followed with P and Q
6751	* (if we have).
6752	*
6753	* It's still mostly the same as other profiles, just with extra rotation.
6754	*/
6755	if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK &&
6756	(op != BTRFS_MAP_READ || io_geom.mirror_num > 1)) {
6757	/*
6758	* For RAID56 @stripe_nr is already the number of full stripes
6759	* before us, which is also the rotation value (needs to modulo
6760	* with num_stripes).
6761	*
6762	* In this case, we just add @stripe_nr with @i, then do the
6763	* modulo, to reduce one modulo call.
6764	*/
6765	bioc->full_stripe_logical = map->start +
6766	btrfs_stripe_nr_to_offset(stripe_nr: io_geom.stripe_nr *
6767	nr_data_stripes(map));
6768	for (int i = 0; i < io_geom.num_stripes; i++) {
6769	struct btrfs_io_stripe *dst = &bioc->stripes[i];
6770	u32 stripe_index;
6771
6772	stripe_index = (i + io_geom.stripe_nr) % io_geom.num_stripes;
6773	dst->dev = map->stripes[stripe_index].dev;
6774	dst->physical =
6775	map->stripes[stripe_index].physical +
6776	io_geom.stripe_offset +
6777	btrfs_stripe_nr_to_offset(stripe_nr: io_geom.stripe_nr);
6778	}
6779	} else {
6780	/*
6781	* For all other non-RAID56 profiles, just copy the target
6782	* stripe into the bioc.
6783	*/
6784	for (i = 0; i < io_geom.num_stripes; i++) {
6785	ret = set_io_stripe(fs_info, logical, length,
6786	dst: &bioc->stripes[i], map, io_geom: &io_geom);
6787	if (ret < 0)
6788	break;
6789	io_geom.stripe_index++;
6790	}
6791	}
6792
6793	if (ret) {
6794	*bioc_ret = NULL;
6795	btrfs_put_bioc(bioc);
6796	goto out;
6797	}
6798
6799	if (op != BTRFS_MAP_READ)
6800	io_geom.max_errors = btrfs_chunk_max_errors(map);
6801
6802	if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
6803	op != BTRFS_MAP_READ) {
6804	handle_ops_on_dev_replace(op, bioc, dev_replace, logical,
6805	num_stripes_ret: &io_geom.num_stripes, max_errors_ret: &io_geom.max_errors);
6806	}
6807
6808	*bioc_ret = bioc;
6809	bioc->num_stripes = io_geom.num_stripes;
6810	bioc->max_errors = io_geom.max_errors;
6811	bioc->mirror_num = io_geom.mirror_num;
6812
6813	out:
6814	if (dev_replace_is_ongoing) {
6815	lockdep_assert_held(&dev_replace->rwsem);
6816	/* Unlock and let waiting writers proceed */
6817	up_read(sem: &dev_replace->rwsem);
6818	}
6819	btrfs_free_chunk_map(map);
6820	return ret;
6821	}
6822
6823	static bool dev_args_match_fs_devices(const struct btrfs_dev_lookup_args *args,
6824	const struct btrfs_fs_devices *fs_devices)
6825	{
6826	if (args->fsid == NULL)
6827	return true;
6828	if (memcmp(p: fs_devices->metadata_uuid, q: args->fsid, BTRFS_FSID_SIZE) == 0)
6829	return true;
6830	return false;
6831	}
6832
6833	static bool dev_args_match_device(const struct btrfs_dev_lookup_args *args,
6834	const struct btrfs_device *device)
6835	{
6836	if (args->missing) {
6837	if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state) &&
6838	!device->bdev)
6839	return true;
6840	return false;
6841	}
6842
6843	if (device->devid != args->devid)
6844	return false;
6845	if (args->uuid && memcmp(p: device->uuid, q: args->uuid, BTRFS_UUID_SIZE) != 0)
6846	return false;
6847	return true;
6848	}
6849
6850	/*
6851	* Find a device specified by @devid or @uuid in the list of @fs_devices, or
6852	* return NULL.
6853	*
6854	* If devid and uuid are both specified, the match must be exact, otherwise
6855	* only devid is used.
6856	*/
6857	struct btrfs_device *btrfs_find_device(const struct btrfs_fs_devices *fs_devices,
6858	const struct btrfs_dev_lookup_args *args)
6859	{
6860	struct btrfs_device *device;
6861	struct btrfs_fs_devices *seed_devs;
6862
6863	if (dev_args_match_fs_devices(args, fs_devices)) {
6864	list_for_each_entry(device, &fs_devices->devices, dev_list) {
6865	if (dev_args_match_device(args, device))
6866	return device;
6867	}
6868	}
6869
6870	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
6871	if (!dev_args_match_fs_devices(args, fs_devices: seed_devs))
6872	continue;
6873	list_for_each_entry(device, &seed_devs->devices, dev_list) {
6874	if (dev_args_match_device(args, device))
6875	return device;
6876	}
6877	}
6878
6879	return NULL;
6880	}
6881
6882	static struct btrfs_device *add_missing_dev(struct btrfs_fs_devices *fs_devices,
6883	u64 devid, u8 *dev_uuid)
6884	{
6885	struct btrfs_device *device;
6886	unsigned int nofs_flag;
6887
6888	/*
6889	* We call this under the chunk_mutex, so we want to use NOFS for this
6890	* allocation, however we don't want to change btrfs_alloc_device() to
6891	* always do NOFS because we use it in a lot of other GFP_KERNEL safe
6892	* places.
6893	*/
6894
6895	nofs_flag = memalloc_nofs_save();
6896	device = btrfs_alloc_device(NULL, devid: &devid, uuid: dev_uuid, NULL);
6897	memalloc_nofs_restore(flags: nofs_flag);
6898	if (IS_ERR(ptr: device))
6899	return device;
6900
6901	list_add(new: &device->dev_list, head: &fs_devices->devices);
6902	device->fs_devices = fs_devices;
6903	fs_devices->num_devices++;
6904
6905	set_bit(BTRFS_DEV_STATE_MISSING, addr: &device->dev_state);
6906	fs_devices->missing_devices++;
6907
6908	return device;
6909	}
6910
6911	/*
6912	* Allocate new device struct, set up devid and UUID.
6913	*
6914	* @fs_info: used only for generating a new devid, can be NULL if
6915	* devid is provided (i.e. @devid != NULL).
6916	* @devid: a pointer to devid for this device. If NULL a new devid
6917	* is generated.
6918	* @uuid: a pointer to UUID for this device. If NULL a new UUID
6919	* is generated.
6920	* @path: a pointer to device path if available, NULL otherwise.
6921	*
6922	* Return: a pointer to a new &struct btrfs_device on success; ERR_PTR()
6923	* on error. Returned struct is not linked onto any lists and must be
6924	* destroyed with btrfs_free_device.
6925	*/
6926	struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info,
6927	const u64 *devid, const u8 *uuid,
6928	const char *path)
6929	{
6930	struct btrfs_device *dev;
6931	u64 tmp;
6932
6933	if (WARN_ON(!devid && !fs_info))
6934	return ERR_PTR(error: -EINVAL);
6935
6936	dev = kzalloc(size: sizeof(*dev), GFP_KERNEL);
6937	if (!dev)
6938	return ERR_PTR(error: -ENOMEM);
6939
6940	INIT_LIST_HEAD(list: &dev->dev_list);
6941	INIT_LIST_HEAD(list: &dev->dev_alloc_list);
6942	INIT_LIST_HEAD(list: &dev->post_commit_list);
6943
6944	atomic_set(v: &dev->dev_stats_ccnt, i: 0);
6945	btrfs_device_data_ordered_init(dev);
6946	extent_io_tree_init(fs_info, tree: &dev->alloc_state, owner: IO_TREE_DEVICE_ALLOC_STATE);
6947
6948	if (devid)
6949	tmp = *devid;
6950	else {
6951	int ret;
6952
6953	ret = find_next_devid(fs_info, devid_ret: &tmp);
6954	if (ret) {
6955	btrfs_free_device(device: dev);
6956	return ERR_PTR(error: ret);
6957	}
6958	}
6959	dev->devid = tmp;
6960
6961	if (uuid)
6962	memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE);
6963	else
6964	generate_random_uuid(uuid: dev->uuid);
6965
6966	if (path) {
6967	struct rcu_string *name;
6968
6969	name = rcu_string_strdup(src: path, GFP_KERNEL);
6970	if (!name) {
6971	btrfs_free_device(device: dev);
6972	return ERR_PTR(error: -ENOMEM);
6973	}
6974	rcu_assign_pointer(dev->name, name);
6975	}
6976
6977	return dev;
6978	}
6979
6980	static void btrfs_report_missing_device(struct btrfs_fs_info *fs_info,
6981	u64 devid, u8 *uuid, bool error)
6982	{
6983	if (error)
6984	btrfs_err_rl(fs_info, "devid %llu uuid %pU is missing",
6985	devid, uuid);
6986	else
6987	btrfs_warn_rl(fs_info, "devid %llu uuid %pU is missing",
6988	devid, uuid);
6989	}
6990
6991	u64 btrfs_calc_stripe_length(const struct btrfs_chunk_map *map)
6992	{
6993	const int data_stripes = calc_data_stripes(type: map->type, num_stripes: map->num_stripes);
6994
6995	return div_u64(dividend: map->chunk_len, divisor: data_stripes);
6996	}
6997
6998	#if BITS_PER_LONG == 32
6999	/*
7000	* Due to page cache limit, metadata beyond BTRFS_32BIT_MAX_FILE_SIZE
7001	* can't be accessed on 32bit systems.
7002	*
7003	* This function do mount time check to reject the fs if it already has
7004	* metadata chunk beyond that limit.
7005	*/
7006	static int check_32bit_meta_chunk(struct btrfs_fs_info *fs_info,
7007	u64 logical, u64 length, u64 type)
7008	{
7009	if (!(type & BTRFS_BLOCK_GROUP_METADATA))
7010	return 0;
7011
7012	if (logical + length < MAX_LFS_FILESIZE)
7013	return 0;
7014
7015	btrfs_err_32bit_limit(fs_info);
7016	return -EOVERFLOW;
7017	}
7018
7019	/*
7020	* This is to give early warning for any metadata chunk reaching
7021	* BTRFS_32BIT_EARLY_WARN_THRESHOLD.
7022	* Although we can still access the metadata, it's not going to be possible
7023	* once the limit is reached.
7024	*/
7025	static void warn_32bit_meta_chunk(struct btrfs_fs_info *fs_info,
7026	u64 logical, u64 length, u64 type)
7027	{
7028	if (!(type & BTRFS_BLOCK_GROUP_METADATA))
7029	return;
7030
7031	if (logical + length < BTRFS_32BIT_EARLY_WARN_THRESHOLD)
7032	return;
7033
7034	btrfs_warn_32bit_limit(fs_info);
7035	}
7036	#endif
7037
7038	static struct btrfs_device *handle_missing_device(struct btrfs_fs_info *fs_info,
7039	u64 devid, u8 *uuid)
7040	{
7041	struct btrfs_device *dev;
7042
7043	if (!btrfs_test_opt(fs_info, DEGRADED)) {
7044	btrfs_report_missing_device(fs_info, devid, uuid, error: true);
7045	return ERR_PTR(error: -ENOENT);
7046	}
7047
7048	dev = add_missing_dev(fs_devices: fs_info->fs_devices, devid, dev_uuid: uuid);
7049	if (IS_ERR(ptr: dev)) {
7050	btrfs_err(fs_info, "failed to init missing device %llu: %ld",
7051	devid, PTR_ERR(dev));
7052	return dev;
7053	}
7054	btrfs_report_missing_device(fs_info, devid, uuid, error: false);
7055
7056	return dev;
7057	}
7058
7059	static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf,
7060	struct btrfs_chunk *chunk)
7061	{
7062	BTRFS_DEV_LOOKUP_ARGS(args);
7063	struct btrfs_fs_info *fs_info = leaf->fs_info;
7064	struct btrfs_chunk_map *map;
7065	u64 logical;
7066	u64 length;
7067	u64 devid;
7068	u64 type;
7069	u8 uuid[BTRFS_UUID_SIZE];
7070	int index;
7071	int num_stripes;
7072	int ret;
7073	int i;
7074
7075	logical = key->offset;
7076	length = btrfs_chunk_length(eb: leaf, s: chunk);
7077	type = btrfs_chunk_type(eb: leaf, s: chunk);
7078	index = btrfs_bg_flags_to_raid_index(flags: type);
7079	num_stripes = btrfs_chunk_num_stripes(eb: leaf, s: chunk);
7080
7081	#if BITS_PER_LONG == 32
7082	ret = check_32bit_meta_chunk(fs_info, logical, length, type);
7083	if (ret < 0)
7084	return ret;
7085	warn_32bit_meta_chunk(fs_info, logical, length, type);
7086	#endif
7087
7088	/*
7089	* Only need to verify chunk item if we're reading from sys chunk array,
7090	* as chunk item in tree block is already verified by tree-checker.
7091	*/
7092	if (leaf->start == BTRFS_SUPER_INFO_OFFSET) {
7093	ret = btrfs_check_chunk_valid(leaf, chunk, logical);
7094	if (ret)
7095	return ret;
7096	}
7097
7098	map = btrfs_find_chunk_map(fs_info, logical, length: 1);
7099
7100	/* already mapped? */
7101	if (map && map->start <= logical && map->start + map->chunk_len > logical) {
7102	btrfs_free_chunk_map(map);
7103	return 0;
7104	} else if (map) {
7105	btrfs_free_chunk_map(map);
7106	}
7107
7108	map = btrfs_alloc_chunk_map(num_stripes, GFP_NOFS);
7109	if (!map)
7110	return -ENOMEM;
7111
7112	map->start = logical;
7113	map->chunk_len = length;
7114	map->num_stripes = num_stripes;
7115	map->io_width = btrfs_chunk_io_width(eb: leaf, s: chunk);
7116	map->io_align = btrfs_chunk_io_align(eb: leaf, s: chunk);
7117	map->type = type;
7118	/*
7119	* We can't use the sub_stripes value, as for profiles other than
7120	* RAID10, they may have 0 as sub_stripes for filesystems created by
7121	* older mkfs (<v5.4).
7122	* In that case, it can cause divide-by-zero errors later.
7123	* Since currently sub_stripes is fixed for each profile, let's
7124	* use the trusted value instead.
7125	*/
7126	map->sub_stripes = btrfs_raid_array[index].sub_stripes;
7127	map->verified_stripes = 0;
7128	map->stripe_size = btrfs_calc_stripe_length(map);
7129	for (i = 0; i < num_stripes; i++) {
7130	map->stripes[i].physical =
7131	btrfs_stripe_offset_nr(eb: leaf, c: chunk, nr: i);
7132	devid = btrfs_stripe_devid_nr(eb: leaf, c: chunk, nr: i);
7133	args.devid = devid;
7134	read_extent_buffer(eb: leaf, dst: uuid, start: (unsigned long)
7135	btrfs_stripe_dev_uuid_nr(c: chunk, nr: i),
7136	BTRFS_UUID_SIZE);
7137	args.uuid = uuid;
7138	map->stripes[i].dev = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
7139	if (!map->stripes[i].dev) {
7140	map->stripes[i].dev = handle_missing_device(fs_info,
7141	devid, uuid);
7142	if (IS_ERR(ptr: map->stripes[i].dev)) {
7143	ret = PTR_ERR(ptr: map->stripes[i].dev);
7144	btrfs_free_chunk_map(map);
7145	return ret;
7146	}
7147	}
7148
7149	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
7150	addr: &(map->stripes[i].dev->dev_state));
7151	}
7152
7153	ret = btrfs_add_chunk_map(fs_info, map);
7154	if (ret < 0) {
7155	btrfs_err(fs_info,
7156	"failed to add chunk map, start=%llu len=%llu: %d",
7157	map->start, map->chunk_len, ret);
7158	}
7159
7160	return ret;
7161	}
7162
7163	static void fill_device_from_item(struct extent_buffer *leaf,
7164	struct btrfs_dev_item *dev_item,
7165	struct btrfs_device *device)
7166	{
7167	unsigned long ptr;
7168
7169	device->devid = btrfs_device_id(eb: leaf, s: dev_item);
7170	device->disk_total_bytes = btrfs_device_total_bytes(eb: leaf, s: dev_item);
7171	device->total_bytes = device->disk_total_bytes;
7172	device->commit_total_bytes = device->disk_total_bytes;
7173	device->bytes_used = btrfs_device_bytes_used(eb: leaf, s: dev_item);
7174	device->commit_bytes_used = device->bytes_used;
7175	device->type = btrfs_device_type(eb: leaf, s: dev_item);
7176	device->io_align = btrfs_device_io_align(eb: leaf, s: dev_item);
7177	device->io_width = btrfs_device_io_width(eb: leaf, s: dev_item);
7178	device->sector_size = btrfs_device_sector_size(eb: leaf, s: dev_item);
7179	WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID);
7180	clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, addr: &device->dev_state);
7181
7182	ptr = btrfs_device_uuid(d: dev_item);
7183	read_extent_buffer(eb: leaf, dst: device->uuid, start: ptr, BTRFS_UUID_SIZE);
7184	}
7185
7186	static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info,
7187	u8 *fsid)
7188	{
7189	struct btrfs_fs_devices *fs_devices;
7190	int ret;
7191
7192	lockdep_assert_held(&uuid_mutex);
7193	ASSERT(fsid);
7194
7195	/* This will match only for multi-device seed fs */
7196	list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list)
7197	if (!memcmp(p: fs_devices->fsid, q: fsid, BTRFS_FSID_SIZE))
7198	return fs_devices;
7199
7200
7201	fs_devices = find_fsid(fsid, NULL);
7202	if (!fs_devices) {
7203	if (!btrfs_test_opt(fs_info, DEGRADED))
7204	return ERR_PTR(error: -ENOENT);
7205
7206	fs_devices = alloc_fs_devices(fsid);
7207	if (IS_ERR(ptr: fs_devices))
7208	return fs_devices;
7209
7210	fs_devices->seeding = true;
7211	fs_devices->opened = 1;
7212	return fs_devices;
7213	}
7214
7215	/*
7216	* Upon first call for a seed fs fsid, just create a private copy of the
7217	* respective fs_devices and anchor it at fs_info->fs_devices->seed_list
7218	*/
7219	fs_devices = clone_fs_devices(orig: fs_devices);
7220	if (IS_ERR(ptr: fs_devices))
7221	return fs_devices;
7222
7223	ret = open_fs_devices(fs_devices, BLK_OPEN_READ, holder: fs_info->bdev_holder);
7224	if (ret) {
7225	free_fs_devices(fs_devices);
7226	return ERR_PTR(error: ret);
7227	}
7228
7229	if (!fs_devices->seeding) {
7230	close_fs_devices(fs_devices);
7231	free_fs_devices(fs_devices);
7232	return ERR_PTR(error: -EINVAL);
7233	}
7234
7235	list_add(new: &fs_devices->seed_list, head: &fs_info->fs_devices->seed_list);
7236
7237	return fs_devices;
7238	}
7239
7240	static int read_one_dev(struct extent_buffer *leaf,
7241	struct btrfs_dev_item *dev_item)
7242	{
7243	BTRFS_DEV_LOOKUP_ARGS(args);
7244	struct btrfs_fs_info *fs_info = leaf->fs_info;
7245	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7246	struct btrfs_device *device;
7247	u64 devid;
7248	int ret;
7249	u8 fs_uuid[BTRFS_FSID_SIZE];
7250	u8 dev_uuid[BTRFS_UUID_SIZE];
7251
7252	devid = btrfs_device_id(eb: leaf, s: dev_item);
7253	args.devid = devid;
7254	read_extent_buffer(eb: leaf, dst: dev_uuid, start: btrfs_device_uuid(d: dev_item),
7255	BTRFS_UUID_SIZE);
7256	read_extent_buffer(eb: leaf, dst: fs_uuid, start: btrfs_device_fsid(d: dev_item),
7257	BTRFS_FSID_SIZE);
7258	args.uuid = dev_uuid;
7259	args.fsid = fs_uuid;
7260
7261	if (memcmp(p: fs_uuid, q: fs_devices->metadata_uuid, BTRFS_FSID_SIZE)) {
7262	fs_devices = open_seed_devices(fs_info, fsid: fs_uuid);
7263	if (IS_ERR(ptr: fs_devices))
7264	return PTR_ERR(ptr: fs_devices);
7265	}
7266
7267	device = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
7268	if (!device) {
7269	if (!btrfs_test_opt(fs_info, DEGRADED)) {
7270	btrfs_report_missing_device(fs_info, devid,
7271	uuid: dev_uuid, error: true);
7272	return -ENOENT;
7273	}
7274
7275	device = add_missing_dev(fs_devices, devid, dev_uuid);
7276	if (IS_ERR(ptr: device)) {
7277	btrfs_err(fs_info,
7278	"failed to add missing dev %llu: %ld",
7279	devid, PTR_ERR(device));
7280	return PTR_ERR(ptr: device);
7281	}
7282	btrfs_report_missing_device(fs_info, devid, uuid: dev_uuid, error: false);
7283	} else {
7284	if (!device->bdev) {
7285	if (!btrfs_test_opt(fs_info, DEGRADED)) {
7286	btrfs_report_missing_device(fs_info,
7287	devid, uuid: dev_uuid, error: true);
7288	return -ENOENT;
7289	}
7290	btrfs_report_missing_device(fs_info, devid,
7291	uuid: dev_uuid, error: false);
7292	}
7293
7294	if (!device->bdev &&
7295	!test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
7296	/*
7297	* this happens when a device that was properly setup
7298	* in the device info lists suddenly goes bad.
7299	* device->bdev is NULL, and so we have to set
7300	* device->missing to one here
7301	*/
7302	device->fs_devices->missing_devices++;
7303	set_bit(BTRFS_DEV_STATE_MISSING, addr: &device->dev_state);
7304	}
7305
7306	/* Move the device to its own fs_devices */
7307	if (device->fs_devices != fs_devices) {
7308	ASSERT(test_bit(BTRFS_DEV_STATE_MISSING,
7309	&device->dev_state));
7310
7311	list_move(list: &device->dev_list, head: &fs_devices->devices);
7312	device->fs_devices->num_devices--;
7313	fs_devices->num_devices++;
7314
7315	device->fs_devices->missing_devices--;
7316	fs_devices->missing_devices++;
7317
7318	device->fs_devices = fs_devices;
7319	}
7320	}
7321
7322	if (device->fs_devices != fs_info->fs_devices) {
7323	BUG_ON(test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state));
7324	if (device->generation !=
7325	btrfs_device_generation(eb: leaf, s: dev_item))
7326	return -EINVAL;
7327	}
7328
7329	fill_device_from_item(leaf, dev_item, device);
7330	if (device->bdev) {
7331	u64 max_total_bytes = bdev_nr_bytes(bdev: device->bdev);
7332
7333	if (device->total_bytes > max_total_bytes) {
7334	btrfs_err(fs_info,
7335	"device total_bytes should be at most %llu but found %llu",
7336	max_total_bytes, device->total_bytes);
7337	return -EINVAL;
7338	}
7339	}
7340	set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, addr: &device->dev_state);
7341	if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
7342	!test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
7343	device->fs_devices->total_rw_bytes += device->total_bytes;
7344	atomic64_add(i: device->total_bytes - device->bytes_used,
7345	v: &fs_info->free_chunk_space);
7346	}
7347	ret = 0;
7348	return ret;
7349	}
7350
7351	int btrfs_read_sys_array(struct btrfs_fs_info *fs_info)
7352	{
7353	struct btrfs_super_block *super_copy = fs_info->super_copy;
7354	struct extent_buffer *sb;
7355	struct btrfs_disk_key *disk_key;
7356	struct btrfs_chunk *chunk;
7357	u8 *array_ptr;
7358	unsigned long sb_array_offset;
7359	int ret = 0;
7360	u32 num_stripes;
7361	u32 array_size;
7362	u32 len = 0;
7363	u32 cur_offset;
7364	u64 type;
7365	struct btrfs_key key;
7366
7367	ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize);
7368
7369	/*
7370	* We allocated a dummy extent, just to use extent buffer accessors.
7371	* There will be unused space after BTRFS_SUPER_INFO_SIZE, but
7372	* that's fine, we will not go beyond system chunk array anyway.
7373	*/
7374	sb = alloc_dummy_extent_buffer(fs_info, BTRFS_SUPER_INFO_OFFSET);
7375	if (!sb)
7376	return -ENOMEM;
7377	set_extent_buffer_uptodate(sb);
7378
7379	write_extent_buffer(eb: sb, src: super_copy, start: 0, BTRFS_SUPER_INFO_SIZE);
7380	array_size = btrfs_super_sys_array_size(s: super_copy);
7381
7382	array_ptr = super_copy->sys_chunk_array;
7383	sb_array_offset = offsetof(struct btrfs_super_block, sys_chunk_array);
7384	cur_offset = 0;
7385
7386	while (cur_offset < array_size) {
7387	disk_key = (struct btrfs_disk_key *)array_ptr;
7388	len = sizeof(*disk_key);
7389	if (cur_offset + len > array_size)
7390	goto out_short_read;
7391
7392	btrfs_disk_key_to_cpu(cpu_key: &key, disk_key);
7393
7394	array_ptr += len;
7395	sb_array_offset += len;
7396	cur_offset += len;
7397
7398	if (key.type != BTRFS_CHUNK_ITEM_KEY) {
7399	btrfs_err(fs_info,
7400	"unexpected item type %u in sys_array at offset %u",
7401	(u32)key.type, cur_offset);
7402	ret = -EIO;
7403	break;
7404	}
7405
7406	chunk = (struct btrfs_chunk *)sb_array_offset;
7407	/*
7408	* At least one btrfs_chunk with one stripe must be present,
7409	* exact stripe count check comes afterwards
7410	*/
7411	len = btrfs_chunk_item_size(num_stripes: 1);
7412	if (cur_offset + len > array_size)
7413	goto out_short_read;
7414
7415	num_stripes = btrfs_chunk_num_stripes(eb: sb, s: chunk);
7416	if (!num_stripes) {
7417	btrfs_err(fs_info,
7418	"invalid number of stripes %u in sys_array at offset %u",
7419	num_stripes, cur_offset);
7420	ret = -EIO;
7421	break;
7422	}
7423
7424	type = btrfs_chunk_type(eb: sb, s: chunk);
7425	if ((type & BTRFS_BLOCK_GROUP_SYSTEM) == 0) {
7426	btrfs_err(fs_info,
7427	"invalid chunk type %llu in sys_array at offset %u",
7428	type, cur_offset);
7429	ret = -EIO;
7430	break;
7431	}
7432
7433	len = btrfs_chunk_item_size(num_stripes);
7434	if (cur_offset + len > array_size)
7435	goto out_short_read;
7436
7437	ret = read_one_chunk(key: &key, leaf: sb, chunk);
7438	if (ret)
7439	break;
7440
7441	array_ptr += len;
7442	sb_array_offset += len;
7443	cur_offset += len;
7444	}
7445	clear_extent_buffer_uptodate(eb: sb);
7446	free_extent_buffer_stale(eb: sb);
7447	return ret;
7448
7449	out_short_read:
7450	btrfs_err(fs_info, "sys_array too short to read %u bytes at offset %u",
7451	len, cur_offset);
7452	clear_extent_buffer_uptodate(eb: sb);
7453	free_extent_buffer_stale(eb: sb);
7454	return -EIO;
7455	}
7456
7457	/*
7458	* Check if all chunks in the fs are OK for read-write degraded mount
7459	*
7460	* If the @failing_dev is specified, it's accounted as missing.
7461	*
7462	* Return true if all chunks meet the minimal RW mount requirements.
7463	* Return false if any chunk doesn't meet the minimal RW mount requirements.
7464	*/
7465	bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info,
7466	struct btrfs_device *failing_dev)
7467	{
7468	struct btrfs_chunk_map *map;
7469	u64 next_start;
7470	bool ret = true;
7471
7472	map = btrfs_find_chunk_map(fs_info, logical: 0, U64_MAX);
7473	/* No chunk at all? Return false anyway */
7474	if (!map) {
7475	ret = false;
7476	goto out;
7477	}
7478	while (map) {
7479	int missing = 0;
7480	int max_tolerated;
7481	int i;
7482
7483	max_tolerated =
7484	btrfs_get_num_tolerated_disk_barrier_failures(
7485	flags: map->type);
7486	for (i = 0; i < map->num_stripes; i++) {
7487	struct btrfs_device *dev = map->stripes[i].dev;
7488
7489	if (!dev || !dev->bdev ||
7490	test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) ||
7491	dev->last_flush_error)
7492	missing++;
7493	else if (failing_dev && failing_dev == dev)
7494	missing++;
7495	}
7496	if (missing > max_tolerated) {
7497	if (!failing_dev)
7498	btrfs_warn(fs_info,
7499	"chunk %llu missing %d devices, max tolerance is %d for writable mount",
7500	map->start, missing, max_tolerated);
7501	btrfs_free_chunk_map(map);
7502	ret = false;
7503	goto out;
7504	}
7505	next_start = map->start + map->chunk_len;
7506	btrfs_free_chunk_map(map);
7507
7508	map = btrfs_find_chunk_map(fs_info, logical: next_start, U64_MAX - next_start);
7509	}
7510	out:
7511	return ret;
7512	}
7513
7514	static void readahead_tree_node_children(struct extent_buffer *node)
7515	{
7516	int i;
7517	const int nr_items = btrfs_header_nritems(eb: node);
7518
7519	for (i = 0; i < nr_items; i++)
7520	btrfs_readahead_node_child(node, slot: i);
7521	}
7522
7523	int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
7524	{
7525	struct btrfs_root *root = fs_info->chunk_root;
7526	struct btrfs_path *path;
7527	struct extent_buffer *leaf;
7528	struct btrfs_key key;
7529	struct btrfs_key found_key;
7530	int ret;
7531	int slot;
7532	int iter_ret = 0;
7533	u64 total_dev = 0;
7534	u64 last_ra_node = 0;
7535
7536	path = btrfs_alloc_path();
7537	if (!path)
7538	return -ENOMEM;
7539
7540	/*
7541	* uuid_mutex is needed only if we are mounting a sprout FS
7542	* otherwise we don't need it.
7543	*/
7544	mutex_lock(&uuid_mutex);
7545
7546	/*
7547	* It is possible for mount and umount to race in such a way that
7548	* we execute this code path, but open_fs_devices failed to clear
7549	* total_rw_bytes. We certainly want it cleared before reading the
7550	* device items, so clear it here.
7551	*/
7552	fs_info->fs_devices->total_rw_bytes = 0;
7553
7554	/*
7555	* Lockdep complains about possible circular locking dependency between
7556	* a disk's open_mutex (struct gendisk.open_mutex), the rw semaphores
7557	* used for freeze procection of a fs (struct super_block.s_writers),
7558	* which we take when starting a transaction, and extent buffers of the
7559	* chunk tree if we call read_one_dev() while holding a lock on an
7560	* extent buffer of the chunk tree. Since we are mounting the filesystem
7561	* and at this point there can't be any concurrent task modifying the
7562	* chunk tree, to keep it simple, just skip locking on the chunk tree.
7563	*/
7564	ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags));
7565	path->skip_locking = 1;
7566
7567	/*
7568	* Read all device items, and then all the chunk items. All
7569	* device items are found before any chunk item (their object id
7570	* is smaller than the lowest possible object id for a chunk
7571	* item - BTRFS_FIRST_CHUNK_TREE_OBJECTID).
7572	*/
7573	key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
7574	key.offset = 0;
7575	key.type = 0;
7576	btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) {
7577	struct extent_buffer *node = path->nodes[1];
7578
7579	leaf = path->nodes[0];
7580	slot = path->slots[0];
7581
7582	if (node) {
7583	if (last_ra_node != node->start) {
7584	readahead_tree_node_children(node);
7585	last_ra_node = node->start;
7586	}
7587	}
7588	if (found_key.type == BTRFS_DEV_ITEM_KEY) {
7589	struct btrfs_dev_item *dev_item;
7590	dev_item = btrfs_item_ptr(leaf, slot,
7591	struct btrfs_dev_item);
7592	ret = read_one_dev(leaf, dev_item);
7593	if (ret)
7594	goto error;
7595	total_dev++;
7596	} else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) {
7597	struct btrfs_chunk *chunk;
7598
7599	/*
7600	* We are only called at mount time, so no need to take
7601	* fs_info->chunk_mutex. Plus, to avoid lockdep warnings,
7602	* we always lock first fs_info->chunk_mutex before
7603	* acquiring any locks on the chunk tree. This is a
7604	* requirement for chunk allocation, see the comment on
7605	* top of btrfs_chunk_alloc() for details.
7606	*/
7607	chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
7608	ret = read_one_chunk(key: &found_key, leaf, chunk);
7609	if (ret)
7610	goto error;
7611	}
7612	}
7613	/* Catch error found during iteration */
7614	if (iter_ret < 0) {
7615	ret = iter_ret;
7616	goto error;
7617	}
7618
7619	/*
7620	* After loading chunk tree, we've got all device information,
7621	* do another round of validation checks.
7622	*/
7623	if (total_dev != fs_info->fs_devices->total_devices) {
7624	btrfs_warn(fs_info,
7625	"super block num_devices %llu mismatch with DEV_ITEM count %llu, will be repaired on next transaction commit",
7626	btrfs_super_num_devices(fs_info->super_copy),
7627	total_dev);
7628	fs_info->fs_devices->total_devices = total_dev;
7629	btrfs_set_super_num_devices(s: fs_info->super_copy, val: total_dev);
7630	}
7631	if (btrfs_super_total_bytes(s: fs_info->super_copy) <
7632	fs_info->fs_devices->total_rw_bytes) {
7633	btrfs_err(fs_info,
7634	"super_total_bytes %llu mismatch with fs_devices total_rw_bytes %llu",
7635	btrfs_super_total_bytes(fs_info->super_copy),
7636	fs_info->fs_devices->total_rw_bytes);
7637	ret = -EINVAL;
7638	goto error;
7639	}
7640	ret = 0;
7641	error:
7642	mutex_unlock(lock: &uuid_mutex);
7643
7644	btrfs_free_path(p: path);
7645	return ret;
7646	}
7647
7648	int btrfs_init_devices_late(struct btrfs_fs_info *fs_info)
7649	{
7650	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7651	struct btrfs_device *device;
7652	int ret = 0;
7653
7654	fs_devices->fs_info = fs_info;
7655
7656	mutex_lock(&fs_devices->device_list_mutex);
7657	list_for_each_entry(device, &fs_devices->devices, dev_list)
7658	device->fs_info = fs_info;
7659
7660	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7661	list_for_each_entry(device, &seed_devs->devices, dev_list) {
7662	device->fs_info = fs_info;
7663	ret = btrfs_get_dev_zone_info(device, populate_cache: false);
7664	if (ret)
7665	break;
7666	}
7667
7668	seed_devs->fs_info = fs_info;
7669	}
7670	mutex_unlock(lock: &fs_devices->device_list_mutex);
7671
7672	return ret;
7673	}
7674
7675	static u64 btrfs_dev_stats_value(const struct extent_buffer *eb,
7676	const struct btrfs_dev_stats_item *ptr,
7677	int index)
7678	{
7679	u64 val;
7680
7681	read_extent_buffer(eb, dst: &val,
7682	offsetof(struct btrfs_dev_stats_item, values) +
7683	((unsigned long)ptr) + (index * sizeof(u64)),
7684	len: sizeof(val));
7685	return val;
7686	}
7687
7688	static void btrfs_set_dev_stats_value(struct extent_buffer *eb,
7689	struct btrfs_dev_stats_item *ptr,
7690	int index, u64 val)
7691	{
7692	write_extent_buffer(eb, src: &val,
7693	offsetof(struct btrfs_dev_stats_item, values) +
7694	((unsigned long)ptr) + (index * sizeof(u64)),
7695	len: sizeof(val));
7696	}
7697
7698	static int btrfs_device_init_dev_stats(struct btrfs_device *device,
7699	struct btrfs_path *path)
7700	{
7701	struct btrfs_dev_stats_item *ptr;
7702	struct extent_buffer *eb;
7703	struct btrfs_key key;
7704	int item_size;
7705	int i, ret, slot;
7706
7707	if (!device->fs_info->dev_root)
7708	return 0;
7709
7710	key.objectid = BTRFS_DEV_STATS_OBJECTID;
7711	key.type = BTRFS_PERSISTENT_ITEM_KEY;
7712	key.offset = device->devid;
7713	ret = btrfs_search_slot(NULL, root: device->fs_info->dev_root, key: &key, p: path, ins_len: 0, cow: 0);
7714	if (ret) {
7715	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7716	btrfs_dev_stat_set(dev: device, index: i, val: 0);
7717	device->dev_stats_valid = 1;
7718	btrfs_release_path(p: path);
7719	return ret < 0 ? ret : 0;
7720	}
7721	slot = path->slots[0];
7722	eb = path->nodes[0];
7723	item_size = btrfs_item_size(eb, slot);
7724
7725	ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_stats_item);
7726
7727	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7728	if (item_size >= (1 + i) * sizeof(__le64))
7729	btrfs_dev_stat_set(dev: device, index: i,
7730	val: btrfs_dev_stats_value(eb, ptr, index: i));
7731	else
7732	btrfs_dev_stat_set(dev: device, index: i, val: 0);
7733	}
7734
7735	device->dev_stats_valid = 1;
7736	btrfs_dev_stat_print_on_load(device);
7737	btrfs_release_path(p: path);
7738
7739	return 0;
7740	}
7741
7742	int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
7743	{
7744	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7745	struct btrfs_device *device;
7746	struct btrfs_path *path = NULL;
7747	int ret = 0;
7748
7749	path = btrfs_alloc_path();
7750	if (!path)
7751	return -ENOMEM;
7752
7753	mutex_lock(&fs_devices->device_list_mutex);
7754	list_for_each_entry(device, &fs_devices->devices, dev_list) {
7755	ret = btrfs_device_init_dev_stats(device, path);
7756	if (ret)
7757	goto out;
7758	}
7759	list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7760	list_for_each_entry(device, &seed_devs->devices, dev_list) {
7761	ret = btrfs_device_init_dev_stats(device, path);
7762	if (ret)
7763	goto out;
7764	}
7765	}
7766	out:
7767	mutex_unlock(lock: &fs_devices->device_list_mutex);
7768
7769	btrfs_free_path(p: path);
7770	return ret;
7771	}
7772
7773	static int update_dev_stat_item(struct btrfs_trans_handle *trans,
7774	struct btrfs_device *device)
7775	{
7776	struct btrfs_fs_info *fs_info = trans->fs_info;
7777	struct btrfs_root *dev_root = fs_info->dev_root;
7778	struct btrfs_path *path;
7779	struct btrfs_key key;
7780	struct extent_buffer *eb;
7781	struct btrfs_dev_stats_item *ptr;
7782	int ret;
7783	int i;
7784
7785	key.objectid = BTRFS_DEV_STATS_OBJECTID;
7786	key.type = BTRFS_PERSISTENT_ITEM_KEY;
7787	key.offset = device->devid;
7788
7789	path = btrfs_alloc_path();
7790	if (!path)
7791	return -ENOMEM;
7792	ret = btrfs_search_slot(trans, root: dev_root, key: &key, p: path, ins_len: -1, cow: 1);
7793	if (ret < 0) {
7794	btrfs_warn_in_rcu(fs_info,
7795	"error %d while searching for dev_stats item for device %s",
7796	ret, btrfs_dev_name(device));
7797	goto out;
7798	}
7799
7800	if (ret == 0 &&
7801	btrfs_item_size(eb: path->nodes[0], slot: path->slots[0]) < sizeof(*ptr)) {
7802	/* need to delete old one and insert a new one */
7803	ret = btrfs_del_item(trans, root: dev_root, path);
7804	if (ret != 0) {
7805	btrfs_warn_in_rcu(fs_info,
7806	"delete too small dev_stats item for device %s failed %d",
7807	btrfs_dev_name(device), ret);
7808	goto out;
7809	}
7810	ret = 1;
7811	}
7812
7813	if (ret == 1) {
7814	/* need to insert a new item */
7815	btrfs_release_path(p: path);
7816	ret = btrfs_insert_empty_item(trans, root: dev_root, path,
7817	key: &key, data_size: sizeof(*ptr));
7818	if (ret < 0) {
7819	btrfs_warn_in_rcu(fs_info,
7820	"insert dev_stats item for device %s failed %d",
7821	btrfs_dev_name(device), ret);
7822	goto out;
7823	}
7824	}
7825
7826	eb = path->nodes[0];
7827	ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item);
7828	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7829	btrfs_set_dev_stats_value(eb, ptr, index: i,
7830	val: btrfs_dev_stat_read(dev: device, index: i));
7831	btrfs_mark_buffer_dirty(trans, buf: eb);
7832
7833	out:
7834	btrfs_free_path(p: path);
7835	return ret;
7836	}
7837
7838	/*
7839	* called from commit_transaction. Writes all changed device stats to disk.
7840	*/
7841	int btrfs_run_dev_stats(struct btrfs_trans_handle *trans)
7842	{
7843	struct btrfs_fs_info *fs_info = trans->fs_info;
7844	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7845	struct btrfs_device *device;
7846	int stats_cnt;
7847	int ret = 0;
7848
7849	mutex_lock(&fs_devices->device_list_mutex);
7850	list_for_each_entry(device, &fs_devices->devices, dev_list) {
7851	stats_cnt = atomic_read(v: &device->dev_stats_ccnt);
7852	if (!device->dev_stats_valid || stats_cnt == 0)
7853	continue;
7854
7855
7856	/*
7857	* There is a LOAD-LOAD control dependency between the value of
7858	* dev_stats_ccnt and updating the on-disk values which requires
7859	* reading the in-memory counters. Such control dependencies
7860	* require explicit read memory barriers.
7861	*
7862	* This memory barriers pairs with smp_mb__before_atomic in
7863	* btrfs_dev_stat_inc/btrfs_dev_stat_set and with the full
7864	* barrier implied by atomic_xchg in
7865	* btrfs_dev_stats_read_and_reset
7866	*/
7867	smp_rmb();
7868
7869	ret = update_dev_stat_item(trans, device);
7870	if (!ret)
7871	atomic_sub(i: stats_cnt, v: &device->dev_stats_ccnt);
7872	}
7873	mutex_unlock(lock: &fs_devices->device_list_mutex);
7874
7875	return ret;
7876	}
7877
7878	void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)
7879	{
7880	btrfs_dev_stat_inc(dev, index);
7881
7882	if (!dev->dev_stats_valid)
7883	return;
7884	btrfs_err_rl_in_rcu(dev->fs_info,
7885	"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7886	btrfs_dev_name(dev),
7887	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7888	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7889	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7890	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7891	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7892	}
7893
7894	static void btrfs_dev_stat_print_on_load(struct btrfs_device *dev)
7895	{
7896	int i;
7897
7898	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7899	if (btrfs_dev_stat_read(dev, index: i) != 0)
7900	break;
7901	if (i == BTRFS_DEV_STAT_VALUES_MAX)
7902	return; /* all values == 0, suppress message */
7903
7904	btrfs_info_in_rcu(dev->fs_info,
7905	"bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7906	btrfs_dev_name(dev),
7907	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7908	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7909	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7910	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7911	btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7912	}
7913
7914	int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info,
7915	struct btrfs_ioctl_get_dev_stats *stats)
7916	{
7917	BTRFS_DEV_LOOKUP_ARGS(args);
7918	struct btrfs_device *dev;
7919	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7920	int i;
7921
7922	mutex_lock(&fs_devices->device_list_mutex);
7923	args.devid = stats->devid;
7924	dev = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
7925	mutex_unlock(lock: &fs_devices->device_list_mutex);
7926
7927	if (!dev) {
7928	btrfs_warn(fs_info, "get dev_stats failed, device not found");
7929	return -ENODEV;
7930	} else if (!dev->dev_stats_valid) {
7931	btrfs_warn(fs_info, "get dev_stats failed, not yet valid");
7932	return -ENODEV;
7933	} else if (stats->flags & BTRFS_DEV_STATS_RESET) {
7934	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7935	if (stats->nr_items > i)
7936	stats->values[i] =
7937	btrfs_dev_stat_read_and_reset(dev, index: i);
7938	else
7939	btrfs_dev_stat_set(dev, index: i, val: 0);
7940	}
7941	btrfs_info(fs_info, "device stats zeroed by %s (%d)",
7942	current->comm, task_pid_nr(current));
7943	} else {
7944	for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7945	if (stats->nr_items > i)
7946	stats->values[i] = btrfs_dev_stat_read(dev, index: i);
7947	}
7948	if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX)
7949	stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX;
7950	return 0;
7951	}
7952
7953	/*
7954	* Update the size and bytes used for each device where it changed. This is
7955	* delayed since we would otherwise get errors while writing out the
7956	* superblocks.
7957	*
7958	* Must be invoked during transaction commit.
7959	*/
7960	void btrfs_commit_device_sizes(struct btrfs_transaction *trans)
7961	{
7962	struct btrfs_device *curr, *next;
7963
7964	ASSERT(trans->state == TRANS_STATE_COMMIT_DOING);
7965
7966	if (list_empty(head: &trans->dev_update_list))
7967	return;
7968
7969	/*
7970	* We don't need the device_list_mutex here. This list is owned by the
7971	* transaction and the transaction must complete before the device is
7972	* released.
7973	*/
7974	mutex_lock(&trans->fs_info->chunk_mutex);
7975	list_for_each_entry_safe(curr, next, &trans->dev_update_list,
7976	post_commit_list) {
7977	list_del_init(entry: &curr->post_commit_list);
7978	curr->commit_total_bytes = curr->disk_total_bytes;
7979	curr->commit_bytes_used = curr->bytes_used;
7980	}
7981	mutex_unlock(lock: &trans->fs_info->chunk_mutex);
7982	}
7983
7984	/*
7985	* Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10.
7986	*/
7987	int btrfs_bg_type_to_factor(u64 flags)
7988	{
7989	const int index = btrfs_bg_flags_to_raid_index(flags);
7990
7991	return btrfs_raid_array[index].ncopies;
7992	}
7993
7994
7995
7996	static int verify_one_dev_extent(struct btrfs_fs_info *fs_info,
7997	u64 chunk_offset, u64 devid,
7998	u64 physical_offset, u64 physical_len)
7999	{
8000	struct btrfs_dev_lookup_args args = { .devid = devid };
8001	struct btrfs_chunk_map *map;
8002	struct btrfs_device *dev;
8003	u64 stripe_len;
8004	bool found = false;
8005	int ret = 0;
8006	int i;
8007
8008	map = btrfs_find_chunk_map(fs_info, logical: chunk_offset, length: 1);
8009	if (!map) {
8010	btrfs_err(fs_info,
8011	"dev extent physical offset %llu on devid %llu doesn't have corresponding chunk",
8012	physical_offset, devid);
8013	ret = -EUCLEAN;
8014	goto out;
8015	}
8016
8017	stripe_len = btrfs_calc_stripe_length(map);
8018	if (physical_len != stripe_len) {
8019	btrfs_err(fs_info,
8020	"dev extent physical offset %llu on devid %llu length doesn't match chunk %llu, have %llu expect %llu",
8021	physical_offset, devid, map->start, physical_len,
8022	stripe_len);
8023	ret = -EUCLEAN;
8024	goto out;
8025	}
8026
8027	/*
8028	* Very old mkfs.btrfs (before v4.1) will not respect the reserved
8029	* space. Although kernel can handle it without problem, better to warn
8030	* the users.
8031	*/
8032	if (physical_offset < BTRFS_DEVICE_RANGE_RESERVED)
8033	btrfs_warn(fs_info,
8034	"devid %llu physical %llu len %llu inside the reserved space",
8035	devid, physical_offset, physical_len);
8036
8037	for (i = 0; i < map->num_stripes; i++) {
8038	if (map->stripes[i].dev->devid == devid &&
8039	map->stripes[i].physical == physical_offset) {
8040	found = true;
8041	if (map->verified_stripes >= map->num_stripes) {
8042	btrfs_err(fs_info,
8043	"too many dev extents for chunk %llu found",
8044	map->start);
8045	ret = -EUCLEAN;
8046	goto out;
8047	}
8048	map->verified_stripes++;
8049	break;
8050	}
8051	}
8052	if (!found) {
8053	btrfs_err(fs_info,
8054	"dev extent physical offset %llu devid %llu has no corresponding chunk",
8055	physical_offset, devid);
8056	ret = -EUCLEAN;
8057	}
8058
8059	/* Make sure no dev extent is beyond device boundary */
8060	dev = btrfs_find_device(fs_devices: fs_info->fs_devices, args: &args);
8061	if (!dev) {
8062	btrfs_err(fs_info, "failed to find devid %llu", devid);
8063	ret = -EUCLEAN;
8064	goto out;
8065	}
8066
8067	if (physical_offset + physical_len > dev->disk_total_bytes) {
8068	btrfs_err(fs_info,
8069	"dev extent devid %llu physical offset %llu len %llu is beyond device boundary %llu",
8070	devid, physical_offset, physical_len,
8071	dev->disk_total_bytes);
8072	ret = -EUCLEAN;
8073	goto out;
8074	}
8075
8076	if (dev->zone_info) {
8077	u64 zone_size = dev->zone_info->zone_size;
8078
8079	if (!IS_ALIGNED(physical_offset, zone_size) ||
8080	!IS_ALIGNED(physical_len, zone_size)) {
8081	btrfs_err(fs_info,
8082	"zoned: dev extent devid %llu physical offset %llu len %llu is not aligned to device zone",
8083	devid, physical_offset, physical_len);
8084	ret = -EUCLEAN;
8085	goto out;
8086	}
8087	}
8088
8089	out:
8090	btrfs_free_chunk_map(map);
8091	return ret;
8092	}
8093
8094	static int verify_chunk_dev_extent_mapping(struct btrfs_fs_info *fs_info)
8095	{
8096	struct rb_node *node;
8097	int ret = 0;
8098
8099	read_lock(&fs_info->mapping_tree_lock);
8100	for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) {
8101	struct btrfs_chunk_map *map;
8102
8103	map = rb_entry(node, struct btrfs_chunk_map, rb_node);
8104	if (map->num_stripes != map->verified_stripes) {
8105	btrfs_err(fs_info,
8106	"chunk %llu has missing dev extent, have %d expect %d",
8107	map->start, map->verified_stripes, map->num_stripes);
8108	ret = -EUCLEAN;
8109	goto out;
8110	}
8111	}
8112	out:
8113	read_unlock(&fs_info->mapping_tree_lock);
8114	return ret;
8115	}
8116
8117	/*
8118	* Ensure that all dev extents are mapped to correct chunk, otherwise
8119	* later chunk allocation/free would cause unexpected behavior.
8120	*
8121	* NOTE: This will iterate through the whole device tree, which should be of
8122	* the same size level as the chunk tree. This slightly increases mount time.
8123	*/
8124	int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info)
8125	{
8126	struct btrfs_path *path;
8127	struct btrfs_root *root = fs_info->dev_root;
8128	struct btrfs_key key;
8129	u64 prev_devid = 0;
8130	u64 prev_dev_ext_end = 0;
8131	int ret = 0;
8132
8133	/*
8134	* We don't have a dev_root because we mounted with ignorebadroots and
8135	* failed to load the root, so we want to skip the verification in this
8136	* case for sure.
8137	*
8138	* However if the dev root is fine, but the tree itself is corrupted
8139	* we'd still fail to mount. This verification is only to make sure
8140	* writes can happen safely, so instead just bypass this check
8141	* completely in the case of IGNOREBADROOTS.
8142	*/
8143	if (btrfs_test_opt(fs_info, IGNOREBADROOTS))
8144	return 0;
8145
8146	key.objectid = 1;
8147	key.type = BTRFS_DEV_EXTENT_KEY;
8148	key.offset = 0;
8149
8150	path = btrfs_alloc_path();
8151	if (!path)
8152	return -ENOMEM;
8153
8154	path->reada = READA_FORWARD;
8155	ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
8156	if (ret < 0)
8157	goto out;
8158
8159	if (path->slots[0] >= btrfs_header_nritems(eb: path->nodes[0])) {
8160	ret = btrfs_next_leaf(root, path);
8161	if (ret < 0)
8162	goto out;
8163	/* No dev extents at all? Not good */
8164	if (ret > 0) {
8165	ret = -EUCLEAN;
8166	goto out;
8167	}
8168	}
8169	while (1) {
8170	struct extent_buffer *leaf = path->nodes[0];
8171	struct btrfs_dev_extent *dext;
8172	int slot = path->slots[0];
8173	u64 chunk_offset;
8174	u64 physical_offset;
8175	u64 physical_len;
8176	u64 devid;
8177
8178	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: slot);
8179	if (key.type != BTRFS_DEV_EXTENT_KEY)
8180	break;
8181	devid = key.objectid;
8182	physical_offset = key.offset;
8183
8184	dext = btrfs_item_ptr(leaf, slot, struct btrfs_dev_extent);
8185	chunk_offset = btrfs_dev_extent_chunk_offset(eb: leaf, s: dext);
8186	physical_len = btrfs_dev_extent_length(eb: leaf, s: dext);
8187
8188	/* Check if this dev extent overlaps with the previous one */
8189	if (devid == prev_devid && physical_offset < prev_dev_ext_end) {
8190	btrfs_err(fs_info,
8191	"dev extent devid %llu physical offset %llu overlap with previous dev extent end %llu",
8192	devid, physical_offset, prev_dev_ext_end);
8193	ret = -EUCLEAN;
8194	goto out;
8195	}
8196
8197	ret = verify_one_dev_extent(fs_info, chunk_offset, devid,
8198	physical_offset, physical_len);
8199	if (ret < 0)
8200	goto out;
8201	prev_devid = devid;
8202	prev_dev_ext_end = physical_offset + physical_len;
8203
8204	ret = btrfs_next_item(root, p: path);
8205	if (ret < 0)
8206	goto out;
8207	if (ret > 0) {
8208	ret = 0;
8209	break;
8210	}
8211	}
8212
8213	/* Ensure all chunks have corresponding dev extents */
8214	ret = verify_chunk_dev_extent_mapping(fs_info);
8215	out:
8216	btrfs_free_path(p: path);
8217	return ret;
8218	}
8219
8220	/*
8221	* Check whether the given block group or device is pinned by any inode being
8222	* used as a swapfile.
8223	*/
8224	bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr)
8225	{
8226	struct btrfs_swapfile_pin *sp;
8227	struct rb_node *node;
8228
8229	spin_lock(lock: &fs_info->swapfile_pins_lock);
8230	node = fs_info->swapfile_pins.rb_node;
8231	while (node) {
8232	sp = rb_entry(node, struct btrfs_swapfile_pin, node);
8233	if (ptr < sp->ptr)
8234	node = node->rb_left;
8235	else if (ptr > sp->ptr)
8236	node = node->rb_right;
8237	else
8238	break;
8239	}
8240	spin_unlock(lock: &fs_info->swapfile_pins_lock);
8241	return node != NULL;
8242	}
8243
8244	static int relocating_repair_kthread(void *data)
8245	{
8246	struct btrfs_block_group *cache = data;
8247	struct btrfs_fs_info *fs_info = cache->fs_info;
8248	u64 target;
8249	int ret = 0;
8250
8251	target = cache->start;
8252	btrfs_put_block_group(cache);
8253
8254	sb_start_write(sb: fs_info->sb);
8255	if (!btrfs_exclop_start(fs_info, type: BTRFS_EXCLOP_BALANCE)) {
8256	btrfs_info(fs_info,
8257	"zoned: skip relocating block group %llu to repair: EBUSY",
8258	target);
8259	sb_end_write(sb: fs_info->sb);
8260	return -EBUSY;
8261	}
8262
8263	mutex_lock(&fs_info->reclaim_bgs_lock);
8264
8265	/* Ensure block group still exists */
8266	cache = btrfs_lookup_block_group(info: fs_info, bytenr: target);
8267	if (!cache)
8268	goto out;
8269
8270	if (!test_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags))
8271	goto out;
8272
8273	ret = btrfs_may_alloc_data_chunk(fs_info, chunk_offset: target);
8274	if (ret < 0)
8275	goto out;
8276
8277	btrfs_info(fs_info,
8278	"zoned: relocating block group %llu to repair IO failure",
8279	target);
8280	ret = btrfs_relocate_chunk(fs_info, chunk_offset: target);
8281
8282	out:
8283	if (cache)
8284	btrfs_put_block_group(cache);
8285	mutex_unlock(lock: &fs_info->reclaim_bgs_lock);
8286	btrfs_exclop_finish(fs_info);
8287	sb_end_write(sb: fs_info->sb);
8288
8289	return ret;
8290	}
8291
8292	bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical)
8293	{
8294	struct btrfs_block_group *cache;
8295
8296	if (!btrfs_is_zoned(fs_info))
8297	return false;
8298
8299	/* Do not attempt to repair in degraded state */
8300	if (btrfs_test_opt(fs_info, DEGRADED))
8301	return true;
8302
8303	cache = btrfs_lookup_block_group(info: fs_info, bytenr: logical);
8304	if (!cache)
8305	return true;
8306
8307	if (test_and_set_bit(nr: BLOCK_GROUP_FLAG_RELOCATING_REPAIR, addr: &cache->runtime_flags)) {
8308	btrfs_put_block_group(cache);
8309	return true;
8310	}
8311
8312	kthread_run(relocating_repair_kthread, cache,
8313	"btrfs-relocating-repair");
8314
8315	return true;
8316	}
8317
8318	static void map_raid56_repair_block(struct btrfs_io_context *bioc,
8319	struct btrfs_io_stripe *smap,
8320	u64 logical)
8321	{
8322	int data_stripes = nr_bioc_data_stripes(bioc);
8323	int i;
8324
8325	for (i = 0; i < data_stripes; i++) {
8326	u64 stripe_start = bioc->full_stripe_logical +
8327	btrfs_stripe_nr_to_offset(stripe_nr: i);
8328
8329	if (logical >= stripe_start &&
8330	logical < stripe_start + BTRFS_STRIPE_LEN)
8331	break;
8332	}
8333	ASSERT(i < data_stripes);
8334	smap->dev = bioc->stripes[i].dev;
8335	smap->physical = bioc->stripes[i].physical +
8336	((logical - bioc->full_stripe_logical) &
8337	BTRFS_STRIPE_LEN_MASK);
8338	}
8339
8340	/*
8341	* Map a repair write into a single device.
8342	*
8343	* A repair write is triggered by read time repair or scrub, which would only
8344	* update the contents of a single device.
8345	* Not update any other mirrors nor go through RMW path.
8346	*
8347	* Callers should ensure:
8348	*
8349	* - Call btrfs_bio_counter_inc_blocked() first
8350	* - The range does not cross stripe boundary
8351	* - Has a valid @mirror_num passed in.
8352	*/
8353	int btrfs_map_repair_block(struct btrfs_fs_info *fs_info,
8354	struct btrfs_io_stripe *smap, u64 logical,
8355	u32 length, int mirror_num)
8356	{
8357	struct btrfs_io_context *bioc = NULL;
8358	u64 map_length = length;
8359	int mirror_ret = mirror_num;
8360	int ret;
8361
8362	ASSERT(mirror_num > 0);
8363
8364	ret = btrfs_map_block(fs_info, op: BTRFS_MAP_WRITE, logical, length: &map_length,
8365	bioc_ret: &bioc, smap, mirror_num_ret: &mirror_ret);
8366	if (ret < 0)
8367	return ret;
8368
8369	/* The map range should not cross stripe boundary. */
8370	ASSERT(map_length >= length);
8371
8372	/* Already mapped to single stripe. */
8373	if (!bioc)
8374	goto out;
8375
8376	/* Map the RAID56 multi-stripe writes to a single one. */
8377	if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
8378	map_raid56_repair_block(bioc, smap, logical);
8379	goto out;
8380	}
8381
8382	ASSERT(mirror_num <= bioc->num_stripes);
8383	smap->dev = bioc->stripes[mirror_num - 1].dev;
8384	smap->physical = bioc->stripes[mirror_num - 1].physical;
8385	out:
8386	btrfs_put_bioc(bioc);
8387	ASSERT(smap->dev);
8388	return 0;
8389	}
8390