1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* raid10.c : Multiple Devices driver for Linux
4	*
5	* Copyright (C) 2000-2004 Neil Brown
6	*
7	* RAID-10 support for md.
8	*
9	* Base on code in raid1.c. See raid1.c for further copyright information.
10	*/
11
12	#include <linux/slab.h>
13	#include <linux/delay.h>
14	#include <linux/blkdev.h>
15	#include <linux/module.h>
16	#include <linux/seq_file.h>
17	#include <linux/ratelimit.h>
18	#include <linux/kthread.h>
19	#include <linux/raid/md_p.h>
20	#include <trace/events/block.h>
21	#include "md.h"
22	#include "raid10.h"
23	#include "raid0.h"
24	#include "md-bitmap.h"
25
26	/*
27	* RAID10 provides a combination of RAID0 and RAID1 functionality.
28	* The layout of data is defined by
29	* chunk_size
30	* raid_disks
31	* near_copies (stored in low byte of layout)
32	* far_copies (stored in second byte of layout)
33	* far_offset (stored in bit 16 of layout )
34	* use_far_sets (stored in bit 17 of layout )
35	* use_far_sets_bugfixed (stored in bit 18 of layout )
36	*
37	* The data to be stored is divided into chunks using chunksize. Each device
38	* is divided into far_copies sections. In each section, chunks are laid out
39	* in a style similar to raid0, but near_copies copies of each chunk is stored
40	* (each on a different drive). The starting device for each section is offset
41	* near_copies from the starting device of the previous section. Thus there
42	* are (near_copies * far_copies) of each chunk, and each is on a different
43	* drive. near_copies and far_copies must be at least one, and their product
44	* is at most raid_disks.
45	*
46	* If far_offset is true, then the far_copies are handled a bit differently.
47	* The copies are still in different stripes, but instead of being very far
48	* apart on disk, there are adjacent stripes.
49	*
50	* The far and offset algorithms are handled slightly differently if
51	* 'use_far_sets' is true. In this case, the array's devices are grouped into
52	* sets that are (near_copies * far_copies) in size. The far copied stripes
53	* are still shifted by 'near_copies' devices, but this shifting stays confined
54	* to the set rather than the entire array. This is done to improve the number
55	* of device combinations that can fail without causing the array to fail.
56	* Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk
57	* on a device):
58	* A B C D A B C D E
59	* ... ...
60	* D A B C E A B C D
61	* Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s):
62	* [A B] [C D] [A B] [C D E]
63	* |...| |...| |...| | ... |
64	* [B A] [D C] [B A] [E C D]
65	*/
66
67	static void allow_barrier(struct r10conf *conf);
68	static void lower_barrier(struct r10conf *conf);
69	static int _enough(struct r10conf *conf, int previous, int ignore);
70	static int enough(struct r10conf *conf, int ignore);
71	static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
72	int *skipped);
73	static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
74	static void end_reshape_write(struct bio *bio);
75	static void end_reshape(struct r10conf *conf);
76
77	#define raid10_log(md, fmt, args...) \
78	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0)
79
80	#include "raid1-10.c"
81
82	#define NULL_CMD
83	#define cmd_before(conf, cmd) \
84	do { \
85	write_sequnlock_irq(&(conf)->resync_lock); \
86	cmd; \
87	} while (0)
88	#define cmd_after(conf) write_seqlock_irq(&(conf)->resync_lock)
89
90	#define wait_event_barrier_cmd(conf, cond, cmd) \
91	wait_event_cmd((conf)->wait_barrier, cond, cmd_before(conf, cmd), \
92	cmd_after(conf))
93
94	#define wait_event_barrier(conf, cond) \
95	wait_event_barrier_cmd(conf, cond, NULL_CMD)
96
97	/*
98	* for resync bio, r10bio pointer can be retrieved from the per-bio
99	* 'struct resync_pages'.
100	*/
101	static inline struct r10bio *get_resync_r10bio(struct bio *bio)
102	{
103	return get_resync_pages(bio)->raid_bio;
104	}
105
106	static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
107	{
108	struct r10conf *conf = data;
109	int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]);
110
111	/* allocate a r10bio with room for raid_disks entries in the
112	* bios array */
113	return kzalloc(size, flags: gfp_flags);
114	}
115
116	#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
117	/* amount of memory to reserve for resync requests */
118	#define RESYNC_WINDOW (1024*1024)
119	/* maximum number of concurrent requests, memory permitting */
120	#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
121	#define CLUSTER_RESYNC_WINDOW (32 * RESYNC_WINDOW)
122	#define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)
123
124	/*
125	* When performing a resync, we need to read and compare, so
126	* we need as many pages are there are copies.
127	* When performing a recovery, we need 2 bios, one for read,
128	* one for write (we recover only one drive per r10buf)
129	*
130	*/
131	static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
132	{
133	struct r10conf *conf = data;
134	struct r10bio *r10_bio;
135	struct bio *bio;
136	int j;
137	int nalloc, nalloc_rp;
138	struct resync_pages *rps;
139
140	r10_bio = r10bio_pool_alloc(gfp_flags, data: conf);
141	if (!r10_bio)
142	return NULL;
143
144	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
145	test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
146	nalloc = conf->copies; /* resync */
147	else
148	nalloc = 2; /* recovery */
149
150	/* allocate once for all bios */
151	if (!conf->have_replacement)
152	nalloc_rp = nalloc;
153	else
154	nalloc_rp = nalloc * 2;
155	rps = kmalloc_array(n: nalloc_rp, size: sizeof(struct resync_pages), flags: gfp_flags);
156	if (!rps)
157	goto out_free_r10bio;
158
159	/*
160	* Allocate bios.
161	*/
162	for (j = nalloc ; j-- ; ) {
163	bio = bio_kmalloc(RESYNC_PAGES, gfp_mask: gfp_flags);
164	if (!bio)
165	goto out_free_bio;
166	bio_init(bio, NULL, table: bio->bi_inline_vecs, RESYNC_PAGES, opf: 0);
167	r10_bio->devs[j].bio = bio;
168	if (!conf->have_replacement)
169	continue;
170	bio = bio_kmalloc(RESYNC_PAGES, gfp_mask: gfp_flags);
171	if (!bio)
172	goto out_free_bio;
173	bio_init(bio, NULL, table: bio->bi_inline_vecs, RESYNC_PAGES, opf: 0);
174	r10_bio->devs[j].repl_bio = bio;
175	}
176	/*
177	* Allocate RESYNC_PAGES data pages and attach them
178	* where needed.
179	*/
180	for (j = 0; j < nalloc; j++) {
181	struct bio *rbio = r10_bio->devs[j].repl_bio;
182	struct resync_pages *rp, *rp_repl;
183
184	rp = &rps[j];
185	if (rbio)
186	rp_repl = &rps[nalloc + j];
187
188	bio = r10_bio->devs[j].bio;
189
190	if (!j || test_bit(MD_RECOVERY_SYNC,
191	&conf->mddev->recovery)) {
192	if (resync_alloc_pages(rp, gfp_flags))
193	goto out_free_pages;
194	} else {
195	memcpy(rp, &rps[0], sizeof(*rp));
196	resync_get_all_pages(rp);
197	}
198
199	rp->raid_bio = r10_bio;
200	bio->bi_private = rp;
201	if (rbio) {
202	memcpy(rp_repl, rp, sizeof(*rp));
203	rbio->bi_private = rp_repl;
204	}
205	}
206
207	return r10_bio;
208
209	out_free_pages:
210	while (--j >= 0)
211	resync_free_pages(rp: &rps[j]);
212
213	j = 0;
214	out_free_bio:
215	for ( ; j < nalloc; j++) {
216	if (r10_bio->devs[j].bio)
217	bio_uninit(r10_bio->devs[j].bio);
218	kfree(objp: r10_bio->devs[j].bio);
219	if (r10_bio->devs[j].repl_bio)
220	bio_uninit(r10_bio->devs[j].repl_bio);
221	kfree(objp: r10_bio->devs[j].repl_bio);
222	}
223	kfree(objp: rps);
224	out_free_r10bio:
225	rbio_pool_free(rbio: r10_bio, data: conf);
226	return NULL;
227	}
228
229	static void r10buf_pool_free(void *__r10_bio, void *data)
230	{
231	struct r10conf *conf = data;
232	struct r10bio *r10bio = __r10_bio;
233	int j;
234	struct resync_pages *rp = NULL;
235
236	for (j = conf->copies; j--; ) {
237	struct bio *bio = r10bio->devs[j].bio;
238
239	if (bio) {
240	rp = get_resync_pages(bio);
241	resync_free_pages(rp);
242	bio_uninit(bio);
243	kfree(objp: bio);
244	}
245
246	bio = r10bio->devs[j].repl_bio;
247	if (bio) {
248	bio_uninit(bio);
249	kfree(objp: bio);
250	}
251	}
252
253	/* resync pages array stored in the 1st bio's .bi_private */
254	kfree(objp: rp);
255
256	rbio_pool_free(rbio: r10bio, data: conf);
257	}
258
259	static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
260	{
261	int i;
262
263	for (i = 0; i < conf->geo.raid_disks; i++) {
264	struct bio **bio = & r10_bio->devs[i].bio;
265	if (!BIO_SPECIAL(*bio))
266	bio_put(*bio);
267	*bio = NULL;
268	bio = &r10_bio->devs[i].repl_bio;
269	if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
270	bio_put(*bio);
271	*bio = NULL;
272	}
273	}
274
275	static void free_r10bio(struct r10bio *r10_bio)
276	{
277	struct r10conf *conf = r10_bio->mddev->private;
278
279	put_all_bios(conf, r10_bio);
280	mempool_free(element: r10_bio, pool: &conf->r10bio_pool);
281	}
282
283	static void put_buf(struct r10bio *r10_bio)
284	{
285	struct r10conf *conf = r10_bio->mddev->private;
286
287	mempool_free(element: r10_bio, pool: &conf->r10buf_pool);
288
289	lower_barrier(conf);
290	}
291
292	static void wake_up_barrier(struct r10conf *conf)
293	{
294	if (wq_has_sleeper(wq_head: &conf->wait_barrier))
295	wake_up(&conf->wait_barrier);
296	}
297
298	static void reschedule_retry(struct r10bio *r10_bio)
299	{
300	unsigned long flags;
301	struct mddev *mddev = r10_bio->mddev;
302	struct r10conf *conf = mddev->private;
303
304	spin_lock_irqsave(&conf->device_lock, flags);
305	list_add(new: &r10_bio->retry_list, head: &conf->retry_list);
306	conf->nr_queued ++;
307	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
308
309	/* wake up frozen array... */
310	wake_up(&conf->wait_barrier);
311
312	md_wakeup_thread(thread: mddev->thread);
313	}
314
315	/*
316	* raid_end_bio_io() is called when we have finished servicing a mirrored
317	* operation and are ready to return a success/failure code to the buffer
318	* cache layer.
319	*/
320	static void raid_end_bio_io(struct r10bio *r10_bio)
321	{
322	struct bio *bio = r10_bio->master_bio;
323	struct r10conf *conf = r10_bio->mddev->private;
324
325	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
326	bio->bi_status = BLK_STS_IOERR;
327
328	bio_endio(bio);
329	/*
330	* Wake up any possible resync thread that waits for the device
331	* to go idle.
332	*/
333	allow_barrier(conf);
334
335	free_r10bio(r10_bio);
336	}
337
338	/*
339	* Update disk head position estimator based on IRQ completion info.
340	*/
341	static inline void update_head_pos(int slot, struct r10bio *r10_bio)
342	{
343	struct r10conf *conf = r10_bio->mddev->private;
344
345	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
346	r10_bio->devs[slot].addr + (r10_bio->sectors);
347	}
348
349	/*
350	* Find the disk number which triggered given bio
351	*/
352	static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
353	struct bio *bio, int *slotp, int *replp)
354	{
355	int slot;
356	int repl = 0;
357
358	for (slot = 0; slot < conf->geo.raid_disks; slot++) {
359	if (r10_bio->devs[slot].bio == bio)
360	break;
361	if (r10_bio->devs[slot].repl_bio == bio) {
362	repl = 1;
363	break;
364	}
365	}
366
367	update_head_pos(slot, r10_bio);
368
369	if (slotp)
370	*slotp = slot;
371	if (replp)
372	*replp = repl;
373	return r10_bio->devs[slot].devnum;
374	}
375
376	static void raid10_end_read_request(struct bio *bio)
377	{
378	int uptodate = !bio->bi_status;
379	struct r10bio *r10_bio = bio->bi_private;
380	int slot;
381	struct md_rdev *rdev;
382	struct r10conf *conf = r10_bio->mddev->private;
383
384	slot = r10_bio->read_slot;
385	rdev = r10_bio->devs[slot].rdev;
386	/*
387	* this branch is our 'one mirror IO has finished' event handler:
388	*/
389	update_head_pos(slot, r10_bio);
390
391	if (uptodate) {
392	/*
393	* Set R10BIO_Uptodate in our master bio, so that
394	* we will return a good error code to the higher
395	* levels even if IO on some other mirrored buffer fails.
396	*
397	* The 'master' represents the composite IO operation to
398	* user-side. So if something waits for IO, then it will
399	* wait for the 'master' bio.
400	*/
401	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
402	} else {
403	/* If all other devices that store this block have
404	* failed, we want to return the error upwards rather
405	* than fail the last device. Here we redefine
406	* "uptodate" to mean "Don't want to retry"
407	*/
408	if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),
409	ignore: rdev->raid_disk))
410	uptodate = 1;
411	}
412	if (uptodate) {
413	raid_end_bio_io(r10_bio);
414	rdev_dec_pending(rdev, mddev: conf->mddev);
415	} else {
416	/*
417	* oops, read error - keep the refcount on the rdev
418	*/
419	pr_err_ratelimited("md/raid10:%s: %pg: rescheduling sector %llu\n",
420	mdname(conf->mddev),
421	rdev->bdev,
422	(unsigned long long)r10_bio->sector);
423	set_bit(nr: R10BIO_ReadError, addr: &r10_bio->state);
424	reschedule_retry(r10_bio);
425	}
426	}
427
428	static void close_write(struct r10bio *r10_bio)
429	{
430	/* clear the bitmap if all writes complete successfully */
431	md_bitmap_endwrite(bitmap: r10_bio->mddev->bitmap, offset: r10_bio->sector,
432	sectors: r10_bio->sectors,
433	success: !test_bit(R10BIO_Degraded, &r10_bio->state),
434	behind: 0);
435	md_write_end(mddev: r10_bio->mddev);
436	}
437
438	static void one_write_done(struct r10bio *r10_bio)
439	{
440	if (atomic_dec_and_test(v: &r10_bio->remaining)) {
441	if (test_bit(R10BIO_WriteError, &r10_bio->state))
442	reschedule_retry(r10_bio);
443	else {
444	close_write(r10_bio);
445	if (test_bit(R10BIO_MadeGood, &r10_bio->state))
446	reschedule_retry(r10_bio);
447	else
448	raid_end_bio_io(r10_bio);
449	}
450	}
451	}
452
453	static void raid10_end_write_request(struct bio *bio)
454	{
455	struct r10bio *r10_bio = bio->bi_private;
456	int dev;
457	int dec_rdev = 1;
458	struct r10conf *conf = r10_bio->mddev->private;
459	int slot, repl;
460	struct md_rdev *rdev = NULL;
461	struct bio *to_put = NULL;
462	bool discard_error;
463
464	discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;
465
466	dev = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
467
468	if (repl)
469	rdev = conf->mirrors[dev].replacement;
470	if (!rdev) {
471	smp_rmb();
472	repl = 0;
473	rdev = conf->mirrors[dev].rdev;
474	}
475	/*
476	* this branch is our 'one mirror IO has finished' event handler:
477	*/
478	if (bio->bi_status && !discard_error) {
479	if (repl)
480	/* Never record new bad blocks to replacement,
481	* just fail it.
482	*/
483	md_error(mddev: rdev->mddev, rdev);
484	else {
485	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
486	if (!test_and_set_bit(nr: WantReplacement, addr: &rdev->flags))
487	set_bit(nr: MD_RECOVERY_NEEDED,
488	addr: &rdev->mddev->recovery);
489
490	dec_rdev = 0;
491	if (test_bit(FailFast, &rdev->flags) &&
492	(bio->bi_opf & MD_FAILFAST)) {
493	md_error(mddev: rdev->mddev, rdev);
494	}
495
496	/*
497	* When the device is faulty, it is not necessary to
498	* handle write error.
499	*/
500	if (!test_bit(Faulty, &rdev->flags))
501	set_bit(nr: R10BIO_WriteError, addr: &r10_bio->state);
502	else {
503	/* Fail the request */
504	set_bit(nr: R10BIO_Degraded, addr: &r10_bio->state);
505	r10_bio->devs[slot].bio = NULL;
506	to_put = bio;
507	dec_rdev = 1;
508	}
509	}
510	} else {
511	/*
512	* Set R10BIO_Uptodate in our master bio, so that
513	* we will return a good error code for to the higher
514	* levels even if IO on some other mirrored buffer fails.
515	*
516	* The 'master' represents the composite IO operation to
517	* user-side. So if something waits for IO, then it will
518	* wait for the 'master' bio.
519	*/
520	sector_t first_bad;
521	int bad_sectors;
522
523	/*
524	* Do not set R10BIO_Uptodate if the current device is
525	* rebuilding or Faulty. This is because we cannot use
526	* such device for properly reading the data back (we could
527	* potentially use it, if the current write would have felt
528	* before rdev->recovery_offset, but for simplicity we don't
529	* check this here.
530	*/
531	if (test_bit(In_sync, &rdev->flags) &&
532	!test_bit(Faulty, &rdev->flags))
533	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
534
535	/* Maybe we can clear some bad blocks. */
536	if (is_badblock(rdev,
537	s: r10_bio->devs[slot].addr,
538	sectors: r10_bio->sectors,
539	first_bad: &first_bad, bad_sectors: &bad_sectors) && !discard_error) {
540	bio_put(bio);
541	if (repl)
542	r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
543	else
544	r10_bio->devs[slot].bio = IO_MADE_GOOD;
545	dec_rdev = 0;
546	set_bit(nr: R10BIO_MadeGood, addr: &r10_bio->state);
547	}
548	}
549
550	/*
551	*
552	* Let's see if all mirrored write operations have finished
553	* already.
554	*/
555	one_write_done(r10_bio);
556	if (dec_rdev)
557	rdev_dec_pending(rdev, mddev: conf->mddev);
558	if (to_put)
559	bio_put(to_put);
560	}
561
562	/*
563	* RAID10 layout manager
564	* As well as the chunksize and raid_disks count, there are two
565	* parameters: near_copies and far_copies.
566	* near_copies * far_copies must be <= raid_disks.
567	* Normally one of these will be 1.
568	* If both are 1, we get raid0.
569	* If near_copies == raid_disks, we get raid1.
570	*
571	* Chunks are laid out in raid0 style with near_copies copies of the
572	* first chunk, followed by near_copies copies of the next chunk and
573	* so on.
574	* If far_copies > 1, then after 1/far_copies of the array has been assigned
575	* as described above, we start again with a device offset of near_copies.
576	* So we effectively have another copy of the whole array further down all
577	* the drives, but with blocks on different drives.
578	* With this layout, and block is never stored twice on the one device.
579	*
580	* raid10_find_phys finds the sector offset of a given virtual sector
581	* on each device that it is on.
582	*
583	* raid10_find_virt does the reverse mapping, from a device and a
584	* sector offset to a virtual address
585	*/
586
587	static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
588	{
589	int n,f;
590	sector_t sector;
591	sector_t chunk;
592	sector_t stripe;
593	int dev;
594	int slot = 0;
595	int last_far_set_start, last_far_set_size;
596
597	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
598	last_far_set_start *= geo->far_set_size;
599
600	last_far_set_size = geo->far_set_size;
601	last_far_set_size += (geo->raid_disks % geo->far_set_size);
602
603	/* now calculate first sector/dev */
604	chunk = r10bio->sector >> geo->chunk_shift;
605	sector = r10bio->sector & geo->chunk_mask;
606
607	chunk *= geo->near_copies;
608	stripe = chunk;
609	dev = sector_div(stripe, geo->raid_disks);
610	if (geo->far_offset)
611	stripe *= geo->far_copies;
612
613	sector += stripe << geo->chunk_shift;
614
615	/* and calculate all the others */
616	for (n = 0; n < geo->near_copies; n++) {
617	int d = dev;
618	int set;
619	sector_t s = sector;
620	r10bio->devs[slot].devnum = d;
621	r10bio->devs[slot].addr = s;
622	slot++;
623
624	for (f = 1; f < geo->far_copies; f++) {
625	set = d / geo->far_set_size;
626	d += geo->near_copies;
627
628	if ((geo->raid_disks % geo->far_set_size) &&
629	(d > last_far_set_start)) {
630	d -= last_far_set_start;
631	d %= last_far_set_size;
632	d += last_far_set_start;
633	} else {
634	d %= geo->far_set_size;
635	d += geo->far_set_size * set;
636	}
637	s += geo->stride;
638	r10bio->devs[slot].devnum = d;
639	r10bio->devs[slot].addr = s;
640	slot++;
641	}
642	dev++;
643	if (dev >= geo->raid_disks) {
644	dev = 0;
645	sector += (geo->chunk_mask + 1);
646	}
647	}
648	}
649
650	static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
651	{
652	struct geom *geo = &conf->geo;
653
654	if (conf->reshape_progress != MaxSector &&
655	((r10bio->sector >= conf->reshape_progress) !=
656	conf->mddev->reshape_backwards)) {
657	set_bit(nr: R10BIO_Previous, addr: &r10bio->state);
658	geo = &conf->prev;
659	} else
660	clear_bit(nr: R10BIO_Previous, addr: &r10bio->state);
661
662	__raid10_find_phys(geo, r10bio);
663	}
664
665	static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
666	{
667	sector_t offset, chunk, vchunk;
668	/* Never use conf->prev as this is only called during resync
669	* or recovery, so reshape isn't happening
670	*/
671	struct geom *geo = &conf->geo;
672	int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;
673	int far_set_size = geo->far_set_size;
674	int last_far_set_start;
675
676	if (geo->raid_disks % geo->far_set_size) {
677	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
678	last_far_set_start *= geo->far_set_size;
679
680	if (dev >= last_far_set_start) {
681	far_set_size = geo->far_set_size;
682	far_set_size += (geo->raid_disks % geo->far_set_size);
683	far_set_start = last_far_set_start;
684	}
685	}
686
687	offset = sector & geo->chunk_mask;
688	if (geo->far_offset) {
689	int fc;
690	chunk = sector >> geo->chunk_shift;
691	fc = sector_div(chunk, geo->far_copies);
692	dev -= fc * geo->near_copies;
693	if (dev < far_set_start)
694	dev += far_set_size;
695	} else {
696	while (sector >= geo->stride) {
697	sector -= geo->stride;
698	if (dev < (geo->near_copies + far_set_start))
699	dev += far_set_size - geo->near_copies;
700	else
701	dev -= geo->near_copies;
702	}
703	chunk = sector >> geo->chunk_shift;
704	}
705	vchunk = chunk * geo->raid_disks + dev;
706	sector_div(vchunk, geo->near_copies);
707	return (vchunk << geo->chunk_shift) + offset;
708	}
709
710	/*
711	* This routine returns the disk from which the requested read should
712	* be done. There is a per-array 'next expected sequential IO' sector
713	* number - if this matches on the next IO then we use the last disk.
714	* There is also a per-disk 'last know head position' sector that is
715	* maintained from IRQ contexts, both the normal and the resync IO
716	* completion handlers update this position correctly. If there is no
717	* perfect sequential match then we pick the disk whose head is closest.
718	*
719	* If there are 2 mirrors in the same 2 devices, performance degrades
720	* because position is mirror, not device based.
721	*
722	* The rdev for the device selected will have nr_pending incremented.
723	*/
724
725	/*
726	* FIXME: possibly should rethink readbalancing and do it differently
727	* depending on near_copies / far_copies geometry.
728	*/
729	static struct md_rdev *read_balance(struct r10conf *conf,
730	struct r10bio *r10_bio,
731	int *max_sectors)
732	{
733	const sector_t this_sector = r10_bio->sector;
734	int disk, slot;
735	int sectors = r10_bio->sectors;
736	int best_good_sectors;
737	sector_t new_distance, best_dist;
738	struct md_rdev *best_dist_rdev, *best_pending_rdev, *rdev = NULL;
739	int do_balance;
740	int best_dist_slot, best_pending_slot;
741	bool has_nonrot_disk = false;
742	unsigned int min_pending;
743	struct geom *geo = &conf->geo;
744
745	raid10_find_phys(conf, r10bio: r10_bio);
746	rcu_read_lock();
747	best_dist_slot = -1;
748	min_pending = UINT_MAX;
749	best_dist_rdev = NULL;
750	best_pending_rdev = NULL;
751	best_dist = MaxSector;
752	best_good_sectors = 0;
753	do_balance = 1;
754	clear_bit(nr: R10BIO_FailFast, addr: &r10_bio->state);
755	/*
756	* Check if we can balance. We can balance on the whole
757	* device if no resync is going on (recovery is ok), or below
758	* the resync window. We take the first readable disk when
759	* above the resync window.
760	*/
761	if ((conf->mddev->recovery_cp < MaxSector
762	&& (this_sector + sectors >= conf->next_resync)) ||
763	(mddev_is_clustered(mddev: conf->mddev) &&
764	md_cluster_ops->area_resyncing(conf->mddev, READ, this_sector,
765	this_sector + sectors)))
766	do_balance = 0;
767
768	for (slot = 0; slot < conf->copies ; slot++) {
769	sector_t first_bad;
770	int bad_sectors;
771	sector_t dev_sector;
772	unsigned int pending;
773	bool nonrot;
774
775	if (r10_bio->devs[slot].bio == IO_BLOCKED)
776	continue;
777	disk = r10_bio->devs[slot].devnum;
778	rdev = rcu_dereference(conf->mirrors[disk].replacement);
779	if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
780	r10_bio->devs[slot].addr + sectors >
781	rdev->recovery_offset) {
782	/*
783	* Read replacement first to prevent reading both rdev
784	* and replacement as NULL during replacement replace
785	* rdev.
786	*/
787	smp_mb();
788	rdev = rcu_dereference(conf->mirrors[disk].rdev);
789	}
790	if (rdev == NULL ||
791	test_bit(Faulty, &rdev->flags))
792	continue;
793	if (!test_bit(In_sync, &rdev->flags) &&
794	r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
795	continue;
796
797	dev_sector = r10_bio->devs[slot].addr;
798	if (is_badblock(rdev, s: dev_sector, sectors,
799	first_bad: &first_bad, bad_sectors: &bad_sectors)) {
800	if (best_dist < MaxSector)
801	/* Already have a better slot */
802	continue;
803	if (first_bad <= dev_sector) {
804	/* Cannot read here. If this is the
805	* 'primary' device, then we must not read
806	* beyond 'bad_sectors' from another device.
807	*/
808	bad_sectors -= (dev_sector - first_bad);
809	if (!do_balance && sectors > bad_sectors)
810	sectors = bad_sectors;
811	if (best_good_sectors > sectors)
812	best_good_sectors = sectors;
813	} else {
814	sector_t good_sectors =
815	first_bad - dev_sector;
816	if (good_sectors > best_good_sectors) {
817	best_good_sectors = good_sectors;
818	best_dist_slot = slot;
819	best_dist_rdev = rdev;
820	}
821	if (!do_balance)
822	/* Must read from here */
823	break;
824	}
825	continue;
826	} else
827	best_good_sectors = sectors;
828
829	if (!do_balance)
830	break;
831
832	nonrot = bdev_nonrot(bdev: rdev->bdev);
833	has_nonrot_disk |= nonrot;
834	pending = atomic_read(v: &rdev->nr_pending);
835	if (min_pending > pending && nonrot) {
836	min_pending = pending;
837	best_pending_slot = slot;
838	best_pending_rdev = rdev;
839	}
840
841	if (best_dist_slot >= 0)
842	/* At least 2 disks to choose from so failfast is OK */
843	set_bit(nr: R10BIO_FailFast, addr: &r10_bio->state);
844	/* This optimisation is debatable, and completely destroys
845	* sequential read speed for 'far copies' arrays. So only
846	* keep it for 'near' arrays, and review those later.
847	*/
848	if (geo->near_copies > 1 && !pending)
849	new_distance = 0;
850
851	/* for far > 1 always use the lowest address */
852	else if (geo->far_copies > 1)
853	new_distance = r10_bio->devs[slot].addr;
854	else
855	new_distance = abs(r10_bio->devs[slot].addr -
856	conf->mirrors[disk].head_position);
857
858	if (new_distance < best_dist) {
859	best_dist = new_distance;
860	best_dist_slot = slot;
861	best_dist_rdev = rdev;
862	}
863	}
864	if (slot >= conf->copies) {
865	if (has_nonrot_disk) {
866	slot = best_pending_slot;
867	rdev = best_pending_rdev;
868	} else {
869	slot = best_dist_slot;
870	rdev = best_dist_rdev;
871	}
872	}
873
874	if (slot >= 0) {
875	atomic_inc(v: &rdev->nr_pending);
876	r10_bio->read_slot = slot;
877	} else
878	rdev = NULL;
879	rcu_read_unlock();
880	*max_sectors = best_good_sectors;
881
882	return rdev;
883	}
884
885	static void flush_pending_writes(struct r10conf *conf)
886	{
887	/* Any writes that have been queued but are awaiting
888	* bitmap updates get flushed here.
889	*/
890	spin_lock_irq(lock: &conf->device_lock);
891
892	if (conf->pending_bio_list.head) {
893	struct blk_plug plug;
894	struct bio *bio;
895
896	bio = bio_list_get(bl: &conf->pending_bio_list);
897	spin_unlock_irq(lock: &conf->device_lock);
898
899	/*
900	* As this is called in a wait_event() loop (see freeze_array),
901	* current->state might be TASK_UNINTERRUPTIBLE which will
902	* cause a warning when we prepare to wait again. As it is
903	* rare that this path is taken, it is perfectly safe to force
904	* us to go around the wait_event() loop again, so the warning
905	* is a false-positive. Silence the warning by resetting
906	* thread state
907	*/
908	__set_current_state(TASK_RUNNING);
909
910	blk_start_plug(&plug);
911	raid1_prepare_flush_writes(bitmap: conf->mddev->bitmap);
912	wake_up(&conf->wait_barrier);
913
914	while (bio) { /* submit pending writes */
915	struct bio *next = bio->bi_next;
916
917	raid1_submit_write(bio);
918	bio = next;
919	cond_resched();
920	}
921	blk_finish_plug(&plug);
922	} else
923	spin_unlock_irq(lock: &conf->device_lock);
924	}
925
926	/* Barriers....
927	* Sometimes we need to suspend IO while we do something else,
928	* either some resync/recovery, or reconfigure the array.
929	* To do this we raise a 'barrier'.
930	* The 'barrier' is a counter that can be raised multiple times
931	* to count how many activities are happening which preclude
932	* normal IO.
933	* We can only raise the barrier if there is no pending IO.
934	* i.e. if nr_pending == 0.
935	* We choose only to raise the barrier if no-one is waiting for the
936	* barrier to go down. This means that as soon as an IO request
937	* is ready, no other operations which require a barrier will start
938	* until the IO request has had a chance.
939	*
940	* So: regular IO calls 'wait_barrier'. When that returns there
941	* is no backgroup IO happening, It must arrange to call
942	* allow_barrier when it has finished its IO.
943	* backgroup IO calls must call raise_barrier. Once that returns
944	* there is no normal IO happeing. It must arrange to call
945	* lower_barrier when the particular background IO completes.
946	*/
947
948	static void raise_barrier(struct r10conf *conf, int force)
949	{
950	write_seqlock_irq(sl: &conf->resync_lock);
951
952	if (WARN_ON_ONCE(force && !conf->barrier))
953	force = false;
954
955	/* Wait until no block IO is waiting (unless 'force') */
956	wait_event_barrier(conf, force || !conf->nr_waiting);
957
958	/* block any new IO from starting */
959	WRITE_ONCE(conf->barrier, conf->barrier + 1);
960
961	/* Now wait for all pending IO to complete */
962	wait_event_barrier(conf, !atomic_read(&conf->nr_pending) &&
963	conf->barrier < RESYNC_DEPTH);
964
965	write_sequnlock_irq(sl: &conf->resync_lock);
966	}
967
968	static void lower_barrier(struct r10conf *conf)
969	{
970	unsigned long flags;
971
972	write_seqlock_irqsave(&conf->resync_lock, flags);
973	WRITE_ONCE(conf->barrier, conf->barrier - 1);
974	write_sequnlock_irqrestore(sl: &conf->resync_lock, flags);
975	wake_up(&conf->wait_barrier);
976	}
977
978	static bool stop_waiting_barrier(struct r10conf *conf)
979	{
980	struct bio_list *bio_list = current->bio_list;
981	struct md_thread *thread;
982
983	/* barrier is dropped */
984	if (!conf->barrier)
985	return true;
986
987	/*
988	* If there are already pending requests (preventing the barrier from
989	* rising completely), and the pre-process bio queue isn't empty, then
990	* don't wait, as we need to empty that queue to get the nr_pending
991	* count down.
992	*/
993	if (atomic_read(v: &conf->nr_pending) && bio_list &&
994	(!bio_list_empty(bl: &bio_list[0]) || !bio_list_empty(bl: &bio_list[1])))
995	return true;
996
997	/* daemon thread must exist while handling io */
998	thread = rcu_dereference_protected(conf->mddev->thread, true);
999	/*
1000	* move on if io is issued from raid10d(), nr_pending is not released
1001	* from original io(see handle_read_error()). All raise barrier is
1002	* blocked until this io is done.
1003	*/
1004	if (thread->tsk == current) {
1005	WARN_ON_ONCE(atomic_read(&conf->nr_pending) == 0);
1006	return true;
1007	}
1008
1009	return false;
1010	}
1011
1012	static bool wait_barrier_nolock(struct r10conf *conf)
1013	{
1014	unsigned int seq = read_seqbegin(sl: &conf->resync_lock);
1015
1016	if (READ_ONCE(conf->barrier))
1017	return false;
1018
1019	atomic_inc(v: &conf->nr_pending);
1020	if (!read_seqretry(sl: &conf->resync_lock, start: seq))
1021	return true;
1022
1023	if (atomic_dec_and_test(v: &conf->nr_pending))
1024	wake_up_barrier(conf);
1025
1026	return false;
1027	}
1028
1029	static bool wait_barrier(struct r10conf *conf, bool nowait)
1030	{
1031	bool ret = true;
1032
1033	if (wait_barrier_nolock(conf))
1034	return true;
1035
1036	write_seqlock_irq(sl: &conf->resync_lock);
1037	if (conf->barrier) {
1038	/* Return false when nowait flag is set */
1039	if (nowait) {
1040	ret = false;
1041	} else {
1042	conf->nr_waiting++;
1043	raid10_log(conf->mddev, "wait barrier");
1044	wait_event_barrier(conf, stop_waiting_barrier(conf));
1045	conf->nr_waiting--;
1046	}
1047	if (!conf->nr_waiting)
1048	wake_up(&conf->wait_barrier);
1049	}
1050	/* Only increment nr_pending when we wait */
1051	if (ret)
1052	atomic_inc(v: &conf->nr_pending);
1053	write_sequnlock_irq(sl: &conf->resync_lock);
1054	return ret;
1055	}
1056
1057	static void allow_barrier(struct r10conf *conf)
1058	{
1059	if ((atomic_dec_and_test(v: &conf->nr_pending)) ||
1060	(conf->array_freeze_pending))
1061	wake_up_barrier(conf);
1062	}
1063
1064	static void freeze_array(struct r10conf *conf, int extra)
1065	{
1066	/* stop syncio and normal IO and wait for everything to
1067	* go quiet.
1068	* We increment barrier and nr_waiting, and then
1069	* wait until nr_pending match nr_queued+extra
1070	* This is called in the context of one normal IO request
1071	* that has failed. Thus any sync request that might be pending
1072	* will be blocked by nr_pending, and we need to wait for
1073	* pending IO requests to complete or be queued for re-try.
1074	* Thus the number queued (nr_queued) plus this request (extra)
1075	* must match the number of pending IOs (nr_pending) before
1076	* we continue.
1077	*/
1078	write_seqlock_irq(sl: &conf->resync_lock);
1079	conf->array_freeze_pending++;
1080	WRITE_ONCE(conf->barrier, conf->barrier + 1);
1081	conf->nr_waiting++;
1082	wait_event_barrier_cmd(conf, atomic_read(&conf->nr_pending) ==
1083	conf->nr_queued + extra, flush_pending_writes(conf));
1084	conf->array_freeze_pending--;
1085	write_sequnlock_irq(sl: &conf->resync_lock);
1086	}
1087
1088	static void unfreeze_array(struct r10conf *conf)
1089	{
1090	/* reverse the effect of the freeze */
1091	write_seqlock_irq(sl: &conf->resync_lock);
1092	WRITE_ONCE(conf->barrier, conf->barrier - 1);
1093	conf->nr_waiting--;
1094	wake_up(&conf->wait_barrier);
1095	write_sequnlock_irq(sl: &conf->resync_lock);
1096	}
1097
1098	static sector_t choose_data_offset(struct r10bio *r10_bio,
1099	struct md_rdev *rdev)
1100	{
1101	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
1102	test_bit(R10BIO_Previous, &r10_bio->state))
1103	return rdev->data_offset;
1104	else
1105	return rdev->new_data_offset;
1106	}
1107
1108	static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
1109	{
1110	struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb, cb);
1111	struct mddev *mddev = plug->cb.data;
1112	struct r10conf *conf = mddev->private;
1113	struct bio *bio;
1114
1115	if (from_schedule) {
1116	spin_lock_irq(lock: &conf->device_lock);
1117	bio_list_merge(bl: &conf->pending_bio_list, bl2: &plug->pending);
1118	spin_unlock_irq(lock: &conf->device_lock);
1119	wake_up_barrier(conf);
1120	md_wakeup_thread(thread: mddev->thread);
1121	kfree(objp: plug);
1122	return;
1123	}
1124
1125	/* we aren't scheduling, so we can do the write-out directly. */
1126	bio = bio_list_get(bl: &plug->pending);
1127	raid1_prepare_flush_writes(bitmap: mddev->bitmap);
1128	wake_up_barrier(conf);
1129
1130	while (bio) { /* submit pending writes */
1131	struct bio *next = bio->bi_next;
1132
1133	raid1_submit_write(bio);
1134	bio = next;
1135	cond_resched();
1136	}
1137	kfree(objp: plug);
1138	}
1139
1140	/*
1141	* 1. Register the new request and wait if the reconstruction thread has put
1142	* up a bar for new requests. Continue immediately if no resync is active
1143	* currently.
1144	* 2. If IO spans the reshape position. Need to wait for reshape to pass.
1145	*/
1146	static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf,
1147	struct bio *bio, sector_t sectors)
1148	{
1149	/* Bail out if REQ_NOWAIT is set for the bio */
1150	if (!wait_barrier(conf, nowait: bio->bi_opf & REQ_NOWAIT)) {
1151	bio_wouldblock_error(bio);
1152	return false;
1153	}
1154	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1155	bio->bi_iter.bi_sector < conf->reshape_progress &&
1156	bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1157	allow_barrier(conf);
1158	if (bio->bi_opf & REQ_NOWAIT) {
1159	bio_wouldblock_error(bio);
1160	return false;
1161	}
1162	raid10_log(conf->mddev, "wait reshape");
1163	wait_event(conf->wait_barrier,
1164	conf->reshape_progress <= bio->bi_iter.bi_sector ||
1165	conf->reshape_progress >= bio->bi_iter.bi_sector +
1166	sectors);
1167	wait_barrier(conf, nowait: false);
1168	}
1169	return true;
1170	}
1171
1172	static void raid10_read_request(struct mddev *mddev, struct bio *bio,
1173	struct r10bio *r10_bio, bool io_accounting)
1174	{
1175	struct r10conf *conf = mddev->private;
1176	struct bio *read_bio;
1177	const enum req_op op = bio_op(bio);
1178	const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
1179	int max_sectors;
1180	struct md_rdev *rdev;
1181	char b[BDEVNAME_SIZE];
1182	int slot = r10_bio->read_slot;
1183	struct md_rdev *err_rdev = NULL;
1184	gfp_t gfp = GFP_NOIO;
1185
1186	if (slot >= 0 && r10_bio->devs[slot].rdev) {
1187	/*
1188	* This is an error retry, but we cannot
1189	* safely dereference the rdev in the r10_bio,
1190	* we must use the one in conf.
1191	* If it has already been disconnected (unlikely)
1192	* we lose the device name in error messages.
1193	*/
1194	int disk;
1195	/*
1196	* As we are blocking raid10, it is a little safer to
1197	* use __GFP_HIGH.
1198	*/
1199	gfp = GFP_NOIO | __GFP_HIGH;
1200
1201	rcu_read_lock();
1202	disk = r10_bio->devs[slot].devnum;
1203	err_rdev = rcu_dereference(conf->mirrors[disk].rdev);
1204	if (err_rdev)
1205	snprintf(buf: b, size: sizeof(b), fmt: "%pg", err_rdev->bdev);
1206	else {
1207	strcpy(p: b, q: "???");
1208	/* This never gets dereferenced */
1209	err_rdev = r10_bio->devs[slot].rdev;
1210	}
1211	rcu_read_unlock();
1212	}
1213
1214	if (!regular_request_wait(mddev, conf, bio, sectors: r10_bio->sectors))
1215	return;
1216	rdev = read_balance(conf, r10_bio, max_sectors: &max_sectors);
1217	if (!rdev) {
1218	if (err_rdev) {
1219	pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
1220	mdname(mddev), b,
1221	(unsigned long long)r10_bio->sector);
1222	}
1223	raid_end_bio_io(r10_bio);
1224	return;
1225	}
1226	if (err_rdev)
1227	pr_err_ratelimited("md/raid10:%s: %pg: redirecting sector %llu to another mirror\n",
1228	mdname(mddev),
1229	rdev->bdev,
1230	(unsigned long long)r10_bio->sector);
1231	if (max_sectors < bio_sectors(bio)) {
1232	struct bio *split = bio_split(bio, sectors: max_sectors,
1233	gfp, bs: &conf->bio_split);
1234	bio_chain(split, bio);
1235	allow_barrier(conf);
1236	submit_bio_noacct(bio);
1237	wait_barrier(conf, nowait: false);
1238	bio = split;
1239	r10_bio->master_bio = bio;
1240	r10_bio->sectors = max_sectors;
1241	}
1242	slot = r10_bio->read_slot;
1243
1244	if (io_accounting) {
1245	md_account_bio(mddev, bio: &bio);
1246	r10_bio->master_bio = bio;
1247	}
1248	read_bio = bio_alloc_clone(bdev: rdev->bdev, bio_src: bio, gfp, bs: &mddev->bio_set);
1249
1250	r10_bio->devs[slot].bio = read_bio;
1251	r10_bio->devs[slot].rdev = rdev;
1252
1253	read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
1254	choose_data_offset(r10_bio, rdev);
1255	read_bio->bi_end_io = raid10_end_read_request;
1256	read_bio->bi_opf = op | do_sync;
1257	if (test_bit(FailFast, &rdev->flags) &&
1258	test_bit(R10BIO_FailFast, &r10_bio->state))
1259	read_bio->bi_opf |= MD_FAILFAST;
1260	read_bio->bi_private = r10_bio;
1261
1262	if (mddev->gendisk)
1263	trace_block_bio_remap(bio: read_bio, dev: disk_devt(disk: mddev->gendisk),
1264	from: r10_bio->sector);
1265	submit_bio_noacct(bio: read_bio);
1266	return;
1267	}
1268
1269	static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
1270	struct bio *bio, bool replacement,
1271	int n_copy)
1272	{
1273	const enum req_op op = bio_op(bio);
1274	const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;
1275	const blk_opf_t do_fua = bio->bi_opf & REQ_FUA;
1276	unsigned long flags;
1277	struct r10conf *conf = mddev->private;
1278	struct md_rdev *rdev;
1279	int devnum = r10_bio->devs[n_copy].devnum;
1280	struct bio *mbio;
1281
1282	if (replacement) {
1283	rdev = conf->mirrors[devnum].replacement;
1284	if (rdev == NULL) {
1285	/* Replacement just got moved to main 'rdev' */
1286	smp_mb();
1287	rdev = conf->mirrors[devnum].rdev;
1288	}
1289	} else
1290	rdev = conf->mirrors[devnum].rdev;
1291
1292	mbio = bio_alloc_clone(bdev: rdev->bdev, bio_src: bio, GFP_NOIO, bs: &mddev->bio_set);
1293	if (replacement)
1294	r10_bio->devs[n_copy].repl_bio = mbio;
1295	else
1296	r10_bio->devs[n_copy].bio = mbio;
1297
1298	mbio->bi_iter.bi_sector = (r10_bio->devs[n_copy].addr +
1299	choose_data_offset(r10_bio, rdev));
1300	mbio->bi_end_io = raid10_end_write_request;
1301	mbio->bi_opf = op | do_sync | do_fua;
1302	if (!replacement && test_bit(FailFast,
1303	&conf->mirrors[devnum].rdev->flags)
1304	&& enough(conf, ignore: devnum))
1305	mbio->bi_opf |= MD_FAILFAST;
1306	mbio->bi_private = r10_bio;
1307
1308	if (conf->mddev->gendisk)
1309	trace_block_bio_remap(bio: mbio, dev: disk_devt(disk: conf->mddev->gendisk),
1310	from: r10_bio->sector);
1311	/* flush_pending_writes() needs access to the rdev so...*/
1312	mbio->bi_bdev = (void *)rdev;
1313
1314	atomic_inc(v: &r10_bio->remaining);
1315
1316	if (!raid1_add_bio_to_plug(mddev, bio: mbio, unplug: raid10_unplug, copies: conf->copies)) {
1317	spin_lock_irqsave(&conf->device_lock, flags);
1318	bio_list_add(bl: &conf->pending_bio_list, bio: mbio);
1319	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
1320	md_wakeup_thread(thread: mddev->thread);
1321	}
1322	}
1323
1324	static struct md_rdev *dereference_rdev_and_rrdev(struct raid10_info *mirror,
1325	struct md_rdev **prrdev)
1326	{
1327	struct md_rdev *rdev, *rrdev;
1328
1329	rrdev = rcu_dereference(mirror->replacement);
1330	/*
1331	* Read replacement first to prevent reading both rdev and
1332	* replacement as NULL during replacement replace rdev.
1333	*/
1334	smp_mb();
1335	rdev = rcu_dereference(mirror->rdev);
1336	if (rdev == rrdev)
1337	rrdev = NULL;
1338
1339	*prrdev = rrdev;
1340	return rdev;
1341	}
1342
1343	static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio)
1344	{
1345	int i;
1346	struct r10conf *conf = mddev->private;
1347	struct md_rdev *blocked_rdev;
1348
1349	retry_wait:
1350	blocked_rdev = NULL;
1351	rcu_read_lock();
1352	for (i = 0; i < conf->copies; i++) {
1353	struct md_rdev *rdev, *rrdev;
1354
1355	rdev = dereference_rdev_and_rrdev(mirror: &conf->mirrors[i], prrdev: &rrdev);
1356	if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1357	atomic_inc(v: &rdev->nr_pending);
1358	blocked_rdev = rdev;
1359	break;
1360	}
1361	if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
1362	atomic_inc(v: &rrdev->nr_pending);
1363	blocked_rdev = rrdev;
1364	break;
1365	}
1366
1367	if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1368	sector_t first_bad;
1369	sector_t dev_sector = r10_bio->devs[i].addr;
1370	int bad_sectors;
1371	int is_bad;
1372
1373	/*
1374	* Discard request doesn't care the write result
1375	* so it doesn't need to wait blocked disk here.
1376	*/
1377	if (!r10_bio->sectors)
1378	continue;
1379
1380	is_bad = is_badblock(rdev, s: dev_sector, sectors: r10_bio->sectors,
1381	first_bad: &first_bad, bad_sectors: &bad_sectors);
1382	if (is_bad < 0) {
1383	/*
1384	* Mustn't write here until the bad block
1385	* is acknowledged
1386	*/
1387	atomic_inc(v: &rdev->nr_pending);
1388	set_bit(nr: BlockedBadBlocks, addr: &rdev->flags);
1389	blocked_rdev = rdev;
1390	break;
1391	}
1392	}
1393	}
1394	rcu_read_unlock();
1395
1396	if (unlikely(blocked_rdev)) {
1397	/* Have to wait for this device to get unblocked, then retry */
1398	allow_barrier(conf);
1399	raid10_log(conf->mddev, "%s wait rdev %d blocked",
1400	__func__, blocked_rdev->raid_disk);
1401	md_wait_for_blocked_rdev(rdev: blocked_rdev, mddev);
1402	wait_barrier(conf, nowait: false);
1403	goto retry_wait;
1404	}
1405	}
1406
1407	static void raid10_write_request(struct mddev *mddev, struct bio *bio,
1408	struct r10bio *r10_bio)
1409	{
1410	struct r10conf *conf = mddev->private;
1411	int i;
1412	sector_t sectors;
1413	int max_sectors;
1414
1415	if ((mddev_is_clustered(mddev) &&
1416	md_cluster_ops->area_resyncing(mddev, WRITE,
1417	bio->bi_iter.bi_sector,
1418	bio_end_sector(bio)))) {
1419	DEFINE_WAIT(w);
1420	/* Bail out if REQ_NOWAIT is set for the bio */
1421	if (bio->bi_opf & REQ_NOWAIT) {
1422	bio_wouldblock_error(bio);
1423	return;
1424	}
1425	for (;;) {
1426	prepare_to_wait(wq_head: &conf->wait_barrier,
1427	wq_entry: &w, TASK_IDLE);
1428	if (!md_cluster_ops->area_resyncing(mddev, WRITE,
1429	bio->bi_iter.bi_sector, bio_end_sector(bio)))
1430	break;
1431	schedule();
1432	}
1433	finish_wait(wq_head: &conf->wait_barrier, wq_entry: &w);
1434	}
1435
1436	sectors = r10_bio->sectors;
1437	if (!regular_request_wait(mddev, conf, bio, sectors))
1438	return;
1439	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1440	(mddev->reshape_backwards
1441	? (bio->bi_iter.bi_sector < conf->reshape_safe &&
1442	bio->bi_iter.bi_sector + sectors > conf->reshape_progress)
1443	: (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&
1444	bio->bi_iter.bi_sector < conf->reshape_progress))) {
1445	/* Need to update reshape_position in metadata */
1446	mddev->reshape_position = conf->reshape_progress;
1447	set_mask_bits(&mddev->sb_flags, 0,
1448	BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1449	md_wakeup_thread(thread: mddev->thread);
1450	if (bio->bi_opf & REQ_NOWAIT) {
1451	allow_barrier(conf);
1452	bio_wouldblock_error(bio);
1453	return;
1454	}
1455	raid10_log(conf->mddev, "wait reshape metadata");
1456	wait_event(mddev->sb_wait,
1457	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
1458
1459	conf->reshape_safe = mddev->reshape_position;
1460	}
1461
1462	/* first select target devices under rcu_lock and
1463	* inc refcount on their rdev. Record them by setting
1464	* bios[x] to bio
1465	* If there are known/acknowledged bad blocks on any device
1466	* on which we have seen a write error, we want to avoid
1467	* writing to those blocks. This potentially requires several
1468	* writes to write around the bad blocks. Each set of writes
1469	* gets its own r10_bio with a set of bios attached.
1470	*/
1471
1472	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
1473	raid10_find_phys(conf, r10bio: r10_bio);
1474
1475	wait_blocked_dev(mddev, r10_bio);
1476
1477	rcu_read_lock();
1478	max_sectors = r10_bio->sectors;
1479
1480	for (i = 0; i < conf->copies; i++) {
1481	int d = r10_bio->devs[i].devnum;
1482	struct md_rdev *rdev, *rrdev;
1483
1484	rdev = dereference_rdev_and_rrdev(mirror: &conf->mirrors[d], prrdev: &rrdev);
1485	if (rdev && (test_bit(Faulty, &rdev->flags)))
1486	rdev = NULL;
1487	if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1488	rrdev = NULL;
1489
1490	r10_bio->devs[i].bio = NULL;
1491	r10_bio->devs[i].repl_bio = NULL;
1492
1493	if (!rdev && !rrdev) {
1494	set_bit(nr: R10BIO_Degraded, addr: &r10_bio->state);
1495	continue;
1496	}
1497	if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1498	sector_t first_bad;
1499	sector_t dev_sector = r10_bio->devs[i].addr;
1500	int bad_sectors;
1501	int is_bad;
1502
1503	is_bad = is_badblock(rdev, s: dev_sector, sectors: max_sectors,
1504	first_bad: &first_bad, bad_sectors: &bad_sectors);
1505	if (is_bad && first_bad <= dev_sector) {
1506	/* Cannot write here at all */
1507	bad_sectors -= (dev_sector - first_bad);
1508	if (bad_sectors < max_sectors)
1509	/* Mustn't write more than bad_sectors
1510	* to other devices yet
1511	*/
1512	max_sectors = bad_sectors;
1513	/* We don't set R10BIO_Degraded as that
1514	* only applies if the disk is missing,
1515	* so it might be re-added, and we want to
1516	* know to recover this chunk.
1517	* In this case the device is here, and the
1518	* fact that this chunk is not in-sync is
1519	* recorded in the bad block log.
1520	*/
1521	continue;
1522	}
1523	if (is_bad) {
1524	int good_sectors = first_bad - dev_sector;
1525	if (good_sectors < max_sectors)
1526	max_sectors = good_sectors;
1527	}
1528	}
1529	if (rdev) {
1530	r10_bio->devs[i].bio = bio;
1531	atomic_inc(v: &rdev->nr_pending);
1532	}
1533	if (rrdev) {
1534	r10_bio->devs[i].repl_bio = bio;
1535	atomic_inc(v: &rrdev->nr_pending);
1536	}
1537	}
1538	rcu_read_unlock();
1539
1540	if (max_sectors < r10_bio->sectors)
1541	r10_bio->sectors = max_sectors;
1542
1543	if (r10_bio->sectors < bio_sectors(bio)) {
1544	struct bio *split = bio_split(bio, sectors: r10_bio->sectors,
1545	GFP_NOIO, bs: &conf->bio_split);
1546	bio_chain(split, bio);
1547	allow_barrier(conf);
1548	submit_bio_noacct(bio);
1549	wait_barrier(conf, nowait: false);
1550	bio = split;
1551	r10_bio->master_bio = bio;
1552	}
1553
1554	md_account_bio(mddev, bio: &bio);
1555	r10_bio->master_bio = bio;
1556	atomic_set(v: &r10_bio->remaining, i: 1);
1557	md_bitmap_startwrite(bitmap: mddev->bitmap, offset: r10_bio->sector, sectors: r10_bio->sectors, behind: 0);
1558
1559	for (i = 0; i < conf->copies; i++) {
1560	if (r10_bio->devs[i].bio)
1561	raid10_write_one_disk(mddev, r10_bio, bio, replacement: false, n_copy: i);
1562	if (r10_bio->devs[i].repl_bio)
1563	raid10_write_one_disk(mddev, r10_bio, bio, replacement: true, n_copy: i);
1564	}
1565	one_write_done(r10_bio);
1566	}
1567
1568	static void __make_request(struct mddev *mddev, struct bio *bio, int sectors)
1569	{
1570	struct r10conf *conf = mddev->private;
1571	struct r10bio *r10_bio;
1572
1573	r10_bio = mempool_alloc(pool: &conf->r10bio_pool, GFP_NOIO);
1574
1575	r10_bio->master_bio = bio;
1576	r10_bio->sectors = sectors;
1577
1578	r10_bio->mddev = mddev;
1579	r10_bio->sector = bio->bi_iter.bi_sector;
1580	r10_bio->state = 0;
1581	r10_bio->read_slot = -1;
1582	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) *
1583	conf->geo.raid_disks);
1584
1585	if (bio_data_dir(bio) == READ)
1586	raid10_read_request(mddev, bio, r10_bio, io_accounting: true);
1587	else
1588	raid10_write_request(mddev, bio, r10_bio);
1589	}
1590
1591	static void raid_end_discard_bio(struct r10bio *r10bio)
1592	{
1593	struct r10conf *conf = r10bio->mddev->private;
1594	struct r10bio *first_r10bio;
1595
1596	while (atomic_dec_and_test(v: &r10bio->remaining)) {
1597
1598	allow_barrier(conf);
1599
1600	if (!test_bit(R10BIO_Discard, &r10bio->state)) {
1601	first_r10bio = (struct r10bio *)r10bio->master_bio;
1602	free_r10bio(r10_bio: r10bio);
1603	r10bio = first_r10bio;
1604	} else {
1605	md_write_end(mddev: r10bio->mddev);
1606	bio_endio(r10bio->master_bio);
1607	free_r10bio(r10_bio: r10bio);
1608	break;
1609	}
1610	}
1611	}
1612
1613	static void raid10_end_discard_request(struct bio *bio)
1614	{
1615	struct r10bio *r10_bio = bio->bi_private;
1616	struct r10conf *conf = r10_bio->mddev->private;
1617	struct md_rdev *rdev = NULL;
1618	int dev;
1619	int slot, repl;
1620
1621	/*
1622	* We don't care the return value of discard bio
1623	*/
1624	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
1625	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
1626
1627	dev = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
1628	if (repl)
1629	rdev = conf->mirrors[dev].replacement;
1630	if (!rdev) {
1631	/*
1632	* raid10_remove_disk uses smp_mb to make sure rdev is set to
1633	* replacement before setting replacement to NULL. It can read
1634	* rdev first without barrier protect even replacement is NULL
1635	*/
1636	smp_rmb();
1637	rdev = conf->mirrors[dev].rdev;
1638	}
1639
1640	raid_end_discard_bio(r10bio: r10_bio);
1641	rdev_dec_pending(rdev, mddev: conf->mddev);
1642	}
1643
1644	/*
1645	* There are some limitations to handle discard bio
1646	* 1st, the discard size is bigger than stripe_size*2.
1647	* 2st, if the discard bio spans reshape progress, we use the old way to
1648	* handle discard bio
1649	*/
1650	static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
1651	{
1652	struct r10conf *conf = mddev->private;
1653	struct geom *geo = &conf->geo;
1654	int far_copies = geo->far_copies;
1655	bool first_copy = true;
1656	struct r10bio *r10_bio, *first_r10bio;
1657	struct bio *split;
1658	int disk;
1659	sector_t chunk;
1660	unsigned int stripe_size;
1661	unsigned int stripe_data_disks;
1662	sector_t split_size;
1663	sector_t bio_start, bio_end;
1664	sector_t first_stripe_index, last_stripe_index;
1665	sector_t start_disk_offset;
1666	unsigned int start_disk_index;
1667	sector_t end_disk_offset;
1668	unsigned int end_disk_index;
1669	unsigned int remainder;
1670
1671	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
1672	return -EAGAIN;
1673
1674	if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) {
1675	bio_wouldblock_error(bio);
1676	return 0;
1677	}
1678	wait_barrier(conf, nowait: false);
1679
1680	/*
1681	* Check reshape again to avoid reshape happens after checking
1682	* MD_RECOVERY_RESHAPE and before wait_barrier
1683	*/
1684	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
1685	goto out;
1686
1687	if (geo->near_copies)
1688	stripe_data_disks = geo->raid_disks / geo->near_copies +
1689	geo->raid_disks % geo->near_copies;
1690	else
1691	stripe_data_disks = geo->raid_disks;
1692
1693	stripe_size = stripe_data_disks << geo->chunk_shift;
1694
1695	bio_start = bio->bi_iter.bi_sector;
1696	bio_end = bio_end_sector(bio);
1697
1698	/*
1699	* Maybe one discard bio is smaller than strip size or across one
1700	* stripe and discard region is larger than one stripe size. For far
1701	* offset layout, if the discard region is not aligned with stripe
1702	* size, there is hole when we submit discard bio to member disk.
1703	* For simplicity, we only handle discard bio which discard region
1704	* is bigger than stripe_size * 2
1705	*/
1706	if (bio_sectors(bio) < stripe_size*2)
1707	goto out;
1708
1709	/*
1710	* Keep bio aligned with strip size.
1711	*/
1712	div_u64_rem(dividend: bio_start, divisor: stripe_size, remainder: &remainder);
1713	if (remainder) {
1714	split_size = stripe_size - remainder;
1715	split = bio_split(bio, sectors: split_size, GFP_NOIO, bs: &conf->bio_split);
1716	bio_chain(split, bio);
1717	allow_barrier(conf);
1718	/* Resend the fist split part */
1719	submit_bio_noacct(bio: split);
1720	wait_barrier(conf, nowait: false);
1721	}
1722	div_u64_rem(dividend: bio_end, divisor: stripe_size, remainder: &remainder);
1723	if (remainder) {
1724	split_size = bio_sectors(bio) - remainder;
1725	split = bio_split(bio, sectors: split_size, GFP_NOIO, bs: &conf->bio_split);
1726	bio_chain(split, bio);
1727	allow_barrier(conf);
1728	/* Resend the second split part */
1729	submit_bio_noacct(bio);
1730	bio = split;
1731	wait_barrier(conf, nowait: false);
1732	}
1733
1734	bio_start = bio->bi_iter.bi_sector;
1735	bio_end = bio_end_sector(bio);
1736
1737	/*
1738	* Raid10 uses chunk as the unit to store data. It's similar like raid0.
1739	* One stripe contains the chunks from all member disk (one chunk from
1740	* one disk at the same HBA address). For layout detail, see 'man md 4'
1741	*/
1742	chunk = bio_start >> geo->chunk_shift;
1743	chunk *= geo->near_copies;
1744	first_stripe_index = chunk;
1745	start_disk_index = sector_div(first_stripe_index, geo->raid_disks);
1746	if (geo->far_offset)
1747	first_stripe_index *= geo->far_copies;
1748	start_disk_offset = (bio_start & geo->chunk_mask) +
1749	(first_stripe_index << geo->chunk_shift);
1750
1751	chunk = bio_end >> geo->chunk_shift;
1752	chunk *= geo->near_copies;
1753	last_stripe_index = chunk;
1754	end_disk_index = sector_div(last_stripe_index, geo->raid_disks);
1755	if (geo->far_offset)
1756	last_stripe_index *= geo->far_copies;
1757	end_disk_offset = (bio_end & geo->chunk_mask) +
1758	(last_stripe_index << geo->chunk_shift);
1759
1760	retry_discard:
1761	r10_bio = mempool_alloc(pool: &conf->r10bio_pool, GFP_NOIO);
1762	r10_bio->mddev = mddev;
1763	r10_bio->state = 0;
1764	r10_bio->sectors = 0;
1765	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * geo->raid_disks);
1766	wait_blocked_dev(mddev, r10_bio);
1767
1768	/*
1769	* For far layout it needs more than one r10bio to cover all regions.
1770	* Inspired by raid10_sync_request, we can use the first r10bio->master_bio
1771	* to record the discard bio. Other r10bio->master_bio record the first
1772	* r10bio. The first r10bio only release after all other r10bios finish.
1773	* The discard bio returns only first r10bio finishes
1774	*/
1775	if (first_copy) {
1776	r10_bio->master_bio = bio;
1777	set_bit(nr: R10BIO_Discard, addr: &r10_bio->state);
1778	first_copy = false;
1779	first_r10bio = r10_bio;
1780	} else
1781	r10_bio->master_bio = (struct bio *)first_r10bio;
1782
1783	/*
1784	* first select target devices under rcu_lock and
1785	* inc refcount on their rdev. Record them by setting
1786	* bios[x] to bio
1787	*/
1788	rcu_read_lock();
1789	for (disk = 0; disk < geo->raid_disks; disk++) {
1790	struct md_rdev *rdev, *rrdev;
1791
1792	rdev = dereference_rdev_and_rrdev(mirror: &conf->mirrors[disk], prrdev: &rrdev);
1793	r10_bio->devs[disk].bio = NULL;
1794	r10_bio->devs[disk].repl_bio = NULL;
1795
1796	if (rdev && (test_bit(Faulty, &rdev->flags)))
1797	rdev = NULL;
1798	if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1799	rrdev = NULL;
1800	if (!rdev && !rrdev)
1801	continue;
1802
1803	if (rdev) {
1804	r10_bio->devs[disk].bio = bio;
1805	atomic_inc(v: &rdev->nr_pending);
1806	}
1807	if (rrdev) {
1808	r10_bio->devs[disk].repl_bio = bio;
1809	atomic_inc(v: &rrdev->nr_pending);
1810	}
1811	}
1812	rcu_read_unlock();
1813
1814	atomic_set(v: &r10_bio->remaining, i: 1);
1815	for (disk = 0; disk < geo->raid_disks; disk++) {
1816	sector_t dev_start, dev_end;
1817	struct bio *mbio, *rbio = NULL;
1818
1819	/*
1820	* Now start to calculate the start and end address for each disk.
1821	* The space between dev_start and dev_end is the discard region.
1822	*
1823	* For dev_start, it needs to consider three conditions:
1824	* 1st, the disk is before start_disk, you can imagine the disk in
1825	* the next stripe. So the dev_start is the start address of next
1826	* stripe.
1827	* 2st, the disk is after start_disk, it means the disk is at the
1828	* same stripe of first disk
1829	* 3st, the first disk itself, we can use start_disk_offset directly
1830	*/
1831	if (disk < start_disk_index)
1832	dev_start = (first_stripe_index + 1) * mddev->chunk_sectors;
1833	else if (disk > start_disk_index)
1834	dev_start = first_stripe_index * mddev->chunk_sectors;
1835	else
1836	dev_start = start_disk_offset;
1837
1838	if (disk < end_disk_index)
1839	dev_end = (last_stripe_index + 1) * mddev->chunk_sectors;
1840	else if (disk > end_disk_index)
1841	dev_end = last_stripe_index * mddev->chunk_sectors;
1842	else
1843	dev_end = end_disk_offset;
1844
1845	/*
1846	* It only handles discard bio which size is >= stripe size, so
1847	* dev_end > dev_start all the time.
1848	* It doesn't need to use rcu lock to get rdev here. We already
1849	* add rdev->nr_pending in the first loop.
1850	*/
1851	if (r10_bio->devs[disk].bio) {
1852	struct md_rdev *rdev = conf->mirrors[disk].rdev;
1853	mbio = bio_alloc_clone(bdev: bio->bi_bdev, bio_src: bio, GFP_NOIO,
1854	bs: &mddev->bio_set);
1855	mbio->bi_end_io = raid10_end_discard_request;
1856	mbio->bi_private = r10_bio;
1857	r10_bio->devs[disk].bio = mbio;
1858	r10_bio->devs[disk].devnum = disk;
1859	atomic_inc(v: &r10_bio->remaining);
1860	md_submit_discard_bio(mddev, rdev, bio: mbio,
1861	start: dev_start + choose_data_offset(r10_bio, rdev),
1862	size: dev_end - dev_start);
1863	bio_endio(mbio);
1864	}
1865	if (r10_bio->devs[disk].repl_bio) {
1866	struct md_rdev *rrdev = conf->mirrors[disk].replacement;
1867	rbio = bio_alloc_clone(bdev: bio->bi_bdev, bio_src: bio, GFP_NOIO,
1868	bs: &mddev->bio_set);
1869	rbio->bi_end_io = raid10_end_discard_request;
1870	rbio->bi_private = r10_bio;
1871	r10_bio->devs[disk].repl_bio = rbio;
1872	r10_bio->devs[disk].devnum = disk;
1873	atomic_inc(v: &r10_bio->remaining);
1874	md_submit_discard_bio(mddev, rdev: rrdev, bio: rbio,
1875	start: dev_start + choose_data_offset(r10_bio, rdev: rrdev),
1876	size: dev_end - dev_start);
1877	bio_endio(rbio);
1878	}
1879	}
1880
1881	if (!geo->far_offset && --far_copies) {
1882	first_stripe_index += geo->stride >> geo->chunk_shift;
1883	start_disk_offset += geo->stride;
1884	last_stripe_index += geo->stride >> geo->chunk_shift;
1885	end_disk_offset += geo->stride;
1886	atomic_inc(v: &first_r10bio->remaining);
1887	raid_end_discard_bio(r10bio: r10_bio);
1888	wait_barrier(conf, nowait: false);
1889	goto retry_discard;
1890	}
1891
1892	raid_end_discard_bio(r10bio: r10_bio);
1893
1894	return 0;
1895	out:
1896	allow_barrier(conf);
1897	return -EAGAIN;
1898	}
1899
1900	static bool raid10_make_request(struct mddev *mddev, struct bio *bio)
1901	{
1902	struct r10conf *conf = mddev->private;
1903	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1904	int chunk_sects = chunk_mask + 1;
1905	int sectors = bio_sectors(bio);
1906
1907	if (unlikely(bio->bi_opf & REQ_PREFLUSH)
1908	&& md_flush_request(mddev, bio))
1909	return true;
1910
1911	if (!md_write_start(mddev, bi: bio))
1912	return false;
1913
1914	if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
1915	if (!raid10_handle_discard(mddev, bio))
1916	return true;
1917
1918	/*
1919	* If this request crosses a chunk boundary, we need to split
1920	* it.
1921	*/
1922	if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
1923	sectors > chunk_sects
1924	&& (conf->geo.near_copies < conf->geo.raid_disks
1925	|| conf->prev.near_copies <
1926	conf->prev.raid_disks)))
1927	sectors = chunk_sects -
1928	(bio->bi_iter.bi_sector &
1929	(chunk_sects - 1));
1930	__make_request(mddev, bio, sectors);
1931
1932	/* In case raid10d snuck in to freeze_array */
1933	wake_up_barrier(conf);
1934	return true;
1935	}
1936
1937	static void raid10_status(struct seq_file *seq, struct mddev *mddev)
1938	{
1939	struct r10conf *conf = mddev->private;
1940	int i;
1941
1942	if (conf->geo.near_copies < conf->geo.raid_disks)
1943	seq_printf(m: seq, fmt: " %dK chunks", mddev->chunk_sectors / 2);
1944	if (conf->geo.near_copies > 1)
1945	seq_printf(m: seq, fmt: " %d near-copies", conf->geo.near_copies);
1946	if (conf->geo.far_copies > 1) {
1947	if (conf->geo.far_offset)
1948	seq_printf(m: seq, fmt: " %d offset-copies", conf->geo.far_copies);
1949	else
1950	seq_printf(m: seq, fmt: " %d far-copies", conf->geo.far_copies);
1951	if (conf->geo.far_set_size != conf->geo.raid_disks)
1952	seq_printf(m: seq, fmt: " %d devices per set", conf->geo.far_set_size);
1953	}
1954	seq_printf(m: seq, fmt: " [%d/%d] [", conf->geo.raid_disks,
1955	conf->geo.raid_disks - mddev->degraded);
1956	rcu_read_lock();
1957	for (i = 0; i < conf->geo.raid_disks; i++) {
1958	struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1959	seq_printf(m: seq, fmt: "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
1960	}
1961	rcu_read_unlock();
1962	seq_printf(m: seq, fmt: "]");
1963	}
1964
1965	/* check if there are enough drives for
1966	* every block to appear on atleast one.
1967	* Don't consider the device numbered 'ignore'
1968	* as we might be about to remove it.
1969	*/
1970	static int _enough(struct r10conf *conf, int previous, int ignore)
1971	{
1972	int first = 0;
1973	int has_enough = 0;
1974	int disks, ncopies;
1975	if (previous) {
1976	disks = conf->prev.raid_disks;
1977	ncopies = conf->prev.near_copies;
1978	} else {
1979	disks = conf->geo.raid_disks;
1980	ncopies = conf->geo.near_copies;
1981	}
1982
1983	rcu_read_lock();
1984	do {
1985	int n = conf->copies;
1986	int cnt = 0;
1987	int this = first;
1988	while (n--) {
1989	struct md_rdev *rdev;
1990	if (this != ignore &&
1991	(rdev = rcu_dereference(conf->mirrors[this].rdev)) &&
1992	test_bit(In_sync, &rdev->flags))
1993	cnt++;
1994	this = (this+1) % disks;
1995	}
1996	if (cnt == 0)
1997	goto out;
1998	first = (first + ncopies) % disks;
1999	} while (first != 0);
2000	has_enough = 1;
2001	out:
2002	rcu_read_unlock();
2003	return has_enough;
2004	}
2005
2006	static int enough(struct r10conf *conf, int ignore)
2007	{
2008	/* when calling 'enough', both 'prev' and 'geo' must
2009	* be stable.
2010	* This is ensured if ->reconfig_mutex or ->device_lock
2011	* is held.
2012	*/
2013	return _enough(conf, previous: 0, ignore) &&
2014	_enough(conf, previous: 1, ignore);
2015	}
2016
2017	/**
2018	* raid10_error() - RAID10 error handler.
2019	* @mddev: affected md device.
2020	* @rdev: member device to fail.
2021	*
2022	* The routine acknowledges &rdev failure and determines new @mddev state.
2023	* If it failed, then:
2024	* - &MD_BROKEN flag is set in &mddev->flags.
2025	* Otherwise, it must be degraded:
2026	* - recovery is interrupted.
2027	* - &mddev->degraded is bumped.
2028	*
2029	* @rdev is marked as &Faulty excluding case when array is failed and
2030	* &mddev->fail_last_dev is off.
2031	*/
2032	static void raid10_error(struct mddev *mddev, struct md_rdev *rdev)
2033	{
2034	struct r10conf *conf = mddev->private;
2035	unsigned long flags;
2036
2037	spin_lock_irqsave(&conf->device_lock, flags);
2038
2039	if (test_bit(In_sync, &rdev->flags) && !enough(conf, ignore: rdev->raid_disk)) {
2040	set_bit(nr: MD_BROKEN, addr: &mddev->flags);
2041
2042	if (!mddev->fail_last_dev) {
2043	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2044	return;
2045	}
2046	}
2047	if (test_and_clear_bit(nr: In_sync, addr: &rdev->flags))
2048	mddev->degraded++;
2049
2050	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
2051	set_bit(nr: Blocked, addr: &rdev->flags);
2052	set_bit(nr: Faulty, addr: &rdev->flags);
2053	set_mask_bits(&mddev->sb_flags, 0,
2054	BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
2055	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2056	pr_crit("md/raid10:%s: Disk failure on %pg, disabling device.\n"
2057	"md/raid10:%s: Operation continuing on %d devices.\n",
2058	mdname(mddev), rdev->bdev,
2059	mdname(mddev), conf->geo.raid_disks - mddev->degraded);
2060	}
2061
2062	static void print_conf(struct r10conf *conf)
2063	{
2064	int i;
2065	struct md_rdev *rdev;
2066
2067	pr_debug("RAID10 conf printout:\n");
2068	if (!conf) {
2069	pr_debug("(!conf)\n");
2070	return;
2071	}
2072	pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
2073	conf->geo.raid_disks);
2074
2075	/* This is only called with ->reconfix_mutex held, so
2076	* rcu protection of rdev is not needed */
2077	for (i = 0; i < conf->geo.raid_disks; i++) {
2078	rdev = conf->mirrors[i].rdev;
2079	if (rdev)
2080	pr_debug(" disk %d, wo:%d, o:%d, dev:%pg\n",
2081	i, !test_bit(In_sync, &rdev->flags),
2082	!test_bit(Faulty, &rdev->flags),
2083	rdev->bdev);
2084	}
2085	}
2086
2087	static void close_sync(struct r10conf *conf)
2088	{
2089	wait_barrier(conf, nowait: false);
2090	allow_barrier(conf);
2091
2092	mempool_exit(pool: &conf->r10buf_pool);
2093	}
2094
2095	static int raid10_spare_active(struct mddev *mddev)
2096	{
2097	int i;
2098	struct r10conf *conf = mddev->private;
2099	struct raid10_info *tmp;
2100	int count = 0;
2101	unsigned long flags;
2102
2103	/*
2104	* Find all non-in_sync disks within the RAID10 configuration
2105	* and mark them in_sync
2106	*/
2107	for (i = 0; i < conf->geo.raid_disks; i++) {
2108	tmp = conf->mirrors + i;
2109	if (tmp->replacement
2110	&& tmp->replacement->recovery_offset == MaxSector
2111	&& !test_bit(Faulty, &tmp->replacement->flags)
2112	&& !test_and_set_bit(nr: In_sync, addr: &tmp->replacement->flags)) {
2113	/* Replacement has just become active */
2114	if (!tmp->rdev
2115	|| !test_and_clear_bit(nr: In_sync, addr: &tmp->rdev->flags))
2116	count++;
2117	if (tmp->rdev) {
2118	/* Replaced device not technically faulty,
2119	* but we need to be sure it gets removed
2120	* and never re-added.
2121	*/
2122	set_bit(nr: Faulty, addr: &tmp->rdev->flags);
2123	sysfs_notify_dirent_safe(
2124	sd: tmp->rdev->sysfs_state);
2125	}
2126	sysfs_notify_dirent_safe(sd: tmp->replacement->sysfs_state);
2127	} else if (tmp->rdev
2128	&& tmp->rdev->recovery_offset == MaxSector
2129	&& !test_bit(Faulty, &tmp->rdev->flags)
2130	&& !test_and_set_bit(nr: In_sync, addr: &tmp->rdev->flags)) {
2131	count++;
2132	sysfs_notify_dirent_safe(sd: tmp->rdev->sysfs_state);
2133	}
2134	}
2135	spin_lock_irqsave(&conf->device_lock, flags);
2136	mddev->degraded -= count;
2137	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
2138
2139	print_conf(conf);
2140	return count;
2141	}
2142
2143	static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
2144	{
2145	struct r10conf *conf = mddev->private;
2146	int err = -EEXIST;
2147	int mirror, repl_slot = -1;
2148	int first = 0;
2149	int last = conf->geo.raid_disks - 1;
2150	struct raid10_info *p;
2151
2152	if (mddev->recovery_cp < MaxSector)
2153	/* only hot-add to in-sync arrays, as recovery is
2154	* very different from resync
2155	*/
2156	return -EBUSY;
2157	if (rdev->saved_raid_disk < 0 && !_enough(conf, previous: 1, ignore: -1))
2158	return -EINVAL;
2159
2160	if (md_integrity_add_rdev(rdev, mddev))
2161	return -ENXIO;
2162
2163	if (rdev->raid_disk >= 0)
2164	first = last = rdev->raid_disk;
2165
2166	if (rdev->saved_raid_disk >= first &&
2167	rdev->saved_raid_disk < conf->geo.raid_disks &&
2168	conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
2169	mirror = rdev->saved_raid_disk;
2170	else
2171	mirror = first;
2172	for ( ; mirror <= last ; mirror++) {
2173	p = &conf->mirrors[mirror];
2174	if (p->recovery_disabled == mddev->recovery_disabled)
2175	continue;
2176	if (p->rdev) {
2177	if (test_bit(WantReplacement, &p->rdev->flags) &&
2178	p->replacement == NULL && repl_slot < 0)
2179	repl_slot = mirror;
2180	continue;
2181	}
2182
2183	if (mddev->gendisk)
2184	disk_stack_limits(disk: mddev->gendisk, bdev: rdev->bdev,
2185	offset: rdev->data_offset << 9);
2186
2187	p->head_position = 0;
2188	p->recovery_disabled = mddev->recovery_disabled - 1;
2189	rdev->raid_disk = mirror;
2190	err = 0;
2191	if (rdev->saved_raid_disk != mirror)
2192	conf->fullsync = 1;
2193	rcu_assign_pointer(p->rdev, rdev);
2194	break;
2195	}
2196
2197	if (err && repl_slot >= 0) {
2198	p = &conf->mirrors[repl_slot];
2199	clear_bit(nr: In_sync, addr: &rdev->flags);
2200	set_bit(nr: Replacement, addr: &rdev->flags);
2201	rdev->raid_disk = repl_slot;
2202	err = 0;
2203	if (mddev->gendisk)
2204	disk_stack_limits(disk: mddev->gendisk, bdev: rdev->bdev,
2205	offset: rdev->data_offset << 9);
2206	conf->fullsync = 1;
2207	rcu_assign_pointer(p->replacement, rdev);
2208	}
2209
2210	print_conf(conf);
2211	return err;
2212	}
2213
2214	static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
2215	{
2216	struct r10conf *conf = mddev->private;
2217	int err = 0;
2218	int number = rdev->raid_disk;
2219	struct md_rdev **rdevp;
2220	struct raid10_info *p;
2221
2222	print_conf(conf);
2223	if (unlikely(number >= mddev->raid_disks))
2224	return 0;
2225	p = conf->mirrors + number;
2226	if (rdev == p->rdev)
2227	rdevp = &p->rdev;
2228	else if (rdev == p->replacement)
2229	rdevp = &p->replacement;
2230	else
2231	return 0;
2232
2233	if (test_bit(In_sync, &rdev->flags) ||
2234	atomic_read(v: &rdev->nr_pending)) {
2235	err = -EBUSY;
2236	goto abort;
2237	}
2238	/* Only remove non-faulty devices if recovery
2239	* is not possible.
2240	*/
2241	if (!test_bit(Faulty, &rdev->flags) &&
2242	mddev->recovery_disabled != p->recovery_disabled &&
2243	(!p->replacement || p->replacement == rdev) &&
2244	number < conf->geo.raid_disks &&
2245	enough(conf, ignore: -1)) {
2246	err = -EBUSY;
2247	goto abort;
2248	}
2249	*rdevp = NULL;
2250	if (!test_bit(RemoveSynchronized, &rdev->flags)) {
2251	synchronize_rcu();
2252	if (atomic_read(v: &rdev->nr_pending)) {
2253	/* lost the race, try later */
2254	err = -EBUSY;
2255	*rdevp = rdev;
2256	goto abort;
2257	}
2258	}
2259	if (p->replacement) {
2260	/* We must have just cleared 'rdev' */
2261	p->rdev = p->replacement;
2262	clear_bit(nr: Replacement, addr: &p->replacement->flags);
2263	smp_mb(); /* Make sure other CPUs may see both as identical
2264	* but will never see neither -- if they are careful.
2265	*/
2266	p->replacement = NULL;
2267	}
2268
2269	clear_bit(nr: WantReplacement, addr: &rdev->flags);
2270	err = md_integrity_register(mddev);
2271
2272	abort:
2273
2274	print_conf(conf);
2275	return err;
2276	}
2277
2278	static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d)
2279	{
2280	struct r10conf *conf = r10_bio->mddev->private;
2281
2282	if (!bio->bi_status)
2283	set_bit(nr: R10BIO_Uptodate, addr: &r10_bio->state);
2284	else
2285	/* The write handler will notice the lack of
2286	* R10BIO_Uptodate and record any errors etc
2287	*/
2288	atomic_add(i: r10_bio->sectors,
2289	v: &conf->mirrors[d].rdev->corrected_errors);
2290
2291	/* for reconstruct, we always reschedule after a read.
2292	* for resync, only after all reads
2293	*/
2294	rdev_dec_pending(rdev: conf->mirrors[d].rdev, mddev: conf->mddev);
2295	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
2296	atomic_dec_and_test(v: &r10_bio->remaining)) {
2297	/* we have read all the blocks,
2298	* do the comparison in process context in raid10d
2299	*/
2300	reschedule_retry(r10_bio);
2301	}
2302	}
2303
2304	static void end_sync_read(struct bio *bio)
2305	{
2306	struct r10bio *r10_bio = get_resync_r10bio(bio);
2307	struct r10conf *conf = r10_bio->mddev->private;
2308	int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
2309
2310	__end_sync_read(r10_bio, bio, d);
2311	}
2312
2313	static void end_reshape_read(struct bio *bio)
2314	{
2315	/* reshape read bio isn't allocated from r10buf_pool */
2316	struct r10bio *r10_bio = bio->bi_private;
2317
2318	__end_sync_read(r10_bio, bio, d: r10_bio->read_slot);
2319	}
2320
2321	static void end_sync_request(struct r10bio *r10_bio)
2322	{
2323	struct mddev *mddev = r10_bio->mddev;
2324
2325	while (atomic_dec_and_test(v: &r10_bio->remaining)) {
2326	if (r10_bio->master_bio == NULL) {
2327	/* the primary of several recovery bios */
2328	sector_t s = r10_bio->sectors;
2329	if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2330	test_bit(R10BIO_WriteError, &r10_bio->state))
2331	reschedule_retry(r10_bio);
2332	else
2333	put_buf(r10_bio);
2334	md_done_sync(mddev, blocks: s, ok: 1);
2335	break;
2336	} else {
2337	struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
2338	if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2339	test_bit(R10BIO_WriteError, &r10_bio->state))
2340	reschedule_retry(r10_bio);
2341	else
2342	put_buf(r10_bio);
2343	r10_bio = r10_bio2;
2344	}
2345	}
2346	}
2347
2348	static void end_sync_write(struct bio *bio)
2349	{
2350	struct r10bio *r10_bio = get_resync_r10bio(bio);
2351	struct mddev *mddev = r10_bio->mddev;
2352	struct r10conf *conf = mddev->private;
2353	int d;
2354	sector_t first_bad;
2355	int bad_sectors;
2356	int slot;
2357	int repl;
2358	struct md_rdev *rdev = NULL;
2359
2360	d = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
2361	if (repl)
2362	rdev = conf->mirrors[d].replacement;
2363	else
2364	rdev = conf->mirrors[d].rdev;
2365
2366	if (bio->bi_status) {
2367	if (repl)
2368	md_error(mddev, rdev);
2369	else {
2370	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
2371	if (!test_and_set_bit(nr: WantReplacement, addr: &rdev->flags))
2372	set_bit(nr: MD_RECOVERY_NEEDED,
2373	addr: &rdev->mddev->recovery);
2374	set_bit(nr: R10BIO_WriteError, addr: &r10_bio->state);
2375	}
2376	} else if (is_badblock(rdev,
2377	s: r10_bio->devs[slot].addr,
2378	sectors: r10_bio->sectors,
2379	first_bad: &first_bad, bad_sectors: &bad_sectors))
2380	set_bit(nr: R10BIO_MadeGood, addr: &r10_bio->state);
2381
2382	rdev_dec_pending(rdev, mddev);
2383
2384	end_sync_request(r10_bio);
2385	}
2386
2387	/*
2388	* Note: sync and recover and handled very differently for raid10
2389	* This code is for resync.
2390	* For resync, we read through virtual addresses and read all blocks.
2391	* If there is any error, we schedule a write. The lowest numbered
2392	* drive is authoritative.
2393	* However requests come for physical address, so we need to map.
2394	* For every physical address there are raid_disks/copies virtual addresses,
2395	* which is always are least one, but is not necessarly an integer.
2396	* This means that a physical address can span multiple chunks, so we may
2397	* have to submit multiple io requests for a single sync request.
2398	*/
2399	/*
2400	* We check if all blocks are in-sync and only write to blocks that
2401	* aren't in sync
2402	*/
2403	static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2404	{
2405	struct r10conf *conf = mddev->private;
2406	int i, first;
2407	struct bio *tbio, *fbio;
2408	int vcnt;
2409	struct page **tpages, **fpages;
2410
2411	atomic_set(v: &r10_bio->remaining, i: 1);
2412
2413	/* find the first device with a block */
2414	for (i=0; i<conf->copies; i++)
2415	if (!r10_bio->devs[i].bio->bi_status)
2416	break;
2417
2418	if (i == conf->copies)
2419	goto done;
2420
2421	first = i;
2422	fbio = r10_bio->devs[i].bio;
2423	fbio->bi_iter.bi_size = r10_bio->sectors << 9;
2424	fbio->bi_iter.bi_idx = 0;
2425	fpages = get_resync_pages(bio: fbio)->pages;
2426
2427	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
2428	/* now find blocks with errors */
2429	for (i=0 ; i < conf->copies ; i++) {
2430	int j, d;
2431	struct md_rdev *rdev;
2432	struct resync_pages *rp;
2433
2434	tbio = r10_bio->devs[i].bio;
2435
2436	if (tbio->bi_end_io != end_sync_read)
2437	continue;
2438	if (i == first)
2439	continue;
2440
2441	tpages = get_resync_pages(bio: tbio)->pages;
2442	d = r10_bio->devs[i].devnum;
2443	rdev = conf->mirrors[d].rdev;
2444	if (!r10_bio->devs[i].bio->bi_status) {
2445	/* We know that the bi_io_vec layout is the same for
2446	* both 'first' and 'i', so we just compare them.
2447	* All vec entries are PAGE_SIZE;
2448	*/
2449	int sectors = r10_bio->sectors;
2450	for (j = 0; j < vcnt; j++) {
2451	int len = PAGE_SIZE;
2452	if (sectors < (len / 512))
2453	len = sectors * 512;
2454	if (memcmp(page_address(fpages[j]),
2455	page_address(tpages[j]),
2456	size: len))
2457	break;
2458	sectors -= len/512;
2459	}
2460	if (j == vcnt)
2461	continue;
2462	atomic64_add(i: r10_bio->sectors, v: &mddev->resync_mismatches);
2463	if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
2464	/* Don't fix anything. */
2465	continue;
2466	} else if (test_bit(FailFast, &rdev->flags)) {
2467	/* Just give up on this device */
2468	md_error(mddev: rdev->mddev, rdev);
2469	continue;
2470	}
2471	/* Ok, we need to write this bio, either to correct an
2472	* inconsistency or to correct an unreadable block.
2473	* First we need to fixup bv_offset, bv_len and
2474	* bi_vecs, as the read request might have corrupted these
2475	*/
2476	rp = get_resync_pages(bio: tbio);
2477	bio_reset(bio: tbio, bdev: conf->mirrors[d].rdev->bdev, opf: REQ_OP_WRITE);
2478
2479	md_bio_reset_resync_pages(bio: tbio, rp, size: fbio->bi_iter.bi_size);
2480
2481	rp->raid_bio = r10_bio;
2482	tbio->bi_private = rp;
2483	tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
2484	tbio->bi_end_io = end_sync_write;
2485
2486	bio_copy_data(dst: tbio, src: fbio);
2487
2488	atomic_inc(v: &conf->mirrors[d].rdev->nr_pending);
2489	atomic_inc(v: &r10_bio->remaining);
2490	md_sync_acct(bdev: conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
2491
2492	if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
2493	tbio->bi_opf |= MD_FAILFAST;
2494	tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
2495	submit_bio_noacct(bio: tbio);
2496	}
2497
2498	/* Now write out to any replacement devices
2499	* that are active
2500	*/
2501	for (i = 0; i < conf->copies; i++) {
2502	int d;
2503
2504	tbio = r10_bio->devs[i].repl_bio;
2505	if (!tbio || !tbio->bi_end_io)
2506	continue;
2507	if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
2508	&& r10_bio->devs[i].bio != fbio)
2509	bio_copy_data(dst: tbio, src: fbio);
2510	d = r10_bio->devs[i].devnum;
2511	atomic_inc(v: &r10_bio->remaining);
2512	md_sync_acct(bdev: conf->mirrors[d].replacement->bdev,
2513	bio_sectors(tbio));
2514	submit_bio_noacct(bio: tbio);
2515	}
2516
2517	done:
2518	if (atomic_dec_and_test(v: &r10_bio->remaining)) {
2519	md_done_sync(mddev, blocks: r10_bio->sectors, ok: 1);
2520	put_buf(r10_bio);
2521	}
2522	}
2523
2524	/*
2525	* Now for the recovery code.
2526	* Recovery happens across physical sectors.
2527	* We recover all non-is_sync drives by finding the virtual address of
2528	* each, and then choose a working drive that also has that virt address.
2529	* There is a separate r10_bio for each non-in_sync drive.
2530	* Only the first two slots are in use. The first for reading,
2531	* The second for writing.
2532	*
2533	*/
2534	static void fix_recovery_read_error(struct r10bio *r10_bio)
2535	{
2536	/* We got a read error during recovery.
2537	* We repeat the read in smaller page-sized sections.
2538	* If a read succeeds, write it to the new device or record
2539	* a bad block if we cannot.
2540	* If a read fails, record a bad block on both old and
2541	* new devices.
2542	*/
2543	struct mddev *mddev = r10_bio->mddev;
2544	struct r10conf *conf = mddev->private;
2545	struct bio *bio = r10_bio->devs[0].bio;
2546	sector_t sect = 0;
2547	int sectors = r10_bio->sectors;
2548	int idx = 0;
2549	int dr = r10_bio->devs[0].devnum;
2550	int dw = r10_bio->devs[1].devnum;
2551	struct page **pages = get_resync_pages(bio)->pages;
2552
2553	while (sectors) {
2554	int s = sectors;
2555	struct md_rdev *rdev;
2556	sector_t addr;
2557	int ok;
2558
2559	if (s > (PAGE_SIZE>>9))
2560	s = PAGE_SIZE >> 9;
2561
2562	rdev = conf->mirrors[dr].rdev;
2563	addr = r10_bio->devs[0].addr + sect,
2564	ok = sync_page_io(rdev,
2565	sector: addr,
2566	size: s << 9,
2567	page: pages[idx],
2568	opf: REQ_OP_READ, metadata_op: false);
2569	if (ok) {
2570	rdev = conf->mirrors[dw].rdev;
2571	addr = r10_bio->devs[1].addr + sect;
2572	ok = sync_page_io(rdev,
2573	sector: addr,
2574	size: s << 9,
2575	page: pages[idx],
2576	opf: REQ_OP_WRITE, metadata_op: false);
2577	if (!ok) {
2578	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
2579	if (!test_and_set_bit(nr: WantReplacement,
2580	addr: &rdev->flags))
2581	set_bit(nr: MD_RECOVERY_NEEDED,
2582	addr: &rdev->mddev->recovery);
2583	}
2584	}
2585	if (!ok) {
2586	/* We don't worry if we cannot set a bad block -
2587	* it really is bad so there is no loss in not
2588	* recording it yet
2589	*/
2590	rdev_set_badblocks(rdev, s: addr, sectors: s, is_new: 0);
2591
2592	if (rdev != conf->mirrors[dw].rdev) {
2593	/* need bad block on destination too */
2594	struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2595	addr = r10_bio->devs[1].addr + sect;
2596	ok = rdev_set_badblocks(rdev: rdev2, s: addr, sectors: s, is_new: 0);
2597	if (!ok) {
2598	/* just abort the recovery */
2599	pr_notice("md/raid10:%s: recovery aborted due to read error\n",
2600	mdname(mddev));
2601
2602	conf->mirrors[dw].recovery_disabled
2603	= mddev->recovery_disabled;
2604	set_bit(nr: MD_RECOVERY_INTR,
2605	addr: &mddev->recovery);
2606	break;
2607	}
2608	}
2609	}
2610
2611	sectors -= s;
2612	sect += s;
2613	idx++;
2614	}
2615	}
2616
2617	static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2618	{
2619	struct r10conf *conf = mddev->private;
2620	int d;
2621	struct bio *wbio = r10_bio->devs[1].bio;
2622	struct bio *wbio2 = r10_bio->devs[1].repl_bio;
2623
2624	/* Need to test wbio2->bi_end_io before we call
2625	* submit_bio_noacct as if the former is NULL,
2626	* the latter is free to free wbio2.
2627	*/
2628	if (wbio2 && !wbio2->bi_end_io)
2629	wbio2 = NULL;
2630
2631	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
2632	fix_recovery_read_error(r10_bio);
2633	if (wbio->bi_end_io)
2634	end_sync_request(r10_bio);
2635	if (wbio2)
2636	end_sync_request(r10_bio);
2637	return;
2638	}
2639
2640	/*
2641	* share the pages with the first bio
2642	* and submit the write request
2643	*/
2644	d = r10_bio->devs[1].devnum;
2645	if (wbio->bi_end_io) {
2646	atomic_inc(v: &conf->mirrors[d].rdev->nr_pending);
2647	md_sync_acct(bdev: conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
2648	submit_bio_noacct(bio: wbio);
2649	}
2650	if (wbio2) {
2651	atomic_inc(v: &conf->mirrors[d].replacement->nr_pending);
2652	md_sync_acct(bdev: conf->mirrors[d].replacement->bdev,
2653	bio_sectors(wbio2));
2654	submit_bio_noacct(bio: wbio2);
2655	}
2656	}
2657
2658	/*
2659	* Used by fix_read_error() to decay the per rdev read_errors.
2660	* We halve the read error count for every hour that has elapsed
2661	* since the last recorded read error.
2662	*
2663	*/
2664	static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2665	{
2666	long cur_time_mon;
2667	unsigned long hours_since_last;
2668	unsigned int read_errors = atomic_read(v: &rdev->read_errors);
2669
2670	cur_time_mon = ktime_get_seconds();
2671
2672	if (rdev->last_read_error == 0) {
2673	/* first time we've seen a read error */
2674	rdev->last_read_error = cur_time_mon;
2675	return;
2676	}
2677
2678	hours_since_last = (long)(cur_time_mon -
2679	rdev->last_read_error) / 3600;
2680
2681	rdev->last_read_error = cur_time_mon;
2682
2683	/*
2684	* if hours_since_last is > the number of bits in read_errors
2685	* just set read errors to 0. We do this to avoid
2686	* overflowing the shift of read_errors by hours_since_last.
2687	*/
2688	if (hours_since_last >= 8 * sizeof(read_errors))
2689	atomic_set(v: &rdev->read_errors, i: 0);
2690	else
2691	atomic_set(v: &rdev->read_errors, i: read_errors >> hours_since_last);
2692	}
2693
2694	static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2695	int sectors, struct page *page, enum req_op op)
2696	{
2697	sector_t first_bad;
2698	int bad_sectors;
2699
2700	if (is_badblock(rdev, s: sector, sectors, first_bad: &first_bad, bad_sectors: &bad_sectors)
2701	&& (op == REQ_OP_READ || test_bit(WriteErrorSeen, &rdev->flags)))
2702	return -1;
2703	if (sync_page_io(rdev, sector, size: sectors << 9, page, opf: op, metadata_op: false))
2704	/* success */
2705	return 1;
2706	if (op == REQ_OP_WRITE) {
2707	set_bit(nr: WriteErrorSeen, addr: &rdev->flags);
2708	if (!test_and_set_bit(nr: WantReplacement, addr: &rdev->flags))
2709	set_bit(nr: MD_RECOVERY_NEEDED,
2710	addr: &rdev->mddev->recovery);
2711	}
2712	/* need to record an error - either for the block or the device */
2713	if (!rdev_set_badblocks(rdev, s: sector, sectors, is_new: 0))
2714	md_error(mddev: rdev->mddev, rdev);
2715	return 0;
2716	}
2717
2718	/*
2719	* This is a kernel thread which:
2720	*
2721	* 1. Retries failed read operations on working mirrors.
2722	* 2. Updates the raid superblock when problems encounter.
2723	* 3. Performs writes following reads for array synchronising.
2724	*/
2725
2726	static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2727	{
2728	int sect = 0; /* Offset from r10_bio->sector */
2729	int sectors = r10_bio->sectors, slot = r10_bio->read_slot;
2730	struct md_rdev *rdev;
2731	int max_read_errors = atomic_read(v: &mddev->max_corr_read_errors);
2732	int d = r10_bio->devs[slot].devnum;
2733
2734	/* still own a reference to this rdev, so it cannot
2735	* have been cleared recently.
2736	*/
2737	rdev = conf->mirrors[d].rdev;
2738
2739	if (test_bit(Faulty, &rdev->flags))
2740	/* drive has already been failed, just ignore any
2741	more fix_read_error() attempts */
2742	return;
2743
2744	check_decay_read_errors(mddev, rdev);
2745	atomic_inc(v: &rdev->read_errors);
2746	if (atomic_read(v: &rdev->read_errors) > max_read_errors) {
2747	pr_notice("md/raid10:%s: %pg: Raid device exceeded read_error threshold [cur %d:max %d]\n",
2748	mdname(mddev), rdev->bdev,
2749	atomic_read(&rdev->read_errors), max_read_errors);
2750	pr_notice("md/raid10:%s: %pg: Failing raid device\n",
2751	mdname(mddev), rdev->bdev);
2752	md_error(mddev, rdev);
2753	r10_bio->devs[slot].bio = IO_BLOCKED;
2754	return;
2755	}
2756
2757	while(sectors) {
2758	int s = sectors;
2759	int sl = slot;
2760	int success = 0;
2761	int start;
2762
2763	if (s > (PAGE_SIZE>>9))
2764	s = PAGE_SIZE >> 9;
2765
2766	rcu_read_lock();
2767	do {
2768	sector_t first_bad;
2769	int bad_sectors;
2770
2771	d = r10_bio->devs[sl].devnum;
2772	rdev = rcu_dereference(conf->mirrors[d].rdev);
2773	if (rdev &&
2774	test_bit(In_sync, &rdev->flags) &&
2775	!test_bit(Faulty, &rdev->flags) &&
2776	is_badblock(rdev, s: r10_bio->devs[sl].addr + sect, sectors: s,
2777	first_bad: &first_bad, bad_sectors: &bad_sectors) == 0) {
2778	atomic_inc(v: &rdev->nr_pending);
2779	rcu_read_unlock();
2780	success = sync_page_io(rdev,
2781	sector: r10_bio->devs[sl].addr +
2782	sect,
2783	size: s<<9,
2784	page: conf->tmppage,
2785	opf: REQ_OP_READ, metadata_op: false);
2786	rdev_dec_pending(rdev, mddev);
2787	rcu_read_lock();
2788	if (success)
2789	break;
2790	}
2791	sl++;
2792	if (sl == conf->copies)
2793	sl = 0;
2794	} while (sl != slot);
2795	rcu_read_unlock();
2796
2797	if (!success) {
2798	/* Cannot read from anywhere, just mark the block
2799	* as bad on the first device to discourage future
2800	* reads.
2801	*/
2802	int dn = r10_bio->devs[slot].devnum;
2803	rdev = conf->mirrors[dn].rdev;
2804
2805	if (!rdev_set_badblocks(
2806	rdev,
2807	s: r10_bio->devs[slot].addr
2808	+ sect,
2809	sectors: s, is_new: 0)) {
2810	md_error(mddev, rdev);
2811	r10_bio->devs[slot].bio
2812	= IO_BLOCKED;
2813	}
2814	break;
2815	}
2816
2817	start = sl;
2818	/* write it back and re-read */
2819	rcu_read_lock();
2820	while (sl != slot) {
2821	if (sl==0)
2822	sl = conf->copies;
2823	sl--;
2824	d = r10_bio->devs[sl].devnum;
2825	rdev = rcu_dereference(conf->mirrors[d].rdev);
2826	if (!rdev ||
2827	test_bit(Faulty, &rdev->flags) ||
2828	!test_bit(In_sync, &rdev->flags))
2829	continue;
2830
2831	atomic_inc(v: &rdev->nr_pending);
2832	rcu_read_unlock();
2833	if (r10_sync_page_io(rdev,
2834	sector: r10_bio->devs[sl].addr +
2835	sect,
2836	sectors: s, page: conf->tmppage, op: REQ_OP_WRITE)
2837	== 0) {
2838	/* Well, this device is dead */
2839	pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %pg)\n",
2840	mdname(mddev), s,
2841	(unsigned long long)(
2842	sect +
2843	choose_data_offset(r10_bio,
2844	rdev)),
2845	rdev->bdev);
2846	pr_notice("md/raid10:%s: %pg: failing drive\n",
2847	mdname(mddev),
2848	rdev->bdev);
2849	}
2850	rdev_dec_pending(rdev, mddev);
2851	rcu_read_lock();
2852	}
2853	sl = start;
2854	while (sl != slot) {
2855	if (sl==0)
2856	sl = conf->copies;
2857	sl--;
2858	d = r10_bio->devs[sl].devnum;
2859	rdev = rcu_dereference(conf->mirrors[d].rdev);
2860	if (!rdev ||
2861	test_bit(Faulty, &rdev->flags) ||
2862	!test_bit(In_sync, &rdev->flags))
2863	continue;
2864
2865	atomic_inc(v: &rdev->nr_pending);
2866	rcu_read_unlock();
2867	switch (r10_sync_page_io(rdev,
2868	sector: r10_bio->devs[sl].addr +
2869	sect,
2870	sectors: s, page: conf->tmppage, op: REQ_OP_READ)) {
2871	case 0:
2872	/* Well, this device is dead */
2873	pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %pg)\n",
2874	mdname(mddev), s,
2875	(unsigned long long)(
2876	sect +
2877	choose_data_offset(r10_bio, rdev)),
2878	rdev->bdev);
2879	pr_notice("md/raid10:%s: %pg: failing drive\n",
2880	mdname(mddev),
2881	rdev->bdev);
2882	break;
2883	case 1:
2884	pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %pg)\n",
2885	mdname(mddev), s,
2886	(unsigned long long)(
2887	sect +
2888	choose_data_offset(r10_bio, rdev)),
2889	rdev->bdev);
2890	atomic_add(i: s, v: &rdev->corrected_errors);
2891	}
2892
2893	rdev_dec_pending(rdev, mddev);
2894	rcu_read_lock();
2895	}
2896	rcu_read_unlock();
2897
2898	sectors -= s;
2899	sect += s;
2900	}
2901	}
2902
2903	static int narrow_write_error(struct r10bio *r10_bio, int i)
2904	{
2905	struct bio *bio = r10_bio->master_bio;
2906	struct mddev *mddev = r10_bio->mddev;
2907	struct r10conf *conf = mddev->private;
2908	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2909	/* bio has the data to be written to slot 'i' where
2910	* we just recently had a write error.
2911	* We repeatedly clone the bio and trim down to one block,
2912	* then try the write. Where the write fails we record
2913	* a bad block.
2914	* It is conceivable that the bio doesn't exactly align with
2915	* blocks. We must handle this.
2916	*
2917	* We currently own a reference to the rdev.
2918	*/
2919
2920	int block_sectors;
2921	sector_t sector;
2922	int sectors;
2923	int sect_to_write = r10_bio->sectors;
2924	int ok = 1;
2925
2926	if (rdev->badblocks.shift < 0)
2927	return 0;
2928
2929	block_sectors = roundup(1 << rdev->badblocks.shift,
2930	bdev_logical_block_size(rdev->bdev) >> 9);
2931	sector = r10_bio->sector;
2932	sectors = ((r10_bio->sector + block_sectors)
2933	& ~(sector_t)(block_sectors - 1))
2934	- sector;
2935
2936	while (sect_to_write) {
2937	struct bio *wbio;
2938	sector_t wsector;
2939	if (sectors > sect_to_write)
2940	sectors = sect_to_write;
2941	/* Write at 'sector' for 'sectors' */
2942	wbio = bio_alloc_clone(bdev: rdev->bdev, bio_src: bio, GFP_NOIO,
2943	bs: &mddev->bio_set);
2944	bio_trim(bio: wbio, offset: sector - bio->bi_iter.bi_sector, size: sectors);
2945	wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
2946	wbio->bi_iter.bi_sector = wsector +
2947	choose_data_offset(r10_bio, rdev);
2948	wbio->bi_opf = REQ_OP_WRITE;
2949
2950	if (submit_bio_wait(bio: wbio) < 0)
2951	/* Failure! */
2952	ok = rdev_set_badblocks(rdev, s: wsector,
2953	sectors, is_new: 0)
2954	&& ok;
2955
2956	bio_put(wbio);
2957	sect_to_write -= sectors;
2958	sector += sectors;
2959	sectors = block_sectors;
2960	}
2961	return ok;
2962	}
2963
2964	static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2965	{
2966	int slot = r10_bio->read_slot;
2967	struct bio *bio;
2968	struct r10conf *conf = mddev->private;
2969	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2970
2971	/* we got a read error. Maybe the drive is bad. Maybe just
2972	* the block and we can fix it.
2973	* We freeze all other IO, and try reading the block from
2974	* other devices. When we find one, we re-write
2975	* and check it that fixes the read error.
2976	* This is all done synchronously while the array is
2977	* frozen.
2978	*/
2979	bio = r10_bio->devs[slot].bio;
2980	bio_put(bio);
2981	r10_bio->devs[slot].bio = NULL;
2982
2983	if (mddev->ro)
2984	r10_bio->devs[slot].bio = IO_BLOCKED;
2985	else if (!test_bit(FailFast, &rdev->flags)) {
2986	freeze_array(conf, extra: 1);
2987	fix_read_error(conf, mddev, r10_bio);
2988	unfreeze_array(conf);
2989	} else
2990	md_error(mddev, rdev);
2991
2992	rdev_dec_pending(rdev, mddev);
2993	r10_bio->state = 0;
2994	raid10_read_request(mddev, bio: r10_bio->master_bio, r10_bio, io_accounting: false);
2995	/*
2996	* allow_barrier after re-submit to ensure no sync io
2997	* can be issued while regular io pending.
2998	*/
2999	allow_barrier(conf);
3000	}
3001
3002	static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
3003	{
3004	/* Some sort of write request has finished and it
3005	* succeeded in writing where we thought there was a
3006	* bad block. So forget the bad block.
3007	* Or possibly if failed and we need to record
3008	* a bad block.
3009	*/
3010	int m;
3011	struct md_rdev *rdev;
3012
3013	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
3014	test_bit(R10BIO_IsRecover, &r10_bio->state)) {
3015	for (m = 0; m < conf->copies; m++) {
3016	int dev = r10_bio->devs[m].devnum;
3017	rdev = conf->mirrors[dev].rdev;
3018	if (r10_bio->devs[m].bio == NULL ||
3019	r10_bio->devs[m].bio->bi_end_io == NULL)
3020	continue;
3021	if (!r10_bio->devs[m].bio->bi_status) {
3022	rdev_clear_badblocks(
3023	rdev,
3024	s: r10_bio->devs[m].addr,
3025	sectors: r10_bio->sectors, is_new: 0);
3026	} else {
3027	if (!rdev_set_badblocks(
3028	rdev,
3029	s: r10_bio->devs[m].addr,
3030	sectors: r10_bio->sectors, is_new: 0))
3031	md_error(mddev: conf->mddev, rdev);
3032	}
3033	rdev = conf->mirrors[dev].replacement;
3034	if (r10_bio->devs[m].repl_bio == NULL ||
3035	r10_bio->devs[m].repl_bio->bi_end_io == NULL)
3036	continue;
3037
3038	if (!r10_bio->devs[m].repl_bio->bi_status) {
3039	rdev_clear_badblocks(
3040	rdev,
3041	s: r10_bio->devs[m].addr,
3042	sectors: r10_bio->sectors, is_new: 0);
3043	} else {
3044	if (!rdev_set_badblocks(
3045	rdev,
3046	s: r10_bio->devs[m].addr,
3047	sectors: r10_bio->sectors, is_new: 0))
3048	md_error(mddev: conf->mddev, rdev);
3049	}
3050	}
3051	put_buf(r10_bio);
3052	} else {
3053	bool fail = false;
3054	for (m = 0; m < conf->copies; m++) {
3055	int dev = r10_bio->devs[m].devnum;
3056	struct bio *bio = r10_bio->devs[m].bio;
3057	rdev = conf->mirrors[dev].rdev;
3058	if (bio == IO_MADE_GOOD) {
3059	rdev_clear_badblocks(
3060	rdev,
3061	s: r10_bio->devs[m].addr,
3062	sectors: r10_bio->sectors, is_new: 0);
3063	rdev_dec_pending(rdev, mddev: conf->mddev);
3064	} else if (bio != NULL && bio->bi_status) {
3065	fail = true;
3066	if (!narrow_write_error(r10_bio, i: m)) {
3067	md_error(mddev: conf->mddev, rdev);
3068	set_bit(nr: R10BIO_Degraded,
3069	addr: &r10_bio->state);
3070	}
3071	rdev_dec_pending(rdev, mddev: conf->mddev);
3072	}
3073	bio = r10_bio->devs[m].repl_bio;
3074	rdev = conf->mirrors[dev].replacement;
3075	if (rdev && bio == IO_MADE_GOOD) {
3076	rdev_clear_badblocks(
3077	rdev,
3078	s: r10_bio->devs[m].addr,
3079	sectors: r10_bio->sectors, is_new: 0);
3080	rdev_dec_pending(rdev, mddev: conf->mddev);
3081	}
3082	}
3083	if (fail) {
3084	spin_lock_irq(lock: &conf->device_lock);
3085	list_add(new: &r10_bio->retry_list, head: &conf->bio_end_io_list);
3086	conf->nr_queued++;
3087	spin_unlock_irq(lock: &conf->device_lock);
3088	/*
3089	* In case freeze_array() is waiting for condition
3090	* nr_pending == nr_queued + extra to be true.
3091	*/
3092	wake_up(&conf->wait_barrier);
3093	md_wakeup_thread(thread: conf->mddev->thread);
3094	} else {
3095	if (test_bit(R10BIO_WriteError,
3096	&r10_bio->state))
3097	close_write(r10_bio);
3098	raid_end_bio_io(r10_bio);
3099	}
3100	}
3101	}
3102
3103	static void raid10d(struct md_thread *thread)
3104	{
3105	struct mddev *mddev = thread->mddev;
3106	struct r10bio *r10_bio;
3107	unsigned long flags;
3108	struct r10conf *conf = mddev->private;
3109	struct list_head *head = &conf->retry_list;
3110	struct blk_plug plug;
3111
3112	md_check_recovery(mddev);
3113
3114	if (!list_empty_careful(head: &conf->bio_end_io_list) &&
3115	!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
3116	LIST_HEAD(tmp);
3117	spin_lock_irqsave(&conf->device_lock, flags);
3118	if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
3119	while (!list_empty(head: &conf->bio_end_io_list)) {
3120	list_move(list: conf->bio_end_io_list.prev, head: &tmp);
3121	conf->nr_queued--;
3122	}
3123	}
3124	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
3125	while (!list_empty(head: &tmp)) {
3126	r10_bio = list_first_entry(&tmp, struct r10bio,
3127	retry_list);
3128	list_del(entry: &r10_bio->retry_list);
3129	if (mddev->degraded)
3130	set_bit(nr: R10BIO_Degraded, addr: &r10_bio->state);
3131
3132	if (test_bit(R10BIO_WriteError,
3133	&r10_bio->state))
3134	close_write(r10_bio);
3135	raid_end_bio_io(r10_bio);
3136	}
3137	}
3138
3139	blk_start_plug(&plug);
3140	for (;;) {
3141
3142	flush_pending_writes(conf);
3143
3144	spin_lock_irqsave(&conf->device_lock, flags);
3145	if (list_empty(head)) {
3146	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
3147	break;
3148	}
3149	r10_bio = list_entry(head->prev, struct r10bio, retry_list);
3150	list_del(entry: head->prev);
3151	conf->nr_queued--;
3152	spin_unlock_irqrestore(lock: &conf->device_lock, flags);
3153
3154	mddev = r10_bio->mddev;
3155	conf = mddev->private;
3156	if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
3157	test_bit(R10BIO_WriteError, &r10_bio->state))
3158	handle_write_completed(conf, r10_bio);
3159	else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
3160	reshape_request_write(mddev, r10_bio);
3161	else if (test_bit(R10BIO_IsSync, &r10_bio->state))
3162	sync_request_write(mddev, r10_bio);
3163	else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
3164	recovery_request_write(mddev, r10_bio);
3165	else if (test_bit(R10BIO_ReadError, &r10_bio->state))
3166	handle_read_error(mddev, r10_bio);
3167	else
3168	WARN_ON_ONCE(1);
3169
3170	cond_resched();
3171	if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
3172	md_check_recovery(mddev);
3173	}
3174	blk_finish_plug(&plug);
3175	}
3176
3177	static int init_resync(struct r10conf *conf)
3178	{
3179	int ret, buffs, i;
3180
3181	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
3182	BUG_ON(mempool_initialized(&conf->r10buf_pool));
3183	conf->have_replacement = 0;
3184	for (i = 0; i < conf->geo.raid_disks; i++)
3185	if (conf->mirrors[i].replacement)
3186	conf->have_replacement = 1;
3187	ret = mempool_init(pool: &conf->r10buf_pool, min_nr: buffs,
3188	alloc_fn: r10buf_pool_alloc, free_fn: r10buf_pool_free, pool_data: conf);
3189	if (ret)
3190	return ret;
3191	conf->next_resync = 0;
3192	return 0;
3193	}
3194
3195	static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf)
3196	{
3197	struct r10bio *r10bio = mempool_alloc(pool: &conf->r10buf_pool, GFP_NOIO);
3198	struct rsync_pages *rp;
3199	struct bio *bio;
3200	int nalloc;
3201	int i;
3202
3203	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
3204	test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
3205	nalloc = conf->copies; /* resync */
3206	else
3207	nalloc = 2; /* recovery */
3208
3209	for (i = 0; i < nalloc; i++) {
3210	bio = r10bio->devs[i].bio;
3211	rp = bio->bi_private;
3212	bio_reset(bio, NULL, opf: 0);
3213	bio->bi_private = rp;
3214	bio = r10bio->devs[i].repl_bio;
3215	if (bio) {
3216	rp = bio->bi_private;
3217	bio_reset(bio, NULL, opf: 0);
3218	bio->bi_private = rp;
3219	}
3220	}
3221	return r10bio;
3222	}
3223
3224	/*
3225	* Set cluster_sync_high since we need other nodes to add the
3226	* range [cluster_sync_low, cluster_sync_high] to suspend list.
3227	*/
3228	static void raid10_set_cluster_sync_high(struct r10conf *conf)
3229	{
3230	sector_t window_size;
3231	int extra_chunk, chunks;
3232
3233	/*
3234	* First, here we define "stripe" as a unit which across
3235	* all member devices one time, so we get chunks by use
3236	* raid_disks / near_copies. Otherwise, if near_copies is
3237	* close to raid_disks, then resync window could increases
3238	* linearly with the increase of raid_disks, which means
3239	* we will suspend a really large IO window while it is not
3240	* necessary. If raid_disks is not divisible by near_copies,
3241	* an extra chunk is needed to ensure the whole "stripe" is
3242	* covered.
3243	*/
3244
3245	chunks = conf->geo.raid_disks / conf->geo.near_copies;
3246	if (conf->geo.raid_disks % conf->geo.near_copies == 0)
3247	extra_chunk = 0;
3248	else
3249	extra_chunk = 1;
3250	window_size = (chunks + extra_chunk) * conf->mddev->chunk_sectors;
3251
3252	/*
3253	* At least use a 32M window to align with raid1's resync window
3254	*/
3255	window_size = (CLUSTER_RESYNC_WINDOW_SECTORS > window_size) ?
3256	CLUSTER_RESYNC_WINDOW_SECTORS : window_size;
3257
3258	conf->cluster_sync_high = conf->cluster_sync_low + window_size;
3259	}
3260
3261	/*
3262	* perform a "sync" on one "block"
3263	*
3264	* We need to make sure that no normal I/O request - particularly write
3265	* requests - conflict with active sync requests.
3266	*
3267	* This is achieved by tracking pending requests and a 'barrier' concept
3268	* that can be installed to exclude normal IO requests.
3269	*
3270	* Resync and recovery are handled very differently.
3271	* We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
3272	*
3273	* For resync, we iterate over virtual addresses, read all copies,
3274	* and update if there are differences. If only one copy is live,
3275	* skip it.
3276	* For recovery, we iterate over physical addresses, read a good
3277	* value for each non-in_sync drive, and over-write.
3278	*
3279	* So, for recovery we may have several outstanding complex requests for a
3280	* given address, one for each out-of-sync device. We model this by allocating
3281	* a number of r10_bio structures, one for each out-of-sync device.
3282	* As we setup these structures, we collect all bio's together into a list
3283	* which we then process collectively to add pages, and then process again
3284	* to pass to submit_bio_noacct.
3285	*
3286	* The r10_bio structures are linked using a borrowed master_bio pointer.
3287	* This link is counted in ->remaining. When the r10_bio that points to NULL
3288	* has its remaining count decremented to 0, the whole complex operation
3289	* is complete.
3290	*
3291	*/
3292
3293	static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
3294	int *skipped)
3295	{
3296	struct r10conf *conf = mddev->private;
3297	struct r10bio *r10_bio;
3298	struct bio *biolist = NULL, *bio;
3299	sector_t max_sector, nr_sectors;
3300	int i;
3301	int max_sync;
3302	sector_t sync_blocks;
3303	sector_t sectors_skipped = 0;
3304	int chunks_skipped = 0;
3305	sector_t chunk_mask = conf->geo.chunk_mask;
3306	int page_idx = 0;
3307	int error_disk = -1;
3308
3309	/*
3310	* Allow skipping a full rebuild for incremental assembly
3311	* of a clean array, like RAID1 does.
3312	*/
3313	if (mddev->bitmap == NULL &&
3314	mddev->recovery_cp == MaxSector &&
3315	mddev->reshape_position == MaxSector &&
3316	!test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
3317	!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
3318	!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
3319	conf->fullsync == 0) {
3320	*skipped = 1;
3321	return mddev->dev_sectors - sector_nr;
3322	}
3323
3324	if (!mempool_initialized(pool: &conf->r10buf_pool))
3325	if (init_resync(conf))
3326	return 0;
3327
3328	skipped:
3329	max_sector = mddev->dev_sectors;
3330	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
3331	test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
3332	max_sector = mddev->resync_max_sectors;
3333	if (sector_nr >= max_sector) {
3334	conf->cluster_sync_low = 0;
3335	conf->cluster_sync_high = 0;
3336
3337	/* If we aborted, we need to abort the
3338	* sync on the 'current' bitmap chucks (there can
3339	* be several when recovering multiple devices).
3340	* as we may have started syncing it but not finished.
3341	* We can find the current address in
3342	* mddev->curr_resync, but for recovery,
3343	* we need to convert that to several
3344	* virtual addresses.
3345	*/
3346	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
3347	end_reshape(conf);
3348	close_sync(conf);
3349	return 0;
3350	}
3351
3352	if (mddev->curr_resync < max_sector) { /* aborted */
3353	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
3354	md_bitmap_end_sync(bitmap: mddev->bitmap, offset: mddev->curr_resync,
3355	blocks: &sync_blocks, aborted: 1);
3356	else for (i = 0; i < conf->geo.raid_disks; i++) {
3357	sector_t sect =
3358	raid10_find_virt(conf, sector: mddev->curr_resync, dev: i);
3359	md_bitmap_end_sync(bitmap: mddev->bitmap, offset: sect,
3360	blocks: &sync_blocks, aborted: 1);
3361	}
3362	} else {
3363	/* completed sync */
3364	if ((!mddev->bitmap || conf->fullsync)
3365	&& conf->have_replacement
3366	&& test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3367	/* Completed a full sync so the replacements
3368	* are now fully recovered.
3369	*/
3370	rcu_read_lock();
3371	for (i = 0; i < conf->geo.raid_disks; i++) {
3372	struct md_rdev *rdev =
3373	rcu_dereference(conf->mirrors[i].replacement);
3374	if (rdev)
3375	rdev->recovery_offset = MaxSector;
3376	}
3377	rcu_read_unlock();
3378	}
3379	conf->fullsync = 0;
3380	}
3381	md_bitmap_close_sync(bitmap: mddev->bitmap);
3382	close_sync(conf);
3383	*skipped = 1;
3384	return sectors_skipped;
3385	}
3386
3387	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
3388	return reshape_request(mddev, sector_nr, skipped);
3389
3390	if (chunks_skipped >= conf->geo.raid_disks) {
3391	pr_err("md/raid10:%s: %s fails\n", mdname(mddev),
3392	test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ? "resync" : "recovery");
3393	if (error_disk >= 0 &&
3394	!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3395	/*
3396	* recovery fails, set mirrors.recovery_disabled,
3397	* device shouldn't be added to there.
3398	*/
3399	conf->mirrors[error_disk].recovery_disabled =
3400	mddev->recovery_disabled;
3401	return 0;
3402	}
3403	/*
3404	* if there has been nothing to do on any drive,
3405	* then there is nothing to do at all.
3406	*/
3407	*skipped = 1;
3408	return (max_sector - sector_nr) + sectors_skipped;
3409	}
3410
3411	if (max_sector > mddev->resync_max)
3412	max_sector = mddev->resync_max; /* Don't do IO beyond here */
3413
3414	/* make sure whole request will fit in a chunk - if chunks
3415	* are meaningful
3416	*/
3417	if (conf->geo.near_copies < conf->geo.raid_disks &&
3418	max_sector > (sector_nr | chunk_mask))
3419	max_sector = (sector_nr | chunk_mask) + 1;
3420
3421	/*
3422	* If there is non-resync activity waiting for a turn, then let it
3423	* though before starting on this new sync request.
3424	*/
3425	if (conf->nr_waiting)
3426	schedule_timeout_uninterruptible(timeout: 1);
3427
3428	/* Again, very different code for resync and recovery.
3429	* Both must result in an r10bio with a list of bios that
3430	* have bi_end_io, bi_sector, bi_bdev set,
3431	* and bi_private set to the r10bio.
3432	* For recovery, we may actually create several r10bios
3433	* with 2 bios in each, that correspond to the bios in the main one.
3434	* In this case, the subordinate r10bios link back through a
3435	* borrowed master_bio pointer, and the counter in the master
3436	* includes a ref from each subordinate.
3437	*/
3438	/* First, we decide what to do and set ->bi_end_io
3439	* To end_sync_read if we want to read, and
3440	* end_sync_write if we will want to write.
3441	*/
3442
3443	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
3444	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3445	/* recovery... the complicated one */
3446	int j;
3447	r10_bio = NULL;
3448
3449	for (i = 0 ; i < conf->geo.raid_disks; i++) {
3450	int still_degraded;
3451	struct r10bio *rb2;
3452	sector_t sect;
3453	int must_sync;
3454	int any_working;
3455	struct raid10_info *mirror = &conf->mirrors[i];
3456	struct md_rdev *mrdev, *mreplace;
3457
3458	rcu_read_lock();
3459	mrdev = rcu_dereference(mirror->rdev);
3460	mreplace = rcu_dereference(mirror->replacement);
3461
3462	if (mrdev && (test_bit(Faulty, &mrdev->flags) ||
3463	test_bit(In_sync, &mrdev->flags)))
3464	mrdev = NULL;
3465	if (mreplace && test_bit(Faulty, &mreplace->flags))
3466	mreplace = NULL;
3467
3468	if (!mrdev && !mreplace) {
3469	rcu_read_unlock();
3470	continue;
3471	}
3472
3473	still_degraded = 0;
3474	/* want to reconstruct this device */
3475	rb2 = r10_bio;
3476	sect = raid10_find_virt(conf, sector: sector_nr, dev: i);
3477	if (sect >= mddev->resync_max_sectors) {
3478	/* last stripe is not complete - don't
3479	* try to recover this sector.
3480	*/
3481	rcu_read_unlock();
3482	continue;
3483	}
3484	/* Unless we are doing a full sync, or a replacement
3485	* we only need to recover the block if it is set in
3486	* the bitmap
3487	*/
3488	must_sync = md_bitmap_start_sync(bitmap: mddev->bitmap, offset: sect,
3489	blocks: &sync_blocks, degraded: 1);
3490	if (sync_blocks < max_sync)
3491	max_sync = sync_blocks;
3492	if (!must_sync &&
3493	mreplace == NULL &&
3494	!conf->fullsync) {
3495	/* yep, skip the sync_blocks here, but don't assume
3496	* that there will never be anything to do here
3497	*/
3498	chunks_skipped = -1;
3499	rcu_read_unlock();
3500	continue;
3501	}
3502	if (mrdev)
3503	atomic_inc(v: &mrdev->nr_pending);
3504	if (mreplace)
3505	atomic_inc(v: &mreplace->nr_pending);
3506	rcu_read_unlock();
3507
3508	r10_bio = raid10_alloc_init_r10buf(conf);
3509	r10_bio->state = 0;
3510	raise_barrier(conf, force: rb2 != NULL);
3511	atomic_set(v: &r10_bio->remaining, i: 0);
3512
3513	r10_bio->master_bio = (struct bio*)rb2;
3514	if (rb2)
3515	atomic_inc(v: &rb2->remaining);
3516	r10_bio->mddev = mddev;
3517	set_bit(nr: R10BIO_IsRecover, addr: &r10_bio->state);
3518	r10_bio->sector = sect;
3519
3520	raid10_find_phys(conf, r10bio: r10_bio);
3521
3522	/* Need to check if the array will still be
3523	* degraded
3524	*/
3525	rcu_read_lock();
3526	for (j = 0; j < conf->geo.raid_disks; j++) {
3527	struct md_rdev *rdev = rcu_dereference(
3528	conf->mirrors[j].rdev);
3529	if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3530	still_degraded = 1;
3531	break;
3532	}
3533	}
3534
3535	must_sync = md_bitmap_start_sync(bitmap: mddev->bitmap, offset: sect,
3536	blocks: &sync_blocks, degraded: still_degraded);
3537
3538	any_working = 0;
3539	for (j=0; j<conf->copies;j++) {
3540	int k;
3541	int d = r10_bio->devs[j].devnum;
3542	sector_t from_addr, to_addr;
3543	struct md_rdev *rdev =
3544	rcu_dereference(conf->mirrors[d].rdev);
3545	sector_t sector, first_bad;
3546	int bad_sectors;
3547	if (!rdev ||
3548	!test_bit(In_sync, &rdev->flags))
3549	continue;
3550	/* This is where we read from */
3551	any_working = 1;
3552	sector = r10_bio->devs[j].addr;
3553
3554	if (is_badblock(rdev, s: sector, sectors: max_sync,
3555	first_bad: &first_bad, bad_sectors: &bad_sectors)) {
3556	if (first_bad > sector)
3557	max_sync = first_bad - sector;
3558	else {
3559	bad_sectors -= (sector
3560	- first_bad);
3561	if (max_sync > bad_sectors)
3562	max_sync = bad_sectors;
3563	continue;
3564	}
3565	}
3566	bio = r10_bio->devs[0].bio;
3567	bio->bi_next = biolist;
3568	biolist = bio;
3569	bio->bi_end_io = end_sync_read;
3570	bio->bi_opf = REQ_OP_READ;
3571	if (test_bit(FailFast, &rdev->flags))
3572	bio->bi_opf |= MD_FAILFAST;
3573	from_addr = r10_bio->devs[j].addr;
3574	bio->bi_iter.bi_sector = from_addr +
3575	rdev->data_offset;
3576	bio_set_dev(bio, bdev: rdev->bdev);
3577	atomic_inc(v: &rdev->nr_pending);
3578	/* and we write to 'i' (if not in_sync) */
3579
3580	for (k=0; k<conf->copies; k++)
3581	if (r10_bio->devs[k].devnum == i)
3582	break;
3583	BUG_ON(k == conf->copies);
3584	to_addr = r10_bio->devs[k].addr;
3585	r10_bio->devs[0].devnum = d;
3586	r10_bio->devs[0].addr = from_addr;
3587	r10_bio->devs[1].devnum = i;
3588	r10_bio->devs[1].addr = to_addr;
3589
3590	if (mrdev) {
3591	bio = r10_bio->devs[1].bio;
3592	bio->bi_next = biolist;
3593	biolist = bio;
3594	bio->bi_end_io = end_sync_write;
3595	bio->bi_opf = REQ_OP_WRITE;
3596	bio->bi_iter.bi_sector = to_addr
3597	+ mrdev->data_offset;
3598	bio_set_dev(bio, bdev: mrdev->bdev);
3599	atomic_inc(v: &r10_bio->remaining);
3600	} else
3601	r10_bio->devs[1].bio->bi_end_io = NULL;
3602
3603	/* and maybe write to replacement */
3604	bio = r10_bio->devs[1].repl_bio;
3605	if (bio)
3606	bio->bi_end_io = NULL;
3607	/* Note: if replace is not NULL, then bio
3608	* cannot be NULL as r10buf_pool_alloc will
3609	* have allocated it.
3610	*/
3611	if (!mreplace)
3612	break;
3613	bio->bi_next = biolist;
3614	biolist = bio;
3615	bio->bi_end_io = end_sync_write;
3616	bio->bi_opf = REQ_OP_WRITE;
3617	bio->bi_iter.bi_sector = to_addr +
3618	mreplace->data_offset;
3619	bio_set_dev(bio, bdev: mreplace->bdev);
3620	atomic_inc(v: &r10_bio->remaining);
3621	break;
3622	}
3623	rcu_read_unlock();
3624	if (j == conf->copies) {
3625	/* Cannot recover, so abort the recovery or
3626	* record a bad block */
3627	if (any_working) {
3628	/* problem is that there are bad blocks
3629	* on other device(s)
3630	*/
3631	int k;
3632	for (k = 0; k < conf->copies; k++)
3633	if (r10_bio->devs[k].devnum == i)
3634	break;
3635	if (mrdev && !test_bit(In_sync,
3636	&mrdev->flags)
3637	&& !rdev_set_badblocks(
3638	rdev: mrdev,
3639	s: r10_bio->devs[k].addr,
3640	sectors: max_sync, is_new: 0))
3641	any_working = 0;
3642	if (mreplace &&
3643	!rdev_set_badblocks(
3644	rdev: mreplace,
3645	s: r10_bio->devs[k].addr,
3646	sectors: max_sync, is_new: 0))
3647	any_working = 0;
3648	}
3649	if (!any_working) {
3650	if (!test_and_set_bit(nr: MD_RECOVERY_INTR,
3651	addr: &mddev->recovery))
3652	pr_warn("md/raid10:%s: insufficient working devices for recovery.\n",
3653	mdname(mddev));
3654	mirror->recovery_disabled
3655	= mddev->recovery_disabled;
3656	} else {
3657	error_disk = i;
3658	}
3659	put_buf(r10_bio);
3660	if (rb2)
3661	atomic_dec(v: &rb2->remaining);
3662	r10_bio = rb2;
3663	if (mrdev)
3664	rdev_dec_pending(rdev: mrdev, mddev);
3665	if (mreplace)
3666	rdev_dec_pending(rdev: mreplace, mddev);
3667	break;
3668	}
3669	if (mrdev)
3670	rdev_dec_pending(rdev: mrdev, mddev);
3671	if (mreplace)
3672	rdev_dec_pending(rdev: mreplace, mddev);
3673	if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) {
3674	/* Only want this if there is elsewhere to
3675	* read from. 'j' is currently the first
3676	* readable copy.
3677	*/
3678	int targets = 1;
3679	for (; j < conf->copies; j++) {
3680	int d = r10_bio->devs[j].devnum;
3681	if (conf->mirrors[d].rdev &&
3682	test_bit(In_sync,
3683	&conf->mirrors[d].rdev->flags))
3684	targets++;
3685	}
3686	if (targets == 1)
3687	r10_bio->devs[0].bio->bi_opf
3688	&= ~MD_FAILFAST;
3689	}
3690	}
3691	if (biolist == NULL) {
3692	while (r10_bio) {
3693	struct r10bio *rb2 = r10_bio;
3694	r10_bio = (struct r10bio*) rb2->master_bio;
3695	rb2->master_bio = NULL;
3696	put_buf(r10_bio: rb2);
3697	}
3698	goto giveup;
3699	}
3700	} else {
3701	/* resync. Schedule a read for every block at this virt offset */
3702	int count = 0;
3703
3704	/*
3705	* Since curr_resync_completed could probably not update in
3706	* time, and we will set cluster_sync_low based on it.
3707	* Let's check against "sector_nr + 2 * RESYNC_SECTORS" for
3708	* safety reason, which ensures curr_resync_completed is
3709	* updated in bitmap_cond_end_sync.
3710	*/
3711	md_bitmap_cond_end_sync(bitmap: mddev->bitmap, sector: sector_nr,
3712	force: mddev_is_clustered(mddev) &&
3713	(sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));
3714
3715	if (!md_bitmap_start_sync(bitmap: mddev->bitmap, offset: sector_nr,
3716	blocks: &sync_blocks, degraded: mddev->degraded) &&
3717	!conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
3718	&mddev->recovery)) {
3719	/* We can skip this block */
3720	*skipped = 1;
3721	return sync_blocks + sectors_skipped;
3722	}
3723	if (sync_blocks < max_sync)
3724	max_sync = sync_blocks;
3725	r10_bio = raid10_alloc_init_r10buf(conf);
3726	r10_bio->state = 0;
3727
3728	r10_bio->mddev = mddev;
3729	atomic_set(v: &r10_bio->remaining, i: 0);
3730	raise_barrier(conf, force: 0);
3731	conf->next_resync = sector_nr;
3732
3733	r10_bio->master_bio = NULL;
3734	r10_bio->sector = sector_nr;
3735	set_bit(nr: R10BIO_IsSync, addr: &r10_bio->state);
3736	raid10_find_phys(conf, r10bio: r10_bio);
3737	r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
3738
3739	for (i = 0; i < conf->copies; i++) {
3740	int d = r10_bio->devs[i].devnum;
3741	sector_t first_bad, sector;
3742	int bad_sectors;
3743	struct md_rdev *rdev;
3744
3745	if (r10_bio->devs[i].repl_bio)
3746	r10_bio->devs[i].repl_bio->bi_end_io = NULL;
3747
3748	bio = r10_bio->devs[i].bio;
3749	bio->bi_status = BLK_STS_IOERR;
3750	rcu_read_lock();
3751	rdev = rcu_dereference(conf->mirrors[d].rdev);
3752	if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3753	rcu_read_unlock();
3754	continue;
3755	}
3756	sector = r10_bio->devs[i].addr;
3757	if (is_badblock(rdev, s: sector, sectors: max_sync,
3758	first_bad: &first_bad, bad_sectors: &bad_sectors)) {
3759	if (first_bad > sector)
3760	max_sync = first_bad - sector;
3761	else {
3762	bad_sectors -= (sector - first_bad);
3763	if (max_sync > bad_sectors)
3764	max_sync = bad_sectors;
3765	rcu_read_unlock();
3766	continue;
3767	}
3768	}
3769	atomic_inc(v: &rdev->nr_pending);
3770	atomic_inc(v: &r10_bio->remaining);
3771	bio->bi_next = biolist;
3772	biolist = bio;
3773	bio->bi_end_io = end_sync_read;
3774	bio->bi_opf = REQ_OP_READ;
3775	if (test_bit(FailFast, &rdev->flags))
3776	bio->bi_opf |= MD_FAILFAST;
3777	bio->bi_iter.bi_sector = sector + rdev->data_offset;
3778	bio_set_dev(bio, bdev: rdev->bdev);
3779	count++;
3780
3781	rdev = rcu_dereference(conf->mirrors[d].replacement);
3782	if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3783	rcu_read_unlock();
3784	continue;
3785	}
3786	atomic_inc(v: &rdev->nr_pending);
3787
3788	/* Need to set up for writing to the replacement */
3789	bio = r10_bio->devs[i].repl_bio;
3790	bio->bi_status = BLK_STS_IOERR;
3791
3792	sector = r10_bio->devs[i].addr;
3793	bio->bi_next = biolist;
3794	biolist = bio;
3795	bio->bi_end_io = end_sync_write;
3796	bio->bi_opf = REQ_OP_WRITE;
3797	if (test_bit(FailFast, &rdev->flags))
3798	bio->bi_opf |= MD_FAILFAST;
3799	bio->bi_iter.bi_sector = sector + rdev->data_offset;
3800	bio_set_dev(bio, bdev: rdev->bdev);
3801	count++;
3802	rcu_read_unlock();
3803	}
3804
3805	if (count < 2) {
3806	for (i=0; i<conf->copies; i++) {
3807	int d = r10_bio->devs[i].devnum;
3808	if (r10_bio->devs[i].bio->bi_end_io)
3809	rdev_dec_pending(rdev: conf->mirrors[d].rdev,
3810	mddev);
3811	if (r10_bio->devs[i].repl_bio &&
3812	r10_bio->devs[i].repl_bio->bi_end_io)
3813	rdev_dec_pending(
3814	rdev: conf->mirrors[d].replacement,
3815	mddev);
3816	}
3817	put_buf(r10_bio);
3818	biolist = NULL;
3819	goto giveup;
3820	}
3821	}
3822
3823	nr_sectors = 0;
3824	if (sector_nr + max_sync < max_sector)
3825	max_sector = sector_nr + max_sync;
3826	do {
3827	struct page *page;
3828	int len = PAGE_SIZE;
3829	if (sector_nr + (len>>9) > max_sector)
3830	len = (max_sector - sector_nr) << 9;
3831	if (len == 0)
3832	break;
3833	for (bio= biolist ; bio ; bio=bio->bi_next) {
3834	struct resync_pages *rp = get_resync_pages(bio);
3835	page = resync_fetch_page(rp, idx: page_idx);
3836	if (WARN_ON(!bio_add_page(bio, page, len, 0))) {
3837	bio->bi_status = BLK_STS_RESOURCE;
3838	bio_endio(bio);
3839	goto giveup;
3840	}
3841	}
3842	nr_sectors += len>>9;
3843	sector_nr += len>>9;
3844	} while (++page_idx < RESYNC_PAGES);
3845	r10_bio->sectors = nr_sectors;
3846
3847	if (mddev_is_clustered(mddev) &&
3848	test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
3849	/* It is resync not recovery */
3850	if (conf->cluster_sync_high < sector_nr + nr_sectors) {
3851	conf->cluster_sync_low = mddev->curr_resync_completed;
3852	raid10_set_cluster_sync_high(conf);
3853	/* Send resync message */
3854	md_cluster_ops->resync_info_update(mddev,
3855	conf->cluster_sync_low,
3856	conf->cluster_sync_high);
3857	}
3858	} else if (mddev_is_clustered(mddev)) {
3859	/* This is recovery not resync */
3860	sector_t sect_va1, sect_va2;
3861	bool broadcast_msg = false;
3862
3863	for (i = 0; i < conf->geo.raid_disks; i++) {
3864	/*
3865	* sector_nr is a device address for recovery, so we
3866	* need translate it to array address before compare
3867	* with cluster_sync_high.
3868	*/
3869	sect_va1 = raid10_find_virt(conf, sector: sector_nr, dev: i);
3870
3871	if (conf->cluster_sync_high < sect_va1 + nr_sectors) {
3872	broadcast_msg = true;
3873	/*
3874	* curr_resync_completed is similar as
3875	* sector_nr, so make the translation too.
3876	*/
3877	sect_va2 = raid10_find_virt(conf,
3878	sector: mddev->curr_resync_completed, dev: i);
3879
3880	if (conf->cluster_sync_low == 0 ||
3881	conf->cluster_sync_low > sect_va2)
3882	conf->cluster_sync_low = sect_va2;
3883	}
3884	}
3885	if (broadcast_msg) {
3886	raid10_set_cluster_sync_high(conf);
3887	md_cluster_ops->resync_info_update(mddev,
3888	conf->cluster_sync_low,
3889	conf->cluster_sync_high);
3890	}
3891	}
3892
3893	while (biolist) {
3894	bio = biolist;
3895	biolist = biolist->bi_next;
3896
3897	bio->bi_next = NULL;
3898	r10_bio = get_resync_r10bio(bio);
3899	r10_bio->sectors = nr_sectors;
3900
3901	if (bio->bi_end_io == end_sync_read) {
3902	md_sync_acct_bio(bio, nr_sectors);
3903	bio->bi_status = 0;
3904	submit_bio_noacct(bio);
3905	}
3906	}
3907
3908	if (sectors_skipped)
3909	/* pretend they weren't skipped, it makes
3910	* no important difference in this case
3911	*/
3912	md_done_sync(mddev, blocks: sectors_skipped, ok: 1);
3913
3914	return sectors_skipped + nr_sectors;
3915	giveup:
3916	/* There is nowhere to write, so all non-sync
3917	* drives must be failed or in resync, all drives
3918	* have a bad block, so try the next chunk...
3919	*/
3920	if (sector_nr + max_sync < max_sector)
3921	max_sector = sector_nr + max_sync;
3922
3923	sectors_skipped += (max_sector - sector_nr);
3924	chunks_skipped ++;
3925	sector_nr = max_sector;
3926	goto skipped;
3927	}
3928
3929	static sector_t
3930	raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3931	{
3932	sector_t size;
3933	struct r10conf *conf = mddev->private;
3934
3935	if (!raid_disks)
3936	raid_disks = min(conf->geo.raid_disks,
3937	conf->prev.raid_disks);
3938	if (!sectors)
3939	sectors = conf->dev_sectors;
3940
3941	size = sectors >> conf->geo.chunk_shift;
3942	sector_div(size, conf->geo.far_copies);
3943	size = size * raid_disks;
3944	sector_div(size, conf->geo.near_copies);
3945
3946	return size << conf->geo.chunk_shift;
3947	}
3948
3949	static void calc_sectors(struct r10conf *conf, sector_t size)
3950	{
3951	/* Calculate the number of sectors-per-device that will
3952	* actually be used, and set conf->dev_sectors and
3953	* conf->stride
3954	*/
3955
3956	size = size >> conf->geo.chunk_shift;
3957	sector_div(size, conf->geo.far_copies);
3958	size = size * conf->geo.raid_disks;
3959	sector_div(size, conf->geo.near_copies);
3960	/* 'size' is now the number of chunks in the array */
3961	/* calculate "used chunks per device" */
3962	size = size * conf->copies;
3963
3964	/* We need to round up when dividing by raid_disks to
3965	* get the stride size.
3966	*/
3967	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3968
3969	conf->dev_sectors = size << conf->geo.chunk_shift;
3970
3971	if (conf->geo.far_offset)
3972	conf->geo.stride = 1 << conf->geo.chunk_shift;
3973	else {
3974	sector_div(size, conf->geo.far_copies);
3975	conf->geo.stride = size << conf->geo.chunk_shift;
3976	}
3977	}
3978
3979	enum geo_type {geo_new, geo_old, geo_start};
3980	static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
3981	{
3982	int nc, fc, fo;
3983	int layout, chunk, disks;
3984	switch (new) {
3985	case geo_old:
3986	layout = mddev->layout;
3987	chunk = mddev->chunk_sectors;
3988	disks = mddev->raid_disks - mddev->delta_disks;
3989	break;
3990	case geo_new:
3991	layout = mddev->new_layout;
3992	chunk = mddev->new_chunk_sectors;
3993	disks = mddev->raid_disks;
3994	break;
3995	default: /* avoid 'may be unused' warnings */
3996	case geo_start: /* new when starting reshape - raid_disks not
3997	* updated yet. */
3998	layout = mddev->new_layout;
3999	chunk = mddev->new_chunk_sectors;
4000	disks = mddev->raid_disks + mddev->delta_disks;
4001	break;
4002	}
4003	if (layout >> 19)
4004	return -1;
4005	if (chunk < (PAGE_SIZE >> 9) ||
4006	!is_power_of_2(n: chunk))
4007	return -2;
4008	nc = layout & 255;
4009	fc = (layout >> 8) & 255;
4010	fo = layout & (1<<16);
4011	geo->raid_disks = disks;
4012	geo->near_copies = nc;
4013	geo->far_copies = fc;
4014	geo->far_offset = fo;
4015	switch (layout >> 17) {
4016	case 0: /* original layout. simple but not always optimal */
4017	geo->far_set_size = disks;
4018	break;
4019	case 1: /* "improved" layout which was buggy. Hopefully no-one is
4020	* actually using this, but leave code here just in case.*/
4021	geo->far_set_size = disks/fc;
4022	WARN(geo->far_set_size < fc,
4023	"This RAID10 layout does not provide data safety - please backup and create new array\n");
4024	break;
4025	case 2: /* "improved" layout fixed to match documentation */
4026	geo->far_set_size = fc * nc;
4027	break;
4028	default: /* Not a valid layout */
4029	return -1;
4030	}
4031	geo->chunk_mask = chunk - 1;
4032	geo->chunk_shift = ffz(~chunk);
4033	return nc*fc;
4034	}
4035
4036	static void raid10_free_conf(struct r10conf *conf)
4037	{
4038	if (!conf)
4039	return;
4040
4041	mempool_exit(pool: &conf->r10bio_pool);
4042	kfree(objp: conf->mirrors);
4043	kfree(objp: conf->mirrors_old);
4044	kfree(objp: conf->mirrors_new);
4045	safe_put_page(p: conf->tmppage);
4046	bioset_exit(&conf->bio_split);
4047	kfree(objp: conf);
4048	}
4049
4050	static struct r10conf *setup_conf(struct mddev *mddev)
4051	{
4052	struct r10conf *conf = NULL;
4053	int err = -EINVAL;
4054	struct geom geo;
4055	int copies;
4056
4057	copies = setup_geo(geo: &geo, mddev, new: geo_new);
4058
4059	if (copies == -2) {
4060	pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n",
4061	mdname(mddev), PAGE_SIZE);
4062	goto out;
4063	}
4064
4065	if (copies < 2 || copies > mddev->raid_disks) {
4066	pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
4067	mdname(mddev), mddev->new_layout);
4068	goto out;
4069	}
4070
4071	err = -ENOMEM;
4072	conf = kzalloc(size: sizeof(struct r10conf), GFP_KERNEL);
4073	if (!conf)
4074	goto out;
4075
4076	/* FIXME calc properly */
4077	conf->mirrors = kcalloc(n: mddev->raid_disks + max(0, -mddev->delta_disks),
4078	size: sizeof(struct raid10_info),
4079	GFP_KERNEL);
4080	if (!conf->mirrors)
4081	goto out;
4082
4083	conf->tmppage = alloc_page(GFP_KERNEL);
4084	if (!conf->tmppage)
4085	goto out;
4086
4087	conf->geo = geo;
4088	conf->copies = copies;
4089	err = mempool_init(pool: &conf->r10bio_pool, NR_RAID_BIOS, alloc_fn: r10bio_pool_alloc,
4090	free_fn: rbio_pool_free, pool_data: conf);
4091	if (err)
4092	goto out;
4093
4094	err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, flags: 0);
4095	if (err)
4096	goto out;
4097
4098	calc_sectors(conf, size: mddev->dev_sectors);
4099	if (mddev->reshape_position == MaxSector) {
4100	conf->prev = conf->geo;
4101	conf->reshape_progress = MaxSector;
4102	} else {
4103	if (setup_geo(geo: &conf->prev, mddev, new: geo_old) != conf->copies) {
4104	err = -EINVAL;
4105	goto out;
4106	}
4107	conf->reshape_progress = mddev->reshape_position;
4108	if (conf->prev.far_offset)
4109	conf->prev.stride = 1 << conf->prev.chunk_shift;
4110	else
4111	/* far_copies must be 1 */
4112	conf->prev.stride = conf->dev_sectors;
4113	}
4114	conf->reshape_safe = conf->reshape_progress;
4115	spin_lock_init(&conf->device_lock);
4116	INIT_LIST_HEAD(list: &conf->retry_list);
4117	INIT_LIST_HEAD(list: &conf->bio_end_io_list);
4118
4119	seqlock_init(&conf->resync_lock);
4120	init_waitqueue_head(&conf->wait_barrier);
4121	atomic_set(v: &conf->nr_pending, i: 0);
4122
4123	err = -ENOMEM;
4124	rcu_assign_pointer(conf->thread,
4125	md_register_thread(raid10d, mddev, "raid10"));
4126	if (!conf->thread)
4127	goto out;
4128
4129	conf->mddev = mddev;
4130	return conf;
4131
4132	out:
4133	raid10_free_conf(conf);
4134	return ERR_PTR(error: err);
4135	}
4136
4137	static void raid10_set_io_opt(struct r10conf *conf)
4138	{
4139	int raid_disks = conf->geo.raid_disks;
4140
4141	if (!(conf->geo.raid_disks % conf->geo.near_copies))
4142	raid_disks /= conf->geo.near_copies;
4143	blk_queue_io_opt(q: conf->mddev->queue, opt: (conf->mddev->chunk_sectors << 9) *
4144	raid_disks);
4145	}
4146
4147	static int raid10_run(struct mddev *mddev)
4148	{
4149	struct r10conf *conf;
4150	int i, disk_idx;
4151	struct raid10_info *disk;
4152	struct md_rdev *rdev;
4153	sector_t size;
4154	sector_t min_offset_diff = 0;
4155	int first = 1;
4156
4157	if (mddev->private == NULL) {
4158	conf = setup_conf(mddev);
4159	if (IS_ERR(ptr: conf))
4160	return PTR_ERR(ptr: conf);
4161	mddev->private = conf;
4162	}
4163	conf = mddev->private;
4164	if (!conf)
4165	goto out;
4166
4167	rcu_assign_pointer(mddev->thread, conf->thread);
4168	rcu_assign_pointer(conf->thread, NULL);
4169
4170	if (mddev_is_clustered(mddev: conf->mddev)) {
4171	int fc, fo;
4172
4173	fc = (mddev->layout >> 8) & 255;
4174	fo = mddev->layout & (1<<16);
4175	if (fc > 1 || fo > 0) {
4176	pr_err("only near layout is supported by clustered"
4177	" raid10\n");
4178	goto out_free_conf;
4179	}
4180	}
4181
4182	if (mddev->queue) {
4183	blk_queue_max_write_zeroes_sectors(q: mddev->queue, max_write_same_sectors: 0);
4184	blk_queue_io_min(q: mddev->queue, min: mddev->chunk_sectors << 9);
4185	raid10_set_io_opt(conf);
4186	}
4187
4188	rdev_for_each(rdev, mddev) {
4189	long long diff;
4190
4191	disk_idx = rdev->raid_disk;
4192	if (disk_idx < 0)
4193	continue;
4194	if (disk_idx >= conf->geo.raid_disks &&
4195	disk_idx >= conf->prev.raid_disks)
4196	continue;
4197	disk = conf->mirrors + disk_idx;
4198
4199	if (test_bit(Replacement, &rdev->flags)) {
4200	if (disk->replacement)
4201	goto out_free_conf;
4202	disk->replacement = rdev;
4203	} else {
4204	if (disk->rdev)
4205	goto out_free_conf;
4206	disk->rdev = rdev;
4207	}
4208	diff = (rdev->new_data_offset - rdev->data_offset);
4209	if (!mddev->reshape_backwards)
4210	diff = -diff;
4211	if (diff < 0)
4212	diff = 0;
4213	if (first || diff < min_offset_diff)
4214	min_offset_diff = diff;
4215
4216	if (mddev->gendisk)
4217	disk_stack_limits(disk: mddev->gendisk, bdev: rdev->bdev,
4218	offset: rdev->data_offset << 9);
4219
4220	disk->head_position = 0;
4221	first = 0;
4222	}
4223
4224	/* need to check that every block has at least one working mirror */
4225	if (!enough(conf, ignore: -1)) {
4226	pr_err("md/raid10:%s: not enough operational mirrors.\n",
4227	mdname(mddev));
4228	goto out_free_conf;
4229	}
4230
4231	if (conf->reshape_progress != MaxSector) {
4232	/* must ensure that shape change is supported */
4233	if (conf->geo.far_copies != 1 &&
4234	conf->geo.far_offset == 0)
4235	goto out_free_conf;
4236	if (conf->prev.far_copies != 1 &&
4237	conf->prev.far_offset == 0)
4238	goto out_free_conf;
4239	}
4240
4241	mddev->degraded = 0;
4242	for (i = 0;
4243	i < conf->geo.raid_disks
4244	|| i < conf->prev.raid_disks;
4245	i++) {
4246
4247	disk = conf->mirrors + i;
4248
4249	if (!disk->rdev && disk->replacement) {
4250	/* The replacement is all we have - use it */
4251	disk->rdev = disk->replacement;
4252	disk->replacement = NULL;
4253	clear_bit(nr: Replacement, addr: &disk->rdev->flags);
4254	}
4255
4256	if (!disk->rdev ||
4257	!test_bit(In_sync, &disk->rdev->flags)) {
4258	disk->head_position = 0;
4259	mddev->degraded++;
4260	if (disk->rdev &&
4261	disk->rdev->saved_raid_disk < 0)
4262	conf->fullsync = 1;
4263	}
4264
4265	if (disk->replacement &&
4266	!test_bit(In_sync, &disk->replacement->flags) &&
4267	disk->replacement->saved_raid_disk < 0) {
4268	conf->fullsync = 1;
4269	}
4270
4271	disk->recovery_disabled = mddev->recovery_disabled - 1;
4272	}
4273
4274	if (mddev->recovery_cp != MaxSector)
4275	pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n",
4276	mdname(mddev));
4277	pr_info("md/raid10:%s: active with %d out of %d devices\n",
4278	mdname(mddev), conf->geo.raid_disks - mddev->degraded,
4279	conf->geo.raid_disks);
4280	/*
4281	* Ok, everything is just fine now
4282	*/
4283	mddev->dev_sectors = conf->dev_sectors;
4284	size = raid10_size(mddev, sectors: 0, raid_disks: 0);
4285	md_set_array_sectors(mddev, array_sectors: size);
4286	mddev->resync_max_sectors = size;
4287	set_bit(nr: MD_FAILFAST_SUPPORTED, addr: &mddev->flags);
4288
4289	if (md_integrity_register(mddev))
4290	goto out_free_conf;
4291
4292	if (conf->reshape_progress != MaxSector) {
4293	unsigned long before_length, after_length;
4294
4295	before_length = ((1 << conf->prev.chunk_shift) *
4296	conf->prev.far_copies);
4297	after_length = ((1 << conf->geo.chunk_shift) *
4298	conf->geo.far_copies);
4299
4300	if (max(before_length, after_length) > min_offset_diff) {
4301	/* This cannot work */
4302	pr_warn("md/raid10: offset difference not enough to continue reshape\n");
4303	goto out_free_conf;
4304	}
4305	conf->offset_diff = min_offset_diff;
4306
4307	clear_bit(nr: MD_RECOVERY_SYNC, addr: &mddev->recovery);
4308	clear_bit(nr: MD_RECOVERY_CHECK, addr: &mddev->recovery);
4309	set_bit(nr: MD_RECOVERY_RESHAPE, addr: &mddev->recovery);
4310	set_bit(nr: MD_RECOVERY_RUNNING, addr: &mddev->recovery);
4311	rcu_assign_pointer(mddev->sync_thread,
4312	md_register_thread(md_do_sync, mddev, "reshape"));
4313	if (!mddev->sync_thread)
4314	goto out_free_conf;
4315	}
4316
4317	return 0;
4318
4319	out_free_conf:
4320	md_unregister_thread(mddev, threadp: &mddev->thread);
4321	raid10_free_conf(conf);
4322	mddev->private = NULL;
4323	out:
4324	return -EIO;
4325	}
4326
4327	static void raid10_free(struct mddev *mddev, void *priv)
4328	{
4329	raid10_free_conf(conf: priv);
4330	}
4331
4332	static void raid10_quiesce(struct mddev *mddev, int quiesce)
4333	{
4334	struct r10conf *conf = mddev->private;
4335
4336	if (quiesce)
4337	raise_barrier(conf, force: 0);
4338	else
4339	lower_barrier(conf);
4340	}
4341
4342	static int raid10_resize(struct mddev *mddev, sector_t sectors)
4343	{
4344	/* Resize of 'far' arrays is not supported.
4345	* For 'near' and 'offset' arrays we can set the
4346	* number of sectors used to be an appropriate multiple
4347	* of the chunk size.
4348	* For 'offset', this is far_copies*chunksize.
4349	* For 'near' the multiplier is the LCM of
4350	* near_copies and raid_disks.
4351	* So if far_copies > 1 && !far_offset, fail.
4352	* Else find LCM(raid_disks, near_copy)*far_copies and
4353	* multiply by chunk_size. Then round to this number.
4354	* This is mostly done by raid10_size()
4355	*/
4356	struct r10conf *conf = mddev->private;
4357	sector_t oldsize, size;
4358
4359	if (mddev->reshape_position != MaxSector)
4360	return -EBUSY;
4361
4362	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
4363	return -EINVAL;
4364
4365	oldsize = raid10_size(mddev, sectors: 0, raid_disks: 0);
4366	size = raid10_size(mddev, sectors, raid_disks: 0);
4367	if (mddev->external_size &&
4368	mddev->array_sectors > size)
4369	return -EINVAL;
4370	if (mddev->bitmap) {
4371	int ret = md_bitmap_resize(bitmap: mddev->bitmap, blocks: size, chunksize: 0, init: 0);
4372	if (ret)
4373	return ret;
4374	}
4375	md_set_array_sectors(mddev, array_sectors: size);
4376	if (sectors > mddev->dev_sectors &&
4377	mddev->recovery_cp > oldsize) {
4378	mddev->recovery_cp = oldsize;
4379	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
4380	}
4381	calc_sectors(conf, size: sectors);
4382	mddev->dev_sectors = conf->dev_sectors;
4383	mddev->resync_max_sectors = size;
4384	return 0;
4385	}
4386
4387	static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
4388	{
4389	struct md_rdev *rdev;
4390	struct r10conf *conf;
4391
4392	if (mddev->degraded > 0) {
4393	pr_warn("md/raid10:%s: Error: degraded raid0!\n",
4394	mdname(mddev));
4395	return ERR_PTR(error: -EINVAL);
4396	}
4397	sector_div(size, devs);
4398
4399	/* Set new parameters */
4400	mddev->new_level = 10;
4401	/* new layout: far_copies = 1, near_copies = 2 */
4402	mddev->new_layout = (1<<8) + 2;
4403	mddev->new_chunk_sectors = mddev->chunk_sectors;
4404	mddev->delta_disks = mddev->raid_disks;
4405	mddev->raid_disks *= 2;
4406	/* make sure it will be not marked as dirty */
4407	mddev->recovery_cp = MaxSector;
4408	mddev->dev_sectors = size;
4409
4410	conf = setup_conf(mddev);
4411	if (!IS_ERR(ptr: conf)) {
4412	rdev_for_each(rdev, mddev)
4413	if (rdev->raid_disk >= 0) {
4414	rdev->new_raid_disk = rdev->raid_disk * 2;
4415	rdev->sectors = size;
4416	}
4417	}
4418
4419	return conf;
4420	}
4421
4422	static void *raid10_takeover(struct mddev *mddev)
4423	{
4424	struct r0conf *raid0_conf;
4425
4426	/* raid10 can take over:
4427	* raid0 - providing it has only two drives
4428	*/
4429	if (mddev->level == 0) {
4430	/* for raid0 takeover only one zone is supported */
4431	raid0_conf = mddev->private;
4432	if (raid0_conf->nr_strip_zones > 1) {
4433	pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n",
4434	mdname(mddev));
4435	return ERR_PTR(error: -EINVAL);
4436	}
4437	return raid10_takeover_raid0(mddev,
4438	size: raid0_conf->strip_zone->zone_end,
4439	devs: raid0_conf->strip_zone->nb_dev);
4440	}
4441	return ERR_PTR(error: -EINVAL);
4442	}
4443
4444	static int raid10_check_reshape(struct mddev *mddev)
4445	{
4446	/* Called when there is a request to change
4447	* - layout (to ->new_layout)
4448	* - chunk size (to ->new_chunk_sectors)
4449	* - raid_disks (by delta_disks)
4450	* or when trying to restart a reshape that was ongoing.
4451	*
4452	* We need to validate the request and possibly allocate
4453	* space if that might be an issue later.
4454	*
4455	* Currently we reject any reshape of a 'far' mode array,
4456	* allow chunk size to change if new is generally acceptable,
4457	* allow raid_disks to increase, and allow
4458	* a switch between 'near' mode and 'offset' mode.
4459	*/
4460	struct r10conf *conf = mddev->private;
4461	struct geom geo;
4462
4463	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
4464	return -EINVAL;
4465
4466	if (setup_geo(geo: &geo, mddev, new: geo_start) != conf->copies)
4467	/* mustn't change number of copies */
4468	return -EINVAL;
4469	if (geo.far_copies > 1 && !geo.far_offset)
4470	/* Cannot switch to 'far' mode */
4471	return -EINVAL;
4472
4473	if (mddev->array_sectors & geo.chunk_mask)
4474	/* not factor of array size */
4475	return -EINVAL;
4476
4477	if (!enough(conf, ignore: -1))
4478	return -EINVAL;
4479
4480	kfree(objp: conf->mirrors_new);
4481	conf->mirrors_new = NULL;
4482	if (mddev->delta_disks > 0) {
4483	/* allocate new 'mirrors' list */
4484	conf->mirrors_new =
4485	kcalloc(n: mddev->raid_disks + mddev->delta_disks,
4486	size: sizeof(struct raid10_info),
4487	GFP_KERNEL);
4488	if (!conf->mirrors_new)
4489	return -ENOMEM;
4490	}
4491	return 0;
4492	}
4493
4494	/*
4495	* Need to check if array has failed when deciding whether to:
4496	* - start an array
4497	* - remove non-faulty devices
4498	* - add a spare
4499	* - allow a reshape
4500	* This determination is simple when no reshape is happening.
4501	* However if there is a reshape, we need to carefully check
4502	* both the before and after sections.
4503	* This is because some failed devices may only affect one
4504	* of the two sections, and some non-in_sync devices may
4505	* be insync in the section most affected by failed devices.
4506	*/
4507	static int calc_degraded(struct r10conf *conf)
4508	{
4509	int degraded, degraded2;
4510	int i;
4511
4512	rcu_read_lock();
4513	degraded = 0;
4514	/* 'prev' section first */
4515	for (i = 0; i < conf->prev.raid_disks; i++) {
4516	struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
4517	if (!rdev || test_bit(Faulty, &rdev->flags))
4518	degraded++;
4519	else if (!test_bit(In_sync, &rdev->flags))
4520	/* When we can reduce the number of devices in
4521	* an array, this might not contribute to
4522	* 'degraded'. It does now.
4523	*/
4524	degraded++;
4525	}
4526	rcu_read_unlock();
4527	if (conf->geo.raid_disks == conf->prev.raid_disks)
4528	return degraded;
4529	rcu_read_lock();
4530	degraded2 = 0;
4531	for (i = 0; i < conf->geo.raid_disks; i++) {
4532	struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
4533	if (!rdev || test_bit(Faulty, &rdev->flags))
4534	degraded2++;
4535	else if (!test_bit(In_sync, &rdev->flags)) {
4536	/* If reshape is increasing the number of devices,
4537	* this section has already been recovered, so
4538	* it doesn't contribute to degraded.
4539	* else it does.
4540	*/
4541	if (conf->geo.raid_disks <= conf->prev.raid_disks)
4542	degraded2++;
4543	}
4544	}
4545	rcu_read_unlock();
4546	if (degraded2 > degraded)
4547	return degraded2;
4548	return degraded;
4549	}
4550
4551	static int raid10_start_reshape(struct mddev *mddev)
4552	{
4553	/* A 'reshape' has been requested. This commits
4554	* the various 'new' fields and sets MD_RECOVER_RESHAPE
4555	* This also checks if there are enough spares and adds them
4556	* to the array.
4557	* We currently require enough spares to make the final
4558	* array non-degraded. We also require that the difference
4559	* between old and new data_offset - on each device - is
4560	* enough that we never risk over-writing.
4561	*/
4562
4563	unsigned long before_length, after_length;
4564	sector_t min_offset_diff = 0;
4565	int first = 1;
4566	struct geom new;
4567	struct r10conf *conf = mddev->private;
4568	struct md_rdev *rdev;
4569	int spares = 0;
4570	int ret;
4571
4572	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
4573	return -EBUSY;
4574
4575	if (setup_geo(geo: &new, mddev, new: geo_start) != conf->copies)
4576	return -EINVAL;
4577
4578	before_length = ((1 << conf->prev.chunk_shift) *
4579	conf->prev.far_copies);
4580	after_length = ((1 << conf->geo.chunk_shift) *
4581	conf->geo.far_copies);
4582
4583	rdev_for_each(rdev, mddev) {
4584	if (!test_bit(In_sync, &rdev->flags)
4585	&& !test_bit(Faulty, &rdev->flags))
4586	spares++;
4587	if (rdev->raid_disk >= 0) {
4588	long long diff = (rdev->new_data_offset
4589	- rdev->data_offset);
4590	if (!mddev->reshape_backwards)
4591	diff = -diff;
4592	if (diff < 0)
4593	diff = 0;
4594	if (first || diff < min_offset_diff)
4595	min_offset_diff = diff;
4596	first = 0;
4597	}
4598	}
4599
4600	if (max(before_length, after_length) > min_offset_diff)
4601	return -EINVAL;
4602
4603	if (spares < mddev->delta_disks)
4604	return -EINVAL;
4605
4606	conf->offset_diff = min_offset_diff;
4607	spin_lock_irq(lock: &conf->device_lock);
4608	if (conf->mirrors_new) {
4609	memcpy(conf->mirrors_new, conf->mirrors,
4610	sizeof(struct raid10_info)*conf->prev.raid_disks);
4611	smp_mb();
4612	kfree(objp: conf->mirrors_old);
4613	conf->mirrors_old = conf->mirrors;
4614	conf->mirrors = conf->mirrors_new;
4615	conf->mirrors_new = NULL;
4616	}
4617	setup_geo(geo: &conf->geo, mddev, new: geo_start);
4618	smp_mb();
4619	if (mddev->reshape_backwards) {
4620	sector_t size = raid10_size(mddev, sectors: 0, raid_disks: 0);
4621	if (size < mddev->array_sectors) {
4622	spin_unlock_irq(lock: &conf->device_lock);
4623	pr_warn("md/raid10:%s: array size must be reduce before number of disks\n",
4624	mdname(mddev));
4625	return -EINVAL;
4626	}
4627	mddev->resync_max_sectors = size;
4628	conf->reshape_progress = size;
4629	} else
4630	conf->reshape_progress = 0;
4631	conf->reshape_safe = conf->reshape_progress;
4632	spin_unlock_irq(lock: &conf->device_lock);
4633
4634	if (mddev->delta_disks && mddev->bitmap) {
4635	struct mdp_superblock_1 *sb = NULL;
4636	sector_t oldsize, newsize;
4637
4638	oldsize = raid10_size(mddev, sectors: 0, raid_disks: 0);
4639	newsize = raid10_size(mddev, sectors: 0, raid_disks: conf->geo.raid_disks);
4640
4641	if (!mddev_is_clustered(mddev)) {
4642	ret = md_bitmap_resize(bitmap: mddev->bitmap, blocks: newsize, chunksize: 0, init: 0);
4643	if (ret)
4644	goto abort;
4645	else
4646	goto out;
4647	}
4648
4649	rdev_for_each(rdev, mddev) {
4650	if (rdev->raid_disk > -1 &&
4651	!test_bit(Faulty, &rdev->flags))
4652	sb = page_address(rdev->sb_page);
4653	}
4654
4655	/*
4656	* some node is already performing reshape, and no need to
4657	* call md_bitmap_resize again since it should be called when
4658	* receiving BITMAP_RESIZE msg
4659	*/
4660	if ((sb && (le32_to_cpu(sb->feature_map) &
4661	MD_FEATURE_RESHAPE_ACTIVE)) || (oldsize == newsize))
4662	goto out;
4663
4664	ret = md_bitmap_resize(bitmap: mddev->bitmap, blocks: newsize, chunksize: 0, init: 0);
4665	if (ret)
4666	goto abort;
4667
4668	ret = md_cluster_ops->resize_bitmaps(mddev, newsize, oldsize);
4669	if (ret) {
4670	md_bitmap_resize(bitmap: mddev->bitmap, blocks: oldsize, chunksize: 0, init: 0);
4671	goto abort;
4672	}
4673	}
4674	out:
4675	if (mddev->delta_disks > 0) {
4676	rdev_for_each(rdev, mddev)
4677	if (rdev->raid_disk < 0 &&
4678	!test_bit(Faulty, &rdev->flags)) {
4679	if (raid10_add_disk(mddev, rdev) == 0) {
4680	if (rdev->raid_disk >=
4681	conf->prev.raid_disks)
4682	set_bit(nr: In_sync, addr: &rdev->flags);
4683	else
4684	rdev->recovery_offset = 0;
4685
4686	/* Failure here is OK */
4687	sysfs_link_rdev(mddev, rdev);
4688	}
4689	} else if (rdev->raid_disk >= conf->prev.raid_disks
4690	&& !test_bit(Faulty, &rdev->flags)) {
4691	/* This is a spare that was manually added */
4692	set_bit(nr: In_sync, addr: &rdev->flags);
4693	}
4694	}
4695	/* When a reshape changes the number of devices,
4696	* ->degraded is measured against the larger of the
4697	* pre and post numbers.
4698	*/
4699	spin_lock_irq(lock: &conf->device_lock);
4700	mddev->degraded = calc_degraded(conf);
4701	spin_unlock_irq(lock: &conf->device_lock);
4702	mddev->raid_disks = conf->geo.raid_disks;
4703	mddev->reshape_position = conf->reshape_progress;
4704	set_bit(nr: MD_SB_CHANGE_DEVS, addr: &mddev->sb_flags);
4705
4706	clear_bit(nr: MD_RECOVERY_SYNC, addr: &mddev->recovery);
4707	clear_bit(nr: MD_RECOVERY_CHECK, addr: &mddev->recovery);
4708	clear_bit(nr: MD_RECOVERY_DONE, addr: &mddev->recovery);
4709	set_bit(nr: MD_RECOVERY_RESHAPE, addr: &mddev->recovery);
4710	set_bit(nr: MD_RECOVERY_RUNNING, addr: &mddev->recovery);
4711
4712	rcu_assign_pointer(mddev->sync_thread,
4713	md_register_thread(md_do_sync, mddev, "reshape"));
4714	if (!mddev->sync_thread) {
4715	ret = -EAGAIN;
4716	goto abort;
4717	}
4718	conf->reshape_checkpoint = jiffies;
4719	md_wakeup_thread(thread: mddev->sync_thread);
4720	md_new_event();
4721	return 0;
4722
4723	abort:
4724	mddev->recovery = 0;
4725	spin_lock_irq(lock: &conf->device_lock);
4726	conf->geo = conf->prev;
4727	mddev->raid_disks = conf->geo.raid_disks;
4728	rdev_for_each(rdev, mddev)
4729	rdev->new_data_offset = rdev->data_offset;
4730	smp_wmb();
4731	conf->reshape_progress = MaxSector;
4732	conf->reshape_safe = MaxSector;
4733	mddev->reshape_position = MaxSector;
4734	spin_unlock_irq(lock: &conf->device_lock);
4735	return ret;
4736	}
4737
4738	/* Calculate the last device-address that could contain
4739	* any block from the chunk that includes the array-address 's'
4740	* and report the next address.
4741	* i.e. the address returned will be chunk-aligned and after
4742	* any data that is in the chunk containing 's'.
4743	*/
4744	static sector_t last_dev_address(sector_t s, struct geom *geo)
4745	{
4746	s = (s | geo->chunk_mask) + 1;
4747	s >>= geo->chunk_shift;
4748	s *= geo->near_copies;
4749	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
4750	s *= geo->far_copies;
4751	s <<= geo->chunk_shift;
4752	return s;
4753	}
4754
4755	/* Calculate the first device-address that could contain
4756	* any block from the chunk that includes the array-address 's'.
4757	* This too will be the start of a chunk
4758	*/
4759	static sector_t first_dev_address(sector_t s, struct geom *geo)
4760	{
4761	s >>= geo->chunk_shift;
4762	s *= geo->near_copies;
4763	sector_div(s, geo->raid_disks);
4764	s *= geo->far_copies;
4765	s <<= geo->chunk_shift;
4766	return s;
4767	}
4768
4769	static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
4770	int *skipped)
4771	{
4772	/* We simply copy at most one chunk (smallest of old and new)
4773	* at a time, possibly less if that exceeds RESYNC_PAGES,
4774	* or we hit a bad block or something.
4775	* This might mean we pause for normal IO in the middle of
4776	* a chunk, but that is not a problem as mddev->reshape_position
4777	* can record any location.
4778	*
4779	* If we will want to write to a location that isn't
4780	* yet recorded as 'safe' (i.e. in metadata on disk) then
4781	* we need to flush all reshape requests and update the metadata.
4782	*
4783	* When reshaping forwards (e.g. to more devices), we interpret
4784	* 'safe' as the earliest block which might not have been copied
4785	* down yet. We divide this by previous stripe size and multiply
4786	* by previous stripe length to get lowest device offset that we
4787	* cannot write to yet.
4788	* We interpret 'sector_nr' as an address that we want to write to.
4789	* From this we use last_device_address() to find where we might
4790	* write to, and first_device_address on the 'safe' position.
4791	* If this 'next' write position is after the 'safe' position,
4792	* we must update the metadata to increase the 'safe' position.
4793	*
4794	* When reshaping backwards, we round in the opposite direction
4795	* and perform the reverse test: next write position must not be
4796	* less than current safe position.
4797	*
4798	* In all this the minimum difference in data offsets
4799	* (conf->offset_diff - always positive) allows a bit of slack,
4800	* so next can be after 'safe', but not by more than offset_diff
4801	*
4802	* We need to prepare all the bios here before we start any IO
4803	* to ensure the size we choose is acceptable to all devices.
4804	* The means one for each copy for write-out and an extra one for
4805	* read-in.
4806	* We store the read-in bio in ->master_bio and the others in
4807	* ->devs[x].bio and ->devs[x].repl_bio.
4808	*/
4809	struct r10conf *conf = mddev->private;
4810	struct r10bio *r10_bio;
4811	sector_t next, safe, last;
4812	int max_sectors;
4813	int nr_sectors;
4814	int s;
4815	struct md_rdev *rdev;
4816	int need_flush = 0;
4817	struct bio *blist;
4818	struct bio *bio, *read_bio;
4819	int sectors_done = 0;
4820	struct page **pages;
4821
4822	if (sector_nr == 0) {
4823	/* If restarting in the middle, skip the initial sectors */
4824	if (mddev->reshape_backwards &&
4825	conf->reshape_progress < raid10_size(mddev, sectors: 0, raid_disks: 0)) {
4826	sector_nr = (raid10_size(mddev, sectors: 0, raid_disks: 0)
4827	- conf->reshape_progress);
4828	} else if (!mddev->reshape_backwards &&
4829	conf->reshape_progress > 0)
4830	sector_nr = conf->reshape_progress;
4831	if (sector_nr) {
4832	mddev->curr_resync_completed = sector_nr;
4833	sysfs_notify_dirent_safe(sd: mddev->sysfs_completed);
4834	*skipped = 1;
4835	return sector_nr;
4836	}
4837	}
4838
4839	/* We don't use sector_nr to track where we are up to
4840	* as that doesn't work well for ->reshape_backwards.
4841	* So just use ->reshape_progress.
4842	*/
4843	if (mddev->reshape_backwards) {
4844	/* 'next' is the earliest device address that we might
4845	* write to for this chunk in the new layout
4846	*/
4847	next = first_dev_address(s: conf->reshape_progress - 1,
4848	geo: &conf->geo);
4849
4850	/* 'safe' is the last device address that we might read from
4851	* in the old layout after a restart
4852	*/
4853	safe = last_dev_address(s: conf->reshape_safe - 1,
4854	geo: &conf->prev);
4855
4856	if (next + conf->offset_diff < safe)
4857	need_flush = 1;
4858
4859	last = conf->reshape_progress - 1;
4860	sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
4861	& conf->prev.chunk_mask);
4862	if (sector_nr + RESYNC_SECTORS < last)
4863	sector_nr = last + 1 - RESYNC_SECTORS;
4864	} else {
4865	/* 'next' is after the last device address that we
4866	* might write to for this chunk in the new layout
4867	*/
4868	next = last_dev_address(s: conf->reshape_progress, geo: &conf->geo);
4869
4870	/* 'safe' is the earliest device address that we might
4871	* read from in the old layout after a restart
4872	*/
4873	safe = first_dev_address(s: conf->reshape_safe, geo: &conf->prev);
4874
4875	/* Need to update metadata if 'next' might be beyond 'safe'
4876	* as that would possibly corrupt data
4877	*/
4878	if (next > safe + conf->offset_diff)
4879	need_flush = 1;
4880
4881	sector_nr = conf->reshape_progress;
4882	last = sector_nr | (conf->geo.chunk_mask
4883	& conf->prev.chunk_mask);
4884
4885	if (sector_nr + RESYNC_SECTORS <= last)
4886	last = sector_nr + RESYNC_SECTORS - 1;
4887	}
4888
4889	if (need_flush ||
4890	time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4891	/* Need to update reshape_position in metadata */
4892	wait_barrier(conf, nowait: false);
4893	mddev->reshape_position = conf->reshape_progress;
4894	if (mddev->reshape_backwards)
4895	mddev->curr_resync_completed = raid10_size(mddev, sectors: 0, raid_disks: 0)
4896	- conf->reshape_progress;
4897	else
4898	mddev->curr_resync_completed = conf->reshape_progress;
4899	conf->reshape_checkpoint = jiffies;
4900	set_bit(nr: MD_SB_CHANGE_DEVS, addr: &mddev->sb_flags);
4901	md_wakeup_thread(thread: mddev->thread);
4902	wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
4903	test_bit(MD_RECOVERY_INTR, &mddev->recovery));
4904	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
4905	allow_barrier(conf);
4906	return sectors_done;
4907	}
4908	conf->reshape_safe = mddev->reshape_position;
4909	allow_barrier(conf);
4910	}
4911
4912	raise_barrier(conf, force: 0);
4913	read_more:
4914	/* Now schedule reads for blocks from sector_nr to last */
4915	r10_bio = raid10_alloc_init_r10buf(conf);
4916	r10_bio->state = 0;
4917	raise_barrier(conf, force: 1);
4918	atomic_set(v: &r10_bio->remaining, i: 0);
4919	r10_bio->mddev = mddev;
4920	r10_bio->sector = sector_nr;
4921	set_bit(nr: R10BIO_IsReshape, addr: &r10_bio->state);
4922	r10_bio->sectors = last - sector_nr + 1;
4923	rdev = read_balance(conf, r10_bio, max_sectors: &max_sectors);
4924	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));
4925
4926	if (!rdev) {
4927	/* Cannot read from here, so need to record bad blocks
4928	* on all the target devices.
4929	*/
4930	// FIXME
4931	mempool_free(element: r10_bio, pool: &conf->r10buf_pool);
4932	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
4933	return sectors_done;
4934	}
4935
4936	read_bio = bio_alloc_bioset(bdev: rdev->bdev, RESYNC_PAGES, opf: REQ_OP_READ,
4937	GFP_KERNEL, bs: &mddev->bio_set);
4938	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
4939	+ rdev->data_offset);
4940	read_bio->bi_private = r10_bio;
4941	read_bio->bi_end_io = end_reshape_read;
4942	r10_bio->master_bio = read_bio;
4943	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;
4944
4945	/*
4946	* Broadcast RESYNC message to other nodes, so all nodes would not
4947	* write to the region to avoid conflict.
4948	*/
4949	if (mddev_is_clustered(mddev) && conf->cluster_sync_high <= sector_nr) {
4950	struct mdp_superblock_1 *sb = NULL;
4951	int sb_reshape_pos = 0;
4952
4953	conf->cluster_sync_low = sector_nr;
4954	conf->cluster_sync_high = sector_nr + CLUSTER_RESYNC_WINDOW_SECTORS;
4955	sb = page_address(rdev->sb_page);
4956	if (sb) {
4957	sb_reshape_pos = le64_to_cpu(sb->reshape_position);
4958	/*
4959	* Set cluster_sync_low again if next address for array
4960	* reshape is less than cluster_sync_low. Since we can't
4961	* update cluster_sync_low until it has finished reshape.
4962	*/
4963	if (sb_reshape_pos < conf->cluster_sync_low)
4964	conf->cluster_sync_low = sb_reshape_pos;
4965	}
4966
4967	md_cluster_ops->resync_info_update(mddev, conf->cluster_sync_low,
4968	conf->cluster_sync_high);
4969	}
4970
4971	/* Now find the locations in the new layout */
4972	__raid10_find_phys(geo: &conf->geo, r10bio: r10_bio);
4973
4974	blist = read_bio;
4975	read_bio->bi_next = NULL;
4976
4977	rcu_read_lock();
4978	for (s = 0; s < conf->copies*2; s++) {
4979	struct bio *b;
4980	int d = r10_bio->devs[s/2].devnum;
4981	struct md_rdev *rdev2;
4982	if (s&1) {
4983	rdev2 = rcu_dereference(conf->mirrors[d].replacement);
4984	b = r10_bio->devs[s/2].repl_bio;
4985	} else {
4986	rdev2 = rcu_dereference(conf->mirrors[d].rdev);
4987	b = r10_bio->devs[s/2].bio;
4988	}
4989	if (!rdev2 || test_bit(Faulty, &rdev2->flags))
4990	continue;
4991
4992	bio_set_dev(bio: b, bdev: rdev2->bdev);
4993	b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
4994	rdev2->new_data_offset;
4995	b->bi_end_io = end_reshape_write;
4996	b->bi_opf = REQ_OP_WRITE;
4997	b->bi_next = blist;
4998	blist = b;
4999	}
5000
5001	/* Now add as many pages as possible to all of these bios. */
5002
5003	nr_sectors = 0;
5004	pages = get_resync_pages(bio: r10_bio->devs[0].bio)->pages;
5005	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
5006	struct page *page = pages[s / (PAGE_SIZE >> 9)];
5007	int len = (max_sectors - s) << 9;
5008	if (len > PAGE_SIZE)
5009	len = PAGE_SIZE;
5010	for (bio = blist; bio ; bio = bio->bi_next) {
5011	if (WARN_ON(!bio_add_page(bio, page, len, 0))) {
5012	bio->bi_status = BLK_STS_RESOURCE;
5013	bio_endio(bio);
5014	return sectors_done;
5015	}
5016	}
5017	sector_nr += len >> 9;
5018	nr_sectors += len >> 9;
5019	}
5020	rcu_read_unlock();
5021	r10_bio->sectors = nr_sectors;
5022
5023	/* Now submit the read */
5024	md_sync_acct_bio(bio: read_bio, nr_sectors: r10_bio->sectors);
5025	atomic_inc(v: &r10_bio->remaining);
5026	read_bio->bi_next = NULL;
5027	submit_bio_noacct(bio: read_bio);
5028	sectors_done += nr_sectors;
5029	if (sector_nr <= last)
5030	goto read_more;
5031
5032	lower_barrier(conf);
5033
5034	/* Now that we have done the whole section we can
5035	* update reshape_progress
5036	*/
5037	if (mddev->reshape_backwards)
5038	conf->reshape_progress -= sectors_done;
5039	else
5040	conf->reshape_progress += sectors_done;
5041
5042	return sectors_done;
5043	}
5044
5045	static void end_reshape_request(struct r10bio *r10_bio);
5046	static int handle_reshape_read_error(struct mddev *mddev,
5047	struct r10bio *r10_bio);
5048	static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
5049	{
5050	/* Reshape read completed. Hopefully we have a block
5051	* to write out.
5052	* If we got a read error then we do sync 1-page reads from
5053	* elsewhere until we find the data - or give up.
5054	*/
5055	struct r10conf *conf = mddev->private;
5056	int s;
5057
5058	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
5059	if (handle_reshape_read_error(mddev, r10_bio) < 0) {
5060	/* Reshape has been aborted */
5061	md_done_sync(mddev, blocks: r10_bio->sectors, ok: 0);
5062	return;
5063	}
5064
5065	/* We definitely have the data in the pages, schedule the
5066	* writes.
5067	*/
5068	atomic_set(v: &r10_bio->remaining, i: 1);
5069	for (s = 0; s < conf->copies*2; s++) {
5070	struct bio *b;
5071	int d = r10_bio->devs[s/2].devnum;
5072	struct md_rdev *rdev;
5073	rcu_read_lock();
5074	if (s&1) {
5075	rdev = rcu_dereference(conf->mirrors[d].replacement);
5076	b = r10_bio->devs[s/2].repl_bio;
5077	} else {
5078	rdev = rcu_dereference(conf->mirrors[d].rdev);
5079	b = r10_bio->devs[s/2].bio;
5080	}
5081	if (!rdev || test_bit(Faulty, &rdev->flags)) {
5082	rcu_read_unlock();
5083	continue;
5084	}
5085	atomic_inc(v: &rdev->nr_pending);
5086	rcu_read_unlock();
5087	md_sync_acct_bio(bio: b, nr_sectors: r10_bio->sectors);
5088	atomic_inc(v: &r10_bio->remaining);
5089	b->bi_next = NULL;
5090	submit_bio_noacct(bio: b);
5091	}
5092	end_reshape_request(r10_bio);
5093	}
5094
5095	static void end_reshape(struct r10conf *conf)
5096	{
5097	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
5098	return;
5099
5100	spin_lock_irq(lock: &conf->device_lock);
5101	conf->prev = conf->geo;
5102	md_finish_reshape(mddev: conf->mddev);
5103	smp_wmb();
5104	conf->reshape_progress = MaxSector;
5105	conf->reshape_safe = MaxSector;
5106	spin_unlock_irq(lock: &conf->device_lock);
5107
5108	if (conf->mddev->queue)
5109	raid10_set_io_opt(conf);
5110	conf->fullsync = 0;
5111	}
5112
5113	static void raid10_update_reshape_pos(struct mddev *mddev)
5114	{
5115	struct r10conf *conf = mddev->private;
5116	sector_t lo, hi;
5117
5118	md_cluster_ops->resync_info_get(mddev, &lo, &hi);
5119	if (((mddev->reshape_position <= hi) && (mddev->reshape_position >= lo))
5120	|| mddev->reshape_position == MaxSector)
5121	conf->reshape_progress = mddev->reshape_position;
5122	else
5123	WARN_ON_ONCE(1);
5124	}
5125
5126	static int handle_reshape_read_error(struct mddev *mddev,
5127	struct r10bio *r10_bio)
5128	{
5129	/* Use sync reads to get the blocks from somewhere else */
5130	int sectors = r10_bio->sectors;
5131	struct r10conf *conf = mddev->private;
5132	struct r10bio *r10b;
5133	int slot = 0;
5134	int idx = 0;
5135	struct page **pages;
5136
5137	r10b = kmalloc(struct_size(r10b, devs, conf->copies), GFP_NOIO);
5138	if (!r10b) {
5139	set_bit(nr: MD_RECOVERY_INTR, addr: &mddev->recovery);
5140	return -ENOMEM;
5141	}
5142
5143	/* reshape IOs share pages from .devs[0].bio */
5144	pages = get_resync_pages(bio: r10_bio->devs[0].bio)->pages;
5145
5146	r10b->sector = r10_bio->sector;
5147	__raid10_find_phys(geo: &conf->prev, r10bio: r10b);
5148
5149	while (sectors) {
5150	int s = sectors;
5151	int success = 0;
5152	int first_slot = slot;
5153
5154	if (s > (PAGE_SIZE >> 9))
5155	s = PAGE_SIZE >> 9;
5156
5157	rcu_read_lock();
5158	while (!success) {
5159	int d = r10b->devs[slot].devnum;
5160	struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
5161	sector_t addr;
5162	if (rdev == NULL ||
5163	test_bit(Faulty, &rdev->flags) ||
5164	!test_bit(In_sync, &rdev->flags))
5165	goto failed;
5166
5167	addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
5168	atomic_inc(v: &rdev->nr_pending);
5169	rcu_read_unlock();
5170	success = sync_page_io(rdev,
5171	sector: addr,
5172	size: s << 9,
5173	page: pages[idx],
5174	opf: REQ_OP_READ, metadata_op: false);
5175	rdev_dec_pending(rdev, mddev);
5176	rcu_read_lock();
5177	if (success)
5178	break;
5179	failed:
5180	slot++;
5181	if (slot >= conf->copies)
5182	slot = 0;
5183	if (slot == first_slot)
5184	break;
5185	}
5186	rcu_read_unlock();
5187	if (!success) {
5188	/* couldn't read this block, must give up */
5189	set_bit(nr: MD_RECOVERY_INTR,
5190	addr: &mddev->recovery);
5191	kfree(objp: r10b);
5192	return -EIO;
5193	}
5194	sectors -= s;
5195	idx++;
5196	}
5197	kfree(objp: r10b);
5198	return 0;
5199	}
5200
5201	static void end_reshape_write(struct bio *bio)
5202	{
5203	struct r10bio *r10_bio = get_resync_r10bio(bio);
5204	struct mddev *mddev = r10_bio->mddev;
5205	struct r10conf *conf = mddev->private;
5206	int d;
5207	int slot;
5208	int repl;
5209	struct md_rdev *rdev = NULL;
5210
5211	d = find_bio_disk(conf, r10_bio, bio, slotp: &slot, replp: &repl);
5212	if (repl)
5213	rdev = conf->mirrors[d].replacement;
5214	if (!rdev) {
5215	smp_mb();
5216	rdev = conf->mirrors[d].rdev;
5217	}
5218
5219	if (bio->bi_status) {
5220	/* FIXME should record badblock */
5221	md_error(mddev, rdev);
5222	}
5223
5224	rdev_dec_pending(rdev, mddev);
5225	end_reshape_request(r10_bio);
5226	}
5227
5228	static void end_reshape_request(struct r10bio *r10_bio)
5229	{
5230	if (!atomic_dec_and_test(v: &r10_bio->remaining))
5231	return;
5232	md_done_sync(mddev: r10_bio->mddev, blocks: r10_bio->sectors, ok: 1);
5233	bio_put(r10_bio->master_bio);
5234	put_buf(r10_bio);
5235	}
5236
5237	static void raid10_finish_reshape(struct mddev *mddev)
5238	{
5239	struct r10conf *conf = mddev->private;
5240
5241	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5242	return;
5243
5244	if (mddev->delta_disks > 0) {
5245	if (mddev->recovery_cp > mddev->resync_max_sectors) {
5246	mddev->recovery_cp = mddev->resync_max_sectors;
5247	set_bit(nr: MD_RECOVERY_NEEDED, addr: &mddev->recovery);
5248	}
5249	mddev->resync_max_sectors = mddev->array_sectors;
5250	} else {
5251	int d;
5252	rcu_read_lock();
5253	for (d = conf->geo.raid_disks ;
5254	d < conf->geo.raid_disks - mddev->delta_disks;
5255	d++) {
5256	struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
5257	if (rdev)
5258	clear_bit(nr: In_sync, addr: &rdev->flags);
5259	rdev = rcu_dereference(conf->mirrors[d].replacement);
5260	if (rdev)
5261	clear_bit(nr: In_sync, addr: &rdev->flags);
5262	}
5263	rcu_read_unlock();
5264	}
5265	mddev->layout = mddev->new_layout;
5266	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
5267	mddev->reshape_position = MaxSector;
5268	mddev->delta_disks = 0;
5269	mddev->reshape_backwards = 0;
5270	}
5271
5272	static struct md_personality raid10_personality =
5273	{
5274	.name = "raid10",
5275	.level = 10,
5276	.owner = THIS_MODULE,
5277	.make_request = raid10_make_request,
5278	.run = raid10_run,
5279	.free = raid10_free,
5280	.status = raid10_status,
5281	.error_handler = raid10_error,
5282	.hot_add_disk = raid10_add_disk,
5283	.hot_remove_disk= raid10_remove_disk,
5284	.spare_active = raid10_spare_active,
5285	.sync_request = raid10_sync_request,
5286	.quiesce = raid10_quiesce,
5287	.size = raid10_size,
5288	.resize = raid10_resize,
5289	.takeover = raid10_takeover,
5290	.check_reshape = raid10_check_reshape,
5291	.start_reshape = raid10_start_reshape,
5292	.finish_reshape = raid10_finish_reshape,
5293	.update_reshape_pos = raid10_update_reshape_pos,
5294	};
5295
5296	static int __init raid_init(void)
5297	{
5298	return register_md_personality(p: &raid10_personality);
5299	}
5300
5301	static void raid_exit(void)
5302	{
5303	unregister_md_personality(p: &raid10_personality);
5304	}
5305
5306	module_init(raid_init);
5307	module_exit(raid_exit);
5308	MODULE_LICENSE("GPL");
5309	MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
5310	MODULE_ALIAS("md-personality-9"); /* RAID10 */
5311	MODULE_ALIAS("md-raid10");
5312	MODULE_ALIAS("md-level-10");
5313