1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * multipath.c : Multiple Devices driver for Linux |
4 | * |
5 | * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat |
6 | * |
7 | * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman |
8 | * |
9 | * MULTIPATH management functions. |
10 | * |
11 | * derived from raid1.c. |
12 | */ |
13 | |
14 | #include <linux/blkdev.h> |
15 | #include <linux/module.h> |
16 | #include <linux/raid/md_u.h> |
17 | #include <linux/seq_file.h> |
18 | #include <linux/slab.h> |
19 | #include "md.h" |
20 | #include "md-multipath.h" |
21 | |
22 | #define MAX_WORK_PER_DISK 128 |
23 | |
24 | #define NR_RESERVED_BUFS 32 |
25 | |
26 | static int multipath_map (struct mpconf *conf) |
27 | { |
28 | int i, disks = conf->raid_disks; |
29 | |
30 | /* |
31 | * Later we do read balancing on the read side |
32 | * now we use the first available disk. |
33 | */ |
34 | |
35 | rcu_read_lock(); |
36 | for (i = 0; i < disks; i++) { |
37 | struct md_rdev *rdev = rcu_dereference(conf->multipaths[i].rdev); |
38 | if (rdev && test_bit(In_sync, &rdev->flags) && |
39 | !test_bit(Faulty, &rdev->flags)) { |
40 | atomic_inc(v: &rdev->nr_pending); |
41 | rcu_read_unlock(); |
42 | return i; |
43 | } |
44 | } |
45 | rcu_read_unlock(); |
46 | |
47 | pr_crit_ratelimited("multipath_map(): no more operational IO paths?\n" ); |
48 | return (-1); |
49 | } |
50 | |
51 | static void multipath_reschedule_retry (struct multipath_bh *mp_bh) |
52 | { |
53 | unsigned long flags; |
54 | struct mddev *mddev = mp_bh->mddev; |
55 | struct mpconf *conf = mddev->private; |
56 | |
57 | spin_lock_irqsave(&conf->device_lock, flags); |
58 | list_add(new: &mp_bh->retry_list, head: &conf->retry_list); |
59 | spin_unlock_irqrestore(lock: &conf->device_lock, flags); |
60 | md_wakeup_thread(thread: mddev->thread); |
61 | } |
62 | |
63 | /* |
64 | * multipath_end_bh_io() is called when we have finished servicing a multipathed |
65 | * operation and are ready to return a success/failure code to the buffer |
66 | * cache layer. |
67 | */ |
68 | static void multipath_end_bh_io(struct multipath_bh *mp_bh, blk_status_t status) |
69 | { |
70 | struct bio *bio = mp_bh->master_bio; |
71 | struct mpconf *conf = mp_bh->mddev->private; |
72 | |
73 | bio->bi_status = status; |
74 | bio_endio(bio); |
75 | mempool_free(element: mp_bh, pool: &conf->pool); |
76 | } |
77 | |
78 | static void multipath_end_request(struct bio *bio) |
79 | { |
80 | struct multipath_bh *mp_bh = bio->bi_private; |
81 | struct mpconf *conf = mp_bh->mddev->private; |
82 | struct md_rdev *rdev = conf->multipaths[mp_bh->path].rdev; |
83 | |
84 | if (!bio->bi_status) |
85 | multipath_end_bh_io(mp_bh, status: 0); |
86 | else if (!(bio->bi_opf & REQ_RAHEAD)) { |
87 | /* |
88 | * oops, IO error: |
89 | */ |
90 | md_error (mddev: mp_bh->mddev, rdev); |
91 | pr_info("multipath: %pg: rescheduling sector %llu\n" , |
92 | rdev->bdev, |
93 | (unsigned long long)bio->bi_iter.bi_sector); |
94 | multipath_reschedule_retry(mp_bh); |
95 | } else |
96 | multipath_end_bh_io(mp_bh, status: bio->bi_status); |
97 | rdev_dec_pending(rdev, mddev: conf->mddev); |
98 | } |
99 | |
100 | static bool multipath_make_request(struct mddev *mddev, struct bio * bio) |
101 | { |
102 | struct mpconf *conf = mddev->private; |
103 | struct multipath_bh * mp_bh; |
104 | struct multipath_info *multipath; |
105 | |
106 | if (unlikely(bio->bi_opf & REQ_PREFLUSH) |
107 | && md_flush_request(mddev, bio)) |
108 | return true; |
109 | |
110 | md_account_bio(mddev, bio: &bio); |
111 | mp_bh = mempool_alloc(pool: &conf->pool, GFP_NOIO); |
112 | |
113 | mp_bh->master_bio = bio; |
114 | mp_bh->mddev = mddev; |
115 | |
116 | mp_bh->path = multipath_map(conf); |
117 | if (mp_bh->path < 0) { |
118 | bio_io_error(bio); |
119 | mempool_free(element: mp_bh, pool: &conf->pool); |
120 | return true; |
121 | } |
122 | multipath = conf->multipaths + mp_bh->path; |
123 | |
124 | bio_init_clone(bdev: multipath->rdev->bdev, bio: &mp_bh->bio, bio_src: bio, GFP_NOIO); |
125 | |
126 | mp_bh->bio.bi_iter.bi_sector += multipath->rdev->data_offset; |
127 | mp_bh->bio.bi_opf |= REQ_FAILFAST_TRANSPORT; |
128 | mp_bh->bio.bi_end_io = multipath_end_request; |
129 | mp_bh->bio.bi_private = mp_bh; |
130 | mddev_check_write_zeroes(mddev, bio: &mp_bh->bio); |
131 | submit_bio_noacct(bio: &mp_bh->bio); |
132 | return true; |
133 | } |
134 | |
135 | static void multipath_status(struct seq_file *seq, struct mddev *mddev) |
136 | { |
137 | struct mpconf *conf = mddev->private; |
138 | int i; |
139 | |
140 | seq_printf (m: seq, fmt: " [%d/%d] [" , conf->raid_disks, |
141 | conf->raid_disks - mddev->degraded); |
142 | rcu_read_lock(); |
143 | for (i = 0; i < conf->raid_disks; i++) { |
144 | struct md_rdev *rdev = rcu_dereference(conf->multipaths[i].rdev); |
145 | seq_printf (m: seq, fmt: "%s" , rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_" ); |
146 | } |
147 | rcu_read_unlock(); |
148 | seq_putc(m: seq, c: ']'); |
149 | } |
150 | |
151 | /* |
152 | * Careful, this can execute in IRQ contexts as well! |
153 | */ |
154 | static void multipath_error (struct mddev *mddev, struct md_rdev *rdev) |
155 | { |
156 | struct mpconf *conf = mddev->private; |
157 | |
158 | if (conf->raid_disks - mddev->degraded <= 1) { |
159 | /* |
160 | * Uh oh, we can do nothing if this is our last path, but |
161 | * first check if this is a queued request for a device |
162 | * which has just failed. |
163 | */ |
164 | pr_warn("multipath: only one IO path left and IO error.\n" ); |
165 | /* leave it active... it's all we have */ |
166 | return; |
167 | } |
168 | /* |
169 | * Mark disk as unusable |
170 | */ |
171 | if (test_and_clear_bit(nr: In_sync, addr: &rdev->flags)) { |
172 | unsigned long flags; |
173 | spin_lock_irqsave(&conf->device_lock, flags); |
174 | mddev->degraded++; |
175 | spin_unlock_irqrestore(lock: &conf->device_lock, flags); |
176 | } |
177 | set_bit(nr: Faulty, addr: &rdev->flags); |
178 | set_bit(nr: MD_SB_CHANGE_DEVS, addr: &mddev->sb_flags); |
179 | pr_err("multipath: IO failure on %pg, disabling IO path.\n" |
180 | "multipath: Operation continuing on %d IO paths.\n" , |
181 | rdev->bdev, |
182 | conf->raid_disks - mddev->degraded); |
183 | } |
184 | |
185 | static void print_multipath_conf (struct mpconf *conf) |
186 | { |
187 | int i; |
188 | struct multipath_info *tmp; |
189 | |
190 | pr_debug("MULTIPATH conf printout:\n" ); |
191 | if (!conf) { |
192 | pr_debug("(conf==NULL)\n" ); |
193 | return; |
194 | } |
195 | pr_debug(" --- wd:%d rd:%d\n" , conf->raid_disks - conf->mddev->degraded, |
196 | conf->raid_disks); |
197 | |
198 | for (i = 0; i < conf->raid_disks; i++) { |
199 | tmp = conf->multipaths + i; |
200 | if (tmp->rdev) |
201 | pr_debug(" disk%d, o:%d, dev:%pg\n" , |
202 | i,!test_bit(Faulty, &tmp->rdev->flags), |
203 | tmp->rdev->bdev); |
204 | } |
205 | } |
206 | |
207 | static int multipath_add_disk(struct mddev *mddev, struct md_rdev *rdev) |
208 | { |
209 | struct mpconf *conf = mddev->private; |
210 | int err = -EEXIST; |
211 | int path; |
212 | struct multipath_info *p; |
213 | int first = 0; |
214 | int last = mddev->raid_disks - 1; |
215 | |
216 | if (rdev->raid_disk >= 0) |
217 | first = last = rdev->raid_disk; |
218 | |
219 | print_multipath_conf(conf); |
220 | |
221 | for (path = first; path <= last; path++) |
222 | if ((p=conf->multipaths+path)->rdev == NULL) { |
223 | disk_stack_limits(disk: mddev->gendisk, bdev: rdev->bdev, |
224 | offset: rdev->data_offset << 9); |
225 | |
226 | err = md_integrity_add_rdev(rdev, mddev); |
227 | if (err) |
228 | break; |
229 | spin_lock_irq(lock: &conf->device_lock); |
230 | mddev->degraded--; |
231 | rdev->raid_disk = path; |
232 | set_bit(nr: In_sync, addr: &rdev->flags); |
233 | spin_unlock_irq(lock: &conf->device_lock); |
234 | rcu_assign_pointer(p->rdev, rdev); |
235 | err = 0; |
236 | break; |
237 | } |
238 | |
239 | print_multipath_conf(conf); |
240 | |
241 | return err; |
242 | } |
243 | |
244 | static int multipath_remove_disk(struct mddev *mddev, struct md_rdev *rdev) |
245 | { |
246 | struct mpconf *conf = mddev->private; |
247 | int err = 0; |
248 | int number = rdev->raid_disk; |
249 | struct multipath_info *p = conf->multipaths + number; |
250 | |
251 | print_multipath_conf(conf); |
252 | |
253 | if (rdev == p->rdev) { |
254 | if (test_bit(In_sync, &rdev->flags) || |
255 | atomic_read(v: &rdev->nr_pending)) { |
256 | pr_warn("hot-remove-disk, slot %d is identified but is still operational!\n" , number); |
257 | err = -EBUSY; |
258 | goto abort; |
259 | } |
260 | p->rdev = NULL; |
261 | if (!test_bit(RemoveSynchronized, &rdev->flags)) { |
262 | synchronize_rcu(); |
263 | if (atomic_read(v: &rdev->nr_pending)) { |
264 | /* lost the race, try later */ |
265 | err = -EBUSY; |
266 | p->rdev = rdev; |
267 | goto abort; |
268 | } |
269 | } |
270 | err = md_integrity_register(mddev); |
271 | } |
272 | abort: |
273 | |
274 | print_multipath_conf(conf); |
275 | return err; |
276 | } |
277 | |
278 | /* |
279 | * This is a kernel thread which: |
280 | * |
281 | * 1. Retries failed read operations on working multipaths. |
282 | * 2. Updates the raid superblock when problems encounter. |
283 | * 3. Performs writes following reads for array syncronising. |
284 | */ |
285 | |
286 | static void multipathd(struct md_thread *thread) |
287 | { |
288 | struct mddev *mddev = thread->mddev; |
289 | struct multipath_bh *mp_bh; |
290 | struct bio *bio; |
291 | unsigned long flags; |
292 | struct mpconf *conf = mddev->private; |
293 | struct list_head *head = &conf->retry_list; |
294 | |
295 | md_check_recovery(mddev); |
296 | for (;;) { |
297 | spin_lock_irqsave(&conf->device_lock, flags); |
298 | if (list_empty(head)) |
299 | break; |
300 | mp_bh = list_entry(head->prev, struct multipath_bh, retry_list); |
301 | list_del(entry: head->prev); |
302 | spin_unlock_irqrestore(lock: &conf->device_lock, flags); |
303 | |
304 | bio = &mp_bh->bio; |
305 | bio->bi_iter.bi_sector = mp_bh->master_bio->bi_iter.bi_sector; |
306 | |
307 | if ((mp_bh->path = multipath_map (conf))<0) { |
308 | pr_err("multipath: %pg: unrecoverable IO read error for block %llu\n" , |
309 | bio->bi_bdev, |
310 | (unsigned long long)bio->bi_iter.bi_sector); |
311 | multipath_end_bh_io(mp_bh, BLK_STS_IOERR); |
312 | } else { |
313 | pr_err("multipath: %pg: redirecting sector %llu to another IO path\n" , |
314 | bio->bi_bdev, |
315 | (unsigned long long)bio->bi_iter.bi_sector); |
316 | *bio = *(mp_bh->master_bio); |
317 | bio->bi_iter.bi_sector += |
318 | conf->multipaths[mp_bh->path].rdev->data_offset; |
319 | bio_set_dev(bio, bdev: conf->multipaths[mp_bh->path].rdev->bdev); |
320 | bio->bi_opf |= REQ_FAILFAST_TRANSPORT; |
321 | bio->bi_end_io = multipath_end_request; |
322 | bio->bi_private = mp_bh; |
323 | submit_bio_noacct(bio); |
324 | } |
325 | } |
326 | spin_unlock_irqrestore(lock: &conf->device_lock, flags); |
327 | } |
328 | |
329 | static sector_t multipath_size(struct mddev *mddev, sector_t sectors, int raid_disks) |
330 | { |
331 | WARN_ONCE(sectors || raid_disks, |
332 | "%s does not support generic reshape\n" , __func__); |
333 | |
334 | return mddev->dev_sectors; |
335 | } |
336 | |
337 | static int multipath_run (struct mddev *mddev) |
338 | { |
339 | struct mpconf *conf; |
340 | int disk_idx; |
341 | struct multipath_info *disk; |
342 | struct md_rdev *rdev; |
343 | int working_disks; |
344 | int ret; |
345 | |
346 | if (md_check_no_bitmap(mddev)) |
347 | return -EINVAL; |
348 | |
349 | if (mddev->level != LEVEL_MULTIPATH) { |
350 | pr_warn("multipath: %s: raid level not set to multipath IO (%d)\n" , |
351 | mdname(mddev), mddev->level); |
352 | goto out; |
353 | } |
354 | /* |
355 | * copy the already verified devices into our private MULTIPATH |
356 | * bookkeeping area. [whatever we allocate in multipath_run(), |
357 | * should be freed in multipath_free()] |
358 | */ |
359 | |
360 | conf = kzalloc(size: sizeof(struct mpconf), GFP_KERNEL); |
361 | mddev->private = conf; |
362 | if (!conf) |
363 | goto out; |
364 | |
365 | conf->multipaths = kcalloc(n: mddev->raid_disks, |
366 | size: sizeof(struct multipath_info), |
367 | GFP_KERNEL); |
368 | if (!conf->multipaths) |
369 | goto out_free_conf; |
370 | |
371 | working_disks = 0; |
372 | rdev_for_each(rdev, mddev) { |
373 | disk_idx = rdev->raid_disk; |
374 | if (disk_idx < 0 || |
375 | disk_idx >= mddev->raid_disks) |
376 | continue; |
377 | |
378 | disk = conf->multipaths + disk_idx; |
379 | disk->rdev = rdev; |
380 | disk_stack_limits(disk: mddev->gendisk, bdev: rdev->bdev, |
381 | offset: rdev->data_offset << 9); |
382 | |
383 | if (!test_bit(Faulty, &rdev->flags)) |
384 | working_disks++; |
385 | } |
386 | |
387 | conf->raid_disks = mddev->raid_disks; |
388 | conf->mddev = mddev; |
389 | spin_lock_init(&conf->device_lock); |
390 | INIT_LIST_HEAD(list: &conf->retry_list); |
391 | |
392 | if (!working_disks) { |
393 | pr_warn("multipath: no operational IO paths for %s\n" , |
394 | mdname(mddev)); |
395 | goto out_free_conf; |
396 | } |
397 | mddev->degraded = conf->raid_disks - working_disks; |
398 | |
399 | ret = mempool_init_kmalloc_pool(pool: &conf->pool, NR_RESERVED_BUFS, |
400 | size: sizeof(struct multipath_bh)); |
401 | if (ret) |
402 | goto out_free_conf; |
403 | |
404 | rcu_assign_pointer(mddev->thread, |
405 | md_register_thread(multipathd, mddev, "multipath" )); |
406 | if (!mddev->thread) |
407 | goto out_free_conf; |
408 | |
409 | pr_info("multipath: array %s active with %d out of %d IO paths\n" , |
410 | mdname(mddev), conf->raid_disks - mddev->degraded, |
411 | mddev->raid_disks); |
412 | /* |
413 | * Ok, everything is just fine now |
414 | */ |
415 | md_set_array_sectors(mddev, array_sectors: multipath_size(mddev, sectors: 0, raid_disks: 0)); |
416 | |
417 | if (md_integrity_register(mddev)) |
418 | goto out_free_conf; |
419 | |
420 | return 0; |
421 | |
422 | out_free_conf: |
423 | mempool_exit(pool: &conf->pool); |
424 | kfree(objp: conf->multipaths); |
425 | kfree(objp: conf); |
426 | mddev->private = NULL; |
427 | out: |
428 | return -EIO; |
429 | } |
430 | |
431 | static void multipath_free(struct mddev *mddev, void *priv) |
432 | { |
433 | struct mpconf *conf = priv; |
434 | |
435 | mempool_exit(pool: &conf->pool); |
436 | kfree(objp: conf->multipaths); |
437 | kfree(objp: conf); |
438 | } |
439 | |
440 | static struct md_personality multipath_personality = |
441 | { |
442 | .name = "multipath" , |
443 | .level = LEVEL_MULTIPATH, |
444 | .owner = THIS_MODULE, |
445 | .make_request = multipath_make_request, |
446 | .run = multipath_run, |
447 | .free = multipath_free, |
448 | .status = multipath_status, |
449 | .error_handler = multipath_error, |
450 | .hot_add_disk = multipath_add_disk, |
451 | .hot_remove_disk= multipath_remove_disk, |
452 | .size = multipath_size, |
453 | }; |
454 | |
455 | static int __init multipath_init (void) |
456 | { |
457 | return register_md_personality (p: &multipath_personality); |
458 | } |
459 | |
460 | static void __exit multipath_exit (void) |
461 | { |
462 | unregister_md_personality (p: &multipath_personality); |
463 | } |
464 | |
465 | module_init(multipath_init); |
466 | module_exit(multipath_exit); |
467 | MODULE_LICENSE("GPL" ); |
468 | MODULE_DESCRIPTION("simple multi-path personality for MD (deprecated)" ); |
469 | MODULE_ALIAS("md-personality-7" ); /* MULTIPATH */ |
470 | MODULE_ALIAS("md-multipath" ); |
471 | MODULE_ALIAS("md-level--4" ); |
472 | |