1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
4 * Shaohua Li <shli@fb.com>
5 */
6#include <linux/module.h>
7
8#include <linux/moduleparam.h>
9#include <linux/sched.h>
10#include <linux/fs.h>
11#include <linux/init.h>
12#include "null_blk.h"
13
14#undef pr_fmt
15#define pr_fmt(fmt) "null_blk: " fmt
16
17#define FREE_BATCH 16
18
19#define TICKS_PER_SEC 50ULL
20#define TIMER_INTERVAL (NSEC_PER_SEC / TICKS_PER_SEC)
21
22#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
23static DECLARE_FAULT_ATTR(null_timeout_attr);
24static DECLARE_FAULT_ATTR(null_requeue_attr);
25static DECLARE_FAULT_ATTR(null_init_hctx_attr);
26#endif
27
28static inline u64 mb_per_tick(int mbps)
29{
30 return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
31}
32
33/*
34 * Status flags for nullb_device.
35 *
36 * CONFIGURED: Device has been configured and turned on. Cannot reconfigure.
37 * UP: Device is currently on and visible in userspace.
38 * THROTTLED: Device is being throttled.
39 * CACHE: Device is using a write-back cache.
40 */
41enum nullb_device_flags {
42 NULLB_DEV_FL_CONFIGURED = 0,
43 NULLB_DEV_FL_UP = 1,
44 NULLB_DEV_FL_THROTTLED = 2,
45 NULLB_DEV_FL_CACHE = 3,
46};
47
48#define MAP_SZ ((PAGE_SIZE >> SECTOR_SHIFT) + 2)
49/*
50 * nullb_page is a page in memory for nullb devices.
51 *
52 * @page: The page holding the data.
53 * @bitmap: The bitmap represents which sector in the page has data.
54 * Each bit represents one block size. For example, sector 8
55 * will use the 7th bit
56 * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
57 * page is being flushing to storage. FREE means the cache page is freed and
58 * should be skipped from flushing to storage. Please see
59 * null_make_cache_space
60 */
61struct nullb_page {
62 struct page *page;
63 DECLARE_BITMAP(bitmap, MAP_SZ);
64};
65#define NULLB_PAGE_LOCK (MAP_SZ - 1)
66#define NULLB_PAGE_FREE (MAP_SZ - 2)
67
68static LIST_HEAD(nullb_list);
69static struct mutex lock;
70static int null_major;
71static DEFINE_IDA(nullb_indexes);
72static struct blk_mq_tag_set tag_set;
73
74enum {
75 NULL_IRQ_NONE = 0,
76 NULL_IRQ_SOFTIRQ = 1,
77 NULL_IRQ_TIMER = 2,
78};
79
80static bool g_virt_boundary;
81module_param_named(virt_boundary, g_virt_boundary, bool, 0444);
82MODULE_PARM_DESC(virt_boundary, "Require a virtual boundary for the device. Default: False");
83
84static int g_no_sched;
85module_param_named(no_sched, g_no_sched, int, 0444);
86MODULE_PARM_DESC(no_sched, "No io scheduler");
87
88static int g_submit_queues = 1;
89module_param_named(submit_queues, g_submit_queues, int, 0444);
90MODULE_PARM_DESC(submit_queues, "Number of submission queues");
91
92static int g_poll_queues = 1;
93module_param_named(poll_queues, g_poll_queues, int, 0444);
94MODULE_PARM_DESC(poll_queues, "Number of IOPOLL submission queues");
95
96static int g_home_node = NUMA_NO_NODE;
97module_param_named(home_node, g_home_node, int, 0444);
98MODULE_PARM_DESC(home_node, "Home node for the device");
99
100#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
101/*
102 * For more details about fault injection, please refer to
103 * Documentation/fault-injection/fault-injection.rst.
104 */
105static char g_timeout_str[80];
106module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
107MODULE_PARM_DESC(timeout, "Fault injection. timeout=<interval>,<probability>,<space>,<times>");
108
109static char g_requeue_str[80];
110module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
111MODULE_PARM_DESC(requeue, "Fault injection. requeue=<interval>,<probability>,<space>,<times>");
112
113static char g_init_hctx_str[80];
114module_param_string(init_hctx, g_init_hctx_str, sizeof(g_init_hctx_str), 0444);
115MODULE_PARM_DESC(init_hctx, "Fault injection to fail hctx init. init_hctx=<interval>,<probability>,<space>,<times>");
116#endif
117
118/*
119 * Historic queue modes.
120 *
121 * These days nothing but NULL_Q_MQ is actually supported, but we keep it the
122 * enum for error reporting.
123 */
124enum {
125 NULL_Q_BIO = 0,
126 NULL_Q_RQ = 1,
127 NULL_Q_MQ = 2,
128};
129
130static int g_queue_mode = NULL_Q_MQ;
131
132static int null_param_store_val(const char *str, int *val, int min, int max)
133{
134 int ret, new_val;
135
136 ret = kstrtoint(s: str, base: 10, res: &new_val);
137 if (ret)
138 return -EINVAL;
139
140 if (new_val < min || new_val > max)
141 return -EINVAL;
142
143 *val = new_val;
144 return 0;
145}
146
147static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
148{
149 return null_param_store_val(str, val: &g_queue_mode, min: NULL_Q_BIO, max: NULL_Q_MQ);
150}
151
152static const struct kernel_param_ops null_queue_mode_param_ops = {
153 .set = null_set_queue_mode,
154 .get = param_get_int,
155};
156
157device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
158MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
159
160static int g_gb = 250;
161module_param_named(gb, g_gb, int, 0444);
162MODULE_PARM_DESC(gb, "Size in GB");
163
164static int g_bs = 512;
165module_param_named(bs, g_bs, int, 0444);
166MODULE_PARM_DESC(bs, "Block size (in bytes)");
167
168static int g_max_sectors;
169module_param_named(max_sectors, g_max_sectors, int, 0444);
170MODULE_PARM_DESC(max_sectors, "Maximum size of a command (in 512B sectors)");
171
172static unsigned int nr_devices = 1;
173module_param(nr_devices, uint, 0444);
174MODULE_PARM_DESC(nr_devices, "Number of devices to register");
175
176static bool g_blocking;
177module_param_named(blocking, g_blocking, bool, 0444);
178MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
179
180static bool g_shared_tags;
181module_param_named(shared_tags, g_shared_tags, bool, 0444);
182MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
183
184static bool g_shared_tag_bitmap;
185module_param_named(shared_tag_bitmap, g_shared_tag_bitmap, bool, 0444);
186MODULE_PARM_DESC(shared_tag_bitmap, "Use shared tag bitmap for all submission queues for blk-mq");
187
188static int g_irqmode = NULL_IRQ_SOFTIRQ;
189
190static int null_set_irqmode(const char *str, const struct kernel_param *kp)
191{
192 return null_param_store_val(str, val: &g_irqmode, min: NULL_IRQ_NONE,
193 max: NULL_IRQ_TIMER);
194}
195
196static const struct kernel_param_ops null_irqmode_param_ops = {
197 .set = null_set_irqmode,
198 .get = param_get_int,
199};
200
201device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
202MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
203
204static unsigned long g_completion_nsec = 10000;
205module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
206MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
207
208static int g_hw_queue_depth = 64;
209module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
210MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
211
212static bool g_use_per_node_hctx;
213module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
214MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
215
216static bool g_memory_backed;
217module_param_named(memory_backed, g_memory_backed, bool, 0444);
218MODULE_PARM_DESC(memory_backed, "Create a memory-backed block device. Default: false");
219
220static bool g_discard;
221module_param_named(discard, g_discard, bool, 0444);
222MODULE_PARM_DESC(discard, "Support discard operations (requires memory-backed null_blk device). Default: false");
223
224static unsigned long g_cache_size;
225module_param_named(cache_size, g_cache_size, ulong, 0444);
226MODULE_PARM_DESC(cache_size, "Cache size in MiB for memory-backed device. Default: 0 (none)");
227
228static bool g_fua = true;
229module_param_named(fua, g_fua, bool, 0444);
230MODULE_PARM_DESC(fua, "Enable/disable FUA support when cache_size is used. Default: true");
231
232static unsigned int g_mbps;
233module_param_named(mbps, g_mbps, uint, 0444);
234MODULE_PARM_DESC(mbps, "Limit maximum bandwidth (in MiB/s). Default: 0 (no limit)");
235
236static bool g_zoned;
237module_param_named(zoned, g_zoned, bool, S_IRUGO);
238MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
239
240static unsigned long g_zone_size = 256;
241module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
242MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
243
244static unsigned long g_zone_capacity;
245module_param_named(zone_capacity, g_zone_capacity, ulong, 0444);
246MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size");
247
248static unsigned int g_zone_nr_conv;
249module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444);
250MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0");
251
252static unsigned int g_zone_max_open;
253module_param_named(zone_max_open, g_zone_max_open, uint, 0444);
254MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones when block device is zoned. Default: 0 (no limit)");
255
256static unsigned int g_zone_max_active;
257module_param_named(zone_max_active, g_zone_max_active, uint, 0444);
258MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)");
259
260static int g_zone_append_max_sectors = INT_MAX;
261module_param_named(zone_append_max_sectors, g_zone_append_max_sectors, int, 0444);
262MODULE_PARM_DESC(zone_append_max_sectors,
263 "Maximum size of a zone append command (in 512B sectors). Specify 0 for zone append emulation");
264
265static bool g_zone_full;
266module_param_named(zone_full, g_zone_full, bool, S_IRUGO);
267MODULE_PARM_DESC(zone_full, "Initialize the sequential write required zones of a zoned device to be full. Default: false");
268
269static bool g_rotational;
270module_param_named(rotational, g_rotational, bool, S_IRUGO);
271MODULE_PARM_DESC(rotational, "Set the rotational feature for the device. Default: false");
272
273static struct nullb_device *null_alloc_dev(void);
274static void null_free_dev(struct nullb_device *dev);
275static void null_del_dev(struct nullb *nullb);
276static int null_add_dev(struct nullb_device *dev);
277static struct nullb *null_find_dev_by_name(const char *name);
278static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
279
280static inline struct nullb_device *to_nullb_device(struct config_item *item)
281{
282 return item ? container_of(to_config_group(item), struct nullb_device, group) : NULL;
283}
284
285static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
286{
287 return snprintf(buf: page, PAGE_SIZE, fmt: "%u\n", val);
288}
289
290static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
291 char *page)
292{
293 return snprintf(buf: page, PAGE_SIZE, fmt: "%lu\n", val);
294}
295
296static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
297{
298 return snprintf(buf: page, PAGE_SIZE, fmt: "%u\n", val);
299}
300
301static ssize_t nullb_device_uint_attr_store(unsigned int *val,
302 const char *page, size_t count)
303{
304 unsigned int tmp;
305 int result;
306
307 result = kstrtouint(s: page, base: 0, res: &tmp);
308 if (result < 0)
309 return result;
310
311 *val = tmp;
312 return count;
313}
314
315static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
316 const char *page, size_t count)
317{
318 int result;
319 unsigned long tmp;
320
321 result = kstrtoul(s: page, base: 0, res: &tmp);
322 if (result < 0)
323 return result;
324
325 *val = tmp;
326 return count;
327}
328
329static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
330 size_t count)
331{
332 bool tmp;
333 int result;
334
335 result = kstrtobool(s: page, res: &tmp);
336 if (result < 0)
337 return result;
338
339 *val = tmp;
340 return count;
341}
342
343/* The following macro should only be used with TYPE = {uint, ulong, bool}. */
344#define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \
345static ssize_t \
346nullb_device_##NAME##_show(struct config_item *item, char *page) \
347{ \
348 return nullb_device_##TYPE##_attr_show( \
349 to_nullb_device(item)->NAME, page); \
350} \
351static ssize_t \
352nullb_device_##NAME##_store(struct config_item *item, const char *page, \
353 size_t count) \
354{ \
355 int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\
356 struct nullb_device *dev = to_nullb_device(item); \
357 TYPE new_value = 0; \
358 int ret; \
359 \
360 ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\
361 if (ret < 0) \
362 return ret; \
363 if (apply_fn) \
364 ret = apply_fn(dev, new_value); \
365 else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \
366 ret = -EBUSY; \
367 if (ret < 0) \
368 return ret; \
369 dev->NAME = new_value; \
370 return count; \
371} \
372CONFIGFS_ATTR(nullb_device_, NAME);
373
374static int nullb_update_nr_hw_queues(struct nullb_device *dev,
375 unsigned int submit_queues,
376 unsigned int poll_queues)
377
378{
379 struct blk_mq_tag_set *set;
380 int ret, nr_hw_queues;
381
382 if (!dev->nullb)
383 return 0;
384
385 /*
386 * Make sure at least one submit queue exists.
387 */
388 if (!submit_queues)
389 return -EINVAL;
390
391 /*
392 * Make sure that null_init_hctx() does not access nullb->queues[] past
393 * the end of that array.
394 */
395 if (submit_queues > nr_cpu_ids || poll_queues > g_poll_queues)
396 return -EINVAL;
397
398 /*
399 * Keep previous and new queue numbers in nullb_device for reference in
400 * the call back function null_map_queues().
401 */
402 dev->prev_submit_queues = dev->submit_queues;
403 dev->prev_poll_queues = dev->poll_queues;
404 dev->submit_queues = submit_queues;
405 dev->poll_queues = poll_queues;
406
407 set = dev->nullb->tag_set;
408 nr_hw_queues = submit_queues + poll_queues;
409 blk_mq_update_nr_hw_queues(set, nr_hw_queues);
410 ret = set->nr_hw_queues == nr_hw_queues ? 0 : -ENOMEM;
411
412 if (ret) {
413 /* on error, revert the queue numbers */
414 dev->submit_queues = dev->prev_submit_queues;
415 dev->poll_queues = dev->prev_poll_queues;
416 }
417
418 return ret;
419}
420
421static int nullb_apply_submit_queues(struct nullb_device *dev,
422 unsigned int submit_queues)
423{
424 int ret;
425
426 mutex_lock(&lock);
427 ret = nullb_update_nr_hw_queues(dev, submit_queues, poll_queues: dev->poll_queues);
428 mutex_unlock(lock: &lock);
429
430 return ret;
431}
432
433static int nullb_apply_poll_queues(struct nullb_device *dev,
434 unsigned int poll_queues)
435{
436 int ret;
437
438 mutex_lock(&lock);
439 ret = nullb_update_nr_hw_queues(dev, submit_queues: dev->submit_queues, poll_queues);
440 mutex_unlock(lock: &lock);
441
442 return ret;
443}
444
445NULLB_DEVICE_ATTR(size, ulong, NULL);
446NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL);
447NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues);
448NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_poll_queues);
449NULLB_DEVICE_ATTR(home_node, uint, NULL);
450NULLB_DEVICE_ATTR(queue_mode, uint, NULL);
451NULLB_DEVICE_ATTR(blocksize, uint, NULL);
452NULLB_DEVICE_ATTR(max_sectors, uint, NULL);
453NULLB_DEVICE_ATTR(irqmode, uint, NULL);
454NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL);
455NULLB_DEVICE_ATTR(index, uint, NULL);
456NULLB_DEVICE_ATTR(blocking, bool, NULL);
457NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL);
458NULLB_DEVICE_ATTR(memory_backed, bool, NULL);
459NULLB_DEVICE_ATTR(discard, bool, NULL);
460NULLB_DEVICE_ATTR(mbps, uint, NULL);
461NULLB_DEVICE_ATTR(cache_size, ulong, NULL);
462NULLB_DEVICE_ATTR(zoned, bool, NULL);
463NULLB_DEVICE_ATTR(zone_size, ulong, NULL);
464NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL);
465NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL);
466NULLB_DEVICE_ATTR(zone_max_open, uint, NULL);
467NULLB_DEVICE_ATTR(zone_max_active, uint, NULL);
468NULLB_DEVICE_ATTR(zone_append_max_sectors, uint, NULL);
469NULLB_DEVICE_ATTR(zone_full, bool, NULL);
470NULLB_DEVICE_ATTR(virt_boundary, bool, NULL);
471NULLB_DEVICE_ATTR(no_sched, bool, NULL);
472NULLB_DEVICE_ATTR(shared_tags, bool, NULL);
473NULLB_DEVICE_ATTR(shared_tag_bitmap, bool, NULL);
474NULLB_DEVICE_ATTR(fua, bool, NULL);
475NULLB_DEVICE_ATTR(rotational, bool, NULL);
476NULLB_DEVICE_ATTR(badblocks_once, bool, NULL);
477NULLB_DEVICE_ATTR(badblocks_partial_io, bool, NULL);
478
479static ssize_t nullb_device_power_show(struct config_item *item, char *page)
480{
481 return nullb_device_bool_attr_show(val: to_nullb_device(item)->power, page);
482}
483
484static ssize_t nullb_device_power_store(struct config_item *item,
485 const char *page, size_t count)
486{
487 struct nullb_device *dev = to_nullb_device(item);
488 bool newp = false;
489 ssize_t ret;
490
491 ret = nullb_device_bool_attr_store(val: &newp, page, count);
492 if (ret < 0)
493 return ret;
494
495 ret = count;
496 mutex_lock(&lock);
497 if (!dev->power && newp) {
498 if (test_and_set_bit(nr: NULLB_DEV_FL_UP, addr: &dev->flags))
499 goto out;
500
501 ret = null_add_dev(dev);
502 if (ret) {
503 clear_bit(nr: NULLB_DEV_FL_UP, addr: &dev->flags);
504 goto out;
505 }
506
507 set_bit(nr: NULLB_DEV_FL_CONFIGURED, addr: &dev->flags);
508 dev->power = newp;
509 ret = count;
510 } else if (dev->power && !newp) {
511 if (test_and_clear_bit(nr: NULLB_DEV_FL_UP, addr: &dev->flags)) {
512 dev->power = newp;
513 null_del_dev(nullb: dev->nullb);
514 }
515 clear_bit(nr: NULLB_DEV_FL_CONFIGURED, addr: &dev->flags);
516 }
517
518out:
519 mutex_unlock(lock: &lock);
520 return ret;
521}
522
523CONFIGFS_ATTR(nullb_device_, power);
524
525static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
526{
527 struct nullb_device *t_dev = to_nullb_device(item);
528
529 return badblocks_show(bb: &t_dev->badblocks, page, unack: 0);
530}
531
532static ssize_t nullb_device_badblocks_store(struct config_item *item,
533 const char *page, size_t count)
534{
535 struct nullb_device *t_dev = to_nullb_device(item);
536 char *orig, *buf, *tmp;
537 u64 start, end;
538 int ret;
539
540 orig = kstrndup(s: page, len: count, GFP_KERNEL);
541 if (!orig)
542 return -ENOMEM;
543
544 buf = strstrip(str: orig);
545
546 ret = -EINVAL;
547 if (buf[0] != '+' && buf[0] != '-')
548 goto out;
549 tmp = strchr(&buf[1], '-');
550 if (!tmp)
551 goto out;
552 *tmp = '\0';
553 ret = kstrtoull(s: buf + 1, base: 0, res: &start);
554 if (ret)
555 goto out;
556 ret = kstrtoull(s: tmp + 1, base: 0, res: &end);
557 if (ret)
558 goto out;
559 ret = -EINVAL;
560 if (start > end)
561 goto out;
562 /* enable badblocks */
563 cmpxchg(&t_dev->badblocks.shift, -1, 0);
564 if (buf[0] == '+') {
565 if (badblocks_set(bb: &t_dev->badblocks, s: start,
566 sectors: end - start + 1, acknowledged: 1))
567 ret = count;
568 } else if (badblocks_clear(bb: &t_dev->badblocks, s: start,
569 sectors: end - start + 1)) {
570 ret = count;
571 }
572out:
573 kfree(objp: orig);
574 return ret;
575}
576CONFIGFS_ATTR(nullb_device_, badblocks);
577
578static ssize_t nullb_device_zone_readonly_store(struct config_item *item,
579 const char *page, size_t count)
580{
581 struct nullb_device *dev = to_nullb_device(item);
582
583 return zone_cond_store(dev, page, count, cond: BLK_ZONE_COND_READONLY);
584}
585CONFIGFS_ATTR_WO(nullb_device_, zone_readonly);
586
587static ssize_t nullb_device_zone_offline_store(struct config_item *item,
588 const char *page, size_t count)
589{
590 struct nullb_device *dev = to_nullb_device(item);
591
592 return zone_cond_store(dev, page, count, cond: BLK_ZONE_COND_OFFLINE);
593}
594CONFIGFS_ATTR_WO(nullb_device_, zone_offline);
595
596static struct configfs_attribute *nullb_device_attrs[] = {
597 &nullb_device_attr_badblocks,
598 &nullb_device_attr_badblocks_once,
599 &nullb_device_attr_badblocks_partial_io,
600 &nullb_device_attr_blocking,
601 &nullb_device_attr_blocksize,
602 &nullb_device_attr_cache_size,
603 &nullb_device_attr_completion_nsec,
604 &nullb_device_attr_discard,
605 &nullb_device_attr_fua,
606 &nullb_device_attr_home_node,
607 &nullb_device_attr_hw_queue_depth,
608 &nullb_device_attr_index,
609 &nullb_device_attr_irqmode,
610 &nullb_device_attr_max_sectors,
611 &nullb_device_attr_mbps,
612 &nullb_device_attr_memory_backed,
613 &nullb_device_attr_no_sched,
614 &nullb_device_attr_poll_queues,
615 &nullb_device_attr_power,
616 &nullb_device_attr_queue_mode,
617 &nullb_device_attr_rotational,
618 &nullb_device_attr_shared_tag_bitmap,
619 &nullb_device_attr_shared_tags,
620 &nullb_device_attr_size,
621 &nullb_device_attr_submit_queues,
622 &nullb_device_attr_use_per_node_hctx,
623 &nullb_device_attr_virt_boundary,
624 &nullb_device_attr_zone_append_max_sectors,
625 &nullb_device_attr_zone_capacity,
626 &nullb_device_attr_zone_full,
627 &nullb_device_attr_zone_max_active,
628 &nullb_device_attr_zone_max_open,
629 &nullb_device_attr_zone_nr_conv,
630 &nullb_device_attr_zone_offline,
631 &nullb_device_attr_zone_readonly,
632 &nullb_device_attr_zone_size,
633 &nullb_device_attr_zoned,
634 NULL,
635};
636
637static void nullb_device_release(struct config_item *item)
638{
639 struct nullb_device *dev = to_nullb_device(item);
640
641 null_free_device_storage(dev, is_cache: false);
642 null_free_dev(dev);
643}
644
645static struct configfs_item_operations nullb_device_ops = {
646 .release = nullb_device_release,
647};
648
649static const struct config_item_type nullb_device_type = {
650 .ct_item_ops = &nullb_device_ops,
651 .ct_attrs = nullb_device_attrs,
652 .ct_owner = THIS_MODULE,
653};
654
655#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
656
657static void nullb_add_fault_config(struct nullb_device *dev)
658{
659 fault_config_init(config: &dev->timeout_config, name: "timeout_inject");
660 fault_config_init(config: &dev->requeue_config, name: "requeue_inject");
661 fault_config_init(config: &dev->init_hctx_fault_config, name: "init_hctx_fault_inject");
662
663 configfs_add_default_group(new_group: &dev->timeout_config.group, group: &dev->group);
664 configfs_add_default_group(new_group: &dev->requeue_config.group, group: &dev->group);
665 configfs_add_default_group(new_group: &dev->init_hctx_fault_config.group, group: &dev->group);
666}
667
668static void nullb_del_fault_config(struct nullb_device *dev)
669{
670 config_item_put(&dev->init_hctx_fault_config.group.cg_item);
671 config_item_put(&dev->requeue_config.group.cg_item);
672 config_item_put(&dev->timeout_config.group.cg_item);
673}
674
675#else
676
677static void nullb_add_fault_config(struct nullb_device *dev)
678{
679}
680
681static void nullb_del_fault_config(struct nullb_device *dev)
682{
683}
684#endif
685
686static struct
687config_group *nullb_group_make_group(struct config_group *group, const char *name)
688{
689 struct nullb_device *dev;
690
691 if (null_find_dev_by_name(name))
692 return ERR_PTR(error: -EEXIST);
693
694 dev = null_alloc_dev();
695 if (!dev)
696 return ERR_PTR(error: -ENOMEM);
697
698 config_group_init_type_name(group: &dev->group, name, type: &nullb_device_type);
699 nullb_add_fault_config(dev);
700
701 return &dev->group;
702}
703
704static void
705nullb_group_drop_item(struct config_group *group, struct config_item *item)
706{
707 struct nullb_device *dev = to_nullb_device(item);
708
709 if (test_and_clear_bit(nr: NULLB_DEV_FL_UP, addr: &dev->flags)) {
710 mutex_lock(&lock);
711 dev->power = false;
712 null_del_dev(nullb: dev->nullb);
713 mutex_unlock(lock: &lock);
714 }
715 nullb_del_fault_config(dev);
716 config_item_put(item);
717}
718
719static ssize_t memb_group_features_show(struct config_item *item, char *page)
720{
721
722 struct configfs_attribute **entry;
723 char delimiter = ',';
724 size_t left = PAGE_SIZE;
725 size_t written = 0;
726 int ret;
727
728 for (entry = &nullb_device_attrs[0]; *entry && left > 0; entry++) {
729 if (!*(entry + 1))
730 delimiter = '\n';
731 ret = snprintf(buf: page + written, size: left, fmt: "%s%c", (*entry)->ca_name,
732 delimiter);
733 if (ret >= left) {
734 WARN_ONCE(1, "Too many null_blk features to print\n");
735 memzero_explicit(s: page, PAGE_SIZE);
736 return -ENOBUFS;
737 }
738 left -= ret;
739 written += ret;
740 }
741
742 return written;
743}
744
745CONFIGFS_ATTR_RO(memb_group_, features);
746
747static struct configfs_attribute *nullb_group_attrs[] = {
748 &memb_group_attr_features,
749 NULL,
750};
751
752static struct configfs_group_operations nullb_group_ops = {
753 .make_group = nullb_group_make_group,
754 .drop_item = nullb_group_drop_item,
755};
756
757static const struct config_item_type nullb_group_type = {
758 .ct_group_ops = &nullb_group_ops,
759 .ct_attrs = nullb_group_attrs,
760 .ct_owner = THIS_MODULE,
761};
762
763static struct configfs_subsystem nullb_subsys = {
764 .su_group = {
765 .cg_item = {
766 .ci_namebuf = "nullb",
767 .ci_type = &nullb_group_type,
768 },
769 },
770};
771
772static inline int null_cache_active(struct nullb *nullb)
773{
774 return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
775}
776
777static struct nullb_device *null_alloc_dev(void)
778{
779 struct nullb_device *dev;
780
781 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
782 if (!dev)
783 return NULL;
784
785#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
786 dev->timeout_config.attr = null_timeout_attr;
787 dev->requeue_config.attr = null_requeue_attr;
788 dev->init_hctx_fault_config.attr = null_init_hctx_attr;
789#endif
790
791 INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
792 INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
793 if (badblocks_init(bb: &dev->badblocks, enable: 0)) {
794 kfree(objp: dev);
795 return NULL;
796 }
797
798 dev->size = g_gb * 1024;
799 dev->completion_nsec = g_completion_nsec;
800 dev->submit_queues = g_submit_queues;
801 dev->prev_submit_queues = g_submit_queues;
802 dev->poll_queues = g_poll_queues;
803 dev->prev_poll_queues = g_poll_queues;
804 dev->home_node = g_home_node;
805 dev->queue_mode = g_queue_mode;
806 dev->blocksize = g_bs;
807 dev->max_sectors = g_max_sectors;
808 dev->irqmode = g_irqmode;
809 dev->hw_queue_depth = g_hw_queue_depth;
810 dev->blocking = g_blocking;
811 dev->memory_backed = g_memory_backed;
812 dev->discard = g_discard;
813 dev->cache_size = g_cache_size;
814 dev->mbps = g_mbps;
815 dev->use_per_node_hctx = g_use_per_node_hctx;
816 dev->zoned = g_zoned;
817 dev->zone_size = g_zone_size;
818 dev->zone_capacity = g_zone_capacity;
819 dev->zone_nr_conv = g_zone_nr_conv;
820 dev->zone_max_open = g_zone_max_open;
821 dev->zone_max_active = g_zone_max_active;
822 dev->zone_append_max_sectors = g_zone_append_max_sectors;
823 dev->zone_full = g_zone_full;
824 dev->virt_boundary = g_virt_boundary;
825 dev->no_sched = g_no_sched;
826 dev->shared_tags = g_shared_tags;
827 dev->shared_tag_bitmap = g_shared_tag_bitmap;
828 dev->fua = g_fua;
829 dev->rotational = g_rotational;
830
831 return dev;
832}
833
834static void null_free_dev(struct nullb_device *dev)
835{
836 if (!dev)
837 return;
838
839 null_free_zoned_dev(dev);
840 badblocks_exit(bb: &dev->badblocks);
841 kfree(objp: dev);
842}
843
844static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
845{
846 struct nullb_cmd *cmd = container_of(timer, struct nullb_cmd, timer);
847
848 blk_mq_end_request(rq: blk_mq_rq_from_pdu(pdu: cmd), error: cmd->error);
849 return HRTIMER_NORESTART;
850}
851
852static void null_cmd_end_timer(struct nullb_cmd *cmd)
853{
854 ktime_t kt = cmd->nq->dev->completion_nsec;
855
856 hrtimer_start(timer: &cmd->timer, tim: kt, mode: HRTIMER_MODE_REL);
857}
858
859static void null_complete_rq(struct request *rq)
860{
861 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
862
863 blk_mq_end_request(rq, error: cmd->error);
864}
865
866static struct nullb_page *null_alloc_page(void)
867{
868 struct nullb_page *t_page;
869
870 t_page = kmalloc(sizeof(struct nullb_page), GFP_NOIO);
871 if (!t_page)
872 return NULL;
873
874 t_page->page = alloc_pages(GFP_NOIO, 0);
875 if (!t_page->page) {
876 kfree(objp: t_page);
877 return NULL;
878 }
879
880 memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
881 return t_page;
882}
883
884static void null_free_page(struct nullb_page *t_page)
885{
886 __set_bit(NULLB_PAGE_FREE, t_page->bitmap);
887 if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
888 return;
889 __free_page(t_page->page);
890 kfree(objp: t_page);
891}
892
893static bool null_page_empty(struct nullb_page *page)
894{
895 int size = MAP_SZ - 2;
896
897 return find_first_bit(addr: page->bitmap, size) == size;
898}
899
900static void null_free_sector(struct nullb *nullb, sector_t sector,
901 bool is_cache)
902{
903 unsigned int sector_bit;
904 u64 idx;
905 struct nullb_page *t_page, *ret;
906 struct radix_tree_root *root;
907
908 root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
909 idx = sector >> PAGE_SECTORS_SHIFT;
910 sector_bit = (sector & SECTOR_MASK);
911
912 t_page = radix_tree_lookup(root, idx);
913 if (t_page) {
914 __clear_bit(sector_bit, t_page->bitmap);
915
916 if (null_page_empty(page: t_page)) {
917 ret = radix_tree_delete_item(root, idx, t_page);
918 WARN_ON(ret != t_page);
919 null_free_page(t_page: ret);
920 if (is_cache)
921 nullb->dev->curr_cache -= PAGE_SIZE;
922 }
923 }
924}
925
926static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
927 struct nullb_page *t_page, bool is_cache)
928{
929 struct radix_tree_root *root;
930
931 root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
932
933 if (radix_tree_insert(root, index: idx, t_page)) {
934 null_free_page(t_page);
935 t_page = radix_tree_lookup(root, idx);
936 WARN_ON(!t_page || t_page->page->private != idx);
937 } else if (is_cache)
938 nullb->dev->curr_cache += PAGE_SIZE;
939
940 return t_page;
941}
942
943static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
944{
945 unsigned long pos = 0;
946 int nr_pages;
947 struct nullb_page *ret, *t_pages[FREE_BATCH];
948 struct radix_tree_root *root;
949
950 root = is_cache ? &dev->cache : &dev->data;
951
952 do {
953 int i;
954
955 nr_pages = radix_tree_gang_lookup(root,
956 results: (void **)t_pages, first_index: pos, FREE_BATCH);
957
958 for (i = 0; i < nr_pages; i++) {
959 pos = t_pages[i]->page->private;
960 ret = radix_tree_delete_item(root, pos, t_pages[i]);
961 WARN_ON(ret != t_pages[i]);
962 null_free_page(t_page: ret);
963 }
964
965 pos++;
966 } while (nr_pages == FREE_BATCH);
967
968 if (is_cache)
969 dev->curr_cache = 0;
970}
971
972static struct nullb_page *__null_lookup_page(struct nullb *nullb,
973 sector_t sector, bool for_write, bool is_cache)
974{
975 unsigned int sector_bit;
976 u64 idx;
977 struct nullb_page *t_page;
978 struct radix_tree_root *root;
979
980 idx = sector >> PAGE_SECTORS_SHIFT;
981 sector_bit = (sector & SECTOR_MASK);
982
983 root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
984 t_page = radix_tree_lookup(root, idx);
985 WARN_ON(t_page && t_page->page->private != idx);
986
987 if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
988 return t_page;
989
990 return NULL;
991}
992
993static struct nullb_page *null_lookup_page(struct nullb *nullb,
994 sector_t sector, bool for_write, bool ignore_cache)
995{
996 struct nullb_page *page = NULL;
997
998 if (!ignore_cache)
999 page = __null_lookup_page(nullb, sector, for_write, is_cache: true);
1000 if (page)
1001 return page;
1002 return __null_lookup_page(nullb, sector, for_write, is_cache: false);
1003}
1004
1005static struct nullb_page *null_insert_page(struct nullb *nullb,
1006 sector_t sector, bool ignore_cache)
1007 __releases(&nullb->lock)
1008 __acquires(&nullb->lock)
1009{
1010 u64 idx;
1011 struct nullb_page *t_page;
1012
1013 t_page = null_lookup_page(nullb, sector, for_write: true, ignore_cache);
1014 if (t_page)
1015 return t_page;
1016
1017 spin_unlock_irq(lock: &nullb->lock);
1018
1019 t_page = null_alloc_page();
1020 if (!t_page)
1021 goto out_lock;
1022
1023 if (radix_tree_preload(GFP_NOIO))
1024 goto out_freepage;
1025
1026 spin_lock_irq(lock: &nullb->lock);
1027 idx = sector >> PAGE_SECTORS_SHIFT;
1028 t_page->page->private = idx;
1029 t_page = null_radix_tree_insert(nullb, idx, t_page, is_cache: !ignore_cache);
1030 radix_tree_preload_end();
1031
1032 return t_page;
1033out_freepage:
1034 null_free_page(t_page);
1035out_lock:
1036 spin_lock_irq(lock: &nullb->lock);
1037 return null_lookup_page(nullb, sector, for_write: true, ignore_cache);
1038}
1039
1040static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
1041{
1042 int i;
1043 unsigned int offset;
1044 u64 idx;
1045 struct nullb_page *t_page, *ret;
1046 void *dst, *src;
1047
1048 idx = c_page->page->private;
1049
1050 t_page = null_insert_page(nullb, sector: idx << PAGE_SECTORS_SHIFT, ignore_cache: true);
1051
1052 __clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
1053 if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
1054 null_free_page(t_page: c_page);
1055 if (t_page && null_page_empty(page: t_page)) {
1056 ret = radix_tree_delete_item(&nullb->dev->data,
1057 idx, t_page);
1058 null_free_page(t_page);
1059 }
1060 return 0;
1061 }
1062
1063 if (!t_page)
1064 return -ENOMEM;
1065
1066 src = kmap_local_page(page: c_page->page);
1067 dst = kmap_local_page(page: t_page->page);
1068
1069 for (i = 0; i < PAGE_SECTORS;
1070 i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
1071 if (test_bit(i, c_page->bitmap)) {
1072 offset = (i << SECTOR_SHIFT);
1073 memcpy(dst + offset, src + offset,
1074 nullb->dev->blocksize);
1075 __set_bit(i, t_page->bitmap);
1076 }
1077 }
1078
1079 kunmap_local(dst);
1080 kunmap_local(src);
1081
1082 ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
1083 null_free_page(t_page: ret);
1084 nullb->dev->curr_cache -= PAGE_SIZE;
1085
1086 return 0;
1087}
1088
1089static int null_make_cache_space(struct nullb *nullb, unsigned long n)
1090{
1091 int i, err, nr_pages;
1092 struct nullb_page *c_pages[FREE_BATCH];
1093 unsigned long flushed = 0, one_round;
1094
1095again:
1096 if ((nullb->dev->cache_size * 1024 * 1024) >
1097 nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
1098 return 0;
1099
1100 nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
1101 results: (void **)c_pages, first_index: nullb->cache_flush_pos, FREE_BATCH);
1102 /*
1103 * nullb_flush_cache_page could unlock before using the c_pages. To
1104 * avoid race, we don't allow page free
1105 */
1106 for (i = 0; i < nr_pages; i++) {
1107 nullb->cache_flush_pos = c_pages[i]->page->private;
1108 /*
1109 * We found the page which is being flushed to disk by other
1110 * threads
1111 */
1112 if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
1113 c_pages[i] = NULL;
1114 else
1115 __set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
1116 }
1117
1118 one_round = 0;
1119 for (i = 0; i < nr_pages; i++) {
1120 if (c_pages[i] == NULL)
1121 continue;
1122 err = null_flush_cache_page(nullb, c_page: c_pages[i]);
1123 if (err)
1124 return err;
1125 one_round++;
1126 }
1127 flushed += one_round << PAGE_SHIFT;
1128
1129 if (n > flushed) {
1130 if (nr_pages == 0)
1131 nullb->cache_flush_pos = 0;
1132 if (one_round == 0) {
1133 /* give other threads a chance */
1134 spin_unlock_irq(lock: &nullb->lock);
1135 spin_lock_irq(lock: &nullb->lock);
1136 }
1137 goto again;
1138 }
1139 return 0;
1140}
1141
1142static blk_status_t copy_to_nullb(struct nullb *nullb, void *source,
1143 loff_t pos, size_t n, bool is_fua)
1144{
1145 size_t temp, count = 0;
1146 struct nullb_page *t_page;
1147 sector_t sector;
1148
1149 while (count < n) {
1150 temp = min3(nullb->dev->blocksize, n - count,
1151 PAGE_SIZE - offset_in_page(pos));
1152 sector = pos >> SECTOR_SHIFT;
1153
1154 if (null_cache_active(nullb) && !is_fua)
1155 null_make_cache_space(nullb, PAGE_SIZE);
1156
1157 t_page = null_insert_page(nullb, sector,
1158 ignore_cache: !null_cache_active(nullb) || is_fua);
1159 if (!t_page)
1160 return BLK_STS_NOSPC;
1161
1162 memcpy_to_page(page: t_page->page, offset_in_page(pos),
1163 from: source + count, len: temp);
1164
1165 __set_bit(sector & SECTOR_MASK, t_page->bitmap);
1166
1167 if (is_fua)
1168 null_free_sector(nullb, sector, is_cache: true);
1169
1170 count += temp;
1171 pos += temp;
1172 }
1173 return BLK_STS_OK;
1174}
1175
1176static void copy_from_nullb(struct nullb *nullb, void *dest, loff_t pos,
1177 size_t n)
1178{
1179 size_t temp, count = 0;
1180 struct nullb_page *t_page;
1181 sector_t sector;
1182
1183 while (count < n) {
1184 temp = min3(nullb->dev->blocksize, n - count,
1185 PAGE_SIZE - offset_in_page(pos));
1186 sector = pos >> SECTOR_SHIFT;
1187
1188 t_page = null_lookup_page(nullb, sector, for_write: false,
1189 ignore_cache: !null_cache_active(nullb));
1190 if (t_page)
1191 memcpy_from_page(to: dest + count, page: t_page->page,
1192 offset_in_page(pos), len: temp);
1193 else
1194 memset(dest + count, 0, temp);
1195
1196 count += temp;
1197 pos += temp;
1198 }
1199}
1200
1201blk_status_t null_handle_discard(struct nullb_device *dev,
1202 sector_t sector, sector_t nr_sectors)
1203{
1204 struct nullb *nullb = dev->nullb;
1205 size_t n = nr_sectors << SECTOR_SHIFT;
1206 size_t temp;
1207
1208 spin_lock_irq(lock: &nullb->lock);
1209 while (n > 0) {
1210 temp = min_t(size_t, n, dev->blocksize);
1211 null_free_sector(nullb, sector, is_cache: false);
1212 if (null_cache_active(nullb))
1213 null_free_sector(nullb, sector, is_cache: true);
1214 sector += temp >> SECTOR_SHIFT;
1215 n -= temp;
1216 }
1217 spin_unlock_irq(lock: &nullb->lock);
1218
1219 return BLK_STS_OK;
1220}
1221
1222static blk_status_t null_handle_flush(struct nullb *nullb)
1223{
1224 int err;
1225
1226 if (!null_cache_active(nullb))
1227 return 0;
1228
1229 spin_lock_irq(lock: &nullb->lock);
1230 while (true) {
1231 err = null_make_cache_space(nullb,
1232 n: nullb->dev->cache_size * 1024 * 1024);
1233 if (err || nullb->dev->curr_cache == 0)
1234 break;
1235 }
1236
1237 WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1238 spin_unlock_irq(lock: &nullb->lock);
1239 return errno_to_blk_status(errno: err);
1240}
1241
1242static blk_status_t null_transfer(struct nullb *nullb, struct page *page,
1243 unsigned int len, unsigned int off, bool is_write, loff_t pos,
1244 bool is_fua)
1245{
1246 struct nullb_device *dev = nullb->dev;
1247 blk_status_t err = BLK_STS_OK;
1248 unsigned int valid_len = len;
1249 void *p;
1250
1251 p = kmap_local_page(page) + off;
1252 if (!is_write) {
1253 if (dev->zoned) {
1254 valid_len = null_zone_valid_read_len(nullb,
1255 sector: pos >> SECTOR_SHIFT, len);
1256 if (valid_len && valid_len != len)
1257 valid_len -= pos & (SECTOR_SIZE - 1);
1258 }
1259
1260 if (valid_len) {
1261 copy_from_nullb(nullb, dest: p, pos, n: valid_len);
1262 off += valid_len;
1263 len -= valid_len;
1264 }
1265
1266 if (len)
1267 memset(p + valid_len, 0xff, len);
1268 flush_dcache_page(page);
1269 } else {
1270 flush_dcache_page(page);
1271 err = copy_to_nullb(nullb, source: p, pos, n: len, is_fua);
1272 }
1273
1274 kunmap_local(p);
1275 return err;
1276}
1277
1278/*
1279 * Transfer data for the given request. The transfer size is capped with the
1280 * nr_sectors argument.
1281 */
1282static blk_status_t null_handle_data_transfer(struct nullb_cmd *cmd,
1283 sector_t nr_sectors)
1284{
1285 struct request *rq = blk_mq_rq_from_pdu(pdu: cmd);
1286 struct nullb *nullb = cmd->nq->dev->nullb;
1287 blk_status_t err = BLK_STS_OK;
1288 unsigned int len;
1289 loff_t pos = blk_rq_pos(rq) << SECTOR_SHIFT;
1290 unsigned int max_bytes = nr_sectors << SECTOR_SHIFT;
1291 unsigned int transferred_bytes = 0;
1292 struct req_iterator iter;
1293 struct bio_vec bvec;
1294
1295 spin_lock_irq(lock: &nullb->lock);
1296 rq_for_each_segment(bvec, rq, iter) {
1297 len = bvec.bv_len;
1298 if (transferred_bytes + len > max_bytes)
1299 len = max_bytes - transferred_bytes;
1300 err = null_transfer(nullb, page: bvec.bv_page, len, off: bvec.bv_offset,
1301 is_write: op_is_write(op: req_op(req: rq)), pos,
1302 is_fua: rq->cmd_flags & REQ_FUA);
1303 if (err)
1304 break;
1305 pos += len;
1306 transferred_bytes += len;
1307 if (transferred_bytes >= max_bytes)
1308 break;
1309 }
1310 spin_unlock_irq(lock: &nullb->lock);
1311
1312 return err;
1313}
1314
1315static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd)
1316{
1317 struct nullb_device *dev = cmd->nq->dev;
1318 struct nullb *nullb = dev->nullb;
1319 blk_status_t sts = BLK_STS_OK;
1320 struct request *rq = blk_mq_rq_from_pdu(pdu: cmd);
1321
1322 if (!hrtimer_active(timer: &nullb->bw_timer))
1323 hrtimer_restart(timer: &nullb->bw_timer);
1324
1325 if (atomic_long_sub_return(i: blk_rq_bytes(rq), v: &nullb->cur_bytes) < 0) {
1326 blk_mq_stop_hw_queues(q: nullb->q);
1327 /* race with timer */
1328 if (atomic_long_read(v: &nullb->cur_bytes) > 0)
1329 blk_mq_start_stopped_hw_queues(q: nullb->q, async: true);
1330 /* requeue request */
1331 sts = BLK_STS_DEV_RESOURCE;
1332 }
1333 return sts;
1334}
1335
1336/*
1337 * Check if the command should fail for the badblocks. If so, return
1338 * BLK_STS_IOERR and return number of partial I/O sectors to be written or read,
1339 * which may be less than the requested number of sectors.
1340 *
1341 * @cmd: The command to handle.
1342 * @sector: The start sector for I/O.
1343 * @nr_sectors: Specifies number of sectors to write or read, and returns the
1344 * number of sectors to be written or read.
1345 */
1346blk_status_t null_handle_badblocks(struct nullb_cmd *cmd, sector_t sector,
1347 unsigned int *nr_sectors)
1348{
1349 struct badblocks *bb = &cmd->nq->dev->badblocks;
1350 struct nullb_device *dev = cmd->nq->dev;
1351 unsigned int block_sectors = dev->blocksize >> SECTOR_SHIFT;
1352 sector_t first_bad, bad_sectors;
1353 unsigned int partial_io_sectors = 0;
1354
1355 if (!badblocks_check(bb, s: sector, sectors: *nr_sectors, first_bad: &first_bad, bad_sectors: &bad_sectors))
1356 return BLK_STS_OK;
1357
1358 if (cmd->nq->dev->badblocks_once)
1359 badblocks_clear(bb, s: first_bad, sectors: bad_sectors);
1360
1361 if (cmd->nq->dev->badblocks_partial_io) {
1362 if (!IS_ALIGNED(first_bad, block_sectors))
1363 first_bad = ALIGN_DOWN(first_bad, block_sectors);
1364 if (sector < first_bad)
1365 partial_io_sectors = first_bad - sector;
1366 }
1367 *nr_sectors = partial_io_sectors;
1368
1369 return BLK_STS_IOERR;
1370}
1371
1372blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd, enum req_op op,
1373 sector_t sector, sector_t nr_sectors)
1374{
1375 struct nullb_device *dev = cmd->nq->dev;
1376
1377 if (op == REQ_OP_DISCARD)
1378 return null_handle_discard(dev, sector, nr_sectors);
1379
1380 return null_handle_data_transfer(cmd, nr_sectors);
1381}
1382
1383static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd)
1384{
1385 struct request *rq = blk_mq_rq_from_pdu(pdu: cmd);
1386 struct nullb_device *dev = cmd->nq->dev;
1387 struct bio *bio;
1388
1389 if (!dev->memory_backed && req_op(req: rq) == REQ_OP_READ) {
1390 __rq_for_each_bio(bio, rq)
1391 zero_fill_bio(bio);
1392 }
1393}
1394
1395static inline void nullb_complete_cmd(struct nullb_cmd *cmd)
1396{
1397 struct request *rq = blk_mq_rq_from_pdu(pdu: cmd);
1398
1399 /*
1400 * Since root privileges are required to configure the null_blk
1401 * driver, it is fine that this driver does not initialize the
1402 * data buffers of read commands. Zero-initialize these buffers
1403 * anyway if KMSAN is enabled to prevent that KMSAN complains
1404 * about null_blk not initializing read data buffers.
1405 */
1406 if (IS_ENABLED(CONFIG_KMSAN))
1407 nullb_zero_read_cmd_buffer(cmd);
1408
1409 /* Complete IO by inline, softirq or timer */
1410 switch (cmd->nq->dev->irqmode) {
1411 case NULL_IRQ_SOFTIRQ:
1412 blk_mq_complete_request(rq);
1413 break;
1414 case NULL_IRQ_NONE:
1415 blk_mq_end_request(rq, error: cmd->error);
1416 break;
1417 case NULL_IRQ_TIMER:
1418 null_cmd_end_timer(cmd);
1419 break;
1420 }
1421}
1422
1423blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op,
1424 sector_t sector, unsigned int nr_sectors)
1425{
1426 struct nullb_device *dev = cmd->nq->dev;
1427 blk_status_t badblocks_ret = BLK_STS_OK;
1428 blk_status_t ret;
1429
1430 if (dev->badblocks.shift != -1)
1431 badblocks_ret = null_handle_badblocks(cmd, sector, nr_sectors: &nr_sectors);
1432
1433 if (dev->memory_backed && nr_sectors) {
1434 ret = null_handle_memory_backed(cmd, op, sector, nr_sectors);
1435 if (ret != BLK_STS_OK)
1436 return ret;
1437 }
1438
1439 return badblocks_ret;
1440}
1441
1442static void null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
1443 sector_t nr_sectors, enum req_op op)
1444{
1445 struct nullb_device *dev = cmd->nq->dev;
1446 struct nullb *nullb = dev->nullb;
1447 blk_status_t sts;
1448
1449 if (op == REQ_OP_FLUSH) {
1450 cmd->error = null_handle_flush(nullb);
1451 goto out;
1452 }
1453
1454 if (dev->zoned)
1455 sts = null_process_zoned_cmd(cmd, op, sector, nr_sectors);
1456 else
1457 sts = null_process_cmd(cmd, op, sector, nr_sectors);
1458
1459 /* Do not overwrite errors (e.g. timeout errors) */
1460 if (cmd->error == BLK_STS_OK)
1461 cmd->error = sts;
1462
1463out:
1464 nullb_complete_cmd(cmd);
1465}
1466
1467static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1468{
1469 struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1470 ktime_t timer_interval = ktime_set(secs: 0, TIMER_INTERVAL);
1471 unsigned int mbps = nullb->dev->mbps;
1472
1473 if (atomic_long_read(v: &nullb->cur_bytes) == mb_per_tick(mbps))
1474 return HRTIMER_NORESTART;
1475
1476 atomic_long_set(v: &nullb->cur_bytes, i: mb_per_tick(mbps));
1477 blk_mq_start_stopped_hw_queues(q: nullb->q, async: true);
1478
1479 hrtimer_forward_now(timer: &nullb->bw_timer, interval: timer_interval);
1480
1481 return HRTIMER_RESTART;
1482}
1483
1484static void nullb_setup_bwtimer(struct nullb *nullb)
1485{
1486 ktime_t timer_interval = ktime_set(secs: 0, TIMER_INTERVAL);
1487
1488 hrtimer_setup(timer: &nullb->bw_timer, function: nullb_bwtimer_fn, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
1489 atomic_long_set(v: &nullb->cur_bytes, i: mb_per_tick(mbps: nullb->dev->mbps));
1490 hrtimer_start(timer: &nullb->bw_timer, tim: timer_interval, mode: HRTIMER_MODE_REL);
1491}
1492
1493#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1494
1495static bool should_timeout_request(struct request *rq)
1496{
1497 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1498 struct nullb_device *dev = cmd->nq->dev;
1499
1500 return should_fail(attr: &dev->timeout_config.attr, size: 1);
1501}
1502
1503static bool should_requeue_request(struct request *rq)
1504{
1505 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1506 struct nullb_device *dev = cmd->nq->dev;
1507
1508 return should_fail(attr: &dev->requeue_config.attr, size: 1);
1509}
1510
1511static bool should_init_hctx_fail(struct nullb_device *dev)
1512{
1513 return should_fail(attr: &dev->init_hctx_fault_config.attr, size: 1);
1514}
1515
1516#else
1517
1518static bool should_timeout_request(struct request *rq)
1519{
1520 return false;
1521}
1522
1523static bool should_requeue_request(struct request *rq)
1524{
1525 return false;
1526}
1527
1528static bool should_init_hctx_fail(struct nullb_device *dev)
1529{
1530 return false;
1531}
1532
1533#endif
1534
1535static void null_map_queues(struct blk_mq_tag_set *set)
1536{
1537 struct nullb *nullb = set->driver_data;
1538 int i, qoff;
1539 unsigned int submit_queues = g_submit_queues;
1540 unsigned int poll_queues = g_poll_queues;
1541
1542 if (nullb) {
1543 struct nullb_device *dev = nullb->dev;
1544
1545 /*
1546 * Refer nr_hw_queues of the tag set to check if the expected
1547 * number of hardware queues are prepared. If block layer failed
1548 * to prepare them, use previous numbers of submit queues and
1549 * poll queues to map queues.
1550 */
1551 if (set->nr_hw_queues ==
1552 dev->submit_queues + dev->poll_queues) {
1553 submit_queues = dev->submit_queues;
1554 poll_queues = dev->poll_queues;
1555 } else if (set->nr_hw_queues ==
1556 dev->prev_submit_queues + dev->prev_poll_queues) {
1557 submit_queues = dev->prev_submit_queues;
1558 poll_queues = dev->prev_poll_queues;
1559 } else {
1560 pr_warn("tag set has unexpected nr_hw_queues: %d\n",
1561 set->nr_hw_queues);
1562 WARN_ON_ONCE(true);
1563 submit_queues = 1;
1564 poll_queues = 0;
1565 }
1566 }
1567
1568 for (i = 0, qoff = 0; i < set->nr_maps; i++) {
1569 struct blk_mq_queue_map *map = &set->map[i];
1570
1571 switch (i) {
1572 case HCTX_TYPE_DEFAULT:
1573 map->nr_queues = submit_queues;
1574 break;
1575 case HCTX_TYPE_READ:
1576 map->nr_queues = 0;
1577 continue;
1578 case HCTX_TYPE_POLL:
1579 map->nr_queues = poll_queues;
1580 break;
1581 }
1582 map->queue_offset = qoff;
1583 qoff += map->nr_queues;
1584 blk_mq_map_queues(qmap: map);
1585 }
1586}
1587
1588static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
1589{
1590 struct nullb_queue *nq = hctx->driver_data;
1591 LIST_HEAD(list);
1592 int nr = 0;
1593 struct request *rq;
1594
1595 spin_lock(lock: &nq->poll_lock);
1596 list_splice_init(list: &nq->poll_list, head: &list);
1597 list_for_each_entry(rq, &list, queuelist)
1598 blk_mq_set_request_complete(rq);
1599 spin_unlock(lock: &nq->poll_lock);
1600
1601 while (!list_empty(head: &list)) {
1602 struct nullb_cmd *cmd;
1603 struct request *req;
1604
1605 req = list_first_entry(&list, struct request, queuelist);
1606 list_del_init(entry: &req->queuelist);
1607 cmd = blk_mq_rq_to_pdu(rq: req);
1608 cmd->error = null_process_cmd(cmd, op: req_op(req), sector: blk_rq_pos(rq: req),
1609 nr_sectors: blk_rq_sectors(rq: req));
1610 if (!blk_mq_add_to_batch(req, iob, is_error: cmd->error != BLK_STS_OK,
1611 complete: blk_mq_end_request_batch))
1612 blk_mq_end_request(rq: req, error: cmd->error);
1613 nr++;
1614 }
1615
1616 return nr;
1617}
1618
1619static enum blk_eh_timer_return null_timeout_rq(struct request *rq)
1620{
1621 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1622 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1623
1624 if (hctx->type == HCTX_TYPE_POLL) {
1625 struct nullb_queue *nq = hctx->driver_data;
1626
1627 spin_lock(lock: &nq->poll_lock);
1628 /* The request may have completed meanwhile. */
1629 if (blk_mq_request_completed(rq)) {
1630 spin_unlock(lock: &nq->poll_lock);
1631 return BLK_EH_DONE;
1632 }
1633 list_del_init(entry: &rq->queuelist);
1634 spin_unlock(lock: &nq->poll_lock);
1635 }
1636
1637 pr_info("rq %p timed out\n", rq);
1638
1639 /*
1640 * If the device is marked as blocking (i.e. memory backed or zoned
1641 * device), the submission path may be blocked waiting for resources
1642 * and cause real timeouts. For these real timeouts, the submission
1643 * path will complete the request using blk_mq_complete_request().
1644 * Only fake timeouts need to execute blk_mq_complete_request() here.
1645 */
1646 cmd->error = BLK_STS_TIMEOUT;
1647 if (cmd->fake_timeout || hctx->type == HCTX_TYPE_POLL)
1648 blk_mq_complete_request(rq);
1649 return BLK_EH_DONE;
1650}
1651
1652static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1653 const struct blk_mq_queue_data *bd)
1654{
1655 struct request *rq = bd->rq;
1656 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1657 struct nullb_queue *nq = hctx->driver_data;
1658 sector_t nr_sectors = blk_rq_sectors(rq);
1659 sector_t sector = blk_rq_pos(rq);
1660 const bool is_poll = hctx->type == HCTX_TYPE_POLL;
1661
1662 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1663
1664 if (!is_poll && nq->dev->irqmode == NULL_IRQ_TIMER) {
1665 hrtimer_setup(timer: &cmd->timer, function: null_cmd_timer_expired, CLOCK_MONOTONIC,
1666 mode: HRTIMER_MODE_REL);
1667 }
1668 cmd->error = BLK_STS_OK;
1669 cmd->nq = nq;
1670 cmd->fake_timeout = should_timeout_request(rq) ||
1671 blk_should_fake_timeout(q: rq->q);
1672
1673 if (should_requeue_request(rq)) {
1674 /*
1675 * Alternate between hitting the core BUSY path, and the
1676 * driver driven requeue path
1677 */
1678 nq->requeue_selection++;
1679 if (nq->requeue_selection & 1)
1680 return BLK_STS_RESOURCE;
1681 blk_mq_requeue_request(rq, kick_requeue_list: true);
1682 return BLK_STS_OK;
1683 }
1684
1685 if (test_bit(NULLB_DEV_FL_THROTTLED, &nq->dev->flags)) {
1686 blk_status_t sts = null_handle_throttled(cmd);
1687
1688 if (sts != BLK_STS_OK)
1689 return sts;
1690 }
1691
1692 blk_mq_start_request(rq);
1693
1694 if (is_poll) {
1695 spin_lock(lock: &nq->poll_lock);
1696 list_add_tail(new: &rq->queuelist, head: &nq->poll_list);
1697 spin_unlock(lock: &nq->poll_lock);
1698 return BLK_STS_OK;
1699 }
1700 if (cmd->fake_timeout)
1701 return BLK_STS_OK;
1702
1703 null_handle_cmd(cmd, sector, nr_sectors, op: req_op(req: rq));
1704 return BLK_STS_OK;
1705}
1706
1707static void null_queue_rqs(struct rq_list *rqlist)
1708{
1709 struct rq_list requeue_list = {};
1710 struct blk_mq_queue_data bd = { };
1711 blk_status_t ret;
1712
1713 do {
1714 struct request *rq = rq_list_pop(rl: rqlist);
1715
1716 bd.rq = rq;
1717 ret = null_queue_rq(hctx: rq->mq_hctx, bd: &bd);
1718 if (ret != BLK_STS_OK)
1719 rq_list_add_tail(rl: &requeue_list, rq);
1720 } while (!rq_list_empty(rl: rqlist));
1721
1722 *rqlist = requeue_list;
1723}
1724
1725static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1726{
1727 nq->dev = nullb->dev;
1728 INIT_LIST_HEAD(list: &nq->poll_list);
1729 spin_lock_init(&nq->poll_lock);
1730}
1731
1732static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1733 unsigned int hctx_idx)
1734{
1735 struct nullb *nullb = hctx->queue->queuedata;
1736 struct nullb_queue *nq;
1737
1738 if (should_init_hctx_fail(dev: nullb->dev))
1739 return -EFAULT;
1740
1741 nq = &nullb->queues[hctx_idx];
1742 hctx->driver_data = nq;
1743 null_init_queue(nullb, nq);
1744
1745 return 0;
1746}
1747
1748static const struct blk_mq_ops null_mq_ops = {
1749 .queue_rq = null_queue_rq,
1750 .queue_rqs = null_queue_rqs,
1751 .complete = null_complete_rq,
1752 .timeout = null_timeout_rq,
1753 .poll = null_poll,
1754 .map_queues = null_map_queues,
1755 .init_hctx = null_init_hctx,
1756};
1757
1758static void null_del_dev(struct nullb *nullb)
1759{
1760 struct nullb_device *dev;
1761
1762 if (!nullb)
1763 return;
1764
1765 dev = nullb->dev;
1766
1767 ida_free(&nullb_indexes, id: nullb->index);
1768
1769 list_del_init(entry: &nullb->list);
1770
1771 del_gendisk(gp: nullb->disk);
1772
1773 if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1774 hrtimer_cancel(timer: &nullb->bw_timer);
1775 atomic_long_set(v: &nullb->cur_bytes, LONG_MAX);
1776 blk_mq_start_stopped_hw_queues(q: nullb->q, async: true);
1777 }
1778
1779 put_disk(disk: nullb->disk);
1780 if (nullb->tag_set == &nullb->__tag_set)
1781 blk_mq_free_tag_set(set: nullb->tag_set);
1782 kfree(objp: nullb->queues);
1783 if (null_cache_active(nullb))
1784 null_free_device_storage(dev: nullb->dev, is_cache: true);
1785 kfree(objp: nullb);
1786 dev->nullb = NULL;
1787}
1788
1789static void null_config_discard(struct nullb *nullb, struct queue_limits *lim)
1790{
1791 if (nullb->dev->discard == false)
1792 return;
1793
1794 if (!nullb->dev->memory_backed) {
1795 nullb->dev->discard = false;
1796 pr_info("discard option is ignored without memory backing\n");
1797 return;
1798 }
1799
1800 if (nullb->dev->zoned) {
1801 nullb->dev->discard = false;
1802 pr_info("discard option is ignored in zoned mode\n");
1803 return;
1804 }
1805
1806 lim->max_hw_discard_sectors = UINT_MAX >> 9;
1807}
1808
1809static const struct block_device_operations null_ops = {
1810 .owner = THIS_MODULE,
1811 .report_zones = null_report_zones,
1812};
1813
1814static int setup_queues(struct nullb *nullb)
1815{
1816 int nqueues = nr_cpu_ids;
1817
1818 if (g_poll_queues)
1819 nqueues += g_poll_queues;
1820
1821 nullb->queues = kcalloc(nqueues, sizeof(struct nullb_queue),
1822 GFP_KERNEL);
1823 if (!nullb->queues)
1824 return -ENOMEM;
1825
1826 return 0;
1827}
1828
1829static int null_init_tag_set(struct blk_mq_tag_set *set, int poll_queues)
1830{
1831 set->ops = &null_mq_ops;
1832 set->cmd_size = sizeof(struct nullb_cmd);
1833 set->timeout = 5 * HZ;
1834 set->nr_maps = 1;
1835 if (poll_queues) {
1836 set->nr_hw_queues += poll_queues;
1837 set->nr_maps += 2;
1838 }
1839 return blk_mq_alloc_tag_set(set);
1840}
1841
1842static int null_init_global_tag_set(void)
1843{
1844 int error;
1845
1846 if (tag_set.ops)
1847 return 0;
1848
1849 tag_set.nr_hw_queues = g_submit_queues;
1850 tag_set.queue_depth = g_hw_queue_depth;
1851 tag_set.numa_node = g_home_node;
1852 if (g_no_sched)
1853 tag_set.flags |= BLK_MQ_F_NO_SCHED_BY_DEFAULT;
1854 if (g_shared_tag_bitmap)
1855 tag_set.flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1856 if (g_blocking)
1857 tag_set.flags |= BLK_MQ_F_BLOCKING;
1858
1859 error = null_init_tag_set(set: &tag_set, poll_queues: g_poll_queues);
1860 if (error)
1861 tag_set.ops = NULL;
1862 return error;
1863}
1864
1865static int null_setup_tagset(struct nullb *nullb)
1866{
1867 if (nullb->dev->shared_tags) {
1868 nullb->tag_set = &tag_set;
1869 return null_init_global_tag_set();
1870 }
1871
1872 nullb->tag_set = &nullb->__tag_set;
1873 nullb->tag_set->driver_data = nullb;
1874 nullb->tag_set->nr_hw_queues = nullb->dev->submit_queues;
1875 nullb->tag_set->queue_depth = nullb->dev->hw_queue_depth;
1876 nullb->tag_set->numa_node = nullb->dev->home_node;
1877 if (nullb->dev->no_sched)
1878 nullb->tag_set->flags |= BLK_MQ_F_NO_SCHED_BY_DEFAULT;
1879 if (nullb->dev->shared_tag_bitmap)
1880 nullb->tag_set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1881 if (nullb->dev->blocking)
1882 nullb->tag_set->flags |= BLK_MQ_F_BLOCKING;
1883 return null_init_tag_set(set: nullb->tag_set, poll_queues: nullb->dev->poll_queues);
1884}
1885
1886static int null_validate_conf(struct nullb_device *dev)
1887{
1888 if (dev->queue_mode == NULL_Q_RQ) {
1889 pr_err("legacy IO path is no longer available\n");
1890 return -EINVAL;
1891 }
1892 if (dev->queue_mode == NULL_Q_BIO) {
1893 pr_err("BIO-based IO path is no longer available, using blk-mq instead.\n");
1894 dev->queue_mode = NULL_Q_MQ;
1895 }
1896
1897 if (dev->use_per_node_hctx) {
1898 if (dev->submit_queues != nr_online_nodes)
1899 dev->submit_queues = nr_online_nodes;
1900 } else if (dev->submit_queues > nr_cpu_ids)
1901 dev->submit_queues = nr_cpu_ids;
1902 else if (dev->submit_queues == 0)
1903 dev->submit_queues = 1;
1904 dev->prev_submit_queues = dev->submit_queues;
1905
1906 if (dev->poll_queues > g_poll_queues)
1907 dev->poll_queues = g_poll_queues;
1908 dev->prev_poll_queues = dev->poll_queues;
1909 dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
1910
1911 /* Do memory allocation, so set blocking */
1912 if (dev->memory_backed)
1913 dev->blocking = true;
1914 else /* cache is meaningless */
1915 dev->cache_size = 0;
1916 dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
1917 dev->cache_size);
1918 dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
1919
1920 if (dev->zoned &&
1921 (!dev->zone_size || !is_power_of_2(n: dev->zone_size))) {
1922 pr_err("zone_size must be power-of-two\n");
1923 return -EINVAL;
1924 }
1925
1926 return 0;
1927}
1928
1929#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1930static bool __null_setup_fault(struct fault_attr *attr, char *str)
1931{
1932 if (!str[0])
1933 return true;
1934
1935 if (!setup_fault_attr(attr, str))
1936 return false;
1937
1938 attr->verbose = 0;
1939 return true;
1940}
1941#endif
1942
1943static bool null_setup_fault(void)
1944{
1945#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1946 if (!__null_setup_fault(attr: &null_timeout_attr, str: g_timeout_str))
1947 return false;
1948 if (!__null_setup_fault(attr: &null_requeue_attr, str: g_requeue_str))
1949 return false;
1950 if (!__null_setup_fault(attr: &null_init_hctx_attr, str: g_init_hctx_str))
1951 return false;
1952#endif
1953 return true;
1954}
1955
1956static int null_add_dev(struct nullb_device *dev)
1957{
1958 struct queue_limits lim = {
1959 .logical_block_size = dev->blocksize,
1960 .physical_block_size = dev->blocksize,
1961 .max_hw_sectors = dev->max_sectors,
1962 .dma_alignment = 1,
1963 };
1964
1965 struct nullb *nullb;
1966 int rv;
1967
1968 rv = null_validate_conf(dev);
1969 if (rv)
1970 return rv;
1971
1972 nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
1973 if (!nullb) {
1974 rv = -ENOMEM;
1975 goto out;
1976 }
1977 nullb->dev = dev;
1978 dev->nullb = nullb;
1979
1980 spin_lock_init(&nullb->lock);
1981
1982 rv = setup_queues(nullb);
1983 if (rv)
1984 goto out_free_nullb;
1985
1986 rv = null_setup_tagset(nullb);
1987 if (rv)
1988 goto out_cleanup_queues;
1989
1990 if (dev->virt_boundary)
1991 lim.virt_boundary_mask = PAGE_SIZE - 1;
1992 null_config_discard(nullb, lim: &lim);
1993 if (dev->zoned) {
1994 rv = null_init_zoned_dev(dev, lim: &lim);
1995 if (rv)
1996 goto out_cleanup_tags;
1997 }
1998
1999 if (dev->cache_size > 0) {
2000 set_bit(nr: NULLB_DEV_FL_CACHE, addr: &nullb->dev->flags);
2001 lim.features |= BLK_FEAT_WRITE_CACHE;
2002 if (dev->fua)
2003 lim.features |= BLK_FEAT_FUA;
2004 }
2005
2006 if (dev->rotational)
2007 lim.features |= BLK_FEAT_ROTATIONAL;
2008
2009 nullb->disk = blk_mq_alloc_disk(nullb->tag_set, &lim, nullb);
2010 if (IS_ERR(ptr: nullb->disk)) {
2011 rv = PTR_ERR(ptr: nullb->disk);
2012 goto out_cleanup_zone;
2013 }
2014 nullb->q = nullb->disk->queue;
2015
2016 if (dev->mbps) {
2017 set_bit(nr: NULLB_DEV_FL_THROTTLED, addr: &dev->flags);
2018 nullb_setup_bwtimer(nullb);
2019 }
2020
2021 nullb->q->queuedata = nullb;
2022
2023 rv = ida_alloc(ida: &nullb_indexes, GFP_KERNEL);
2024 if (rv < 0)
2025 goto out_cleanup_disk;
2026
2027 nullb->index = rv;
2028 dev->index = rv;
2029
2030 if (config_item_name(item: &dev->group.cg_item)) {
2031 /* Use configfs dir name as the device name */
2032 snprintf(buf: nullb->disk_name, size: sizeof(nullb->disk_name),
2033 fmt: "%s", config_item_name(item: &dev->group.cg_item));
2034 } else {
2035 sprintf(buf: nullb->disk_name, fmt: "nullb%d", nullb->index);
2036 }
2037
2038 set_capacity(disk: nullb->disk,
2039 size: ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT);
2040 nullb->disk->major = null_major;
2041 nullb->disk->first_minor = nullb->index;
2042 nullb->disk->minors = 1;
2043 nullb->disk->fops = &null_ops;
2044 nullb->disk->private_data = nullb;
2045 strscpy(nullb->disk->disk_name, nullb->disk_name);
2046
2047 if (nullb->dev->zoned) {
2048 rv = null_register_zoned_dev(nullb);
2049 if (rv)
2050 goto out_ida_free;
2051 }
2052
2053 rv = add_disk(disk: nullb->disk);
2054 if (rv)
2055 goto out_ida_free;
2056
2057 list_add_tail(new: &nullb->list, head: &nullb_list);
2058
2059 pr_info("disk %s created\n", nullb->disk_name);
2060
2061 return 0;
2062
2063out_ida_free:
2064 ida_free(&nullb_indexes, id: nullb->index);
2065out_cleanup_disk:
2066 put_disk(disk: nullb->disk);
2067out_cleanup_zone:
2068 null_free_zoned_dev(dev);
2069out_cleanup_tags:
2070 if (nullb->tag_set == &nullb->__tag_set)
2071 blk_mq_free_tag_set(set: nullb->tag_set);
2072out_cleanup_queues:
2073 kfree(objp: nullb->queues);
2074out_free_nullb:
2075 kfree(objp: nullb);
2076 dev->nullb = NULL;
2077out:
2078 return rv;
2079}
2080
2081static struct nullb *null_find_dev_by_name(const char *name)
2082{
2083 struct nullb *nullb = NULL, *nb;
2084
2085 mutex_lock(&lock);
2086 list_for_each_entry(nb, &nullb_list, list) {
2087 if (strcmp(nb->disk_name, name) == 0) {
2088 nullb = nb;
2089 break;
2090 }
2091 }
2092 mutex_unlock(lock: &lock);
2093
2094 return nullb;
2095}
2096
2097static int null_create_dev(void)
2098{
2099 struct nullb_device *dev;
2100 int ret;
2101
2102 dev = null_alloc_dev();
2103 if (!dev)
2104 return -ENOMEM;
2105
2106 mutex_lock(&lock);
2107 ret = null_add_dev(dev);
2108 mutex_unlock(lock: &lock);
2109 if (ret) {
2110 null_free_dev(dev);
2111 return ret;
2112 }
2113
2114 return 0;
2115}
2116
2117static void null_destroy_dev(struct nullb *nullb)
2118{
2119 struct nullb_device *dev = nullb->dev;
2120
2121 null_del_dev(nullb);
2122 null_free_device_storage(dev, is_cache: false);
2123 null_free_dev(dev);
2124}
2125
2126static int __init null_init(void)
2127{
2128 int ret = 0;
2129 unsigned int i;
2130 struct nullb *nullb;
2131
2132 if (g_bs > PAGE_SIZE) {
2133 pr_warn("invalid block size\n");
2134 pr_warn("defaults block size to %lu\n", PAGE_SIZE);
2135 g_bs = PAGE_SIZE;
2136 }
2137
2138 if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
2139 pr_err("invalid home_node value\n");
2140 g_home_node = NUMA_NO_NODE;
2141 }
2142
2143 if (!null_setup_fault())
2144 return -EINVAL;
2145
2146 if (g_queue_mode == NULL_Q_RQ) {
2147 pr_err("legacy IO path is no longer available\n");
2148 return -EINVAL;
2149 }
2150
2151 if (g_use_per_node_hctx) {
2152 if (g_submit_queues != nr_online_nodes) {
2153 pr_warn("submit_queues param is set to %u.\n",
2154 nr_online_nodes);
2155 g_submit_queues = nr_online_nodes;
2156 }
2157 } else if (g_submit_queues > nr_cpu_ids) {
2158 g_submit_queues = nr_cpu_ids;
2159 } else if (g_submit_queues <= 0) {
2160 g_submit_queues = 1;
2161 }
2162
2163 config_group_init(group: &nullb_subsys.su_group);
2164 mutex_init(&nullb_subsys.su_mutex);
2165
2166 ret = configfs_register_subsystem(subsys: &nullb_subsys);
2167 if (ret)
2168 return ret;
2169
2170 mutex_init(&lock);
2171
2172 null_major = register_blkdev(0, "nullb");
2173 if (null_major < 0) {
2174 ret = null_major;
2175 goto err_conf;
2176 }
2177
2178 for (i = 0; i < nr_devices; i++) {
2179 ret = null_create_dev();
2180 if (ret)
2181 goto err_dev;
2182 }
2183
2184 pr_info("module loaded\n");
2185 return 0;
2186
2187err_dev:
2188 while (!list_empty(head: &nullb_list)) {
2189 nullb = list_entry(nullb_list.next, struct nullb, list);
2190 null_destroy_dev(nullb);
2191 }
2192 unregister_blkdev(major: null_major, name: "nullb");
2193err_conf:
2194 configfs_unregister_subsystem(subsys: &nullb_subsys);
2195 return ret;
2196}
2197
2198static void __exit null_exit(void)
2199{
2200 struct nullb *nullb;
2201
2202 configfs_unregister_subsystem(subsys: &nullb_subsys);
2203
2204 unregister_blkdev(major: null_major, name: "nullb");
2205
2206 mutex_lock(&lock);
2207 while (!list_empty(head: &nullb_list)) {
2208 nullb = list_entry(nullb_list.next, struct nullb, list);
2209 null_destroy_dev(nullb);
2210 }
2211 mutex_unlock(lock: &lock);
2212
2213 if (tag_set.ops)
2214 blk_mq_free_tag_set(set: &tag_set);
2215
2216 mutex_destroy(lock: &lock);
2217}
2218
2219module_init(null_init);
2220module_exit(null_exit);
2221
2222MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
2223MODULE_DESCRIPTION("multi queue aware block test driver");
2224MODULE_LICENSE("GPL");
2225

source code of linux/drivers/block/null_blk/main.c