1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Functions related to interrupt-poll handling in the block layer. This |
4 | * is similar to NAPI for network devices. |
5 | */ |
6 | #include <linux/kernel.h> |
7 | #include <linux/module.h> |
8 | #include <linux/init.h> |
9 | #include <linux/bio.h> |
10 | #include <linux/interrupt.h> |
11 | #include <linux/cpu.h> |
12 | #include <linux/irq_poll.h> |
13 | #include <linux/delay.h> |
14 | |
15 | static unsigned int irq_poll_budget __read_mostly = 256; |
16 | |
17 | static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); |
18 | |
19 | /** |
20 | * irq_poll_sched - Schedule a run of the iopoll handler |
21 | * @iop: The parent iopoll structure |
22 | * |
23 | * Description: |
24 | * Add this irq_poll structure to the pending poll list and trigger the |
25 | * raise of the blk iopoll softirq. |
26 | **/ |
27 | void irq_poll_sched(struct irq_poll *iop) |
28 | { |
29 | unsigned long flags; |
30 | |
31 | if (test_bit(IRQ_POLL_F_DISABLE, &iop->state)) |
32 | return; |
33 | if (test_and_set_bit(nr: IRQ_POLL_F_SCHED, addr: &iop->state)) |
34 | return; |
35 | |
36 | local_irq_save(flags); |
37 | list_add_tail(new: &iop->list, this_cpu_ptr(&blk_cpu_iopoll)); |
38 | raise_softirq_irqoff(nr: IRQ_POLL_SOFTIRQ); |
39 | local_irq_restore(flags); |
40 | } |
41 | EXPORT_SYMBOL(irq_poll_sched); |
42 | |
43 | /** |
44 | * __irq_poll_complete - Mark this @iop as un-polled again |
45 | * @iop: The parent iopoll structure |
46 | * |
47 | * Description: |
48 | * See irq_poll_complete(). This function must be called with interrupts |
49 | * disabled. |
50 | **/ |
51 | static void __irq_poll_complete(struct irq_poll *iop) |
52 | { |
53 | list_del(entry: &iop->list); |
54 | smp_mb__before_atomic(); |
55 | clear_bit_unlock(nr: IRQ_POLL_F_SCHED, addr: &iop->state); |
56 | } |
57 | |
58 | /** |
59 | * irq_poll_complete - Mark this @iop as un-polled again |
60 | * @iop: The parent iopoll structure |
61 | * |
62 | * Description: |
63 | * If a driver consumes less than the assigned budget in its run of the |
64 | * iopoll handler, it'll end the polled mode by calling this function. The |
65 | * iopoll handler will not be invoked again before irq_poll_sched() |
66 | * is called. |
67 | **/ |
68 | void irq_poll_complete(struct irq_poll *iop) |
69 | { |
70 | unsigned long flags; |
71 | |
72 | local_irq_save(flags); |
73 | __irq_poll_complete(iop); |
74 | local_irq_restore(flags); |
75 | } |
76 | EXPORT_SYMBOL(irq_poll_complete); |
77 | |
78 | static void __latent_entropy irq_poll_softirq(struct softirq_action *h) |
79 | { |
80 | struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); |
81 | int rearm = 0, budget = irq_poll_budget; |
82 | unsigned long start_time = jiffies; |
83 | |
84 | local_irq_disable(); |
85 | |
86 | while (!list_empty(head: list)) { |
87 | struct irq_poll *iop; |
88 | int work, weight; |
89 | |
90 | /* |
91 | * If softirq window is exhausted then punt. |
92 | */ |
93 | if (budget <= 0 || time_after(jiffies, start_time)) { |
94 | rearm = 1; |
95 | break; |
96 | } |
97 | |
98 | local_irq_enable(); |
99 | |
100 | /* Even though interrupts have been re-enabled, this |
101 | * access is safe because interrupts can only add new |
102 | * entries to the tail of this list, and only ->poll() |
103 | * calls can remove this head entry from the list. |
104 | */ |
105 | iop = list_entry(list->next, struct irq_poll, list); |
106 | |
107 | weight = iop->weight; |
108 | work = 0; |
109 | if (test_bit(IRQ_POLL_F_SCHED, &iop->state)) |
110 | work = iop->poll(iop, weight); |
111 | |
112 | budget -= work; |
113 | |
114 | local_irq_disable(); |
115 | |
116 | /* |
117 | * Drivers must not modify the iopoll state, if they |
118 | * consume their assigned weight (or more, some drivers can't |
119 | * easily just stop processing, they have to complete an |
120 | * entire mask of commands).In such cases this code |
121 | * still "owns" the iopoll instance and therefore can |
122 | * move the instance around on the list at-will. |
123 | */ |
124 | if (work >= weight) { |
125 | if (test_bit(IRQ_POLL_F_DISABLE, &iop->state)) |
126 | __irq_poll_complete(iop); |
127 | else |
128 | list_move_tail(list: &iop->list, head: list); |
129 | } |
130 | } |
131 | |
132 | if (rearm) |
133 | __raise_softirq_irqoff(nr: IRQ_POLL_SOFTIRQ); |
134 | |
135 | local_irq_enable(); |
136 | } |
137 | |
138 | /** |
139 | * irq_poll_disable - Disable iopoll on this @iop |
140 | * @iop: The parent iopoll structure |
141 | * |
142 | * Description: |
143 | * Disable io polling and wait for any pending callbacks to have completed. |
144 | **/ |
145 | void irq_poll_disable(struct irq_poll *iop) |
146 | { |
147 | set_bit(nr: IRQ_POLL_F_DISABLE, addr: &iop->state); |
148 | while (test_and_set_bit(nr: IRQ_POLL_F_SCHED, addr: &iop->state)) |
149 | msleep(msecs: 1); |
150 | clear_bit(nr: IRQ_POLL_F_DISABLE, addr: &iop->state); |
151 | } |
152 | EXPORT_SYMBOL(irq_poll_disable); |
153 | |
154 | /** |
155 | * irq_poll_enable - Enable iopoll on this @iop |
156 | * @iop: The parent iopoll structure |
157 | * |
158 | * Description: |
159 | * Enable iopoll on this @iop. Note that the handler run will not be |
160 | * scheduled, it will only mark it as active. |
161 | **/ |
162 | void irq_poll_enable(struct irq_poll *iop) |
163 | { |
164 | BUG_ON(!test_bit(IRQ_POLL_F_SCHED, &iop->state)); |
165 | smp_mb__before_atomic(); |
166 | clear_bit_unlock(nr: IRQ_POLL_F_SCHED, addr: &iop->state); |
167 | } |
168 | EXPORT_SYMBOL(irq_poll_enable); |
169 | |
170 | /** |
171 | * irq_poll_init - Initialize this @iop |
172 | * @iop: The parent iopoll structure |
173 | * @weight: The default weight (or command completion budget) |
174 | * @poll_fn: The handler to invoke |
175 | * |
176 | * Description: |
177 | * Initialize and enable this irq_poll structure. |
178 | **/ |
179 | void irq_poll_init(struct irq_poll *iop, int weight, irq_poll_fn *poll_fn) |
180 | { |
181 | memset(iop, 0, sizeof(*iop)); |
182 | INIT_LIST_HEAD(list: &iop->list); |
183 | iop->weight = weight; |
184 | iop->poll = poll_fn; |
185 | } |
186 | EXPORT_SYMBOL(irq_poll_init); |
187 | |
188 | static int irq_poll_cpu_dead(unsigned int cpu) |
189 | { |
190 | /* |
191 | * If a CPU goes away, splice its entries to the current CPU and |
192 | * set the POLL softirq bit. The local_bh_disable()/enable() pair |
193 | * ensures that it is handled. Otherwise the current CPU could |
194 | * reach idle with the POLL softirq pending. |
195 | */ |
196 | local_bh_disable(); |
197 | local_irq_disable(); |
198 | list_splice_init(list: &per_cpu(blk_cpu_iopoll, cpu), |
199 | this_cpu_ptr(&blk_cpu_iopoll)); |
200 | __raise_softirq_irqoff(nr: IRQ_POLL_SOFTIRQ); |
201 | local_irq_enable(); |
202 | local_bh_enable(); |
203 | |
204 | return 0; |
205 | } |
206 | |
207 | static __init int irq_poll_setup(void) |
208 | { |
209 | int i; |
210 | |
211 | for_each_possible_cpu(i) |
212 | INIT_LIST_HEAD(list: &per_cpu(blk_cpu_iopoll, i)); |
213 | |
214 | open_softirq(nr: IRQ_POLL_SOFTIRQ, action: irq_poll_softirq); |
215 | cpuhp_setup_state_nocalls(state: CPUHP_IRQ_POLL_DEAD, name: "irq_poll:dead" , NULL, |
216 | teardown: irq_poll_cpu_dead); |
217 | return 0; |
218 | } |
219 | subsys_initcall(irq_poll_setup); |
220 | |