1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright 2008 Cisco Systems, Inc. All rights reserved. |
4 | * Copyright 2007 Nuova Systems, Inc. All rights reserved. |
5 | */ |
6 | #include <linux/string.h> |
7 | #include <linux/errno.h> |
8 | #include <linux/pci.h> |
9 | #include <linux/interrupt.h> |
10 | #include <scsi/libfc.h> |
11 | #include <scsi/fc_frame.h> |
12 | #include "vnic_dev.h" |
13 | #include "vnic_intr.h" |
14 | #include "vnic_stats.h" |
15 | #include "fnic_io.h" |
16 | #include "fnic.h" |
17 | |
18 | static irqreturn_t fnic_isr_legacy(int irq, void *data) |
19 | { |
20 | struct fnic *fnic = data; |
21 | u32 pba; |
22 | unsigned long work_done = 0; |
23 | |
24 | pba = vnic_intr_legacy_pba(legacy_pba: fnic->legacy_pba); |
25 | if (!pba) |
26 | return IRQ_NONE; |
27 | |
28 | fnic->fnic_stats.misc_stats.last_isr_time = jiffies; |
29 | atomic64_inc(v: &fnic->fnic_stats.misc_stats.isr_count); |
30 | |
31 | if (pba & (1 << FNIC_INTX_NOTIFY)) { |
32 | vnic_intr_return_all_credits(intr: &fnic->intr[FNIC_INTX_NOTIFY]); |
33 | fnic_handle_link_event(fnic); |
34 | } |
35 | |
36 | if (pba & (1 << FNIC_INTX_ERR)) { |
37 | vnic_intr_return_all_credits(intr: &fnic->intr[FNIC_INTX_ERR]); |
38 | fnic_log_q_error(fnic); |
39 | } |
40 | |
41 | if (pba & (1 << FNIC_INTX_DUMMY)) { |
42 | atomic64_inc(v: &fnic->fnic_stats.misc_stats.intx_dummy); |
43 | vnic_intr_return_all_credits(intr: &fnic->intr[FNIC_INTX_DUMMY]); |
44 | } |
45 | |
46 | if (pba & (1 << FNIC_INTX_WQ_RQ_COPYWQ)) { |
47 | work_done += fnic_wq_copy_cmpl_handler(fnic, copy_work_to_do: io_completions, FNIC_MQ_CQ_INDEX); |
48 | work_done += fnic_wq_cmpl_handler(fnic, -1); |
49 | work_done += fnic_rq_cmpl_handler(fnic, -1); |
50 | |
51 | vnic_intr_return_credits(intr: &fnic->intr[FNIC_INTX_WQ_RQ_COPYWQ], |
52 | credits: work_done, |
53 | unmask: 1 /* unmask intr */, |
54 | reset_timer: 1 /* reset intr timer */); |
55 | } |
56 | |
57 | return IRQ_HANDLED; |
58 | } |
59 | |
60 | static irqreturn_t fnic_isr_msi(int irq, void *data) |
61 | { |
62 | struct fnic *fnic = data; |
63 | unsigned long work_done = 0; |
64 | |
65 | fnic->fnic_stats.misc_stats.last_isr_time = jiffies; |
66 | atomic64_inc(v: &fnic->fnic_stats.misc_stats.isr_count); |
67 | |
68 | work_done += fnic_wq_copy_cmpl_handler(fnic, copy_work_to_do: io_completions, FNIC_MQ_CQ_INDEX); |
69 | work_done += fnic_wq_cmpl_handler(fnic, -1); |
70 | work_done += fnic_rq_cmpl_handler(fnic, -1); |
71 | |
72 | vnic_intr_return_credits(intr: &fnic->intr[0], |
73 | credits: work_done, |
74 | unmask: 1 /* unmask intr */, |
75 | reset_timer: 1 /* reset intr timer */); |
76 | |
77 | return IRQ_HANDLED; |
78 | } |
79 | |
80 | static irqreturn_t fnic_isr_msix_rq(int irq, void *data) |
81 | { |
82 | struct fnic *fnic = data; |
83 | unsigned long rq_work_done = 0; |
84 | |
85 | fnic->fnic_stats.misc_stats.last_isr_time = jiffies; |
86 | atomic64_inc(v: &fnic->fnic_stats.misc_stats.isr_count); |
87 | |
88 | rq_work_done = fnic_rq_cmpl_handler(fnic, -1); |
89 | vnic_intr_return_credits(intr: &fnic->intr[FNIC_MSIX_RQ], |
90 | credits: rq_work_done, |
91 | unmask: 1 /* unmask intr */, |
92 | reset_timer: 1 /* reset intr timer */); |
93 | |
94 | return IRQ_HANDLED; |
95 | } |
96 | |
97 | static irqreturn_t fnic_isr_msix_wq(int irq, void *data) |
98 | { |
99 | struct fnic *fnic = data; |
100 | unsigned long wq_work_done = 0; |
101 | |
102 | fnic->fnic_stats.misc_stats.last_isr_time = jiffies; |
103 | atomic64_inc(v: &fnic->fnic_stats.misc_stats.isr_count); |
104 | |
105 | wq_work_done = fnic_wq_cmpl_handler(fnic, -1); |
106 | vnic_intr_return_credits(intr: &fnic->intr[FNIC_MSIX_WQ], |
107 | credits: wq_work_done, |
108 | unmask: 1 /* unmask intr */, |
109 | reset_timer: 1 /* reset intr timer */); |
110 | return IRQ_HANDLED; |
111 | } |
112 | |
113 | static irqreturn_t fnic_isr_msix_wq_copy(int irq, void *data) |
114 | { |
115 | struct fnic *fnic = data; |
116 | unsigned long wq_copy_work_done = 0; |
117 | int i; |
118 | |
119 | fnic->fnic_stats.misc_stats.last_isr_time = jiffies; |
120 | atomic64_inc(v: &fnic->fnic_stats.misc_stats.isr_count); |
121 | |
122 | i = irq - fnic->msix[0].irq_num; |
123 | if (i >= fnic->wq_copy_count + fnic->copy_wq_base || |
124 | i < 0 || fnic->msix[i].irq_num != irq) { |
125 | for (i = fnic->copy_wq_base; i < fnic->wq_copy_count + fnic->copy_wq_base ; i++) { |
126 | if (fnic->msix[i].irq_num == irq) |
127 | break; |
128 | } |
129 | } |
130 | |
131 | wq_copy_work_done = fnic_wq_copy_cmpl_handler(fnic, copy_work_to_do: io_completions, cq_index: i); |
132 | vnic_intr_return_credits(intr: &fnic->intr[i], |
133 | credits: wq_copy_work_done, |
134 | unmask: 1 /* unmask intr */, |
135 | reset_timer: 1 /* reset intr timer */); |
136 | return IRQ_HANDLED; |
137 | } |
138 | |
139 | static irqreturn_t fnic_isr_msix_err_notify(int irq, void *data) |
140 | { |
141 | struct fnic *fnic = data; |
142 | |
143 | fnic->fnic_stats.misc_stats.last_isr_time = jiffies; |
144 | atomic64_inc(v: &fnic->fnic_stats.misc_stats.isr_count); |
145 | |
146 | vnic_intr_return_all_credits(intr: &fnic->intr[fnic->err_intr_offset]); |
147 | fnic_log_q_error(fnic); |
148 | fnic_handle_link_event(fnic); |
149 | |
150 | return IRQ_HANDLED; |
151 | } |
152 | |
153 | void fnic_free_intr(struct fnic *fnic) |
154 | { |
155 | int i; |
156 | |
157 | switch (vnic_dev_get_intr_mode(vdev: fnic->vdev)) { |
158 | case VNIC_DEV_INTR_MODE_INTX: |
159 | case VNIC_DEV_INTR_MODE_MSI: |
160 | free_irq(pci_irq_vector(dev: fnic->pdev, nr: 0), fnic); |
161 | break; |
162 | |
163 | case VNIC_DEV_INTR_MODE_MSIX: |
164 | for (i = 0; i < ARRAY_SIZE(fnic->msix); i++) |
165 | if (fnic->msix[i].requested) |
166 | free_irq(pci_irq_vector(dev: fnic->pdev, nr: i), |
167 | fnic->msix[i].devid); |
168 | break; |
169 | |
170 | default: |
171 | break; |
172 | } |
173 | } |
174 | |
175 | int fnic_request_intr(struct fnic *fnic) |
176 | { |
177 | int err = 0; |
178 | int i; |
179 | |
180 | switch (vnic_dev_get_intr_mode(vdev: fnic->vdev)) { |
181 | |
182 | case VNIC_DEV_INTR_MODE_INTX: |
183 | err = request_irq(irq: pci_irq_vector(dev: fnic->pdev, nr: 0), |
184 | handler: &fnic_isr_legacy, IRQF_SHARED, DRV_NAME, dev: fnic); |
185 | break; |
186 | |
187 | case VNIC_DEV_INTR_MODE_MSI: |
188 | err = request_irq(irq: pci_irq_vector(dev: fnic->pdev, nr: 0), handler: &fnic_isr_msi, |
189 | flags: 0, name: fnic->name, dev: fnic); |
190 | break; |
191 | |
192 | case VNIC_DEV_INTR_MODE_MSIX: |
193 | |
194 | sprintf(buf: fnic->msix[FNIC_MSIX_RQ].devname, |
195 | fmt: "%.11s-fcs-rq" , fnic->name); |
196 | fnic->msix[FNIC_MSIX_RQ].isr = fnic_isr_msix_rq; |
197 | fnic->msix[FNIC_MSIX_RQ].devid = fnic; |
198 | |
199 | sprintf(buf: fnic->msix[FNIC_MSIX_WQ].devname, |
200 | fmt: "%.11s-fcs-wq" , fnic->name); |
201 | fnic->msix[FNIC_MSIX_WQ].isr = fnic_isr_msix_wq; |
202 | fnic->msix[FNIC_MSIX_WQ].devid = fnic; |
203 | |
204 | for (i = fnic->copy_wq_base; i < fnic->wq_copy_count + fnic->copy_wq_base; i++) { |
205 | sprintf(buf: fnic->msix[i].devname, |
206 | fmt: "%.11s-scsi-wq-%d" , fnic->name, i-FNIC_MSIX_WQ_COPY); |
207 | fnic->msix[i].isr = fnic_isr_msix_wq_copy; |
208 | fnic->msix[i].devid = fnic; |
209 | } |
210 | |
211 | sprintf(buf: fnic->msix[fnic->err_intr_offset].devname, |
212 | fmt: "%.11s-err-notify" , fnic->name); |
213 | fnic->msix[fnic->err_intr_offset].isr = |
214 | fnic_isr_msix_err_notify; |
215 | fnic->msix[fnic->err_intr_offset].devid = fnic; |
216 | |
217 | for (i = 0; i < fnic->intr_count; i++) { |
218 | fnic->msix[i].irq_num = pci_irq_vector(dev: fnic->pdev, nr: i); |
219 | |
220 | err = request_irq(irq: fnic->msix[i].irq_num, |
221 | handler: fnic->msix[i].isr, flags: 0, |
222 | name: fnic->msix[i].devname, |
223 | dev: fnic->msix[i].devid); |
224 | if (err) { |
225 | FNIC_ISR_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, |
226 | "request_irq failed with error: %d\n" , |
227 | err); |
228 | fnic_free_intr(fnic); |
229 | break; |
230 | } |
231 | fnic->msix[i].requested = 1; |
232 | } |
233 | break; |
234 | |
235 | default: |
236 | break; |
237 | } |
238 | |
239 | return err; |
240 | } |
241 | |
242 | int fnic_set_intr_mode_msix(struct fnic *fnic) |
243 | { |
244 | unsigned int n = ARRAY_SIZE(fnic->rq); |
245 | unsigned int m = ARRAY_SIZE(fnic->wq); |
246 | unsigned int o = ARRAY_SIZE(fnic->hw_copy_wq); |
247 | unsigned int min_irqs = n + m + 1 + 1; /*rq, raw wq, wq, err*/ |
248 | |
249 | /* |
250 | * We need n RQs, m WQs, o Copy WQs, n+m+o CQs, and n+m+o+1 INTRs |
251 | * (last INTR is used for WQ/RQ errors and notification area) |
252 | */ |
253 | FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, |
254 | "rq-array size: %d wq-array size: %d copy-wq array size: %d\n" , |
255 | n, m, o); |
256 | FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, |
257 | "rq_count: %d raw_wq_count: %d wq_copy_count: %d cq_count: %d\n" , |
258 | fnic->rq_count, fnic->raw_wq_count, |
259 | fnic->wq_copy_count, fnic->cq_count); |
260 | |
261 | if (fnic->rq_count <= n && fnic->raw_wq_count <= m && |
262 | fnic->wq_copy_count <= o) { |
263 | int vec_count = 0; |
264 | int vecs = fnic->rq_count + fnic->raw_wq_count + fnic->wq_copy_count + 1; |
265 | |
266 | vec_count = pci_alloc_irq_vectors(dev: fnic->pdev, min_vecs: min_irqs, max_vecs: vecs, |
267 | PCI_IRQ_MSIX | PCI_IRQ_AFFINITY); |
268 | FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, |
269 | "allocated %d MSI-X vectors\n" , |
270 | vec_count); |
271 | |
272 | if (vec_count > 0) { |
273 | if (vec_count < vecs) { |
274 | FNIC_ISR_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, |
275 | "interrupts number mismatch: vec_count: %d vecs: %d\n" , |
276 | vec_count, vecs); |
277 | if (vec_count < min_irqs) { |
278 | FNIC_ISR_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, |
279 | "no interrupts for copy wq\n" ); |
280 | return 1; |
281 | } |
282 | } |
283 | |
284 | fnic->rq_count = n; |
285 | fnic->raw_wq_count = m; |
286 | fnic->copy_wq_base = fnic->rq_count + fnic->raw_wq_count; |
287 | fnic->wq_copy_count = vec_count - n - m - 1; |
288 | fnic->wq_count = fnic->raw_wq_count + fnic->wq_copy_count; |
289 | if (fnic->cq_count != vec_count - 1) { |
290 | FNIC_ISR_DBG(KERN_ERR, fnic->lport->host, fnic->fnic_num, |
291 | "CQ count: %d does not match MSI-X vector count: %d\n" , |
292 | fnic->cq_count, vec_count); |
293 | fnic->cq_count = vec_count - 1; |
294 | } |
295 | fnic->intr_count = vec_count; |
296 | fnic->err_intr_offset = fnic->rq_count + fnic->wq_count; |
297 | |
298 | FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, |
299 | "rq_count: %d raw_wq_count: %d copy_wq_base: %d\n" , |
300 | fnic->rq_count, |
301 | fnic->raw_wq_count, fnic->copy_wq_base); |
302 | |
303 | FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, |
304 | "wq_copy_count: %d wq_count: %d cq_count: %d\n" , |
305 | fnic->wq_copy_count, |
306 | fnic->wq_count, fnic->cq_count); |
307 | |
308 | FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, |
309 | "intr_count: %d err_intr_offset: %u" , |
310 | fnic->intr_count, |
311 | fnic->err_intr_offset); |
312 | |
313 | vnic_dev_set_intr_mode(vdev: fnic->vdev, intr_mode: VNIC_DEV_INTR_MODE_MSIX); |
314 | FNIC_ISR_DBG(KERN_INFO, fnic->lport->host, fnic->fnic_num, |
315 | "fnic using MSI-X\n" ); |
316 | return 0; |
317 | } |
318 | } |
319 | return 1; |
320 | } |
321 | |
322 | int fnic_set_intr_mode(struct fnic *fnic) |
323 | { |
324 | int ret_status = 0; |
325 | |
326 | /* |
327 | * Set interrupt mode (INTx, MSI, MSI-X) depending |
328 | * system capabilities. |
329 | * |
330 | * Try MSI-X first |
331 | */ |
332 | ret_status = fnic_set_intr_mode_msix(fnic); |
333 | if (ret_status == 0) |
334 | return ret_status; |
335 | |
336 | /* |
337 | * Next try MSI |
338 | * We need 1 RQ, 1 WQ, 1 WQ_COPY, 3 CQs, and 1 INTR |
339 | */ |
340 | if (fnic->rq_count >= 1 && |
341 | fnic->raw_wq_count >= 1 && |
342 | fnic->wq_copy_count >= 1 && |
343 | fnic->cq_count >= 3 && |
344 | fnic->intr_count >= 1 && |
345 | pci_alloc_irq_vectors(dev: fnic->pdev, min_vecs: 1, max_vecs: 1, PCI_IRQ_MSI) == 1) { |
346 | fnic->rq_count = 1; |
347 | fnic->raw_wq_count = 1; |
348 | fnic->wq_copy_count = 1; |
349 | fnic->wq_count = 2; |
350 | fnic->cq_count = 3; |
351 | fnic->intr_count = 1; |
352 | fnic->err_intr_offset = 0; |
353 | |
354 | FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, |
355 | "Using MSI Interrupts\n" ); |
356 | vnic_dev_set_intr_mode(vdev: fnic->vdev, intr_mode: VNIC_DEV_INTR_MODE_MSI); |
357 | |
358 | return 0; |
359 | } |
360 | |
361 | /* |
362 | * Next try INTx |
363 | * We need 1 RQ, 1 WQ, 1 WQ_COPY, 3 CQs, and 3 INTRs |
364 | * 1 INTR is used for all 3 queues, 1 INTR for queue errors |
365 | * 1 INTR for notification area |
366 | */ |
367 | |
368 | if (fnic->rq_count >= 1 && |
369 | fnic->raw_wq_count >= 1 && |
370 | fnic->wq_copy_count >= 1 && |
371 | fnic->cq_count >= 3 && |
372 | fnic->intr_count >= 3) { |
373 | |
374 | fnic->rq_count = 1; |
375 | fnic->raw_wq_count = 1; |
376 | fnic->wq_copy_count = 1; |
377 | fnic->cq_count = 3; |
378 | fnic->intr_count = 3; |
379 | |
380 | FNIC_ISR_DBG(KERN_DEBUG, fnic->lport->host, fnic->fnic_num, |
381 | "Using Legacy Interrupts\n" ); |
382 | vnic_dev_set_intr_mode(vdev: fnic->vdev, intr_mode: VNIC_DEV_INTR_MODE_INTX); |
383 | |
384 | return 0; |
385 | } |
386 | |
387 | vnic_dev_set_intr_mode(vdev: fnic->vdev, intr_mode: VNIC_DEV_INTR_MODE_UNKNOWN); |
388 | |
389 | return -EINVAL; |
390 | } |
391 | |
392 | void fnic_clear_intr_mode(struct fnic *fnic) |
393 | { |
394 | pci_free_irq_vectors(dev: fnic->pdev); |
395 | vnic_dev_set_intr_mode(vdev: fnic->vdev, intr_mode: VNIC_DEV_INTR_MODE_INTX); |
396 | } |
397 | |