1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #define KMSG_COMPONENT "zpci" |
3 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt |
4 | |
5 | #include <linux/kernel.h> |
6 | #include <linux/irq.h> |
7 | #include <linux/kernel_stat.h> |
8 | #include <linux/pci.h> |
9 | #include <linux/msi.h> |
10 | #include <linux/smp.h> |
11 | |
12 | #include <asm/isc.h> |
13 | #include <asm/airq.h> |
14 | #include <asm/tpi.h> |
15 | |
16 | static enum {FLOATING, DIRECTED} irq_delivery; |
17 | |
18 | /* |
19 | * summary bit vector |
20 | * FLOATING - summary bit per function |
21 | * DIRECTED - summary bit per cpu (only used in fallback path) |
22 | */ |
23 | static struct airq_iv *zpci_sbv; |
24 | |
25 | /* |
26 | * interrupt bit vectors |
27 | * FLOATING - interrupt bit vector per function |
28 | * DIRECTED - interrupt bit vector per cpu |
29 | */ |
30 | static struct airq_iv **zpci_ibv; |
31 | |
32 | /* Modify PCI: Register floating adapter interruptions */ |
33 | static int zpci_set_airq(struct zpci_dev *zdev) |
34 | { |
35 | u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT); |
36 | struct zpci_fib fib = {0}; |
37 | u8 status; |
38 | |
39 | fib.fmt0.isc = PCI_ISC; |
40 | fib.fmt0.sum = 1; /* enable summary notifications */ |
41 | fib.fmt0.noi = airq_iv_end(zdev->aibv); |
42 | fib.fmt0.aibv = virt_to_phys(address: zdev->aibv->vector); |
43 | fib.fmt0.aibvo = 0; /* each zdev has its own interrupt vector */ |
44 | fib.fmt0.aisb = virt_to_phys(address: zpci_sbv->vector) + (zdev->aisb / 64) * 8; |
45 | fib.fmt0.aisbo = zdev->aisb & 63; |
46 | fib.gd = zdev->gisa; |
47 | |
48 | return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; |
49 | } |
50 | |
51 | /* Modify PCI: Unregister floating adapter interruptions */ |
52 | static int zpci_clear_airq(struct zpci_dev *zdev) |
53 | { |
54 | u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT); |
55 | struct zpci_fib fib = {0}; |
56 | u8 cc, status; |
57 | |
58 | fib.gd = zdev->gisa; |
59 | |
60 | cc = zpci_mod_fc(req, &fib, &status); |
61 | if (cc == 3 || (cc == 1 && status == 24)) |
62 | /* Function already gone or IRQs already deregistered. */ |
63 | cc = 0; |
64 | |
65 | return cc ? -EIO : 0; |
66 | } |
67 | |
68 | /* Modify PCI: Register CPU directed interruptions */ |
69 | static int zpci_set_directed_irq(struct zpci_dev *zdev) |
70 | { |
71 | u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_REG_INT_D); |
72 | struct zpci_fib fib = {0}; |
73 | u8 status; |
74 | |
75 | fib.fmt = 1; |
76 | fib.fmt1.noi = zdev->msi_nr_irqs; |
77 | fib.fmt1.dibvo = zdev->msi_first_bit; |
78 | fib.gd = zdev->gisa; |
79 | |
80 | return zpci_mod_fc(req, &fib, &status) ? -EIO : 0; |
81 | } |
82 | |
83 | /* Modify PCI: Unregister CPU directed interruptions */ |
84 | static int zpci_clear_directed_irq(struct zpci_dev *zdev) |
85 | { |
86 | u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_DEREG_INT_D); |
87 | struct zpci_fib fib = {0}; |
88 | u8 cc, status; |
89 | |
90 | fib.fmt = 1; |
91 | fib.gd = zdev->gisa; |
92 | cc = zpci_mod_fc(req, &fib, &status); |
93 | if (cc == 3 || (cc == 1 && status == 24)) |
94 | /* Function already gone or IRQs already deregistered. */ |
95 | cc = 0; |
96 | |
97 | return cc ? -EIO : 0; |
98 | } |
99 | |
100 | /* Register adapter interruptions */ |
101 | static int zpci_set_irq(struct zpci_dev *zdev) |
102 | { |
103 | int rc; |
104 | |
105 | if (irq_delivery == DIRECTED) |
106 | rc = zpci_set_directed_irq(zdev); |
107 | else |
108 | rc = zpci_set_airq(zdev); |
109 | |
110 | if (!rc) |
111 | zdev->irqs_registered = 1; |
112 | |
113 | return rc; |
114 | } |
115 | |
116 | /* Clear adapter interruptions */ |
117 | static int zpci_clear_irq(struct zpci_dev *zdev) |
118 | { |
119 | int rc; |
120 | |
121 | if (irq_delivery == DIRECTED) |
122 | rc = zpci_clear_directed_irq(zdev); |
123 | else |
124 | rc = zpci_clear_airq(zdev); |
125 | |
126 | if (!rc) |
127 | zdev->irqs_registered = 0; |
128 | |
129 | return rc; |
130 | } |
131 | |
132 | static int zpci_set_irq_affinity(struct irq_data *data, const struct cpumask *dest, |
133 | bool force) |
134 | { |
135 | struct msi_desc *entry = irq_data_get_msi_desc(d: data); |
136 | struct msi_msg msg = entry->msg; |
137 | int cpu_addr = smp_cpu_get_cpu_address(cpumask_first(srcp: dest)); |
138 | |
139 | msg.address_lo &= 0xff0000ff; |
140 | msg.address_lo |= (cpu_addr << 8); |
141 | pci_write_msi_msg(irq: data->irq, msg: &msg); |
142 | |
143 | return IRQ_SET_MASK_OK; |
144 | } |
145 | |
146 | static struct irq_chip zpci_irq_chip = { |
147 | .name = "PCI-MSI" , |
148 | .irq_unmask = pci_msi_unmask_irq, |
149 | .irq_mask = pci_msi_mask_irq, |
150 | }; |
151 | |
152 | static void zpci_handle_cpu_local_irq(bool rescan) |
153 | { |
154 | struct airq_iv *dibv = zpci_ibv[smp_processor_id()]; |
155 | union zpci_sic_iib iib = {{0}}; |
156 | unsigned long bit; |
157 | int irqs_on = 0; |
158 | |
159 | for (bit = 0;;) { |
160 | /* Scan the directed IRQ bit vector */ |
161 | bit = airq_iv_scan(dibv, bit, airq_iv_end(dibv)); |
162 | if (bit == -1UL) { |
163 | if (!rescan || irqs_on++) |
164 | /* End of second scan with interrupts on. */ |
165 | break; |
166 | /* First scan complete, re-enable interrupts. */ |
167 | if (zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC, &iib)) |
168 | break; |
169 | bit = 0; |
170 | continue; |
171 | } |
172 | inc_irq_stat(IRQIO_MSI); |
173 | generic_handle_irq(irq: airq_iv_get_data(dibv, bit)); |
174 | } |
175 | } |
176 | |
177 | struct cpu_irq_data { |
178 | call_single_data_t csd; |
179 | atomic_t scheduled; |
180 | }; |
181 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_irq_data, irq_data); |
182 | |
183 | static void zpci_handle_remote_irq(void *data) |
184 | { |
185 | atomic_t *scheduled = data; |
186 | |
187 | do { |
188 | zpci_handle_cpu_local_irq(rescan: false); |
189 | } while (atomic_dec_return(v: scheduled)); |
190 | } |
191 | |
192 | static void zpci_handle_fallback_irq(void) |
193 | { |
194 | struct cpu_irq_data *cpu_data; |
195 | union zpci_sic_iib iib = {{0}}; |
196 | unsigned long cpu; |
197 | int irqs_on = 0; |
198 | |
199 | for (cpu = 0;;) { |
200 | cpu = airq_iv_scan(zpci_sbv, cpu, airq_iv_end(zpci_sbv)); |
201 | if (cpu == -1UL) { |
202 | if (irqs_on++) |
203 | /* End of second scan with interrupts on. */ |
204 | break; |
205 | /* First scan complete, re-enable interrupts. */ |
206 | if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC, &iib)) |
207 | break; |
208 | cpu = 0; |
209 | continue; |
210 | } |
211 | cpu_data = &per_cpu(irq_data, cpu); |
212 | if (atomic_inc_return(v: &cpu_data->scheduled) > 1) |
213 | continue; |
214 | |
215 | INIT_CSD(&cpu_data->csd, zpci_handle_remote_irq, &cpu_data->scheduled); |
216 | smp_call_function_single_async(cpu, csd: &cpu_data->csd); |
217 | } |
218 | } |
219 | |
220 | static void zpci_directed_irq_handler(struct airq_struct *airq, |
221 | struct tpi_info *tpi_info) |
222 | { |
223 | bool floating = !tpi_info->directed_irq; |
224 | |
225 | if (floating) { |
226 | inc_irq_stat(IRQIO_PCF); |
227 | zpci_handle_fallback_irq(); |
228 | } else { |
229 | inc_irq_stat(IRQIO_PCD); |
230 | zpci_handle_cpu_local_irq(rescan: true); |
231 | } |
232 | } |
233 | |
234 | static void zpci_floating_irq_handler(struct airq_struct *airq, |
235 | struct tpi_info *tpi_info) |
236 | { |
237 | union zpci_sic_iib iib = {{0}}; |
238 | unsigned long si, ai; |
239 | struct airq_iv *aibv; |
240 | int irqs_on = 0; |
241 | |
242 | inc_irq_stat(IRQIO_PCF); |
243 | for (si = 0;;) { |
244 | /* Scan adapter summary indicator bit vector */ |
245 | si = airq_iv_scan(zpci_sbv, si, airq_iv_end(zpci_sbv)); |
246 | if (si == -1UL) { |
247 | if (irqs_on++) |
248 | /* End of second scan with interrupts on. */ |
249 | break; |
250 | /* First scan complete, re-enable interrupts. */ |
251 | if (zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC, &iib)) |
252 | break; |
253 | si = 0; |
254 | continue; |
255 | } |
256 | |
257 | /* Scan the adapter interrupt vector for this device. */ |
258 | aibv = zpci_ibv[si]; |
259 | for (ai = 0;;) { |
260 | ai = airq_iv_scan(aibv, ai, airq_iv_end(aibv)); |
261 | if (ai == -1UL) |
262 | break; |
263 | inc_irq_stat(IRQIO_MSI); |
264 | airq_iv_lock(aibv, ai); |
265 | generic_handle_irq(irq: airq_iv_get_data(aibv, ai)); |
266 | airq_iv_unlock(aibv, ai); |
267 | } |
268 | } |
269 | } |
270 | |
271 | int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) |
272 | { |
273 | struct zpci_dev *zdev = to_zpci(pdev); |
274 | unsigned int hwirq, msi_vecs, cpu; |
275 | unsigned long bit; |
276 | struct msi_desc *msi; |
277 | struct msi_msg msg; |
278 | int cpu_addr; |
279 | int rc, irq; |
280 | |
281 | zdev->aisb = -1UL; |
282 | zdev->msi_first_bit = -1U; |
283 | if (type == PCI_CAP_ID_MSI && nvec > 1) |
284 | return 1; |
285 | msi_vecs = min_t(unsigned int, nvec, zdev->max_msi); |
286 | |
287 | if (irq_delivery == DIRECTED) { |
288 | /* Allocate cpu vector bits */ |
289 | bit = airq_iv_alloc(zpci_ibv[0], msi_vecs); |
290 | if (bit == -1UL) |
291 | return -EIO; |
292 | } else { |
293 | /* Allocate adapter summary indicator bit */ |
294 | bit = airq_iv_alloc_bit(zpci_sbv); |
295 | if (bit == -1UL) |
296 | return -EIO; |
297 | zdev->aisb = bit; |
298 | |
299 | /* Create adapter interrupt vector */ |
300 | zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK, NULL); |
301 | if (!zdev->aibv) |
302 | return -ENOMEM; |
303 | |
304 | /* Wire up shortcut pointer */ |
305 | zpci_ibv[bit] = zdev->aibv; |
306 | /* Each function has its own interrupt vector */ |
307 | bit = 0; |
308 | } |
309 | |
310 | /* Request MSI interrupts */ |
311 | hwirq = bit; |
312 | msi_for_each_desc(msi, &pdev->dev, MSI_DESC_NOTASSOCIATED) { |
313 | rc = -EIO; |
314 | if (hwirq - bit >= msi_vecs) |
315 | break; |
316 | irq = __irq_alloc_descs(irq: -1, from: 0, cnt: 1, node: 0, THIS_MODULE, |
317 | affinity: (irq_delivery == DIRECTED) ? |
318 | msi->affinity : NULL); |
319 | if (irq < 0) |
320 | return -ENOMEM; |
321 | rc = irq_set_msi_desc(irq, entry: msi); |
322 | if (rc) |
323 | return rc; |
324 | irq_set_chip_and_handler(irq, chip: &zpci_irq_chip, |
325 | handle: handle_percpu_irq); |
326 | msg.data = hwirq - bit; |
327 | if (irq_delivery == DIRECTED) { |
328 | if (msi->affinity) |
329 | cpu = cpumask_first(srcp: &msi->affinity->mask); |
330 | else |
331 | cpu = 0; |
332 | cpu_addr = smp_cpu_get_cpu_address(cpu); |
333 | |
334 | msg.address_lo = zdev->msi_addr & 0xff0000ff; |
335 | msg.address_lo |= (cpu_addr << 8); |
336 | |
337 | for_each_possible_cpu(cpu) { |
338 | airq_iv_set_data(zpci_ibv[cpu], hwirq, irq); |
339 | } |
340 | } else { |
341 | msg.address_lo = zdev->msi_addr & 0xffffffff; |
342 | airq_iv_set_data(zdev->aibv, hwirq, irq); |
343 | } |
344 | msg.address_hi = zdev->msi_addr >> 32; |
345 | pci_write_msi_msg(irq, msg: &msg); |
346 | hwirq++; |
347 | } |
348 | |
349 | zdev->msi_first_bit = bit; |
350 | zdev->msi_nr_irqs = msi_vecs; |
351 | |
352 | rc = zpci_set_irq(zdev); |
353 | if (rc) |
354 | return rc; |
355 | |
356 | return (msi_vecs == nvec) ? 0 : msi_vecs; |
357 | } |
358 | |
359 | void arch_teardown_msi_irqs(struct pci_dev *pdev) |
360 | { |
361 | struct zpci_dev *zdev = to_zpci(pdev); |
362 | struct msi_desc *msi; |
363 | int rc; |
364 | |
365 | /* Disable interrupts */ |
366 | rc = zpci_clear_irq(zdev); |
367 | if (rc) |
368 | return; |
369 | |
370 | /* Release MSI interrupts */ |
371 | msi_for_each_desc(msi, &pdev->dev, MSI_DESC_ASSOCIATED) { |
372 | irq_set_msi_desc(irq: msi->irq, NULL); |
373 | irq_free_desc(irq: msi->irq); |
374 | msi->msg.address_lo = 0; |
375 | msi->msg.address_hi = 0; |
376 | msi->msg.data = 0; |
377 | msi->irq = 0; |
378 | } |
379 | |
380 | if (zdev->aisb != -1UL) { |
381 | zpci_ibv[zdev->aisb] = NULL; |
382 | airq_iv_free_bit(zpci_sbv, zdev->aisb); |
383 | zdev->aisb = -1UL; |
384 | } |
385 | if (zdev->aibv) { |
386 | airq_iv_release(zdev->aibv); |
387 | zdev->aibv = NULL; |
388 | } |
389 | |
390 | if ((irq_delivery == DIRECTED) && zdev->msi_first_bit != -1U) |
391 | airq_iv_free(zpci_ibv[0], zdev->msi_first_bit, zdev->msi_nr_irqs); |
392 | } |
393 | |
394 | bool arch_restore_msi_irqs(struct pci_dev *pdev) |
395 | { |
396 | struct zpci_dev *zdev = to_zpci(pdev); |
397 | |
398 | if (!zdev->irqs_registered) |
399 | zpci_set_irq(zdev); |
400 | return true; |
401 | } |
402 | |
403 | static struct airq_struct zpci_airq = { |
404 | .handler = zpci_floating_irq_handler, |
405 | .isc = PCI_ISC, |
406 | }; |
407 | |
408 | static void __init cpu_enable_directed_irq(void *unused) |
409 | { |
410 | union zpci_sic_iib iib = {{0}}; |
411 | union zpci_sic_iib ziib = {{0}}; |
412 | |
413 | iib.cdiib.dibv_addr = (u64) zpci_ibv[smp_processor_id()]->vector; |
414 | |
415 | zpci_set_irq_ctrl(SIC_IRQ_MODE_SET_CPU, 0, &iib); |
416 | zpci_set_irq_ctrl(SIC_IRQ_MODE_D_SINGLE, PCI_ISC, &ziib); |
417 | } |
418 | |
419 | static int __init zpci_directed_irq_init(void) |
420 | { |
421 | union zpci_sic_iib iib = {{0}}; |
422 | unsigned int cpu; |
423 | |
424 | zpci_sbv = airq_iv_create(num_possible_cpus(), 0, NULL); |
425 | if (!zpci_sbv) |
426 | return -ENOMEM; |
427 | |
428 | iib.diib.isc = PCI_ISC; |
429 | iib.diib.nr_cpus = num_possible_cpus(); |
430 | iib.diib.disb_addr = virt_to_phys(address: zpci_sbv->vector); |
431 | zpci_set_irq_ctrl(SIC_IRQ_MODE_DIRECT, 0, &iib); |
432 | |
433 | zpci_ibv = kcalloc(num_possible_cpus(), size: sizeof(*zpci_ibv), |
434 | GFP_KERNEL); |
435 | if (!zpci_ibv) |
436 | return -ENOMEM; |
437 | |
438 | for_each_possible_cpu(cpu) { |
439 | /* |
440 | * Per CPU IRQ vectors look the same but bit-allocation |
441 | * is only done on the first vector. |
442 | */ |
443 | zpci_ibv[cpu] = airq_iv_create(cache_line_size() * BITS_PER_BYTE, |
444 | AIRQ_IV_DATA | |
445 | AIRQ_IV_CACHELINE | |
446 | (!cpu ? AIRQ_IV_ALLOC : 0), NULL); |
447 | if (!zpci_ibv[cpu]) |
448 | return -ENOMEM; |
449 | } |
450 | on_each_cpu(func: cpu_enable_directed_irq, NULL, wait: 1); |
451 | |
452 | zpci_irq_chip.irq_set_affinity = zpci_set_irq_affinity; |
453 | |
454 | return 0; |
455 | } |
456 | |
457 | static int __init zpci_floating_irq_init(void) |
458 | { |
459 | zpci_ibv = kcalloc(ZPCI_NR_DEVICES, sizeof(*zpci_ibv), GFP_KERNEL); |
460 | if (!zpci_ibv) |
461 | return -ENOMEM; |
462 | |
463 | zpci_sbv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC, NULL); |
464 | if (!zpci_sbv) |
465 | goto out_free; |
466 | |
467 | return 0; |
468 | |
469 | out_free: |
470 | kfree(objp: zpci_ibv); |
471 | return -ENOMEM; |
472 | } |
473 | |
474 | int __init zpci_irq_init(void) |
475 | { |
476 | union zpci_sic_iib iib = {{0}}; |
477 | int rc; |
478 | |
479 | irq_delivery = sclp.has_dirq ? DIRECTED : FLOATING; |
480 | if (s390_pci_force_floating) |
481 | irq_delivery = FLOATING; |
482 | |
483 | if (irq_delivery == DIRECTED) |
484 | zpci_airq.handler = zpci_directed_irq_handler; |
485 | |
486 | rc = register_adapter_interrupt(&zpci_airq); |
487 | if (rc) |
488 | goto out; |
489 | /* Set summary to 1 to be called every time for the ISC. */ |
490 | *zpci_airq.lsi_ptr = 1; |
491 | |
492 | switch (irq_delivery) { |
493 | case FLOATING: |
494 | rc = zpci_floating_irq_init(); |
495 | break; |
496 | case DIRECTED: |
497 | rc = zpci_directed_irq_init(); |
498 | break; |
499 | } |
500 | |
501 | if (rc) |
502 | goto out_airq; |
503 | |
504 | /* |
505 | * Enable floating IRQs (with suppression after one IRQ). When using |
506 | * directed IRQs this enables the fallback path. |
507 | */ |
508 | zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, PCI_ISC, &iib); |
509 | |
510 | return 0; |
511 | out_airq: |
512 | unregister_adapter_interrupt(&zpci_airq); |
513 | out: |
514 | return rc; |
515 | } |
516 | |
517 | void __init zpci_irq_exit(void) |
518 | { |
519 | unsigned int cpu; |
520 | |
521 | if (irq_delivery == DIRECTED) { |
522 | for_each_possible_cpu(cpu) { |
523 | airq_iv_release(zpci_ibv[cpu]); |
524 | } |
525 | } |
526 | kfree(objp: zpci_ibv); |
527 | if (zpci_sbv) |
528 | airq_iv_release(zpci_sbv); |
529 | unregister_adapter_interrupt(&zpci_airq); |
530 | } |
531 | |