1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* |
3 | * APM X-Gene MSI Driver |
4 | * |
5 | * Copyright (c) 2014, Applied Micro Circuits Corporation |
6 | * Author: Tanmay Inamdar <tinamdar@apm.com> |
7 | * Duc Dang <dhdang@apm.com> |
8 | */ |
9 | #include <linux/cpu.h> |
10 | #include <linux/interrupt.h> |
11 | #include <linux/irqdomain.h> |
12 | #include <linux/module.h> |
13 | #include <linux/msi.h> |
14 | #include <linux/irqchip/chained_irq.h> |
15 | #include <linux/pci.h> |
16 | #include <linux/platform_device.h> |
17 | #include <linux/of_pci.h> |
18 | |
19 | #define MSI_IR0 0x000000 |
20 | #define MSI_INT0 0x800000 |
21 | #define IDX_PER_GROUP 8 |
22 | #define IRQS_PER_IDX 16 |
23 | #define NR_HW_IRQS 16 |
24 | #define NR_MSI_VEC (IDX_PER_GROUP * IRQS_PER_IDX * NR_HW_IRQS) |
25 | |
26 | struct xgene_msi_group { |
27 | struct xgene_msi *msi; |
28 | int gic_irq; |
29 | u32 msi_grp; |
30 | }; |
31 | |
32 | struct xgene_msi { |
33 | struct device_node *node; |
34 | struct irq_domain *inner_domain; |
35 | struct irq_domain *msi_domain; |
36 | u64 msi_addr; |
37 | void __iomem *msi_regs; |
38 | unsigned long *bitmap; |
39 | struct mutex bitmap_lock; |
40 | struct xgene_msi_group *msi_groups; |
41 | int num_cpus; |
42 | }; |
43 | |
44 | /* Global data */ |
45 | static struct xgene_msi xgene_msi_ctrl; |
46 | |
47 | static struct irq_chip xgene_msi_top_irq_chip = { |
48 | .name = "X-Gene1 MSI" , |
49 | .irq_enable = pci_msi_unmask_irq, |
50 | .irq_disable = pci_msi_mask_irq, |
51 | .irq_mask = pci_msi_mask_irq, |
52 | .irq_unmask = pci_msi_unmask_irq, |
53 | }; |
54 | |
55 | static struct msi_domain_info xgene_msi_domain_info = { |
56 | .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | |
57 | MSI_FLAG_PCI_MSIX), |
58 | .chip = &xgene_msi_top_irq_chip, |
59 | }; |
60 | |
61 | /* |
62 | * X-Gene v1 has 16 groups of MSI termination registers MSInIRx, where |
63 | * n is group number (0..F), x is index of registers in each group (0..7) |
64 | * The register layout is as follows: |
65 | * MSI0IR0 base_addr |
66 | * MSI0IR1 base_addr + 0x10000 |
67 | * ... ... |
68 | * MSI0IR6 base_addr + 0x60000 |
69 | * MSI0IR7 base_addr + 0x70000 |
70 | * MSI1IR0 base_addr + 0x80000 |
71 | * MSI1IR1 base_addr + 0x90000 |
72 | * ... ... |
73 | * MSI1IR7 base_addr + 0xF0000 |
74 | * MSI2IR0 base_addr + 0x100000 |
75 | * ... ... |
76 | * MSIFIR0 base_addr + 0x780000 |
77 | * MSIFIR1 base_addr + 0x790000 |
78 | * ... ... |
79 | * MSIFIR7 base_addr + 0x7F0000 |
80 | * MSIINT0 base_addr + 0x800000 |
81 | * MSIINT1 base_addr + 0x810000 |
82 | * ... ... |
83 | * MSIINTF base_addr + 0x8F0000 |
84 | * |
85 | * Each index register supports 16 MSI vectors (0..15) to generate interrupt. |
86 | * There are total 16 GIC IRQs assigned for these 16 groups of MSI termination |
87 | * registers. |
88 | * |
89 | * Each MSI termination group has 1 MSIINTn register (n is 0..15) to indicate |
90 | * the MSI pending status caused by 1 of its 8 index registers. |
91 | */ |
92 | |
93 | /* MSInIRx read helper */ |
94 | static u32 xgene_msi_ir_read(struct xgene_msi *msi, |
95 | u32 msi_grp, u32 msir_idx) |
96 | { |
97 | return readl_relaxed(msi->msi_regs + MSI_IR0 + |
98 | (msi_grp << 19) + (msir_idx << 16)); |
99 | } |
100 | |
101 | /* MSIINTn read helper */ |
102 | static u32 xgene_msi_int_read(struct xgene_msi *msi, u32 msi_grp) |
103 | { |
104 | return readl_relaxed(msi->msi_regs + MSI_INT0 + (msi_grp << 16)); |
105 | } |
106 | |
107 | /* |
108 | * With 2048 MSI vectors supported, the MSI message can be constructed using |
109 | * following scheme: |
110 | * - Divide into 8 256-vector groups |
111 | * Group 0: 0-255 |
112 | * Group 1: 256-511 |
113 | * Group 2: 512-767 |
114 | * ... |
115 | * Group 7: 1792-2047 |
116 | * - Each 256-vector group is divided into 16 16-vector groups |
117 | * As an example: 16 16-vector groups for 256-vector group 0-255 is |
118 | * Group 0: 0-15 |
119 | * Group 1: 16-32 |
120 | * ... |
121 | * Group 15: 240-255 |
122 | * - The termination address of MSI vector in 256-vector group n and 16-vector |
123 | * group x is the address of MSIxIRn |
124 | * - The data for MSI vector in 16-vector group x is x |
125 | */ |
126 | static u32 hwirq_to_reg_set(unsigned long hwirq) |
127 | { |
128 | return (hwirq / (NR_HW_IRQS * IRQS_PER_IDX)); |
129 | } |
130 | |
131 | static u32 hwirq_to_group(unsigned long hwirq) |
132 | { |
133 | return (hwirq % NR_HW_IRQS); |
134 | } |
135 | |
136 | static u32 hwirq_to_msi_data(unsigned long hwirq) |
137 | { |
138 | return ((hwirq / NR_HW_IRQS) % IRQS_PER_IDX); |
139 | } |
140 | |
141 | static void xgene_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) |
142 | { |
143 | struct xgene_msi *msi = irq_data_get_irq_chip_data(d: data); |
144 | u32 reg_set = hwirq_to_reg_set(hwirq: data->hwirq); |
145 | u32 group = hwirq_to_group(hwirq: data->hwirq); |
146 | u64 target_addr = msi->msi_addr + (((8 * group) + reg_set) << 16); |
147 | |
148 | msg->address_hi = upper_32_bits(target_addr); |
149 | msg->address_lo = lower_32_bits(target_addr); |
150 | msg->data = hwirq_to_msi_data(hwirq: data->hwirq); |
151 | } |
152 | |
153 | /* |
154 | * X-Gene v1 only has 16 MSI GIC IRQs for 2048 MSI vectors. To maintain |
155 | * the expected behaviour of .set_affinity for each MSI interrupt, the 16 |
156 | * MSI GIC IRQs are statically allocated to 8 X-Gene v1 cores (2 GIC IRQs |
157 | * for each core). The MSI vector is moved fom 1 MSI GIC IRQ to another |
158 | * MSI GIC IRQ to steer its MSI interrupt to correct X-Gene v1 core. As a |
159 | * consequence, the total MSI vectors that X-Gene v1 supports will be |
160 | * reduced to 256 (2048/8) vectors. |
161 | */ |
162 | static int hwirq_to_cpu(unsigned long hwirq) |
163 | { |
164 | return (hwirq % xgene_msi_ctrl.num_cpus); |
165 | } |
166 | |
167 | static unsigned long hwirq_to_canonical_hwirq(unsigned long hwirq) |
168 | { |
169 | return (hwirq - hwirq_to_cpu(hwirq)); |
170 | } |
171 | |
172 | static int xgene_msi_set_affinity(struct irq_data *irqdata, |
173 | const struct cpumask *mask, bool force) |
174 | { |
175 | int target_cpu = cpumask_first(srcp: mask); |
176 | int curr_cpu; |
177 | |
178 | curr_cpu = hwirq_to_cpu(hwirq: irqdata->hwirq); |
179 | if (curr_cpu == target_cpu) |
180 | return IRQ_SET_MASK_OK_DONE; |
181 | |
182 | /* Update MSI number to target the new CPU */ |
183 | irqdata->hwirq = hwirq_to_canonical_hwirq(hwirq: irqdata->hwirq) + target_cpu; |
184 | |
185 | return IRQ_SET_MASK_OK; |
186 | } |
187 | |
188 | static struct irq_chip xgene_msi_bottom_irq_chip = { |
189 | .name = "MSI" , |
190 | .irq_set_affinity = xgene_msi_set_affinity, |
191 | .irq_compose_msi_msg = xgene_compose_msi_msg, |
192 | }; |
193 | |
194 | static int xgene_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, |
195 | unsigned int nr_irqs, void *args) |
196 | { |
197 | struct xgene_msi *msi = domain->host_data; |
198 | int msi_irq; |
199 | |
200 | mutex_lock(&msi->bitmap_lock); |
201 | |
202 | msi_irq = bitmap_find_next_zero_area(map: msi->bitmap, NR_MSI_VEC, start: 0, |
203 | nr: msi->num_cpus, align_mask: 0); |
204 | if (msi_irq < NR_MSI_VEC) |
205 | bitmap_set(map: msi->bitmap, start: msi_irq, nbits: msi->num_cpus); |
206 | else |
207 | msi_irq = -ENOSPC; |
208 | |
209 | mutex_unlock(lock: &msi->bitmap_lock); |
210 | |
211 | if (msi_irq < 0) |
212 | return msi_irq; |
213 | |
214 | irq_domain_set_info(domain, virq, hwirq: msi_irq, |
215 | chip: &xgene_msi_bottom_irq_chip, chip_data: domain->host_data, |
216 | handler: handle_simple_irq, NULL, NULL); |
217 | |
218 | return 0; |
219 | } |
220 | |
221 | static void xgene_irq_domain_free(struct irq_domain *domain, |
222 | unsigned int virq, unsigned int nr_irqs) |
223 | { |
224 | struct irq_data *d = irq_domain_get_irq_data(domain, virq); |
225 | struct xgene_msi *msi = irq_data_get_irq_chip_data(d); |
226 | u32 hwirq; |
227 | |
228 | mutex_lock(&msi->bitmap_lock); |
229 | |
230 | hwirq = hwirq_to_canonical_hwirq(hwirq: d->hwirq); |
231 | bitmap_clear(map: msi->bitmap, start: hwirq, nbits: msi->num_cpus); |
232 | |
233 | mutex_unlock(lock: &msi->bitmap_lock); |
234 | |
235 | irq_domain_free_irqs_parent(domain, irq_base: virq, nr_irqs); |
236 | } |
237 | |
238 | static const struct irq_domain_ops msi_domain_ops = { |
239 | .alloc = xgene_irq_domain_alloc, |
240 | .free = xgene_irq_domain_free, |
241 | }; |
242 | |
243 | static int xgene_allocate_domains(struct xgene_msi *msi) |
244 | { |
245 | msi->inner_domain = irq_domain_add_linear(NULL, NR_MSI_VEC, |
246 | ops: &msi_domain_ops, host_data: msi); |
247 | if (!msi->inner_domain) |
248 | return -ENOMEM; |
249 | |
250 | msi->msi_domain = pci_msi_create_irq_domain(fwnode: of_node_to_fwnode(node: msi->node), |
251 | info: &xgene_msi_domain_info, |
252 | parent: msi->inner_domain); |
253 | |
254 | if (!msi->msi_domain) { |
255 | irq_domain_remove(host: msi->inner_domain); |
256 | return -ENOMEM; |
257 | } |
258 | |
259 | return 0; |
260 | } |
261 | |
262 | static void xgene_free_domains(struct xgene_msi *msi) |
263 | { |
264 | if (msi->msi_domain) |
265 | irq_domain_remove(host: msi->msi_domain); |
266 | if (msi->inner_domain) |
267 | irq_domain_remove(host: msi->inner_domain); |
268 | } |
269 | |
270 | static int xgene_msi_init_allocator(struct xgene_msi *xgene_msi) |
271 | { |
272 | xgene_msi->bitmap = bitmap_zalloc(NR_MSI_VEC, GFP_KERNEL); |
273 | if (!xgene_msi->bitmap) |
274 | return -ENOMEM; |
275 | |
276 | mutex_init(&xgene_msi->bitmap_lock); |
277 | |
278 | xgene_msi->msi_groups = kcalloc(NR_HW_IRQS, |
279 | size: sizeof(struct xgene_msi_group), |
280 | GFP_KERNEL); |
281 | if (!xgene_msi->msi_groups) |
282 | return -ENOMEM; |
283 | |
284 | return 0; |
285 | } |
286 | |
287 | static void xgene_msi_isr(struct irq_desc *desc) |
288 | { |
289 | struct irq_chip *chip = irq_desc_get_chip(desc); |
290 | struct xgene_msi_group *msi_groups; |
291 | struct xgene_msi *xgene_msi; |
292 | int msir_index, msir_val, hw_irq, ret; |
293 | u32 intr_index, grp_select, msi_grp; |
294 | |
295 | chained_irq_enter(chip, desc); |
296 | |
297 | msi_groups = irq_desc_get_handler_data(desc); |
298 | xgene_msi = msi_groups->msi; |
299 | msi_grp = msi_groups->msi_grp; |
300 | |
301 | /* |
302 | * MSIINTn (n is 0..F) indicates if there is a pending MSI interrupt |
303 | * If bit x of this register is set (x is 0..7), one or more interrupts |
304 | * corresponding to MSInIRx is set. |
305 | */ |
306 | grp_select = xgene_msi_int_read(msi: xgene_msi, msi_grp); |
307 | while (grp_select) { |
308 | msir_index = ffs(grp_select) - 1; |
309 | /* |
310 | * Calculate MSInIRx address to read to check for interrupts |
311 | * (refer to termination address and data assignment |
312 | * described in xgene_compose_msi_msg() ) |
313 | */ |
314 | msir_val = xgene_msi_ir_read(msi: xgene_msi, msi_grp, msir_idx: msir_index); |
315 | while (msir_val) { |
316 | intr_index = ffs(msir_val) - 1; |
317 | /* |
318 | * Calculate MSI vector number (refer to the termination |
319 | * address and data assignment described in |
320 | * xgene_compose_msi_msg function) |
321 | */ |
322 | hw_irq = (((msir_index * IRQS_PER_IDX) + intr_index) * |
323 | NR_HW_IRQS) + msi_grp; |
324 | /* |
325 | * As we have multiple hw_irq that maps to single MSI, |
326 | * always look up the virq using the hw_irq as seen from |
327 | * CPU0 |
328 | */ |
329 | hw_irq = hwirq_to_canonical_hwirq(hwirq: hw_irq); |
330 | ret = generic_handle_domain_irq(domain: xgene_msi->inner_domain, hwirq: hw_irq); |
331 | WARN_ON_ONCE(ret); |
332 | msir_val &= ~(1 << intr_index); |
333 | } |
334 | grp_select &= ~(1 << msir_index); |
335 | |
336 | if (!grp_select) { |
337 | /* |
338 | * We handled all interrupts happened in this group, |
339 | * resample this group MSI_INTx register in case |
340 | * something else has been made pending in the meantime |
341 | */ |
342 | grp_select = xgene_msi_int_read(msi: xgene_msi, msi_grp); |
343 | } |
344 | } |
345 | |
346 | chained_irq_exit(chip, desc); |
347 | } |
348 | |
349 | static enum cpuhp_state pci_xgene_online; |
350 | |
351 | static void xgene_msi_remove(struct platform_device *pdev) |
352 | { |
353 | struct xgene_msi *msi = platform_get_drvdata(pdev); |
354 | |
355 | if (pci_xgene_online) |
356 | cpuhp_remove_state(state: pci_xgene_online); |
357 | cpuhp_remove_state(state: CPUHP_PCI_XGENE_DEAD); |
358 | |
359 | kfree(objp: msi->msi_groups); |
360 | |
361 | bitmap_free(bitmap: msi->bitmap); |
362 | msi->bitmap = NULL; |
363 | |
364 | xgene_free_domains(msi); |
365 | } |
366 | |
367 | static int xgene_msi_hwirq_alloc(unsigned int cpu) |
368 | { |
369 | struct xgene_msi *msi = &xgene_msi_ctrl; |
370 | struct xgene_msi_group *msi_group; |
371 | cpumask_var_t mask; |
372 | int i; |
373 | int err; |
374 | |
375 | for (i = cpu; i < NR_HW_IRQS; i += msi->num_cpus) { |
376 | msi_group = &msi->msi_groups[i]; |
377 | if (!msi_group->gic_irq) |
378 | continue; |
379 | |
380 | irq_set_chained_handler_and_data(irq: msi_group->gic_irq, |
381 | handle: xgene_msi_isr, data: msi_group); |
382 | |
383 | /* |
384 | * Statically allocate MSI GIC IRQs to each CPU core. |
385 | * With 8-core X-Gene v1, 2 MSI GIC IRQs are allocated |
386 | * to each core. |
387 | */ |
388 | if (alloc_cpumask_var(mask: &mask, GFP_KERNEL)) { |
389 | cpumask_clear(dstp: mask); |
390 | cpumask_set_cpu(cpu, dstp: mask); |
391 | err = irq_set_affinity(irq: msi_group->gic_irq, cpumask: mask); |
392 | if (err) |
393 | pr_err("failed to set affinity for GIC IRQ" ); |
394 | free_cpumask_var(mask); |
395 | } else { |
396 | pr_err("failed to alloc CPU mask for affinity\n" ); |
397 | err = -EINVAL; |
398 | } |
399 | |
400 | if (err) { |
401 | irq_set_chained_handler_and_data(irq: msi_group->gic_irq, |
402 | NULL, NULL); |
403 | return err; |
404 | } |
405 | } |
406 | |
407 | return 0; |
408 | } |
409 | |
410 | static int xgene_msi_hwirq_free(unsigned int cpu) |
411 | { |
412 | struct xgene_msi *msi = &xgene_msi_ctrl; |
413 | struct xgene_msi_group *msi_group; |
414 | int i; |
415 | |
416 | for (i = cpu; i < NR_HW_IRQS; i += msi->num_cpus) { |
417 | msi_group = &msi->msi_groups[i]; |
418 | if (!msi_group->gic_irq) |
419 | continue; |
420 | |
421 | irq_set_chained_handler_and_data(irq: msi_group->gic_irq, NULL, |
422 | NULL); |
423 | } |
424 | return 0; |
425 | } |
426 | |
427 | static const struct of_device_id xgene_msi_match_table[] = { |
428 | {.compatible = "apm,xgene1-msi" }, |
429 | {}, |
430 | }; |
431 | |
432 | static int xgene_msi_probe(struct platform_device *pdev) |
433 | { |
434 | struct resource *res; |
435 | int rc, irq_index; |
436 | struct xgene_msi *xgene_msi; |
437 | int virt_msir; |
438 | u32 msi_val, msi_idx; |
439 | |
440 | xgene_msi = &xgene_msi_ctrl; |
441 | |
442 | platform_set_drvdata(pdev, data: xgene_msi); |
443 | |
444 | xgene_msi->msi_regs = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res); |
445 | if (IS_ERR(ptr: xgene_msi->msi_regs)) { |
446 | rc = PTR_ERR(ptr: xgene_msi->msi_regs); |
447 | goto error; |
448 | } |
449 | xgene_msi->msi_addr = res->start; |
450 | xgene_msi->node = pdev->dev.of_node; |
451 | xgene_msi->num_cpus = num_possible_cpus(); |
452 | |
453 | rc = xgene_msi_init_allocator(xgene_msi); |
454 | if (rc) { |
455 | dev_err(&pdev->dev, "Error allocating MSI bitmap\n" ); |
456 | goto error; |
457 | } |
458 | |
459 | rc = xgene_allocate_domains(msi: xgene_msi); |
460 | if (rc) { |
461 | dev_err(&pdev->dev, "Failed to allocate MSI domain\n" ); |
462 | goto error; |
463 | } |
464 | |
465 | for (irq_index = 0; irq_index < NR_HW_IRQS; irq_index++) { |
466 | virt_msir = platform_get_irq(pdev, irq_index); |
467 | if (virt_msir < 0) { |
468 | rc = virt_msir; |
469 | goto error; |
470 | } |
471 | xgene_msi->msi_groups[irq_index].gic_irq = virt_msir; |
472 | xgene_msi->msi_groups[irq_index].msi_grp = irq_index; |
473 | xgene_msi->msi_groups[irq_index].msi = xgene_msi; |
474 | } |
475 | |
476 | /* |
477 | * MSInIRx registers are read-to-clear; before registering |
478 | * interrupt handlers, read all of them to clear spurious |
479 | * interrupts that may occur before the driver is probed. |
480 | */ |
481 | for (irq_index = 0; irq_index < NR_HW_IRQS; irq_index++) { |
482 | for (msi_idx = 0; msi_idx < IDX_PER_GROUP; msi_idx++) |
483 | xgene_msi_ir_read(msi: xgene_msi, msi_grp: irq_index, msir_idx: msi_idx); |
484 | |
485 | /* Read MSIINTn to confirm */ |
486 | msi_val = xgene_msi_int_read(msi: xgene_msi, msi_grp: irq_index); |
487 | if (msi_val) { |
488 | dev_err(&pdev->dev, "Failed to clear spurious IRQ\n" ); |
489 | rc = -EINVAL; |
490 | goto error; |
491 | } |
492 | } |
493 | |
494 | rc = cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, name: "pci/xgene:online" , |
495 | startup: xgene_msi_hwirq_alloc, NULL); |
496 | if (rc < 0) |
497 | goto err_cpuhp; |
498 | pci_xgene_online = rc; |
499 | rc = cpuhp_setup_state(state: CPUHP_PCI_XGENE_DEAD, name: "pci/xgene:dead" , NULL, |
500 | teardown: xgene_msi_hwirq_free); |
501 | if (rc) |
502 | goto err_cpuhp; |
503 | |
504 | dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n" ); |
505 | |
506 | return 0; |
507 | |
508 | err_cpuhp: |
509 | dev_err(&pdev->dev, "failed to add CPU MSI notifier\n" ); |
510 | error: |
511 | xgene_msi_remove(pdev); |
512 | return rc; |
513 | } |
514 | |
515 | static struct platform_driver xgene_msi_driver = { |
516 | .driver = { |
517 | .name = "xgene-msi" , |
518 | .of_match_table = xgene_msi_match_table, |
519 | }, |
520 | .probe = xgene_msi_probe, |
521 | .remove_new = xgene_msi_remove, |
522 | }; |
523 | |
524 | static int __init xgene_pcie_msi_init(void) |
525 | { |
526 | return platform_driver_register(&xgene_msi_driver); |
527 | } |
528 | subsys_initcall(xgene_pcie_msi_init); |
529 | |