1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 */
6
7#define pr_fmt(fmt) "riscv-imsic: " fmt
8#include <linux/acpi.h>
9#include <linux/bitmap.h>
10#include <linux/cpu.h>
11#include <linux/interrupt.h>
12#include <linux/io.h>
13#include <linux/irq.h>
14#include <linux/irqchip.h>
15#include <linux/irqdomain.h>
16#include <linux/module.h>
17#include <linux/msi.h>
18#include <linux/pci.h>
19#include <linux/platform_device.h>
20#include <linux/spinlock.h>
21#include <linux/smp.h>
22
23#include <linux/irqchip/irq-msi-lib.h>
24#include "irq-riscv-imsic-state.h"
25
26static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index,
27 phys_addr_t *out_msi_pa)
28{
29 struct imsic_global_config *global;
30 struct imsic_local_config *local;
31
32 global = &imsic->global;
33 local = per_cpu_ptr(global->local, cpu);
34
35 if (BIT(global->guest_index_bits) <= guest_index)
36 return false;
37
38 if (out_msi_pa)
39 *out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ);
40
41 return true;
42}
43
44static void imsic_irq_mask(struct irq_data *d)
45{
46 imsic_vector_mask(vec: irq_data_get_irq_chip_data(d));
47}
48
49static void imsic_irq_unmask(struct irq_data *d)
50{
51 imsic_vector_unmask(vec: irq_data_get_irq_chip_data(d));
52}
53
54static int imsic_irq_retrigger(struct irq_data *d)
55{
56 struct imsic_vector *vec = irq_data_get_irq_chip_data(d);
57 struct imsic_local_config *local;
58
59 if (WARN_ON(!vec))
60 return -ENOENT;
61
62 local = per_cpu_ptr(imsic->global.local, vec->cpu);
63 writel_relaxed(vec->local_id, local->msi_va);
64 return 0;
65}
66
67static void imsic_irq_ack(struct irq_data *d)
68{
69 irq_move_irq(data: d);
70}
71
72static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg)
73{
74 phys_addr_t msi_addr;
75
76 if (WARN_ON(!vec))
77 return;
78
79 if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr)))
80 return;
81
82 msg->address_hi = upper_32_bits(msi_addr);
83 msg->address_lo = lower_32_bits(msi_addr);
84 msg->data = vec->local_id;
85}
86
87static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg)
88{
89 imsic_irq_compose_vector_msg(vec: irq_data_get_irq_chip_data(d), msg);
90}
91
92#ifdef CONFIG_SMP
93static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec)
94{
95 struct msi_msg msg = { };
96
97 imsic_irq_compose_vector_msg(vec, msg: &msg);
98 irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg);
99}
100
101static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
102 bool force)
103{
104 struct imsic_vector *old_vec, *new_vec;
105 struct imsic_vector tmp_vec;
106
107 /*
108 * Requirements for the downstream irqdomains (or devices):
109 *
110 * 1) Downstream irqdomains (or devices) with atomic MSI update can
111 * happily do imsic_irq_set_affinity() in the process-context on
112 * any CPU so the irqchip of such irqdomains must not set the
113 * IRQCHIP_MOVE_DEFERRED flag.
114 *
115 * 2) Downstream irqdomains (or devices) with non-atomic MSI update
116 * must use imsic_irq_set_affinity() in nterrupt-context upon
117 * the next device interrupt so the irqchip of such irqdomains
118 * must set the IRQCHIP_MOVE_DEFERRED flag.
119 */
120
121 old_vec = irq_data_get_irq_chip_data(d);
122 if (WARN_ON(!old_vec))
123 return -ENOENT;
124
125 /* If old vector cpu belongs to the target cpumask then do nothing */
126 if (cpumask_test_cpu(cpu: old_vec->cpu, cpumask: mask_val))
127 return IRQ_SET_MASK_OK_DONE;
128
129 /* If move is already in-flight then return failure */
130 if (imsic_vector_get_move(vec: old_vec))
131 return -EBUSY;
132
133 /* Get a new vector on the desired set of CPUs */
134 new_vec = imsic_vector_alloc(irq: old_vec->irq, mask: mask_val);
135 if (!new_vec)
136 return -ENOSPC;
137
138 /*
139 * Device having non-atomic MSI update might see an intermediate
140 * state when changing target IMSIC vector from one CPU to another.
141 *
142 * To avoid losing interrupt to such intermediate state, do the
143 * following (just like x86 APIC):
144 *
145 * 1) First write a temporary IMSIC vector to the device which
146 * has MSI address same as the old IMSIC vector but MSI data
147 * matches the new IMSIC vector.
148 *
149 * 2) Next write the new IMSIC vector to the device.
150 *
151 * Based on the above, __imsic_local_sync() must check pending
152 * status of both old MSI data and new MSI data on the old CPU.
153 */
154 if (!irq_can_move_in_process_context(data: d) &&
155 new_vec->local_id != old_vec->local_id) {
156 /* Setup temporary vector */
157 tmp_vec.cpu = old_vec->cpu;
158 tmp_vec.local_id = new_vec->local_id;
159
160 /* Point device to the temporary vector */
161 imsic_msi_update_msg(d: irq_get_irq_data(irq: d->irq), vec: &tmp_vec);
162 }
163
164 /* Point device to the new vector */
165 imsic_msi_update_msg(d: irq_get_irq_data(irq: d->irq), vec: new_vec);
166
167 /* Update irq descriptors with the new vector */
168 d->chip_data = new_vec;
169
170 /* Update effective affinity */
171 irq_data_update_effective_affinity(d, cpumask_of(new_vec->cpu));
172
173 /* Move state of the old vector to the new vector */
174 imsic_vector_move(old_vec, new_vec);
175
176 return IRQ_SET_MASK_OK_DONE;
177}
178
179static void imsic_irq_force_complete_move(struct irq_data *d)
180{
181 struct imsic_vector *mvec, *vec = irq_data_get_irq_chip_data(d);
182 unsigned int cpu = smp_processor_id();
183
184 if (WARN_ON(!vec))
185 return;
186
187 /* Do nothing if there is no in-flight move */
188 mvec = imsic_vector_get_move(vec);
189 if (!mvec)
190 return;
191
192 /* Do nothing if the old IMSIC vector does not belong to current CPU */
193 if (mvec->cpu != cpu)
194 return;
195
196 /*
197 * The best we can do is force cleanup the old IMSIC vector.
198 *
199 * The challenges over here are same as x86 vector domain so
200 * refer to the comments in irq_force_complete_move() function
201 * implemented at arch/x86/kernel/apic/vector.c.
202 */
203
204 /* Force cleanup in-flight move */
205 pr_info("IRQ fixup: irq %d move in progress, old vector cpu %d local_id %d\n",
206 d->irq, mvec->cpu, mvec->local_id);
207 imsic_vector_force_move_cleanup(vec);
208}
209#endif
210
211static struct irq_chip imsic_irq_base_chip = {
212 .name = "IMSIC",
213 .irq_mask = imsic_irq_mask,
214 .irq_unmask = imsic_irq_unmask,
215#ifdef CONFIG_SMP
216 .irq_set_affinity = imsic_irq_set_affinity,
217 .irq_force_complete_move = imsic_irq_force_complete_move,
218#endif
219 .irq_retrigger = imsic_irq_retrigger,
220 .irq_ack = imsic_irq_ack,
221 .irq_compose_msi_msg = imsic_irq_compose_msg,
222 .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
223};
224
225static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
226 unsigned int nr_irqs, void *args)
227{
228 struct imsic_vector *vec;
229
230 /* Multi-MSI is not supported yet. */
231 if (nr_irqs > 1)
232 return -EOPNOTSUPP;
233
234 vec = imsic_vector_alloc(irq: virq, cpu_online_mask);
235 if (!vec)
236 return -ENOSPC;
237
238 irq_domain_set_info(domain, virq, hwirq: virq, chip: &imsic_irq_base_chip, chip_data: vec,
239 handler: handle_edge_irq, NULL, NULL);
240 irq_set_noprobe(irq: virq);
241 irq_set_affinity(irq: virq, cpu_online_mask);
242 irq_data_update_effective_affinity(d: irq_get_irq_data(irq: virq), cpumask_of(vec->cpu));
243
244 return 0;
245}
246
247static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
248 unsigned int nr_irqs)
249{
250 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
251
252 imsic_vector_free(vector: irq_data_get_irq_chip_data(d));
253 irq_domain_free_irqs_parent(domain, irq_base: virq, nr_irqs);
254}
255
256#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
257static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d,
258 struct irq_data *irqd, int ind)
259{
260 if (!irqd) {
261 imsic_vector_debug_show_summary(m, ind);
262 return;
263 }
264
265 imsic_vector_debug_show(m, vec: irq_data_get_irq_chip_data(d: irqd), ind);
266}
267#endif
268
269static const struct irq_domain_ops imsic_base_domain_ops = {
270 .alloc = imsic_irq_domain_alloc,
271 .free = imsic_irq_domain_free,
272 .select = msi_lib_irq_domain_select,
273#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
274 .debug_show = imsic_irq_debug_show,
275#endif
276};
277
278static bool imsic_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
279 struct irq_domain *real_parent, struct msi_domain_info *info)
280{
281 if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
282 return false;
283
284 switch (info->bus_token) {
285 case DOMAIN_BUS_PCI_DEVICE_MSI:
286 case DOMAIN_BUS_PCI_DEVICE_MSIX:
287 info->chip->flags |= IRQCHIP_MOVE_DEFERRED;
288 break;
289 default:
290 break;
291 }
292
293 return true;
294}
295
296static const struct msi_parent_ops imsic_msi_parent_ops = {
297 .supported_flags = MSI_GENERIC_FLAGS_MASK |
298 MSI_FLAG_PCI_MSIX,
299 .required_flags = MSI_FLAG_USE_DEF_DOM_OPS |
300 MSI_FLAG_USE_DEF_CHIP_OPS |
301 MSI_FLAG_PCI_MSI_MASK_PARENT,
302 .chip_flags = MSI_CHIP_FLAG_SET_ACK,
303 .bus_select_token = DOMAIN_BUS_NEXUS,
304 .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
305 .init_dev_msi_info = imsic_init_dev_msi_info,
306};
307
308int imsic_irqdomain_init(void)
309{
310 struct imsic_global_config *global;
311
312 if (!imsic || !imsic->fwnode) {
313 pr_err("early driver not probed\n");
314 return -ENODEV;
315 }
316
317 if (imsic->base_domain) {
318 pr_err("%pfwP: irq domain already created\n", imsic->fwnode);
319 return -ENODEV;
320 }
321
322 /* Create Base IRQ domain */
323 imsic->base_domain = irq_domain_create_tree(fwnode: imsic->fwnode,
324 ops: &imsic_base_domain_ops, host_data: imsic);
325 if (!imsic->base_domain) {
326 pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode);
327 return -ENOMEM;
328 }
329 imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT;
330 imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops;
331
332 irq_domain_update_bus_token(domain: imsic->base_domain, bus_token: DOMAIN_BUS_NEXUS);
333
334 global = &imsic->global;
335 pr_info("%pfwP: hart-index-bits: %d, guest-index-bits: %d\n",
336 imsic->fwnode, global->hart_index_bits, global->guest_index_bits);
337 pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n",
338 imsic->fwnode, global->group_index_bits, global->group_index_shift);
339 pr_info("%pfwP: per-CPU IDs %d at base address %pa\n",
340 imsic->fwnode, global->nr_ids, &global->base_addr);
341 pr_info("%pfwP: total %d interrupts available\n",
342 imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1));
343
344 return 0;
345}
346
347static int imsic_platform_probe_common(struct fwnode_handle *fwnode)
348{
349 if (imsic && imsic->fwnode != fwnode) {
350 pr_err("%pfwP: fwnode mismatch\n", fwnode);
351 return -ENODEV;
352 }
353
354 return imsic_irqdomain_init();
355}
356
357static int imsic_platform_dt_probe(struct platform_device *pdev)
358{
359 return imsic_platform_probe_common(fwnode: pdev->dev.fwnode);
360}
361
362#ifdef CONFIG_ACPI
363
364/*
365 * On ACPI based systems, PCI enumeration happens early during boot in
366 * acpi_scan_init(). PCI enumeration expects MSI domain setup before
367 * it calls pci_set_msi_domain(). Hence, unlike in DT where
368 * imsic-platform drive probe happens late during boot, ACPI based
369 * systems need to setup the MSI domain early.
370 */
371int imsic_platform_acpi_probe(struct fwnode_handle *fwnode)
372{
373 return imsic_platform_probe_common(fwnode);
374}
375
376#endif
377
378static const struct of_device_id imsic_platform_match[] = {
379 { .compatible = "riscv,imsics" },
380 {}
381};
382
383static struct platform_driver imsic_platform_driver = {
384 .driver = {
385 .name = "riscv-imsic",
386 .of_match_table = imsic_platform_match,
387 },
388 .probe = imsic_platform_dt_probe,
389};
390builtin_platform_driver(imsic_platform_driver);
391

source code of linux/drivers/irqchip/irq-riscv-imsic-platform.c