1// SPDX-License-Identifier: GPL-2.0
2/*
3 * PCI Message Signaled Interrupt (MSI) - irqdomain support
4 */
5#include <linux/acpi_iort.h>
6#include <linux/irqdomain.h>
7#include <linux/of_irq.h>
8
9#include "msi.h"
10
11int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
12{
13 struct irq_domain *domain;
14
15 domain = dev_get_msi_domain(dev: &dev->dev);
16 if (domain && irq_domain_is_hierarchy(domain))
17 return msi_domain_alloc_irqs_all_locked(dev: &dev->dev, domid: MSI_DEFAULT_DOMAIN, nirqs: nvec);
18
19 return pci_msi_legacy_setup_msi_irqs(dev, nvec, type);
20}
21
22void pci_msi_teardown_msi_irqs(struct pci_dev *dev)
23{
24 struct irq_domain *domain;
25
26 domain = dev_get_msi_domain(dev: &dev->dev);
27 if (domain && irq_domain_is_hierarchy(domain)) {
28 msi_domain_free_irqs_all_locked(dev: &dev->dev, domid: MSI_DEFAULT_DOMAIN);
29 } else {
30 pci_msi_legacy_teardown_msi_irqs(dev);
31 msi_free_msi_descs(dev: &dev->dev);
32 }
33}
34
35/**
36 * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space
37 * @irq_data: Pointer to interrupt data of the MSI interrupt
38 * @msg: Pointer to the message
39 */
40static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg)
41{
42 struct msi_desc *desc = irq_data_get_msi_desc(d: irq_data);
43
44 /*
45 * For MSI-X desc->irq is always equal to irq_data->irq. For
46 * MSI only the first interrupt of MULTI MSI passes the test.
47 */
48 if (desc->irq == irq_data->irq)
49 __pci_write_msi_msg(entry: desc, msg);
50}
51
52/**
53 * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source
54 * @desc: Pointer to the MSI descriptor
55 *
56 * The ID number is only used within the irqdomain.
57 */
58static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc)
59{
60 struct pci_dev *dev = msi_desc_to_pci_dev(desc);
61
62 return (irq_hw_number_t)desc->msi_index |
63 pci_dev_id(dev) << 11 |
64 ((irq_hw_number_t)(pci_domain_nr(bus: dev->bus) & 0xFFFFFFFF)) << 27;
65}
66
67static void pci_msi_domain_set_desc(msi_alloc_info_t *arg,
68 struct msi_desc *desc)
69{
70 arg->desc = desc;
71 arg->hwirq = pci_msi_domain_calc_hwirq(desc);
72}
73
74static struct msi_domain_ops pci_msi_domain_ops_default = {
75 .set_desc = pci_msi_domain_set_desc,
76};
77
78static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info)
79{
80 struct msi_domain_ops *ops = info->ops;
81
82 if (ops == NULL) {
83 info->ops = &pci_msi_domain_ops_default;
84 } else {
85 if (ops->set_desc == NULL)
86 ops->set_desc = pci_msi_domain_set_desc;
87 }
88}
89
90static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info)
91{
92 struct irq_chip *chip = info->chip;
93
94 BUG_ON(!chip);
95 if (!chip->irq_write_msi_msg)
96 chip->irq_write_msi_msg = pci_msi_domain_write_msg;
97 if (!chip->irq_mask)
98 chip->irq_mask = pci_msi_mask_irq;
99 if (!chip->irq_unmask)
100 chip->irq_unmask = pci_msi_unmask_irq;
101}
102
103/**
104 * pci_msi_create_irq_domain - Create a MSI interrupt domain
105 * @fwnode: Optional fwnode of the interrupt controller
106 * @info: MSI domain info
107 * @parent: Parent irq domain
108 *
109 * Updates the domain and chip ops and creates a MSI interrupt domain.
110 *
111 * Returns:
112 * A domain pointer or NULL in case of failure.
113 */
114struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
115 struct msi_domain_info *info,
116 struct irq_domain *parent)
117{
118 if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE))
119 info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
120
121 if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
122 pci_msi_domain_update_dom_ops(info);
123 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
124 pci_msi_domain_update_chip_ops(info);
125
126 /* Let the core code free MSI descriptors when freeing interrupts */
127 info->flags |= MSI_FLAG_FREE_MSI_DESCS;
128
129 info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS;
130 if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))
131 info->flags |= MSI_FLAG_MUST_REACTIVATE;
132
133 /* PCI-MSI is oneshot-safe */
134 info->chip->flags |= IRQCHIP_ONESHOT_SAFE;
135 /* Let the core update the bus token */
136 info->bus_token = DOMAIN_BUS_PCI_MSI;
137
138 return msi_create_irq_domain(fwnode, info, parent);
139}
140EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain);
141
142/*
143 * Per device MSI[-X] domain functionality
144 */
145static void pci_device_domain_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
146{
147 arg->desc = desc;
148 arg->hwirq = desc->msi_index;
149}
150
151static void pci_irq_mask_msi(struct irq_data *data)
152{
153 struct msi_desc *desc = irq_data_get_msi_desc(d: data);
154
155 pci_msi_mask(desc, BIT(data->irq - desc->irq));
156}
157
158static void pci_irq_unmask_msi(struct irq_data *data)
159{
160 struct msi_desc *desc = irq_data_get_msi_desc(d: data);
161
162 pci_msi_unmask(desc, BIT(data->irq - desc->irq));
163}
164
165#ifdef CONFIG_GENERIC_IRQ_RESERVATION_MODE
166# define MSI_REACTIVATE MSI_FLAG_MUST_REACTIVATE
167#else
168# define MSI_REACTIVATE 0
169#endif
170
171#define MSI_COMMON_FLAGS (MSI_FLAG_FREE_MSI_DESCS | \
172 MSI_FLAG_ACTIVATE_EARLY | \
173 MSI_FLAG_DEV_SYSFS | \
174 MSI_REACTIVATE)
175
176static const struct msi_domain_template pci_msi_template = {
177 .chip = {
178 .name = "PCI-MSI",
179 .irq_mask = pci_irq_mask_msi,
180 .irq_unmask = pci_irq_unmask_msi,
181 .irq_write_msi_msg = pci_msi_domain_write_msg,
182 .flags = IRQCHIP_ONESHOT_SAFE,
183 },
184
185 .ops = {
186 .set_desc = pci_device_domain_set_desc,
187 },
188
189 .info = {
190 .flags = MSI_COMMON_FLAGS | MSI_FLAG_MULTI_PCI_MSI,
191 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSI,
192 },
193};
194
195static void pci_irq_mask_msix(struct irq_data *data)
196{
197 pci_msix_mask(desc: irq_data_get_msi_desc(d: data));
198}
199
200static void pci_irq_unmask_msix(struct irq_data *data)
201{
202 pci_msix_unmask(desc: irq_data_get_msi_desc(d: data));
203}
204
205static void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_info_t *arg,
206 struct msi_desc *desc)
207{
208 /* Don't fiddle with preallocated MSI descriptors */
209 if (!desc->pci.mask_base)
210 msix_prepare_msi_desc(to_pci_dev(desc->dev), desc);
211}
212
213static const struct msi_domain_template pci_msix_template = {
214 .chip = {
215 .name = "PCI-MSIX",
216 .irq_mask = pci_irq_mask_msix,
217 .irq_unmask = pci_irq_unmask_msix,
218 .irq_write_msi_msg = pci_msi_domain_write_msg,
219 .flags = IRQCHIP_ONESHOT_SAFE,
220 },
221
222 .ops = {
223 .prepare_desc = pci_msix_prepare_desc,
224 .set_desc = pci_device_domain_set_desc,
225 },
226
227 .info = {
228 .flags = MSI_COMMON_FLAGS | MSI_FLAG_PCI_MSIX |
229 MSI_FLAG_PCI_MSIX_ALLOC_DYN,
230 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSIX,
231 },
232};
233
234static bool pci_match_device_domain(struct pci_dev *pdev, enum irq_domain_bus_token bus_token)
235{
236 return msi_match_device_irq_domain(dev: &pdev->dev, domid: MSI_DEFAULT_DOMAIN, bus_token);
237}
238
239static bool pci_create_device_domain(struct pci_dev *pdev, const struct msi_domain_template *tmpl,
240 unsigned int hwsize)
241{
242 struct irq_domain *domain = dev_get_msi_domain(dev: &pdev->dev);
243
244 if (!domain || !irq_domain_is_msi_parent(domain))
245 return true;
246
247 return msi_create_device_irq_domain(dev: &pdev->dev, domid: MSI_DEFAULT_DOMAIN, template: tmpl,
248 hwsize, NULL, NULL);
249}
250
251/**
252 * pci_setup_msi_device_domain - Setup a device MSI interrupt domain
253 * @pdev: The PCI device to create the domain on
254 *
255 * Return:
256 * True when:
257 * - The device does not have a MSI parent irq domain associated,
258 * which keeps the legacy architecture specific and the global
259 * PCI/MSI domain models working
260 * - The MSI domain exists already
261 * - The MSI domain was successfully allocated
262 * False when:
263 * - MSI-X is enabled
264 * - The domain creation fails.
265 *
266 * The created MSI domain is preserved until:
267 * - The device is removed
268 * - MSI is disabled and a MSI-X domain is created
269 */
270bool pci_setup_msi_device_domain(struct pci_dev *pdev)
271{
272 if (WARN_ON_ONCE(pdev->msix_enabled))
273 return false;
274
275 if (pci_match_device_domain(pdev, bus_token: DOMAIN_BUS_PCI_DEVICE_MSI))
276 return true;
277 if (pci_match_device_domain(pdev, bus_token: DOMAIN_BUS_PCI_DEVICE_MSIX))
278 msi_remove_device_irq_domain(dev: &pdev->dev, domid: MSI_DEFAULT_DOMAIN);
279
280 return pci_create_device_domain(pdev, tmpl: &pci_msi_template, hwsize: 1);
281}
282
283/**
284 * pci_setup_msix_device_domain - Setup a device MSI-X interrupt domain
285 * @pdev: The PCI device to create the domain on
286 * @hwsize: The size of the MSI-X vector table
287 *
288 * Return:
289 * True when:
290 * - The device does not have a MSI parent irq domain associated,
291 * which keeps the legacy architecture specific and the global
292 * PCI/MSI domain models working
293 * - The MSI-X domain exists already
294 * - The MSI-X domain was successfully allocated
295 * False when:
296 * - MSI is enabled
297 * - The domain creation fails.
298 *
299 * The created MSI-X domain is preserved until:
300 * - The device is removed
301 * - MSI-X is disabled and a MSI domain is created
302 */
303bool pci_setup_msix_device_domain(struct pci_dev *pdev, unsigned int hwsize)
304{
305 if (WARN_ON_ONCE(pdev->msi_enabled))
306 return false;
307
308 if (pci_match_device_domain(pdev, bus_token: DOMAIN_BUS_PCI_DEVICE_MSIX))
309 return true;
310 if (pci_match_device_domain(pdev, bus_token: DOMAIN_BUS_PCI_DEVICE_MSI))
311 msi_remove_device_irq_domain(dev: &pdev->dev, domid: MSI_DEFAULT_DOMAIN);
312
313 return pci_create_device_domain(pdev, tmpl: &pci_msix_template, hwsize);
314}
315
316/**
317 * pci_msi_domain_supports - Check for support of a particular feature flag
318 * @pdev: The PCI device to operate on
319 * @feature_mask: The feature mask to check for (full match)
320 * @mode: If ALLOW_LEGACY this grants the feature when there is no irq domain
321 * associated to the device. If DENY_LEGACY the lack of an irq domain
322 * makes the feature unsupported
323 */
324bool pci_msi_domain_supports(struct pci_dev *pdev, unsigned int feature_mask,
325 enum support_mode mode)
326{
327 struct msi_domain_info *info;
328 struct irq_domain *domain;
329 unsigned int supported;
330
331 domain = dev_get_msi_domain(dev: &pdev->dev);
332
333 if (!domain || !irq_domain_is_hierarchy(domain))
334 return mode == ALLOW_LEGACY;
335
336 if (!irq_domain_is_msi_parent(domain)) {
337 /*
338 * For "global" PCI/MSI interrupt domains the associated
339 * msi_domain_info::flags is the authoritative source of
340 * information.
341 */
342 info = domain->host_data;
343 supported = info->flags;
344 } else {
345 /*
346 * For MSI parent domains the supported feature set
347 * is available in the parent ops. This makes checks
348 * possible before actually instantiating the
349 * per device domain because the parent is never
350 * expanding the PCI/MSI functionality.
351 */
352 supported = domain->msi_parent_ops->supported_flags;
353 }
354
355 return (supported & feature_mask) == feature_mask;
356}
357
358/**
359 * pci_create_ims_domain - Create a secondary IMS domain for a PCI device
360 * @pdev: The PCI device to operate on
361 * @template: The MSI info template which describes the domain
362 * @hwsize: The size of the hardware entry table or 0 if the domain
363 * is purely software managed
364 * @data: Optional pointer to domain specific data to be stored
365 * in msi_domain_info::data
366 *
367 * Return: True on success, false otherwise
368 *
369 * An IMS domain is expected to have the following constraints:
370 * - The index space is managed by the core code
371 *
372 * - There is no requirement for consecutive index ranges
373 *
374 * - The interrupt chip must provide the following callbacks:
375 * - irq_mask()
376 * - irq_unmask()
377 * - irq_write_msi_msg()
378 *
379 * - The interrupt chip must provide the following optional callbacks
380 * when the irq_mask(), irq_unmask() and irq_write_msi_msg() callbacks
381 * cannot operate directly on hardware, e.g. in the case that the
382 * interrupt message store is in queue memory:
383 * - irq_bus_lock()
384 * - irq_bus_unlock()
385 *
386 * These callbacks are invoked from preemptible task context and are
387 * allowed to sleep. In this case the mandatory callbacks above just
388 * store the information. The irq_bus_unlock() callback is supposed
389 * to make the change effective before returning.
390 *
391 * - Interrupt affinity setting is handled by the underlying parent
392 * interrupt domain and communicated to the IMS domain via
393 * irq_write_msi_msg().
394 *
395 * The domain is automatically destroyed when the PCI device is removed.
396 */
397bool pci_create_ims_domain(struct pci_dev *pdev, const struct msi_domain_template *template,
398 unsigned int hwsize, void *data)
399{
400 struct irq_domain *domain = dev_get_msi_domain(dev: &pdev->dev);
401
402 if (!domain || !irq_domain_is_msi_parent(domain))
403 return false;
404
405 if (template->info.bus_token != DOMAIN_BUS_PCI_DEVICE_IMS ||
406 !(template->info.flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS) ||
407 !(template->info.flags & MSI_FLAG_FREE_MSI_DESCS) ||
408 !template->chip.irq_mask || !template->chip.irq_unmask ||
409 !template->chip.irq_write_msi_msg || template->chip.irq_set_affinity)
410 return false;
411
412 return msi_create_device_irq_domain(dev: &pdev->dev, domid: MSI_SECONDARY_DOMAIN, template,
413 hwsize, domain_data: data, NULL);
414}
415EXPORT_SYMBOL_GPL(pci_create_ims_domain);
416
417/*
418 * Users of the generic MSI infrastructure expect a device to have a single ID,
419 * so with DMA aliases we have to pick the least-worst compromise. Devices with
420 * DMA phantom functions tend to still emit MSIs from the real function number,
421 * so we ignore those and only consider topological aliases where either the
422 * alias device or RID appears on a different bus number. We also make the
423 * reasonable assumption that bridges are walked in an upstream direction (so
424 * the last one seen wins), and the much braver assumption that the most likely
425 * case is that of PCI->PCIe so we should always use the alias RID. This echoes
426 * the logic from intel_irq_remapping's set_msi_sid(), which presumably works
427 * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions
428 * for taking ownership all we can really do is close our eyes and hope...
429 */
430static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data)
431{
432 u32 *pa = data;
433 u8 bus = PCI_BUS_NUM(*pa);
434
435 if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus)
436 *pa = alias;
437
438 return 0;
439}
440
441/**
442 * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID)
443 * @domain: The interrupt domain
444 * @pdev: The PCI device.
445 *
446 * The RID for a device is formed from the alias, with a firmware
447 * supplied mapping applied
448 *
449 * Returns: The RID.
450 */
451u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
452{
453 struct device_node *of_node;
454 u32 rid = pci_dev_id(dev: pdev);
455
456 pci_for_each_dma_alias(pdev, fn: get_msi_id_cb, data: &rid);
457
458 of_node = irq_domain_get_of_node(d: domain);
459 rid = of_node ? of_msi_map_id(dev: &pdev->dev, msi_np: of_node, id_in: rid) :
460 iort_msi_map_id(dev: &pdev->dev, id: rid);
461
462 return rid;
463}
464
465/**
466 * pci_msi_get_device_domain - Get the MSI domain for a given PCI device
467 * @pdev: The PCI device
468 *
469 * Use the firmware data to find a device-specific MSI domain
470 * (i.e. not one that is set as a default).
471 *
472 * Returns: The corresponding MSI domain or NULL if none has been found.
473 */
474struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
475{
476 struct irq_domain *dom;
477 u32 rid = pci_dev_id(dev: pdev);
478
479 pci_for_each_dma_alias(pdev, fn: get_msi_id_cb, data: &rid);
480 dom = of_msi_map_get_device_domain(dev: &pdev->dev, id: rid, bus_token: DOMAIN_BUS_PCI_MSI);
481 if (!dom)
482 dom = iort_get_device_domain(dev: &pdev->dev, id: rid,
483 bus_token: DOMAIN_BUS_PCI_MSI);
484 return dom;
485}
486

source code of linux/drivers/pci/msi/irqdomain.c