1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * PCI Message Signaled Interrupt (MSI) - irqdomain support |
4 | */ |
5 | #include <linux/acpi_iort.h> |
6 | #include <linux/irqdomain.h> |
7 | #include <linux/of_irq.h> |
8 | |
9 | #include "msi.h" |
10 | |
11 | int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) |
12 | { |
13 | struct irq_domain *domain; |
14 | |
15 | domain = dev_get_msi_domain(dev: &dev->dev); |
16 | if (domain && irq_domain_is_hierarchy(domain)) |
17 | return msi_domain_alloc_irqs_all_locked(dev: &dev->dev, domid: MSI_DEFAULT_DOMAIN, nirqs: nvec); |
18 | |
19 | return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); |
20 | } |
21 | |
22 | void pci_msi_teardown_msi_irqs(struct pci_dev *dev) |
23 | { |
24 | struct irq_domain *domain; |
25 | |
26 | domain = dev_get_msi_domain(dev: &dev->dev); |
27 | if (domain && irq_domain_is_hierarchy(domain)) { |
28 | msi_domain_free_irqs_all_locked(dev: &dev->dev, domid: MSI_DEFAULT_DOMAIN); |
29 | } else { |
30 | pci_msi_legacy_teardown_msi_irqs(dev); |
31 | msi_free_msi_descs(dev: &dev->dev); |
32 | } |
33 | } |
34 | |
35 | /** |
36 | * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space |
37 | * @irq_data: Pointer to interrupt data of the MSI interrupt |
38 | * @msg: Pointer to the message |
39 | */ |
40 | static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) |
41 | { |
42 | struct msi_desc *desc = irq_data_get_msi_desc(d: irq_data); |
43 | |
44 | /* |
45 | * For MSI-X desc->irq is always equal to irq_data->irq. For |
46 | * MSI only the first interrupt of MULTI MSI passes the test. |
47 | */ |
48 | if (desc->irq == irq_data->irq) |
49 | __pci_write_msi_msg(entry: desc, msg); |
50 | } |
51 | |
52 | /** |
53 | * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source |
54 | * @desc: Pointer to the MSI descriptor |
55 | * |
56 | * The ID number is only used within the irqdomain. |
57 | */ |
58 | static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) |
59 | { |
60 | struct pci_dev *dev = msi_desc_to_pci_dev(desc); |
61 | |
62 | return (irq_hw_number_t)desc->msi_index | |
63 | pci_dev_id(dev) << 11 | |
64 | ((irq_hw_number_t)(pci_domain_nr(bus: dev->bus) & 0xFFFFFFFF)) << 27; |
65 | } |
66 | |
67 | static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, |
68 | struct msi_desc *desc) |
69 | { |
70 | arg->desc = desc; |
71 | arg->hwirq = pci_msi_domain_calc_hwirq(desc); |
72 | } |
73 | |
74 | static struct msi_domain_ops pci_msi_domain_ops_default = { |
75 | .set_desc = pci_msi_domain_set_desc, |
76 | }; |
77 | |
78 | static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) |
79 | { |
80 | struct msi_domain_ops *ops = info->ops; |
81 | |
82 | if (ops == NULL) { |
83 | info->ops = &pci_msi_domain_ops_default; |
84 | } else { |
85 | if (ops->set_desc == NULL) |
86 | ops->set_desc = pci_msi_domain_set_desc; |
87 | } |
88 | } |
89 | |
90 | static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) |
91 | { |
92 | struct irq_chip *chip = info->chip; |
93 | |
94 | BUG_ON(!chip); |
95 | if (!chip->irq_write_msi_msg) |
96 | chip->irq_write_msi_msg = pci_msi_domain_write_msg; |
97 | if (!chip->irq_mask) |
98 | chip->irq_mask = pci_msi_mask_irq; |
99 | if (!chip->irq_unmask) |
100 | chip->irq_unmask = pci_msi_unmask_irq; |
101 | } |
102 | |
103 | /** |
104 | * pci_msi_create_irq_domain - Create a MSI interrupt domain |
105 | * @fwnode: Optional fwnode of the interrupt controller |
106 | * @info: MSI domain info |
107 | * @parent: Parent irq domain |
108 | * |
109 | * Updates the domain and chip ops and creates a MSI interrupt domain. |
110 | * |
111 | * Returns: |
112 | * A domain pointer or NULL in case of failure. |
113 | */ |
114 | struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, |
115 | struct msi_domain_info *info, |
116 | struct irq_domain *parent) |
117 | { |
118 | if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE)) |
119 | info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; |
120 | |
121 | if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) |
122 | pci_msi_domain_update_dom_ops(info); |
123 | if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) |
124 | pci_msi_domain_update_chip_ops(info); |
125 | |
126 | /* Let the core code free MSI descriptors when freeing interrupts */ |
127 | info->flags |= MSI_FLAG_FREE_MSI_DESCS; |
128 | |
129 | info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS; |
130 | if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) |
131 | info->flags |= MSI_FLAG_MUST_REACTIVATE; |
132 | |
133 | /* PCI-MSI is oneshot-safe */ |
134 | info->chip->flags |= IRQCHIP_ONESHOT_SAFE; |
135 | /* Let the core update the bus token */ |
136 | info->bus_token = DOMAIN_BUS_PCI_MSI; |
137 | |
138 | return msi_create_irq_domain(fwnode, info, parent); |
139 | } |
140 | EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); |
141 | |
142 | /* |
143 | * Per device MSI[-X] domain functionality |
144 | */ |
145 | static void pci_device_domain_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) |
146 | { |
147 | arg->desc = desc; |
148 | arg->hwirq = desc->msi_index; |
149 | } |
150 | |
151 | static void pci_irq_mask_msi(struct irq_data *data) |
152 | { |
153 | struct msi_desc *desc = irq_data_get_msi_desc(d: data); |
154 | |
155 | pci_msi_mask(desc, BIT(data->irq - desc->irq)); |
156 | } |
157 | |
158 | static void pci_irq_unmask_msi(struct irq_data *data) |
159 | { |
160 | struct msi_desc *desc = irq_data_get_msi_desc(d: data); |
161 | |
162 | pci_msi_unmask(desc, BIT(data->irq - desc->irq)); |
163 | } |
164 | |
165 | #ifdef CONFIG_GENERIC_IRQ_RESERVATION_MODE |
166 | # define MSI_REACTIVATE MSI_FLAG_MUST_REACTIVATE |
167 | #else |
168 | # define MSI_REACTIVATE 0 |
169 | #endif |
170 | |
171 | #define MSI_COMMON_FLAGS (MSI_FLAG_FREE_MSI_DESCS | \ |
172 | MSI_FLAG_ACTIVATE_EARLY | \ |
173 | MSI_FLAG_DEV_SYSFS | \ |
174 | MSI_REACTIVATE) |
175 | |
176 | static const struct msi_domain_template pci_msi_template = { |
177 | .chip = { |
178 | .name = "PCI-MSI" , |
179 | .irq_mask = pci_irq_mask_msi, |
180 | .irq_unmask = pci_irq_unmask_msi, |
181 | .irq_write_msi_msg = pci_msi_domain_write_msg, |
182 | .flags = IRQCHIP_ONESHOT_SAFE, |
183 | }, |
184 | |
185 | .ops = { |
186 | .set_desc = pci_device_domain_set_desc, |
187 | }, |
188 | |
189 | .info = { |
190 | .flags = MSI_COMMON_FLAGS | MSI_FLAG_MULTI_PCI_MSI, |
191 | .bus_token = DOMAIN_BUS_PCI_DEVICE_MSI, |
192 | }, |
193 | }; |
194 | |
195 | static void pci_irq_mask_msix(struct irq_data *data) |
196 | { |
197 | pci_msix_mask(desc: irq_data_get_msi_desc(d: data)); |
198 | } |
199 | |
200 | static void pci_irq_unmask_msix(struct irq_data *data) |
201 | { |
202 | pci_msix_unmask(desc: irq_data_get_msi_desc(d: data)); |
203 | } |
204 | |
205 | static void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_info_t *arg, |
206 | struct msi_desc *desc) |
207 | { |
208 | /* Don't fiddle with preallocated MSI descriptors */ |
209 | if (!desc->pci.mask_base) |
210 | msix_prepare_msi_desc(to_pci_dev(desc->dev), desc); |
211 | } |
212 | |
213 | static const struct msi_domain_template pci_msix_template = { |
214 | .chip = { |
215 | .name = "PCI-MSIX" , |
216 | .irq_mask = pci_irq_mask_msix, |
217 | .irq_unmask = pci_irq_unmask_msix, |
218 | .irq_write_msi_msg = pci_msi_domain_write_msg, |
219 | .flags = IRQCHIP_ONESHOT_SAFE, |
220 | }, |
221 | |
222 | .ops = { |
223 | .prepare_desc = pci_msix_prepare_desc, |
224 | .set_desc = pci_device_domain_set_desc, |
225 | }, |
226 | |
227 | .info = { |
228 | .flags = MSI_COMMON_FLAGS | MSI_FLAG_PCI_MSIX | |
229 | MSI_FLAG_PCI_MSIX_ALLOC_DYN, |
230 | .bus_token = DOMAIN_BUS_PCI_DEVICE_MSIX, |
231 | }, |
232 | }; |
233 | |
234 | static bool pci_match_device_domain(struct pci_dev *pdev, enum irq_domain_bus_token bus_token) |
235 | { |
236 | return msi_match_device_irq_domain(dev: &pdev->dev, domid: MSI_DEFAULT_DOMAIN, bus_token); |
237 | } |
238 | |
239 | static bool pci_create_device_domain(struct pci_dev *pdev, const struct msi_domain_template *tmpl, |
240 | unsigned int hwsize) |
241 | { |
242 | struct irq_domain *domain = dev_get_msi_domain(dev: &pdev->dev); |
243 | |
244 | if (!domain || !irq_domain_is_msi_parent(domain)) |
245 | return true; |
246 | |
247 | return msi_create_device_irq_domain(dev: &pdev->dev, domid: MSI_DEFAULT_DOMAIN, template: tmpl, |
248 | hwsize, NULL, NULL); |
249 | } |
250 | |
251 | /** |
252 | * pci_setup_msi_device_domain - Setup a device MSI interrupt domain |
253 | * @pdev: The PCI device to create the domain on |
254 | * |
255 | * Return: |
256 | * True when: |
257 | * - The device does not have a MSI parent irq domain associated, |
258 | * which keeps the legacy architecture specific and the global |
259 | * PCI/MSI domain models working |
260 | * - The MSI domain exists already |
261 | * - The MSI domain was successfully allocated |
262 | * False when: |
263 | * - MSI-X is enabled |
264 | * - The domain creation fails. |
265 | * |
266 | * The created MSI domain is preserved until: |
267 | * - The device is removed |
268 | * - MSI is disabled and a MSI-X domain is created |
269 | */ |
270 | bool pci_setup_msi_device_domain(struct pci_dev *pdev) |
271 | { |
272 | if (WARN_ON_ONCE(pdev->msix_enabled)) |
273 | return false; |
274 | |
275 | if (pci_match_device_domain(pdev, bus_token: DOMAIN_BUS_PCI_DEVICE_MSI)) |
276 | return true; |
277 | if (pci_match_device_domain(pdev, bus_token: DOMAIN_BUS_PCI_DEVICE_MSIX)) |
278 | msi_remove_device_irq_domain(dev: &pdev->dev, domid: MSI_DEFAULT_DOMAIN); |
279 | |
280 | return pci_create_device_domain(pdev, tmpl: &pci_msi_template, hwsize: 1); |
281 | } |
282 | |
283 | /** |
284 | * pci_setup_msix_device_domain - Setup a device MSI-X interrupt domain |
285 | * @pdev: The PCI device to create the domain on |
286 | * @hwsize: The size of the MSI-X vector table |
287 | * |
288 | * Return: |
289 | * True when: |
290 | * - The device does not have a MSI parent irq domain associated, |
291 | * which keeps the legacy architecture specific and the global |
292 | * PCI/MSI domain models working |
293 | * - The MSI-X domain exists already |
294 | * - The MSI-X domain was successfully allocated |
295 | * False when: |
296 | * - MSI is enabled |
297 | * - The domain creation fails. |
298 | * |
299 | * The created MSI-X domain is preserved until: |
300 | * - The device is removed |
301 | * - MSI-X is disabled and a MSI domain is created |
302 | */ |
303 | bool pci_setup_msix_device_domain(struct pci_dev *pdev, unsigned int hwsize) |
304 | { |
305 | if (WARN_ON_ONCE(pdev->msi_enabled)) |
306 | return false; |
307 | |
308 | if (pci_match_device_domain(pdev, bus_token: DOMAIN_BUS_PCI_DEVICE_MSIX)) |
309 | return true; |
310 | if (pci_match_device_domain(pdev, bus_token: DOMAIN_BUS_PCI_DEVICE_MSI)) |
311 | msi_remove_device_irq_domain(dev: &pdev->dev, domid: MSI_DEFAULT_DOMAIN); |
312 | |
313 | return pci_create_device_domain(pdev, tmpl: &pci_msix_template, hwsize); |
314 | } |
315 | |
316 | /** |
317 | * pci_msi_domain_supports - Check for support of a particular feature flag |
318 | * @pdev: The PCI device to operate on |
319 | * @feature_mask: The feature mask to check for (full match) |
320 | * @mode: If ALLOW_LEGACY this grants the feature when there is no irq domain |
321 | * associated to the device. If DENY_LEGACY the lack of an irq domain |
322 | * makes the feature unsupported |
323 | */ |
324 | bool pci_msi_domain_supports(struct pci_dev *pdev, unsigned int feature_mask, |
325 | enum support_mode mode) |
326 | { |
327 | struct msi_domain_info *info; |
328 | struct irq_domain *domain; |
329 | unsigned int supported; |
330 | |
331 | domain = dev_get_msi_domain(dev: &pdev->dev); |
332 | |
333 | if (!domain || !irq_domain_is_hierarchy(domain)) |
334 | return mode == ALLOW_LEGACY; |
335 | |
336 | if (!irq_domain_is_msi_parent(domain)) { |
337 | /* |
338 | * For "global" PCI/MSI interrupt domains the associated |
339 | * msi_domain_info::flags is the authoritative source of |
340 | * information. |
341 | */ |
342 | info = domain->host_data; |
343 | supported = info->flags; |
344 | } else { |
345 | /* |
346 | * For MSI parent domains the supported feature set |
347 | * is available in the parent ops. This makes checks |
348 | * possible before actually instantiating the |
349 | * per device domain because the parent is never |
350 | * expanding the PCI/MSI functionality. |
351 | */ |
352 | supported = domain->msi_parent_ops->supported_flags; |
353 | } |
354 | |
355 | return (supported & feature_mask) == feature_mask; |
356 | } |
357 | |
358 | /** |
359 | * pci_create_ims_domain - Create a secondary IMS domain for a PCI device |
360 | * @pdev: The PCI device to operate on |
361 | * @template: The MSI info template which describes the domain |
362 | * @hwsize: The size of the hardware entry table or 0 if the domain |
363 | * is purely software managed |
364 | * @data: Optional pointer to domain specific data to be stored |
365 | * in msi_domain_info::data |
366 | * |
367 | * Return: True on success, false otherwise |
368 | * |
369 | * An IMS domain is expected to have the following constraints: |
370 | * - The index space is managed by the core code |
371 | * |
372 | * - There is no requirement for consecutive index ranges |
373 | * |
374 | * - The interrupt chip must provide the following callbacks: |
375 | * - irq_mask() |
376 | * - irq_unmask() |
377 | * - irq_write_msi_msg() |
378 | * |
379 | * - The interrupt chip must provide the following optional callbacks |
380 | * when the irq_mask(), irq_unmask() and irq_write_msi_msg() callbacks |
381 | * cannot operate directly on hardware, e.g. in the case that the |
382 | * interrupt message store is in queue memory: |
383 | * - irq_bus_lock() |
384 | * - irq_bus_unlock() |
385 | * |
386 | * These callbacks are invoked from preemptible task context and are |
387 | * allowed to sleep. In this case the mandatory callbacks above just |
388 | * store the information. The irq_bus_unlock() callback is supposed |
389 | * to make the change effective before returning. |
390 | * |
391 | * - Interrupt affinity setting is handled by the underlying parent |
392 | * interrupt domain and communicated to the IMS domain via |
393 | * irq_write_msi_msg(). |
394 | * |
395 | * The domain is automatically destroyed when the PCI device is removed. |
396 | */ |
397 | bool pci_create_ims_domain(struct pci_dev *pdev, const struct msi_domain_template *template, |
398 | unsigned int hwsize, void *data) |
399 | { |
400 | struct irq_domain *domain = dev_get_msi_domain(dev: &pdev->dev); |
401 | |
402 | if (!domain || !irq_domain_is_msi_parent(domain)) |
403 | return false; |
404 | |
405 | if (template->info.bus_token != DOMAIN_BUS_PCI_DEVICE_IMS || |
406 | !(template->info.flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS) || |
407 | !(template->info.flags & MSI_FLAG_FREE_MSI_DESCS) || |
408 | !template->chip.irq_mask || !template->chip.irq_unmask || |
409 | !template->chip.irq_write_msi_msg || template->chip.irq_set_affinity) |
410 | return false; |
411 | |
412 | return msi_create_device_irq_domain(dev: &pdev->dev, domid: MSI_SECONDARY_DOMAIN, template, |
413 | hwsize, domain_data: data, NULL); |
414 | } |
415 | EXPORT_SYMBOL_GPL(pci_create_ims_domain); |
416 | |
417 | /* |
418 | * Users of the generic MSI infrastructure expect a device to have a single ID, |
419 | * so with DMA aliases we have to pick the least-worst compromise. Devices with |
420 | * DMA phantom functions tend to still emit MSIs from the real function number, |
421 | * so we ignore those and only consider topological aliases where either the |
422 | * alias device or RID appears on a different bus number. We also make the |
423 | * reasonable assumption that bridges are walked in an upstream direction (so |
424 | * the last one seen wins), and the much braver assumption that the most likely |
425 | * case is that of PCI->PCIe so we should always use the alias RID. This echoes |
426 | * the logic from intel_irq_remapping's set_msi_sid(), which presumably works |
427 | * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions |
428 | * for taking ownership all we can really do is close our eyes and hope... |
429 | */ |
430 | static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data) |
431 | { |
432 | u32 *pa = data; |
433 | u8 bus = PCI_BUS_NUM(*pa); |
434 | |
435 | if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus) |
436 | *pa = alias; |
437 | |
438 | return 0; |
439 | } |
440 | |
441 | /** |
442 | * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID) |
443 | * @domain: The interrupt domain |
444 | * @pdev: The PCI device. |
445 | * |
446 | * The RID for a device is formed from the alias, with a firmware |
447 | * supplied mapping applied |
448 | * |
449 | * Returns: The RID. |
450 | */ |
451 | u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev) |
452 | { |
453 | struct device_node *of_node; |
454 | u32 rid = pci_dev_id(dev: pdev); |
455 | |
456 | pci_for_each_dma_alias(pdev, fn: get_msi_id_cb, data: &rid); |
457 | |
458 | of_node = irq_domain_get_of_node(d: domain); |
459 | rid = of_node ? of_msi_map_id(dev: &pdev->dev, msi_np: of_node, id_in: rid) : |
460 | iort_msi_map_id(dev: &pdev->dev, id: rid); |
461 | |
462 | return rid; |
463 | } |
464 | |
465 | /** |
466 | * pci_msi_get_device_domain - Get the MSI domain for a given PCI device |
467 | * @pdev: The PCI device |
468 | * |
469 | * Use the firmware data to find a device-specific MSI domain |
470 | * (i.e. not one that is set as a default). |
471 | * |
472 | * Returns: The corresponding MSI domain or NULL if none has been found. |
473 | */ |
474 | struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) |
475 | { |
476 | struct irq_domain *dom; |
477 | u32 rid = pci_dev_id(dev: pdev); |
478 | |
479 | pci_for_each_dma_alias(pdev, fn: get_msi_id_cb, data: &rid); |
480 | dom = of_msi_map_get_device_domain(dev: &pdev->dev, id: rid, bus_token: DOMAIN_BUS_PCI_MSI); |
481 | if (!dom) |
482 | dom = iort_get_device_domain(dev: &pdev->dev, id: rid, |
483 | bus_token: DOMAIN_BUS_PCI_MSI); |
484 | return dom; |
485 | } |
486 | |