1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * PCI MSI/MSI-X — Exported APIs for device drivers |
4 | * |
5 | * Copyright (C) 2003-2004 Intel |
6 | * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) |
7 | * Copyright (C) 2016 Christoph Hellwig. |
8 | * Copyright (C) 2022 Linutronix GmbH |
9 | */ |
10 | |
11 | #include <linux/export.h> |
12 | #include <linux/irq.h> |
13 | |
14 | #include "msi.h" |
15 | |
16 | /** |
17 | * pci_enable_msi() - Enable MSI interrupt mode on device |
18 | * @dev: the PCI device to operate on |
19 | * |
20 | * Legacy device driver API to enable MSI interrupts mode on device and |
21 | * allocate a single interrupt vector. On success, the allocated vector |
22 | * Linux IRQ will be saved at @dev->irq. The driver must invoke |
23 | * pci_disable_msi() on cleanup. |
24 | * |
25 | * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API |
26 | * pair should, in general, be used instead. |
27 | * |
28 | * Return: 0 on success, errno otherwise |
29 | */ |
30 | int pci_enable_msi(struct pci_dev *dev) |
31 | { |
32 | int rc = __pci_enable_msi_range(dev, minvec: 1, maxvec: 1, NULL); |
33 | if (rc < 0) |
34 | return rc; |
35 | return 0; |
36 | } |
37 | EXPORT_SYMBOL(pci_enable_msi); |
38 | |
39 | /** |
40 | * pci_disable_msi() - Disable MSI interrupt mode on device |
41 | * @dev: the PCI device to operate on |
42 | * |
43 | * Legacy device driver API to disable MSI interrupt mode on device, |
44 | * free earlier allocated interrupt vectors, and restore INTx emulation. |
45 | * The PCI device Linux IRQ (@dev->irq) is restored to its default |
46 | * pin-assertion IRQ. This is the cleanup pair of pci_enable_msi(). |
47 | * |
48 | * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API |
49 | * pair should, in general, be used instead. |
50 | */ |
51 | void pci_disable_msi(struct pci_dev *dev) |
52 | { |
53 | if (!pci_msi_enabled() || !dev || !dev->msi_enabled) |
54 | return; |
55 | |
56 | msi_lock_descs(dev: &dev->dev); |
57 | pci_msi_shutdown(dev); |
58 | pci_free_msi_irqs(dev); |
59 | msi_unlock_descs(dev: &dev->dev); |
60 | } |
61 | EXPORT_SYMBOL(pci_disable_msi); |
62 | |
63 | /** |
64 | * pci_msix_vec_count() - Get number of MSI-X interrupt vectors on device |
65 | * @dev: the PCI device to operate on |
66 | * |
67 | * Return: number of MSI-X interrupt vectors available on this device |
68 | * (i.e., the device's MSI-X capability structure "table size"), -EINVAL |
69 | * if the device is not MSI-X capable, other errnos otherwise. |
70 | */ |
71 | int pci_msix_vec_count(struct pci_dev *dev) |
72 | { |
73 | u16 control; |
74 | |
75 | if (!dev->msix_cap) |
76 | return -EINVAL; |
77 | |
78 | pci_read_config_word(dev, where: dev->msix_cap + PCI_MSIX_FLAGS, val: &control); |
79 | return msix_table_size(control); |
80 | } |
81 | EXPORT_SYMBOL(pci_msix_vec_count); |
82 | |
83 | /** |
84 | * pci_enable_msix_range() - Enable MSI-X interrupt mode on device |
85 | * @dev: the PCI device to operate on |
86 | * @entries: input/output parameter, array of MSI-X configuration entries |
87 | * @minvec: minimum required number of MSI-X vectors |
88 | * @maxvec: maximum desired number of MSI-X vectors |
89 | * |
90 | * Legacy device driver API to enable MSI-X interrupt mode on device and |
91 | * configure its MSI-X capability structure as appropriate. The passed |
92 | * @entries array must have each of its members "entry" field set to a |
93 | * desired (valid) MSI-X vector number, where the range of valid MSI-X |
94 | * vector numbers can be queried through pci_msix_vec_count(). If |
95 | * successful, the driver must invoke pci_disable_msix() on cleanup. |
96 | * |
97 | * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API |
98 | * pair should, in general, be used instead. |
99 | * |
100 | * Return: number of MSI-X vectors allocated (which might be smaller |
101 | * than @maxvecs), where Linux IRQ numbers for such allocated vectors |
102 | * are saved back in the @entries array elements' "vector" field. Return |
103 | * -ENOSPC if less than @minvecs interrupt vectors are available. |
104 | * Return -EINVAL if one of the passed @entries members "entry" field |
105 | * was invalid or a duplicate, or if plain MSI interrupts mode was |
106 | * earlier enabled on device. Return other errnos otherwise. |
107 | */ |
108 | int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, |
109 | int minvec, int maxvec) |
110 | { |
111 | return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, flags: 0); |
112 | } |
113 | EXPORT_SYMBOL(pci_enable_msix_range); |
114 | |
115 | /** |
116 | * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling |
117 | * MSI-X is supported |
118 | * |
119 | * @dev: PCI device to operate on |
120 | * |
121 | * Return: True if supported, false otherwise |
122 | */ |
123 | bool pci_msix_can_alloc_dyn(struct pci_dev *dev) |
124 | { |
125 | if (!dev->msix_cap) |
126 | return false; |
127 | |
128 | return pci_msi_domain_supports(dev, feature_mask: MSI_FLAG_PCI_MSIX_ALLOC_DYN, mode: DENY_LEGACY); |
129 | } |
130 | EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn); |
131 | |
132 | /** |
133 | * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X |
134 | * at a given MSI-X vector index or any free vector index |
135 | * |
136 | * @dev: PCI device to operate on |
137 | * @index: Index to allocate. If @index == MSI_ANY_INDEX this allocates |
138 | * the next free index in the MSI-X table |
139 | * @affdesc: Optional pointer to an affinity descriptor structure. NULL otherwise |
140 | * |
141 | * Return: A struct msi_map |
142 | * |
143 | * On success msi_map::index contains the allocated index (>= 0) and |
144 | * msi_map::virq contains the allocated Linux interrupt number (> 0). |
145 | * |
146 | * On fail msi_map::index contains the error code and msi_map::virq |
147 | * is set to 0. |
148 | */ |
149 | struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index, |
150 | const struct irq_affinity_desc *affdesc) |
151 | { |
152 | struct msi_map map = { .index = -ENOTSUPP }; |
153 | |
154 | if (!dev->msix_enabled) |
155 | return map; |
156 | |
157 | if (!pci_msix_can_alloc_dyn(dev)) |
158 | return map; |
159 | |
160 | return msi_domain_alloc_irq_at(dev: &dev->dev, domid: MSI_DEFAULT_DOMAIN, index, affdesc, NULL); |
161 | } |
162 | EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at); |
163 | |
164 | /** |
165 | * pci_msix_free_irq - Free an interrupt on a PCI/MSIX interrupt domain |
166 | * |
167 | * @dev: The PCI device to operate on |
168 | * @map: A struct msi_map describing the interrupt to free |
169 | * |
170 | * Undo an interrupt vector allocation. Does not disable MSI-X. |
171 | */ |
172 | void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map) |
173 | { |
174 | if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0)) |
175 | return; |
176 | if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev))) |
177 | return; |
178 | msi_domain_free_irqs_range(dev: &dev->dev, domid: MSI_DEFAULT_DOMAIN, first: map.index, last: map.index); |
179 | } |
180 | EXPORT_SYMBOL_GPL(pci_msix_free_irq); |
181 | |
182 | /** |
183 | * pci_disable_msix() - Disable MSI-X interrupt mode on device |
184 | * @dev: the PCI device to operate on |
185 | * |
186 | * Legacy device driver API to disable MSI-X interrupt mode on device, |
187 | * free earlier-allocated interrupt vectors, and restore INTx. |
188 | * The PCI device Linux IRQ (@dev->irq) is restored to its default pin |
189 | * assertion IRQ. This is the cleanup pair of pci_enable_msix_range(). |
190 | * |
191 | * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API |
192 | * pair should, in general, be used instead. |
193 | */ |
194 | void pci_disable_msix(struct pci_dev *dev) |
195 | { |
196 | if (!pci_msi_enabled() || !dev || !dev->msix_enabled) |
197 | return; |
198 | |
199 | msi_lock_descs(dev: &dev->dev); |
200 | pci_msix_shutdown(dev); |
201 | pci_free_msi_irqs(dev); |
202 | msi_unlock_descs(dev: &dev->dev); |
203 | } |
204 | EXPORT_SYMBOL(pci_disable_msix); |
205 | |
206 | /** |
207 | * pci_alloc_irq_vectors() - Allocate multiple device interrupt vectors |
208 | * @dev: the PCI device to operate on |
209 | * @min_vecs: minimum required number of vectors (must be >= 1) |
210 | * @max_vecs: maximum desired number of vectors |
211 | * @flags: One or more of: |
212 | * |
213 | * * %PCI_IRQ_MSIX Allow trying MSI-X vector allocations |
214 | * * %PCI_IRQ_MSI Allow trying MSI vector allocations |
215 | * |
216 | * * %PCI_IRQ_LEGACY Allow trying legacy INTx interrupts, if |
217 | * and only if @min_vecs == 1 |
218 | * |
219 | * * %PCI_IRQ_AFFINITY Auto-manage IRQs affinity by spreading |
220 | * the vectors around available CPUs |
221 | * |
222 | * Allocate up to @max_vecs interrupt vectors on device. MSI-X irq |
223 | * vector allocation has a higher precedence over plain MSI, which has a |
224 | * higher precedence over legacy INTx emulation. |
225 | * |
226 | * Upon a successful allocation, the caller should use pci_irq_vector() |
227 | * to get the Linux IRQ number to be passed to request_threaded_irq(). |
228 | * The driver must call pci_free_irq_vectors() on cleanup. |
229 | * |
230 | * Return: number of allocated vectors (which might be smaller than |
231 | * @max_vecs), -ENOSPC if less than @min_vecs interrupt vectors are |
232 | * available, other errnos otherwise. |
233 | */ |
234 | int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, |
235 | unsigned int max_vecs, unsigned int flags) |
236 | { |
237 | return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs, |
238 | flags, NULL); |
239 | } |
240 | EXPORT_SYMBOL(pci_alloc_irq_vectors); |
241 | |
242 | /** |
243 | * pci_alloc_irq_vectors_affinity() - Allocate multiple device interrupt |
244 | * vectors with affinity requirements |
245 | * @dev: the PCI device to operate on |
246 | * @min_vecs: minimum required number of vectors (must be >= 1) |
247 | * @max_vecs: maximum desired number of vectors |
248 | * @flags: allocation flags, as in pci_alloc_irq_vectors() |
249 | * @affd: affinity requirements (can be %NULL). |
250 | * |
251 | * Same as pci_alloc_irq_vectors(), but with the extra @affd parameter. |
252 | * Check that function docs, and &struct irq_affinity, for more details. |
253 | */ |
254 | int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs, |
255 | unsigned int max_vecs, unsigned int flags, |
256 | struct irq_affinity *affd) |
257 | { |
258 | struct irq_affinity msi_default_affd = {0}; |
259 | int nvecs = -ENOSPC; |
260 | |
261 | if (flags & PCI_IRQ_AFFINITY) { |
262 | if (!affd) |
263 | affd = &msi_default_affd; |
264 | } else { |
265 | if (WARN_ON(affd)) |
266 | affd = NULL; |
267 | } |
268 | |
269 | if (flags & PCI_IRQ_MSIX) { |
270 | nvecs = __pci_enable_msix_range(dev, NULL, minvec: min_vecs, maxvec: max_vecs, |
271 | affd, flags); |
272 | if (nvecs > 0) |
273 | return nvecs; |
274 | } |
275 | |
276 | if (flags & PCI_IRQ_MSI) { |
277 | nvecs = __pci_enable_msi_range(dev, minvec: min_vecs, maxvec: max_vecs, affd); |
278 | if (nvecs > 0) |
279 | return nvecs; |
280 | } |
281 | |
282 | /* use legacy IRQ if allowed */ |
283 | if (flags & PCI_IRQ_LEGACY) { |
284 | if (min_vecs == 1 && dev->irq) { |
285 | /* |
286 | * Invoke the affinity spreading logic to ensure that |
287 | * the device driver can adjust queue configuration |
288 | * for the single interrupt case. |
289 | */ |
290 | if (affd) |
291 | irq_create_affinity_masks(nvec: 1, affd); |
292 | pci_intx(dev, enable: 1); |
293 | return 1; |
294 | } |
295 | } |
296 | |
297 | return nvecs; |
298 | } |
299 | EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity); |
300 | |
301 | /** |
302 | * pci_irq_vector() - Get Linux IRQ number of a device interrupt vector |
303 | * @dev: the PCI device to operate on |
304 | * @nr: device-relative interrupt vector index (0-based); has different |
305 | * meanings, depending on interrupt mode: |
306 | * |
307 | * * MSI-X the index in the MSI-X vector table |
308 | * * MSI the index of the enabled MSI vectors |
309 | * * INTx must be 0 |
310 | * |
311 | * Return: the Linux IRQ number, or -EINVAL if @nr is out of range |
312 | */ |
313 | int pci_irq_vector(struct pci_dev *dev, unsigned int nr) |
314 | { |
315 | unsigned int irq; |
316 | |
317 | if (!dev->msi_enabled && !dev->msix_enabled) |
318 | return !nr ? dev->irq : -EINVAL; |
319 | |
320 | irq = msi_get_virq(dev: &dev->dev, index: nr); |
321 | return irq ? irq : -EINVAL; |
322 | } |
323 | EXPORT_SYMBOL(pci_irq_vector); |
324 | |
325 | /** |
326 | * pci_irq_get_affinity() - Get a device interrupt vector affinity |
327 | * @dev: the PCI device to operate on |
328 | * @nr: device-relative interrupt vector index (0-based); has different |
329 | * meanings, depending on interrupt mode: |
330 | * |
331 | * * MSI-X the index in the MSI-X vector table |
332 | * * MSI the index of the enabled MSI vectors |
333 | * * INTx must be 0 |
334 | * |
335 | * Return: MSI/MSI-X vector affinity, NULL if @nr is out of range or if |
336 | * the MSI(-X) vector was allocated without explicit affinity |
337 | * requirements (e.g., by pci_enable_msi(), pci_enable_msix_range(), or |
338 | * pci_alloc_irq_vectors() without the %PCI_IRQ_AFFINITY flag). Return a |
339 | * generic set of CPU IDs representing all possible CPUs available |
340 | * during system boot if the device is in legacy INTx mode. |
341 | */ |
342 | const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr) |
343 | { |
344 | int idx, irq = pci_irq_vector(dev, nr); |
345 | struct msi_desc *desc; |
346 | |
347 | if (WARN_ON_ONCE(irq <= 0)) |
348 | return NULL; |
349 | |
350 | desc = irq_get_msi_desc(irq); |
351 | /* Non-MSI does not have the information handy */ |
352 | if (!desc) |
353 | return cpu_possible_mask; |
354 | |
355 | /* MSI[X] interrupts can be allocated without affinity descriptor */ |
356 | if (!desc->affinity) |
357 | return NULL; |
358 | |
359 | /* |
360 | * MSI has a mask array in the descriptor. |
361 | * MSI-X has a single mask. |
362 | */ |
363 | idx = dev->msi_enabled ? nr : 0; |
364 | return &desc->affinity[idx].mask; |
365 | } |
366 | EXPORT_SYMBOL(pci_irq_get_affinity); |
367 | |
368 | /** |
369 | * pci_ims_alloc_irq - Allocate an interrupt on a PCI/IMS interrupt domain |
370 | * @dev: The PCI device to operate on |
371 | * @icookie: Pointer to an IMS implementation specific cookie for this |
372 | * IMS instance (PASID, queue ID, pointer...). |
373 | * The cookie content is copied into the MSI descriptor for the |
374 | * interrupt chip callbacks or domain specific setup functions. |
375 | * @affdesc: Optional pointer to an interrupt affinity descriptor |
376 | * |
377 | * There is no index for IMS allocations as IMS is an implementation |
378 | * specific storage and does not have any direct associations between |
379 | * index, which might be a pure software construct, and device |
380 | * functionality. This association is established by the driver either via |
381 | * the index - if there is a hardware table - or in case of purely software |
382 | * managed IMS implementation the association happens via the |
383 | * irq_write_msi_msg() callback of the implementation specific interrupt |
384 | * chip, which utilizes the provided @icookie to store the MSI message in |
385 | * the appropriate place. |
386 | * |
387 | * Return: A struct msi_map |
388 | * |
389 | * On success msi_map::index contains the allocated index (>= 0) and |
390 | * msi_map::virq the allocated Linux interrupt number (> 0). |
391 | * |
392 | * On fail msi_map::index contains the error code and msi_map::virq |
393 | * is set to 0. |
394 | */ |
395 | struct msi_map pci_ims_alloc_irq(struct pci_dev *dev, union msi_instance_cookie *icookie, |
396 | const struct irq_affinity_desc *affdesc) |
397 | { |
398 | return msi_domain_alloc_irq_at(dev: &dev->dev, domid: MSI_SECONDARY_DOMAIN, MSI_ANY_INDEX, |
399 | affdesc, cookie: icookie); |
400 | } |
401 | EXPORT_SYMBOL_GPL(pci_ims_alloc_irq); |
402 | |
403 | /** |
404 | * pci_ims_free_irq - Allocate an interrupt on a PCI/IMS interrupt domain |
405 | * which was allocated via pci_ims_alloc_irq() |
406 | * @dev: The PCI device to operate on |
407 | * @map: A struct msi_map describing the interrupt to free as |
408 | * returned from pci_ims_alloc_irq() |
409 | */ |
410 | void pci_ims_free_irq(struct pci_dev *dev, struct msi_map map) |
411 | { |
412 | if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0)) |
413 | return; |
414 | msi_domain_free_irqs_range(dev: &dev->dev, domid: MSI_SECONDARY_DOMAIN, first: map.index, last: map.index); |
415 | } |
416 | EXPORT_SYMBOL_GPL(pci_ims_free_irq); |
417 | |
418 | /** |
419 | * pci_free_irq_vectors() - Free previously allocated IRQs for a device |
420 | * @dev: the PCI device to operate on |
421 | * |
422 | * Undo the interrupt vector allocations and possible device MSI/MSI-X |
423 | * enablement earlier done through pci_alloc_irq_vectors_affinity() or |
424 | * pci_alloc_irq_vectors(). |
425 | */ |
426 | void pci_free_irq_vectors(struct pci_dev *dev) |
427 | { |
428 | pci_disable_msix(dev); |
429 | pci_disable_msi(dev); |
430 | } |
431 | EXPORT_SYMBOL(pci_free_irq_vectors); |
432 | |
433 | /** |
434 | * pci_restore_msi_state() - Restore cached MSI(-X) state on device |
435 | * @dev: the PCI device to operate on |
436 | * |
437 | * Write the Linux-cached MSI(-X) state back on device. This is |
438 | * typically useful upon system resume, or after an error-recovery PCI |
439 | * adapter reset. |
440 | */ |
441 | void pci_restore_msi_state(struct pci_dev *dev) |
442 | { |
443 | __pci_restore_msi_state(dev); |
444 | __pci_restore_msix_state(dev); |
445 | } |
446 | EXPORT_SYMBOL_GPL(pci_restore_msi_state); |
447 | |
448 | /** |
449 | * pci_msi_enabled() - Are MSI(-X) interrupts enabled system-wide? |
450 | * |
451 | * Return: true if MSI has not been globally disabled through ACPI FADT, |
452 | * PCI bridge quirks, or the "pci=nomsi" kernel command-line option. |
453 | */ |
454 | int pci_msi_enabled(void) |
455 | { |
456 | return pci_msi_enable; |
457 | } |
458 | EXPORT_SYMBOL(pci_msi_enabled); |
459 | |