1// SPDX-License-Identifier: GPL-2.0
2/*
3 * PCI MSI/MSI-X — Exported APIs for device drivers
4 *
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 * Copyright (C) 2016 Christoph Hellwig.
8 * Copyright (C) 2022 Linutronix GmbH
9 */
10
11#include <linux/export.h>
12#include <linux/irq.h>
13
14#include "msi.h"
15
16/**
17 * pci_enable_msi() - Enable MSI interrupt mode on device
18 * @dev: the PCI device to operate on
19 *
20 * Legacy device driver API to enable MSI interrupts mode on device and
21 * allocate a single interrupt vector. On success, the allocated vector
22 * Linux IRQ will be saved at @dev->irq. The driver must invoke
23 * pci_disable_msi() on cleanup.
24 *
25 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
26 * pair should, in general, be used instead.
27 *
28 * Return: 0 on success, errno otherwise
29 */
30int pci_enable_msi(struct pci_dev *dev)
31{
32 int rc = __pci_enable_msi_range(dev, minvec: 1, maxvec: 1, NULL);
33 if (rc < 0)
34 return rc;
35 return 0;
36}
37EXPORT_SYMBOL(pci_enable_msi);
38
39/**
40 * pci_disable_msi() - Disable MSI interrupt mode on device
41 * @dev: the PCI device to operate on
42 *
43 * Legacy device driver API to disable MSI interrupt mode on device,
44 * free earlier allocated interrupt vectors, and restore INTx emulation.
45 * The PCI device Linux IRQ (@dev->irq) is restored to its default
46 * pin-assertion IRQ. This is the cleanup pair of pci_enable_msi().
47 *
48 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
49 * pair should, in general, be used instead.
50 */
51void pci_disable_msi(struct pci_dev *dev)
52{
53 if (!pci_msi_enabled() || !dev || !dev->msi_enabled)
54 return;
55
56 msi_lock_descs(dev: &dev->dev);
57 pci_msi_shutdown(dev);
58 pci_free_msi_irqs(dev);
59 msi_unlock_descs(dev: &dev->dev);
60}
61EXPORT_SYMBOL(pci_disable_msi);
62
63/**
64 * pci_msix_vec_count() - Get number of MSI-X interrupt vectors on device
65 * @dev: the PCI device to operate on
66 *
67 * Return: number of MSI-X interrupt vectors available on this device
68 * (i.e., the device's MSI-X capability structure "table size"), -EINVAL
69 * if the device is not MSI-X capable, other errnos otherwise.
70 */
71int pci_msix_vec_count(struct pci_dev *dev)
72{
73 u16 control;
74
75 if (!dev->msix_cap)
76 return -EINVAL;
77
78 pci_read_config_word(dev, where: dev->msix_cap + PCI_MSIX_FLAGS, val: &control);
79 return msix_table_size(control);
80}
81EXPORT_SYMBOL(pci_msix_vec_count);
82
83/**
84 * pci_enable_msix_range() - Enable MSI-X interrupt mode on device
85 * @dev: the PCI device to operate on
86 * @entries: input/output parameter, array of MSI-X configuration entries
87 * @minvec: minimum required number of MSI-X vectors
88 * @maxvec: maximum desired number of MSI-X vectors
89 *
90 * Legacy device driver API to enable MSI-X interrupt mode on device and
91 * configure its MSI-X capability structure as appropriate. The passed
92 * @entries array must have each of its members "entry" field set to a
93 * desired (valid) MSI-X vector number, where the range of valid MSI-X
94 * vector numbers can be queried through pci_msix_vec_count(). If
95 * successful, the driver must invoke pci_disable_msix() on cleanup.
96 *
97 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
98 * pair should, in general, be used instead.
99 *
100 * Return: number of MSI-X vectors allocated (which might be smaller
101 * than @maxvecs), where Linux IRQ numbers for such allocated vectors
102 * are saved back in the @entries array elements' "vector" field. Return
103 * -ENOSPC if less than @minvecs interrupt vectors are available.
104 * Return -EINVAL if one of the passed @entries members "entry" field
105 * was invalid or a duplicate, or if plain MSI interrupts mode was
106 * earlier enabled on device. Return other errnos otherwise.
107 */
108int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
109 int minvec, int maxvec)
110{
111 return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, flags: 0);
112}
113EXPORT_SYMBOL(pci_enable_msix_range);
114
115/**
116 * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling
117 * MSI-X is supported
118 *
119 * @dev: PCI device to operate on
120 *
121 * Return: True if supported, false otherwise
122 */
123bool pci_msix_can_alloc_dyn(struct pci_dev *dev)
124{
125 if (!dev->msix_cap)
126 return false;
127
128 return pci_msi_domain_supports(dev, feature_mask: MSI_FLAG_PCI_MSIX_ALLOC_DYN, mode: DENY_LEGACY);
129}
130EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn);
131
132/**
133 * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X
134 * at a given MSI-X vector index or any free vector index
135 *
136 * @dev: PCI device to operate on
137 * @index: Index to allocate. If @index == MSI_ANY_INDEX this allocates
138 * the next free index in the MSI-X table
139 * @affdesc: Optional pointer to an affinity descriptor structure. NULL otherwise
140 *
141 * Return: A struct msi_map
142 *
143 * On success msi_map::index contains the allocated index (>= 0) and
144 * msi_map::virq contains the allocated Linux interrupt number (> 0).
145 *
146 * On fail msi_map::index contains the error code and msi_map::virq
147 * is set to 0.
148 */
149struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index,
150 const struct irq_affinity_desc *affdesc)
151{
152 struct msi_map map = { .index = -ENOTSUPP };
153
154 if (!dev->msix_enabled)
155 return map;
156
157 if (!pci_msix_can_alloc_dyn(dev))
158 return map;
159
160 return msi_domain_alloc_irq_at(dev: &dev->dev, domid: MSI_DEFAULT_DOMAIN, index, affdesc, NULL);
161}
162EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at);
163
164/**
165 * pci_msix_free_irq - Free an interrupt on a PCI/MSIX interrupt domain
166 *
167 * @dev: The PCI device to operate on
168 * @map: A struct msi_map describing the interrupt to free
169 *
170 * Undo an interrupt vector allocation. Does not disable MSI-X.
171 */
172void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map)
173{
174 if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))
175 return;
176 if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev)))
177 return;
178 msi_domain_free_irqs_range(dev: &dev->dev, domid: MSI_DEFAULT_DOMAIN, first: map.index, last: map.index);
179}
180EXPORT_SYMBOL_GPL(pci_msix_free_irq);
181
182/**
183 * pci_disable_msix() - Disable MSI-X interrupt mode on device
184 * @dev: the PCI device to operate on
185 *
186 * Legacy device driver API to disable MSI-X interrupt mode on device,
187 * free earlier-allocated interrupt vectors, and restore INTx.
188 * The PCI device Linux IRQ (@dev->irq) is restored to its default pin
189 * assertion IRQ. This is the cleanup pair of pci_enable_msix_range().
190 *
191 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
192 * pair should, in general, be used instead.
193 */
194void pci_disable_msix(struct pci_dev *dev)
195{
196 if (!pci_msi_enabled() || !dev || !dev->msix_enabled)
197 return;
198
199 msi_lock_descs(dev: &dev->dev);
200 pci_msix_shutdown(dev);
201 pci_free_msi_irqs(dev);
202 msi_unlock_descs(dev: &dev->dev);
203}
204EXPORT_SYMBOL(pci_disable_msix);
205
206/**
207 * pci_alloc_irq_vectors() - Allocate multiple device interrupt vectors
208 * @dev: the PCI device to operate on
209 * @min_vecs: minimum required number of vectors (must be >= 1)
210 * @max_vecs: maximum desired number of vectors
211 * @flags: One or more of:
212 *
213 * * %PCI_IRQ_MSIX Allow trying MSI-X vector allocations
214 * * %PCI_IRQ_MSI Allow trying MSI vector allocations
215 *
216 * * %PCI_IRQ_LEGACY Allow trying legacy INTx interrupts, if
217 * and only if @min_vecs == 1
218 *
219 * * %PCI_IRQ_AFFINITY Auto-manage IRQs affinity by spreading
220 * the vectors around available CPUs
221 *
222 * Allocate up to @max_vecs interrupt vectors on device. MSI-X irq
223 * vector allocation has a higher precedence over plain MSI, which has a
224 * higher precedence over legacy INTx emulation.
225 *
226 * Upon a successful allocation, the caller should use pci_irq_vector()
227 * to get the Linux IRQ number to be passed to request_threaded_irq().
228 * The driver must call pci_free_irq_vectors() on cleanup.
229 *
230 * Return: number of allocated vectors (which might be smaller than
231 * @max_vecs), -ENOSPC if less than @min_vecs interrupt vectors are
232 * available, other errnos otherwise.
233 */
234int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs,
235 unsigned int max_vecs, unsigned int flags)
236{
237 return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs,
238 flags, NULL);
239}
240EXPORT_SYMBOL(pci_alloc_irq_vectors);
241
242/**
243 * pci_alloc_irq_vectors_affinity() - Allocate multiple device interrupt
244 * vectors with affinity requirements
245 * @dev: the PCI device to operate on
246 * @min_vecs: minimum required number of vectors (must be >= 1)
247 * @max_vecs: maximum desired number of vectors
248 * @flags: allocation flags, as in pci_alloc_irq_vectors()
249 * @affd: affinity requirements (can be %NULL).
250 *
251 * Same as pci_alloc_irq_vectors(), but with the extra @affd parameter.
252 * Check that function docs, and &struct irq_affinity, for more details.
253 */
254int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
255 unsigned int max_vecs, unsigned int flags,
256 struct irq_affinity *affd)
257{
258 struct irq_affinity msi_default_affd = {0};
259 int nvecs = -ENOSPC;
260
261 if (flags & PCI_IRQ_AFFINITY) {
262 if (!affd)
263 affd = &msi_default_affd;
264 } else {
265 if (WARN_ON(affd))
266 affd = NULL;
267 }
268
269 if (flags & PCI_IRQ_MSIX) {
270 nvecs = __pci_enable_msix_range(dev, NULL, minvec: min_vecs, maxvec: max_vecs,
271 affd, flags);
272 if (nvecs > 0)
273 return nvecs;
274 }
275
276 if (flags & PCI_IRQ_MSI) {
277 nvecs = __pci_enable_msi_range(dev, minvec: min_vecs, maxvec: max_vecs, affd);
278 if (nvecs > 0)
279 return nvecs;
280 }
281
282 /* use legacy IRQ if allowed */
283 if (flags & PCI_IRQ_LEGACY) {
284 if (min_vecs == 1 && dev->irq) {
285 /*
286 * Invoke the affinity spreading logic to ensure that
287 * the device driver can adjust queue configuration
288 * for the single interrupt case.
289 */
290 if (affd)
291 irq_create_affinity_masks(nvec: 1, affd);
292 pci_intx(dev, enable: 1);
293 return 1;
294 }
295 }
296
297 return nvecs;
298}
299EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);
300
301/**
302 * pci_irq_vector() - Get Linux IRQ number of a device interrupt vector
303 * @dev: the PCI device to operate on
304 * @nr: device-relative interrupt vector index (0-based); has different
305 * meanings, depending on interrupt mode:
306 *
307 * * MSI-X the index in the MSI-X vector table
308 * * MSI the index of the enabled MSI vectors
309 * * INTx must be 0
310 *
311 * Return: the Linux IRQ number, or -EINVAL if @nr is out of range
312 */
313int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
314{
315 unsigned int irq;
316
317 if (!dev->msi_enabled && !dev->msix_enabled)
318 return !nr ? dev->irq : -EINVAL;
319
320 irq = msi_get_virq(dev: &dev->dev, index: nr);
321 return irq ? irq : -EINVAL;
322}
323EXPORT_SYMBOL(pci_irq_vector);
324
325/**
326 * pci_irq_get_affinity() - Get a device interrupt vector affinity
327 * @dev: the PCI device to operate on
328 * @nr: device-relative interrupt vector index (0-based); has different
329 * meanings, depending on interrupt mode:
330 *
331 * * MSI-X the index in the MSI-X vector table
332 * * MSI the index of the enabled MSI vectors
333 * * INTx must be 0
334 *
335 * Return: MSI/MSI-X vector affinity, NULL if @nr is out of range or if
336 * the MSI(-X) vector was allocated without explicit affinity
337 * requirements (e.g., by pci_enable_msi(), pci_enable_msix_range(), or
338 * pci_alloc_irq_vectors() without the %PCI_IRQ_AFFINITY flag). Return a
339 * generic set of CPU IDs representing all possible CPUs available
340 * during system boot if the device is in legacy INTx mode.
341 */
342const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
343{
344 int idx, irq = pci_irq_vector(dev, nr);
345 struct msi_desc *desc;
346
347 if (WARN_ON_ONCE(irq <= 0))
348 return NULL;
349
350 desc = irq_get_msi_desc(irq);
351 /* Non-MSI does not have the information handy */
352 if (!desc)
353 return cpu_possible_mask;
354
355 /* MSI[X] interrupts can be allocated without affinity descriptor */
356 if (!desc->affinity)
357 return NULL;
358
359 /*
360 * MSI has a mask array in the descriptor.
361 * MSI-X has a single mask.
362 */
363 idx = dev->msi_enabled ? nr : 0;
364 return &desc->affinity[idx].mask;
365}
366EXPORT_SYMBOL(pci_irq_get_affinity);
367
368/**
369 * pci_ims_alloc_irq - Allocate an interrupt on a PCI/IMS interrupt domain
370 * @dev: The PCI device to operate on
371 * @icookie: Pointer to an IMS implementation specific cookie for this
372 * IMS instance (PASID, queue ID, pointer...).
373 * The cookie content is copied into the MSI descriptor for the
374 * interrupt chip callbacks or domain specific setup functions.
375 * @affdesc: Optional pointer to an interrupt affinity descriptor
376 *
377 * There is no index for IMS allocations as IMS is an implementation
378 * specific storage and does not have any direct associations between
379 * index, which might be a pure software construct, and device
380 * functionality. This association is established by the driver either via
381 * the index - if there is a hardware table - or in case of purely software
382 * managed IMS implementation the association happens via the
383 * irq_write_msi_msg() callback of the implementation specific interrupt
384 * chip, which utilizes the provided @icookie to store the MSI message in
385 * the appropriate place.
386 *
387 * Return: A struct msi_map
388 *
389 * On success msi_map::index contains the allocated index (>= 0) and
390 * msi_map::virq the allocated Linux interrupt number (> 0).
391 *
392 * On fail msi_map::index contains the error code and msi_map::virq
393 * is set to 0.
394 */
395struct msi_map pci_ims_alloc_irq(struct pci_dev *dev, union msi_instance_cookie *icookie,
396 const struct irq_affinity_desc *affdesc)
397{
398 return msi_domain_alloc_irq_at(dev: &dev->dev, domid: MSI_SECONDARY_DOMAIN, MSI_ANY_INDEX,
399 affdesc, cookie: icookie);
400}
401EXPORT_SYMBOL_GPL(pci_ims_alloc_irq);
402
403/**
404 * pci_ims_free_irq - Allocate an interrupt on a PCI/IMS interrupt domain
405 * which was allocated via pci_ims_alloc_irq()
406 * @dev: The PCI device to operate on
407 * @map: A struct msi_map describing the interrupt to free as
408 * returned from pci_ims_alloc_irq()
409 */
410void pci_ims_free_irq(struct pci_dev *dev, struct msi_map map)
411{
412 if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))
413 return;
414 msi_domain_free_irqs_range(dev: &dev->dev, domid: MSI_SECONDARY_DOMAIN, first: map.index, last: map.index);
415}
416EXPORT_SYMBOL_GPL(pci_ims_free_irq);
417
418/**
419 * pci_free_irq_vectors() - Free previously allocated IRQs for a device
420 * @dev: the PCI device to operate on
421 *
422 * Undo the interrupt vector allocations and possible device MSI/MSI-X
423 * enablement earlier done through pci_alloc_irq_vectors_affinity() or
424 * pci_alloc_irq_vectors().
425 */
426void pci_free_irq_vectors(struct pci_dev *dev)
427{
428 pci_disable_msix(dev);
429 pci_disable_msi(dev);
430}
431EXPORT_SYMBOL(pci_free_irq_vectors);
432
433/**
434 * pci_restore_msi_state() - Restore cached MSI(-X) state on device
435 * @dev: the PCI device to operate on
436 *
437 * Write the Linux-cached MSI(-X) state back on device. This is
438 * typically useful upon system resume, or after an error-recovery PCI
439 * adapter reset.
440 */
441void pci_restore_msi_state(struct pci_dev *dev)
442{
443 __pci_restore_msi_state(dev);
444 __pci_restore_msix_state(dev);
445}
446EXPORT_SYMBOL_GPL(pci_restore_msi_state);
447
448/**
449 * pci_msi_enabled() - Are MSI(-X) interrupts enabled system-wide?
450 *
451 * Return: true if MSI has not been globally disabled through ACPI FADT,
452 * PCI bridge quirks, or the "pci=nomsi" kernel command-line option.
453 */
454int pci_msi_enabled(void)
455{
456 return pci_msi_enable;
457}
458EXPORT_SYMBOL(pci_msi_enabled);
459

source code of linux/drivers/pci/msi/api.c