1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VFIO PCI interrupt handling
4 *
5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
6 * Author: Alex Williamson <alex.williamson@redhat.com>
7 *
8 * Derived from original vfio:
9 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
10 * Author: Tom Lyon, pugs@cisco.com
11 */
12
13#include <linux/device.h>
14#include <linux/interrupt.h>
15#include <linux/eventfd.h>
16#include <linux/msi.h>
17#include <linux/pci.h>
18#include <linux/file.h>
19#include <linux/vfio.h>
20#include <linux/wait.h>
21#include <linux/slab.h>
22
23#include "vfio_pci_priv.h"
24
25struct vfio_pci_irq_ctx {
26 struct eventfd_ctx *trigger;
27 struct virqfd *unmask;
28 struct virqfd *mask;
29 char *name;
30 bool masked;
31 struct irq_bypass_producer producer;
32};
33
34static bool irq_is(struct vfio_pci_core_device *vdev, int type)
35{
36 return vdev->irq_type == type;
37}
38
39static bool is_intx(struct vfio_pci_core_device *vdev)
40{
41 return vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX;
42}
43
44static bool is_irq_none(struct vfio_pci_core_device *vdev)
45{
46 return !(vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX ||
47 vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX ||
48 vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX);
49}
50
51static
52struct vfio_pci_irq_ctx *vfio_irq_ctx_get(struct vfio_pci_core_device *vdev,
53 unsigned long index)
54{
55 return xa_load(&vdev->ctx, index);
56}
57
58static void vfio_irq_ctx_free(struct vfio_pci_core_device *vdev,
59 struct vfio_pci_irq_ctx *ctx, unsigned long index)
60{
61 xa_erase(&vdev->ctx, index);
62 kfree(objp: ctx);
63}
64
65static struct vfio_pci_irq_ctx *
66vfio_irq_ctx_alloc(struct vfio_pci_core_device *vdev, unsigned long index)
67{
68 struct vfio_pci_irq_ctx *ctx;
69 int ret;
70
71 ctx = kzalloc(size: sizeof(*ctx), GFP_KERNEL_ACCOUNT);
72 if (!ctx)
73 return NULL;
74
75 ret = xa_insert(xa: &vdev->ctx, index, entry: ctx, GFP_KERNEL_ACCOUNT);
76 if (ret) {
77 kfree(objp: ctx);
78 return NULL;
79 }
80
81 return ctx;
82}
83
84/*
85 * INTx
86 */
87static void vfio_send_intx_eventfd(void *opaque, void *unused)
88{
89 struct vfio_pci_core_device *vdev = opaque;
90
91 if (likely(is_intx(vdev) && !vdev->virq_disabled)) {
92 struct vfio_pci_irq_ctx *ctx;
93 struct eventfd_ctx *trigger;
94
95 ctx = vfio_irq_ctx_get(vdev, index: 0);
96 if (WARN_ON_ONCE(!ctx))
97 return;
98
99 trigger = READ_ONCE(ctx->trigger);
100 if (likely(trigger))
101 eventfd_signal(ctx: trigger);
102 }
103}
104
105/* Returns true if the INTx vfio_pci_irq_ctx.masked value is changed. */
106static bool __vfio_pci_intx_mask(struct vfio_pci_core_device *vdev)
107{
108 struct pci_dev *pdev = vdev->pdev;
109 struct vfio_pci_irq_ctx *ctx;
110 unsigned long flags;
111 bool masked_changed = false;
112
113 lockdep_assert_held(&vdev->igate);
114
115 spin_lock_irqsave(&vdev->irqlock, flags);
116
117 /*
118 * Masking can come from interrupt, ioctl, or config space
119 * via INTx disable. The latter means this can get called
120 * even when not using intx delivery. In this case, just
121 * try to have the physical bit follow the virtual bit.
122 */
123 if (unlikely(!is_intx(vdev))) {
124 if (vdev->pci_2_3)
125 pci_intx(dev: pdev, enable: 0);
126 goto out_unlock;
127 }
128
129 ctx = vfio_irq_ctx_get(vdev, index: 0);
130 if (WARN_ON_ONCE(!ctx))
131 goto out_unlock;
132
133 if (!ctx->masked) {
134 /*
135 * Can't use check_and_mask here because we always want to
136 * mask, not just when something is pending.
137 */
138 if (vdev->pci_2_3)
139 pci_intx(dev: pdev, enable: 0);
140 else
141 disable_irq_nosync(irq: pdev->irq);
142
143 ctx->masked = true;
144 masked_changed = true;
145 }
146
147out_unlock:
148 spin_unlock_irqrestore(lock: &vdev->irqlock, flags);
149 return masked_changed;
150}
151
152bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev)
153{
154 bool mask_changed;
155
156 mutex_lock(&vdev->igate);
157 mask_changed = __vfio_pci_intx_mask(vdev);
158 mutex_unlock(lock: &vdev->igate);
159
160 return mask_changed;
161}
162
163/*
164 * If this is triggered by an eventfd, we can't call eventfd_signal
165 * or else we'll deadlock on the eventfd wait queue. Return >0 when
166 * a signal is necessary, which can then be handled via a work queue
167 * or directly depending on the caller.
168 */
169static int vfio_pci_intx_unmask_handler(void *opaque, void *unused)
170{
171 struct vfio_pci_core_device *vdev = opaque;
172 struct pci_dev *pdev = vdev->pdev;
173 struct vfio_pci_irq_ctx *ctx;
174 unsigned long flags;
175 int ret = 0;
176
177 spin_lock_irqsave(&vdev->irqlock, flags);
178
179 /*
180 * Unmasking comes from ioctl or config, so again, have the
181 * physical bit follow the virtual even when not using INTx.
182 */
183 if (unlikely(!is_intx(vdev))) {
184 if (vdev->pci_2_3)
185 pci_intx(dev: pdev, enable: 1);
186 goto out_unlock;
187 }
188
189 ctx = vfio_irq_ctx_get(vdev, index: 0);
190 if (WARN_ON_ONCE(!ctx))
191 goto out_unlock;
192
193 if (ctx->masked && !vdev->virq_disabled) {
194 /*
195 * A pending interrupt here would immediately trigger,
196 * but we can avoid that overhead by just re-sending
197 * the interrupt to the user.
198 */
199 if (vdev->pci_2_3) {
200 if (!pci_check_and_unmask_intx(dev: pdev))
201 ret = 1;
202 } else
203 enable_irq(irq: pdev->irq);
204
205 ctx->masked = (ret > 0);
206 }
207
208out_unlock:
209 spin_unlock_irqrestore(lock: &vdev->irqlock, flags);
210
211 return ret;
212}
213
214static void __vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev)
215{
216 lockdep_assert_held(&vdev->igate);
217
218 if (vfio_pci_intx_unmask_handler(opaque: vdev, NULL) > 0)
219 vfio_send_intx_eventfd(opaque: vdev, NULL);
220}
221
222void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev)
223{
224 mutex_lock(&vdev->igate);
225 __vfio_pci_intx_unmask(vdev);
226 mutex_unlock(lock: &vdev->igate);
227}
228
229static irqreturn_t vfio_intx_handler(int irq, void *dev_id)
230{
231 struct vfio_pci_core_device *vdev = dev_id;
232 struct vfio_pci_irq_ctx *ctx;
233 unsigned long flags;
234 int ret = IRQ_NONE;
235
236 ctx = vfio_irq_ctx_get(vdev, index: 0);
237 if (WARN_ON_ONCE(!ctx))
238 return ret;
239
240 spin_lock_irqsave(&vdev->irqlock, flags);
241
242 if (!vdev->pci_2_3) {
243 disable_irq_nosync(irq: vdev->pdev->irq);
244 ctx->masked = true;
245 ret = IRQ_HANDLED;
246 } else if (!ctx->masked && /* may be shared */
247 pci_check_and_mask_intx(dev: vdev->pdev)) {
248 ctx->masked = true;
249 ret = IRQ_HANDLED;
250 }
251
252 spin_unlock_irqrestore(lock: &vdev->irqlock, flags);
253
254 if (ret == IRQ_HANDLED)
255 vfio_send_intx_eventfd(opaque: vdev, NULL);
256
257 return ret;
258}
259
260static int vfio_intx_enable(struct vfio_pci_core_device *vdev,
261 struct eventfd_ctx *trigger)
262{
263 struct pci_dev *pdev = vdev->pdev;
264 struct vfio_pci_irq_ctx *ctx;
265 unsigned long irqflags;
266 char *name;
267 int ret;
268
269 if (!is_irq_none(vdev))
270 return -EINVAL;
271
272 if (!pdev->irq)
273 return -ENODEV;
274
275 name = kasprintf(GFP_KERNEL_ACCOUNT, fmt: "vfio-intx(%s)", pci_name(pdev));
276 if (!name)
277 return -ENOMEM;
278
279 ctx = vfio_irq_ctx_alloc(vdev, index: 0);
280 if (!ctx)
281 return -ENOMEM;
282
283 ctx->name = name;
284 ctx->trigger = trigger;
285
286 /*
287 * Fill the initial masked state based on virq_disabled. After
288 * enable, changing the DisINTx bit in vconfig directly changes INTx
289 * masking. igate prevents races during setup, once running masked
290 * is protected via irqlock.
291 *
292 * Devices supporting DisINTx also reflect the current mask state in
293 * the physical DisINTx bit, which is not affected during IRQ setup.
294 *
295 * Devices without DisINTx support require an exclusive interrupt.
296 * IRQ masking is performed at the IRQ chip. Again, igate protects
297 * against races during setup and IRQ handlers and irqfds are not
298 * yet active, therefore masked is stable and can be used to
299 * conditionally auto-enable the IRQ.
300 *
301 * irq_type must be stable while the IRQ handler is registered,
302 * therefore it must be set before request_irq().
303 */
304 ctx->masked = vdev->virq_disabled;
305 if (vdev->pci_2_3) {
306 pci_intx(dev: pdev, enable: !ctx->masked);
307 irqflags = IRQF_SHARED;
308 } else {
309 irqflags = ctx->masked ? IRQF_NO_AUTOEN : 0;
310 }
311
312 vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX;
313
314 ret = request_irq(irq: pdev->irq, handler: vfio_intx_handler,
315 flags: irqflags, name: ctx->name, dev: vdev);
316 if (ret) {
317 vdev->irq_type = VFIO_PCI_NUM_IRQS;
318 kfree(objp: name);
319 vfio_irq_ctx_free(vdev, ctx, index: 0);
320 return ret;
321 }
322
323 return 0;
324}
325
326static int vfio_intx_set_signal(struct vfio_pci_core_device *vdev,
327 struct eventfd_ctx *trigger)
328{
329 struct pci_dev *pdev = vdev->pdev;
330 struct vfio_pci_irq_ctx *ctx;
331 struct eventfd_ctx *old;
332
333 ctx = vfio_irq_ctx_get(vdev, index: 0);
334 if (WARN_ON_ONCE(!ctx))
335 return -EINVAL;
336
337 old = ctx->trigger;
338
339 WRITE_ONCE(ctx->trigger, trigger);
340
341 /* Releasing an old ctx requires synchronizing in-flight users */
342 if (old) {
343 synchronize_irq(irq: pdev->irq);
344 vfio_virqfd_flush_thread(pvirqfd: &ctx->unmask);
345 eventfd_ctx_put(ctx: old);
346 }
347
348 return 0;
349}
350
351static void vfio_intx_disable(struct vfio_pci_core_device *vdev)
352{
353 struct pci_dev *pdev = vdev->pdev;
354 struct vfio_pci_irq_ctx *ctx;
355
356 ctx = vfio_irq_ctx_get(vdev, index: 0);
357 WARN_ON_ONCE(!ctx);
358 if (ctx) {
359 vfio_virqfd_disable(pvirqfd: &ctx->unmask);
360 vfio_virqfd_disable(pvirqfd: &ctx->mask);
361 free_irq(pdev->irq, vdev);
362 if (ctx->trigger)
363 eventfd_ctx_put(ctx: ctx->trigger);
364 kfree(objp: ctx->name);
365 vfio_irq_ctx_free(vdev, ctx, index: 0);
366 }
367 vdev->irq_type = VFIO_PCI_NUM_IRQS;
368}
369
370/*
371 * MSI/MSI-X
372 */
373static irqreturn_t vfio_msihandler(int irq, void *arg)
374{
375 struct eventfd_ctx *trigger = arg;
376
377 eventfd_signal(ctx: trigger);
378 return IRQ_HANDLED;
379}
380
381static int vfio_msi_enable(struct vfio_pci_core_device *vdev, int nvec, bool msix)
382{
383 struct pci_dev *pdev = vdev->pdev;
384 unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI;
385 int ret;
386 u16 cmd;
387
388 if (!is_irq_none(vdev))
389 return -EINVAL;
390
391 /* return the number of supported vectors if we can't get all: */
392 cmd = vfio_pci_memory_lock_and_enable(vdev);
393 ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: nvec, flags: flag);
394 if (ret < nvec) {
395 if (ret > 0)
396 pci_free_irq_vectors(dev: pdev);
397 vfio_pci_memory_unlock_and_restore(vdev, cmd);
398 return ret;
399 }
400 vfio_pci_memory_unlock_and_restore(vdev, cmd);
401
402 vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX :
403 VFIO_PCI_MSI_IRQ_INDEX;
404
405 if (!msix) {
406 /*
407 * Compute the virtual hardware field for max msi vectors -
408 * it is the log base 2 of the number of vectors.
409 */
410 vdev->msi_qmax = fls(x: nvec * 2 - 1) - 1;
411 }
412
413 return 0;
414}
415
416/*
417 * vfio_msi_alloc_irq() returns the Linux IRQ number of an MSI or MSI-X device
418 * interrupt vector. If a Linux IRQ number is not available then a new
419 * interrupt is allocated if dynamic MSI-X is supported.
420 *
421 * Where is vfio_msi_free_irq()? Allocated interrupts are maintained,
422 * essentially forming a cache that subsequent allocations can draw from.
423 * Interrupts are freed using pci_free_irq_vectors() when MSI/MSI-X is
424 * disabled.
425 */
426static int vfio_msi_alloc_irq(struct vfio_pci_core_device *vdev,
427 unsigned int vector, bool msix)
428{
429 struct pci_dev *pdev = vdev->pdev;
430 struct msi_map map;
431 int irq;
432 u16 cmd;
433
434 irq = pci_irq_vector(dev: pdev, nr: vector);
435 if (WARN_ON_ONCE(irq == 0))
436 return -EINVAL;
437 if (irq > 0 || !msix || !vdev->has_dyn_msix)
438 return irq;
439
440 cmd = vfio_pci_memory_lock_and_enable(vdev);
441 map = pci_msix_alloc_irq_at(dev: pdev, index: vector, NULL);
442 vfio_pci_memory_unlock_and_restore(vdev, cmd);
443
444 return map.index < 0 ? map.index : map.virq;
445}
446
447static int vfio_msi_set_vector_signal(struct vfio_pci_core_device *vdev,
448 unsigned int vector, int fd, bool msix)
449{
450 struct pci_dev *pdev = vdev->pdev;
451 struct vfio_pci_irq_ctx *ctx;
452 struct eventfd_ctx *trigger;
453 int irq = -EINVAL, ret;
454 u16 cmd;
455
456 ctx = vfio_irq_ctx_get(vdev, index: vector);
457
458 if (ctx) {
459 irq_bypass_unregister_producer(&ctx->producer);
460 irq = pci_irq_vector(dev: pdev, nr: vector);
461 cmd = vfio_pci_memory_lock_and_enable(vdev);
462 free_irq(irq, ctx->trigger);
463 vfio_pci_memory_unlock_and_restore(vdev, cmd);
464 /* Interrupt stays allocated, will be freed at MSI-X disable. */
465 kfree(objp: ctx->name);
466 eventfd_ctx_put(ctx: ctx->trigger);
467 vfio_irq_ctx_free(vdev, ctx, index: vector);
468 }
469
470 if (fd < 0)
471 return 0;
472
473 if (irq == -EINVAL) {
474 /* Interrupt stays allocated, will be freed at MSI-X disable. */
475 irq = vfio_msi_alloc_irq(vdev, vector, msix);
476 if (irq < 0)
477 return irq;
478 }
479
480 ctx = vfio_irq_ctx_alloc(vdev, index: vector);
481 if (!ctx)
482 return -ENOMEM;
483
484 ctx->name = kasprintf(GFP_KERNEL_ACCOUNT, fmt: "vfio-msi%s[%d](%s)",
485 msix ? "x" : "", vector, pci_name(pdev));
486 if (!ctx->name) {
487 ret = -ENOMEM;
488 goto out_free_ctx;
489 }
490
491 trigger = eventfd_ctx_fdget(fd);
492 if (IS_ERR(ptr: trigger)) {
493 ret = PTR_ERR(ptr: trigger);
494 goto out_free_name;
495 }
496
497 /*
498 * If the vector was previously allocated, refresh the on-device
499 * message data before enabling in case it had been cleared or
500 * corrupted (e.g. due to backdoor resets) since writing.
501 */
502 cmd = vfio_pci_memory_lock_and_enable(vdev);
503 if (msix) {
504 struct msi_msg msg;
505
506 get_cached_msi_msg(irq, msg: &msg);
507 pci_write_msi_msg(irq, msg: &msg);
508 }
509
510 ret = request_irq(irq, handler: vfio_msihandler, flags: 0, name: ctx->name, dev: trigger);
511 vfio_pci_memory_unlock_and_restore(vdev, cmd);
512 if (ret)
513 goto out_put_eventfd_ctx;
514
515 ctx->producer.token = trigger;
516 ctx->producer.irq = irq;
517 ret = irq_bypass_register_producer(&ctx->producer);
518 if (unlikely(ret)) {
519 dev_info(&pdev->dev,
520 "irq bypass producer (token %p) registration fails: %d\n",
521 ctx->producer.token, ret);
522
523 ctx->producer.token = NULL;
524 }
525 ctx->trigger = trigger;
526
527 return 0;
528
529out_put_eventfd_ctx:
530 eventfd_ctx_put(ctx: trigger);
531out_free_name:
532 kfree(objp: ctx->name);
533out_free_ctx:
534 vfio_irq_ctx_free(vdev, ctx, index: vector);
535 return ret;
536}
537
538static int vfio_msi_set_block(struct vfio_pci_core_device *vdev, unsigned start,
539 unsigned count, int32_t *fds, bool msix)
540{
541 unsigned int i, j;
542 int ret = 0;
543
544 for (i = 0, j = start; i < count && !ret; i++, j++) {
545 int fd = fds ? fds[i] : -1;
546 ret = vfio_msi_set_vector_signal(vdev, vector: j, fd, msix);
547 }
548
549 if (ret) {
550 for (i = start; i < j; i++)
551 vfio_msi_set_vector_signal(vdev, vector: i, fd: -1, msix);
552 }
553
554 return ret;
555}
556
557static void vfio_msi_disable(struct vfio_pci_core_device *vdev, bool msix)
558{
559 struct pci_dev *pdev = vdev->pdev;
560 struct vfio_pci_irq_ctx *ctx;
561 unsigned long i;
562 u16 cmd;
563
564 xa_for_each(&vdev->ctx, i, ctx) {
565 vfio_virqfd_disable(pvirqfd: &ctx->unmask);
566 vfio_virqfd_disable(pvirqfd: &ctx->mask);
567 vfio_msi_set_vector_signal(vdev, vector: i, fd: -1, msix);
568 }
569
570 cmd = vfio_pci_memory_lock_and_enable(vdev);
571 pci_free_irq_vectors(dev: pdev);
572 vfio_pci_memory_unlock_and_restore(vdev, cmd);
573
574 /*
575 * Both disable paths above use pci_intx_for_msi() to clear DisINTx
576 * via their shutdown paths. Restore for NoINTx devices.
577 */
578 if (vdev->nointx)
579 pci_intx(dev: pdev, enable: 0);
580
581 vdev->irq_type = VFIO_PCI_NUM_IRQS;
582}
583
584/*
585 * IOCTL support
586 */
587static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev,
588 unsigned index, unsigned start,
589 unsigned count, uint32_t flags, void *data)
590{
591 if (!is_intx(vdev) || start != 0 || count != 1)
592 return -EINVAL;
593
594 if (flags & VFIO_IRQ_SET_DATA_NONE) {
595 __vfio_pci_intx_unmask(vdev);
596 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
597 uint8_t unmask = *(uint8_t *)data;
598 if (unmask)
599 __vfio_pci_intx_unmask(vdev);
600 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
601 struct vfio_pci_irq_ctx *ctx = vfio_irq_ctx_get(vdev, index: 0);
602 int32_t fd = *(int32_t *)data;
603
604 if (WARN_ON_ONCE(!ctx))
605 return -EINVAL;
606 if (fd >= 0)
607 return vfio_virqfd_enable(opaque: (void *) vdev,
608 handler: vfio_pci_intx_unmask_handler,
609 thread: vfio_send_intx_eventfd, NULL,
610 pvirqfd: &ctx->unmask, fd);
611
612 vfio_virqfd_disable(pvirqfd: &ctx->unmask);
613 }
614
615 return 0;
616}
617
618static int vfio_pci_set_intx_mask(struct vfio_pci_core_device *vdev,
619 unsigned index, unsigned start,
620 unsigned count, uint32_t flags, void *data)
621{
622 if (!is_intx(vdev) || start != 0 || count != 1)
623 return -EINVAL;
624
625 if (flags & VFIO_IRQ_SET_DATA_NONE) {
626 __vfio_pci_intx_mask(vdev);
627 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
628 uint8_t mask = *(uint8_t *)data;
629 if (mask)
630 __vfio_pci_intx_mask(vdev);
631 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
632 return -ENOTTY; /* XXX implement me */
633 }
634
635 return 0;
636}
637
638static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev,
639 unsigned index, unsigned start,
640 unsigned count, uint32_t flags, void *data)
641{
642 if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
643 vfio_intx_disable(vdev);
644 return 0;
645 }
646
647 if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1)
648 return -EINVAL;
649
650 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
651 struct eventfd_ctx *trigger = NULL;
652 int32_t fd = *(int32_t *)data;
653 int ret;
654
655 if (fd >= 0) {
656 trigger = eventfd_ctx_fdget(fd);
657 if (IS_ERR(ptr: trigger))
658 return PTR_ERR(ptr: trigger);
659 }
660
661 if (is_intx(vdev))
662 ret = vfio_intx_set_signal(vdev, trigger);
663 else
664 ret = vfio_intx_enable(vdev, trigger);
665
666 if (ret && trigger)
667 eventfd_ctx_put(ctx: trigger);
668
669 return ret;
670 }
671
672 if (!is_intx(vdev))
673 return -EINVAL;
674
675 if (flags & VFIO_IRQ_SET_DATA_NONE) {
676 vfio_send_intx_eventfd(opaque: vdev, NULL);
677 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
678 uint8_t trigger = *(uint8_t *)data;
679 if (trigger)
680 vfio_send_intx_eventfd(opaque: vdev, NULL);
681 }
682 return 0;
683}
684
685static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev,
686 unsigned index, unsigned start,
687 unsigned count, uint32_t flags, void *data)
688{
689 struct vfio_pci_irq_ctx *ctx;
690 unsigned int i;
691 bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false;
692
693 if (irq_is(vdev, type: index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
694 vfio_msi_disable(vdev, msix);
695 return 0;
696 }
697
698 if (!(irq_is(vdev, type: index) || is_irq_none(vdev)))
699 return -EINVAL;
700
701 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
702 int32_t *fds = data;
703 int ret;
704
705 if (vdev->irq_type == index)
706 return vfio_msi_set_block(vdev, start, count,
707 fds, msix);
708
709 ret = vfio_msi_enable(vdev, nvec: start + count, msix);
710 if (ret)
711 return ret;
712
713 ret = vfio_msi_set_block(vdev, start, count, fds, msix);
714 if (ret)
715 vfio_msi_disable(vdev, msix);
716
717 return ret;
718 }
719
720 if (!irq_is(vdev, type: index))
721 return -EINVAL;
722
723 for (i = start; i < start + count; i++) {
724 ctx = vfio_irq_ctx_get(vdev, index: i);
725 if (!ctx)
726 continue;
727 if (flags & VFIO_IRQ_SET_DATA_NONE) {
728 eventfd_signal(ctx: ctx->trigger);
729 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
730 uint8_t *bools = data;
731 if (bools[i - start])
732 eventfd_signal(ctx: ctx->trigger);
733 }
734 }
735 return 0;
736}
737
738static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx,
739 unsigned int count, uint32_t flags,
740 void *data)
741{
742 /* DATA_NONE/DATA_BOOL enables loopback testing */
743 if (flags & VFIO_IRQ_SET_DATA_NONE) {
744 if (*ctx) {
745 if (count) {
746 eventfd_signal(ctx: *ctx);
747 } else {
748 eventfd_ctx_put(ctx: *ctx);
749 *ctx = NULL;
750 }
751 return 0;
752 }
753 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
754 uint8_t trigger;
755
756 if (!count)
757 return -EINVAL;
758
759 trigger = *(uint8_t *)data;
760 if (trigger && *ctx)
761 eventfd_signal(ctx: *ctx);
762
763 return 0;
764 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
765 int32_t fd;
766
767 if (!count)
768 return -EINVAL;
769
770 fd = *(int32_t *)data;
771 if (fd == -1) {
772 if (*ctx)
773 eventfd_ctx_put(ctx: *ctx);
774 *ctx = NULL;
775 } else if (fd >= 0) {
776 struct eventfd_ctx *efdctx;
777
778 efdctx = eventfd_ctx_fdget(fd);
779 if (IS_ERR(ptr: efdctx))
780 return PTR_ERR(ptr: efdctx);
781
782 if (*ctx)
783 eventfd_ctx_put(ctx: *ctx);
784
785 *ctx = efdctx;
786 }
787 return 0;
788 }
789
790 return -EINVAL;
791}
792
793static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev,
794 unsigned index, unsigned start,
795 unsigned count, uint32_t flags, void *data)
796{
797 if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1)
798 return -EINVAL;
799
800 return vfio_pci_set_ctx_trigger_single(ctx: &vdev->err_trigger,
801 count, flags, data);
802}
803
804static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev,
805 unsigned index, unsigned start,
806 unsigned count, uint32_t flags, void *data)
807{
808 if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1)
809 return -EINVAL;
810
811 return vfio_pci_set_ctx_trigger_single(ctx: &vdev->req_trigger,
812 count, flags, data);
813}
814
815int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags,
816 unsigned index, unsigned start, unsigned count,
817 void *data)
818{
819 int (*func)(struct vfio_pci_core_device *vdev, unsigned index,
820 unsigned start, unsigned count, uint32_t flags,
821 void *data) = NULL;
822
823 switch (index) {
824 case VFIO_PCI_INTX_IRQ_INDEX:
825 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
826 case VFIO_IRQ_SET_ACTION_MASK:
827 func = vfio_pci_set_intx_mask;
828 break;
829 case VFIO_IRQ_SET_ACTION_UNMASK:
830 func = vfio_pci_set_intx_unmask;
831 break;
832 case VFIO_IRQ_SET_ACTION_TRIGGER:
833 func = vfio_pci_set_intx_trigger;
834 break;
835 }
836 break;
837 case VFIO_PCI_MSI_IRQ_INDEX:
838 case VFIO_PCI_MSIX_IRQ_INDEX:
839 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
840 case VFIO_IRQ_SET_ACTION_MASK:
841 case VFIO_IRQ_SET_ACTION_UNMASK:
842 /* XXX Need masking support exported */
843 break;
844 case VFIO_IRQ_SET_ACTION_TRIGGER:
845 func = vfio_pci_set_msi_trigger;
846 break;
847 }
848 break;
849 case VFIO_PCI_ERR_IRQ_INDEX:
850 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
851 case VFIO_IRQ_SET_ACTION_TRIGGER:
852 if (pci_is_pcie(dev: vdev->pdev))
853 func = vfio_pci_set_err_trigger;
854 break;
855 }
856 break;
857 case VFIO_PCI_REQ_IRQ_INDEX:
858 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
859 case VFIO_IRQ_SET_ACTION_TRIGGER:
860 func = vfio_pci_set_req_trigger;
861 break;
862 }
863 break;
864 }
865
866 if (!func)
867 return -ENOTTY;
868
869 return func(vdev, index, start, count, flags, data);
870}
871

source code of linux/drivers/vfio/pci/vfio_pci_intrs.c