1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * VFIO PCI interrupt handling |
4 | * |
5 | * Copyright (C) 2012 Red Hat, Inc. All rights reserved. |
6 | * Author: Alex Williamson <alex.williamson@redhat.com> |
7 | * |
8 | * Derived from original vfio: |
9 | * Copyright 2010 Cisco Systems, Inc. All rights reserved. |
10 | * Author: Tom Lyon, pugs@cisco.com |
11 | */ |
12 | |
13 | #include <linux/device.h> |
14 | #include <linux/interrupt.h> |
15 | #include <linux/eventfd.h> |
16 | #include <linux/msi.h> |
17 | #include <linux/pci.h> |
18 | #include <linux/file.h> |
19 | #include <linux/vfio.h> |
20 | #include <linux/wait.h> |
21 | #include <linux/slab.h> |
22 | |
23 | #include "vfio_pci_priv.h" |
24 | |
25 | struct vfio_pci_irq_ctx { |
26 | struct eventfd_ctx *trigger; |
27 | struct virqfd *unmask; |
28 | struct virqfd *mask; |
29 | char *name; |
30 | bool masked; |
31 | struct irq_bypass_producer producer; |
32 | }; |
33 | |
34 | static bool irq_is(struct vfio_pci_core_device *vdev, int type) |
35 | { |
36 | return vdev->irq_type == type; |
37 | } |
38 | |
39 | static bool is_intx(struct vfio_pci_core_device *vdev) |
40 | { |
41 | return vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX; |
42 | } |
43 | |
44 | static bool is_irq_none(struct vfio_pci_core_device *vdev) |
45 | { |
46 | return !(vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX || |
47 | vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX || |
48 | vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX); |
49 | } |
50 | |
51 | static |
52 | struct vfio_pci_irq_ctx *vfio_irq_ctx_get(struct vfio_pci_core_device *vdev, |
53 | unsigned long index) |
54 | { |
55 | return xa_load(&vdev->ctx, index); |
56 | } |
57 | |
58 | static void vfio_irq_ctx_free(struct vfio_pci_core_device *vdev, |
59 | struct vfio_pci_irq_ctx *ctx, unsigned long index) |
60 | { |
61 | xa_erase(&vdev->ctx, index); |
62 | kfree(objp: ctx); |
63 | } |
64 | |
65 | static struct vfio_pci_irq_ctx * |
66 | vfio_irq_ctx_alloc(struct vfio_pci_core_device *vdev, unsigned long index) |
67 | { |
68 | struct vfio_pci_irq_ctx *ctx; |
69 | int ret; |
70 | |
71 | ctx = kzalloc(size: sizeof(*ctx), GFP_KERNEL_ACCOUNT); |
72 | if (!ctx) |
73 | return NULL; |
74 | |
75 | ret = xa_insert(xa: &vdev->ctx, index, entry: ctx, GFP_KERNEL_ACCOUNT); |
76 | if (ret) { |
77 | kfree(objp: ctx); |
78 | return NULL; |
79 | } |
80 | |
81 | return ctx; |
82 | } |
83 | |
84 | /* |
85 | * INTx |
86 | */ |
87 | static void vfio_send_intx_eventfd(void *opaque, void *unused) |
88 | { |
89 | struct vfio_pci_core_device *vdev = opaque; |
90 | |
91 | if (likely(is_intx(vdev) && !vdev->virq_disabled)) { |
92 | struct vfio_pci_irq_ctx *ctx; |
93 | struct eventfd_ctx *trigger; |
94 | |
95 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
96 | if (WARN_ON_ONCE(!ctx)) |
97 | return; |
98 | |
99 | trigger = READ_ONCE(ctx->trigger); |
100 | if (likely(trigger)) |
101 | eventfd_signal(ctx: trigger); |
102 | } |
103 | } |
104 | |
105 | /* Returns true if the INTx vfio_pci_irq_ctx.masked value is changed. */ |
106 | static bool __vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) |
107 | { |
108 | struct pci_dev *pdev = vdev->pdev; |
109 | struct vfio_pci_irq_ctx *ctx; |
110 | unsigned long flags; |
111 | bool masked_changed = false; |
112 | |
113 | lockdep_assert_held(&vdev->igate); |
114 | |
115 | spin_lock_irqsave(&vdev->irqlock, flags); |
116 | |
117 | /* |
118 | * Masking can come from interrupt, ioctl, or config space |
119 | * via INTx disable. The latter means this can get called |
120 | * even when not using intx delivery. In this case, just |
121 | * try to have the physical bit follow the virtual bit. |
122 | */ |
123 | if (unlikely(!is_intx(vdev))) { |
124 | if (vdev->pci_2_3) |
125 | pci_intx(dev: pdev, enable: 0); |
126 | goto out_unlock; |
127 | } |
128 | |
129 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
130 | if (WARN_ON_ONCE(!ctx)) |
131 | goto out_unlock; |
132 | |
133 | if (!ctx->masked) { |
134 | /* |
135 | * Can't use check_and_mask here because we always want to |
136 | * mask, not just when something is pending. |
137 | */ |
138 | if (vdev->pci_2_3) |
139 | pci_intx(dev: pdev, enable: 0); |
140 | else |
141 | disable_irq_nosync(irq: pdev->irq); |
142 | |
143 | ctx->masked = true; |
144 | masked_changed = true; |
145 | } |
146 | |
147 | out_unlock: |
148 | spin_unlock_irqrestore(lock: &vdev->irqlock, flags); |
149 | return masked_changed; |
150 | } |
151 | |
152 | bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) |
153 | { |
154 | bool mask_changed; |
155 | |
156 | mutex_lock(&vdev->igate); |
157 | mask_changed = __vfio_pci_intx_mask(vdev); |
158 | mutex_unlock(lock: &vdev->igate); |
159 | |
160 | return mask_changed; |
161 | } |
162 | |
163 | /* |
164 | * If this is triggered by an eventfd, we can't call eventfd_signal |
165 | * or else we'll deadlock on the eventfd wait queue. Return >0 when |
166 | * a signal is necessary, which can then be handled via a work queue |
167 | * or directly depending on the caller. |
168 | */ |
169 | static int vfio_pci_intx_unmask_handler(void *opaque, void *unused) |
170 | { |
171 | struct vfio_pci_core_device *vdev = opaque; |
172 | struct pci_dev *pdev = vdev->pdev; |
173 | struct vfio_pci_irq_ctx *ctx; |
174 | unsigned long flags; |
175 | int ret = 0; |
176 | |
177 | spin_lock_irqsave(&vdev->irqlock, flags); |
178 | |
179 | /* |
180 | * Unmasking comes from ioctl or config, so again, have the |
181 | * physical bit follow the virtual even when not using INTx. |
182 | */ |
183 | if (unlikely(!is_intx(vdev))) { |
184 | if (vdev->pci_2_3) |
185 | pci_intx(dev: pdev, enable: 1); |
186 | goto out_unlock; |
187 | } |
188 | |
189 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
190 | if (WARN_ON_ONCE(!ctx)) |
191 | goto out_unlock; |
192 | |
193 | if (ctx->masked && !vdev->virq_disabled) { |
194 | /* |
195 | * A pending interrupt here would immediately trigger, |
196 | * but we can avoid that overhead by just re-sending |
197 | * the interrupt to the user. |
198 | */ |
199 | if (vdev->pci_2_3) { |
200 | if (!pci_check_and_unmask_intx(dev: pdev)) |
201 | ret = 1; |
202 | } else |
203 | enable_irq(irq: pdev->irq); |
204 | |
205 | ctx->masked = (ret > 0); |
206 | } |
207 | |
208 | out_unlock: |
209 | spin_unlock_irqrestore(lock: &vdev->irqlock, flags); |
210 | |
211 | return ret; |
212 | } |
213 | |
214 | static void __vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) |
215 | { |
216 | lockdep_assert_held(&vdev->igate); |
217 | |
218 | if (vfio_pci_intx_unmask_handler(opaque: vdev, NULL) > 0) |
219 | vfio_send_intx_eventfd(opaque: vdev, NULL); |
220 | } |
221 | |
222 | void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) |
223 | { |
224 | mutex_lock(&vdev->igate); |
225 | __vfio_pci_intx_unmask(vdev); |
226 | mutex_unlock(lock: &vdev->igate); |
227 | } |
228 | |
229 | static irqreturn_t vfio_intx_handler(int irq, void *dev_id) |
230 | { |
231 | struct vfio_pci_core_device *vdev = dev_id; |
232 | struct vfio_pci_irq_ctx *ctx; |
233 | unsigned long flags; |
234 | int ret = IRQ_NONE; |
235 | |
236 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
237 | if (WARN_ON_ONCE(!ctx)) |
238 | return ret; |
239 | |
240 | spin_lock_irqsave(&vdev->irqlock, flags); |
241 | |
242 | if (!vdev->pci_2_3) { |
243 | disable_irq_nosync(irq: vdev->pdev->irq); |
244 | ctx->masked = true; |
245 | ret = IRQ_HANDLED; |
246 | } else if (!ctx->masked && /* may be shared */ |
247 | pci_check_and_mask_intx(dev: vdev->pdev)) { |
248 | ctx->masked = true; |
249 | ret = IRQ_HANDLED; |
250 | } |
251 | |
252 | spin_unlock_irqrestore(lock: &vdev->irqlock, flags); |
253 | |
254 | if (ret == IRQ_HANDLED) |
255 | vfio_send_intx_eventfd(opaque: vdev, NULL); |
256 | |
257 | return ret; |
258 | } |
259 | |
260 | static int vfio_intx_enable(struct vfio_pci_core_device *vdev, |
261 | struct eventfd_ctx *trigger) |
262 | { |
263 | struct pci_dev *pdev = vdev->pdev; |
264 | struct vfio_pci_irq_ctx *ctx; |
265 | unsigned long irqflags; |
266 | char *name; |
267 | int ret; |
268 | |
269 | if (!is_irq_none(vdev)) |
270 | return -EINVAL; |
271 | |
272 | if (!pdev->irq) |
273 | return -ENODEV; |
274 | |
275 | name = kasprintf(GFP_KERNEL_ACCOUNT, fmt: "vfio-intx(%s)" , pci_name(pdev)); |
276 | if (!name) |
277 | return -ENOMEM; |
278 | |
279 | ctx = vfio_irq_ctx_alloc(vdev, index: 0); |
280 | if (!ctx) |
281 | return -ENOMEM; |
282 | |
283 | ctx->name = name; |
284 | ctx->trigger = trigger; |
285 | |
286 | /* |
287 | * Fill the initial masked state based on virq_disabled. After |
288 | * enable, changing the DisINTx bit in vconfig directly changes INTx |
289 | * masking. igate prevents races during setup, once running masked |
290 | * is protected via irqlock. |
291 | * |
292 | * Devices supporting DisINTx also reflect the current mask state in |
293 | * the physical DisINTx bit, which is not affected during IRQ setup. |
294 | * |
295 | * Devices without DisINTx support require an exclusive interrupt. |
296 | * IRQ masking is performed at the IRQ chip. Again, igate protects |
297 | * against races during setup and IRQ handlers and irqfds are not |
298 | * yet active, therefore masked is stable and can be used to |
299 | * conditionally auto-enable the IRQ. |
300 | * |
301 | * irq_type must be stable while the IRQ handler is registered, |
302 | * therefore it must be set before request_irq(). |
303 | */ |
304 | ctx->masked = vdev->virq_disabled; |
305 | if (vdev->pci_2_3) { |
306 | pci_intx(dev: pdev, enable: !ctx->masked); |
307 | irqflags = IRQF_SHARED; |
308 | } else { |
309 | irqflags = ctx->masked ? IRQF_NO_AUTOEN : 0; |
310 | } |
311 | |
312 | vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; |
313 | |
314 | ret = request_irq(irq: pdev->irq, handler: vfio_intx_handler, |
315 | flags: irqflags, name: ctx->name, dev: vdev); |
316 | if (ret) { |
317 | vdev->irq_type = VFIO_PCI_NUM_IRQS; |
318 | kfree(objp: name); |
319 | vfio_irq_ctx_free(vdev, ctx, index: 0); |
320 | return ret; |
321 | } |
322 | |
323 | return 0; |
324 | } |
325 | |
326 | static int vfio_intx_set_signal(struct vfio_pci_core_device *vdev, |
327 | struct eventfd_ctx *trigger) |
328 | { |
329 | struct pci_dev *pdev = vdev->pdev; |
330 | struct vfio_pci_irq_ctx *ctx; |
331 | struct eventfd_ctx *old; |
332 | |
333 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
334 | if (WARN_ON_ONCE(!ctx)) |
335 | return -EINVAL; |
336 | |
337 | old = ctx->trigger; |
338 | |
339 | WRITE_ONCE(ctx->trigger, trigger); |
340 | |
341 | /* Releasing an old ctx requires synchronizing in-flight users */ |
342 | if (old) { |
343 | synchronize_irq(irq: pdev->irq); |
344 | vfio_virqfd_flush_thread(pvirqfd: &ctx->unmask); |
345 | eventfd_ctx_put(ctx: old); |
346 | } |
347 | |
348 | return 0; |
349 | } |
350 | |
351 | static void vfio_intx_disable(struct vfio_pci_core_device *vdev) |
352 | { |
353 | struct pci_dev *pdev = vdev->pdev; |
354 | struct vfio_pci_irq_ctx *ctx; |
355 | |
356 | ctx = vfio_irq_ctx_get(vdev, index: 0); |
357 | WARN_ON_ONCE(!ctx); |
358 | if (ctx) { |
359 | vfio_virqfd_disable(pvirqfd: &ctx->unmask); |
360 | vfio_virqfd_disable(pvirqfd: &ctx->mask); |
361 | free_irq(pdev->irq, vdev); |
362 | if (ctx->trigger) |
363 | eventfd_ctx_put(ctx: ctx->trigger); |
364 | kfree(objp: ctx->name); |
365 | vfio_irq_ctx_free(vdev, ctx, index: 0); |
366 | } |
367 | vdev->irq_type = VFIO_PCI_NUM_IRQS; |
368 | } |
369 | |
370 | /* |
371 | * MSI/MSI-X |
372 | */ |
373 | static irqreturn_t vfio_msihandler(int irq, void *arg) |
374 | { |
375 | struct eventfd_ctx *trigger = arg; |
376 | |
377 | eventfd_signal(ctx: trigger); |
378 | return IRQ_HANDLED; |
379 | } |
380 | |
381 | static int vfio_msi_enable(struct vfio_pci_core_device *vdev, int nvec, bool msix) |
382 | { |
383 | struct pci_dev *pdev = vdev->pdev; |
384 | unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; |
385 | int ret; |
386 | u16 cmd; |
387 | |
388 | if (!is_irq_none(vdev)) |
389 | return -EINVAL; |
390 | |
391 | /* return the number of supported vectors if we can't get all: */ |
392 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
393 | ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: nvec, flags: flag); |
394 | if (ret < nvec) { |
395 | if (ret > 0) |
396 | pci_free_irq_vectors(dev: pdev); |
397 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
398 | return ret; |
399 | } |
400 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
401 | |
402 | vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX : |
403 | VFIO_PCI_MSI_IRQ_INDEX; |
404 | |
405 | if (!msix) { |
406 | /* |
407 | * Compute the virtual hardware field for max msi vectors - |
408 | * it is the log base 2 of the number of vectors. |
409 | */ |
410 | vdev->msi_qmax = fls(x: nvec * 2 - 1) - 1; |
411 | } |
412 | |
413 | return 0; |
414 | } |
415 | |
416 | /* |
417 | * vfio_msi_alloc_irq() returns the Linux IRQ number of an MSI or MSI-X device |
418 | * interrupt vector. If a Linux IRQ number is not available then a new |
419 | * interrupt is allocated if dynamic MSI-X is supported. |
420 | * |
421 | * Where is vfio_msi_free_irq()? Allocated interrupts are maintained, |
422 | * essentially forming a cache that subsequent allocations can draw from. |
423 | * Interrupts are freed using pci_free_irq_vectors() when MSI/MSI-X is |
424 | * disabled. |
425 | */ |
426 | static int vfio_msi_alloc_irq(struct vfio_pci_core_device *vdev, |
427 | unsigned int vector, bool msix) |
428 | { |
429 | struct pci_dev *pdev = vdev->pdev; |
430 | struct msi_map map; |
431 | int irq; |
432 | u16 cmd; |
433 | |
434 | irq = pci_irq_vector(dev: pdev, nr: vector); |
435 | if (WARN_ON_ONCE(irq == 0)) |
436 | return -EINVAL; |
437 | if (irq > 0 || !msix || !vdev->has_dyn_msix) |
438 | return irq; |
439 | |
440 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
441 | map = pci_msix_alloc_irq_at(dev: pdev, index: vector, NULL); |
442 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
443 | |
444 | return map.index < 0 ? map.index : map.virq; |
445 | } |
446 | |
447 | static int vfio_msi_set_vector_signal(struct vfio_pci_core_device *vdev, |
448 | unsigned int vector, int fd, bool msix) |
449 | { |
450 | struct pci_dev *pdev = vdev->pdev; |
451 | struct vfio_pci_irq_ctx *ctx; |
452 | struct eventfd_ctx *trigger; |
453 | int irq = -EINVAL, ret; |
454 | u16 cmd; |
455 | |
456 | ctx = vfio_irq_ctx_get(vdev, index: vector); |
457 | |
458 | if (ctx) { |
459 | irq_bypass_unregister_producer(&ctx->producer); |
460 | irq = pci_irq_vector(dev: pdev, nr: vector); |
461 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
462 | free_irq(irq, ctx->trigger); |
463 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
464 | /* Interrupt stays allocated, will be freed at MSI-X disable. */ |
465 | kfree(objp: ctx->name); |
466 | eventfd_ctx_put(ctx: ctx->trigger); |
467 | vfio_irq_ctx_free(vdev, ctx, index: vector); |
468 | } |
469 | |
470 | if (fd < 0) |
471 | return 0; |
472 | |
473 | if (irq == -EINVAL) { |
474 | /* Interrupt stays allocated, will be freed at MSI-X disable. */ |
475 | irq = vfio_msi_alloc_irq(vdev, vector, msix); |
476 | if (irq < 0) |
477 | return irq; |
478 | } |
479 | |
480 | ctx = vfio_irq_ctx_alloc(vdev, index: vector); |
481 | if (!ctx) |
482 | return -ENOMEM; |
483 | |
484 | ctx->name = kasprintf(GFP_KERNEL_ACCOUNT, fmt: "vfio-msi%s[%d](%s)" , |
485 | msix ? "x" : "" , vector, pci_name(pdev)); |
486 | if (!ctx->name) { |
487 | ret = -ENOMEM; |
488 | goto out_free_ctx; |
489 | } |
490 | |
491 | trigger = eventfd_ctx_fdget(fd); |
492 | if (IS_ERR(ptr: trigger)) { |
493 | ret = PTR_ERR(ptr: trigger); |
494 | goto out_free_name; |
495 | } |
496 | |
497 | /* |
498 | * If the vector was previously allocated, refresh the on-device |
499 | * message data before enabling in case it had been cleared or |
500 | * corrupted (e.g. due to backdoor resets) since writing. |
501 | */ |
502 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
503 | if (msix) { |
504 | struct msi_msg msg; |
505 | |
506 | get_cached_msi_msg(irq, msg: &msg); |
507 | pci_write_msi_msg(irq, msg: &msg); |
508 | } |
509 | |
510 | ret = request_irq(irq, handler: vfio_msihandler, flags: 0, name: ctx->name, dev: trigger); |
511 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
512 | if (ret) |
513 | goto out_put_eventfd_ctx; |
514 | |
515 | ctx->producer.token = trigger; |
516 | ctx->producer.irq = irq; |
517 | ret = irq_bypass_register_producer(&ctx->producer); |
518 | if (unlikely(ret)) { |
519 | dev_info(&pdev->dev, |
520 | "irq bypass producer (token %p) registration fails: %d\n" , |
521 | ctx->producer.token, ret); |
522 | |
523 | ctx->producer.token = NULL; |
524 | } |
525 | ctx->trigger = trigger; |
526 | |
527 | return 0; |
528 | |
529 | out_put_eventfd_ctx: |
530 | eventfd_ctx_put(ctx: trigger); |
531 | out_free_name: |
532 | kfree(objp: ctx->name); |
533 | out_free_ctx: |
534 | vfio_irq_ctx_free(vdev, ctx, index: vector); |
535 | return ret; |
536 | } |
537 | |
538 | static int vfio_msi_set_block(struct vfio_pci_core_device *vdev, unsigned start, |
539 | unsigned count, int32_t *fds, bool msix) |
540 | { |
541 | unsigned int i, j; |
542 | int ret = 0; |
543 | |
544 | for (i = 0, j = start; i < count && !ret; i++, j++) { |
545 | int fd = fds ? fds[i] : -1; |
546 | ret = vfio_msi_set_vector_signal(vdev, vector: j, fd, msix); |
547 | } |
548 | |
549 | if (ret) { |
550 | for (i = start; i < j; i++) |
551 | vfio_msi_set_vector_signal(vdev, vector: i, fd: -1, msix); |
552 | } |
553 | |
554 | return ret; |
555 | } |
556 | |
557 | static void vfio_msi_disable(struct vfio_pci_core_device *vdev, bool msix) |
558 | { |
559 | struct pci_dev *pdev = vdev->pdev; |
560 | struct vfio_pci_irq_ctx *ctx; |
561 | unsigned long i; |
562 | u16 cmd; |
563 | |
564 | xa_for_each(&vdev->ctx, i, ctx) { |
565 | vfio_virqfd_disable(pvirqfd: &ctx->unmask); |
566 | vfio_virqfd_disable(pvirqfd: &ctx->mask); |
567 | vfio_msi_set_vector_signal(vdev, vector: i, fd: -1, msix); |
568 | } |
569 | |
570 | cmd = vfio_pci_memory_lock_and_enable(vdev); |
571 | pci_free_irq_vectors(dev: pdev); |
572 | vfio_pci_memory_unlock_and_restore(vdev, cmd); |
573 | |
574 | /* |
575 | * Both disable paths above use pci_intx_for_msi() to clear DisINTx |
576 | * via their shutdown paths. Restore for NoINTx devices. |
577 | */ |
578 | if (vdev->nointx) |
579 | pci_intx(dev: pdev, enable: 0); |
580 | |
581 | vdev->irq_type = VFIO_PCI_NUM_IRQS; |
582 | } |
583 | |
584 | /* |
585 | * IOCTL support |
586 | */ |
587 | static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev, |
588 | unsigned index, unsigned start, |
589 | unsigned count, uint32_t flags, void *data) |
590 | { |
591 | if (!is_intx(vdev) || start != 0 || count != 1) |
592 | return -EINVAL; |
593 | |
594 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
595 | __vfio_pci_intx_unmask(vdev); |
596 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
597 | uint8_t unmask = *(uint8_t *)data; |
598 | if (unmask) |
599 | __vfio_pci_intx_unmask(vdev); |
600 | } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
601 | struct vfio_pci_irq_ctx *ctx = vfio_irq_ctx_get(vdev, index: 0); |
602 | int32_t fd = *(int32_t *)data; |
603 | |
604 | if (WARN_ON_ONCE(!ctx)) |
605 | return -EINVAL; |
606 | if (fd >= 0) |
607 | return vfio_virqfd_enable(opaque: (void *) vdev, |
608 | handler: vfio_pci_intx_unmask_handler, |
609 | thread: vfio_send_intx_eventfd, NULL, |
610 | pvirqfd: &ctx->unmask, fd); |
611 | |
612 | vfio_virqfd_disable(pvirqfd: &ctx->unmask); |
613 | } |
614 | |
615 | return 0; |
616 | } |
617 | |
618 | static int vfio_pci_set_intx_mask(struct vfio_pci_core_device *vdev, |
619 | unsigned index, unsigned start, |
620 | unsigned count, uint32_t flags, void *data) |
621 | { |
622 | if (!is_intx(vdev) || start != 0 || count != 1) |
623 | return -EINVAL; |
624 | |
625 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
626 | __vfio_pci_intx_mask(vdev); |
627 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
628 | uint8_t mask = *(uint8_t *)data; |
629 | if (mask) |
630 | __vfio_pci_intx_mask(vdev); |
631 | } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
632 | return -ENOTTY; /* XXX implement me */ |
633 | } |
634 | |
635 | return 0; |
636 | } |
637 | |
638 | static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev, |
639 | unsigned index, unsigned start, |
640 | unsigned count, uint32_t flags, void *data) |
641 | { |
642 | if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { |
643 | vfio_intx_disable(vdev); |
644 | return 0; |
645 | } |
646 | |
647 | if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1) |
648 | return -EINVAL; |
649 | |
650 | if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
651 | struct eventfd_ctx *trigger = NULL; |
652 | int32_t fd = *(int32_t *)data; |
653 | int ret; |
654 | |
655 | if (fd >= 0) { |
656 | trigger = eventfd_ctx_fdget(fd); |
657 | if (IS_ERR(ptr: trigger)) |
658 | return PTR_ERR(ptr: trigger); |
659 | } |
660 | |
661 | if (is_intx(vdev)) |
662 | ret = vfio_intx_set_signal(vdev, trigger); |
663 | else |
664 | ret = vfio_intx_enable(vdev, trigger); |
665 | |
666 | if (ret && trigger) |
667 | eventfd_ctx_put(ctx: trigger); |
668 | |
669 | return ret; |
670 | } |
671 | |
672 | if (!is_intx(vdev)) |
673 | return -EINVAL; |
674 | |
675 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
676 | vfio_send_intx_eventfd(opaque: vdev, NULL); |
677 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
678 | uint8_t trigger = *(uint8_t *)data; |
679 | if (trigger) |
680 | vfio_send_intx_eventfd(opaque: vdev, NULL); |
681 | } |
682 | return 0; |
683 | } |
684 | |
685 | static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev, |
686 | unsigned index, unsigned start, |
687 | unsigned count, uint32_t flags, void *data) |
688 | { |
689 | struct vfio_pci_irq_ctx *ctx; |
690 | unsigned int i; |
691 | bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false; |
692 | |
693 | if (irq_is(vdev, type: index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { |
694 | vfio_msi_disable(vdev, msix); |
695 | return 0; |
696 | } |
697 | |
698 | if (!(irq_is(vdev, type: index) || is_irq_none(vdev))) |
699 | return -EINVAL; |
700 | |
701 | if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
702 | int32_t *fds = data; |
703 | int ret; |
704 | |
705 | if (vdev->irq_type == index) |
706 | return vfio_msi_set_block(vdev, start, count, |
707 | fds, msix); |
708 | |
709 | ret = vfio_msi_enable(vdev, nvec: start + count, msix); |
710 | if (ret) |
711 | return ret; |
712 | |
713 | ret = vfio_msi_set_block(vdev, start, count, fds, msix); |
714 | if (ret) |
715 | vfio_msi_disable(vdev, msix); |
716 | |
717 | return ret; |
718 | } |
719 | |
720 | if (!irq_is(vdev, type: index)) |
721 | return -EINVAL; |
722 | |
723 | for (i = start; i < start + count; i++) { |
724 | ctx = vfio_irq_ctx_get(vdev, index: i); |
725 | if (!ctx) |
726 | continue; |
727 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
728 | eventfd_signal(ctx: ctx->trigger); |
729 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
730 | uint8_t *bools = data; |
731 | if (bools[i - start]) |
732 | eventfd_signal(ctx: ctx->trigger); |
733 | } |
734 | } |
735 | return 0; |
736 | } |
737 | |
738 | static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, |
739 | unsigned int count, uint32_t flags, |
740 | void *data) |
741 | { |
742 | /* DATA_NONE/DATA_BOOL enables loopback testing */ |
743 | if (flags & VFIO_IRQ_SET_DATA_NONE) { |
744 | if (*ctx) { |
745 | if (count) { |
746 | eventfd_signal(ctx: *ctx); |
747 | } else { |
748 | eventfd_ctx_put(ctx: *ctx); |
749 | *ctx = NULL; |
750 | } |
751 | return 0; |
752 | } |
753 | } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { |
754 | uint8_t trigger; |
755 | |
756 | if (!count) |
757 | return -EINVAL; |
758 | |
759 | trigger = *(uint8_t *)data; |
760 | if (trigger && *ctx) |
761 | eventfd_signal(ctx: *ctx); |
762 | |
763 | return 0; |
764 | } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { |
765 | int32_t fd; |
766 | |
767 | if (!count) |
768 | return -EINVAL; |
769 | |
770 | fd = *(int32_t *)data; |
771 | if (fd == -1) { |
772 | if (*ctx) |
773 | eventfd_ctx_put(ctx: *ctx); |
774 | *ctx = NULL; |
775 | } else if (fd >= 0) { |
776 | struct eventfd_ctx *efdctx; |
777 | |
778 | efdctx = eventfd_ctx_fdget(fd); |
779 | if (IS_ERR(ptr: efdctx)) |
780 | return PTR_ERR(ptr: efdctx); |
781 | |
782 | if (*ctx) |
783 | eventfd_ctx_put(ctx: *ctx); |
784 | |
785 | *ctx = efdctx; |
786 | } |
787 | return 0; |
788 | } |
789 | |
790 | return -EINVAL; |
791 | } |
792 | |
793 | static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev, |
794 | unsigned index, unsigned start, |
795 | unsigned count, uint32_t flags, void *data) |
796 | { |
797 | if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1) |
798 | return -EINVAL; |
799 | |
800 | return vfio_pci_set_ctx_trigger_single(ctx: &vdev->err_trigger, |
801 | count, flags, data); |
802 | } |
803 | |
804 | static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev, |
805 | unsigned index, unsigned start, |
806 | unsigned count, uint32_t flags, void *data) |
807 | { |
808 | if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1) |
809 | return -EINVAL; |
810 | |
811 | return vfio_pci_set_ctx_trigger_single(ctx: &vdev->req_trigger, |
812 | count, flags, data); |
813 | } |
814 | |
815 | int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, |
816 | unsigned index, unsigned start, unsigned count, |
817 | void *data) |
818 | { |
819 | int (*func)(struct vfio_pci_core_device *vdev, unsigned index, |
820 | unsigned start, unsigned count, uint32_t flags, |
821 | void *data) = NULL; |
822 | |
823 | switch (index) { |
824 | case VFIO_PCI_INTX_IRQ_INDEX: |
825 | switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { |
826 | case VFIO_IRQ_SET_ACTION_MASK: |
827 | func = vfio_pci_set_intx_mask; |
828 | break; |
829 | case VFIO_IRQ_SET_ACTION_UNMASK: |
830 | func = vfio_pci_set_intx_unmask; |
831 | break; |
832 | case VFIO_IRQ_SET_ACTION_TRIGGER: |
833 | func = vfio_pci_set_intx_trigger; |
834 | break; |
835 | } |
836 | break; |
837 | case VFIO_PCI_MSI_IRQ_INDEX: |
838 | case VFIO_PCI_MSIX_IRQ_INDEX: |
839 | switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { |
840 | case VFIO_IRQ_SET_ACTION_MASK: |
841 | case VFIO_IRQ_SET_ACTION_UNMASK: |
842 | /* XXX Need masking support exported */ |
843 | break; |
844 | case VFIO_IRQ_SET_ACTION_TRIGGER: |
845 | func = vfio_pci_set_msi_trigger; |
846 | break; |
847 | } |
848 | break; |
849 | case VFIO_PCI_ERR_IRQ_INDEX: |
850 | switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { |
851 | case VFIO_IRQ_SET_ACTION_TRIGGER: |
852 | if (pci_is_pcie(dev: vdev->pdev)) |
853 | func = vfio_pci_set_err_trigger; |
854 | break; |
855 | } |
856 | break; |
857 | case VFIO_PCI_REQ_IRQ_INDEX: |
858 | switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { |
859 | case VFIO_IRQ_SET_ACTION_TRIGGER: |
860 | func = vfio_pci_set_req_trigger; |
861 | break; |
862 | } |
863 | break; |
864 | } |
865 | |
866 | if (!func) |
867 | return -ENOTTY; |
868 | |
869 | return func(vdev, index, start, count, flags, data); |
870 | } |
871 | |