1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2020 Intel Corporation |
4 | * Author: Johannes Berg <johannes@sipsolutions.net> |
5 | */ |
6 | #include <linux/module.h> |
7 | #include <linux/pci.h> |
8 | #include <linux/virtio.h> |
9 | #include <linux/virtio_config.h> |
10 | #include <linux/logic_iomem.h> |
11 | #include <linux/of_platform.h> |
12 | #include <linux/irqdomain.h> |
13 | #include <linux/virtio_pcidev.h> |
14 | #include <linux/virtio-uml.h> |
15 | #include <linux/delay.h> |
16 | #include <linux/msi.h> |
17 | #include <asm/unaligned.h> |
18 | #include <irq_kern.h> |
19 | |
20 | #define MAX_DEVICES 8 |
21 | #define MAX_MSI_VECTORS 32 |
22 | #define CFG_SPACE_SIZE 4096 |
23 | |
24 | /* for MSI-X we have a 32-bit payload */ |
25 | #define MAX_IRQ_MSG_SIZE (sizeof(struct virtio_pcidev_msg) + sizeof(u32)) |
26 | #define NUM_IRQ_MSGS 10 |
27 | |
28 | #define HANDLE_NO_FREE(ptr) ((void *)((unsigned long)(ptr) | 1)) |
29 | #define HANDLE_IS_NO_FREE(ptr) ((unsigned long)(ptr) & 1) |
30 | |
31 | struct um_pci_device { |
32 | struct virtio_device *vdev; |
33 | |
34 | /* for now just standard BARs */ |
35 | u8 resptr[PCI_STD_NUM_BARS]; |
36 | |
37 | struct virtqueue *cmd_vq, *irq_vq; |
38 | |
39 | #define UM_PCI_STAT_WAITING 0 |
40 | unsigned long status; |
41 | |
42 | int irq; |
43 | |
44 | bool platform; |
45 | }; |
46 | |
47 | struct um_pci_device_reg { |
48 | struct um_pci_device *dev; |
49 | void __iomem *iomem; |
50 | }; |
51 | |
52 | static struct pci_host_bridge *bridge; |
53 | static DEFINE_MUTEX(um_pci_mtx); |
54 | static struct um_pci_device *um_pci_platform_device; |
55 | static struct um_pci_device_reg um_pci_devices[MAX_DEVICES]; |
56 | static struct fwnode_handle *um_pci_fwnode; |
57 | static struct irq_domain *um_pci_inner_domain; |
58 | static struct irq_domain *um_pci_msi_domain; |
59 | static unsigned long um_pci_msi_used[BITS_TO_LONGS(MAX_MSI_VECTORS)]; |
60 | |
61 | static unsigned int um_pci_max_delay_us = 40000; |
62 | module_param_named(max_delay_us, um_pci_max_delay_us, uint, 0644); |
63 | |
64 | struct um_pci_message_buffer { |
65 | struct virtio_pcidev_msg hdr; |
66 | u8 data[8]; |
67 | }; |
68 | |
69 | static struct um_pci_message_buffer __percpu *um_pci_msg_bufs; |
70 | |
71 | static int um_pci_send_cmd(struct um_pci_device *dev, |
72 | struct virtio_pcidev_msg *cmd, |
73 | unsigned int cmd_size, |
74 | const void *, unsigned int , |
75 | void *out, unsigned int out_size) |
76 | { |
77 | struct scatterlist out_sg, , in_sg; |
78 | struct scatterlist *sgs_list[] = { |
79 | [0] = &out_sg, |
80 | [1] = extra ? &extra_sg : &in_sg, |
81 | [2] = extra ? &in_sg : NULL, |
82 | }; |
83 | struct um_pci_message_buffer *buf; |
84 | int delay_count = 0; |
85 | int ret, len; |
86 | bool posted; |
87 | |
88 | if (WARN_ON(cmd_size < sizeof(*cmd) || cmd_size > sizeof(*buf))) |
89 | return -EINVAL; |
90 | |
91 | switch (cmd->op) { |
92 | case VIRTIO_PCIDEV_OP_CFG_WRITE: |
93 | case VIRTIO_PCIDEV_OP_MMIO_WRITE: |
94 | case VIRTIO_PCIDEV_OP_MMIO_MEMSET: |
95 | /* in PCI, writes are posted, so don't wait */ |
96 | posted = !out; |
97 | WARN_ON(!posted); |
98 | break; |
99 | default: |
100 | posted = false; |
101 | break; |
102 | } |
103 | |
104 | buf = get_cpu_var(um_pci_msg_bufs); |
105 | if (buf) |
106 | memcpy(buf, cmd, cmd_size); |
107 | |
108 | if (posted) { |
109 | u8 *ncmd = kmalloc(size: cmd_size + extra_size, GFP_ATOMIC); |
110 | |
111 | if (ncmd) { |
112 | memcpy(ncmd, cmd, cmd_size); |
113 | if (extra) |
114 | memcpy(ncmd + cmd_size, extra, extra_size); |
115 | cmd = (void *)ncmd; |
116 | cmd_size += extra_size; |
117 | extra = NULL; |
118 | extra_size = 0; |
119 | } else { |
120 | /* try without allocating memory */ |
121 | posted = false; |
122 | cmd = (void *)buf; |
123 | } |
124 | } else { |
125 | cmd = (void *)buf; |
126 | } |
127 | |
128 | sg_init_one(&out_sg, cmd, cmd_size); |
129 | if (extra) |
130 | sg_init_one(&extra_sg, extra, extra_size); |
131 | if (out) |
132 | sg_init_one(&in_sg, out, out_size); |
133 | |
134 | /* add to internal virtio queue */ |
135 | ret = virtqueue_add_sgs(vq: dev->cmd_vq, sgs: sgs_list, |
136 | out_sgs: extra ? 2 : 1, |
137 | in_sgs: out ? 1 : 0, |
138 | data: posted ? cmd : HANDLE_NO_FREE(cmd), |
139 | GFP_ATOMIC); |
140 | if (ret) { |
141 | if (posted) |
142 | kfree(objp: cmd); |
143 | goto out; |
144 | } |
145 | |
146 | if (posted) { |
147 | virtqueue_kick(vq: dev->cmd_vq); |
148 | ret = 0; |
149 | goto out; |
150 | } |
151 | |
152 | /* kick and poll for getting a response on the queue */ |
153 | set_bit(UM_PCI_STAT_WAITING, addr: &dev->status); |
154 | virtqueue_kick(vq: dev->cmd_vq); |
155 | |
156 | while (1) { |
157 | void *completed = virtqueue_get_buf(vq: dev->cmd_vq, len: &len); |
158 | |
159 | if (completed == HANDLE_NO_FREE(cmd)) |
160 | break; |
161 | |
162 | if (completed && !HANDLE_IS_NO_FREE(completed)) |
163 | kfree(objp: completed); |
164 | |
165 | if (WARN_ONCE(virtqueue_is_broken(dev->cmd_vq) || |
166 | ++delay_count > um_pci_max_delay_us, |
167 | "um virt-pci delay: %d" , delay_count)) { |
168 | ret = -EIO; |
169 | break; |
170 | } |
171 | udelay(1); |
172 | } |
173 | clear_bit(UM_PCI_STAT_WAITING, addr: &dev->status); |
174 | |
175 | out: |
176 | put_cpu_var(um_pci_msg_bufs); |
177 | return ret; |
178 | } |
179 | |
180 | static unsigned long um_pci_cfgspace_read(void *priv, unsigned int offset, |
181 | int size) |
182 | { |
183 | struct um_pci_device_reg *reg = priv; |
184 | struct um_pci_device *dev = reg->dev; |
185 | struct virtio_pcidev_msg hdr = { |
186 | .op = VIRTIO_PCIDEV_OP_CFG_READ, |
187 | .size = size, |
188 | .addr = offset, |
189 | }; |
190 | /* buf->data is maximum size - we may only use parts of it */ |
191 | struct um_pci_message_buffer *buf; |
192 | u8 *data; |
193 | unsigned long ret = ULONG_MAX; |
194 | size_t bytes = sizeof(buf->data); |
195 | |
196 | if (!dev) |
197 | return ULONG_MAX; |
198 | |
199 | buf = get_cpu_var(um_pci_msg_bufs); |
200 | data = buf->data; |
201 | |
202 | if (buf) |
203 | memset(data, 0xff, bytes); |
204 | |
205 | switch (size) { |
206 | case 1: |
207 | case 2: |
208 | case 4: |
209 | #ifdef CONFIG_64BIT |
210 | case 8: |
211 | #endif |
212 | break; |
213 | default: |
214 | WARN(1, "invalid config space read size %d\n" , size); |
215 | goto out; |
216 | } |
217 | |
218 | if (um_pci_send_cmd(dev, cmd: &hdr, cmd_size: sizeof(hdr), NULL, extra_size: 0, out: data, out_size: bytes)) |
219 | goto out; |
220 | |
221 | switch (size) { |
222 | case 1: |
223 | ret = data[0]; |
224 | break; |
225 | case 2: |
226 | ret = le16_to_cpup(p: (void *)data); |
227 | break; |
228 | case 4: |
229 | ret = le32_to_cpup(p: (void *)data); |
230 | break; |
231 | #ifdef CONFIG_64BIT |
232 | case 8: |
233 | ret = le64_to_cpup(p: (void *)data); |
234 | break; |
235 | #endif |
236 | default: |
237 | break; |
238 | } |
239 | |
240 | out: |
241 | put_cpu_var(um_pci_msg_bufs); |
242 | return ret; |
243 | } |
244 | |
245 | static void um_pci_cfgspace_write(void *priv, unsigned int offset, int size, |
246 | unsigned long val) |
247 | { |
248 | struct um_pci_device_reg *reg = priv; |
249 | struct um_pci_device *dev = reg->dev; |
250 | struct { |
251 | struct virtio_pcidev_msg hdr; |
252 | /* maximum size - we may only use parts of it */ |
253 | u8 data[8]; |
254 | } msg = { |
255 | .hdr = { |
256 | .op = VIRTIO_PCIDEV_OP_CFG_WRITE, |
257 | .size = size, |
258 | .addr = offset, |
259 | }, |
260 | }; |
261 | |
262 | if (!dev) |
263 | return; |
264 | |
265 | switch (size) { |
266 | case 1: |
267 | msg.data[0] = (u8)val; |
268 | break; |
269 | case 2: |
270 | put_unaligned_le16(val, p: (void *)msg.data); |
271 | break; |
272 | case 4: |
273 | put_unaligned_le32(val, p: (void *)msg.data); |
274 | break; |
275 | #ifdef CONFIG_64BIT |
276 | case 8: |
277 | put_unaligned_le64(val, p: (void *)msg.data); |
278 | break; |
279 | #endif |
280 | default: |
281 | WARN(1, "invalid config space write size %d\n" , size); |
282 | return; |
283 | } |
284 | |
285 | WARN_ON(um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0)); |
286 | } |
287 | |
288 | static const struct logic_iomem_ops um_pci_device_cfgspace_ops = { |
289 | .read = um_pci_cfgspace_read, |
290 | .write = um_pci_cfgspace_write, |
291 | }; |
292 | |
293 | static void um_pci_bar_copy_from(void *priv, void *buffer, |
294 | unsigned int offset, int size) |
295 | { |
296 | u8 *resptr = priv; |
297 | struct um_pci_device *dev = container_of(resptr - *resptr, |
298 | struct um_pci_device, |
299 | resptr[0]); |
300 | struct virtio_pcidev_msg hdr = { |
301 | .op = VIRTIO_PCIDEV_OP_MMIO_READ, |
302 | .bar = *resptr, |
303 | .size = size, |
304 | .addr = offset, |
305 | }; |
306 | |
307 | memset(buffer, 0xff, size); |
308 | |
309 | um_pci_send_cmd(dev, cmd: &hdr, cmd_size: sizeof(hdr), NULL, extra_size: 0, out: buffer, out_size: size); |
310 | } |
311 | |
312 | static unsigned long um_pci_bar_read(void *priv, unsigned int offset, |
313 | int size) |
314 | { |
315 | /* buf->data is maximum size - we may only use parts of it */ |
316 | struct um_pci_message_buffer *buf; |
317 | u8 *data; |
318 | unsigned long ret = ULONG_MAX; |
319 | |
320 | buf = get_cpu_var(um_pci_msg_bufs); |
321 | data = buf->data; |
322 | |
323 | switch (size) { |
324 | case 1: |
325 | case 2: |
326 | case 4: |
327 | #ifdef CONFIG_64BIT |
328 | case 8: |
329 | #endif |
330 | break; |
331 | default: |
332 | WARN(1, "invalid config space read size %d\n" , size); |
333 | goto out; |
334 | } |
335 | |
336 | um_pci_bar_copy_from(priv, buffer: data, offset, size); |
337 | |
338 | switch (size) { |
339 | case 1: |
340 | ret = data[0]; |
341 | break; |
342 | case 2: |
343 | ret = le16_to_cpup(p: (void *)data); |
344 | break; |
345 | case 4: |
346 | ret = le32_to_cpup(p: (void *)data); |
347 | break; |
348 | #ifdef CONFIG_64BIT |
349 | case 8: |
350 | ret = le64_to_cpup(p: (void *)data); |
351 | break; |
352 | #endif |
353 | default: |
354 | break; |
355 | } |
356 | |
357 | out: |
358 | put_cpu_var(um_pci_msg_bufs); |
359 | return ret; |
360 | } |
361 | |
362 | static void um_pci_bar_copy_to(void *priv, unsigned int offset, |
363 | const void *buffer, int size) |
364 | { |
365 | u8 *resptr = priv; |
366 | struct um_pci_device *dev = container_of(resptr - *resptr, |
367 | struct um_pci_device, |
368 | resptr[0]); |
369 | struct virtio_pcidev_msg hdr = { |
370 | .op = VIRTIO_PCIDEV_OP_MMIO_WRITE, |
371 | .bar = *resptr, |
372 | .size = size, |
373 | .addr = offset, |
374 | }; |
375 | |
376 | um_pci_send_cmd(dev, cmd: &hdr, cmd_size: sizeof(hdr), extra: buffer, extra_size: size, NULL, out_size: 0); |
377 | } |
378 | |
379 | static void um_pci_bar_write(void *priv, unsigned int offset, int size, |
380 | unsigned long val) |
381 | { |
382 | /* maximum size - we may only use parts of it */ |
383 | u8 data[8]; |
384 | |
385 | switch (size) { |
386 | case 1: |
387 | data[0] = (u8)val; |
388 | break; |
389 | case 2: |
390 | put_unaligned_le16(val, p: (void *)data); |
391 | break; |
392 | case 4: |
393 | put_unaligned_le32(val, p: (void *)data); |
394 | break; |
395 | #ifdef CONFIG_64BIT |
396 | case 8: |
397 | put_unaligned_le64(val, p: (void *)data); |
398 | break; |
399 | #endif |
400 | default: |
401 | WARN(1, "invalid config space write size %d\n" , size); |
402 | return; |
403 | } |
404 | |
405 | um_pci_bar_copy_to(priv, offset, buffer: data, size); |
406 | } |
407 | |
408 | static void um_pci_bar_set(void *priv, unsigned int offset, u8 value, int size) |
409 | { |
410 | u8 *resptr = priv; |
411 | struct um_pci_device *dev = container_of(resptr - *resptr, |
412 | struct um_pci_device, |
413 | resptr[0]); |
414 | struct { |
415 | struct virtio_pcidev_msg hdr; |
416 | u8 data; |
417 | } msg = { |
418 | .hdr = { |
419 | .op = VIRTIO_PCIDEV_OP_CFG_WRITE, |
420 | .bar = *resptr, |
421 | .size = size, |
422 | .addr = offset, |
423 | }, |
424 | .data = value, |
425 | }; |
426 | |
427 | um_pci_send_cmd(dev, cmd: &msg.hdr, cmd_size: sizeof(msg), NULL, extra_size: 0, NULL, out_size: 0); |
428 | } |
429 | |
430 | static const struct logic_iomem_ops um_pci_device_bar_ops = { |
431 | .read = um_pci_bar_read, |
432 | .write = um_pci_bar_write, |
433 | .set = um_pci_bar_set, |
434 | .copy_from = um_pci_bar_copy_from, |
435 | .copy_to = um_pci_bar_copy_to, |
436 | }; |
437 | |
438 | static void __iomem *um_pci_map_bus(struct pci_bus *bus, unsigned int devfn, |
439 | int where) |
440 | { |
441 | struct um_pci_device_reg *dev; |
442 | unsigned int busn = bus->number; |
443 | |
444 | if (busn > 0) |
445 | return NULL; |
446 | |
447 | /* not allowing functions for now ... */ |
448 | if (devfn % 8) |
449 | return NULL; |
450 | |
451 | if (devfn / 8 >= ARRAY_SIZE(um_pci_devices)) |
452 | return NULL; |
453 | |
454 | dev = &um_pci_devices[devfn / 8]; |
455 | if (!dev) |
456 | return NULL; |
457 | |
458 | return (void __iomem *)((unsigned long)dev->iomem + where); |
459 | } |
460 | |
461 | static struct pci_ops um_pci_ops = { |
462 | .map_bus = um_pci_map_bus, |
463 | .read = pci_generic_config_read, |
464 | .write = pci_generic_config_write, |
465 | }; |
466 | |
467 | static void um_pci_rescan(void) |
468 | { |
469 | pci_lock_rescan_remove(); |
470 | pci_rescan_bus(bus: bridge->bus); |
471 | pci_unlock_rescan_remove(); |
472 | } |
473 | |
474 | static void um_pci_irq_vq_addbuf(struct virtqueue *vq, void *buf, bool kick) |
475 | { |
476 | struct scatterlist sg[1]; |
477 | |
478 | sg_init_one(sg, buf, MAX_IRQ_MSG_SIZE); |
479 | if (virtqueue_add_inbuf(vq, sg, num: 1, data: buf, GFP_ATOMIC)) |
480 | kfree(objp: buf); |
481 | else if (kick) |
482 | virtqueue_kick(vq); |
483 | } |
484 | |
485 | static void um_pci_handle_irq_message(struct virtqueue *vq, |
486 | struct virtio_pcidev_msg *msg) |
487 | { |
488 | struct virtio_device *vdev = vq->vdev; |
489 | struct um_pci_device *dev = vdev->priv; |
490 | |
491 | if (!dev->irq) |
492 | return; |
493 | |
494 | /* we should properly chain interrupts, but on ARCH=um we don't care */ |
495 | |
496 | switch (msg->op) { |
497 | case VIRTIO_PCIDEV_OP_INT: |
498 | generic_handle_irq(irq: dev->irq); |
499 | break; |
500 | case VIRTIO_PCIDEV_OP_MSI: |
501 | /* our MSI message is just the interrupt number */ |
502 | if (msg->size == sizeof(u32)) |
503 | generic_handle_irq(le32_to_cpup(p: (void *)msg->data)); |
504 | else |
505 | generic_handle_irq(le16_to_cpup(p: (void *)msg->data)); |
506 | break; |
507 | case VIRTIO_PCIDEV_OP_PME: |
508 | /* nothing to do - we already woke up due to the message */ |
509 | break; |
510 | default: |
511 | dev_err(&vdev->dev, "unexpected virt-pci message %d\n" , msg->op); |
512 | break; |
513 | } |
514 | } |
515 | |
516 | static void um_pci_cmd_vq_cb(struct virtqueue *vq) |
517 | { |
518 | struct virtio_device *vdev = vq->vdev; |
519 | struct um_pci_device *dev = vdev->priv; |
520 | void *cmd; |
521 | int len; |
522 | |
523 | if (test_bit(UM_PCI_STAT_WAITING, &dev->status)) |
524 | return; |
525 | |
526 | while ((cmd = virtqueue_get_buf(vq, len: &len))) { |
527 | if (WARN_ON(HANDLE_IS_NO_FREE(cmd))) |
528 | continue; |
529 | kfree(objp: cmd); |
530 | } |
531 | } |
532 | |
533 | static void um_pci_irq_vq_cb(struct virtqueue *vq) |
534 | { |
535 | struct virtio_pcidev_msg *msg; |
536 | int len; |
537 | |
538 | while ((msg = virtqueue_get_buf(vq, len: &len))) { |
539 | if (len >= sizeof(*msg)) |
540 | um_pci_handle_irq_message(vq, msg); |
541 | |
542 | /* recycle the message buffer */ |
543 | um_pci_irq_vq_addbuf(vq, buf: msg, kick: true); |
544 | } |
545 | } |
546 | |
547 | #ifdef CONFIG_OF |
548 | /* Copied from arch/x86/kernel/devicetree.c */ |
549 | struct device_node *pcibios_get_phb_of_node(struct pci_bus *bus) |
550 | { |
551 | struct device_node *np; |
552 | |
553 | for_each_node_by_type(np, "pci" ) { |
554 | const void *prop; |
555 | unsigned int bus_min; |
556 | |
557 | prop = of_get_property(node: np, name: "bus-range" , NULL); |
558 | if (!prop) |
559 | continue; |
560 | bus_min = be32_to_cpup(p: prop); |
561 | if (bus->number == bus_min) |
562 | return np; |
563 | } |
564 | return NULL; |
565 | } |
566 | #endif |
567 | |
568 | static int um_pci_init_vqs(struct um_pci_device *dev) |
569 | { |
570 | struct virtqueue *vqs[2]; |
571 | static const char *const names[2] = { "cmd" , "irq" }; |
572 | vq_callback_t *cbs[2] = { um_pci_cmd_vq_cb, um_pci_irq_vq_cb }; |
573 | int err, i; |
574 | |
575 | err = virtio_find_vqs(vdev: dev->vdev, nvqs: 2, vqs, callbacks: cbs, names, NULL); |
576 | if (err) |
577 | return err; |
578 | |
579 | dev->cmd_vq = vqs[0]; |
580 | dev->irq_vq = vqs[1]; |
581 | |
582 | virtio_device_ready(dev: dev->vdev); |
583 | |
584 | for (i = 0; i < NUM_IRQ_MSGS; i++) { |
585 | void *msg = kzalloc(MAX_IRQ_MSG_SIZE, GFP_KERNEL); |
586 | |
587 | if (msg) |
588 | um_pci_irq_vq_addbuf(vq: dev->irq_vq, buf: msg, kick: false); |
589 | } |
590 | |
591 | virtqueue_kick(vq: dev->irq_vq); |
592 | |
593 | return 0; |
594 | } |
595 | |
596 | static void __um_pci_virtio_platform_remove(struct virtio_device *vdev, |
597 | struct um_pci_device *dev) |
598 | { |
599 | virtio_reset_device(dev: vdev); |
600 | vdev->config->del_vqs(vdev); |
601 | |
602 | mutex_lock(&um_pci_mtx); |
603 | um_pci_platform_device = NULL; |
604 | mutex_unlock(lock: &um_pci_mtx); |
605 | |
606 | kfree(objp: dev); |
607 | } |
608 | |
609 | static int um_pci_virtio_platform_probe(struct virtio_device *vdev, |
610 | struct um_pci_device *dev) |
611 | { |
612 | int ret; |
613 | |
614 | dev->platform = true; |
615 | |
616 | mutex_lock(&um_pci_mtx); |
617 | |
618 | if (um_pci_platform_device) { |
619 | mutex_unlock(lock: &um_pci_mtx); |
620 | ret = -EBUSY; |
621 | goto out_free; |
622 | } |
623 | |
624 | ret = um_pci_init_vqs(dev); |
625 | if (ret) { |
626 | mutex_unlock(lock: &um_pci_mtx); |
627 | goto out_free; |
628 | } |
629 | |
630 | um_pci_platform_device = dev; |
631 | |
632 | mutex_unlock(lock: &um_pci_mtx); |
633 | |
634 | ret = of_platform_default_populate(root: vdev->dev.of_node, NULL, parent: &vdev->dev); |
635 | if (ret) |
636 | __um_pci_virtio_platform_remove(vdev, dev); |
637 | |
638 | return ret; |
639 | |
640 | out_free: |
641 | kfree(objp: dev); |
642 | return ret; |
643 | } |
644 | |
645 | static int um_pci_virtio_probe(struct virtio_device *vdev) |
646 | { |
647 | struct um_pci_device *dev; |
648 | int i, free = -1; |
649 | int err = -ENOSPC; |
650 | |
651 | dev = kzalloc(size: sizeof(*dev), GFP_KERNEL); |
652 | if (!dev) |
653 | return -ENOMEM; |
654 | |
655 | dev->vdev = vdev; |
656 | vdev->priv = dev; |
657 | |
658 | if (of_device_is_compatible(device: vdev->dev.of_node, "simple-bus" )) |
659 | return um_pci_virtio_platform_probe(vdev, dev); |
660 | |
661 | mutex_lock(&um_pci_mtx); |
662 | for (i = 0; i < MAX_DEVICES; i++) { |
663 | if (um_pci_devices[i].dev) |
664 | continue; |
665 | free = i; |
666 | break; |
667 | } |
668 | |
669 | if (free < 0) |
670 | goto error; |
671 | |
672 | err = um_pci_init_vqs(dev); |
673 | if (err) |
674 | goto error; |
675 | |
676 | dev->irq = irq_alloc_desc(numa_node_id()); |
677 | if (dev->irq < 0) { |
678 | err = dev->irq; |
679 | goto err_reset; |
680 | } |
681 | um_pci_devices[free].dev = dev; |
682 | vdev->priv = dev; |
683 | |
684 | mutex_unlock(lock: &um_pci_mtx); |
685 | |
686 | device_set_wakeup_enable(dev: &vdev->dev, enable: true); |
687 | |
688 | /* |
689 | * In order to do suspend-resume properly, don't allow VQs |
690 | * to be suspended. |
691 | */ |
692 | virtio_uml_set_no_vq_suspend(vdev, true); |
693 | |
694 | um_pci_rescan(); |
695 | return 0; |
696 | err_reset: |
697 | virtio_reset_device(dev: vdev); |
698 | vdev->config->del_vqs(vdev); |
699 | error: |
700 | mutex_unlock(lock: &um_pci_mtx); |
701 | kfree(objp: dev); |
702 | return err; |
703 | } |
704 | |
705 | static void um_pci_virtio_remove(struct virtio_device *vdev) |
706 | { |
707 | struct um_pci_device *dev = vdev->priv; |
708 | int i; |
709 | |
710 | if (dev->platform) { |
711 | of_platform_depopulate(parent: &vdev->dev); |
712 | __um_pci_virtio_platform_remove(vdev, dev); |
713 | return; |
714 | } |
715 | |
716 | device_set_wakeup_enable(dev: &vdev->dev, enable: false); |
717 | |
718 | mutex_lock(&um_pci_mtx); |
719 | for (i = 0; i < MAX_DEVICES; i++) { |
720 | if (um_pci_devices[i].dev != dev) |
721 | continue; |
722 | |
723 | um_pci_devices[i].dev = NULL; |
724 | irq_free_desc(irq: dev->irq); |
725 | |
726 | break; |
727 | } |
728 | mutex_unlock(lock: &um_pci_mtx); |
729 | |
730 | if (i < MAX_DEVICES) { |
731 | struct pci_dev *pci_dev; |
732 | |
733 | pci_dev = pci_get_slot(bus: bridge->bus, devfn: i); |
734 | if (pci_dev) |
735 | pci_stop_and_remove_bus_device_locked(dev: pci_dev); |
736 | } |
737 | |
738 | /* Stop all virtqueues */ |
739 | virtio_reset_device(dev: vdev); |
740 | dev->cmd_vq = NULL; |
741 | dev->irq_vq = NULL; |
742 | vdev->config->del_vqs(vdev); |
743 | |
744 | kfree(objp: dev); |
745 | } |
746 | |
747 | static struct virtio_device_id id_table[] = { |
748 | { CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID, VIRTIO_DEV_ANY_ID }, |
749 | { 0 }, |
750 | }; |
751 | MODULE_DEVICE_TABLE(virtio, id_table); |
752 | |
753 | static struct virtio_driver um_pci_virtio_driver = { |
754 | .driver.name = "virtio-pci" , |
755 | .driver.owner = THIS_MODULE, |
756 | .id_table = id_table, |
757 | .probe = um_pci_virtio_probe, |
758 | .remove = um_pci_virtio_remove, |
759 | }; |
760 | |
761 | static struct resource virt_cfgspace_resource = { |
762 | .name = "PCI config space" , |
763 | .start = 0xf0000000 - MAX_DEVICES * CFG_SPACE_SIZE, |
764 | .end = 0xf0000000 - 1, |
765 | .flags = IORESOURCE_MEM, |
766 | }; |
767 | |
768 | static long um_pci_map_cfgspace(unsigned long offset, size_t size, |
769 | const struct logic_iomem_ops **ops, |
770 | void **priv) |
771 | { |
772 | if (WARN_ON(size > CFG_SPACE_SIZE || offset % CFG_SPACE_SIZE)) |
773 | return -EINVAL; |
774 | |
775 | if (offset / CFG_SPACE_SIZE < MAX_DEVICES) { |
776 | *ops = &um_pci_device_cfgspace_ops; |
777 | *priv = &um_pci_devices[offset / CFG_SPACE_SIZE]; |
778 | return 0; |
779 | } |
780 | |
781 | WARN(1, "cannot map offset 0x%lx/0x%zx\n" , offset, size); |
782 | return -ENOENT; |
783 | } |
784 | |
785 | static const struct logic_iomem_region_ops um_pci_cfgspace_ops = { |
786 | .map = um_pci_map_cfgspace, |
787 | }; |
788 | |
789 | static struct resource virt_iomem_resource = { |
790 | .name = "PCI iomem" , |
791 | .start = 0xf0000000, |
792 | .end = 0xffffffff, |
793 | .flags = IORESOURCE_MEM, |
794 | }; |
795 | |
796 | struct um_pci_map_iomem_data { |
797 | unsigned long offset; |
798 | size_t size; |
799 | const struct logic_iomem_ops **ops; |
800 | void **priv; |
801 | long ret; |
802 | }; |
803 | |
804 | static int um_pci_map_iomem_walk(struct pci_dev *pdev, void *_data) |
805 | { |
806 | struct um_pci_map_iomem_data *data = _data; |
807 | struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8]; |
808 | struct um_pci_device *dev; |
809 | int i; |
810 | |
811 | if (!reg->dev) |
812 | return 0; |
813 | |
814 | for (i = 0; i < ARRAY_SIZE(dev->resptr); i++) { |
815 | struct resource *r = &pdev->resource[i]; |
816 | |
817 | if ((r->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM) |
818 | continue; |
819 | |
820 | /* |
821 | * must be the whole or part of the resource, |
822 | * not allowed to only overlap |
823 | */ |
824 | if (data->offset < r->start || data->offset > r->end) |
825 | continue; |
826 | if (data->offset + data->size - 1 > r->end) |
827 | continue; |
828 | |
829 | dev = reg->dev; |
830 | *data->ops = &um_pci_device_bar_ops; |
831 | dev->resptr[i] = i; |
832 | *data->priv = &dev->resptr[i]; |
833 | data->ret = data->offset - r->start; |
834 | |
835 | /* no need to continue */ |
836 | return 1; |
837 | } |
838 | |
839 | return 0; |
840 | } |
841 | |
842 | static long um_pci_map_iomem(unsigned long offset, size_t size, |
843 | const struct logic_iomem_ops **ops, |
844 | void **priv) |
845 | { |
846 | struct um_pci_map_iomem_data data = { |
847 | /* we want the full address here */ |
848 | .offset = offset + virt_iomem_resource.start, |
849 | .size = size, |
850 | .ops = ops, |
851 | .priv = priv, |
852 | .ret = -ENOENT, |
853 | }; |
854 | |
855 | pci_walk_bus(top: bridge->bus, cb: um_pci_map_iomem_walk, userdata: &data); |
856 | return data.ret; |
857 | } |
858 | |
859 | static const struct logic_iomem_region_ops um_pci_iomem_ops = { |
860 | .map = um_pci_map_iomem, |
861 | }; |
862 | |
863 | static void um_pci_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) |
864 | { |
865 | /* |
866 | * This is a very low address and not actually valid 'physical' memory |
867 | * in UML, so we can simply map MSI(-X) vectors to there, it cannot be |
868 | * legitimately written to by the device in any other way. |
869 | * We use the (virtual) IRQ number here as the message to simplify the |
870 | * code that receives the message, where for now we simply trust the |
871 | * device to send the correct message. |
872 | */ |
873 | msg->address_hi = 0; |
874 | msg->address_lo = 0xa0000; |
875 | msg->data = data->irq; |
876 | } |
877 | |
878 | static struct irq_chip um_pci_msi_bottom_irq_chip = { |
879 | .name = "UM virtio MSI" , |
880 | .irq_compose_msi_msg = um_pci_compose_msi_msg, |
881 | }; |
882 | |
883 | static int um_pci_inner_domain_alloc(struct irq_domain *domain, |
884 | unsigned int virq, unsigned int nr_irqs, |
885 | void *args) |
886 | { |
887 | unsigned long bit; |
888 | |
889 | WARN_ON(nr_irqs != 1); |
890 | |
891 | mutex_lock(&um_pci_mtx); |
892 | bit = find_first_zero_bit(addr: um_pci_msi_used, MAX_MSI_VECTORS); |
893 | if (bit >= MAX_MSI_VECTORS) { |
894 | mutex_unlock(lock: &um_pci_mtx); |
895 | return -ENOSPC; |
896 | } |
897 | |
898 | set_bit(nr: bit, addr: um_pci_msi_used); |
899 | mutex_unlock(lock: &um_pci_mtx); |
900 | |
901 | irq_domain_set_info(domain, virq, hwirq: bit, chip: &um_pci_msi_bottom_irq_chip, |
902 | chip_data: domain->host_data, handler: handle_simple_irq, |
903 | NULL, NULL); |
904 | |
905 | return 0; |
906 | } |
907 | |
908 | static void um_pci_inner_domain_free(struct irq_domain *domain, |
909 | unsigned int virq, unsigned int nr_irqs) |
910 | { |
911 | struct irq_data *d = irq_domain_get_irq_data(domain, virq); |
912 | |
913 | mutex_lock(&um_pci_mtx); |
914 | |
915 | if (!test_bit(d->hwirq, um_pci_msi_used)) |
916 | pr_err("trying to free unused MSI#%lu\n" , d->hwirq); |
917 | else |
918 | __clear_bit(d->hwirq, um_pci_msi_used); |
919 | |
920 | mutex_unlock(lock: &um_pci_mtx); |
921 | } |
922 | |
923 | static const struct irq_domain_ops um_pci_inner_domain_ops = { |
924 | .alloc = um_pci_inner_domain_alloc, |
925 | .free = um_pci_inner_domain_free, |
926 | }; |
927 | |
928 | static struct irq_chip um_pci_msi_irq_chip = { |
929 | .name = "UM virtio PCIe MSI" , |
930 | .irq_mask = pci_msi_mask_irq, |
931 | .irq_unmask = pci_msi_unmask_irq, |
932 | }; |
933 | |
934 | static struct msi_domain_info um_pci_msi_domain_info = { |
935 | .flags = MSI_FLAG_USE_DEF_DOM_OPS | |
936 | MSI_FLAG_USE_DEF_CHIP_OPS | |
937 | MSI_FLAG_PCI_MSIX, |
938 | .chip = &um_pci_msi_irq_chip, |
939 | }; |
940 | |
941 | static struct resource busn_resource = { |
942 | .name = "PCI busn" , |
943 | .start = 0, |
944 | .end = 0, |
945 | .flags = IORESOURCE_BUS, |
946 | }; |
947 | |
948 | static int um_pci_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin) |
949 | { |
950 | struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8]; |
951 | |
952 | if (WARN_ON(!reg->dev)) |
953 | return -EINVAL; |
954 | |
955 | /* Yes, we map all pins to the same IRQ ... doesn't matter for now. */ |
956 | return reg->dev->irq; |
957 | } |
958 | |
959 | void *pci_root_bus_fwnode(struct pci_bus *bus) |
960 | { |
961 | return um_pci_fwnode; |
962 | } |
963 | |
964 | static long um_pci_map_platform(unsigned long offset, size_t size, |
965 | const struct logic_iomem_ops **ops, |
966 | void **priv) |
967 | { |
968 | if (!um_pci_platform_device) |
969 | return -ENOENT; |
970 | |
971 | *ops = &um_pci_device_bar_ops; |
972 | *priv = &um_pci_platform_device->resptr[0]; |
973 | |
974 | return 0; |
975 | } |
976 | |
977 | static const struct logic_iomem_region_ops um_pci_platform_ops = { |
978 | .map = um_pci_map_platform, |
979 | }; |
980 | |
981 | static struct resource virt_platform_resource = { |
982 | .name = "platform" , |
983 | .start = 0x10000000, |
984 | .end = 0x1fffffff, |
985 | .flags = IORESOURCE_MEM, |
986 | }; |
987 | |
988 | static int __init um_pci_init(void) |
989 | { |
990 | int err, i; |
991 | |
992 | WARN_ON(logic_iomem_add_region(&virt_cfgspace_resource, |
993 | &um_pci_cfgspace_ops)); |
994 | WARN_ON(logic_iomem_add_region(&virt_iomem_resource, |
995 | &um_pci_iomem_ops)); |
996 | WARN_ON(logic_iomem_add_region(&virt_platform_resource, |
997 | &um_pci_platform_ops)); |
998 | |
999 | if (WARN(CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID < 0, |
1000 | "No virtio device ID configured for PCI - no PCI support\n" )) |
1001 | return 0; |
1002 | |
1003 | um_pci_msg_bufs = alloc_percpu(struct um_pci_message_buffer); |
1004 | if (!um_pci_msg_bufs) |
1005 | return -ENOMEM; |
1006 | |
1007 | bridge = pci_alloc_host_bridge(priv: 0); |
1008 | if (!bridge) { |
1009 | err = -ENOMEM; |
1010 | goto free; |
1011 | } |
1012 | |
1013 | um_pci_fwnode = irq_domain_alloc_named_fwnode(name: "um-pci" ); |
1014 | if (!um_pci_fwnode) { |
1015 | err = -ENOMEM; |
1016 | goto free; |
1017 | } |
1018 | |
1019 | um_pci_inner_domain = __irq_domain_add(fwnode: um_pci_fwnode, MAX_MSI_VECTORS, |
1020 | MAX_MSI_VECTORS, direct_max: 0, |
1021 | ops: &um_pci_inner_domain_ops, NULL); |
1022 | if (!um_pci_inner_domain) { |
1023 | err = -ENOMEM; |
1024 | goto free; |
1025 | } |
1026 | |
1027 | um_pci_msi_domain = pci_msi_create_irq_domain(fwnode: um_pci_fwnode, |
1028 | info: &um_pci_msi_domain_info, |
1029 | parent: um_pci_inner_domain); |
1030 | if (!um_pci_msi_domain) { |
1031 | err = -ENOMEM; |
1032 | goto free; |
1033 | } |
1034 | |
1035 | pci_add_resource(resources: &bridge->windows, res: &virt_iomem_resource); |
1036 | pci_add_resource(resources: &bridge->windows, res: &busn_resource); |
1037 | bridge->ops = &um_pci_ops; |
1038 | bridge->map_irq = um_pci_map_irq; |
1039 | |
1040 | for (i = 0; i < MAX_DEVICES; i++) { |
1041 | resource_size_t start; |
1042 | |
1043 | start = virt_cfgspace_resource.start + i * CFG_SPACE_SIZE; |
1044 | um_pci_devices[i].iomem = ioremap(offset: start, CFG_SPACE_SIZE); |
1045 | if (WARN(!um_pci_devices[i].iomem, "failed to map %d\n" , i)) { |
1046 | err = -ENOMEM; |
1047 | goto free; |
1048 | } |
1049 | } |
1050 | |
1051 | err = pci_host_probe(bridge); |
1052 | if (err) |
1053 | goto free; |
1054 | |
1055 | err = register_virtio_driver(drv: &um_pci_virtio_driver); |
1056 | if (err) |
1057 | goto free; |
1058 | return 0; |
1059 | free: |
1060 | if (um_pci_inner_domain) |
1061 | irq_domain_remove(host: um_pci_inner_domain); |
1062 | if (um_pci_fwnode) |
1063 | irq_domain_free_fwnode(fwnode: um_pci_fwnode); |
1064 | if (bridge) { |
1065 | pci_free_resource_list(resources: &bridge->windows); |
1066 | pci_free_host_bridge(bridge); |
1067 | } |
1068 | free_percpu(pdata: um_pci_msg_bufs); |
1069 | return err; |
1070 | } |
1071 | module_init(um_pci_init); |
1072 | |
1073 | static void __exit um_pci_exit(void) |
1074 | { |
1075 | unregister_virtio_driver(drv: &um_pci_virtio_driver); |
1076 | irq_domain_remove(host: um_pci_msi_domain); |
1077 | irq_domain_remove(host: um_pci_inner_domain); |
1078 | pci_free_resource_list(resources: &bridge->windows); |
1079 | pci_free_host_bridge(bridge); |
1080 | free_percpu(pdata: um_pci_msg_bufs); |
1081 | } |
1082 | module_exit(um_pci_exit); |
1083 | |