1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Virtio vhost-user driver |
4 | * |
5 | * Copyright(c) 2019 Intel Corporation |
6 | * |
7 | * This driver allows virtio devices to be used over a vhost-user socket. |
8 | * |
9 | * Guest devices can be instantiated by kernel module or command line |
10 | * parameters. One device will be created for each parameter. Syntax: |
11 | * |
12 | * virtio_uml.device=<socket>:<virtio_id>[:<platform_id>] |
13 | * where: |
14 | * <socket> := vhost-user socket path to connect |
15 | * <virtio_id> := virtio device id (as in virtio_ids.h) |
16 | * <platform_id> := (optional) platform device id |
17 | * |
18 | * example: |
19 | * virtio_uml.device=/var/uml.socket:1 |
20 | * |
21 | * Based on Virtio MMIO driver by Pawel Moll, copyright 2011-2014, ARM Ltd. |
22 | */ |
23 | #include <linux/module.h> |
24 | #include <linux/of.h> |
25 | #include <linux/platform_device.h> |
26 | #include <linux/slab.h> |
27 | #include <linux/virtio.h> |
28 | #include <linux/virtio_config.h> |
29 | #include <linux/virtio_ring.h> |
30 | #include <linux/time-internal.h> |
31 | #include <linux/virtio-uml.h> |
32 | #include <shared/as-layout.h> |
33 | #include <irq_kern.h> |
34 | #include <init.h> |
35 | #include <os.h> |
36 | #include "vhost_user.h" |
37 | |
38 | #define MAX_SUPPORTED_QUEUE_SIZE 256 |
39 | |
40 | #define to_virtio_uml_device(_vdev) \ |
41 | container_of(_vdev, struct virtio_uml_device, vdev) |
42 | |
43 | struct virtio_uml_platform_data { |
44 | u32 virtio_device_id; |
45 | const char *socket_path; |
46 | struct work_struct conn_broken_wk; |
47 | struct platform_device *pdev; |
48 | }; |
49 | |
50 | struct virtio_uml_device { |
51 | struct virtio_device vdev; |
52 | struct platform_device *pdev; |
53 | struct virtio_uml_platform_data *pdata; |
54 | |
55 | spinlock_t sock_lock; |
56 | int sock, req_fd, irq; |
57 | u64 features; |
58 | u64 protocol_features; |
59 | u8 status; |
60 | u8 registered:1; |
61 | u8 suspended:1; |
62 | u8 no_vq_suspend:1; |
63 | |
64 | u8 config_changed_irq:1; |
65 | uint64_t vq_irq_vq_map; |
66 | int recv_rc; |
67 | }; |
68 | |
69 | struct virtio_uml_vq_info { |
70 | int kick_fd, call_fd; |
71 | char name[32]; |
72 | bool suspended; |
73 | }; |
74 | |
75 | extern unsigned long long physmem_size, highmem; |
76 | |
77 | #define vu_err(vu_dev, ...) dev_err(&(vu_dev)->pdev->dev, ##__VA_ARGS__) |
78 | |
79 | /* Vhost-user protocol */ |
80 | |
81 | static int full_sendmsg_fds(int fd, const void *buf, unsigned int len, |
82 | const int *fds, unsigned int fds_num) |
83 | { |
84 | int rc; |
85 | |
86 | do { |
87 | rc = os_sendmsg_fds(fd, buf, len, fds, fds_num); |
88 | if (rc > 0) { |
89 | buf += rc; |
90 | len -= rc; |
91 | fds = NULL; |
92 | fds_num = 0; |
93 | } |
94 | } while (len && (rc >= 0 || rc == -EINTR)); |
95 | |
96 | if (rc < 0) |
97 | return rc; |
98 | return 0; |
99 | } |
100 | |
101 | static int full_read(int fd, void *buf, int len, bool abortable) |
102 | { |
103 | int rc; |
104 | |
105 | if (!len) |
106 | return 0; |
107 | |
108 | do { |
109 | rc = os_read_file(fd, buf, len); |
110 | if (rc > 0) { |
111 | buf += rc; |
112 | len -= rc; |
113 | } |
114 | } while (len && (rc > 0 || rc == -EINTR || (!abortable && rc == -EAGAIN))); |
115 | |
116 | if (rc < 0) |
117 | return rc; |
118 | if (rc == 0) |
119 | return -ECONNRESET; |
120 | return 0; |
121 | } |
122 | |
123 | static int (int fd, struct vhost_user_msg *msg) |
124 | { |
125 | return full_read(fd, buf: msg, len: sizeof(msg->header), abortable: true); |
126 | } |
127 | |
128 | static int vhost_user_recv(struct virtio_uml_device *vu_dev, |
129 | int fd, struct vhost_user_msg *msg, |
130 | size_t max_payload_size, bool wait) |
131 | { |
132 | size_t size; |
133 | int rc; |
134 | |
135 | /* |
136 | * In virtio time-travel mode, we're handling all the vhost-user |
137 | * FDs by polling them whenever appropriate. However, we may get |
138 | * into a situation where we're sending out an interrupt message |
139 | * to a device (e.g. a net device) and need to handle a simulation |
140 | * time message while doing so, e.g. one that tells us to update |
141 | * our idea of how long we can run without scheduling. |
142 | * |
143 | * Thus, we need to not just read() from the given fd, but need |
144 | * to also handle messages for the simulation time - this function |
145 | * does that for us while waiting for the given fd to be readable. |
146 | */ |
147 | if (wait) |
148 | time_travel_wait_readable(fd); |
149 | |
150 | rc = vhost_user_recv_header(fd, msg); |
151 | |
152 | if (rc) |
153 | return rc; |
154 | size = msg->header.size; |
155 | if (size > max_payload_size) |
156 | return -EPROTO; |
157 | return full_read(fd, buf: &msg->payload, len: size, abortable: false); |
158 | } |
159 | |
160 | static void vhost_user_check_reset(struct virtio_uml_device *vu_dev, |
161 | int rc) |
162 | { |
163 | struct virtio_uml_platform_data *pdata = vu_dev->pdata; |
164 | |
165 | if (rc != -ECONNRESET) |
166 | return; |
167 | |
168 | if (!vu_dev->registered) |
169 | return; |
170 | |
171 | vu_dev->registered = 0; |
172 | |
173 | schedule_work(work: &pdata->conn_broken_wk); |
174 | } |
175 | |
176 | static int vhost_user_recv_resp(struct virtio_uml_device *vu_dev, |
177 | struct vhost_user_msg *msg, |
178 | size_t max_payload_size) |
179 | { |
180 | int rc = vhost_user_recv(vu_dev, fd: vu_dev->sock, msg, |
181 | max_payload_size, wait: true); |
182 | |
183 | if (rc) { |
184 | vhost_user_check_reset(vu_dev, rc); |
185 | return rc; |
186 | } |
187 | |
188 | if (msg->header.flags != (VHOST_USER_FLAG_REPLY | VHOST_USER_VERSION)) |
189 | return -EPROTO; |
190 | |
191 | return 0; |
192 | } |
193 | |
194 | static int vhost_user_recv_u64(struct virtio_uml_device *vu_dev, |
195 | u64 *value) |
196 | { |
197 | struct vhost_user_msg msg; |
198 | int rc = vhost_user_recv_resp(vu_dev, msg: &msg, |
199 | max_payload_size: sizeof(msg.payload.integer)); |
200 | |
201 | if (rc) |
202 | return rc; |
203 | if (msg.header.size != sizeof(msg.payload.integer)) |
204 | return -EPROTO; |
205 | *value = msg.payload.integer; |
206 | return 0; |
207 | } |
208 | |
209 | static int vhost_user_recv_req(struct virtio_uml_device *vu_dev, |
210 | struct vhost_user_msg *msg, |
211 | size_t max_payload_size) |
212 | { |
213 | int rc = vhost_user_recv(vu_dev, fd: vu_dev->req_fd, msg, |
214 | max_payload_size, wait: false); |
215 | |
216 | if (rc) |
217 | return rc; |
218 | |
219 | if ((msg->header.flags & ~VHOST_USER_FLAG_NEED_REPLY) != |
220 | VHOST_USER_VERSION) |
221 | return -EPROTO; |
222 | |
223 | return 0; |
224 | } |
225 | |
226 | static int vhost_user_send(struct virtio_uml_device *vu_dev, |
227 | bool need_response, struct vhost_user_msg *msg, |
228 | int *fds, size_t num_fds) |
229 | { |
230 | size_t size = sizeof(msg->header) + msg->header.size; |
231 | unsigned long flags; |
232 | bool request_ack; |
233 | int rc; |
234 | |
235 | msg->header.flags |= VHOST_USER_VERSION; |
236 | |
237 | /* |
238 | * The need_response flag indicates that we already need a response, |
239 | * e.g. to read the features. In these cases, don't request an ACK as |
240 | * it is meaningless. Also request an ACK only if supported. |
241 | */ |
242 | request_ack = !need_response; |
243 | if (!(vu_dev->protocol_features & |
244 | BIT_ULL(VHOST_USER_PROTOCOL_F_REPLY_ACK))) |
245 | request_ack = false; |
246 | |
247 | if (request_ack) |
248 | msg->header.flags |= VHOST_USER_FLAG_NEED_REPLY; |
249 | |
250 | spin_lock_irqsave(&vu_dev->sock_lock, flags); |
251 | rc = full_sendmsg_fds(fd: vu_dev->sock, buf: msg, len: size, fds, fds_num: num_fds); |
252 | if (rc < 0) |
253 | goto out; |
254 | |
255 | if (request_ack) { |
256 | uint64_t status; |
257 | |
258 | rc = vhost_user_recv_u64(vu_dev, value: &status); |
259 | if (rc) |
260 | goto out; |
261 | |
262 | if (status) { |
263 | vu_err(vu_dev, "slave reports error: %llu\n" , status); |
264 | rc = -EIO; |
265 | goto out; |
266 | } |
267 | } |
268 | |
269 | out: |
270 | spin_unlock_irqrestore(lock: &vu_dev->sock_lock, flags); |
271 | return rc; |
272 | } |
273 | |
274 | static int vhost_user_send_no_payload(struct virtio_uml_device *vu_dev, |
275 | bool need_response, u32 request) |
276 | { |
277 | struct vhost_user_msg msg = { |
278 | .header.request = request, |
279 | }; |
280 | |
281 | return vhost_user_send(vu_dev, need_response, msg: &msg, NULL, num_fds: 0); |
282 | } |
283 | |
284 | static int vhost_user_send_no_payload_fd(struct virtio_uml_device *vu_dev, |
285 | u32 request, int fd) |
286 | { |
287 | struct vhost_user_msg msg = { |
288 | .header.request = request, |
289 | }; |
290 | |
291 | return vhost_user_send(vu_dev, need_response: false, msg: &msg, fds: &fd, num_fds: 1); |
292 | } |
293 | |
294 | static int vhost_user_send_u64(struct virtio_uml_device *vu_dev, |
295 | u32 request, u64 value) |
296 | { |
297 | struct vhost_user_msg msg = { |
298 | .header.request = request, |
299 | .header.size = sizeof(msg.payload.integer), |
300 | .payload.integer = value, |
301 | }; |
302 | |
303 | return vhost_user_send(vu_dev, need_response: false, msg: &msg, NULL, num_fds: 0); |
304 | } |
305 | |
306 | static int vhost_user_set_owner(struct virtio_uml_device *vu_dev) |
307 | { |
308 | return vhost_user_send_no_payload(vu_dev, need_response: false, request: VHOST_USER_SET_OWNER); |
309 | } |
310 | |
311 | static int vhost_user_get_features(struct virtio_uml_device *vu_dev, |
312 | u64 *features) |
313 | { |
314 | int rc = vhost_user_send_no_payload(vu_dev, need_response: true, |
315 | request: VHOST_USER_GET_FEATURES); |
316 | |
317 | if (rc) |
318 | return rc; |
319 | return vhost_user_recv_u64(vu_dev, value: features); |
320 | } |
321 | |
322 | static int vhost_user_set_features(struct virtio_uml_device *vu_dev, |
323 | u64 features) |
324 | { |
325 | return vhost_user_send_u64(vu_dev, request: VHOST_USER_SET_FEATURES, value: features); |
326 | } |
327 | |
328 | static int vhost_user_get_protocol_features(struct virtio_uml_device *vu_dev, |
329 | u64 *protocol_features) |
330 | { |
331 | int rc = vhost_user_send_no_payload(vu_dev, need_response: true, |
332 | request: VHOST_USER_GET_PROTOCOL_FEATURES); |
333 | |
334 | if (rc) |
335 | return rc; |
336 | return vhost_user_recv_u64(vu_dev, value: protocol_features); |
337 | } |
338 | |
339 | static int vhost_user_set_protocol_features(struct virtio_uml_device *vu_dev, |
340 | u64 protocol_features) |
341 | { |
342 | return vhost_user_send_u64(vu_dev, request: VHOST_USER_SET_PROTOCOL_FEATURES, |
343 | value: protocol_features); |
344 | } |
345 | |
346 | static void vhost_user_reply(struct virtio_uml_device *vu_dev, |
347 | struct vhost_user_msg *msg, int response) |
348 | { |
349 | struct vhost_user_msg reply = { |
350 | .payload.integer = response, |
351 | }; |
352 | size_t size = sizeof(reply.header) + sizeof(reply.payload.integer); |
353 | int rc; |
354 | |
355 | reply.header = msg->header; |
356 | reply.header.flags &= ~VHOST_USER_FLAG_NEED_REPLY; |
357 | reply.header.flags |= VHOST_USER_FLAG_REPLY; |
358 | reply.header.size = sizeof(reply.payload.integer); |
359 | |
360 | rc = full_sendmsg_fds(fd: vu_dev->req_fd, buf: &reply, len: size, NULL, fds_num: 0); |
361 | |
362 | if (rc) |
363 | vu_err(vu_dev, |
364 | "sending reply to slave request failed: %d (size %zu)\n" , |
365 | rc, size); |
366 | } |
367 | |
368 | static irqreturn_t vu_req_read_message(struct virtio_uml_device *vu_dev, |
369 | struct time_travel_event *ev) |
370 | { |
371 | struct virtqueue *vq; |
372 | int response = 1; |
373 | struct { |
374 | struct vhost_user_msg msg; |
375 | u8 [512]; |
376 | } msg; |
377 | int rc; |
378 | irqreturn_t irq_rc = IRQ_NONE; |
379 | |
380 | while (1) { |
381 | rc = vhost_user_recv_req(vu_dev, msg: &msg.msg, |
382 | max_payload_size: sizeof(msg.msg.payload) + |
383 | sizeof(msg.extra_payload)); |
384 | if (rc) |
385 | break; |
386 | |
387 | switch (msg.msg.header.request) { |
388 | case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG: |
389 | vu_dev->config_changed_irq = true; |
390 | response = 0; |
391 | break; |
392 | case VHOST_USER_SLAVE_VRING_CALL: |
393 | virtio_device_for_each_vq((&vu_dev->vdev), vq) { |
394 | if (vq->index == msg.msg.payload.vring_state.index) { |
395 | response = 0; |
396 | vu_dev->vq_irq_vq_map |= BIT_ULL(vq->index); |
397 | break; |
398 | } |
399 | } |
400 | break; |
401 | case VHOST_USER_SLAVE_IOTLB_MSG: |
402 | /* not supported - VIRTIO_F_ACCESS_PLATFORM */ |
403 | case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG: |
404 | /* not supported - VHOST_USER_PROTOCOL_F_HOST_NOTIFIER */ |
405 | default: |
406 | vu_err(vu_dev, "unexpected slave request %d\n" , |
407 | msg.msg.header.request); |
408 | } |
409 | |
410 | if (ev && !vu_dev->suspended) |
411 | time_travel_add_irq_event(ev); |
412 | |
413 | if (msg.msg.header.flags & VHOST_USER_FLAG_NEED_REPLY) |
414 | vhost_user_reply(vu_dev, msg: &msg.msg, response); |
415 | irq_rc = IRQ_HANDLED; |
416 | } |
417 | /* mask EAGAIN as we try non-blocking read until socket is empty */ |
418 | vu_dev->recv_rc = (rc == -EAGAIN) ? 0 : rc; |
419 | return irq_rc; |
420 | } |
421 | |
422 | static irqreturn_t vu_req_interrupt(int irq, void *data) |
423 | { |
424 | struct virtio_uml_device *vu_dev = data; |
425 | irqreturn_t ret = IRQ_HANDLED; |
426 | |
427 | if (!um_irq_timetravel_handler_used()) |
428 | ret = vu_req_read_message(vu_dev, NULL); |
429 | |
430 | if (vu_dev->recv_rc) { |
431 | vhost_user_check_reset(vu_dev, rc: vu_dev->recv_rc); |
432 | } else if (vu_dev->vq_irq_vq_map) { |
433 | struct virtqueue *vq; |
434 | |
435 | virtio_device_for_each_vq((&vu_dev->vdev), vq) { |
436 | if (vu_dev->vq_irq_vq_map & BIT_ULL(vq->index)) |
437 | vring_interrupt(irq: 0 /* ignored */, vq: vq); |
438 | } |
439 | vu_dev->vq_irq_vq_map = 0; |
440 | } else if (vu_dev->config_changed_irq) { |
441 | virtio_config_changed(dev: &vu_dev->vdev); |
442 | vu_dev->config_changed_irq = false; |
443 | } |
444 | |
445 | return ret; |
446 | } |
447 | |
448 | static void vu_req_interrupt_comm_handler(int irq, int fd, void *data, |
449 | struct time_travel_event *ev) |
450 | { |
451 | vu_req_read_message(vu_dev: data, ev); |
452 | } |
453 | |
454 | static int vhost_user_init_slave_req(struct virtio_uml_device *vu_dev) |
455 | { |
456 | int rc, req_fds[2]; |
457 | |
458 | /* Use a pipe for slave req fd, SIGIO is not supported for eventfd */ |
459 | rc = os_pipe(req_fds, true, true); |
460 | if (rc < 0) |
461 | return rc; |
462 | vu_dev->req_fd = req_fds[0]; |
463 | |
464 | rc = um_request_irq_tt(UM_IRQ_ALLOC, vu_dev->req_fd, IRQ_READ, |
465 | vu_req_interrupt, IRQF_SHARED, |
466 | vu_dev->pdev->name, vu_dev, |
467 | vu_req_interrupt_comm_handler); |
468 | if (rc < 0) |
469 | goto err_close; |
470 | |
471 | vu_dev->irq = rc; |
472 | |
473 | rc = vhost_user_send_no_payload_fd(vu_dev, request: VHOST_USER_SET_SLAVE_REQ_FD, |
474 | fd: req_fds[1]); |
475 | if (rc) |
476 | goto err_free_irq; |
477 | |
478 | goto out; |
479 | |
480 | err_free_irq: |
481 | um_free_irq(vu_dev->irq, vu_dev); |
482 | err_close: |
483 | os_close_file(req_fds[0]); |
484 | out: |
485 | /* Close unused write end of request fds */ |
486 | os_close_file(req_fds[1]); |
487 | return rc; |
488 | } |
489 | |
490 | static int vhost_user_init(struct virtio_uml_device *vu_dev) |
491 | { |
492 | int rc = vhost_user_set_owner(vu_dev); |
493 | |
494 | if (rc) |
495 | return rc; |
496 | rc = vhost_user_get_features(vu_dev, features: &vu_dev->features); |
497 | if (rc) |
498 | return rc; |
499 | |
500 | if (vu_dev->features & BIT_ULL(VHOST_USER_F_PROTOCOL_FEATURES)) { |
501 | rc = vhost_user_get_protocol_features(vu_dev, |
502 | protocol_features: &vu_dev->protocol_features); |
503 | if (rc) |
504 | return rc; |
505 | vu_dev->protocol_features &= VHOST_USER_SUPPORTED_PROTOCOL_F; |
506 | rc = vhost_user_set_protocol_features(vu_dev, |
507 | protocol_features: vu_dev->protocol_features); |
508 | if (rc) |
509 | return rc; |
510 | } |
511 | |
512 | if (vu_dev->protocol_features & |
513 | BIT_ULL(VHOST_USER_PROTOCOL_F_SLAVE_REQ)) { |
514 | rc = vhost_user_init_slave_req(vu_dev); |
515 | if (rc) |
516 | return rc; |
517 | } |
518 | |
519 | return 0; |
520 | } |
521 | |
522 | static void vhost_user_get_config(struct virtio_uml_device *vu_dev, |
523 | u32 offset, void *buf, u32 len) |
524 | { |
525 | u32 cfg_size = offset + len; |
526 | struct vhost_user_msg *msg; |
527 | size_t payload_size = sizeof(msg->payload.config) + cfg_size; |
528 | size_t msg_size = sizeof(msg->header) + payload_size; |
529 | int rc; |
530 | |
531 | if (!(vu_dev->protocol_features & |
532 | BIT_ULL(VHOST_USER_PROTOCOL_F_CONFIG))) |
533 | return; |
534 | |
535 | msg = kzalloc(size: msg_size, GFP_KERNEL); |
536 | if (!msg) |
537 | return; |
538 | msg->header.request = VHOST_USER_GET_CONFIG; |
539 | msg->header.size = payload_size; |
540 | msg->payload.config.offset = 0; |
541 | msg->payload.config.size = cfg_size; |
542 | |
543 | rc = vhost_user_send(vu_dev, need_response: true, msg, NULL, num_fds: 0); |
544 | if (rc) { |
545 | vu_err(vu_dev, "sending VHOST_USER_GET_CONFIG failed: %d\n" , |
546 | rc); |
547 | goto free; |
548 | } |
549 | |
550 | rc = vhost_user_recv_resp(vu_dev, msg, max_payload_size: msg_size); |
551 | if (rc) { |
552 | vu_err(vu_dev, |
553 | "receiving VHOST_USER_GET_CONFIG response failed: %d\n" , |
554 | rc); |
555 | goto free; |
556 | } |
557 | |
558 | if (msg->header.size != payload_size || |
559 | msg->payload.config.size != cfg_size) { |
560 | rc = -EPROTO; |
561 | vu_err(vu_dev, |
562 | "Invalid VHOST_USER_GET_CONFIG sizes (payload %d expected %zu, config %u expected %u)\n" , |
563 | msg->header.size, payload_size, |
564 | msg->payload.config.size, cfg_size); |
565 | goto free; |
566 | } |
567 | memcpy(buf, msg->payload.config.payload + offset, len); |
568 | |
569 | free: |
570 | kfree(objp: msg); |
571 | } |
572 | |
573 | static void vhost_user_set_config(struct virtio_uml_device *vu_dev, |
574 | u32 offset, const void *buf, u32 len) |
575 | { |
576 | struct vhost_user_msg *msg; |
577 | size_t payload_size = sizeof(msg->payload.config) + len; |
578 | size_t msg_size = sizeof(msg->header) + payload_size; |
579 | int rc; |
580 | |
581 | if (!(vu_dev->protocol_features & |
582 | BIT_ULL(VHOST_USER_PROTOCOL_F_CONFIG))) |
583 | return; |
584 | |
585 | msg = kzalloc(size: msg_size, GFP_KERNEL); |
586 | if (!msg) |
587 | return; |
588 | msg->header.request = VHOST_USER_SET_CONFIG; |
589 | msg->header.size = payload_size; |
590 | msg->payload.config.offset = offset; |
591 | msg->payload.config.size = len; |
592 | memcpy(msg->payload.config.payload, buf, len); |
593 | |
594 | rc = vhost_user_send(vu_dev, need_response: false, msg, NULL, num_fds: 0); |
595 | if (rc) |
596 | vu_err(vu_dev, "sending VHOST_USER_SET_CONFIG failed: %d\n" , |
597 | rc); |
598 | |
599 | kfree(objp: msg); |
600 | } |
601 | |
602 | static int vhost_user_init_mem_region(u64 addr, u64 size, int *fd_out, |
603 | struct vhost_user_mem_region *region_out) |
604 | { |
605 | unsigned long long mem_offset; |
606 | int rc = phys_mapping(addr, &mem_offset); |
607 | |
608 | if (WARN(rc < 0, "phys_mapping of 0x%llx returned %d\n" , addr, rc)) |
609 | return -EFAULT; |
610 | *fd_out = rc; |
611 | region_out->guest_addr = addr; |
612 | region_out->user_addr = addr; |
613 | region_out->size = size; |
614 | region_out->mmap_offset = mem_offset; |
615 | |
616 | /* Ensure mapping is valid for the entire region */ |
617 | rc = phys_mapping(addr + size - 1, &mem_offset); |
618 | if (WARN(rc != *fd_out, "phys_mapping of 0x%llx failed: %d != %d\n" , |
619 | addr + size - 1, rc, *fd_out)) |
620 | return -EFAULT; |
621 | return 0; |
622 | } |
623 | |
624 | static int vhost_user_set_mem_table(struct virtio_uml_device *vu_dev) |
625 | { |
626 | struct vhost_user_msg msg = { |
627 | .header.request = VHOST_USER_SET_MEM_TABLE, |
628 | .header.size = sizeof(msg.payload.mem_regions), |
629 | .payload.mem_regions.num = 1, |
630 | }; |
631 | unsigned long reserved = uml_reserved - uml_physmem; |
632 | int fds[2]; |
633 | int rc; |
634 | |
635 | /* |
636 | * This is a bit tricky, see also the comment with setup_physmem(). |
637 | * |
638 | * Essentially, setup_physmem() uses a file to mmap() our physmem, |
639 | * but the code and data we *already* have is omitted. To us, this |
640 | * is no difference, since they both become part of our address |
641 | * space and memory consumption. To somebody looking in from the |
642 | * outside, however, it is different because the part of our memory |
643 | * consumption that's already part of the binary (code/data) is not |
644 | * mapped from the file, so it's not visible to another mmap from |
645 | * the file descriptor. |
646 | * |
647 | * Thus, don't advertise this space to the vhost-user slave. This |
648 | * means that the slave will likely abort or similar when we give |
649 | * it an address from the hidden range, since it's not marked as |
650 | * a valid address, but at least that way we detect the issue and |
651 | * don't just have the slave read an all-zeroes buffer from the |
652 | * shared memory file, or write something there that we can never |
653 | * see (depending on the direction of the virtqueue traffic.) |
654 | * |
655 | * Since we usually don't want to use .text for virtio buffers, |
656 | * this effectively means that you cannot use |
657 | * 1) global variables, which are in the .bss and not in the shm |
658 | * file-backed memory |
659 | * 2) the stack in some processes, depending on where they have |
660 | * their stack (or maybe only no interrupt stack?) |
661 | * |
662 | * The stack is already not typically valid for DMA, so this isn't |
663 | * much of a restriction, but global variables might be encountered. |
664 | * |
665 | * It might be possible to fix it by copying around the data that's |
666 | * between bss_start and where we map the file now, but it's not |
667 | * something that you typically encounter with virtio drivers, so |
668 | * it didn't seem worthwhile. |
669 | */ |
670 | rc = vhost_user_init_mem_region(addr: reserved, size: physmem_size - reserved, |
671 | fd_out: &fds[0], |
672 | region_out: &msg.payload.mem_regions.regions[0]); |
673 | |
674 | if (rc < 0) |
675 | return rc; |
676 | if (highmem) { |
677 | msg.payload.mem_regions.num++; |
678 | rc = vhost_user_init_mem_region(__pa(end_iomem), highmem, |
679 | &fds[1], &msg.payload.mem_regions.regions[1]); |
680 | if (rc < 0) |
681 | return rc; |
682 | } |
683 | |
684 | return vhost_user_send(vu_dev, need_response: false, msg: &msg, fds, |
685 | num_fds: msg.payload.mem_regions.num); |
686 | } |
687 | |
688 | static int vhost_user_set_vring_state(struct virtio_uml_device *vu_dev, |
689 | u32 request, u32 index, u32 num) |
690 | { |
691 | struct vhost_user_msg msg = { |
692 | .header.request = request, |
693 | .header.size = sizeof(msg.payload.vring_state), |
694 | .payload.vring_state.index = index, |
695 | .payload.vring_state.num = num, |
696 | }; |
697 | |
698 | return vhost_user_send(vu_dev, need_response: false, msg: &msg, NULL, num_fds: 0); |
699 | } |
700 | |
701 | static int vhost_user_set_vring_num(struct virtio_uml_device *vu_dev, |
702 | u32 index, u32 num) |
703 | { |
704 | return vhost_user_set_vring_state(vu_dev, request: VHOST_USER_SET_VRING_NUM, |
705 | index, num); |
706 | } |
707 | |
708 | static int vhost_user_set_vring_base(struct virtio_uml_device *vu_dev, |
709 | u32 index, u32 offset) |
710 | { |
711 | return vhost_user_set_vring_state(vu_dev, request: VHOST_USER_SET_VRING_BASE, |
712 | index, num: offset); |
713 | } |
714 | |
715 | static int vhost_user_set_vring_addr(struct virtio_uml_device *vu_dev, |
716 | u32 index, u64 desc, u64 used, u64 avail, |
717 | u64 log) |
718 | { |
719 | struct vhost_user_msg msg = { |
720 | .header.request = VHOST_USER_SET_VRING_ADDR, |
721 | .header.size = sizeof(msg.payload.vring_addr), |
722 | .payload.vring_addr.index = index, |
723 | .payload.vring_addr.desc = desc, |
724 | .payload.vring_addr.used = used, |
725 | .payload.vring_addr.avail = avail, |
726 | .payload.vring_addr.log = log, |
727 | }; |
728 | |
729 | return vhost_user_send(vu_dev, need_response: false, msg: &msg, NULL, num_fds: 0); |
730 | } |
731 | |
732 | static int vhost_user_set_vring_fd(struct virtio_uml_device *vu_dev, |
733 | u32 request, int index, int fd) |
734 | { |
735 | struct vhost_user_msg msg = { |
736 | .header.request = request, |
737 | .header.size = sizeof(msg.payload.integer), |
738 | .payload.integer = index, |
739 | }; |
740 | |
741 | if (index & ~VHOST_USER_VRING_INDEX_MASK) |
742 | return -EINVAL; |
743 | if (fd < 0) { |
744 | msg.payload.integer |= VHOST_USER_VRING_POLL_MASK; |
745 | return vhost_user_send(vu_dev, need_response: false, msg: &msg, NULL, num_fds: 0); |
746 | } |
747 | return vhost_user_send(vu_dev, need_response: false, msg: &msg, fds: &fd, num_fds: 1); |
748 | } |
749 | |
750 | static int vhost_user_set_vring_call(struct virtio_uml_device *vu_dev, |
751 | int index, int fd) |
752 | { |
753 | return vhost_user_set_vring_fd(vu_dev, request: VHOST_USER_SET_VRING_CALL, |
754 | index, fd); |
755 | } |
756 | |
757 | static int vhost_user_set_vring_kick(struct virtio_uml_device *vu_dev, |
758 | int index, int fd) |
759 | { |
760 | return vhost_user_set_vring_fd(vu_dev, request: VHOST_USER_SET_VRING_KICK, |
761 | index, fd); |
762 | } |
763 | |
764 | static int vhost_user_set_vring_enable(struct virtio_uml_device *vu_dev, |
765 | u32 index, bool enable) |
766 | { |
767 | if (!(vu_dev->features & BIT_ULL(VHOST_USER_F_PROTOCOL_FEATURES))) |
768 | return 0; |
769 | |
770 | return vhost_user_set_vring_state(vu_dev, request: VHOST_USER_SET_VRING_ENABLE, |
771 | index, num: enable); |
772 | } |
773 | |
774 | |
775 | /* Virtio interface */ |
776 | |
777 | static bool vu_notify(struct virtqueue *vq) |
778 | { |
779 | struct virtio_uml_vq_info *info = vq->priv; |
780 | const uint64_t n = 1; |
781 | int rc; |
782 | |
783 | if (info->suspended) |
784 | return true; |
785 | |
786 | time_travel_propagate_time(); |
787 | |
788 | if (info->kick_fd < 0) { |
789 | struct virtio_uml_device *vu_dev; |
790 | |
791 | vu_dev = to_virtio_uml_device(vq->vdev); |
792 | |
793 | return vhost_user_set_vring_state(vu_dev, request: VHOST_USER_VRING_KICK, |
794 | index: vq->index, num: 0) == 0; |
795 | } |
796 | |
797 | do { |
798 | rc = os_write_file(info->kick_fd, &n, sizeof(n)); |
799 | } while (rc == -EINTR); |
800 | return !WARN(rc != sizeof(n), "write returned %d\n" , rc); |
801 | } |
802 | |
803 | static irqreturn_t vu_interrupt(int irq, void *opaque) |
804 | { |
805 | struct virtqueue *vq = opaque; |
806 | struct virtio_uml_vq_info *info = vq->priv; |
807 | uint64_t n; |
808 | int rc; |
809 | irqreturn_t ret = IRQ_NONE; |
810 | |
811 | do { |
812 | rc = os_read_file(info->call_fd, &n, sizeof(n)); |
813 | if (rc == sizeof(n)) |
814 | ret |= vring_interrupt(irq, vq: vq); |
815 | } while (rc == sizeof(n) || rc == -EINTR); |
816 | WARN(rc != -EAGAIN, "read returned %d\n" , rc); |
817 | return ret; |
818 | } |
819 | |
820 | |
821 | static void vu_get(struct virtio_device *vdev, unsigned offset, |
822 | void *buf, unsigned len) |
823 | { |
824 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
825 | |
826 | vhost_user_get_config(vu_dev, offset, buf, len); |
827 | } |
828 | |
829 | static void vu_set(struct virtio_device *vdev, unsigned offset, |
830 | const void *buf, unsigned len) |
831 | { |
832 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
833 | |
834 | vhost_user_set_config(vu_dev, offset, buf, len); |
835 | } |
836 | |
837 | static u8 vu_get_status(struct virtio_device *vdev) |
838 | { |
839 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
840 | |
841 | return vu_dev->status; |
842 | } |
843 | |
844 | static void vu_set_status(struct virtio_device *vdev, u8 status) |
845 | { |
846 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
847 | |
848 | vu_dev->status = status; |
849 | } |
850 | |
851 | static void vu_reset(struct virtio_device *vdev) |
852 | { |
853 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
854 | |
855 | vu_dev->status = 0; |
856 | } |
857 | |
858 | static void vu_del_vq(struct virtqueue *vq) |
859 | { |
860 | struct virtio_uml_vq_info *info = vq->priv; |
861 | |
862 | if (info->call_fd >= 0) { |
863 | struct virtio_uml_device *vu_dev; |
864 | |
865 | vu_dev = to_virtio_uml_device(vq->vdev); |
866 | |
867 | um_free_irq(vu_dev->irq, vq); |
868 | os_close_file(info->call_fd); |
869 | } |
870 | |
871 | if (info->kick_fd >= 0) |
872 | os_close_file(info->kick_fd); |
873 | |
874 | vring_del_virtqueue(vq); |
875 | kfree(objp: info); |
876 | } |
877 | |
878 | static void vu_del_vqs(struct virtio_device *vdev) |
879 | { |
880 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
881 | struct virtqueue *vq, *n; |
882 | u64 features; |
883 | |
884 | /* Note: reverse order as a workaround to a decoding bug in snabb */ |
885 | list_for_each_entry_reverse(vq, &vdev->vqs, list) |
886 | WARN_ON(vhost_user_set_vring_enable(vu_dev, vq->index, false)); |
887 | |
888 | /* Ensure previous messages have been processed */ |
889 | WARN_ON(vhost_user_get_features(vu_dev, &features)); |
890 | |
891 | list_for_each_entry_safe(vq, n, &vdev->vqs, list) |
892 | vu_del_vq(vq); |
893 | } |
894 | |
895 | static int vu_setup_vq_call_fd(struct virtio_uml_device *vu_dev, |
896 | struct virtqueue *vq) |
897 | { |
898 | struct virtio_uml_vq_info *info = vq->priv; |
899 | int call_fds[2]; |
900 | int rc; |
901 | |
902 | /* no call FD needed/desired in this case */ |
903 | if (vu_dev->protocol_features & |
904 | BIT_ULL(VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS) && |
905 | vu_dev->protocol_features & |
906 | BIT_ULL(VHOST_USER_PROTOCOL_F_SLAVE_REQ)) { |
907 | info->call_fd = -1; |
908 | return 0; |
909 | } |
910 | |
911 | /* Use a pipe for call fd, since SIGIO is not supported for eventfd */ |
912 | rc = os_pipe(call_fds, true, true); |
913 | if (rc < 0) |
914 | return rc; |
915 | |
916 | info->call_fd = call_fds[0]; |
917 | rc = um_request_irq(vu_dev->irq, info->call_fd, IRQ_READ, |
918 | vu_interrupt, IRQF_SHARED, info->name, vq); |
919 | if (rc < 0) |
920 | goto close_both; |
921 | |
922 | rc = vhost_user_set_vring_call(vu_dev, index: vq->index, fd: call_fds[1]); |
923 | if (rc) |
924 | goto release_irq; |
925 | |
926 | goto out; |
927 | |
928 | release_irq: |
929 | um_free_irq(vu_dev->irq, vq); |
930 | close_both: |
931 | os_close_file(call_fds[0]); |
932 | out: |
933 | /* Close (unused) write end of call fds */ |
934 | os_close_file(call_fds[1]); |
935 | |
936 | return rc; |
937 | } |
938 | |
939 | static struct virtqueue *vu_setup_vq(struct virtio_device *vdev, |
940 | unsigned index, vq_callback_t *callback, |
941 | const char *name, bool ctx) |
942 | { |
943 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
944 | struct platform_device *pdev = vu_dev->pdev; |
945 | struct virtio_uml_vq_info *info; |
946 | struct virtqueue *vq; |
947 | int num = MAX_SUPPORTED_QUEUE_SIZE; |
948 | int rc; |
949 | |
950 | info = kzalloc(size: sizeof(*info), GFP_KERNEL); |
951 | if (!info) { |
952 | rc = -ENOMEM; |
953 | goto error_kzalloc; |
954 | } |
955 | snprintf(buf: info->name, size: sizeof(info->name), fmt: "%s.%d-%s" , pdev->name, |
956 | pdev->id, name); |
957 | |
958 | vq = vring_create_virtqueue(index, num, PAGE_SIZE, vdev, weak_barriers: true, may_reduce_num: true, |
959 | ctx, notify: vu_notify, callback, name: info->name); |
960 | if (!vq) { |
961 | rc = -ENOMEM; |
962 | goto error_create; |
963 | } |
964 | vq->priv = info; |
965 | vq->num_max = num; |
966 | num = virtqueue_get_vring_size(vq); |
967 | |
968 | if (vu_dev->protocol_features & |
969 | BIT_ULL(VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS)) { |
970 | info->kick_fd = -1; |
971 | } else { |
972 | rc = os_eventfd(0, 0); |
973 | if (rc < 0) |
974 | goto error_kick; |
975 | info->kick_fd = rc; |
976 | } |
977 | |
978 | rc = vu_setup_vq_call_fd(vu_dev, vq); |
979 | if (rc) |
980 | goto error_call; |
981 | |
982 | rc = vhost_user_set_vring_num(vu_dev, index, num); |
983 | if (rc) |
984 | goto error_setup; |
985 | |
986 | rc = vhost_user_set_vring_base(vu_dev, index, offset: 0); |
987 | if (rc) |
988 | goto error_setup; |
989 | |
990 | rc = vhost_user_set_vring_addr(vu_dev, index, |
991 | desc: virtqueue_get_desc_addr(vq), |
992 | used: virtqueue_get_used_addr(vq), |
993 | avail: virtqueue_get_avail_addr(vq), |
994 | log: (u64) -1); |
995 | if (rc) |
996 | goto error_setup; |
997 | |
998 | return vq; |
999 | |
1000 | error_setup: |
1001 | if (info->call_fd >= 0) { |
1002 | um_free_irq(vu_dev->irq, vq); |
1003 | os_close_file(info->call_fd); |
1004 | } |
1005 | error_call: |
1006 | if (info->kick_fd >= 0) |
1007 | os_close_file(info->kick_fd); |
1008 | error_kick: |
1009 | vring_del_virtqueue(vq); |
1010 | error_create: |
1011 | kfree(objp: info); |
1012 | error_kzalloc: |
1013 | return ERR_PTR(error: rc); |
1014 | } |
1015 | |
1016 | static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs, |
1017 | struct virtqueue *vqs[], vq_callback_t *callbacks[], |
1018 | const char * const names[], const bool *ctx, |
1019 | struct irq_affinity *desc) |
1020 | { |
1021 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
1022 | int i, queue_idx = 0, rc; |
1023 | struct virtqueue *vq; |
1024 | |
1025 | /* not supported for now */ |
1026 | if (WARN_ON(nvqs > 64)) |
1027 | return -EINVAL; |
1028 | |
1029 | rc = vhost_user_set_mem_table(vu_dev); |
1030 | if (rc) |
1031 | return rc; |
1032 | |
1033 | for (i = 0; i < nvqs; ++i) { |
1034 | if (!names[i]) { |
1035 | vqs[i] = NULL; |
1036 | continue; |
1037 | } |
1038 | |
1039 | vqs[i] = vu_setup_vq(vdev, index: queue_idx++, callback: callbacks[i], name: names[i], |
1040 | ctx: ctx ? ctx[i] : false); |
1041 | if (IS_ERR(ptr: vqs[i])) { |
1042 | rc = PTR_ERR(ptr: vqs[i]); |
1043 | goto error_setup; |
1044 | } |
1045 | } |
1046 | |
1047 | list_for_each_entry(vq, &vdev->vqs, list) { |
1048 | struct virtio_uml_vq_info *info = vq->priv; |
1049 | |
1050 | if (info->kick_fd >= 0) { |
1051 | rc = vhost_user_set_vring_kick(vu_dev, index: vq->index, |
1052 | fd: info->kick_fd); |
1053 | if (rc) |
1054 | goto error_setup; |
1055 | } |
1056 | |
1057 | rc = vhost_user_set_vring_enable(vu_dev, index: vq->index, enable: true); |
1058 | if (rc) |
1059 | goto error_setup; |
1060 | } |
1061 | |
1062 | return 0; |
1063 | |
1064 | error_setup: |
1065 | vu_del_vqs(vdev); |
1066 | return rc; |
1067 | } |
1068 | |
1069 | static u64 vu_get_features(struct virtio_device *vdev) |
1070 | { |
1071 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
1072 | |
1073 | return vu_dev->features; |
1074 | } |
1075 | |
1076 | static int vu_finalize_features(struct virtio_device *vdev) |
1077 | { |
1078 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
1079 | u64 supported = vdev->features & VHOST_USER_SUPPORTED_F; |
1080 | |
1081 | vring_transport_features(vdev); |
1082 | vu_dev->features = vdev->features | supported; |
1083 | |
1084 | return vhost_user_set_features(vu_dev, features: vu_dev->features); |
1085 | } |
1086 | |
1087 | static const char *vu_bus_name(struct virtio_device *vdev) |
1088 | { |
1089 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
1090 | |
1091 | return vu_dev->pdev->name; |
1092 | } |
1093 | |
1094 | static const struct virtio_config_ops virtio_uml_config_ops = { |
1095 | .get = vu_get, |
1096 | .set = vu_set, |
1097 | .get_status = vu_get_status, |
1098 | .set_status = vu_set_status, |
1099 | .reset = vu_reset, |
1100 | .find_vqs = vu_find_vqs, |
1101 | .del_vqs = vu_del_vqs, |
1102 | .get_features = vu_get_features, |
1103 | .finalize_features = vu_finalize_features, |
1104 | .bus_name = vu_bus_name, |
1105 | }; |
1106 | |
1107 | static void virtio_uml_release_dev(struct device *d) |
1108 | { |
1109 | struct virtio_device *vdev = |
1110 | container_of(d, struct virtio_device, dev); |
1111 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
1112 | |
1113 | time_travel_propagate_time(); |
1114 | |
1115 | /* might not have been opened due to not negotiating the feature */ |
1116 | if (vu_dev->req_fd >= 0) { |
1117 | um_free_irq(vu_dev->irq, vu_dev); |
1118 | os_close_file(vu_dev->req_fd); |
1119 | } |
1120 | |
1121 | os_close_file(vu_dev->sock); |
1122 | kfree(objp: vu_dev); |
1123 | } |
1124 | |
1125 | void virtio_uml_set_no_vq_suspend(struct virtio_device *vdev, |
1126 | bool no_vq_suspend) |
1127 | { |
1128 | struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev); |
1129 | |
1130 | if (WARN_ON(vdev->config != &virtio_uml_config_ops)) |
1131 | return; |
1132 | |
1133 | vu_dev->no_vq_suspend = no_vq_suspend; |
1134 | dev_info(&vdev->dev, "%sabled VQ suspend\n" , |
1135 | no_vq_suspend ? "dis" : "en" ); |
1136 | } |
1137 | |
1138 | static void vu_of_conn_broken(struct work_struct *wk) |
1139 | { |
1140 | struct virtio_uml_platform_data *pdata; |
1141 | struct virtio_uml_device *vu_dev; |
1142 | |
1143 | pdata = container_of(wk, struct virtio_uml_platform_data, conn_broken_wk); |
1144 | |
1145 | vu_dev = platform_get_drvdata(pdev: pdata->pdev); |
1146 | |
1147 | virtio_break_device(dev: &vu_dev->vdev); |
1148 | |
1149 | /* |
1150 | * We can't remove the device from the devicetree so the only thing we |
1151 | * can do is warn. |
1152 | */ |
1153 | WARN_ON(1); |
1154 | } |
1155 | |
1156 | /* Platform device */ |
1157 | |
1158 | static struct virtio_uml_platform_data * |
1159 | virtio_uml_create_pdata(struct platform_device *pdev) |
1160 | { |
1161 | struct device_node *np = pdev->dev.of_node; |
1162 | struct virtio_uml_platform_data *pdata; |
1163 | int ret; |
1164 | |
1165 | if (!np) |
1166 | return ERR_PTR(error: -EINVAL); |
1167 | |
1168 | pdata = devm_kzalloc(dev: &pdev->dev, size: sizeof(*pdata), GFP_KERNEL); |
1169 | if (!pdata) |
1170 | return ERR_PTR(error: -ENOMEM); |
1171 | |
1172 | INIT_WORK(&pdata->conn_broken_wk, vu_of_conn_broken); |
1173 | pdata->pdev = pdev; |
1174 | |
1175 | ret = of_property_read_string(np, propname: "socket-path" , out_string: &pdata->socket_path); |
1176 | if (ret) |
1177 | return ERR_PTR(error: ret); |
1178 | |
1179 | ret = of_property_read_u32(np, propname: "virtio-device-id" , |
1180 | out_value: &pdata->virtio_device_id); |
1181 | if (ret) |
1182 | return ERR_PTR(error: ret); |
1183 | |
1184 | return pdata; |
1185 | } |
1186 | |
1187 | static int virtio_uml_probe(struct platform_device *pdev) |
1188 | { |
1189 | struct virtio_uml_platform_data *pdata = pdev->dev.platform_data; |
1190 | struct virtio_uml_device *vu_dev; |
1191 | int rc; |
1192 | |
1193 | if (!pdata) { |
1194 | pdata = virtio_uml_create_pdata(pdev); |
1195 | if (IS_ERR(ptr: pdata)) |
1196 | return PTR_ERR(ptr: pdata); |
1197 | } |
1198 | |
1199 | vu_dev = kzalloc(size: sizeof(*vu_dev), GFP_KERNEL); |
1200 | if (!vu_dev) |
1201 | return -ENOMEM; |
1202 | |
1203 | vu_dev->pdata = pdata; |
1204 | vu_dev->vdev.dev.parent = &pdev->dev; |
1205 | vu_dev->vdev.dev.release = virtio_uml_release_dev; |
1206 | vu_dev->vdev.config = &virtio_uml_config_ops; |
1207 | vu_dev->vdev.id.device = pdata->virtio_device_id; |
1208 | vu_dev->vdev.id.vendor = VIRTIO_DEV_ANY_ID; |
1209 | vu_dev->pdev = pdev; |
1210 | vu_dev->req_fd = -1; |
1211 | |
1212 | time_travel_propagate_time(); |
1213 | |
1214 | do { |
1215 | rc = os_connect_socket(pdata->socket_path); |
1216 | } while (rc == -EINTR); |
1217 | if (rc < 0) |
1218 | goto error_free; |
1219 | vu_dev->sock = rc; |
1220 | |
1221 | spin_lock_init(&vu_dev->sock_lock); |
1222 | |
1223 | rc = vhost_user_init(vu_dev); |
1224 | if (rc) |
1225 | goto error_init; |
1226 | |
1227 | platform_set_drvdata(pdev, data: vu_dev); |
1228 | |
1229 | device_set_wakeup_capable(dev: &vu_dev->vdev.dev, capable: true); |
1230 | |
1231 | rc = register_virtio_device(dev: &vu_dev->vdev); |
1232 | if (rc) |
1233 | put_device(dev: &vu_dev->vdev.dev); |
1234 | vu_dev->registered = 1; |
1235 | return rc; |
1236 | |
1237 | error_init: |
1238 | os_close_file(vu_dev->sock); |
1239 | error_free: |
1240 | kfree(objp: vu_dev); |
1241 | return rc; |
1242 | } |
1243 | |
1244 | static int virtio_uml_remove(struct platform_device *pdev) |
1245 | { |
1246 | struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev); |
1247 | |
1248 | unregister_virtio_device(dev: &vu_dev->vdev); |
1249 | return 0; |
1250 | } |
1251 | |
1252 | /* Command line device list */ |
1253 | |
1254 | static void vu_cmdline_release_dev(struct device *d) |
1255 | { |
1256 | } |
1257 | |
1258 | static struct device vu_cmdline_parent = { |
1259 | .init_name = "virtio-uml-cmdline" , |
1260 | .release = vu_cmdline_release_dev, |
1261 | }; |
1262 | |
1263 | static bool vu_cmdline_parent_registered; |
1264 | static int vu_cmdline_id; |
1265 | |
1266 | static int vu_unregister_cmdline_device(struct device *dev, void *data) |
1267 | { |
1268 | struct platform_device *pdev = to_platform_device(dev); |
1269 | struct virtio_uml_platform_data *pdata = pdev->dev.platform_data; |
1270 | |
1271 | kfree(objp: pdata->socket_path); |
1272 | platform_device_unregister(pdev); |
1273 | return 0; |
1274 | } |
1275 | |
1276 | static void vu_conn_broken(struct work_struct *wk) |
1277 | { |
1278 | struct virtio_uml_platform_data *pdata; |
1279 | struct virtio_uml_device *vu_dev; |
1280 | |
1281 | pdata = container_of(wk, struct virtio_uml_platform_data, conn_broken_wk); |
1282 | |
1283 | vu_dev = platform_get_drvdata(pdev: pdata->pdev); |
1284 | |
1285 | virtio_break_device(dev: &vu_dev->vdev); |
1286 | |
1287 | vu_unregister_cmdline_device(dev: &pdata->pdev->dev, NULL); |
1288 | } |
1289 | |
1290 | static int vu_cmdline_set(const char *device, const struct kernel_param *kp) |
1291 | { |
1292 | const char *ids = strchr(device, ':'); |
1293 | unsigned int virtio_device_id; |
1294 | int processed, consumed, err; |
1295 | char *socket_path; |
1296 | struct virtio_uml_platform_data pdata, *ppdata; |
1297 | struct platform_device *pdev; |
1298 | |
1299 | if (!ids || ids == device) |
1300 | return -EINVAL; |
1301 | |
1302 | processed = sscanf(ids, ":%u%n:%d%n" , |
1303 | &virtio_device_id, &consumed, |
1304 | &vu_cmdline_id, &consumed); |
1305 | |
1306 | if (processed < 1 || ids[consumed]) |
1307 | return -EINVAL; |
1308 | |
1309 | if (!vu_cmdline_parent_registered) { |
1310 | err = device_register(dev: &vu_cmdline_parent); |
1311 | if (err) { |
1312 | pr_err("Failed to register parent device!\n" ); |
1313 | put_device(dev: &vu_cmdline_parent); |
1314 | return err; |
1315 | } |
1316 | vu_cmdline_parent_registered = true; |
1317 | } |
1318 | |
1319 | socket_path = kmemdup_nul(s: device, len: ids - device, GFP_KERNEL); |
1320 | if (!socket_path) |
1321 | return -ENOMEM; |
1322 | |
1323 | pdata.virtio_device_id = (u32) virtio_device_id; |
1324 | pdata.socket_path = socket_path; |
1325 | |
1326 | pr_info("Registering device virtio-uml.%d id=%d at %s\n" , |
1327 | vu_cmdline_id, virtio_device_id, socket_path); |
1328 | |
1329 | pdev = platform_device_register_data(parent: &vu_cmdline_parent, name: "virtio-uml" , |
1330 | id: vu_cmdline_id++, data: &pdata, |
1331 | size: sizeof(pdata)); |
1332 | err = PTR_ERR_OR_ZERO(ptr: pdev); |
1333 | if (err) |
1334 | goto free; |
1335 | |
1336 | ppdata = pdev->dev.platform_data; |
1337 | ppdata->pdev = pdev; |
1338 | INIT_WORK(&ppdata->conn_broken_wk, vu_conn_broken); |
1339 | |
1340 | return 0; |
1341 | |
1342 | free: |
1343 | kfree(objp: socket_path); |
1344 | return err; |
1345 | } |
1346 | |
1347 | static int vu_cmdline_get_device(struct device *dev, void *data) |
1348 | { |
1349 | struct platform_device *pdev = to_platform_device(dev); |
1350 | struct virtio_uml_platform_data *pdata = pdev->dev.platform_data; |
1351 | char *buffer = data; |
1352 | unsigned int len = strlen(buffer); |
1353 | |
1354 | snprintf(buf: buffer + len, PAGE_SIZE - len, fmt: "%s:%d:%d\n" , |
1355 | pdata->socket_path, pdata->virtio_device_id, pdev->id); |
1356 | return 0; |
1357 | } |
1358 | |
1359 | static int vu_cmdline_get(char *buffer, const struct kernel_param *kp) |
1360 | { |
1361 | buffer[0] = '\0'; |
1362 | if (vu_cmdline_parent_registered) |
1363 | device_for_each_child(dev: &vu_cmdline_parent, data: buffer, |
1364 | fn: vu_cmdline_get_device); |
1365 | return strlen(buffer) + 1; |
1366 | } |
1367 | |
1368 | static const struct kernel_param_ops vu_cmdline_param_ops = { |
1369 | .set = vu_cmdline_set, |
1370 | .get = vu_cmdline_get, |
1371 | }; |
1372 | |
1373 | device_param_cb(device, &vu_cmdline_param_ops, NULL, S_IRUSR); |
1374 | __uml_help(vu_cmdline_param_ops, |
1375 | "virtio_uml.device=<socket>:<virtio_id>[:<platform_id>]\n" |
1376 | " Configure a virtio device over a vhost-user socket.\n" |
1377 | " See virtio_ids.h for a list of possible virtio device id values.\n" |
1378 | " Optionally use a specific platform_device id.\n\n" |
1379 | ); |
1380 | |
1381 | |
1382 | static void vu_unregister_cmdline_devices(void) |
1383 | { |
1384 | if (vu_cmdline_parent_registered) { |
1385 | device_for_each_child(dev: &vu_cmdline_parent, NULL, |
1386 | fn: vu_unregister_cmdline_device); |
1387 | device_unregister(dev: &vu_cmdline_parent); |
1388 | vu_cmdline_parent_registered = false; |
1389 | } |
1390 | } |
1391 | |
1392 | /* Platform driver */ |
1393 | |
1394 | static const struct of_device_id virtio_uml_match[] = { |
1395 | { .compatible = "virtio,uml" , }, |
1396 | { } |
1397 | }; |
1398 | MODULE_DEVICE_TABLE(of, virtio_uml_match); |
1399 | |
1400 | static int virtio_uml_suspend(struct platform_device *pdev, pm_message_t state) |
1401 | { |
1402 | struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev); |
1403 | |
1404 | if (!vu_dev->no_vq_suspend) { |
1405 | struct virtqueue *vq; |
1406 | |
1407 | virtio_device_for_each_vq((&vu_dev->vdev), vq) { |
1408 | struct virtio_uml_vq_info *info = vq->priv; |
1409 | |
1410 | info->suspended = true; |
1411 | vhost_user_set_vring_enable(vu_dev, index: vq->index, enable: false); |
1412 | } |
1413 | } |
1414 | |
1415 | if (!device_may_wakeup(dev: &vu_dev->vdev.dev)) { |
1416 | vu_dev->suspended = true; |
1417 | return 0; |
1418 | } |
1419 | |
1420 | return irq_set_irq_wake(vu_dev->irq, 1); |
1421 | } |
1422 | |
1423 | static int virtio_uml_resume(struct platform_device *pdev) |
1424 | { |
1425 | struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev); |
1426 | |
1427 | if (!vu_dev->no_vq_suspend) { |
1428 | struct virtqueue *vq; |
1429 | |
1430 | virtio_device_for_each_vq((&vu_dev->vdev), vq) { |
1431 | struct virtio_uml_vq_info *info = vq->priv; |
1432 | |
1433 | info->suspended = false; |
1434 | vhost_user_set_vring_enable(vu_dev, index: vq->index, enable: true); |
1435 | } |
1436 | } |
1437 | |
1438 | vu_dev->suspended = false; |
1439 | |
1440 | if (!device_may_wakeup(dev: &vu_dev->vdev.dev)) |
1441 | return 0; |
1442 | |
1443 | return irq_set_irq_wake(vu_dev->irq, 0); |
1444 | } |
1445 | |
1446 | static struct platform_driver virtio_uml_driver = { |
1447 | .probe = virtio_uml_probe, |
1448 | .remove = virtio_uml_remove, |
1449 | .driver = { |
1450 | .name = "virtio-uml" , |
1451 | .of_match_table = virtio_uml_match, |
1452 | }, |
1453 | .suspend = virtio_uml_suspend, |
1454 | .resume = virtio_uml_resume, |
1455 | }; |
1456 | |
1457 | static int __init virtio_uml_init(void) |
1458 | { |
1459 | return platform_driver_register(&virtio_uml_driver); |
1460 | } |
1461 | |
1462 | static void __exit virtio_uml_exit(void) |
1463 | { |
1464 | platform_driver_unregister(&virtio_uml_driver); |
1465 | vu_unregister_cmdline_devices(); |
1466 | } |
1467 | |
1468 | module_init(virtio_uml_init); |
1469 | module_exit(virtio_uml_exit); |
1470 | __uml_exitcall(virtio_uml_exit); |
1471 | |
1472 | MODULE_DESCRIPTION("UML driver for vhost-user virtio devices" ); |
1473 | MODULE_LICENSE("GPL" ); |
1474 | |