1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (C) 2009 Red Hat, Inc. |
3 | * Author: Michael S. Tsirkin <mst@redhat.com> |
4 | * |
5 | * test virtio server in host kernel. |
6 | */ |
7 | |
8 | #include <linux/compat.h> |
9 | #include <linux/eventfd.h> |
10 | #include <linux/vhost.h> |
11 | #include <linux/miscdevice.h> |
12 | #include <linux/module.h> |
13 | #include <linux/mutex.h> |
14 | #include <linux/workqueue.h> |
15 | #include <linux/file.h> |
16 | #include <linux/slab.h> |
17 | |
18 | #include "test.h" |
19 | #include "vhost.h" |
20 | |
21 | /* Max number of bytes transferred before requeueing the job. |
22 | * Using this limit prevents one virtqueue from starving others. */ |
23 | #define VHOST_TEST_WEIGHT 0x80000 |
24 | |
25 | /* Max number of packets transferred before requeueing the job. |
26 | * Using this limit prevents one virtqueue from starving others with |
27 | * pkts. |
28 | */ |
29 | #define VHOST_TEST_PKT_WEIGHT 256 |
30 | |
31 | enum { |
32 | VHOST_TEST_VQ = 0, |
33 | VHOST_TEST_VQ_MAX = 1, |
34 | }; |
35 | |
36 | struct vhost_test { |
37 | struct vhost_dev dev; |
38 | struct vhost_virtqueue vqs[VHOST_TEST_VQ_MAX]; |
39 | }; |
40 | |
41 | /* Expects to be always run from workqueue - which acts as |
42 | * read-size critical section for our kind of RCU. */ |
43 | static void handle_vq(struct vhost_test *n) |
44 | { |
45 | struct vhost_virtqueue *vq = &n->vqs[VHOST_TEST_VQ]; |
46 | unsigned out, in; |
47 | int head; |
48 | size_t len, total_len = 0; |
49 | void *private; |
50 | |
51 | mutex_lock(&vq->mutex); |
52 | private = vhost_vq_get_backend(vq); |
53 | if (!private) { |
54 | mutex_unlock(lock: &vq->mutex); |
55 | return; |
56 | } |
57 | |
58 | vhost_disable_notify(&n->dev, vq); |
59 | |
60 | for (;;) { |
61 | head = vhost_get_vq_desc(vq, iov: vq->iov, |
62 | ARRAY_SIZE(vq->iov), |
63 | out_num: &out, in_num: &in, |
64 | NULL, NULL); |
65 | /* On error, stop handling until the next kick. */ |
66 | if (unlikely(head < 0)) |
67 | break; |
68 | /* Nothing new? Wait for eventfd to tell us they refilled. */ |
69 | if (head == vq->num) { |
70 | if (unlikely(vhost_enable_notify(&n->dev, vq))) { |
71 | vhost_disable_notify(&n->dev, vq); |
72 | continue; |
73 | } |
74 | break; |
75 | } |
76 | if (in) { |
77 | vq_err(vq, "Unexpected descriptor format for TX: " |
78 | "out %d, int %d\n" , out, in); |
79 | break; |
80 | } |
81 | len = iov_length(iov: vq->iov, nr_segs: out); |
82 | /* Sanity check */ |
83 | if (!len) { |
84 | vq_err(vq, "Unexpected 0 len for TX\n" ); |
85 | break; |
86 | } |
87 | vhost_add_used_and_signal(&n->dev, vq, id: head, len: 0); |
88 | total_len += len; |
89 | if (unlikely(vhost_exceeds_weight(vq, 0, total_len))) |
90 | break; |
91 | } |
92 | |
93 | mutex_unlock(lock: &vq->mutex); |
94 | } |
95 | |
96 | static void handle_vq_kick(struct vhost_work *work) |
97 | { |
98 | struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, |
99 | poll.work); |
100 | struct vhost_test *n = container_of(vq->dev, struct vhost_test, dev); |
101 | |
102 | handle_vq(n); |
103 | } |
104 | |
105 | static int vhost_test_open(struct inode *inode, struct file *f) |
106 | { |
107 | struct vhost_test *n = kmalloc(size: sizeof *n, GFP_KERNEL); |
108 | struct vhost_dev *dev; |
109 | struct vhost_virtqueue **vqs; |
110 | |
111 | if (!n) |
112 | return -ENOMEM; |
113 | vqs = kmalloc_array(n: VHOST_TEST_VQ_MAX, size: sizeof(*vqs), GFP_KERNEL); |
114 | if (!vqs) { |
115 | kfree(objp: n); |
116 | return -ENOMEM; |
117 | } |
118 | |
119 | dev = &n->dev; |
120 | vqs[VHOST_TEST_VQ] = &n->vqs[VHOST_TEST_VQ]; |
121 | n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick; |
122 | vhost_dev_init(dev, vqs, nvqs: VHOST_TEST_VQ_MAX, UIO_MAXIOV, |
123 | VHOST_TEST_PKT_WEIGHT, VHOST_TEST_WEIGHT, use_worker: true, NULL); |
124 | |
125 | f->private_data = n; |
126 | |
127 | return 0; |
128 | } |
129 | |
130 | static void *vhost_test_stop_vq(struct vhost_test *n, |
131 | struct vhost_virtqueue *vq) |
132 | { |
133 | void *private; |
134 | |
135 | mutex_lock(&vq->mutex); |
136 | private = vhost_vq_get_backend(vq); |
137 | vhost_vq_set_backend(vq, NULL); |
138 | mutex_unlock(lock: &vq->mutex); |
139 | return private; |
140 | } |
141 | |
142 | static void vhost_test_stop(struct vhost_test *n, void **privatep) |
143 | { |
144 | *privatep = vhost_test_stop_vq(n, vq: n->vqs + VHOST_TEST_VQ); |
145 | } |
146 | |
147 | static void vhost_test_flush(struct vhost_test *n) |
148 | { |
149 | vhost_dev_flush(dev: &n->dev); |
150 | } |
151 | |
152 | static int vhost_test_release(struct inode *inode, struct file *f) |
153 | { |
154 | struct vhost_test *n = f->private_data; |
155 | void *private; |
156 | |
157 | vhost_test_stop(n, privatep: &private); |
158 | vhost_test_flush(n); |
159 | vhost_dev_stop(&n->dev); |
160 | vhost_dev_cleanup(&n->dev); |
161 | kfree(objp: n->dev.vqs); |
162 | kfree(objp: n); |
163 | return 0; |
164 | } |
165 | |
166 | static long vhost_test_run(struct vhost_test *n, int test) |
167 | { |
168 | void *priv, *oldpriv; |
169 | struct vhost_virtqueue *vq; |
170 | int r, index; |
171 | |
172 | if (test < 0 || test > 1) |
173 | return -EINVAL; |
174 | |
175 | mutex_lock(&n->dev.mutex); |
176 | r = vhost_dev_check_owner(&n->dev); |
177 | if (r) |
178 | goto err; |
179 | |
180 | for (index = 0; index < n->dev.nvqs; ++index) { |
181 | /* Verify that ring has been setup correctly. */ |
182 | if (!vhost_vq_access_ok(vq: &n->vqs[index])) { |
183 | r = -EFAULT; |
184 | goto err; |
185 | } |
186 | } |
187 | |
188 | for (index = 0; index < n->dev.nvqs; ++index) { |
189 | vq = n->vqs + index; |
190 | mutex_lock(&vq->mutex); |
191 | priv = test ? n : NULL; |
192 | |
193 | /* start polling new socket */ |
194 | oldpriv = vhost_vq_get_backend(vq); |
195 | vhost_vq_set_backend(vq, private_data: priv); |
196 | |
197 | r = vhost_vq_init_access(&n->vqs[index]); |
198 | |
199 | mutex_unlock(lock: &vq->mutex); |
200 | |
201 | if (r) |
202 | goto err; |
203 | |
204 | if (oldpriv) { |
205 | vhost_test_flush(n); |
206 | } |
207 | } |
208 | |
209 | mutex_unlock(lock: &n->dev.mutex); |
210 | return 0; |
211 | |
212 | err: |
213 | mutex_unlock(lock: &n->dev.mutex); |
214 | return r; |
215 | } |
216 | |
217 | static long vhost_test_reset_owner(struct vhost_test *n) |
218 | { |
219 | void *priv = NULL; |
220 | long err; |
221 | struct vhost_iotlb *umem; |
222 | |
223 | mutex_lock(&n->dev.mutex); |
224 | err = vhost_dev_check_owner(&n->dev); |
225 | if (err) |
226 | goto done; |
227 | umem = vhost_dev_reset_owner_prepare(); |
228 | if (!umem) { |
229 | err = -ENOMEM; |
230 | goto done; |
231 | } |
232 | vhost_test_stop(n, privatep: &priv); |
233 | vhost_test_flush(n); |
234 | vhost_dev_stop(&n->dev); |
235 | vhost_dev_reset_owner(dev: &n->dev, iotlb: umem); |
236 | done: |
237 | mutex_unlock(lock: &n->dev.mutex); |
238 | return err; |
239 | } |
240 | |
241 | static int vhost_test_set_features(struct vhost_test *n, u64 features) |
242 | { |
243 | struct vhost_virtqueue *vq; |
244 | |
245 | mutex_lock(&n->dev.mutex); |
246 | if ((features & (1 << VHOST_F_LOG_ALL)) && |
247 | !vhost_log_access_ok(&n->dev)) { |
248 | mutex_unlock(lock: &n->dev.mutex); |
249 | return -EFAULT; |
250 | } |
251 | vq = &n->vqs[VHOST_TEST_VQ]; |
252 | mutex_lock(&vq->mutex); |
253 | vq->acked_features = features; |
254 | mutex_unlock(lock: &vq->mutex); |
255 | mutex_unlock(lock: &n->dev.mutex); |
256 | return 0; |
257 | } |
258 | |
259 | static long vhost_test_set_backend(struct vhost_test *n, unsigned index, int fd) |
260 | { |
261 | static void *backend; |
262 | |
263 | const bool enable = fd != -1; |
264 | struct vhost_virtqueue *vq; |
265 | int r; |
266 | |
267 | mutex_lock(&n->dev.mutex); |
268 | r = vhost_dev_check_owner(&n->dev); |
269 | if (r) |
270 | goto err; |
271 | |
272 | if (index >= VHOST_TEST_VQ_MAX) { |
273 | r = -ENOBUFS; |
274 | goto err; |
275 | } |
276 | vq = &n->vqs[index]; |
277 | mutex_lock(&vq->mutex); |
278 | |
279 | /* Verify that ring has been setup correctly. */ |
280 | if (!vhost_vq_access_ok(vq)) { |
281 | r = -EFAULT; |
282 | goto err_vq; |
283 | } |
284 | if (!enable) { |
285 | vhost_poll_stop(poll: &vq->poll); |
286 | backend = vhost_vq_get_backend(vq); |
287 | vhost_vq_set_backend(vq, NULL); |
288 | } else { |
289 | vhost_vq_set_backend(vq, private_data: backend); |
290 | r = vhost_vq_init_access(vq); |
291 | if (r == 0) |
292 | r = vhost_poll_start(poll: &vq->poll, file: vq->kick); |
293 | } |
294 | |
295 | mutex_unlock(lock: &vq->mutex); |
296 | |
297 | if (enable) { |
298 | vhost_test_flush(n); |
299 | } |
300 | |
301 | mutex_unlock(lock: &n->dev.mutex); |
302 | return 0; |
303 | |
304 | err_vq: |
305 | mutex_unlock(lock: &vq->mutex); |
306 | err: |
307 | mutex_unlock(lock: &n->dev.mutex); |
308 | return r; |
309 | } |
310 | |
311 | static long vhost_test_ioctl(struct file *f, unsigned int ioctl, |
312 | unsigned long arg) |
313 | { |
314 | struct vhost_vring_file backend; |
315 | struct vhost_test *n = f->private_data; |
316 | void __user *argp = (void __user *)arg; |
317 | u64 __user *featurep = argp; |
318 | int test; |
319 | u64 features; |
320 | int r; |
321 | switch (ioctl) { |
322 | case VHOST_TEST_RUN: |
323 | if (copy_from_user(to: &test, from: argp, n: sizeof test)) |
324 | return -EFAULT; |
325 | return vhost_test_run(n, test); |
326 | case VHOST_TEST_SET_BACKEND: |
327 | if (copy_from_user(to: &backend, from: argp, n: sizeof backend)) |
328 | return -EFAULT; |
329 | return vhost_test_set_backend(n, index: backend.index, fd: backend.fd); |
330 | case VHOST_GET_FEATURES: |
331 | features = VHOST_FEATURES; |
332 | if (copy_to_user(to: featurep, from: &features, n: sizeof features)) |
333 | return -EFAULT; |
334 | return 0; |
335 | case VHOST_SET_FEATURES: |
336 | if (copy_from_user(to: &features, from: featurep, n: sizeof features)) |
337 | return -EFAULT; |
338 | if (features & ~VHOST_FEATURES) |
339 | return -EOPNOTSUPP; |
340 | return vhost_test_set_features(n, features); |
341 | case VHOST_RESET_OWNER: |
342 | return vhost_test_reset_owner(n); |
343 | default: |
344 | mutex_lock(&n->dev.mutex); |
345 | r = vhost_dev_ioctl(&n->dev, ioctl, argp); |
346 | if (r == -ENOIOCTLCMD) |
347 | r = vhost_vring_ioctl(d: &n->dev, ioctl, argp); |
348 | vhost_test_flush(n); |
349 | mutex_unlock(lock: &n->dev.mutex); |
350 | return r; |
351 | } |
352 | } |
353 | |
354 | static const struct file_operations vhost_test_fops = { |
355 | .owner = THIS_MODULE, |
356 | .release = vhost_test_release, |
357 | .unlocked_ioctl = vhost_test_ioctl, |
358 | .compat_ioctl = compat_ptr_ioctl, |
359 | .open = vhost_test_open, |
360 | .llseek = noop_llseek, |
361 | }; |
362 | |
363 | static struct miscdevice vhost_test_misc = { |
364 | MISC_DYNAMIC_MINOR, |
365 | "vhost-test" , |
366 | &vhost_test_fops, |
367 | }; |
368 | module_misc_device(vhost_test_misc); |
369 | |
370 | MODULE_VERSION("0.0.1" ); |
371 | MODULE_LICENSE("GPL v2" ); |
372 | MODULE_AUTHOR("Michael S. Tsirkin" ); |
373 | MODULE_DESCRIPTION("Host kernel side for virtio simulator" ); |
374 | |