1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VDPA device simulator core.
4 *
5 * Copyright (c) 2020, Red Hat Inc. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/device.h>
13#include <linux/kernel.h>
14#include <linux/kthread.h>
15#include <linux/slab.h>
16#include <linux/dma-map-ops.h>
17#include <linux/vringh.h>
18#include <linux/vdpa.h>
19#include <linux/vhost_iotlb.h>
20#include <uapi/linux/vdpa.h>
21#include <uapi/linux/vhost_types.h>
22
23#include "vdpa_sim.h"
24
25#define DRV_VERSION "0.1"
26#define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>"
27#define DRV_DESC "vDPA Device Simulator core"
28#define DRV_LICENSE "GPL v2"
29
30static int batch_mapping = 1;
31module_param(batch_mapping, int, 0444);
32MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable");
33
34static int max_iotlb_entries = 2048;
35module_param(max_iotlb_entries, int, 0444);
36MODULE_PARM_DESC(max_iotlb_entries,
37 "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)");
38
39static bool use_va = true;
40module_param(use_va, bool, 0444);
41MODULE_PARM_DESC(use_va, "Enable/disable the device's ability to use VA");
42
43#define VDPASIM_QUEUE_ALIGN PAGE_SIZE
44#define VDPASIM_QUEUE_MAX 256
45#define VDPASIM_VENDOR_ID 0
46
47struct vdpasim_mm_work {
48 struct kthread_work work;
49 struct vdpasim *vdpasim;
50 struct mm_struct *mm_to_bind;
51 int ret;
52};
53
54static void vdpasim_mm_work_fn(struct kthread_work *work)
55{
56 struct vdpasim_mm_work *mm_work =
57 container_of(work, struct vdpasim_mm_work, work);
58 struct vdpasim *vdpasim = mm_work->vdpasim;
59
60 mm_work->ret = 0;
61
62 //TODO: should we attach the cgroup of the mm owner?
63 vdpasim->mm_bound = mm_work->mm_to_bind;
64}
65
66static void vdpasim_worker_change_mm_sync(struct vdpasim *vdpasim,
67 struct vdpasim_mm_work *mm_work)
68{
69 struct kthread_work *work = &mm_work->work;
70
71 kthread_init_work(work, vdpasim_mm_work_fn);
72 kthread_queue_work(worker: vdpasim->worker, work);
73
74 kthread_flush_work(work);
75}
76
77static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)
78{
79 return container_of(vdpa, struct vdpasim, vdpa);
80}
81
82static void vdpasim_vq_notify(struct vringh *vring)
83{
84 struct vdpasim_virtqueue *vq =
85 container_of(vring, struct vdpasim_virtqueue, vring);
86
87 if (!vq->cb)
88 return;
89
90 vq->cb(vq->private);
91}
92
93static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
94{
95 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
96 uint16_t last_avail_idx = vq->vring.last_avail_idx;
97 struct vring_desc *desc = (struct vring_desc *)
98 (uintptr_t)vq->desc_addr;
99 struct vring_avail *avail = (struct vring_avail *)
100 (uintptr_t)vq->driver_addr;
101 struct vring_used *used = (struct vring_used *)
102 (uintptr_t)vq->device_addr;
103
104 if (use_va && vdpasim->mm_bound) {
105 vringh_init_iotlb_va(vrh: &vq->vring, features: vdpasim->features, num: vq->num,
106 weak_barriers: true, desc, avail, used);
107 } else {
108 vringh_init_iotlb(vrh: &vq->vring, features: vdpasim->features, num: vq->num,
109 weak_barriers: true, desc, avail, used);
110 }
111
112 vq->vring.last_avail_idx = last_avail_idx;
113
114 /*
115 * Since vdpa_sim does not support receive inflight descriptors as a
116 * destination of a migration, let's set both avail_idx and used_idx
117 * the same at vq start. This is how vhost-user works in a
118 * VHOST_SET_VRING_BASE call.
119 *
120 * Although the simple fix is to set last_used_idx at
121 * vdpasim_set_vq_state, it would be reset at vdpasim_queue_ready.
122 */
123 vq->vring.last_used_idx = last_avail_idx;
124 vq->vring.notify = vdpasim_vq_notify;
125}
126
127static void vdpasim_vq_reset(struct vdpasim *vdpasim,
128 struct vdpasim_virtqueue *vq)
129{
130 vq->ready = false;
131 vq->desc_addr = 0;
132 vq->driver_addr = 0;
133 vq->device_addr = 0;
134 vq->cb = NULL;
135 vq->private = NULL;
136 vringh_init_iotlb(vrh: &vq->vring, features: vdpasim->dev_attr.supported_features,
137 VDPASIM_QUEUE_MAX, weak_barriers: false, NULL, NULL, NULL);
138
139 vq->vring.notify = NULL;
140}
141
142static void vdpasim_do_reset(struct vdpasim *vdpasim, u32 flags)
143{
144 int i;
145
146 spin_lock(lock: &vdpasim->iommu_lock);
147
148 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
149 vdpasim_vq_reset(vdpasim, vq: &vdpasim->vqs[i]);
150 vringh_set_iotlb(vrh: &vdpasim->vqs[i].vring, iotlb: &vdpasim->iommu[0],
151 iotlb_lock: &vdpasim->iommu_lock);
152 }
153
154 if (flags & VDPA_RESET_F_CLEAN_MAP) {
155 for (i = 0; i < vdpasim->dev_attr.nas; i++) {
156 vhost_iotlb_reset(iotlb: &vdpasim->iommu[i]);
157 vhost_iotlb_add_range(iotlb: &vdpasim->iommu[i], start: 0, ULONG_MAX,
158 addr: 0, VHOST_MAP_RW);
159 vdpasim->iommu_pt[i] = true;
160 }
161 }
162
163 vdpasim->running = false;
164 spin_unlock(lock: &vdpasim->iommu_lock);
165
166 vdpasim->features = 0;
167 vdpasim->status = 0;
168 ++vdpasim->generation;
169}
170
171static const struct vdpa_config_ops vdpasim_config_ops;
172static const struct vdpa_config_ops vdpasim_batch_config_ops;
173
174static void vdpasim_work_fn(struct kthread_work *work)
175{
176 struct vdpasim *vdpasim = container_of(work, struct vdpasim, work);
177 struct mm_struct *mm = vdpasim->mm_bound;
178
179 if (use_va && mm) {
180 if (!mmget_not_zero(mm))
181 return;
182 kthread_use_mm(mm);
183 }
184
185 vdpasim->dev_attr.work_fn(vdpasim);
186
187 if (use_va && mm) {
188 kthread_unuse_mm(mm);
189 mmput(mm);
190 }
191}
192
193struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
194 const struct vdpa_dev_set_config *config)
195{
196 const struct vdpa_config_ops *ops;
197 struct vdpa_device *vdpa;
198 struct vdpasim *vdpasim;
199 struct device *dev;
200 int i, ret = -ENOMEM;
201
202 if (!dev_attr->alloc_size)
203 return ERR_PTR(error: -EINVAL);
204
205 if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
206 if (config->device_features &
207 ~dev_attr->supported_features)
208 return ERR_PTR(error: -EINVAL);
209 dev_attr->supported_features =
210 config->device_features;
211 }
212
213 if (batch_mapping)
214 ops = &vdpasim_batch_config_ops;
215 else
216 ops = &vdpasim_config_ops;
217
218 vdpa = __vdpa_alloc_device(NULL, config: ops, NULL,
219 ngroups: dev_attr->ngroups, nas: dev_attr->nas,
220 size: dev_attr->alloc_size,
221 name: dev_attr->name, use_va);
222 if (IS_ERR(ptr: vdpa)) {
223 ret = PTR_ERR(ptr: vdpa);
224 goto err_alloc;
225 }
226
227 vdpasim = vdpa_to_sim(vdpa);
228 vdpasim->dev_attr = *dev_attr;
229 dev = &vdpasim->vdpa.dev;
230
231 kthread_init_work(&vdpasim->work, vdpasim_work_fn);
232 vdpasim->worker = kthread_run_worker(0, "vDPA sim worker: %s",
233 dev_attr->name);
234 if (IS_ERR(ptr: vdpasim->worker))
235 goto err_iommu;
236
237 mutex_init(&vdpasim->mutex);
238 spin_lock_init(&vdpasim->iommu_lock);
239
240 dev->dma_mask = &dev->coherent_dma_mask;
241 if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
242 goto err_iommu;
243 vdpasim->vdpa.mdev = dev_attr->mgmt_dev;
244
245 vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL);
246 if (!vdpasim->config)
247 goto err_iommu;
248
249 vdpasim->vqs = kcalloc(dev_attr->nvqs, sizeof(struct vdpasim_virtqueue),
250 GFP_KERNEL);
251 if (!vdpasim->vqs)
252 goto err_iommu;
253
254 vdpasim->iommu = kmalloc_array(vdpasim->dev_attr.nas,
255 sizeof(*vdpasim->iommu), GFP_KERNEL);
256 if (!vdpasim->iommu)
257 goto err_iommu;
258
259 vdpasim->iommu_pt = kmalloc_array(vdpasim->dev_attr.nas,
260 sizeof(*vdpasim->iommu_pt), GFP_KERNEL);
261 if (!vdpasim->iommu_pt)
262 goto err_iommu;
263
264 for (i = 0; i < vdpasim->dev_attr.nas; i++) {
265 vhost_iotlb_init(iotlb: &vdpasim->iommu[i], limit: max_iotlb_entries, flags: 0);
266 vhost_iotlb_add_range(iotlb: &vdpasim->iommu[i], start: 0, ULONG_MAX, addr: 0,
267 VHOST_MAP_RW);
268 vdpasim->iommu_pt[i] = true;
269 }
270
271 for (i = 0; i < dev_attr->nvqs; i++)
272 vringh_set_iotlb(vrh: &vdpasim->vqs[i].vring, iotlb: &vdpasim->iommu[0],
273 iotlb_lock: &vdpasim->iommu_lock);
274
275 vdpasim->vdpa.vmap.dma_dev = dev;
276
277 return vdpasim;
278
279err_iommu:
280 put_device(dev);
281err_alloc:
282 return ERR_PTR(error: ret);
283}
284EXPORT_SYMBOL_GPL(vdpasim_create);
285
286void vdpasim_schedule_work(struct vdpasim *vdpasim)
287{
288 kthread_queue_work(worker: vdpasim->worker, work: &vdpasim->work);
289}
290EXPORT_SYMBOL_GPL(vdpasim_schedule_work);
291
292static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx,
293 u64 desc_area, u64 driver_area,
294 u64 device_area)
295{
296 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
297 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
298
299 vq->desc_addr = desc_area;
300 vq->driver_addr = driver_area;
301 vq->device_addr = device_area;
302
303 return 0;
304}
305
306static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
307{
308 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
309 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
310
311 vq->num = num;
312}
313
314static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx)
315{
316 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
317 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
318
319 if (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)
320 return vq->num;
321 else
322 return VDPASIM_QUEUE_MAX;
323}
324
325static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
326{
327 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
328 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
329
330 if (!vdpasim->running &&
331 (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
332 vdpasim->pending_kick = true;
333 return;
334 }
335
336 if (vq->ready)
337 vdpasim_schedule_work(vdpasim);
338}
339
340static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
341 struct vdpa_callback *cb)
342{
343 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
344 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
345
346 vq->cb = cb->callback;
347 vq->private = cb->private;
348}
349
350static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready)
351{
352 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
353 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
354 bool old_ready;
355
356 mutex_lock(&vdpasim->mutex);
357 old_ready = vq->ready;
358 vq->ready = ready;
359 if (vq->ready && !old_ready) {
360 vdpasim_queue_ready(vdpasim, idx);
361 }
362 mutex_unlock(lock: &vdpasim->mutex);
363}
364
365static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
366{
367 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
368 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
369
370 return vq->ready;
371}
372
373static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx,
374 const struct vdpa_vq_state *state)
375{
376 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
377 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
378 struct vringh *vrh = &vq->vring;
379
380 mutex_lock(&vdpasim->mutex);
381 vrh->last_avail_idx = state->split.avail_index;
382 mutex_unlock(lock: &vdpasim->mutex);
383
384 return 0;
385}
386
387static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx,
388 struct vdpa_vq_state *state)
389{
390 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
391 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
392 struct vringh *vrh = &vq->vring;
393
394 state->split.avail_index = vrh->last_avail_idx;
395 return 0;
396}
397
398static int vdpasim_get_vq_stats(struct vdpa_device *vdpa, u16 idx,
399 struct sk_buff *msg,
400 struct netlink_ext_ack *extack)
401{
402 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
403
404 if (vdpasim->dev_attr.get_stats)
405 return vdpasim->dev_attr.get_stats(vdpasim, idx,
406 msg, extack);
407 return -EOPNOTSUPP;
408}
409
410static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa)
411{
412 return VDPASIM_QUEUE_ALIGN;
413}
414
415static u32 vdpasim_get_vq_group(struct vdpa_device *vdpa, u16 idx)
416{
417 /* RX and TX belongs to group 0, CVQ belongs to group 1 */
418 if (idx == 2)
419 return 1;
420 else
421 return 0;
422}
423
424static u64 vdpasim_get_device_features(struct vdpa_device *vdpa)
425{
426 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
427
428 return vdpasim->dev_attr.supported_features;
429}
430
431static u64 vdpasim_get_backend_features(const struct vdpa_device *vdpa)
432{
433 return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
434}
435
436static int vdpasim_set_driver_features(struct vdpa_device *vdpa, u64 features)
437{
438 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
439
440 /* DMA mapping must be done by driver */
441 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
442 return -EINVAL;
443
444 vdpasim->features = features & vdpasim->dev_attr.supported_features;
445
446 return 0;
447}
448
449static u64 vdpasim_get_driver_features(struct vdpa_device *vdpa)
450{
451 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
452
453 return vdpasim->features;
454}
455
456static void vdpasim_set_config_cb(struct vdpa_device *vdpa,
457 struct vdpa_callback *cb)
458{
459 /* We don't support config interrupt */
460}
461
462static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa)
463{
464 return VDPASIM_QUEUE_MAX;
465}
466
467static u32 vdpasim_get_device_id(struct vdpa_device *vdpa)
468{
469 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
470
471 return vdpasim->dev_attr.id;
472}
473
474static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa)
475{
476 return VDPASIM_VENDOR_ID;
477}
478
479static u8 vdpasim_get_status(struct vdpa_device *vdpa)
480{
481 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
482 u8 status;
483
484 mutex_lock(&vdpasim->mutex);
485 status = vdpasim->status;
486 mutex_unlock(lock: &vdpasim->mutex);
487
488 return status;
489}
490
491static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status)
492{
493 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
494
495 mutex_lock(&vdpasim->mutex);
496 vdpasim->status = status;
497 vdpasim->running = (status & VIRTIO_CONFIG_S_DRIVER_OK) != 0;
498 mutex_unlock(lock: &vdpasim->mutex);
499}
500
501static int vdpasim_compat_reset(struct vdpa_device *vdpa, u32 flags)
502{
503 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
504
505 mutex_lock(&vdpasim->mutex);
506 vdpasim->status = 0;
507 vdpasim_do_reset(vdpasim, flags);
508 mutex_unlock(lock: &vdpasim->mutex);
509
510 return 0;
511}
512
513static int vdpasim_reset(struct vdpa_device *vdpa)
514{
515 return vdpasim_compat_reset(vdpa, flags: 0);
516}
517
518static int vdpasim_suspend(struct vdpa_device *vdpa)
519{
520 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
521
522 mutex_lock(&vdpasim->mutex);
523 vdpasim->running = false;
524 mutex_unlock(lock: &vdpasim->mutex);
525
526 return 0;
527}
528
529static int vdpasim_resume(struct vdpa_device *vdpa)
530{
531 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
532 int i;
533
534 mutex_lock(&vdpasim->mutex);
535 vdpasim->running = true;
536
537 if (vdpasim->pending_kick) {
538 /* Process pending descriptors */
539 for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
540 vdpasim_kick_vq(vdpa, idx: i);
541
542 vdpasim->pending_kick = false;
543 }
544
545 mutex_unlock(lock: &vdpasim->mutex);
546
547 return 0;
548}
549
550static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)
551{
552 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
553
554 return vdpasim->dev_attr.config_size;
555}
556
557static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset,
558 void *buf, unsigned int len)
559{
560 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
561
562 if (offset + len > vdpasim->dev_attr.config_size)
563 return;
564
565 if (vdpasim->dev_attr.get_config)
566 vdpasim->dev_attr.get_config(vdpasim, vdpasim->config);
567
568 memcpy(buf, vdpasim->config + offset, len);
569}
570
571static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset,
572 const void *buf, unsigned int len)
573{
574 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
575
576 if (offset + len > vdpasim->dev_attr.config_size)
577 return;
578
579 memcpy(vdpasim->config + offset, buf, len);
580
581 if (vdpasim->dev_attr.set_config)
582 vdpasim->dev_attr.set_config(vdpasim, vdpasim->config);
583}
584
585static u32 vdpasim_get_generation(struct vdpa_device *vdpa)
586{
587 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
588
589 return vdpasim->generation;
590}
591
592static struct vdpa_iova_range vdpasim_get_iova_range(struct vdpa_device *vdpa)
593{
594 struct vdpa_iova_range range = {
595 .first = 0ULL,
596 .last = ULLONG_MAX,
597 };
598
599 return range;
600}
601
602static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group,
603 unsigned int asid)
604{
605 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
606 struct vhost_iotlb *iommu;
607 int i;
608
609 if (group > vdpasim->dev_attr.ngroups)
610 return -EINVAL;
611
612 if (asid >= vdpasim->dev_attr.nas)
613 return -EINVAL;
614
615 iommu = &vdpasim->iommu[asid];
616
617 mutex_lock(&vdpasim->mutex);
618
619 for (i = 0; i < vdpasim->dev_attr.nvqs; i++)
620 if (vdpasim_get_vq_group(vdpa, idx: i) == group)
621 vringh_set_iotlb(vrh: &vdpasim->vqs[i].vring, iotlb: iommu,
622 iotlb_lock: &vdpasim->iommu_lock);
623
624 mutex_unlock(lock: &vdpasim->mutex);
625
626 return 0;
627}
628
629static int vdpasim_set_map(struct vdpa_device *vdpa, unsigned int asid,
630 struct vhost_iotlb *iotlb)
631{
632 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
633 struct vhost_iotlb_map *map;
634 struct vhost_iotlb *iommu;
635 u64 start = 0ULL, last = 0ULL - 1;
636 int ret;
637
638 if (asid >= vdpasim->dev_attr.nas)
639 return -EINVAL;
640
641 spin_lock(lock: &vdpasim->iommu_lock);
642
643 iommu = &vdpasim->iommu[asid];
644 vhost_iotlb_reset(iotlb: iommu);
645 vdpasim->iommu_pt[asid] = false;
646
647 for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
648 map = vhost_iotlb_itree_next(map, start, last)) {
649 ret = vhost_iotlb_add_range(iotlb: iommu, start: map->start,
650 last: map->last, addr: map->addr, perm: map->perm);
651 if (ret)
652 goto err;
653 }
654 spin_unlock(lock: &vdpasim->iommu_lock);
655 return 0;
656
657err:
658 vhost_iotlb_reset(iotlb: iommu);
659 spin_unlock(lock: &vdpasim->iommu_lock);
660 return ret;
661}
662
663static int vdpasim_reset_map(struct vdpa_device *vdpa, unsigned int asid)
664{
665 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
666
667 if (asid >= vdpasim->dev_attr.nas)
668 return -EINVAL;
669
670 spin_lock(lock: &vdpasim->iommu_lock);
671 if (vdpasim->iommu_pt[asid])
672 goto out;
673 vhost_iotlb_reset(iotlb: &vdpasim->iommu[asid]);
674 vhost_iotlb_add_range(iotlb: &vdpasim->iommu[asid], start: 0, ULONG_MAX,
675 addr: 0, VHOST_MAP_RW);
676 vdpasim->iommu_pt[asid] = true;
677out:
678 spin_unlock(lock: &vdpasim->iommu_lock);
679 return 0;
680}
681
682static int vdpasim_bind_mm(struct vdpa_device *vdpa, struct mm_struct *mm)
683{
684 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
685 struct vdpasim_mm_work mm_work;
686
687 mm_work.vdpasim = vdpasim;
688 mm_work.mm_to_bind = mm;
689
690 vdpasim_worker_change_mm_sync(vdpasim, mm_work: &mm_work);
691
692 return mm_work.ret;
693}
694
695static void vdpasim_unbind_mm(struct vdpa_device *vdpa)
696{
697 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
698 struct vdpasim_mm_work mm_work;
699
700 mm_work.vdpasim = vdpasim;
701 mm_work.mm_to_bind = NULL;
702
703 vdpasim_worker_change_mm_sync(vdpasim, mm_work: &mm_work);
704}
705
706static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid,
707 u64 iova, u64 size,
708 u64 pa, u32 perm, void *opaque)
709{
710 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
711 int ret;
712
713 if (asid >= vdpasim->dev_attr.nas)
714 return -EINVAL;
715
716 spin_lock(lock: &vdpasim->iommu_lock);
717 if (vdpasim->iommu_pt[asid]) {
718 vhost_iotlb_reset(iotlb: &vdpasim->iommu[asid]);
719 vdpasim->iommu_pt[asid] = false;
720 }
721 ret = vhost_iotlb_add_range_ctx(iotlb: &vdpasim->iommu[asid], start: iova,
722 last: iova + size - 1, addr: pa, perm, opaque);
723 spin_unlock(lock: &vdpasim->iommu_lock);
724
725 return ret;
726}
727
728static int vdpasim_dma_unmap(struct vdpa_device *vdpa, unsigned int asid,
729 u64 iova, u64 size)
730{
731 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
732
733 if (asid >= vdpasim->dev_attr.nas)
734 return -EINVAL;
735
736 if (vdpasim->iommu_pt[asid]) {
737 vhost_iotlb_reset(iotlb: &vdpasim->iommu[asid]);
738 vdpasim->iommu_pt[asid] = false;
739 }
740
741 spin_lock(lock: &vdpasim->iommu_lock);
742 vhost_iotlb_del_range(iotlb: &vdpasim->iommu[asid], start: iova, last: iova + size - 1);
743 spin_unlock(lock: &vdpasim->iommu_lock);
744
745 return 0;
746}
747
748static void vdpasim_free(struct vdpa_device *vdpa)
749{
750 struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
751 int i;
752
753 kthread_cancel_work_sync(work: &vdpasim->work);
754 kthread_destroy_worker(worker: vdpasim->worker);
755
756 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {
757 vringh_kiov_cleanup(kiov: &vdpasim->vqs[i].out_iov);
758 vringh_kiov_cleanup(kiov: &vdpasim->vqs[i].in_iov);
759 }
760
761 vdpasim->dev_attr.free(vdpasim);
762
763 for (i = 0; i < vdpasim->dev_attr.nas; i++)
764 vhost_iotlb_reset(iotlb: &vdpasim->iommu[i]);
765 kfree(objp: vdpasim->iommu);
766 kfree(objp: vdpasim->iommu_pt);
767 kfree(objp: vdpasim->vqs);
768 kfree(objp: vdpasim->config);
769}
770
771static const struct vdpa_config_ops vdpasim_config_ops = {
772 .set_vq_address = vdpasim_set_vq_address,
773 .set_vq_num = vdpasim_set_vq_num,
774 .kick_vq = vdpasim_kick_vq,
775 .set_vq_cb = vdpasim_set_vq_cb,
776 .set_vq_ready = vdpasim_set_vq_ready,
777 .get_vq_ready = vdpasim_get_vq_ready,
778 .set_vq_state = vdpasim_set_vq_state,
779 .get_vendor_vq_stats = vdpasim_get_vq_stats,
780 .get_vq_state = vdpasim_get_vq_state,
781 .get_vq_align = vdpasim_get_vq_align,
782 .get_vq_group = vdpasim_get_vq_group,
783 .get_device_features = vdpasim_get_device_features,
784 .get_backend_features = vdpasim_get_backend_features,
785 .set_driver_features = vdpasim_set_driver_features,
786 .get_driver_features = vdpasim_get_driver_features,
787 .set_config_cb = vdpasim_set_config_cb,
788 .get_vq_num_max = vdpasim_get_vq_num_max,
789 .get_vq_size = vdpasim_get_vq_size,
790 .get_device_id = vdpasim_get_device_id,
791 .get_vendor_id = vdpasim_get_vendor_id,
792 .get_status = vdpasim_get_status,
793 .set_status = vdpasim_set_status,
794 .reset = vdpasim_reset,
795 .compat_reset = vdpasim_compat_reset,
796 .suspend = vdpasim_suspend,
797 .resume = vdpasim_resume,
798 .get_config_size = vdpasim_get_config_size,
799 .get_config = vdpasim_get_config,
800 .set_config = vdpasim_set_config,
801 .get_generation = vdpasim_get_generation,
802 .get_iova_range = vdpasim_get_iova_range,
803 .set_group_asid = vdpasim_set_group_asid,
804 .dma_map = vdpasim_dma_map,
805 .dma_unmap = vdpasim_dma_unmap,
806 .reset_map = vdpasim_reset_map,
807 .bind_mm = vdpasim_bind_mm,
808 .unbind_mm = vdpasim_unbind_mm,
809 .free = vdpasim_free,
810};
811
812static const struct vdpa_config_ops vdpasim_batch_config_ops = {
813 .set_vq_address = vdpasim_set_vq_address,
814 .set_vq_num = vdpasim_set_vq_num,
815 .kick_vq = vdpasim_kick_vq,
816 .set_vq_cb = vdpasim_set_vq_cb,
817 .set_vq_ready = vdpasim_set_vq_ready,
818 .get_vq_ready = vdpasim_get_vq_ready,
819 .set_vq_state = vdpasim_set_vq_state,
820 .get_vendor_vq_stats = vdpasim_get_vq_stats,
821 .get_vq_state = vdpasim_get_vq_state,
822 .get_vq_align = vdpasim_get_vq_align,
823 .get_vq_group = vdpasim_get_vq_group,
824 .get_device_features = vdpasim_get_device_features,
825 .get_backend_features = vdpasim_get_backend_features,
826 .set_driver_features = vdpasim_set_driver_features,
827 .get_driver_features = vdpasim_get_driver_features,
828 .set_config_cb = vdpasim_set_config_cb,
829 .get_vq_num_max = vdpasim_get_vq_num_max,
830 .get_device_id = vdpasim_get_device_id,
831 .get_vendor_id = vdpasim_get_vendor_id,
832 .get_status = vdpasim_get_status,
833 .set_status = vdpasim_set_status,
834 .reset = vdpasim_reset,
835 .compat_reset = vdpasim_compat_reset,
836 .suspend = vdpasim_suspend,
837 .resume = vdpasim_resume,
838 .get_config_size = vdpasim_get_config_size,
839 .get_config = vdpasim_get_config,
840 .set_config = vdpasim_set_config,
841 .get_generation = vdpasim_get_generation,
842 .get_iova_range = vdpasim_get_iova_range,
843 .set_group_asid = vdpasim_set_group_asid,
844 .set_map = vdpasim_set_map,
845 .reset_map = vdpasim_reset_map,
846 .bind_mm = vdpasim_bind_mm,
847 .unbind_mm = vdpasim_unbind_mm,
848 .free = vdpasim_free,
849};
850
851MODULE_VERSION(DRV_VERSION);
852MODULE_LICENSE(DRV_LICENSE);
853MODULE_AUTHOR(DRV_AUTHOR);
854MODULE_DESCRIPTION(DRV_DESC);
855

source code of linux/drivers/vdpa/vdpa_sim/vdpa_sim.c