1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/compat.h>
3#include <linux/dma-mapping.h>
4#include <linux/iommu.h>
5#include <linux/module.h>
6#include <linux/poll.h>
7#include <linux/slab.h>
8#include <linux/uacce.h>
9
10static dev_t uacce_devt;
11static DEFINE_XARRAY_ALLOC(uacce_xa);
12
13static const struct class uacce_class = {
14 .name = UACCE_NAME,
15};
16
17/*
18 * If the parent driver or the device disappears, the queue state is invalid and
19 * ops are not usable anymore.
20 */
21static bool uacce_queue_is_valid(struct uacce_queue *q)
22{
23 return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED;
24}
25
26static int uacce_start_queue(struct uacce_queue *q)
27{
28 int ret;
29
30 if (q->state != UACCE_Q_INIT)
31 return -EINVAL;
32
33 if (q->uacce->ops->start_queue) {
34 ret = q->uacce->ops->start_queue(q);
35 if (ret < 0)
36 return ret;
37 }
38
39 q->state = UACCE_Q_STARTED;
40 return 0;
41}
42
43static int uacce_put_queue(struct uacce_queue *q)
44{
45 struct uacce_device *uacce = q->uacce;
46
47 if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue)
48 uacce->ops->stop_queue(q);
49
50 if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) &&
51 uacce->ops->put_queue)
52 uacce->ops->put_queue(q);
53
54 q->state = UACCE_Q_ZOMBIE;
55
56 return 0;
57}
58
59static long uacce_fops_unl_ioctl(struct file *filep,
60 unsigned int cmd, unsigned long arg)
61{
62 struct uacce_queue *q = filep->private_data;
63 struct uacce_device *uacce = q->uacce;
64 long ret = -ENXIO;
65
66 /*
67 * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from
68 * user. Avoid a circular lock dependency with uacce_fops_mmap(), which
69 * gets called with mmap_lock held, by taking uacce->mutex instead of
70 * q->mutex. Doing this in uacce_fops_mmap() is not possible because
71 * uacce_fops_open() calls iommu_sva_bind_device(), which takes
72 * mmap_lock, while holding uacce->mutex.
73 */
74 mutex_lock(&uacce->mutex);
75 if (!uacce_queue_is_valid(q))
76 goto out_unlock;
77
78 switch (cmd) {
79 case UACCE_CMD_START_Q:
80 ret = uacce_start_queue(q);
81 break;
82 case UACCE_CMD_PUT_Q:
83 ret = uacce_put_queue(q);
84 break;
85 default:
86 if (uacce->ops->ioctl)
87 ret = uacce->ops->ioctl(q, cmd, arg);
88 else
89 ret = -EINVAL;
90 }
91out_unlock:
92 mutex_unlock(lock: &uacce->mutex);
93 return ret;
94}
95
96#ifdef CONFIG_COMPAT
97static long uacce_fops_compat_ioctl(struct file *filep,
98 unsigned int cmd, unsigned long arg)
99{
100 arg = (unsigned long)compat_ptr(uptr: arg);
101
102 return uacce_fops_unl_ioctl(filep, cmd, arg);
103}
104#endif
105
106static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)
107{
108 u32 pasid;
109 struct iommu_sva *handle;
110
111 if (!(uacce->flags & UACCE_DEV_SVA))
112 return 0;
113
114 handle = iommu_sva_bind_device(dev: uacce->parent, current->mm);
115 if (IS_ERR(ptr: handle))
116 return PTR_ERR(ptr: handle);
117
118 pasid = iommu_sva_get_pasid(handle);
119 if (pasid == IOMMU_PASID_INVALID) {
120 iommu_sva_unbind_device(handle);
121 return -ENODEV;
122 }
123
124 q->handle = handle;
125 q->pasid = pasid;
126 return 0;
127}
128
129static void uacce_unbind_queue(struct uacce_queue *q)
130{
131 if (!q->handle)
132 return;
133 iommu_sva_unbind_device(handle: q->handle);
134 q->handle = NULL;
135}
136
137static int uacce_fops_open(struct inode *inode, struct file *filep)
138{
139 struct uacce_device *uacce;
140 struct uacce_queue *q;
141 int ret;
142
143 uacce = xa_load(&uacce_xa, index: iminor(inode));
144 if (!uacce)
145 return -ENODEV;
146
147 q = kzalloc(size: sizeof(struct uacce_queue), GFP_KERNEL);
148 if (!q)
149 return -ENOMEM;
150
151 mutex_lock(&uacce->mutex);
152
153 if (!uacce->parent) {
154 ret = -EINVAL;
155 goto out_with_mem;
156 }
157
158 ret = uacce_bind_queue(uacce, q);
159 if (ret)
160 goto out_with_mem;
161
162 q->uacce = uacce;
163
164 if (uacce->ops->get_queue) {
165 ret = uacce->ops->get_queue(uacce, q->pasid, q);
166 if (ret < 0)
167 goto out_with_bond;
168 }
169
170 init_waitqueue_head(&q->wait);
171 filep->private_data = q;
172 q->state = UACCE_Q_INIT;
173 q->mapping = filep->f_mapping;
174 mutex_init(&q->mutex);
175 list_add(new: &q->list, head: &uacce->queues);
176 mutex_unlock(lock: &uacce->mutex);
177
178 return 0;
179
180out_with_bond:
181 uacce_unbind_queue(q);
182out_with_mem:
183 kfree(objp: q);
184 mutex_unlock(lock: &uacce->mutex);
185 return ret;
186}
187
188static int uacce_fops_release(struct inode *inode, struct file *filep)
189{
190 struct uacce_queue *q = filep->private_data;
191 struct uacce_device *uacce = q->uacce;
192
193 mutex_lock(&uacce->mutex);
194 uacce_put_queue(q);
195 uacce_unbind_queue(q);
196 list_del(entry: &q->list);
197 mutex_unlock(lock: &uacce->mutex);
198 kfree(objp: q);
199
200 return 0;
201}
202
203static void uacce_vma_close(struct vm_area_struct *vma)
204{
205 struct uacce_queue *q = vma->vm_private_data;
206
207 if (vma->vm_pgoff < UACCE_MAX_REGION) {
208 struct uacce_qfile_region *qfr = q->qfrs[vma->vm_pgoff];
209
210 mutex_lock(&q->mutex);
211 q->qfrs[vma->vm_pgoff] = NULL;
212 mutex_unlock(lock: &q->mutex);
213 kfree(objp: qfr);
214 }
215}
216
217static const struct vm_operations_struct uacce_vm_ops = {
218 .close = uacce_vma_close,
219};
220
221static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma)
222{
223 struct uacce_queue *q = filep->private_data;
224 struct uacce_device *uacce = q->uacce;
225 struct uacce_qfile_region *qfr;
226 enum uacce_qfrt type = UACCE_MAX_REGION;
227 int ret = 0;
228
229 if (vma->vm_pgoff < UACCE_MAX_REGION)
230 type = vma->vm_pgoff;
231 else
232 return -EINVAL;
233
234 qfr = kzalloc(size: sizeof(*qfr), GFP_KERNEL);
235 if (!qfr)
236 return -ENOMEM;
237
238 vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK);
239 vma->vm_ops = &uacce_vm_ops;
240 vma->vm_private_data = q;
241 qfr->type = type;
242
243 mutex_lock(&q->mutex);
244 if (!uacce_queue_is_valid(q)) {
245 ret = -ENXIO;
246 goto out_with_lock;
247 }
248
249 if (q->qfrs[type]) {
250 ret = -EEXIST;
251 goto out_with_lock;
252 }
253
254 switch (type) {
255 case UACCE_QFRT_MMIO:
256 case UACCE_QFRT_DUS:
257 if (!uacce->ops->mmap) {
258 ret = -EINVAL;
259 goto out_with_lock;
260 }
261
262 ret = uacce->ops->mmap(q, vma, qfr);
263 if (ret)
264 goto out_with_lock;
265 break;
266
267 default:
268 ret = -EINVAL;
269 goto out_with_lock;
270 }
271
272 q->qfrs[type] = qfr;
273 mutex_unlock(lock: &q->mutex);
274
275 return ret;
276
277out_with_lock:
278 mutex_unlock(lock: &q->mutex);
279 kfree(objp: qfr);
280 return ret;
281}
282
283static __poll_t uacce_fops_poll(struct file *file, poll_table *wait)
284{
285 struct uacce_queue *q = file->private_data;
286 struct uacce_device *uacce = q->uacce;
287 __poll_t ret = 0;
288
289 mutex_lock(&q->mutex);
290 if (!uacce_queue_is_valid(q))
291 goto out_unlock;
292
293 poll_wait(filp: file, wait_address: &q->wait, p: wait);
294
295 if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q))
296 ret = EPOLLIN | EPOLLRDNORM;
297
298out_unlock:
299 mutex_unlock(lock: &q->mutex);
300 return ret;
301}
302
303static const struct file_operations uacce_fops = {
304 .owner = THIS_MODULE,
305 .open = uacce_fops_open,
306 .release = uacce_fops_release,
307 .unlocked_ioctl = uacce_fops_unl_ioctl,
308#ifdef CONFIG_COMPAT
309 .compat_ioctl = uacce_fops_compat_ioctl,
310#endif
311 .mmap = uacce_fops_mmap,
312 .poll = uacce_fops_poll,
313};
314
315#define to_uacce_device(dev) container_of(dev, struct uacce_device, dev)
316
317static ssize_t api_show(struct device *dev,
318 struct device_attribute *attr, char *buf)
319{
320 struct uacce_device *uacce = to_uacce_device(dev);
321
322 return sysfs_emit(buf, fmt: "%s\n", uacce->api_ver);
323}
324
325static ssize_t flags_show(struct device *dev,
326 struct device_attribute *attr, char *buf)
327{
328 struct uacce_device *uacce = to_uacce_device(dev);
329
330 return sysfs_emit(buf, fmt: "%u\n", uacce->flags);
331}
332
333static ssize_t available_instances_show(struct device *dev,
334 struct device_attribute *attr,
335 char *buf)
336{
337 struct uacce_device *uacce = to_uacce_device(dev);
338
339 if (!uacce->ops->get_available_instances)
340 return -ENODEV;
341
342 return sysfs_emit(buf, fmt: "%d\n",
343 uacce->ops->get_available_instances(uacce));
344}
345
346static ssize_t algorithms_show(struct device *dev,
347 struct device_attribute *attr, char *buf)
348{
349 struct uacce_device *uacce = to_uacce_device(dev);
350
351 return sysfs_emit(buf, fmt: "%s\n", uacce->algs);
352}
353
354static ssize_t region_mmio_size_show(struct device *dev,
355 struct device_attribute *attr, char *buf)
356{
357 struct uacce_device *uacce = to_uacce_device(dev);
358
359 return sysfs_emit(buf, fmt: "%lu\n",
360 uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT);
361}
362
363static ssize_t region_dus_size_show(struct device *dev,
364 struct device_attribute *attr, char *buf)
365{
366 struct uacce_device *uacce = to_uacce_device(dev);
367
368 return sysfs_emit(buf, fmt: "%lu\n",
369 uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT);
370}
371
372static ssize_t isolate_show(struct device *dev,
373 struct device_attribute *attr, char *buf)
374{
375 struct uacce_device *uacce = to_uacce_device(dev);
376
377 return sysfs_emit(buf, fmt: "%d\n", uacce->ops->get_isolate_state(uacce));
378}
379
380static ssize_t isolate_strategy_show(struct device *dev, struct device_attribute *attr, char *buf)
381{
382 struct uacce_device *uacce = to_uacce_device(dev);
383 u32 val;
384
385 val = uacce->ops->isolate_err_threshold_read(uacce);
386
387 return sysfs_emit(buf, fmt: "%u\n", val);
388}
389
390static ssize_t isolate_strategy_store(struct device *dev, struct device_attribute *attr,
391 const char *buf, size_t count)
392{
393 struct uacce_device *uacce = to_uacce_device(dev);
394 unsigned long val;
395 int ret;
396
397 if (kstrtoul(s: buf, base: 0, res: &val) < 0)
398 return -EINVAL;
399
400 if (val > UACCE_MAX_ERR_THRESHOLD)
401 return -EINVAL;
402
403 ret = uacce->ops->isolate_err_threshold_write(uacce, val);
404 if (ret)
405 return ret;
406
407 return count;
408}
409
410static DEVICE_ATTR_RO(api);
411static DEVICE_ATTR_RO(flags);
412static DEVICE_ATTR_RO(available_instances);
413static DEVICE_ATTR_RO(algorithms);
414static DEVICE_ATTR_RO(region_mmio_size);
415static DEVICE_ATTR_RO(region_dus_size);
416static DEVICE_ATTR_RO(isolate);
417static DEVICE_ATTR_RW(isolate_strategy);
418
419static struct attribute *uacce_dev_attrs[] = {
420 &dev_attr_api.attr,
421 &dev_attr_flags.attr,
422 &dev_attr_available_instances.attr,
423 &dev_attr_algorithms.attr,
424 &dev_attr_region_mmio_size.attr,
425 &dev_attr_region_dus_size.attr,
426 &dev_attr_isolate.attr,
427 &dev_attr_isolate_strategy.attr,
428 NULL,
429};
430
431static umode_t uacce_dev_is_visible(struct kobject *kobj,
432 struct attribute *attr, int n)
433{
434 struct device *dev = kobj_to_dev(kobj);
435 struct uacce_device *uacce = to_uacce_device(dev);
436
437 if (((attr == &dev_attr_region_mmio_size.attr) &&
438 (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) ||
439 ((attr == &dev_attr_region_dus_size.attr) &&
440 (!uacce->qf_pg_num[UACCE_QFRT_DUS])))
441 return 0;
442
443 if (attr == &dev_attr_isolate_strategy.attr &&
444 (!uacce->ops->isolate_err_threshold_read &&
445 !uacce->ops->isolate_err_threshold_write))
446 return 0;
447
448 if (attr == &dev_attr_isolate.attr && !uacce->ops->get_isolate_state)
449 return 0;
450
451 return attr->mode;
452}
453
454static struct attribute_group uacce_dev_group = {
455 .is_visible = uacce_dev_is_visible,
456 .attrs = uacce_dev_attrs,
457};
458
459__ATTRIBUTE_GROUPS(uacce_dev);
460
461static void uacce_release(struct device *dev)
462{
463 struct uacce_device *uacce = to_uacce_device(dev);
464
465 kfree(objp: uacce);
466}
467
468static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags)
469{
470 int ret;
471
472 if (!(flags & UACCE_DEV_SVA))
473 return flags;
474
475 flags &= ~UACCE_DEV_SVA;
476
477 ret = iommu_dev_enable_feature(dev: parent, f: IOMMU_DEV_FEAT_IOPF);
478 if (ret) {
479 dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret));
480 return flags;
481 }
482
483 ret = iommu_dev_enable_feature(dev: parent, f: IOMMU_DEV_FEAT_SVA);
484 if (ret) {
485 dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret));
486 iommu_dev_disable_feature(dev: parent, f: IOMMU_DEV_FEAT_IOPF);
487 return flags;
488 }
489
490 return flags | UACCE_DEV_SVA;
491}
492
493static void uacce_disable_sva(struct uacce_device *uacce)
494{
495 if (!(uacce->flags & UACCE_DEV_SVA))
496 return;
497
498 iommu_dev_disable_feature(dev: uacce->parent, f: IOMMU_DEV_FEAT_SVA);
499 iommu_dev_disable_feature(dev: uacce->parent, f: IOMMU_DEV_FEAT_IOPF);
500}
501
502/**
503 * uacce_alloc() - alloc an accelerator
504 * @parent: pointer of uacce parent device
505 * @interface: pointer of uacce_interface for register
506 *
507 * Returns uacce pointer if success and ERR_PTR if not
508 * Need check returned negotiated uacce->flags
509 */
510struct uacce_device *uacce_alloc(struct device *parent,
511 struct uacce_interface *interface)
512{
513 unsigned int flags = interface->flags;
514 struct uacce_device *uacce;
515 int ret;
516
517 uacce = kzalloc(size: sizeof(struct uacce_device), GFP_KERNEL);
518 if (!uacce)
519 return ERR_PTR(error: -ENOMEM);
520
521 flags = uacce_enable_sva(parent, flags);
522
523 uacce->parent = parent;
524 uacce->flags = flags;
525 uacce->ops = interface->ops;
526
527 ret = xa_alloc(xa: &uacce_xa, id: &uacce->dev_id, entry: uacce, xa_limit_32b,
528 GFP_KERNEL);
529 if (ret < 0)
530 goto err_with_uacce;
531
532 INIT_LIST_HEAD(list: &uacce->queues);
533 mutex_init(&uacce->mutex);
534 device_initialize(dev: &uacce->dev);
535 uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id);
536 uacce->dev.class = &uacce_class;
537 uacce->dev.groups = uacce_dev_groups;
538 uacce->dev.parent = uacce->parent;
539 uacce->dev.release = uacce_release;
540 dev_set_name(dev: &uacce->dev, name: "%s-%d", interface->name, uacce->dev_id);
541
542 return uacce;
543
544err_with_uacce:
545 uacce_disable_sva(uacce);
546 kfree(objp: uacce);
547 return ERR_PTR(error: ret);
548}
549EXPORT_SYMBOL_GPL(uacce_alloc);
550
551/**
552 * uacce_register() - add the accelerator to cdev and export to user space
553 * @uacce: The initialized uacce device
554 *
555 * Return 0 if register succeeded, or an error.
556 */
557int uacce_register(struct uacce_device *uacce)
558{
559 if (!uacce)
560 return -ENODEV;
561
562 uacce->cdev = cdev_alloc();
563 if (!uacce->cdev)
564 return -ENOMEM;
565
566 uacce->cdev->ops = &uacce_fops;
567 uacce->cdev->owner = THIS_MODULE;
568
569 return cdev_device_add(cdev: uacce->cdev, dev: &uacce->dev);
570}
571EXPORT_SYMBOL_GPL(uacce_register);
572
573/**
574 * uacce_remove() - remove the accelerator
575 * @uacce: the accelerator to remove
576 */
577void uacce_remove(struct uacce_device *uacce)
578{
579 struct uacce_queue *q, *next_q;
580
581 if (!uacce)
582 return;
583
584 /*
585 * uacce_fops_open() may be running concurrently, even after we remove
586 * the cdev. Holding uacce->mutex ensures that open() does not obtain a
587 * removed uacce device.
588 */
589 mutex_lock(&uacce->mutex);
590 /* ensure no open queue remains */
591 list_for_each_entry_safe(q, next_q, &uacce->queues, list) {
592 /*
593 * Taking q->mutex ensures that fops do not use the defunct
594 * uacce->ops after the queue is disabled.
595 */
596 mutex_lock(&q->mutex);
597 uacce_put_queue(q);
598 mutex_unlock(lock: &q->mutex);
599 uacce_unbind_queue(q);
600
601 /*
602 * unmap remaining mapping from user space, preventing user still
603 * access the mmaped area while parent device is already removed
604 */
605 unmap_mapping_range(mapping: q->mapping, holebegin: 0, holelen: 0, even_cows: 1);
606 }
607
608 /* disable sva now since no opened queues */
609 uacce_disable_sva(uacce);
610
611 if (uacce->cdev)
612 cdev_device_del(cdev: uacce->cdev, dev: &uacce->dev);
613 xa_erase(&uacce_xa, index: uacce->dev_id);
614 /*
615 * uacce exists as long as there are open fds, but ops will be freed
616 * now. Ensure that bugs cause NULL deref rather than use-after-free.
617 */
618 uacce->ops = NULL;
619 uacce->parent = NULL;
620 mutex_unlock(lock: &uacce->mutex);
621 put_device(dev: &uacce->dev);
622}
623EXPORT_SYMBOL_GPL(uacce_remove);
624
625static int __init uacce_init(void)
626{
627 int ret;
628
629 ret = class_register(class: &uacce_class);
630 if (ret)
631 return ret;
632
633 ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME);
634 if (ret)
635 class_unregister(class: &uacce_class);
636
637 return ret;
638}
639
640static __exit void uacce_exit(void)
641{
642 unregister_chrdev_region(uacce_devt, MINORMASK);
643 class_unregister(class: &uacce_class);
644}
645
646subsys_initcall(uacce_init);
647module_exit(uacce_exit);
648
649MODULE_LICENSE("GPL");
650MODULE_AUTHOR("HiSilicon Tech. Co., Ltd.");
651MODULE_DESCRIPTION("Accelerator interface for Userland applications");
652

source code of linux/drivers/misc/uacce/uacce.c