1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/dma-buf.h>
7#include <linux/highmem.h>
8#include <linux/module.h>
9#include <linux/set_memory.h>
10#include <linux/xarray.h>
11
12#include <drm/drm_cache.h>
13#include <drm/drm_debugfs.h>
14#include <drm/drm_file.h>
15#include <drm/drm_utils.h>
16
17#include "ivpu_drv.h"
18#include "ivpu_fw.h"
19#include "ivpu_gem.h"
20#include "ivpu_hw.h"
21#include "ivpu_mmu.h"
22#include "ivpu_mmu_context.h"
23
24MODULE_IMPORT_NS("DMA_BUF");
25
26static const struct drm_gem_object_funcs ivpu_gem_funcs;
27
28static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
29{
30 ivpu_dbg(vdev, BO,
31 "%6s: bo %8p size %9zu ctx %d vpu_addr %9llx pages %d sgt %d mmu_mapped %d wc %d imported %d\n",
32 action, bo, ivpu_bo_size(bo), bo->ctx_id, bo->vpu_addr,
33 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
34 (bool)drm_gem_is_imported(&bo->base.base));
35}
36
37static inline int ivpu_bo_lock(struct ivpu_bo *bo)
38{
39 return dma_resv_lock(obj: bo->base.base.resv, NULL);
40}
41
42static inline void ivpu_bo_unlock(struct ivpu_bo *bo)
43{
44 dma_resv_unlock(obj: bo->base.base.resv);
45}
46
47static struct sg_table *ivpu_bo_map_attachment(struct ivpu_device *vdev, struct ivpu_bo *bo)
48{
49 struct sg_table *sgt;
50
51 drm_WARN_ON(&vdev->drm, !bo->base.base.import_attach);
52
53 ivpu_bo_lock(bo);
54
55 sgt = bo->base.sgt;
56 if (!sgt) {
57 sgt = dma_buf_map_attachment(bo->base.base.import_attach, DMA_BIDIRECTIONAL);
58 if (IS_ERR(ptr: sgt))
59 ivpu_err(vdev, "Failed to map BO in IOMMU: %ld\n", PTR_ERR(sgt));
60 else
61 bo->base.sgt = sgt;
62 }
63
64 ivpu_bo_unlock(bo);
65
66 return sgt;
67}
68
69/*
70 * ivpu_bo_bind() - pin the backing physical pages and map them to VPU.
71 *
72 * This function pins physical memory pages, then maps the physical pages
73 * to IOMMU address space and finally updates the VPU MMU page tables
74 * to allow the VPU to translate VPU address to IOMMU address.
75 */
76int __must_check ivpu_bo_bind(struct ivpu_bo *bo)
77{
78 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
79 struct sg_table *sgt;
80 int ret = 0;
81
82 ivpu_dbg_bo(vdev, bo, action: "bind");
83
84 if (bo->base.base.import_attach)
85 sgt = ivpu_bo_map_attachment(vdev, bo);
86 else
87 sgt = drm_gem_shmem_get_pages_sgt(shmem: &bo->base);
88 if (IS_ERR(ptr: sgt)) {
89 ret = PTR_ERR(ptr: sgt);
90 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
91 return ret;
92 }
93
94 ivpu_bo_lock(bo);
95
96 if (!bo->mmu_mapped) {
97 drm_WARN_ON(&vdev->drm, !bo->ctx);
98 ret = ivpu_mmu_context_map_sgt(vdev, ctx: bo->ctx, vpu_addr: bo->vpu_addr, sgt,
99 llc_coherent: ivpu_bo_is_snooped(bo), read_only: ivpu_bo_is_read_only(bo));
100 if (ret) {
101 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
102 goto unlock;
103 }
104 bo->mmu_mapped = true;
105 }
106
107unlock:
108 ivpu_bo_unlock(bo);
109
110 return ret;
111}
112
113static int
114ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
115 const struct ivpu_addr_range *range)
116{
117 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
118 int idx, ret;
119
120 if (!drm_dev_enter(dev: &vdev->drm, idx: &idx))
121 return -ENODEV;
122
123 ivpu_bo_lock(bo);
124
125 ret = ivpu_mmu_context_insert_node(ctx, range, size: ivpu_bo_size(bo), node: &bo->mm_node);
126 if (!ret) {
127 bo->ctx = ctx;
128 bo->ctx_id = ctx->id;
129 bo->vpu_addr = bo->mm_node.start;
130 ivpu_dbg_bo(vdev, bo, action: "vaddr");
131 }
132
133 ivpu_bo_unlock(bo);
134
135 drm_dev_exit(idx);
136
137 return ret;
138}
139
140static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
141{
142 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
143
144 dma_resv_assert_held(bo->base.base.resv);
145
146 if (bo->mmu_mapped) {
147 drm_WARN_ON(&vdev->drm, !bo->ctx);
148 drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
149 drm_WARN_ON(&vdev->drm, !bo->base.sgt);
150 ivpu_mmu_context_unmap_sgt(vdev, ctx: bo->ctx, vpu_addr: bo->vpu_addr, sgt: bo->base.sgt);
151 bo->mmu_mapped = false;
152 }
153
154 if (bo->ctx) {
155 ivpu_mmu_context_remove_node(ctx: bo->ctx, node: &bo->mm_node);
156 bo->ctx = NULL;
157 }
158
159 if (bo->base.sgt) {
160 if (bo->base.base.import_attach) {
161 dma_buf_unmap_attachment(bo->base.base.import_attach,
162 bo->base.sgt, DMA_BIDIRECTIONAL);
163 } else {
164 dma_unmap_sgtable(dev: vdev->drm.dev, sgt: bo->base.sgt, dir: DMA_BIDIRECTIONAL, attrs: 0);
165 sg_free_table(bo->base.sgt);
166 kfree(objp: bo->base.sgt);
167 }
168 bo->base.sgt = NULL;
169 }
170}
171
172void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
173{
174 struct ivpu_bo *bo;
175
176 if (drm_WARN_ON(&vdev->drm, !ctx))
177 return;
178
179 mutex_lock(&vdev->bo_list_lock);
180 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
181 ivpu_bo_lock(bo);
182 if (bo->ctx == ctx) {
183 ivpu_dbg_bo(vdev, bo, action: "unbind");
184 ivpu_bo_unbind_locked(bo);
185 }
186 ivpu_bo_unlock(bo);
187 }
188 mutex_unlock(lock: &vdev->bo_list_lock);
189}
190
191struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
192{
193 struct ivpu_bo *bo;
194
195 if (size == 0 || !PAGE_ALIGNED(size))
196 return ERR_PTR(error: -EINVAL);
197
198 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
199 if (!bo)
200 return ERR_PTR(error: -ENOMEM);
201
202 bo->base.base.funcs = &ivpu_gem_funcs;
203 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
204
205 INIT_LIST_HEAD(list: &bo->bo_list_node);
206
207 return &bo->base.base;
208}
209
210struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev,
211 struct dma_buf *dma_buf)
212{
213 struct ivpu_device *vdev = to_ivpu_device(dev);
214 struct device *attach_dev = dev->dev;
215 struct dma_buf_attachment *attach;
216 struct drm_gem_object *obj;
217 struct ivpu_bo *bo;
218 int ret;
219
220 attach = dma_buf_attach(dmabuf: dma_buf, dev: attach_dev);
221 if (IS_ERR(ptr: attach))
222 return ERR_CAST(ptr: attach);
223
224 get_dma_buf(dmabuf: dma_buf);
225
226 obj = drm_gem_shmem_prime_import_sg_table(dev, attach, NULL);
227 if (IS_ERR(ptr: obj)) {
228 ret = PTR_ERR(ptr: obj);
229 goto fail_detach;
230 }
231
232 obj->import_attach = attach;
233 obj->resv = dma_buf->resv;
234
235 bo = to_ivpu_bo(obj);
236
237 mutex_lock(&vdev->bo_list_lock);
238 list_add_tail(new: &bo->bo_list_node, head: &vdev->bo_list);
239 mutex_unlock(lock: &vdev->bo_list_lock);
240
241 ivpu_dbg(vdev, BO, "import: bo %8p size %9zu\n", bo, ivpu_bo_size(bo));
242
243 return obj;
244
245fail_detach:
246 dma_buf_detach(dmabuf: dma_buf, attach);
247 dma_buf_put(dmabuf: dma_buf);
248
249 return ERR_PTR(error: ret);
250}
251
252static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags)
253{
254 struct drm_gem_shmem_object *shmem;
255 struct ivpu_bo *bo;
256
257 switch (flags & DRM_IVPU_BO_CACHE_MASK) {
258 case DRM_IVPU_BO_CACHED:
259 case DRM_IVPU_BO_WC:
260 break;
261 default:
262 return ERR_PTR(error: -EINVAL);
263 }
264
265 shmem = drm_gem_shmem_create(dev: &vdev->drm, size);
266 if (IS_ERR(ptr: shmem))
267 return ERR_CAST(ptr: shmem);
268
269 bo = to_ivpu_bo(obj: &shmem->base);
270 bo->base.map_wc = flags & DRM_IVPU_BO_WC;
271 bo->flags = flags;
272
273 mutex_lock(&vdev->bo_list_lock);
274 list_add_tail(new: &bo->bo_list_node, head: &vdev->bo_list);
275 mutex_unlock(lock: &vdev->bo_list_lock);
276
277 ivpu_dbg(vdev, BO, " alloc: bo %8p size %9llu\n", bo, size);
278
279 return bo;
280}
281
282static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
283{
284 struct ivpu_file_priv *file_priv = file->driver_priv;
285 struct ivpu_device *vdev = file_priv->vdev;
286 struct ivpu_bo *bo = to_ivpu_bo(obj);
287 struct ivpu_addr_range *range;
288
289 if (bo->ctx) {
290 ivpu_dbg(vdev, IOCTL, "Can't add BO %pe to ctx %u: already in ctx %u\n",
291 bo, file_priv->ctx.id, bo->ctx->id);
292 return -EALREADY;
293 }
294
295 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
296 range = &vdev->hw->ranges.shave;
297 else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
298 range = &vdev->hw->ranges.dma;
299 else
300 range = &vdev->hw->ranges.user;
301
302 return ivpu_bo_alloc_vpu_addr(bo, ctx: &file_priv->ctx, range);
303}
304
305static void ivpu_gem_bo_free(struct drm_gem_object *obj)
306{
307 struct ivpu_device *vdev = to_ivpu_device(dev: obj->dev);
308 struct ivpu_bo *bo = to_ivpu_bo(obj);
309
310 ivpu_dbg_bo(vdev, bo, action: "free");
311
312 drm_WARN_ON(&vdev->drm, list_empty(&bo->bo_list_node));
313
314 mutex_lock(&vdev->bo_list_lock);
315 list_del(entry: &bo->bo_list_node);
316
317 drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) &&
318 !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
319 drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0);
320 drm_WARN_ON(&vdev->drm, bo->base.vaddr);
321
322 ivpu_bo_lock(bo);
323 ivpu_bo_unbind_locked(bo);
324 ivpu_bo_unlock(bo);
325
326 mutex_unlock(lock: &vdev->bo_list_lock);
327
328 drm_WARN_ON(&vdev->drm, bo->mmu_mapped);
329 drm_WARN_ON(&vdev->drm, bo->ctx);
330
331 drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1);
332 drm_WARN_ON(obj->dev, bo->base.base.vma_node.vm_files.rb_node);
333 drm_gem_shmem_free(shmem: &bo->base);
334}
335
336static enum drm_gem_object_status ivpu_gem_status(struct drm_gem_object *obj)
337{
338 struct ivpu_bo *bo = to_ivpu_bo(obj);
339 enum drm_gem_object_status status = 0;
340
341 if (ivpu_bo_is_resident(bo))
342 status |= DRM_GEM_OBJECT_RESIDENT;
343
344 return status;
345}
346
347static const struct drm_gem_object_funcs ivpu_gem_funcs = {
348 .free = ivpu_gem_bo_free,
349 .open = ivpu_gem_bo_open,
350 .print_info = drm_gem_shmem_object_print_info,
351 .pin = drm_gem_shmem_object_pin,
352 .unpin = drm_gem_shmem_object_unpin,
353 .get_sg_table = drm_gem_shmem_object_get_sg_table,
354 .vmap = drm_gem_shmem_object_vmap,
355 .vunmap = drm_gem_shmem_object_vunmap,
356 .mmap = drm_gem_shmem_object_mmap,
357 .status = ivpu_gem_status,
358 .vm_ops = &drm_gem_shmem_vm_ops,
359};
360
361int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
362{
363 struct ivpu_file_priv *file_priv = file->driver_priv;
364 struct ivpu_device *vdev = file_priv->vdev;
365 struct drm_ivpu_bo_create *args = data;
366 u64 size = PAGE_ALIGN(args->size);
367 struct ivpu_bo *bo;
368 int ret;
369
370 if (args->flags & ~DRM_IVPU_BO_FLAGS) {
371 ivpu_dbg(vdev, IOCTL, "Invalid BO flags 0x%x\n", args->flags);
372 return -EINVAL;
373 }
374
375 if (size == 0) {
376 ivpu_dbg(vdev, IOCTL, "Invalid BO size %llu\n", args->size);
377 return -EINVAL;
378 }
379
380 bo = ivpu_bo_alloc(vdev, size, flags: args->flags);
381 if (IS_ERR(ptr: bo)) {
382 ivpu_dbg(vdev, IOCTL, "Failed to allocate BO: %pe ctx %u size %llu flags 0x%x\n",
383 bo, file_priv->ctx.id, args->size, args->flags);
384 return PTR_ERR(ptr: bo);
385 }
386
387 drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 0);
388
389 ret = drm_gem_handle_create(file_priv: file, obj: &bo->base.base, handlep: &args->handle);
390 if (ret) {
391 ivpu_dbg(vdev, IOCTL, "Failed to create handle for BO: %pe ctx %u size %llu flags 0x%x\n",
392 bo, file_priv->ctx.id, args->size, args->flags);
393 } else {
394 args->vpu_addr = bo->vpu_addr;
395 drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 1);
396 }
397
398 drm_gem_object_put(obj: &bo->base.base);
399
400 return ret;
401}
402
403struct ivpu_bo *
404ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
405 struct ivpu_addr_range *range, u64 size, u32 flags)
406{
407 struct iosys_map map;
408 struct ivpu_bo *bo;
409 int ret;
410
411 if (drm_WARN_ON(&vdev->drm, !range))
412 return NULL;
413
414 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
415 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
416 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
417
418 bo = ivpu_bo_alloc(vdev, size, flags);
419 if (IS_ERR(ptr: bo)) {
420 ivpu_err(vdev, "Failed to allocate BO: %pe vpu_addr 0x%llx size %llu flags 0x%x\n",
421 bo, range->start, size, flags);
422 return NULL;
423 }
424
425 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
426 if (ret) {
427 ivpu_err(vdev, "Failed to allocate NPU address for BO: %pe ctx %u size %llu: %d\n",
428 bo, ctx->id, size, ret);
429 goto err_put;
430 }
431
432 ret = ivpu_bo_bind(bo);
433 if (ret)
434 goto err_put;
435
436 if (flags & DRM_IVPU_BO_MAPPABLE) {
437 ivpu_bo_lock(bo);
438 ret = drm_gem_shmem_vmap_locked(shmem: &bo->base, map: &map);
439 ivpu_bo_unlock(bo);
440
441 if (ret)
442 goto err_put;
443 }
444
445 return bo;
446
447err_put:
448 drm_gem_object_put(obj: &bo->base.base);
449 return NULL;
450}
451
452struct ivpu_bo *ivpu_bo_create_runtime(struct ivpu_device *vdev, u64 addr, u64 size, u32 flags)
453{
454 struct ivpu_addr_range range;
455
456 if (!ivpu_is_within_range(addr, size, range: &vdev->hw->ranges.runtime)) {
457 ivpu_err(vdev, "Invalid runtime BO address 0x%llx size %llu\n", addr, size);
458 return NULL;
459 }
460
461 if (ivpu_hw_range_init(vdev, range: &range, start: addr, size))
462 return NULL;
463
464 return ivpu_bo_create(vdev, ctx: &vdev->gctx, range: &range, size, flags);
465}
466
467struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
468{
469 return ivpu_bo_create(vdev, ctx: &vdev->gctx, range: &vdev->hw->ranges.global, size, flags);
470}
471
472void ivpu_bo_free(struct ivpu_bo *bo)
473{
474 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
475
476 if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
477 ivpu_bo_lock(bo);
478 drm_gem_shmem_vunmap_locked(shmem: &bo->base, map: &map);
479 ivpu_bo_unlock(bo);
480 }
481
482 drm_gem_object_put(obj: &bo->base.base);
483}
484
485int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
486{
487 struct drm_ivpu_bo_info *args = data;
488 struct drm_gem_object *obj;
489 struct ivpu_bo *bo;
490 int ret = 0;
491
492 obj = drm_gem_object_lookup(filp: file, handle: args->handle);
493 if (!obj)
494 return -ENOENT;
495
496 bo = to_ivpu_bo(obj);
497
498 ivpu_bo_lock(bo);
499 args->flags = bo->flags;
500 args->mmap_offset = drm_vma_node_offset_addr(node: &obj->vma_node);
501 args->vpu_addr = bo->vpu_addr;
502 args->size = obj->size;
503 ivpu_bo_unlock(bo);
504
505 drm_gem_object_put(obj);
506 return ret;
507}
508
509int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
510{
511 struct drm_ivpu_bo_wait *args = data;
512 struct drm_gem_object *obj;
513 unsigned long timeout;
514 long ret;
515
516 timeout = drm_timeout_abs_to_jiffies(timeout_nsec: args->timeout_ns);
517
518 /* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
519 timeout += 1;
520
521 obj = drm_gem_object_lookup(filp: file, handle: args->handle);
522 if (!obj)
523 return -EINVAL;
524
525 ret = dma_resv_wait_timeout(obj: obj->resv, usage: DMA_RESV_USAGE_READ, intr: true, timeout);
526 if (ret == 0) {
527 ret = -ETIMEDOUT;
528 } else if (ret > 0) {
529 ret = 0;
530 args->job_status = to_ivpu_bo(obj)->job_status;
531 }
532
533 drm_gem_object_put(obj);
534
535 return ret;
536}
537
538static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
539{
540 ivpu_bo_lock(bo);
541
542 drm_printf(p, f: "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
543 bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size,
544 bo->flags, kref_read(kref: &bo->base.base.refcount));
545
546 if (bo->base.pages)
547 drm_printf(p, f: " has_pages");
548
549 if (bo->mmu_mapped)
550 drm_printf(p, f: " mmu_mapped");
551
552 if (drm_gem_is_imported(obj: &bo->base.base))
553 drm_printf(p, f: " imported");
554
555 drm_printf(p, f: "\n");
556
557 ivpu_bo_unlock(bo);
558}
559
560void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
561{
562 struct ivpu_device *vdev = to_ivpu_device(dev);
563 struct ivpu_bo *bo;
564
565 drm_printf(p, f: "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
566 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
567
568 mutex_lock(&vdev->bo_list_lock);
569 list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
570 ivpu_bo_print_info(bo, p);
571 mutex_unlock(lock: &vdev->bo_list_lock);
572}
573
574void ivpu_bo_list_print(struct drm_device *dev)
575{
576 struct drm_printer p = drm_info_printer(dev: dev->dev);
577
578 ivpu_bo_list(dev, p: &p);
579}
580

source code of linux/drivers/accel/ivpu/ivpu_gem.c