1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2020-2023 Intel Corporation |
4 | */ |
5 | |
6 | #include <linux/dma-buf.h> |
7 | #include <linux/highmem.h> |
8 | #include <linux/module.h> |
9 | #include <linux/set_memory.h> |
10 | #include <linux/xarray.h> |
11 | |
12 | #include <drm/drm_cache.h> |
13 | #include <drm/drm_debugfs.h> |
14 | #include <drm/drm_file.h> |
15 | #include <drm/drm_utils.h> |
16 | |
17 | #include "ivpu_drv.h" |
18 | #include "ivpu_gem.h" |
19 | #include "ivpu_hw.h" |
20 | #include "ivpu_mmu.h" |
21 | #include "ivpu_mmu_context.h" |
22 | |
23 | static const struct drm_gem_object_funcs ivpu_gem_funcs; |
24 | |
25 | static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) |
26 | { |
27 | ivpu_dbg(vdev, BO, |
28 | "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n" , |
29 | action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx ? bo->ctx->id : 0, |
30 | (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, |
31 | (bool)bo->base.base.import_attach); |
32 | } |
33 | |
34 | /* |
35 | * ivpu_bo_pin() - pin the backing physical pages and map them to VPU. |
36 | * |
37 | * This function pins physical memory pages, then maps the physical pages |
38 | * to IOMMU address space and finally updates the VPU MMU page tables |
39 | * to allow the VPU to translate VPU address to IOMMU address. |
40 | */ |
41 | int __must_check ivpu_bo_pin(struct ivpu_bo *bo) |
42 | { |
43 | struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); |
44 | int ret = 0; |
45 | |
46 | mutex_lock(&bo->lock); |
47 | |
48 | ivpu_dbg_bo(vdev, bo, action: "pin" ); |
49 | drm_WARN_ON(&vdev->drm, !bo->ctx); |
50 | |
51 | if (!bo->mmu_mapped) { |
52 | struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(shmem: &bo->base); |
53 | |
54 | if (IS_ERR(ptr: sgt)) { |
55 | ret = PTR_ERR(ptr: sgt); |
56 | ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n" , ret); |
57 | goto unlock; |
58 | } |
59 | |
60 | ret = ivpu_mmu_context_map_sgt(vdev, ctx: bo->ctx, vpu_addr: bo->vpu_addr, sgt, |
61 | llc_coherent: ivpu_bo_is_snooped(bo)); |
62 | if (ret) { |
63 | ivpu_err(vdev, "Failed to map BO in MMU: %d\n" , ret); |
64 | goto unlock; |
65 | } |
66 | bo->mmu_mapped = true; |
67 | } |
68 | |
69 | unlock: |
70 | mutex_unlock(lock: &bo->lock); |
71 | |
72 | return ret; |
73 | } |
74 | |
75 | static int |
76 | ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, |
77 | const struct ivpu_addr_range *range) |
78 | { |
79 | struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); |
80 | int idx, ret; |
81 | |
82 | if (!drm_dev_enter(dev: &vdev->drm, idx: &idx)) |
83 | return -ENODEV; |
84 | |
85 | mutex_lock(&bo->lock); |
86 | |
87 | ret = ivpu_mmu_context_insert_node(ctx, range, size: ivpu_bo_size(bo), node: &bo->mm_node); |
88 | if (!ret) { |
89 | bo->ctx = ctx; |
90 | bo->vpu_addr = bo->mm_node.start; |
91 | } else { |
92 | ivpu_err(vdev, "Failed to add BO to context %u: %d\n" , ctx->id, ret); |
93 | } |
94 | |
95 | ivpu_dbg_bo(vdev, bo, action: "alloc" ); |
96 | |
97 | mutex_unlock(lock: &bo->lock); |
98 | |
99 | drm_dev_exit(idx); |
100 | |
101 | return ret; |
102 | } |
103 | |
104 | static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) |
105 | { |
106 | struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); |
107 | |
108 | lockdep_assert(lockdep_is_held(&bo->lock) || !kref_read(&bo->base.base.refcount)); |
109 | |
110 | if (bo->mmu_mapped) { |
111 | drm_WARN_ON(&vdev->drm, !bo->ctx); |
112 | drm_WARN_ON(&vdev->drm, !bo->vpu_addr); |
113 | drm_WARN_ON(&vdev->drm, !bo->base.sgt); |
114 | ivpu_mmu_context_unmap_sgt(vdev, ctx: bo->ctx, vpu_addr: bo->vpu_addr, sgt: bo->base.sgt); |
115 | bo->mmu_mapped = false; |
116 | } |
117 | |
118 | if (bo->ctx) { |
119 | ivpu_mmu_context_remove_node(ctx: bo->ctx, node: &bo->mm_node); |
120 | bo->ctx = NULL; |
121 | } |
122 | |
123 | if (bo->base.base.import_attach) |
124 | return; |
125 | |
126 | dma_resv_lock(obj: bo->base.base.resv, NULL); |
127 | if (bo->base.sgt) { |
128 | dma_unmap_sgtable(dev: vdev->drm.dev, sgt: bo->base.sgt, dir: DMA_BIDIRECTIONAL, attrs: 0); |
129 | sg_free_table(bo->base.sgt); |
130 | kfree(objp: bo->base.sgt); |
131 | bo->base.sgt = NULL; |
132 | } |
133 | dma_resv_unlock(obj: bo->base.base.resv); |
134 | } |
135 | |
136 | void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) |
137 | { |
138 | struct ivpu_bo *bo; |
139 | |
140 | if (drm_WARN_ON(&vdev->drm, !ctx)) |
141 | return; |
142 | |
143 | mutex_lock(&vdev->bo_list_lock); |
144 | list_for_each_entry(bo, &vdev->bo_list, bo_list_node) { |
145 | mutex_lock(&bo->lock); |
146 | if (bo->ctx == ctx) { |
147 | ivpu_dbg_bo(vdev, bo, action: "unbind" ); |
148 | ivpu_bo_unbind_locked(bo); |
149 | } |
150 | mutex_unlock(lock: &bo->lock); |
151 | } |
152 | mutex_unlock(lock: &vdev->bo_list_lock); |
153 | } |
154 | |
155 | struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size) |
156 | { |
157 | struct ivpu_bo *bo; |
158 | |
159 | if (size == 0 || !PAGE_ALIGNED(size)) |
160 | return ERR_PTR(error: -EINVAL); |
161 | |
162 | bo = kzalloc(size: sizeof(*bo), GFP_KERNEL); |
163 | if (!bo) |
164 | return ERR_PTR(error: -ENOMEM); |
165 | |
166 | bo->base.base.funcs = &ivpu_gem_funcs; |
167 | bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */ |
168 | |
169 | INIT_LIST_HEAD(list: &bo->bo_list_node); |
170 | mutex_init(&bo->lock); |
171 | |
172 | return &bo->base.base; |
173 | } |
174 | |
175 | static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags) |
176 | { |
177 | struct drm_gem_shmem_object *shmem; |
178 | struct ivpu_bo *bo; |
179 | |
180 | switch (flags & DRM_IVPU_BO_CACHE_MASK) { |
181 | case DRM_IVPU_BO_CACHED: |
182 | case DRM_IVPU_BO_WC: |
183 | break; |
184 | default: |
185 | return ERR_PTR(error: -EINVAL); |
186 | } |
187 | |
188 | shmem = drm_gem_shmem_create(dev: &vdev->drm, size); |
189 | if (IS_ERR(ptr: shmem)) |
190 | return ERR_CAST(ptr: shmem); |
191 | |
192 | bo = to_ivpu_bo(obj: &shmem->base); |
193 | bo->base.map_wc = flags & DRM_IVPU_BO_WC; |
194 | bo->flags = flags; |
195 | |
196 | mutex_lock(&vdev->bo_list_lock); |
197 | list_add_tail(new: &bo->bo_list_node, head: &vdev->bo_list); |
198 | mutex_unlock(lock: &vdev->bo_list_lock); |
199 | |
200 | return bo; |
201 | } |
202 | |
203 | static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file) |
204 | { |
205 | struct ivpu_file_priv *file_priv = file->driver_priv; |
206 | struct ivpu_device *vdev = file_priv->vdev; |
207 | struct ivpu_bo *bo = to_ivpu_bo(obj); |
208 | struct ivpu_addr_range *range; |
209 | |
210 | if (bo->ctx) { |
211 | ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n" , |
212 | file_priv->ctx.id, bo->ctx->id); |
213 | return -EALREADY; |
214 | } |
215 | |
216 | if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) |
217 | range = &vdev->hw->ranges.shave; |
218 | else if (bo->flags & DRM_IVPU_BO_DMA_MEM) |
219 | range = &vdev->hw->ranges.dma; |
220 | else |
221 | range = &vdev->hw->ranges.user; |
222 | |
223 | return ivpu_bo_alloc_vpu_addr(bo, ctx: &file_priv->ctx, range); |
224 | } |
225 | |
226 | static void ivpu_gem_bo_free(struct drm_gem_object *obj) |
227 | { |
228 | struct ivpu_device *vdev = to_ivpu_device(dev: obj->dev); |
229 | struct ivpu_bo *bo = to_ivpu_bo(obj); |
230 | |
231 | ivpu_dbg_bo(vdev, bo, action: "free" ); |
232 | |
233 | mutex_lock(&vdev->bo_list_lock); |
234 | list_del(entry: &bo->bo_list_node); |
235 | mutex_unlock(lock: &vdev->bo_list_lock); |
236 | |
237 | drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); |
238 | |
239 | ivpu_bo_unbind_locked(bo); |
240 | mutex_destroy(lock: &bo->lock); |
241 | |
242 | drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1); |
243 | drm_gem_shmem_free(shmem: &bo->base); |
244 | } |
245 | |
246 | static const struct drm_gem_object_funcs ivpu_gem_funcs = { |
247 | .free = ivpu_gem_bo_free, |
248 | .open = ivpu_gem_bo_open, |
249 | .print_info = drm_gem_shmem_object_print_info, |
250 | .pin = drm_gem_shmem_object_pin, |
251 | .unpin = drm_gem_shmem_object_unpin, |
252 | .get_sg_table = drm_gem_shmem_object_get_sg_table, |
253 | .vmap = drm_gem_shmem_object_vmap, |
254 | .vunmap = drm_gem_shmem_object_vunmap, |
255 | .mmap = drm_gem_shmem_object_mmap, |
256 | .vm_ops = &drm_gem_shmem_vm_ops, |
257 | }; |
258 | |
259 | int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) |
260 | { |
261 | struct ivpu_file_priv *file_priv = file->driver_priv; |
262 | struct ivpu_device *vdev = file_priv->vdev; |
263 | struct drm_ivpu_bo_create *args = data; |
264 | u64 size = PAGE_ALIGN(args->size); |
265 | struct ivpu_bo *bo; |
266 | int ret; |
267 | |
268 | if (args->flags & ~DRM_IVPU_BO_FLAGS) |
269 | return -EINVAL; |
270 | |
271 | if (size == 0) |
272 | return -EINVAL; |
273 | |
274 | bo = ivpu_bo_alloc(vdev, size, flags: args->flags); |
275 | if (IS_ERR(ptr: bo)) { |
276 | ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)" , |
277 | bo, file_priv->ctx.id, args->size, args->flags); |
278 | return PTR_ERR(ptr: bo); |
279 | } |
280 | |
281 | ret = drm_gem_handle_create(file_priv: file, obj: &bo->base.base, handlep: &args->handle); |
282 | if (!ret) |
283 | args->vpu_addr = bo->vpu_addr; |
284 | |
285 | drm_gem_object_put(obj: &bo->base.base); |
286 | |
287 | return ret; |
288 | } |
289 | |
290 | struct ivpu_bo * |
291 | ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, |
292 | struct ivpu_addr_range *range, u64 size, u32 flags) |
293 | { |
294 | struct iosys_map map; |
295 | struct ivpu_bo *bo; |
296 | int ret; |
297 | |
298 | if (drm_WARN_ON(&vdev->drm, !range)) |
299 | return NULL; |
300 | |
301 | drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start)); |
302 | drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end)); |
303 | drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); |
304 | |
305 | bo = ivpu_bo_alloc(vdev, size, flags); |
306 | if (IS_ERR(ptr: bo)) { |
307 | ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)" , |
308 | bo, range->start, size, flags); |
309 | return NULL; |
310 | } |
311 | |
312 | ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range); |
313 | if (ret) |
314 | goto err_put; |
315 | |
316 | ret = ivpu_bo_pin(bo); |
317 | if (ret) |
318 | goto err_put; |
319 | |
320 | if (flags & DRM_IVPU_BO_MAPPABLE) { |
321 | dma_resv_lock(obj: bo->base.base.resv, NULL); |
322 | ret = drm_gem_shmem_vmap(shmem: &bo->base, map: &map); |
323 | dma_resv_unlock(obj: bo->base.base.resv); |
324 | |
325 | if (ret) |
326 | goto err_put; |
327 | } |
328 | |
329 | return bo; |
330 | |
331 | err_put: |
332 | drm_gem_object_put(obj: &bo->base.base); |
333 | return NULL; |
334 | } |
335 | |
336 | struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags) |
337 | { |
338 | return ivpu_bo_create(vdev, ctx: &vdev->gctx, range: &vdev->hw->ranges.global, size, flags); |
339 | } |
340 | |
341 | void ivpu_bo_free(struct ivpu_bo *bo) |
342 | { |
343 | struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr); |
344 | |
345 | if (bo->flags & DRM_IVPU_BO_MAPPABLE) { |
346 | dma_resv_lock(obj: bo->base.base.resv, NULL); |
347 | drm_gem_shmem_vunmap(shmem: &bo->base, map: &map); |
348 | dma_resv_unlock(obj: bo->base.base.resv); |
349 | } |
350 | |
351 | drm_gem_object_put(obj: &bo->base.base); |
352 | } |
353 | |
354 | int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) |
355 | { |
356 | struct drm_ivpu_bo_info *args = data; |
357 | struct drm_gem_object *obj; |
358 | struct ivpu_bo *bo; |
359 | int ret = 0; |
360 | |
361 | obj = drm_gem_object_lookup(filp: file, handle: args->handle); |
362 | if (!obj) |
363 | return -ENOENT; |
364 | |
365 | bo = to_ivpu_bo(obj); |
366 | |
367 | mutex_lock(&bo->lock); |
368 | args->flags = bo->flags; |
369 | args->mmap_offset = drm_vma_node_offset_addr(node: &obj->vma_node); |
370 | args->vpu_addr = bo->vpu_addr; |
371 | args->size = obj->size; |
372 | mutex_unlock(lock: &bo->lock); |
373 | |
374 | drm_gem_object_put(obj); |
375 | return ret; |
376 | } |
377 | |
378 | int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) |
379 | { |
380 | struct drm_ivpu_bo_wait *args = data; |
381 | struct drm_gem_object *obj; |
382 | unsigned long timeout; |
383 | long ret; |
384 | |
385 | timeout = drm_timeout_abs_to_jiffies(timeout_nsec: args->timeout_ns); |
386 | |
387 | obj = drm_gem_object_lookup(filp: file, handle: args->handle); |
388 | if (!obj) |
389 | return -EINVAL; |
390 | |
391 | ret = dma_resv_wait_timeout(obj: obj->resv, usage: DMA_RESV_USAGE_READ, intr: true, timeout); |
392 | if (ret == 0) { |
393 | ret = -ETIMEDOUT; |
394 | } else if (ret > 0) { |
395 | ret = 0; |
396 | args->job_status = to_ivpu_bo(obj)->job_status; |
397 | } |
398 | |
399 | drm_gem_object_put(obj); |
400 | |
401 | return ret; |
402 | } |
403 | |
404 | static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) |
405 | { |
406 | mutex_lock(&bo->lock); |
407 | |
408 | drm_printf(p, f: "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u" , |
409 | bo, bo->ctx->id, bo->vpu_addr, bo->base.base.size, |
410 | bo->flags, kref_read(kref: &bo->base.base.refcount)); |
411 | |
412 | if (bo->base.pages) |
413 | drm_printf(p, f: " has_pages" ); |
414 | |
415 | if (bo->mmu_mapped) |
416 | drm_printf(p, f: " mmu_mapped" ); |
417 | |
418 | if (bo->base.base.import_attach) |
419 | drm_printf(p, f: " imported" ); |
420 | |
421 | drm_printf(p, f: "\n" ); |
422 | |
423 | mutex_unlock(lock: &bo->lock); |
424 | } |
425 | |
426 | void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) |
427 | { |
428 | struct ivpu_device *vdev = to_ivpu_device(dev); |
429 | struct ivpu_bo *bo; |
430 | |
431 | drm_printf(p, f: "%-9s %-3s %-14s %-10s %-10s %-4s %s\n" , |
432 | "bo" , "ctx" , "vpu_addr" , "size" , "flags" , "refs" , "attribs" ); |
433 | |
434 | mutex_lock(&vdev->bo_list_lock); |
435 | list_for_each_entry(bo, &vdev->bo_list, bo_list_node) |
436 | ivpu_bo_print_info(bo, p); |
437 | mutex_unlock(lock: &vdev->bo_list_lock); |
438 | } |
439 | |
440 | void ivpu_bo_list_print(struct drm_device *dev) |
441 | { |
442 | struct drm_printer p = drm_info_printer(dev: dev->dev); |
443 | |
444 | ivpu_bo_list(dev, p: &p); |
445 | } |
446 | |