1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ |
3 | |
4 | #include <linux/mm.h> |
5 | #include <linux/iosys-map.h> |
6 | #include <linux/sync_file.h> |
7 | #include <linux/pagemap.h> |
8 | #include <linux/shmem_fs.h> |
9 | #include <linux/dma-mapping.h> |
10 | |
11 | #include <drm/drm_file.h> |
12 | #include <drm/drm_syncobj.h> |
13 | #include <drm/drm_utils.h> |
14 | |
15 | #include <drm/lima_drm.h> |
16 | |
17 | #include "lima_drv.h" |
18 | #include "lima_gem.h" |
19 | #include "lima_vm.h" |
20 | |
21 | int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) |
22 | { |
23 | struct page **pages; |
24 | struct address_space *mapping = bo->base.base.filp->f_mapping; |
25 | struct device *dev = bo->base.base.dev->dev; |
26 | size_t old_size = bo->heap_size; |
27 | size_t new_size = bo->heap_size ? bo->heap_size * 2 : |
28 | (lima_heap_init_nr_pages << PAGE_SHIFT); |
29 | struct sg_table sgt; |
30 | int i, ret; |
31 | |
32 | if (bo->heap_size >= bo->base.base.size) |
33 | return -ENOSPC; |
34 | |
35 | new_size = min(new_size, bo->base.base.size); |
36 | |
37 | dma_resv_lock(obj: bo->base.base.resv, NULL); |
38 | |
39 | if (bo->base.pages) { |
40 | pages = bo->base.pages; |
41 | } else { |
42 | pages = kvmalloc_array(n: bo->base.base.size >> PAGE_SHIFT, |
43 | size: sizeof(*pages), GFP_KERNEL | __GFP_ZERO); |
44 | if (!pages) { |
45 | dma_resv_unlock(obj: bo->base.base.resv); |
46 | return -ENOMEM; |
47 | } |
48 | |
49 | bo->base.pages = pages; |
50 | bo->base.pages_use_count = 1; |
51 | |
52 | mapping_set_unevictable(mapping); |
53 | } |
54 | |
55 | for (i = old_size >> PAGE_SHIFT; i < new_size >> PAGE_SHIFT; i++) { |
56 | struct page *page = shmem_read_mapping_page(mapping, index: i); |
57 | |
58 | if (IS_ERR(ptr: page)) { |
59 | dma_resv_unlock(obj: bo->base.base.resv); |
60 | return PTR_ERR(ptr: page); |
61 | } |
62 | pages[i] = page; |
63 | } |
64 | |
65 | dma_resv_unlock(obj: bo->base.base.resv); |
66 | |
67 | ret = sg_alloc_table_from_pages(sgt: &sgt, pages, n_pages: i, offset: 0, |
68 | size: new_size, GFP_KERNEL); |
69 | if (ret) |
70 | return ret; |
71 | |
72 | if (bo->base.sgt) { |
73 | dma_unmap_sgtable(dev, sgt: bo->base.sgt, dir: DMA_BIDIRECTIONAL, attrs: 0); |
74 | sg_free_table(bo->base.sgt); |
75 | } else { |
76 | bo->base.sgt = kmalloc(size: sizeof(*bo->base.sgt), GFP_KERNEL); |
77 | if (!bo->base.sgt) { |
78 | sg_free_table(&sgt); |
79 | return -ENOMEM; |
80 | } |
81 | } |
82 | |
83 | ret = dma_map_sgtable(dev, sgt: &sgt, dir: DMA_BIDIRECTIONAL, attrs: 0); |
84 | if (ret) { |
85 | sg_free_table(&sgt); |
86 | kfree(objp: bo->base.sgt); |
87 | bo->base.sgt = NULL; |
88 | return ret; |
89 | } |
90 | |
91 | *bo->base.sgt = sgt; |
92 | |
93 | if (vm) { |
94 | ret = lima_vm_map_bo(vm, bo, pageoff: old_size >> PAGE_SHIFT); |
95 | if (ret) |
96 | return ret; |
97 | } |
98 | |
99 | bo->heap_size = new_size; |
100 | return 0; |
101 | } |
102 | |
103 | int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file, |
104 | u32 size, u32 flags, u32 *handle) |
105 | { |
106 | int err; |
107 | gfp_t mask; |
108 | struct drm_gem_shmem_object *shmem; |
109 | struct drm_gem_object *obj; |
110 | struct lima_bo *bo; |
111 | bool is_heap = flags & LIMA_BO_FLAG_HEAP; |
112 | |
113 | shmem = drm_gem_shmem_create(dev, size); |
114 | if (IS_ERR(ptr: shmem)) |
115 | return PTR_ERR(ptr: shmem); |
116 | |
117 | obj = &shmem->base; |
118 | |
119 | /* Mali Utgard GPU can only support 32bit address space */ |
120 | mask = mapping_gfp_mask(mapping: obj->filp->f_mapping); |
121 | mask &= ~__GFP_HIGHMEM; |
122 | mask |= __GFP_DMA32; |
123 | mapping_set_gfp_mask(m: obj->filp->f_mapping, mask); |
124 | |
125 | if (is_heap) { |
126 | bo = to_lima_bo(obj); |
127 | err = lima_heap_alloc(bo, NULL); |
128 | if (err) |
129 | goto out; |
130 | } else { |
131 | struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(shmem); |
132 | |
133 | if (IS_ERR(ptr: sgt)) { |
134 | err = PTR_ERR(ptr: sgt); |
135 | goto out; |
136 | } |
137 | } |
138 | |
139 | err = drm_gem_handle_create(file_priv: file, obj, handlep: handle); |
140 | |
141 | out: |
142 | /* drop reference from allocate - handle holds it now */ |
143 | drm_gem_object_put(obj); |
144 | |
145 | return err; |
146 | } |
147 | |
148 | static void lima_gem_free_object(struct drm_gem_object *obj) |
149 | { |
150 | struct lima_bo *bo = to_lima_bo(obj); |
151 | |
152 | if (!list_empty(head: &bo->va)) |
153 | dev_err(obj->dev->dev, "lima gem free bo still has va\n" ); |
154 | |
155 | drm_gem_shmem_free(shmem: &bo->base); |
156 | } |
157 | |
158 | static int lima_gem_object_open(struct drm_gem_object *obj, struct drm_file *file) |
159 | { |
160 | struct lima_bo *bo = to_lima_bo(obj); |
161 | struct lima_drm_priv *priv = to_lima_drm_priv(file); |
162 | struct lima_vm *vm = priv->vm; |
163 | |
164 | return lima_vm_bo_add(vm, bo, create: true); |
165 | } |
166 | |
167 | static void lima_gem_object_close(struct drm_gem_object *obj, struct drm_file *file) |
168 | { |
169 | struct lima_bo *bo = to_lima_bo(obj); |
170 | struct lima_drm_priv *priv = to_lima_drm_priv(file); |
171 | struct lima_vm *vm = priv->vm; |
172 | |
173 | lima_vm_bo_del(vm, bo); |
174 | } |
175 | |
176 | static int lima_gem_pin(struct drm_gem_object *obj) |
177 | { |
178 | struct lima_bo *bo = to_lima_bo(obj); |
179 | |
180 | if (bo->heap_size) |
181 | return -EINVAL; |
182 | |
183 | return drm_gem_shmem_pin(shmem: &bo->base); |
184 | } |
185 | |
186 | static int lima_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map) |
187 | { |
188 | struct lima_bo *bo = to_lima_bo(obj); |
189 | |
190 | if (bo->heap_size) |
191 | return -EINVAL; |
192 | |
193 | return drm_gem_shmem_vmap(shmem: &bo->base, map); |
194 | } |
195 | |
196 | static int lima_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) |
197 | { |
198 | struct lima_bo *bo = to_lima_bo(obj); |
199 | |
200 | if (bo->heap_size) |
201 | return -EINVAL; |
202 | |
203 | return drm_gem_shmem_mmap(shmem: &bo->base, vma); |
204 | } |
205 | |
206 | static const struct drm_gem_object_funcs lima_gem_funcs = { |
207 | .free = lima_gem_free_object, |
208 | .open = lima_gem_object_open, |
209 | .close = lima_gem_object_close, |
210 | .print_info = drm_gem_shmem_object_print_info, |
211 | .pin = lima_gem_pin, |
212 | .unpin = drm_gem_shmem_object_unpin, |
213 | .get_sg_table = drm_gem_shmem_object_get_sg_table, |
214 | .vmap = lima_gem_vmap, |
215 | .vunmap = drm_gem_shmem_object_vunmap, |
216 | .mmap = lima_gem_mmap, |
217 | .vm_ops = &drm_gem_shmem_vm_ops, |
218 | }; |
219 | |
220 | struct drm_gem_object *lima_gem_create_object(struct drm_device *dev, size_t size) |
221 | { |
222 | struct lima_bo *bo; |
223 | |
224 | bo = kzalloc(size: sizeof(*bo), GFP_KERNEL); |
225 | if (!bo) |
226 | return ERR_PTR(error: -ENOMEM); |
227 | |
228 | mutex_init(&bo->lock); |
229 | INIT_LIST_HEAD(list: &bo->va); |
230 | bo->base.map_wc = true; |
231 | bo->base.base.funcs = &lima_gem_funcs; |
232 | |
233 | return &bo->base.base; |
234 | } |
235 | |
236 | int lima_gem_get_info(struct drm_file *file, u32 handle, u32 *va, u64 *offset) |
237 | { |
238 | struct drm_gem_object *obj; |
239 | struct lima_bo *bo; |
240 | struct lima_drm_priv *priv = to_lima_drm_priv(file); |
241 | struct lima_vm *vm = priv->vm; |
242 | |
243 | obj = drm_gem_object_lookup(filp: file, handle); |
244 | if (!obj) |
245 | return -ENOENT; |
246 | |
247 | bo = to_lima_bo(obj); |
248 | |
249 | *va = lima_vm_get_va(vm, bo); |
250 | |
251 | *offset = drm_vma_node_offset_addr(node: &obj->vma_node); |
252 | |
253 | drm_gem_object_put(obj); |
254 | return 0; |
255 | } |
256 | |
257 | static int lima_gem_sync_bo(struct lima_sched_task *task, struct lima_bo *bo, |
258 | bool write, bool explicit) |
259 | { |
260 | int err; |
261 | |
262 | err = dma_resv_reserve_fences(obj: lima_bo_resv(bo), num_fences: 1); |
263 | if (err) |
264 | return err; |
265 | |
266 | /* explicit sync use user passed dep fence */ |
267 | if (explicit) |
268 | return 0; |
269 | |
270 | return drm_sched_job_add_implicit_dependencies(job: &task->base, |
271 | obj: &bo->base.base, |
272 | write); |
273 | } |
274 | |
275 | static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit) |
276 | { |
277 | int i, err; |
278 | |
279 | for (i = 0; i < ARRAY_SIZE(submit->in_sync); i++) { |
280 | if (!submit->in_sync[i]) |
281 | continue; |
282 | |
283 | err = drm_sched_job_add_syncobj_dependency(job: &submit->task->base, file, |
284 | handle: submit->in_sync[i], point: 0); |
285 | if (err) |
286 | return err; |
287 | } |
288 | |
289 | return 0; |
290 | } |
291 | |
292 | int lima_gem_submit(struct drm_file *file, struct lima_submit *submit) |
293 | { |
294 | int i, err = 0; |
295 | struct ww_acquire_ctx ctx; |
296 | struct lima_drm_priv *priv = to_lima_drm_priv(file); |
297 | struct lima_vm *vm = priv->vm; |
298 | struct drm_syncobj *out_sync = NULL; |
299 | struct dma_fence *fence; |
300 | struct lima_bo **bos = submit->lbos; |
301 | |
302 | if (submit->out_sync) { |
303 | out_sync = drm_syncobj_find(file_private: file, handle: submit->out_sync); |
304 | if (!out_sync) |
305 | return -ENOENT; |
306 | } |
307 | |
308 | for (i = 0; i < submit->nr_bos; i++) { |
309 | struct drm_gem_object *obj; |
310 | struct lima_bo *bo; |
311 | |
312 | obj = drm_gem_object_lookup(filp: file, handle: submit->bos[i].handle); |
313 | if (!obj) { |
314 | err = -ENOENT; |
315 | goto err_out0; |
316 | } |
317 | |
318 | bo = to_lima_bo(obj); |
319 | |
320 | /* increase refcnt of gpu va map to prevent unmapped when executing, |
321 | * will be decreased when task done |
322 | */ |
323 | err = lima_vm_bo_add(vm, bo, create: false); |
324 | if (err) { |
325 | drm_gem_object_put(obj); |
326 | goto err_out0; |
327 | } |
328 | |
329 | bos[i] = bo; |
330 | } |
331 | |
332 | err = drm_gem_lock_reservations(objs: (struct drm_gem_object **)bos, |
333 | count: submit->nr_bos, acquire_ctx: &ctx); |
334 | if (err) |
335 | goto err_out0; |
336 | |
337 | err = lima_sched_task_init( |
338 | task: submit->task, context: submit->ctx->context + submit->pipe, |
339 | bos, num_bos: submit->nr_bos, vm); |
340 | if (err) |
341 | goto err_out1; |
342 | |
343 | err = lima_gem_add_deps(file, submit); |
344 | if (err) |
345 | goto err_out2; |
346 | |
347 | for (i = 0; i < submit->nr_bos; i++) { |
348 | err = lima_gem_sync_bo( |
349 | task: submit->task, bo: bos[i], |
350 | write: submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE, |
351 | explicit: submit->flags & LIMA_SUBMIT_FLAG_EXPLICIT_FENCE); |
352 | if (err) |
353 | goto err_out2; |
354 | } |
355 | |
356 | fence = lima_sched_context_queue_task(task: submit->task); |
357 | |
358 | for (i = 0; i < submit->nr_bos; i++) { |
359 | dma_resv_add_fence(obj: lima_bo_resv(bo: bos[i]), fence, |
360 | usage: submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE ? |
361 | DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ); |
362 | } |
363 | |
364 | drm_gem_unlock_reservations(objs: (struct drm_gem_object **)bos, |
365 | count: submit->nr_bos, acquire_ctx: &ctx); |
366 | |
367 | for (i = 0; i < submit->nr_bos; i++) |
368 | drm_gem_object_put(obj: &bos[i]->base.base); |
369 | |
370 | if (out_sync) { |
371 | drm_syncobj_replace_fence(syncobj: out_sync, fence); |
372 | drm_syncobj_put(obj: out_sync); |
373 | } |
374 | |
375 | dma_fence_put(fence); |
376 | |
377 | return 0; |
378 | |
379 | err_out2: |
380 | lima_sched_task_fini(task: submit->task); |
381 | err_out1: |
382 | drm_gem_unlock_reservations(objs: (struct drm_gem_object **)bos, |
383 | count: submit->nr_bos, acquire_ctx: &ctx); |
384 | err_out0: |
385 | for (i = 0; i < submit->nr_bos; i++) { |
386 | if (!bos[i]) |
387 | break; |
388 | lima_vm_bo_del(vm, bo: bos[i]); |
389 | drm_gem_object_put(obj: &bos[i]->base.base); |
390 | } |
391 | if (out_sync) |
392 | drm_syncobj_put(obj: out_sync); |
393 | return err; |
394 | } |
395 | |
396 | int lima_gem_wait(struct drm_file *file, u32 handle, u32 op, s64 timeout_ns) |
397 | { |
398 | bool write = op & LIMA_GEM_WAIT_WRITE; |
399 | long ret, timeout; |
400 | |
401 | if (!op) |
402 | return 0; |
403 | |
404 | timeout = drm_timeout_abs_to_jiffies(timeout_nsec: timeout_ns); |
405 | |
406 | ret = drm_gem_dma_resv_wait(filep: file, handle, wait_all: write, timeout); |
407 | if (ret == -ETIME) |
408 | ret = timeout ? -ETIMEDOUT : -EBUSY; |
409 | |
410 | return ret; |
411 | } |
412 | |