1 | /* |
2 | * Copyright (C) 2008 Ben Skeggs. |
3 | * All Rights Reserved. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining |
6 | * a copy of this software and associated documentation files (the |
7 | * "Software"), to deal in the Software without restriction, including |
8 | * without limitation the rights to use, copy, modify, merge, publish, |
9 | * distribute, sublicense, and/or sell copies of the Software, and to |
10 | * permit persons to whom the Software is furnished to do so, subject to |
11 | * the following conditions: |
12 | * |
13 | * The above copyright notice and this permission notice (including the |
14 | * next paragraph) shall be included in all copies or substantial |
15 | * portions of the Software. |
16 | * |
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. |
20 | * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE |
21 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
22 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
24 | * |
25 | */ |
26 | |
27 | #include <drm/drm_gem_ttm_helper.h> |
28 | |
29 | #include "nouveau_drv.h" |
30 | #include "nouveau_dma.h" |
31 | #include "nouveau_fence.h" |
32 | #include "nouveau_abi16.h" |
33 | |
34 | #include "nouveau_ttm.h" |
35 | #include "nouveau_gem.h" |
36 | #include "nouveau_mem.h" |
37 | #include "nouveau_vmm.h" |
38 | |
39 | #include <nvif/class.h> |
40 | #include <nvif/push206e.h> |
41 | |
42 | static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf) |
43 | { |
44 | struct vm_area_struct *vma = vmf->vma; |
45 | struct ttm_buffer_object *bo = vma->vm_private_data; |
46 | pgprot_t prot; |
47 | vm_fault_t ret; |
48 | |
49 | ret = ttm_bo_vm_reserve(bo, vmf); |
50 | if (ret) |
51 | return ret; |
52 | |
53 | ret = nouveau_ttm_fault_reserve_notify(bo); |
54 | if (ret) |
55 | goto error_unlock; |
56 | |
57 | nouveau_bo_del_io_reserve_lru(bo); |
58 | prot = vm_get_page_prot(vm_flags: vma->vm_flags); |
59 | ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); |
60 | nouveau_bo_add_io_reserve_lru(bo); |
61 | if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) |
62 | return ret; |
63 | |
64 | error_unlock: |
65 | dma_resv_unlock(obj: bo->base.resv); |
66 | return ret; |
67 | } |
68 | |
69 | static const struct vm_operations_struct nouveau_ttm_vm_ops = { |
70 | .fault = nouveau_ttm_fault, |
71 | .open = ttm_bo_vm_open, |
72 | .close = ttm_bo_vm_close, |
73 | .access = ttm_bo_vm_access |
74 | }; |
75 | |
76 | void |
77 | nouveau_gem_object_del(struct drm_gem_object *gem) |
78 | { |
79 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
80 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
81 | struct device *dev = drm->dev->dev; |
82 | int ret; |
83 | |
84 | ret = pm_runtime_get_sync(dev); |
85 | if (WARN_ON(ret < 0 && ret != -EACCES)) { |
86 | pm_runtime_put_autosuspend(dev); |
87 | return; |
88 | } |
89 | |
90 | if (gem->import_attach) |
91 | drm_prime_gem_destroy(obj: gem, sg: nvbo->bo.sg); |
92 | |
93 | ttm_bo_put(bo: &nvbo->bo); |
94 | |
95 | pm_runtime_mark_last_busy(dev); |
96 | pm_runtime_put_autosuspend(dev); |
97 | } |
98 | |
99 | int |
100 | nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) |
101 | { |
102 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
103 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
104 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
105 | struct device *dev = drm->dev->dev; |
106 | struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli); |
107 | struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); |
108 | struct nouveau_vma *vma; |
109 | int ret; |
110 | |
111 | if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) |
112 | return 0; |
113 | |
114 | if (nvbo->no_share && uvmm && &uvmm->resv != nvbo->bo.base.resv) |
115 | return -EPERM; |
116 | |
117 | ret = ttm_bo_reserve(bo: &nvbo->bo, interruptible: false, no_wait: false, NULL); |
118 | if (ret) |
119 | return ret; |
120 | |
121 | ret = pm_runtime_get_sync(dev); |
122 | if (ret < 0 && ret != -EACCES) { |
123 | pm_runtime_put_autosuspend(dev); |
124 | goto out; |
125 | } |
126 | |
127 | /* only create a VMA on binding */ |
128 | if (!nouveau_cli_uvmm(cli)) |
129 | ret = nouveau_vma_new(nvbo, vmm, &vma); |
130 | else |
131 | ret = 0; |
132 | pm_runtime_mark_last_busy(dev); |
133 | pm_runtime_put_autosuspend(dev); |
134 | out: |
135 | ttm_bo_unreserve(bo: &nvbo->bo); |
136 | return ret; |
137 | } |
138 | |
139 | struct nouveau_gem_object_unmap { |
140 | struct nouveau_cli_work work; |
141 | struct nouveau_vma *vma; |
142 | }; |
143 | |
144 | static void |
145 | nouveau_gem_object_delete(struct nouveau_vma *vma) |
146 | { |
147 | nouveau_fence_unref(&vma->fence); |
148 | nouveau_vma_del(&vma); |
149 | } |
150 | |
151 | static void |
152 | nouveau_gem_object_delete_work(struct nouveau_cli_work *w) |
153 | { |
154 | struct nouveau_gem_object_unmap *work = |
155 | container_of(w, typeof(*work), work); |
156 | nouveau_gem_object_delete(vma: work->vma); |
157 | kfree(objp: work); |
158 | } |
159 | |
160 | static void |
161 | nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) |
162 | { |
163 | struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; |
164 | struct nouveau_gem_object_unmap *work; |
165 | |
166 | list_del_init(entry: &vma->head); |
167 | |
168 | if (!fence) { |
169 | nouveau_gem_object_delete(vma); |
170 | return; |
171 | } |
172 | |
173 | if (!(work = kmalloc(size: sizeof(*work), GFP_KERNEL))) { |
174 | WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); |
175 | nouveau_gem_object_delete(vma); |
176 | return; |
177 | } |
178 | |
179 | work->work.func = nouveau_gem_object_delete_work; |
180 | work->vma = vma; |
181 | nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); |
182 | } |
183 | |
184 | void |
185 | nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) |
186 | { |
187 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
188 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
189 | struct nouveau_drm *drm = nouveau_bdev(bd: nvbo->bo.bdev); |
190 | struct device *dev = drm->dev->dev; |
191 | struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); |
192 | struct nouveau_vma *vma; |
193 | int ret; |
194 | |
195 | if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) |
196 | return; |
197 | |
198 | if (nouveau_cli_uvmm(cli)) |
199 | return; |
200 | |
201 | ret = ttm_bo_reserve(bo: &nvbo->bo, interruptible: false, no_wait: false, NULL); |
202 | if (ret) |
203 | return; |
204 | |
205 | vma = nouveau_vma_find(nvbo, vmm); |
206 | if (vma) { |
207 | if (--vma->refs == 0) { |
208 | ret = pm_runtime_get_sync(dev); |
209 | if (!WARN_ON(ret < 0 && ret != -EACCES)) { |
210 | nouveau_gem_object_unmap(nvbo, vma); |
211 | pm_runtime_mark_last_busy(dev); |
212 | } |
213 | pm_runtime_put_autosuspend(dev); |
214 | } |
215 | } |
216 | ttm_bo_unreserve(bo: &nvbo->bo); |
217 | } |
218 | |
219 | const struct drm_gem_object_funcs nouveau_gem_object_funcs = { |
220 | .free = nouveau_gem_object_del, |
221 | .open = nouveau_gem_object_open, |
222 | .close = nouveau_gem_object_close, |
223 | .export = nouveau_gem_prime_export, |
224 | .pin = nouveau_gem_prime_pin, |
225 | .unpin = nouveau_gem_prime_unpin, |
226 | .get_sg_table = nouveau_gem_prime_get_sg_table, |
227 | .vmap = drm_gem_ttm_vmap, |
228 | .vunmap = drm_gem_ttm_vunmap, |
229 | .mmap = drm_gem_ttm_mmap, |
230 | .vm_ops = &nouveau_ttm_vm_ops, |
231 | }; |
232 | |
233 | int |
234 | nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, |
235 | uint32_t tile_mode, uint32_t tile_flags, |
236 | struct nouveau_bo **pnvbo) |
237 | { |
238 | struct nouveau_drm *drm = cli->drm; |
239 | struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli); |
240 | struct dma_resv *resv = NULL; |
241 | struct nouveau_bo *nvbo; |
242 | int ret; |
243 | |
244 | if (domain & NOUVEAU_GEM_DOMAIN_NO_SHARE) { |
245 | if (unlikely(!uvmm)) |
246 | return -EINVAL; |
247 | |
248 | resv = &uvmm->resv; |
249 | } |
250 | |
251 | if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) |
252 | domain |= NOUVEAU_GEM_DOMAIN_CPU; |
253 | |
254 | nvbo = nouveau_bo_alloc(cli, size: &size, align: &align, domain, tile_mode, |
255 | tile_flags, internal: false); |
256 | if (IS_ERR(ptr: nvbo)) |
257 | return PTR_ERR(ptr: nvbo); |
258 | |
259 | nvbo->bo.base.funcs = &nouveau_gem_object_funcs; |
260 | nvbo->no_share = domain & NOUVEAU_GEM_DOMAIN_NO_SHARE; |
261 | |
262 | /* Initialize the embedded gem-object. We return a single gem-reference |
263 | * to the caller, instead of a normal nouveau_bo ttm reference. */ |
264 | ret = drm_gem_object_init(dev: drm->dev, obj: &nvbo->bo.base, size); |
265 | if (ret) { |
266 | drm_gem_object_release(obj: &nvbo->bo.base); |
267 | kfree(objp: nvbo); |
268 | return ret; |
269 | } |
270 | |
271 | if (resv) |
272 | dma_resv_lock(obj: resv, NULL); |
273 | |
274 | ret = nouveau_bo_init(nvbo, size, align, domain, NULL, robj: resv); |
275 | |
276 | if (resv) |
277 | dma_resv_unlock(obj: resv); |
278 | |
279 | if (ret) |
280 | return ret; |
281 | |
282 | /* we restrict allowed domains on nv50+ to only the types |
283 | * that were requested at creation time. not possibly on |
284 | * earlier chips without busting the ABI. |
285 | */ |
286 | nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | |
287 | NOUVEAU_GEM_DOMAIN_GART; |
288 | if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) |
289 | nvbo->valid_domains &= domain; |
290 | |
291 | *pnvbo = nvbo; |
292 | return 0; |
293 | } |
294 | |
295 | static int |
296 | nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, |
297 | struct drm_nouveau_gem_info *rep) |
298 | { |
299 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
300 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
301 | struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); |
302 | struct nouveau_vma *vma; |
303 | |
304 | if (is_power_of_2(n: nvbo->valid_domains)) |
305 | rep->domain = nvbo->valid_domains; |
306 | else if (nvbo->bo.resource->mem_type == TTM_PL_TT) |
307 | rep->domain = NOUVEAU_GEM_DOMAIN_GART; |
308 | else |
309 | rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; |
310 | rep->offset = nvbo->offset; |
311 | if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50 && |
312 | !nouveau_cli_uvmm(cli)) { |
313 | vma = nouveau_vma_find(nvbo, vmm); |
314 | if (!vma) |
315 | return -EINVAL; |
316 | |
317 | rep->offset = vma->addr; |
318 | } else |
319 | rep->offset = 0; |
320 | |
321 | rep->size = nvbo->bo.base.size; |
322 | rep->map_handle = drm_vma_node_offset_addr(node: &nvbo->bo.base.vma_node); |
323 | rep->tile_mode = nvbo->mode; |
324 | rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; |
325 | if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) |
326 | rep->tile_flags |= nvbo->kind << 8; |
327 | else |
328 | if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) |
329 | rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; |
330 | else |
331 | rep->tile_flags |= nvbo->zeta; |
332 | return 0; |
333 | } |
334 | |
335 | int |
336 | nouveau_gem_ioctl_new(struct drm_device *dev, void *data, |
337 | struct drm_file *file_priv) |
338 | { |
339 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
340 | struct drm_nouveau_gem_new *req = data; |
341 | struct nouveau_bo *nvbo = NULL; |
342 | int ret = 0; |
343 | |
344 | /* If uvmm wasn't initialized until now disable it completely to prevent |
345 | * userspace from mixing up UAPIs. |
346 | */ |
347 | nouveau_cli_disable_uvmm_noinit(cli); |
348 | |
349 | ret = nouveau_gem_new(cli, size: req->info.size, align: req->align, |
350 | domain: req->info.domain, tile_mode: req->info.tile_mode, |
351 | tile_flags: req->info.tile_flags, pnvbo: &nvbo); |
352 | if (ret) |
353 | return ret; |
354 | |
355 | ret = drm_gem_handle_create(file_priv, obj: &nvbo->bo.base, |
356 | handlep: &req->info.handle); |
357 | if (ret == 0) { |
358 | ret = nouveau_gem_info(file_priv, gem: &nvbo->bo.base, rep: &req->info); |
359 | if (ret) |
360 | drm_gem_handle_delete(filp: file_priv, handle: req->info.handle); |
361 | } |
362 | |
363 | /* drop reference from allocate - handle holds it now */ |
364 | drm_gem_object_put(obj: &nvbo->bo.base); |
365 | return ret; |
366 | } |
367 | |
368 | static int |
369 | nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, |
370 | uint32_t write_domains, uint32_t valid_domains) |
371 | { |
372 | struct nouveau_bo *nvbo = nouveau_gem_object(gem); |
373 | struct ttm_buffer_object *bo = &nvbo->bo; |
374 | uint32_t domains = valid_domains & nvbo->valid_domains & |
375 | (write_domains ? write_domains : read_domains); |
376 | uint32_t pref_domains = 0; |
377 | |
378 | if (!domains) |
379 | return -EINVAL; |
380 | |
381 | valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART); |
382 | |
383 | if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && |
384 | bo->resource->mem_type == TTM_PL_VRAM) |
385 | pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; |
386 | |
387 | else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && |
388 | bo->resource->mem_type == TTM_PL_TT) |
389 | pref_domains |= NOUVEAU_GEM_DOMAIN_GART; |
390 | |
391 | else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) |
392 | pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; |
393 | |
394 | else |
395 | pref_domains |= NOUVEAU_GEM_DOMAIN_GART; |
396 | |
397 | nouveau_bo_placement_set(nvbo, type: pref_domains, busy: valid_domains); |
398 | |
399 | return 0; |
400 | } |
401 | |
402 | struct validate_op { |
403 | struct list_head list; |
404 | struct ww_acquire_ctx ticket; |
405 | }; |
406 | |
407 | static void |
408 | validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, |
409 | struct nouveau_fence *fence, |
410 | struct drm_nouveau_gem_pushbuf_bo *pbbo) |
411 | { |
412 | struct nouveau_bo *nvbo; |
413 | struct drm_nouveau_gem_pushbuf_bo *b; |
414 | |
415 | while (!list_empty(head: &op->list)) { |
416 | nvbo = list_entry(op->list.next, struct nouveau_bo, entry); |
417 | b = &pbbo[nvbo->pbbo_index]; |
418 | |
419 | if (likely(fence)) { |
420 | nouveau_bo_fence(nvbo, fence, exclusive: !!b->write_domains); |
421 | |
422 | if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { |
423 | struct nouveau_vma *vma = |
424 | (void *)(unsigned long)b->user_priv; |
425 | nouveau_fence_unref(&vma->fence); |
426 | dma_fence_get(fence: &fence->base); |
427 | vma->fence = fence; |
428 | } |
429 | } |
430 | |
431 | if (unlikely(nvbo->validate_mapped)) { |
432 | ttm_bo_kunmap(map: &nvbo->kmap); |
433 | nvbo->validate_mapped = false; |
434 | } |
435 | |
436 | list_del(entry: &nvbo->entry); |
437 | nvbo->reserved_by = NULL; |
438 | ttm_bo_unreserve(bo: &nvbo->bo); |
439 | drm_gem_object_put(obj: &nvbo->bo.base); |
440 | } |
441 | } |
442 | |
443 | static void |
444 | validate_fini(struct validate_op *op, struct nouveau_channel *chan, |
445 | struct nouveau_fence *fence, |
446 | struct drm_nouveau_gem_pushbuf_bo *pbbo) |
447 | { |
448 | validate_fini_no_ticket(op, chan, fence, pbbo); |
449 | ww_acquire_fini(ctx: &op->ticket); |
450 | } |
451 | |
452 | static int |
453 | validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, |
454 | struct drm_nouveau_gem_pushbuf_bo *pbbo, |
455 | int nr_buffers, struct validate_op *op) |
456 | { |
457 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
458 | int trycnt = 0; |
459 | int ret = -EINVAL, i; |
460 | struct nouveau_bo *res_bo = NULL; |
461 | LIST_HEAD(gart_list); |
462 | LIST_HEAD(vram_list); |
463 | LIST_HEAD(both_list); |
464 | |
465 | ww_acquire_init(ctx: &op->ticket, ww_class: &reservation_ww_class); |
466 | retry: |
467 | if (++trycnt > 100000) { |
468 | NV_PRINTK(err, cli, "%s failed and gave up.\n" , __func__); |
469 | return -EINVAL; |
470 | } |
471 | |
472 | for (i = 0; i < nr_buffers; i++) { |
473 | struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; |
474 | struct drm_gem_object *gem; |
475 | struct nouveau_bo *nvbo; |
476 | |
477 | gem = drm_gem_object_lookup(filp: file_priv, handle: b->handle); |
478 | if (!gem) { |
479 | NV_PRINTK(err, cli, "Unknown handle 0x%08x\n" , b->handle); |
480 | ret = -ENOENT; |
481 | break; |
482 | } |
483 | nvbo = nouveau_gem_object(gem); |
484 | if (nvbo == res_bo) { |
485 | res_bo = NULL; |
486 | drm_gem_object_put(obj: gem); |
487 | continue; |
488 | } |
489 | |
490 | if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { |
491 | NV_PRINTK(err, cli, "multiple instances of buffer %d on " |
492 | "validation list\n" , b->handle); |
493 | drm_gem_object_put(obj: gem); |
494 | ret = -EINVAL; |
495 | break; |
496 | } |
497 | |
498 | ret = ttm_bo_reserve(bo: &nvbo->bo, interruptible: true, no_wait: false, ticket: &op->ticket); |
499 | if (ret) { |
500 | list_splice_tail_init(list: &vram_list, head: &op->list); |
501 | list_splice_tail_init(list: &gart_list, head: &op->list); |
502 | list_splice_tail_init(list: &both_list, head: &op->list); |
503 | validate_fini_no_ticket(op, chan, NULL, NULL); |
504 | if (unlikely(ret == -EDEADLK)) { |
505 | ret = ttm_bo_reserve_slowpath(bo: &nvbo->bo, interruptible: true, |
506 | ticket: &op->ticket); |
507 | if (!ret) |
508 | res_bo = nvbo; |
509 | } |
510 | if (unlikely(ret)) { |
511 | if (ret != -ERESTARTSYS) |
512 | NV_PRINTK(err, cli, "fail reserve\n" ); |
513 | break; |
514 | } |
515 | } |
516 | |
517 | if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { |
518 | struct nouveau_vmm *vmm = chan->vmm; |
519 | struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); |
520 | if (!vma) { |
521 | NV_PRINTK(err, cli, "vma not found!\n" ); |
522 | ret = -EINVAL; |
523 | break; |
524 | } |
525 | |
526 | b->user_priv = (uint64_t)(unsigned long)vma; |
527 | } else { |
528 | b->user_priv = (uint64_t)(unsigned long)nvbo; |
529 | } |
530 | |
531 | nvbo->reserved_by = file_priv; |
532 | nvbo->pbbo_index = i; |
533 | if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && |
534 | (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) |
535 | list_add_tail(new: &nvbo->entry, head: &both_list); |
536 | else |
537 | if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) |
538 | list_add_tail(new: &nvbo->entry, head: &vram_list); |
539 | else |
540 | if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) |
541 | list_add_tail(new: &nvbo->entry, head: &gart_list); |
542 | else { |
543 | NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n" , |
544 | b->valid_domains); |
545 | list_add_tail(new: &nvbo->entry, head: &both_list); |
546 | ret = -EINVAL; |
547 | break; |
548 | } |
549 | if (nvbo == res_bo) |
550 | goto retry; |
551 | } |
552 | |
553 | ww_acquire_done(ctx: &op->ticket); |
554 | list_splice_tail(list: &vram_list, head: &op->list); |
555 | list_splice_tail(list: &gart_list, head: &op->list); |
556 | list_splice_tail(list: &both_list, head: &op->list); |
557 | if (ret) |
558 | validate_fini(op, chan, NULL, NULL); |
559 | return ret; |
560 | |
561 | } |
562 | |
563 | static int |
564 | validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, |
565 | struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) |
566 | { |
567 | struct nouveau_drm *drm = chan->drm; |
568 | struct nouveau_bo *nvbo; |
569 | int ret, relocs = 0; |
570 | |
571 | list_for_each_entry(nvbo, list, entry) { |
572 | struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; |
573 | |
574 | ret = nouveau_gem_set_domain(gem: &nvbo->bo.base, read_domains: b->read_domains, |
575 | write_domains: b->write_domains, |
576 | valid_domains: b->valid_domains); |
577 | if (unlikely(ret)) { |
578 | NV_PRINTK(err, cli, "fail set_domain\n" ); |
579 | return ret; |
580 | } |
581 | |
582 | ret = nouveau_bo_validate(nvbo, interruptible: true, no_wait_gpu: false); |
583 | if (unlikely(ret)) { |
584 | if (ret != -ERESTARTSYS) |
585 | NV_PRINTK(err, cli, "fail ttm_validate\n" ); |
586 | return ret; |
587 | } |
588 | |
589 | ret = nouveau_fence_sync(nvbo, chan, exclusive: !!b->write_domains, intr: true); |
590 | if (unlikely(ret)) { |
591 | if (ret != -ERESTARTSYS) |
592 | NV_PRINTK(err, cli, "fail post-validate sync\n" ); |
593 | return ret; |
594 | } |
595 | |
596 | if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { |
597 | if (nvbo->offset == b->presumed.offset && |
598 | ((nvbo->bo.resource->mem_type == TTM_PL_VRAM && |
599 | b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || |
600 | (nvbo->bo.resource->mem_type == TTM_PL_TT && |
601 | b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) |
602 | continue; |
603 | |
604 | if (nvbo->bo.resource->mem_type == TTM_PL_TT) |
605 | b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; |
606 | else |
607 | b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; |
608 | b->presumed.offset = nvbo->offset; |
609 | b->presumed.valid = 0; |
610 | relocs++; |
611 | } |
612 | } |
613 | |
614 | return relocs; |
615 | } |
616 | |
617 | static int |
618 | nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, |
619 | struct drm_file *file_priv, |
620 | struct drm_nouveau_gem_pushbuf_bo *pbbo, |
621 | int nr_buffers, |
622 | struct validate_op *op, bool *apply_relocs) |
623 | { |
624 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
625 | int ret; |
626 | |
627 | INIT_LIST_HEAD(list: &op->list); |
628 | |
629 | if (nr_buffers == 0) |
630 | return 0; |
631 | |
632 | ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); |
633 | if (unlikely(ret)) { |
634 | if (ret != -ERESTARTSYS) |
635 | NV_PRINTK(err, cli, "validate_init\n" ); |
636 | return ret; |
637 | } |
638 | |
639 | ret = validate_list(chan, cli, list: &op->list, pbbo); |
640 | if (unlikely(ret < 0)) { |
641 | if (ret != -ERESTARTSYS) |
642 | NV_PRINTK(err, cli, "validating bo list\n" ); |
643 | validate_fini(op, chan, NULL, NULL); |
644 | return ret; |
645 | } else if (ret > 0) { |
646 | *apply_relocs = true; |
647 | } |
648 | |
649 | return 0; |
650 | } |
651 | |
652 | static int |
653 | nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, |
654 | struct drm_nouveau_gem_pushbuf *req, |
655 | struct drm_nouveau_gem_pushbuf_reloc *reloc, |
656 | struct drm_nouveau_gem_pushbuf_bo *bo) |
657 | { |
658 | int ret = 0; |
659 | unsigned i; |
660 | |
661 | for (i = 0; i < req->nr_relocs; i++) { |
662 | struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; |
663 | struct drm_nouveau_gem_pushbuf_bo *b; |
664 | struct nouveau_bo *nvbo; |
665 | uint32_t data; |
666 | long lret; |
667 | |
668 | if (unlikely(r->bo_index >= req->nr_buffers)) { |
669 | NV_PRINTK(err, cli, "reloc bo index invalid\n" ); |
670 | ret = -EINVAL; |
671 | break; |
672 | } |
673 | |
674 | b = &bo[r->bo_index]; |
675 | if (b->presumed.valid) |
676 | continue; |
677 | |
678 | if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { |
679 | NV_PRINTK(err, cli, "reloc container bo index invalid\n" ); |
680 | ret = -EINVAL; |
681 | break; |
682 | } |
683 | nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; |
684 | |
685 | if (unlikely(r->reloc_bo_offset + 4 > |
686 | nvbo->bo.base.size)) { |
687 | NV_PRINTK(err, cli, "reloc outside of bo\n" ); |
688 | ret = -EINVAL; |
689 | break; |
690 | } |
691 | |
692 | if (!nvbo->kmap.virtual) { |
693 | ret = ttm_bo_kmap(bo: &nvbo->bo, start_page: 0, PFN_UP(nvbo->bo.base.size), |
694 | map: &nvbo->kmap); |
695 | if (ret) { |
696 | NV_PRINTK(err, cli, "failed kmap for reloc\n" ); |
697 | break; |
698 | } |
699 | nvbo->validate_mapped = true; |
700 | } |
701 | |
702 | if (r->flags & NOUVEAU_GEM_RELOC_LOW) |
703 | data = b->presumed.offset + r->data; |
704 | else |
705 | if (r->flags & NOUVEAU_GEM_RELOC_HIGH) |
706 | data = (b->presumed.offset + r->data) >> 32; |
707 | else |
708 | data = r->data; |
709 | |
710 | if (r->flags & NOUVEAU_GEM_RELOC_OR) { |
711 | if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) |
712 | data |= r->tor; |
713 | else |
714 | data |= r->vor; |
715 | } |
716 | |
717 | lret = dma_resv_wait_timeout(obj: nvbo->bo.base.resv, |
718 | usage: DMA_RESV_USAGE_BOOKKEEP, |
719 | intr: false, timeout: 15 * HZ); |
720 | if (!lret) |
721 | ret = -EBUSY; |
722 | else if (lret > 0) |
723 | ret = 0; |
724 | else |
725 | ret = lret; |
726 | |
727 | if (ret) { |
728 | NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n" , |
729 | ret); |
730 | break; |
731 | } |
732 | |
733 | nouveau_bo_wr32(nvbo, index: r->reloc_bo_offset >> 2, val: data); |
734 | } |
735 | |
736 | return ret; |
737 | } |
738 | |
739 | int |
740 | nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, |
741 | struct drm_file *file_priv) |
742 | { |
743 | struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); |
744 | struct nouveau_cli *cli = nouveau_cli(fpriv: file_priv); |
745 | struct nouveau_abi16_chan *temp; |
746 | struct nouveau_drm *drm = nouveau_drm(dev); |
747 | struct drm_nouveau_gem_pushbuf *req = data; |
748 | struct drm_nouveau_gem_pushbuf_push *push; |
749 | struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; |
750 | struct drm_nouveau_gem_pushbuf_bo *bo; |
751 | struct nouveau_channel *chan = NULL; |
752 | struct validate_op op; |
753 | struct nouveau_fence *fence = NULL; |
754 | int i, j, ret = 0; |
755 | bool do_reloc = false, sync = false; |
756 | |
757 | if (unlikely(!abi16)) |
758 | return -ENOMEM; |
759 | |
760 | if (unlikely(nouveau_cli_uvmm(cli))) |
761 | return -ENOSYS; |
762 | |
763 | list_for_each_entry(temp, &abi16->channels, head) { |
764 | if (temp->chan->chid == req->channel) { |
765 | chan = temp->chan; |
766 | break; |
767 | } |
768 | } |
769 | |
770 | if (!chan) |
771 | return nouveau_abi16_put(abi16, -ENOENT); |
772 | if (unlikely(atomic_read(&chan->killed))) |
773 | return nouveau_abi16_put(abi16, -ENODEV); |
774 | |
775 | sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; |
776 | |
777 | req->vram_available = drm->gem.vram_available; |
778 | req->gart_available = drm->gem.gart_available; |
779 | if (unlikely(req->nr_push == 0)) |
780 | goto out_next; |
781 | |
782 | if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { |
783 | NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n" , |
784 | req->nr_push, NOUVEAU_GEM_MAX_PUSH); |
785 | return nouveau_abi16_put(abi16, -EINVAL); |
786 | } |
787 | |
788 | if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { |
789 | NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n" , |
790 | req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); |
791 | return nouveau_abi16_put(abi16, -EINVAL); |
792 | } |
793 | |
794 | if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { |
795 | NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n" , |
796 | req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); |
797 | return nouveau_abi16_put(abi16, -EINVAL); |
798 | } |
799 | |
800 | push = u_memcpya(user: req->push, nmemb: req->nr_push, size: sizeof(*push)); |
801 | if (IS_ERR(ptr: push)) |
802 | return nouveau_abi16_put(abi16, PTR_ERR(ptr: push)); |
803 | |
804 | bo = u_memcpya(user: req->buffers, nmemb: req->nr_buffers, size: sizeof(*bo)); |
805 | if (IS_ERR(ptr: bo)) { |
806 | u_free(addr: push); |
807 | return nouveau_abi16_put(abi16, PTR_ERR(ptr: bo)); |
808 | } |
809 | |
810 | /* Ensure all push buffers are on validate list */ |
811 | for (i = 0; i < req->nr_push; i++) { |
812 | if (push[i].bo_index >= req->nr_buffers) { |
813 | NV_PRINTK(err, cli, "push %d buffer not in list\n" , i); |
814 | ret = -EINVAL; |
815 | goto out_prevalid; |
816 | } |
817 | } |
818 | |
819 | /* Validate buffer list */ |
820 | revalidate: |
821 | ret = nouveau_gem_pushbuf_validate(chan, file_priv, pbbo: bo, |
822 | nr_buffers: req->nr_buffers, op: &op, apply_relocs: &do_reloc); |
823 | if (ret) { |
824 | if (ret != -ERESTARTSYS) |
825 | NV_PRINTK(err, cli, "validate: %d\n" , ret); |
826 | goto out_prevalid; |
827 | } |
828 | |
829 | /* Apply any relocations that are required */ |
830 | if (do_reloc) { |
831 | if (!reloc) { |
832 | validate_fini(op: &op, chan, NULL, pbbo: bo); |
833 | reloc = u_memcpya(user: req->relocs, nmemb: req->nr_relocs, size: sizeof(*reloc)); |
834 | if (IS_ERR(ptr: reloc)) { |
835 | ret = PTR_ERR(ptr: reloc); |
836 | goto out_prevalid; |
837 | } |
838 | |
839 | goto revalidate; |
840 | } |
841 | |
842 | ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); |
843 | if (ret) { |
844 | NV_PRINTK(err, cli, "reloc apply: %d\n" , ret); |
845 | goto out; |
846 | } |
847 | } |
848 | |
849 | if (chan->dma.ib_max) { |
850 | ret = nouveau_dma_wait(chan, slots: req->nr_push + 1, size: 16); |
851 | if (ret) { |
852 | NV_PRINTK(err, cli, "nv50cal_space: %d\n" , ret); |
853 | goto out; |
854 | } |
855 | |
856 | for (i = 0; i < req->nr_push; i++) { |
857 | struct nouveau_vma *vma = (void *)(unsigned long) |
858 | bo[push[i].bo_index].user_priv; |
859 | u64 addr = vma->addr + push[i].offset; |
860 | u32 length = push[i].length & ~NOUVEAU_GEM_PUSHBUF_NO_PREFETCH; |
861 | bool no_prefetch = push[i].length & NOUVEAU_GEM_PUSHBUF_NO_PREFETCH; |
862 | |
863 | nv50_dma_push(chan, addr, length, no_prefetch); |
864 | } |
865 | } else |
866 | if (drm->client.device.info.chipset >= 0x25) { |
867 | ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); |
868 | if (ret) { |
869 | NV_PRINTK(err, cli, "cal_space: %d\n" , ret); |
870 | goto out; |
871 | } |
872 | |
873 | for (i = 0; i < req->nr_push; i++) { |
874 | struct nouveau_bo *nvbo = (void *)(unsigned long) |
875 | bo[push[i].bo_index].user_priv; |
876 | |
877 | PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); |
878 | PUSH_DATA(chan->chan.push, 0); |
879 | } |
880 | } else { |
881 | ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); |
882 | if (ret) { |
883 | NV_PRINTK(err, cli, "jmp_space: %d\n" , ret); |
884 | goto out; |
885 | } |
886 | |
887 | for (i = 0; i < req->nr_push; i++) { |
888 | struct nouveau_bo *nvbo = (void *)(unsigned long) |
889 | bo[push[i].bo_index].user_priv; |
890 | uint32_t cmd; |
891 | |
892 | cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); |
893 | cmd |= 0x20000000; |
894 | if (unlikely(cmd != req->suffix0)) { |
895 | if (!nvbo->kmap.virtual) { |
896 | ret = ttm_bo_kmap(bo: &nvbo->bo, start_page: 0, |
897 | PFN_UP(nvbo->bo.base.size), |
898 | map: &nvbo->kmap); |
899 | if (ret) { |
900 | WIND_RING(chan); |
901 | goto out; |
902 | } |
903 | nvbo->validate_mapped = true; |
904 | } |
905 | |
906 | nouveau_bo_wr32(nvbo, index: (push[i].offset + |
907 | push[i].length - 8) / 4, val: cmd); |
908 | } |
909 | |
910 | PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); |
911 | PUSH_DATA(chan->chan.push, 0); |
912 | for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) |
913 | PUSH_DATA(chan->chan.push, 0); |
914 | } |
915 | } |
916 | |
917 | ret = nouveau_fence_new(&fence, chan); |
918 | if (ret) { |
919 | NV_PRINTK(err, cli, "error fencing pushbuf: %d\n" , ret); |
920 | WIND_RING(chan); |
921 | goto out; |
922 | } |
923 | |
924 | if (sync) { |
925 | if (!(ret = nouveau_fence_wait(fence, lazy: false, intr: false))) { |
926 | if ((ret = dma_fence_get_status(fence: &fence->base)) == 1) |
927 | ret = 0; |
928 | } |
929 | } |
930 | |
931 | out: |
932 | validate_fini(op: &op, chan, fence, pbbo: bo); |
933 | nouveau_fence_unref(&fence); |
934 | |
935 | if (do_reloc) { |
936 | struct drm_nouveau_gem_pushbuf_bo __user *upbbo = |
937 | u64_to_user_ptr(req->buffers); |
938 | |
939 | for (i = 0; i < req->nr_buffers; i++) { |
940 | if (bo[i].presumed.valid) |
941 | continue; |
942 | |
943 | if (copy_to_user(to: &upbbo[i].presumed, from: &bo[i].presumed, |
944 | n: sizeof(bo[i].presumed))) { |
945 | ret = -EFAULT; |
946 | break; |
947 | } |
948 | } |
949 | } |
950 | out_prevalid: |
951 | if (!IS_ERR(ptr: reloc)) |
952 | u_free(addr: reloc); |
953 | u_free(addr: bo); |
954 | u_free(addr: push); |
955 | |
956 | out_next: |
957 | if (chan->dma.ib_max) { |
958 | req->suffix0 = 0x00000000; |
959 | req->suffix1 = 0x00000000; |
960 | } else |
961 | if (drm->client.device.info.chipset >= 0x25) { |
962 | req->suffix0 = 0x00020000; |
963 | req->suffix1 = 0x00000000; |
964 | } else { |
965 | req->suffix0 = 0x20000000 | |
966 | (chan->push.addr + ((chan->dma.cur + 2) << 2)); |
967 | req->suffix1 = 0x00000000; |
968 | } |
969 | |
970 | return nouveau_abi16_put(abi16, ret); |
971 | } |
972 | |
973 | int |
974 | nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, |
975 | struct drm_file *file_priv) |
976 | { |
977 | struct drm_nouveau_gem_cpu_prep *req = data; |
978 | struct drm_gem_object *gem; |
979 | struct nouveau_bo *nvbo; |
980 | bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); |
981 | bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); |
982 | long lret; |
983 | int ret; |
984 | |
985 | gem = drm_gem_object_lookup(filp: file_priv, handle: req->handle); |
986 | if (!gem) |
987 | return -ENOENT; |
988 | nvbo = nouveau_gem_object(gem); |
989 | |
990 | lret = dma_resv_wait_timeout(obj: nvbo->bo.base.resv, |
991 | usage: dma_resv_usage_rw(write), intr: true, |
992 | timeout: no_wait ? 0 : 30 * HZ); |
993 | if (!lret) |
994 | ret = -EBUSY; |
995 | else if (lret > 0) |
996 | ret = 0; |
997 | else |
998 | ret = lret; |
999 | |
1000 | nouveau_bo_sync_for_cpu(nvbo); |
1001 | drm_gem_object_put(obj: gem); |
1002 | |
1003 | return ret; |
1004 | } |
1005 | |
1006 | int |
1007 | nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, |
1008 | struct drm_file *file_priv) |
1009 | { |
1010 | struct drm_nouveau_gem_cpu_fini *req = data; |
1011 | struct drm_gem_object *gem; |
1012 | struct nouveau_bo *nvbo; |
1013 | |
1014 | gem = drm_gem_object_lookup(filp: file_priv, handle: req->handle); |
1015 | if (!gem) |
1016 | return -ENOENT; |
1017 | nvbo = nouveau_gem_object(gem); |
1018 | |
1019 | nouveau_bo_sync_for_device(nvbo); |
1020 | drm_gem_object_put(obj: gem); |
1021 | return 0; |
1022 | } |
1023 | |
1024 | int |
1025 | nouveau_gem_ioctl_info(struct drm_device *dev, void *data, |
1026 | struct drm_file *file_priv) |
1027 | { |
1028 | struct drm_nouveau_gem_info *req = data; |
1029 | struct drm_gem_object *gem; |
1030 | int ret; |
1031 | |
1032 | gem = drm_gem_object_lookup(filp: file_priv, handle: req->handle); |
1033 | if (!gem) |
1034 | return -ENOENT; |
1035 | |
1036 | ret = nouveau_gem_info(file_priv, gem, rep: req); |
1037 | drm_gem_object_put(obj: gem); |
1038 | return ret; |
1039 | } |
1040 | |
1041 | |