1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (c) 2020 NVIDIA Corporation */ |
3 | |
4 | #include <linux/dma-fence-array.h> |
5 | #include <linux/dma-mapping.h> |
6 | #include <linux/file.h> |
7 | #include <linux/host1x.h> |
8 | #include <linux/iommu.h> |
9 | #include <linux/kref.h> |
10 | #include <linux/list.h> |
11 | #include <linux/nospec.h> |
12 | #include <linux/pm_runtime.h> |
13 | #include <linux/scatterlist.h> |
14 | #include <linux/slab.h> |
15 | #include <linux/sync_file.h> |
16 | |
17 | #include <drm/drm_drv.h> |
18 | #include <drm/drm_file.h> |
19 | #include <drm/drm_syncobj.h> |
20 | |
21 | #include "drm.h" |
22 | #include "gem.h" |
23 | #include "submit.h" |
24 | #include "uapi.h" |
25 | |
26 | #define SUBMIT_ERR(context, fmt, ...) \ |
27 | dev_err_ratelimited(context->client->base.dev, \ |
28 | "%s: job submission failed: " fmt "\n", \ |
29 | current->comm, ##__VA_ARGS__) |
30 | |
31 | struct gather_bo { |
32 | struct host1x_bo base; |
33 | |
34 | struct kref ref; |
35 | |
36 | struct device *dev; |
37 | u32 *gather_data; |
38 | dma_addr_t gather_data_dma; |
39 | size_t gather_data_words; |
40 | }; |
41 | |
42 | static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo) |
43 | { |
44 | struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); |
45 | |
46 | kref_get(kref: &bo->ref); |
47 | |
48 | return host_bo; |
49 | } |
50 | |
51 | static void gather_bo_release(struct kref *ref) |
52 | { |
53 | struct gather_bo *bo = container_of(ref, struct gather_bo, ref); |
54 | |
55 | dma_free_attrs(dev: bo->dev, size: bo->gather_data_words * 4, cpu_addr: bo->gather_data, dma_handle: bo->gather_data_dma, |
56 | attrs: 0); |
57 | kfree(objp: bo); |
58 | } |
59 | |
60 | static void gather_bo_put(struct host1x_bo *host_bo) |
61 | { |
62 | struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); |
63 | |
64 | kref_put(kref: &bo->ref, release: gather_bo_release); |
65 | } |
66 | |
67 | static struct host1x_bo_mapping * |
68 | gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) |
69 | { |
70 | struct gather_bo *gather = container_of(bo, struct gather_bo, base); |
71 | struct host1x_bo_mapping *map; |
72 | int err; |
73 | |
74 | map = kzalloc(size: sizeof(*map), GFP_KERNEL); |
75 | if (!map) |
76 | return ERR_PTR(error: -ENOMEM); |
77 | |
78 | kref_init(kref: &map->ref); |
79 | map->bo = host1x_bo_get(bo); |
80 | map->direction = direction; |
81 | map->dev = dev; |
82 | |
83 | map->sgt = kzalloc(size: sizeof(*map->sgt), GFP_KERNEL); |
84 | if (!map->sgt) { |
85 | err = -ENOMEM; |
86 | goto free; |
87 | } |
88 | |
89 | err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, |
90 | gather->gather_data_words * 4); |
91 | if (err) |
92 | goto free_sgt; |
93 | |
94 | err = dma_map_sgtable(dev, sgt: map->sgt, dir: direction, attrs: 0); |
95 | if (err) |
96 | goto free_sgt; |
97 | |
98 | map->phys = sg_dma_address(map->sgt->sgl); |
99 | map->size = gather->gather_data_words * 4; |
100 | map->chunks = err; |
101 | |
102 | return map; |
103 | |
104 | free_sgt: |
105 | sg_free_table(map->sgt); |
106 | kfree(objp: map->sgt); |
107 | free: |
108 | kfree(objp: map); |
109 | return ERR_PTR(error: err); |
110 | } |
111 | |
112 | static void gather_bo_unpin(struct host1x_bo_mapping *map) |
113 | { |
114 | if (!map) |
115 | return; |
116 | |
117 | dma_unmap_sgtable(dev: map->dev, sgt: map->sgt, dir: map->direction, attrs: 0); |
118 | sg_free_table(map->sgt); |
119 | kfree(objp: map->sgt); |
120 | host1x_bo_put(bo: map->bo); |
121 | |
122 | kfree(objp: map); |
123 | } |
124 | |
125 | static void *gather_bo_mmap(struct host1x_bo *host_bo) |
126 | { |
127 | struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); |
128 | |
129 | return bo->gather_data; |
130 | } |
131 | |
132 | static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr) |
133 | { |
134 | } |
135 | |
136 | static const struct host1x_bo_ops gather_bo_ops = { |
137 | .get = gather_bo_get, |
138 | .put = gather_bo_put, |
139 | .pin = gather_bo_pin, |
140 | .unpin = gather_bo_unpin, |
141 | .mmap = gather_bo_mmap, |
142 | .munmap = gather_bo_munmap, |
143 | }; |
144 | |
145 | static struct tegra_drm_mapping * |
146 | tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id) |
147 | { |
148 | struct tegra_drm_mapping *mapping; |
149 | |
150 | xa_lock(&context->mappings); |
151 | |
152 | mapping = xa_load(&context->mappings, index: id); |
153 | if (mapping) |
154 | kref_get(kref: &mapping->ref); |
155 | |
156 | xa_unlock(&context->mappings); |
157 | |
158 | return mapping; |
159 | } |
160 | |
161 | static void *alloc_copy_user_array(void __user *from, size_t count, size_t size) |
162 | { |
163 | size_t copy_len; |
164 | void *data; |
165 | |
166 | if (check_mul_overflow(count, size, ©_len)) |
167 | return ERR_PTR(error: -EINVAL); |
168 | |
169 | if (copy_len > 0x4000) |
170 | return ERR_PTR(error: -E2BIG); |
171 | |
172 | data = vmemdup_user(from, copy_len); |
173 | if (IS_ERR(ptr: data)) |
174 | return ERR_CAST(ptr: data); |
175 | |
176 | return data; |
177 | } |
178 | |
179 | static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev, |
180 | struct tegra_drm_context *context, |
181 | struct drm_tegra_channel_submit *args) |
182 | { |
183 | struct gather_bo *bo; |
184 | size_t copy_len; |
185 | |
186 | if (args->gather_data_words == 0) { |
187 | SUBMIT_ERR(context, "gather_data_words cannot be zero" ); |
188 | return -EINVAL; |
189 | } |
190 | |
191 | if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) { |
192 | SUBMIT_ERR(context, "gather_data_words is too large" ); |
193 | return -EINVAL; |
194 | } |
195 | |
196 | bo = kzalloc(size: sizeof(*bo), GFP_KERNEL); |
197 | if (!bo) { |
198 | SUBMIT_ERR(context, "failed to allocate memory for bo info" ); |
199 | return -ENOMEM; |
200 | } |
201 | |
202 | host1x_bo_init(bo: &bo->base, ops: &gather_bo_ops); |
203 | kref_init(kref: &bo->ref); |
204 | bo->dev = dev; |
205 | |
206 | bo->gather_data = dma_alloc_attrs(dev, size: copy_len, dma_handle: &bo->gather_data_dma, |
207 | GFP_KERNEL | __GFP_NOWARN, attrs: 0); |
208 | if (!bo->gather_data) { |
209 | SUBMIT_ERR(context, "failed to allocate memory for gather data" ); |
210 | kfree(objp: bo); |
211 | return -ENOMEM; |
212 | } |
213 | |
214 | if (copy_from_user(to: bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), n: copy_len)) { |
215 | SUBMIT_ERR(context, "failed to copy gather data from userspace" ); |
216 | dma_free_attrs(dev, size: copy_len, cpu_addr: bo->gather_data, dma_handle: bo->gather_data_dma, attrs: 0); |
217 | kfree(objp: bo); |
218 | return -EFAULT; |
219 | } |
220 | |
221 | bo->gather_data_words = args->gather_data_words; |
222 | |
223 | *pbo = bo; |
224 | |
225 | return 0; |
226 | } |
227 | |
228 | static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo, |
229 | struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping) |
230 | { |
231 | /* TODO check that target_offset is within bounds */ |
232 | dma_addr_t iova = mapping->iova + buf->reloc.target_offset; |
233 | u32 written_ptr; |
234 | |
235 | #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT |
236 | if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) |
237 | iova |= BIT_ULL(39); |
238 | #endif |
239 | |
240 | written_ptr = iova >> buf->reloc.shift; |
241 | |
242 | if (buf->reloc.gather_offset_words >= bo->gather_data_words) { |
243 | SUBMIT_ERR(context, |
244 | "relocation has too large gather offset (%u vs gather length %zu)" , |
245 | buf->reloc.gather_offset_words, bo->gather_data_words); |
246 | return -EINVAL; |
247 | } |
248 | |
249 | buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words, |
250 | bo->gather_data_words); |
251 | |
252 | bo->gather_data[buf->reloc.gather_offset_words] = written_ptr; |
253 | |
254 | return 0; |
255 | } |
256 | |
257 | static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo, |
258 | struct drm_tegra_channel_submit *args, |
259 | struct tegra_drm_submit_data *job_data) |
260 | { |
261 | struct tegra_drm_used_mapping *mappings; |
262 | struct drm_tegra_submit_buf *bufs; |
263 | int err; |
264 | u32 i; |
265 | |
266 | bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), count: args->num_bufs, |
267 | size: sizeof(*bufs)); |
268 | if (IS_ERR(ptr: bufs)) { |
269 | SUBMIT_ERR(context, "failed to copy bufs array from userspace" ); |
270 | return PTR_ERR(ptr: bufs); |
271 | } |
272 | |
273 | mappings = kcalloc(n: args->num_bufs, size: sizeof(*mappings), GFP_KERNEL); |
274 | if (!mappings) { |
275 | SUBMIT_ERR(context, "failed to allocate memory for mapping info" ); |
276 | err = -ENOMEM; |
277 | goto done; |
278 | } |
279 | |
280 | for (i = 0; i < args->num_bufs; i++) { |
281 | struct drm_tegra_submit_buf *buf = &bufs[i]; |
282 | struct tegra_drm_mapping *mapping; |
283 | |
284 | if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) { |
285 | SUBMIT_ERR(context, "invalid flag specified for buffer" ); |
286 | err = -EINVAL; |
287 | goto drop_refs; |
288 | } |
289 | |
290 | mapping = tegra_drm_mapping_get(context, id: buf->mapping); |
291 | if (!mapping) { |
292 | SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer" , buf->mapping); |
293 | err = -EINVAL; |
294 | goto drop_refs; |
295 | } |
296 | |
297 | err = submit_write_reloc(context, bo, buf, mapping); |
298 | if (err) { |
299 | tegra_drm_mapping_put(mapping); |
300 | goto drop_refs; |
301 | } |
302 | |
303 | mappings[i].mapping = mapping; |
304 | mappings[i].flags = buf->flags; |
305 | } |
306 | |
307 | job_data->used_mappings = mappings; |
308 | job_data->num_used_mappings = i; |
309 | |
310 | err = 0; |
311 | |
312 | goto done; |
313 | |
314 | drop_refs: |
315 | while (i--) |
316 | tegra_drm_mapping_put(mapping: mappings[i].mapping); |
317 | |
318 | kfree(objp: mappings); |
319 | job_data->used_mappings = NULL; |
320 | |
321 | done: |
322 | kvfree(addr: bufs); |
323 | |
324 | return err; |
325 | } |
326 | |
327 | static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job, |
328 | struct xarray *syncpoints, struct drm_tegra_channel_submit *args) |
329 | { |
330 | struct host1x_syncpt *sp; |
331 | |
332 | if (args->syncpt.flags) { |
333 | SUBMIT_ERR(context, "invalid flag specified for syncpt" ); |
334 | return -EINVAL; |
335 | } |
336 | |
337 | /* Syncpt ref will be dropped on job release */ |
338 | sp = xa_load(syncpoints, index: args->syncpt.id); |
339 | if (!sp) { |
340 | SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated" ); |
341 | return -EINVAL; |
342 | } |
343 | |
344 | job->syncpt = host1x_syncpt_get(sp); |
345 | job->syncpt_incrs = args->syncpt.increments; |
346 | |
347 | return 0; |
348 | } |
349 | |
350 | static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context, |
351 | struct drm_tegra_submit_cmd_gather_uptr *cmd, |
352 | struct gather_bo *bo, u32 *offset, |
353 | struct tegra_drm_submit_data *job_data, |
354 | u32 *class) |
355 | { |
356 | u32 next_offset; |
357 | |
358 | if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) { |
359 | SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command" ); |
360 | return -EINVAL; |
361 | } |
362 | |
363 | /* Check for maximum gather size */ |
364 | if (cmd->words > 16383) { |
365 | SUBMIT_ERR(context, "too many words in GATHER_UPTR command" ); |
366 | return -EINVAL; |
367 | } |
368 | |
369 | if (check_add_overflow(*offset, cmd->words, &next_offset)) { |
370 | SUBMIT_ERR(context, "too many total words in job" ); |
371 | return -EINVAL; |
372 | } |
373 | |
374 | if (next_offset > bo->gather_data_words) { |
375 | SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data" ); |
376 | return -EINVAL; |
377 | } |
378 | |
379 | if (tegra_drm_fw_validate(client: context->client, data: bo->gather_data, start: *offset, |
380 | words: cmd->words, submit: job_data, job_class: class)) { |
381 | SUBMIT_ERR(context, "job was rejected by firewall" ); |
382 | return -EINVAL; |
383 | } |
384 | |
385 | host1x_job_add_gather(job, bo: &bo->base, words: cmd->words, offset: *offset * 4); |
386 | |
387 | *offset = next_offset; |
388 | |
389 | return 0; |
390 | } |
391 | |
392 | static struct host1x_job * |
393 | submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo, |
394 | struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data, |
395 | struct xarray *syncpoints) |
396 | { |
397 | struct drm_tegra_submit_cmd *cmds; |
398 | u32 i, gather_offset = 0, class; |
399 | struct host1x_job *job; |
400 | int err; |
401 | |
402 | /* Set initial class for firewall. */ |
403 | class = context->client->base.class; |
404 | |
405 | cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), count: args->num_cmds, |
406 | size: sizeof(*cmds)); |
407 | if (IS_ERR(ptr: cmds)) { |
408 | SUBMIT_ERR(context, "failed to copy cmds array from userspace" ); |
409 | return ERR_CAST(ptr: cmds); |
410 | } |
411 | |
412 | job = host1x_job_alloc(ch: context->channel, num_cmdbufs: args->num_cmds, num_relocs: 0, skip_firewall: true); |
413 | if (!job) { |
414 | SUBMIT_ERR(context, "failed to allocate memory for job" ); |
415 | job = ERR_PTR(error: -ENOMEM); |
416 | goto done; |
417 | } |
418 | |
419 | err = submit_get_syncpt(context, job, syncpoints, args); |
420 | if (err < 0) |
421 | goto free_job; |
422 | |
423 | job->client = &context->client->base; |
424 | job->class = context->client->base.class; |
425 | job->serialize = true; |
426 | |
427 | for (i = 0; i < args->num_cmds; i++) { |
428 | struct drm_tegra_submit_cmd *cmd = &cmds[i]; |
429 | |
430 | if (cmd->flags) { |
431 | SUBMIT_ERR(context, "unknown flags given for cmd" ); |
432 | err = -EINVAL; |
433 | goto free_job; |
434 | } |
435 | |
436 | if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) { |
437 | err = submit_job_add_gather(job, context, cmd: &cmd->gather_uptr, bo, |
438 | offset: &gather_offset, job_data, class: &class); |
439 | if (err) |
440 | goto free_job; |
441 | } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) { |
442 | if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { |
443 | SUBMIT_ERR(context, "non-zero reserved value" ); |
444 | err = -EINVAL; |
445 | goto free_job; |
446 | } |
447 | |
448 | host1x_job_add_wait(job, id: cmd->wait_syncpt.id, thresh: cmd->wait_syncpt.value, |
449 | relative: false, next_class: class); |
450 | } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) { |
451 | if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { |
452 | SUBMIT_ERR(context, "non-zero reserved value" ); |
453 | err = -EINVAL; |
454 | goto free_job; |
455 | } |
456 | |
457 | if (cmd->wait_syncpt.id != args->syncpt.id) { |
458 | SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job" ); |
459 | err = -EINVAL; |
460 | goto free_job; |
461 | } |
462 | |
463 | host1x_job_add_wait(job, id: cmd->wait_syncpt.id, thresh: cmd->wait_syncpt.value, |
464 | relative: true, next_class: class); |
465 | } else { |
466 | SUBMIT_ERR(context, "unknown cmd type" ); |
467 | err = -EINVAL; |
468 | goto free_job; |
469 | } |
470 | } |
471 | |
472 | if (gather_offset == 0) { |
473 | SUBMIT_ERR(context, "job must have at least one gather" ); |
474 | err = -EINVAL; |
475 | goto free_job; |
476 | } |
477 | |
478 | goto done; |
479 | |
480 | free_job: |
481 | host1x_job_put(job); |
482 | job = ERR_PTR(error: err); |
483 | |
484 | done: |
485 | kvfree(addr: cmds); |
486 | |
487 | return job; |
488 | } |
489 | |
490 | static void release_job(struct host1x_job *job) |
491 | { |
492 | struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base); |
493 | struct tegra_drm_submit_data *job_data = job->user_data; |
494 | u32 i; |
495 | |
496 | if (job->memory_context) |
497 | host1x_memory_context_put(cd: job->memory_context); |
498 | |
499 | for (i = 0; i < job_data->num_used_mappings; i++) |
500 | tegra_drm_mapping_put(mapping: job_data->used_mappings[i].mapping); |
501 | |
502 | kfree(objp: job_data->used_mappings); |
503 | kfree(objp: job_data); |
504 | |
505 | pm_runtime_mark_last_busy(dev: client->base.dev); |
506 | pm_runtime_put_autosuspend(dev: client->base.dev); |
507 | } |
508 | |
509 | int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, |
510 | struct drm_file *file) |
511 | { |
512 | struct tegra_drm_file *fpriv = file->driver_priv; |
513 | struct drm_tegra_channel_submit *args = data; |
514 | struct tegra_drm_submit_data *job_data; |
515 | struct drm_syncobj *syncobj = NULL; |
516 | struct tegra_drm_context *context; |
517 | struct host1x_job *job; |
518 | struct gather_bo *bo; |
519 | u32 i; |
520 | int err; |
521 | |
522 | mutex_lock(&fpriv->lock); |
523 | |
524 | context = xa_load(&fpriv->contexts, index: args->context); |
525 | if (!context) { |
526 | mutex_unlock(lock: &fpriv->lock); |
527 | pr_err_ratelimited("%s: %s: invalid channel context '%#x'" , __func__, |
528 | current->comm, args->context); |
529 | return -EINVAL; |
530 | } |
531 | |
532 | if (args->syncobj_in) { |
533 | struct dma_fence *fence; |
534 | |
535 | err = drm_syncobj_find_fence(file_private: file, handle: args->syncobj_in, point: 0, flags: 0, fence: &fence); |
536 | if (err) { |
537 | SUBMIT_ERR(context, "invalid syncobj_in '%#x'" , args->syncobj_in); |
538 | goto unlock; |
539 | } |
540 | |
541 | err = dma_fence_wait_timeout(fence, intr: true, timeout: msecs_to_jiffies(m: 10000)); |
542 | dma_fence_put(fence); |
543 | if (err) { |
544 | SUBMIT_ERR(context, "wait for syncobj_in timed out" ); |
545 | goto unlock; |
546 | } |
547 | } |
548 | |
549 | if (args->syncobj_out) { |
550 | syncobj = drm_syncobj_find(file_private: file, handle: args->syncobj_out); |
551 | if (!syncobj) { |
552 | SUBMIT_ERR(context, "invalid syncobj_out '%#x'" , args->syncobj_out); |
553 | err = -ENOENT; |
554 | goto unlock; |
555 | } |
556 | } |
557 | |
558 | /* Allocate gather BO and copy gather words in. */ |
559 | err = submit_copy_gather_data(pbo: &bo, dev: drm->dev, context, args); |
560 | if (err) |
561 | goto unlock; |
562 | |
563 | job_data = kzalloc(size: sizeof(*job_data), GFP_KERNEL); |
564 | if (!job_data) { |
565 | SUBMIT_ERR(context, "failed to allocate memory for job data" ); |
566 | err = -ENOMEM; |
567 | goto put_bo; |
568 | } |
569 | |
570 | /* Get data buffer mappings and do relocation patching. */ |
571 | err = submit_process_bufs(context, bo, args, job_data); |
572 | if (err) |
573 | goto free_job_data; |
574 | |
575 | /* Allocate host1x_job and add gathers and waits to it. */ |
576 | job = submit_create_job(context, bo, args, job_data, syncpoints: &fpriv->syncpoints); |
577 | if (IS_ERR(ptr: job)) { |
578 | err = PTR_ERR(ptr: job); |
579 | goto free_job_data; |
580 | } |
581 | |
582 | /* Map gather data for Host1x. */ |
583 | err = host1x_job_pin(job, dev: context->client->base.dev); |
584 | if (err) { |
585 | SUBMIT_ERR(context, "failed to pin job: %d" , err); |
586 | goto put_job; |
587 | } |
588 | |
589 | if (context->client->ops->get_streamid_offset) { |
590 | err = context->client->ops->get_streamid_offset( |
591 | context->client, &job->engine_streamid_offset); |
592 | if (err) { |
593 | SUBMIT_ERR(context, "failed to get streamid offset: %d" , err); |
594 | goto unpin_job; |
595 | } |
596 | } |
597 | |
598 | if (context->memory_context && context->client->ops->can_use_memory_ctx) { |
599 | bool supported; |
600 | |
601 | err = context->client->ops->can_use_memory_ctx(context->client, &supported); |
602 | if (err) { |
603 | SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d" , err); |
604 | goto unpin_job; |
605 | } |
606 | |
607 | if (supported) { |
608 | job->memory_context = context->memory_context; |
609 | host1x_memory_context_get(cd: job->memory_context); |
610 | } |
611 | } else if (context->client->ops->get_streamid_offset) { |
612 | /* |
613 | * Job submission will need to temporarily change stream ID, |
614 | * so need to tell it what to change it back to. |
615 | */ |
616 | if (!tegra_dev_iommu_get_stream_id(dev: context->client->base.dev, |
617 | stream_id: &job->engine_fallback_streamid)) |
618 | job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS; |
619 | } |
620 | |
621 | /* Boot engine. */ |
622 | err = pm_runtime_resume_and_get(dev: context->client->base.dev); |
623 | if (err < 0) { |
624 | SUBMIT_ERR(context, "could not power up engine: %d" , err); |
625 | goto put_memory_context; |
626 | } |
627 | |
628 | job->user_data = job_data; |
629 | job->release = release_job; |
630 | job->timeout = 10000; |
631 | |
632 | /* |
633 | * job_data is now part of job reference counting, so don't release |
634 | * it from here. |
635 | */ |
636 | job_data = NULL; |
637 | |
638 | /* Submit job to hardware. */ |
639 | err = host1x_job_submit(job); |
640 | if (err) { |
641 | SUBMIT_ERR(context, "host1x job submission failed: %d" , err); |
642 | goto unpin_job; |
643 | } |
644 | |
645 | /* Return postfences to userspace and add fences to DMA reservations. */ |
646 | args->syncpt.value = job->syncpt_end; |
647 | |
648 | if (syncobj) { |
649 | struct dma_fence *fence = host1x_fence_create(sp: job->syncpt, threshold: job->syncpt_end, timeout: true); |
650 | if (IS_ERR(ptr: fence)) { |
651 | err = PTR_ERR(ptr: fence); |
652 | SUBMIT_ERR(context, "failed to create postfence: %d" , err); |
653 | } |
654 | |
655 | drm_syncobj_replace_fence(syncobj, fence); |
656 | } |
657 | |
658 | goto put_job; |
659 | |
660 | put_memory_context: |
661 | if (job->memory_context) |
662 | host1x_memory_context_put(cd: job->memory_context); |
663 | unpin_job: |
664 | host1x_job_unpin(job); |
665 | put_job: |
666 | host1x_job_put(job); |
667 | free_job_data: |
668 | if (job_data && job_data->used_mappings) { |
669 | for (i = 0; i < job_data->num_used_mappings; i++) |
670 | tegra_drm_mapping_put(mapping: job_data->used_mappings[i].mapping); |
671 | |
672 | kfree(objp: job_data->used_mappings); |
673 | } |
674 | |
675 | kfree(objp: job_data); |
676 | put_bo: |
677 | gather_bo_put(host_bo: &bo->base); |
678 | unlock: |
679 | if (syncobj) |
680 | drm_syncobj_put(obj: syncobj); |
681 | |
682 | mutex_unlock(lock: &fpriv->lock); |
683 | return err; |
684 | } |
685 | |