1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ |
3 | |
4 | #include <linux/module.h> |
5 | #include <linux/of.h> |
6 | #include <linux/platform_device.h> |
7 | #include <linux/uaccess.h> |
8 | #include <linux/slab.h> |
9 | #include <linux/pm_runtime.h> |
10 | #include <drm/drm_ioctl.h> |
11 | #include <drm/drm_drv.h> |
12 | #include <drm/drm_prime.h> |
13 | #include <drm/lima_drm.h> |
14 | |
15 | #include "lima_device.h" |
16 | #include "lima_drv.h" |
17 | #include "lima_gem.h" |
18 | #include "lima_vm.h" |
19 | |
20 | int lima_sched_timeout_ms; |
21 | uint lima_heap_init_nr_pages = 8; |
22 | uint lima_max_error_tasks; |
23 | uint lima_job_hang_limit; |
24 | |
25 | MODULE_PARM_DESC(sched_timeout_ms, "task run timeout in ms" ); |
26 | module_param_named(sched_timeout_ms, lima_sched_timeout_ms, int, 0444); |
27 | |
28 | MODULE_PARM_DESC(heap_init_nr_pages, "heap buffer init number of pages" ); |
29 | module_param_named(heap_init_nr_pages, lima_heap_init_nr_pages, uint, 0444); |
30 | |
31 | MODULE_PARM_DESC(max_error_tasks, "max number of error tasks to save" ); |
32 | module_param_named(max_error_tasks, lima_max_error_tasks, uint, 0644); |
33 | |
34 | MODULE_PARM_DESC(job_hang_limit, "number of times to allow a job to hang before dropping it (default 0)" ); |
35 | module_param_named(job_hang_limit, lima_job_hang_limit, uint, 0444); |
36 | |
37 | static int lima_ioctl_get_param(struct drm_device *dev, void *data, struct drm_file *file) |
38 | { |
39 | struct drm_lima_get_param *args = data; |
40 | struct lima_device *ldev = to_lima_dev(dev); |
41 | |
42 | if (args->pad) |
43 | return -EINVAL; |
44 | |
45 | switch (args->param) { |
46 | case DRM_LIMA_PARAM_GPU_ID: |
47 | switch (ldev->id) { |
48 | case lima_gpu_mali400: |
49 | args->value = DRM_LIMA_PARAM_GPU_ID_MALI400; |
50 | break; |
51 | case lima_gpu_mali450: |
52 | args->value = DRM_LIMA_PARAM_GPU_ID_MALI450; |
53 | break; |
54 | default: |
55 | args->value = DRM_LIMA_PARAM_GPU_ID_UNKNOWN; |
56 | break; |
57 | } |
58 | break; |
59 | |
60 | case DRM_LIMA_PARAM_NUM_PP: |
61 | args->value = ldev->pipe[lima_pipe_pp].num_processor; |
62 | break; |
63 | |
64 | case DRM_LIMA_PARAM_GP_VERSION: |
65 | args->value = ldev->gp_version; |
66 | break; |
67 | |
68 | case DRM_LIMA_PARAM_PP_VERSION: |
69 | args->value = ldev->pp_version; |
70 | break; |
71 | |
72 | default: |
73 | return -EINVAL; |
74 | } |
75 | |
76 | return 0; |
77 | } |
78 | |
79 | static int lima_ioctl_gem_create(struct drm_device *dev, void *data, struct drm_file *file) |
80 | { |
81 | struct drm_lima_gem_create *args = data; |
82 | |
83 | if (args->pad) |
84 | return -EINVAL; |
85 | |
86 | if (args->flags & ~(LIMA_BO_FLAG_HEAP)) |
87 | return -EINVAL; |
88 | |
89 | if (args->size == 0) |
90 | return -EINVAL; |
91 | |
92 | return lima_gem_create_handle(dev, file, size: args->size, flags: args->flags, handle: &args->handle); |
93 | } |
94 | |
95 | static int lima_ioctl_gem_info(struct drm_device *dev, void *data, struct drm_file *file) |
96 | { |
97 | struct drm_lima_gem_info *args = data; |
98 | |
99 | return lima_gem_get_info(file, handle: args->handle, va: &args->va, offset: &args->offset); |
100 | } |
101 | |
102 | static int lima_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file) |
103 | { |
104 | struct drm_lima_gem_submit *args = data; |
105 | struct lima_device *ldev = to_lima_dev(dev); |
106 | struct lima_drm_priv *priv = file->driver_priv; |
107 | struct drm_lima_gem_submit_bo *bos; |
108 | struct lima_sched_pipe *pipe; |
109 | struct lima_sched_task *task; |
110 | struct lima_ctx *ctx; |
111 | struct lima_submit submit = {0}; |
112 | size_t size; |
113 | int err = 0; |
114 | |
115 | if (args->pipe >= lima_pipe_num || args->nr_bos == 0) |
116 | return -EINVAL; |
117 | |
118 | if (args->flags & ~(LIMA_SUBMIT_FLAG_EXPLICIT_FENCE)) |
119 | return -EINVAL; |
120 | |
121 | pipe = ldev->pipe + args->pipe; |
122 | if (args->frame_size != pipe->frame_size) |
123 | return -EINVAL; |
124 | |
125 | bos = kvcalloc(n: args->nr_bos, size: sizeof(*submit.bos) + sizeof(*submit.lbos), GFP_KERNEL); |
126 | if (!bos) |
127 | return -ENOMEM; |
128 | |
129 | size = args->nr_bos * sizeof(*submit.bos); |
130 | if (copy_from_user(to: bos, u64_to_user_ptr(args->bos), n: size)) { |
131 | err = -EFAULT; |
132 | goto out0; |
133 | } |
134 | |
135 | task = kmem_cache_zalloc(k: pipe->task_slab, GFP_KERNEL); |
136 | if (!task) { |
137 | err = -ENOMEM; |
138 | goto out0; |
139 | } |
140 | |
141 | task->frame = task + 1; |
142 | if (copy_from_user(to: task->frame, u64_to_user_ptr(args->frame), n: args->frame_size)) { |
143 | err = -EFAULT; |
144 | goto out1; |
145 | } |
146 | |
147 | err = pipe->task_validate(pipe, task); |
148 | if (err) |
149 | goto out1; |
150 | |
151 | ctx = lima_ctx_get(mgr: &priv->ctx_mgr, id: args->ctx); |
152 | if (!ctx) { |
153 | err = -ENOENT; |
154 | goto out1; |
155 | } |
156 | |
157 | submit.pipe = args->pipe; |
158 | submit.bos = bos; |
159 | submit.lbos = (void *)bos + size; |
160 | submit.nr_bos = args->nr_bos; |
161 | submit.task = task; |
162 | submit.ctx = ctx; |
163 | submit.flags = args->flags; |
164 | submit.in_sync[0] = args->in_sync[0]; |
165 | submit.in_sync[1] = args->in_sync[1]; |
166 | submit.out_sync = args->out_sync; |
167 | |
168 | err = lima_gem_submit(file, submit: &submit); |
169 | |
170 | lima_ctx_put(ctx); |
171 | out1: |
172 | if (err) |
173 | kmem_cache_free(s: pipe->task_slab, objp: task); |
174 | out0: |
175 | kvfree(addr: bos); |
176 | return err; |
177 | } |
178 | |
179 | static int lima_ioctl_gem_wait(struct drm_device *dev, void *data, struct drm_file *file) |
180 | { |
181 | struct drm_lima_gem_wait *args = data; |
182 | |
183 | if (args->op & ~(LIMA_GEM_WAIT_READ|LIMA_GEM_WAIT_WRITE)) |
184 | return -EINVAL; |
185 | |
186 | return lima_gem_wait(file, handle: args->handle, op: args->op, timeout_ns: args->timeout_ns); |
187 | } |
188 | |
189 | static int lima_ioctl_ctx_create(struct drm_device *dev, void *data, struct drm_file *file) |
190 | { |
191 | struct drm_lima_ctx_create *args = data; |
192 | struct lima_drm_priv *priv = file->driver_priv; |
193 | struct lima_device *ldev = to_lima_dev(dev); |
194 | |
195 | if (args->_pad) |
196 | return -EINVAL; |
197 | |
198 | return lima_ctx_create(dev: ldev, mgr: &priv->ctx_mgr, id: &args->id); |
199 | } |
200 | |
201 | static int lima_ioctl_ctx_free(struct drm_device *dev, void *data, struct drm_file *file) |
202 | { |
203 | struct drm_lima_ctx_create *args = data; |
204 | struct lima_drm_priv *priv = file->driver_priv; |
205 | |
206 | if (args->_pad) |
207 | return -EINVAL; |
208 | |
209 | return lima_ctx_free(mgr: &priv->ctx_mgr, id: args->id); |
210 | } |
211 | |
212 | static int lima_drm_driver_open(struct drm_device *dev, struct drm_file *file) |
213 | { |
214 | int err; |
215 | struct lima_drm_priv *priv; |
216 | struct lima_device *ldev = to_lima_dev(dev); |
217 | |
218 | priv = kzalloc(size: sizeof(*priv), GFP_KERNEL); |
219 | if (!priv) |
220 | return -ENOMEM; |
221 | |
222 | priv->vm = lima_vm_create(dev: ldev); |
223 | if (!priv->vm) { |
224 | err = -ENOMEM; |
225 | goto err_out0; |
226 | } |
227 | |
228 | lima_ctx_mgr_init(mgr: &priv->ctx_mgr); |
229 | |
230 | file->driver_priv = priv; |
231 | return 0; |
232 | |
233 | err_out0: |
234 | kfree(objp: priv); |
235 | return err; |
236 | } |
237 | |
238 | static void lima_drm_driver_postclose(struct drm_device *dev, struct drm_file *file) |
239 | { |
240 | struct lima_drm_priv *priv = file->driver_priv; |
241 | |
242 | lima_ctx_mgr_fini(mgr: &priv->ctx_mgr); |
243 | lima_vm_put(vm: priv->vm); |
244 | kfree(objp: priv); |
245 | } |
246 | |
247 | static const struct drm_ioctl_desc lima_drm_driver_ioctls[] = { |
248 | DRM_IOCTL_DEF_DRV(LIMA_GET_PARAM, lima_ioctl_get_param, DRM_RENDER_ALLOW), |
249 | DRM_IOCTL_DEF_DRV(LIMA_GEM_CREATE, lima_ioctl_gem_create, DRM_RENDER_ALLOW), |
250 | DRM_IOCTL_DEF_DRV(LIMA_GEM_INFO, lima_ioctl_gem_info, DRM_RENDER_ALLOW), |
251 | DRM_IOCTL_DEF_DRV(LIMA_GEM_SUBMIT, lima_ioctl_gem_submit, DRM_RENDER_ALLOW), |
252 | DRM_IOCTL_DEF_DRV(LIMA_GEM_WAIT, lima_ioctl_gem_wait, DRM_RENDER_ALLOW), |
253 | DRM_IOCTL_DEF_DRV(LIMA_CTX_CREATE, lima_ioctl_ctx_create, DRM_RENDER_ALLOW), |
254 | DRM_IOCTL_DEF_DRV(LIMA_CTX_FREE, lima_ioctl_ctx_free, DRM_RENDER_ALLOW), |
255 | }; |
256 | |
257 | DEFINE_DRM_GEM_FOPS(lima_drm_driver_fops); |
258 | |
259 | /* |
260 | * Changelog: |
261 | * |
262 | * - 1.1.0 - add heap buffer support |
263 | */ |
264 | |
265 | static const struct drm_driver lima_drm_driver = { |
266 | .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, |
267 | .open = lima_drm_driver_open, |
268 | .postclose = lima_drm_driver_postclose, |
269 | .ioctls = lima_drm_driver_ioctls, |
270 | .num_ioctls = ARRAY_SIZE(lima_drm_driver_ioctls), |
271 | .fops = &lima_drm_driver_fops, |
272 | .name = "lima" , |
273 | .desc = "lima DRM" , |
274 | .date = "20191231" , |
275 | .major = 1, |
276 | .minor = 1, |
277 | .patchlevel = 0, |
278 | |
279 | .gem_create_object = lima_gem_create_object, |
280 | .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, |
281 | }; |
282 | |
283 | struct lima_block_reader { |
284 | void *dst; |
285 | size_t base; |
286 | size_t count; |
287 | size_t off; |
288 | ssize_t read; |
289 | }; |
290 | |
291 | static bool lima_read_block(struct lima_block_reader *reader, |
292 | void *src, size_t src_size) |
293 | { |
294 | size_t max_off = reader->base + src_size; |
295 | |
296 | if (reader->off < max_off) { |
297 | size_t size = min_t(size_t, max_off - reader->off, |
298 | reader->count); |
299 | |
300 | memcpy(reader->dst, src + (reader->off - reader->base), size); |
301 | |
302 | reader->dst += size; |
303 | reader->off += size; |
304 | reader->read += size; |
305 | reader->count -= size; |
306 | } |
307 | |
308 | reader->base = max_off; |
309 | |
310 | return !!reader->count; |
311 | } |
312 | |
313 | static ssize_t lima_error_state_read(struct file *filp, struct kobject *kobj, |
314 | struct bin_attribute *attr, char *buf, |
315 | loff_t off, size_t count) |
316 | { |
317 | struct device *dev = kobj_to_dev(kobj); |
318 | struct lima_device *ldev = dev_get_drvdata(dev); |
319 | struct lima_sched_error_task *et; |
320 | struct lima_block_reader reader = { |
321 | .dst = buf, |
322 | .count = count, |
323 | .off = off, |
324 | }; |
325 | |
326 | mutex_lock(&ldev->error_task_list_lock); |
327 | |
328 | if (lima_read_block(reader: &reader, src: &ldev->dump, src_size: sizeof(ldev->dump))) { |
329 | list_for_each_entry(et, &ldev->error_task_list, list) { |
330 | if (!lima_read_block(reader: &reader, src: et->data, src_size: et->size)) |
331 | break; |
332 | } |
333 | } |
334 | |
335 | mutex_unlock(lock: &ldev->error_task_list_lock); |
336 | return reader.read; |
337 | } |
338 | |
339 | static ssize_t lima_error_state_write(struct file *file, struct kobject *kobj, |
340 | struct bin_attribute *attr, char *buf, |
341 | loff_t off, size_t count) |
342 | { |
343 | struct device *dev = kobj_to_dev(kobj); |
344 | struct lima_device *ldev = dev_get_drvdata(dev); |
345 | struct lima_sched_error_task *et, *tmp; |
346 | |
347 | mutex_lock(&ldev->error_task_list_lock); |
348 | |
349 | list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) { |
350 | list_del(entry: &et->list); |
351 | kvfree(addr: et); |
352 | } |
353 | |
354 | ldev->dump.size = 0; |
355 | ldev->dump.num_tasks = 0; |
356 | |
357 | mutex_unlock(lock: &ldev->error_task_list_lock); |
358 | |
359 | return count; |
360 | } |
361 | |
362 | static const struct bin_attribute lima_error_state_attr = { |
363 | .attr.name = "error" , |
364 | .attr.mode = 0600, |
365 | .size = 0, |
366 | .read = lima_error_state_read, |
367 | .write = lima_error_state_write, |
368 | }; |
369 | |
370 | static int lima_pdev_probe(struct platform_device *pdev) |
371 | { |
372 | struct lima_device *ldev; |
373 | struct drm_device *ddev; |
374 | int err; |
375 | |
376 | err = lima_sched_slab_init(); |
377 | if (err) |
378 | return err; |
379 | |
380 | ldev = devm_kzalloc(dev: &pdev->dev, size: sizeof(*ldev), GFP_KERNEL); |
381 | if (!ldev) { |
382 | err = -ENOMEM; |
383 | goto err_out0; |
384 | } |
385 | |
386 | ldev->dev = &pdev->dev; |
387 | ldev->id = (enum lima_gpu_id)of_device_get_match_data(dev: &pdev->dev); |
388 | |
389 | platform_set_drvdata(pdev, data: ldev); |
390 | |
391 | /* Allocate and initialize the DRM device. */ |
392 | ddev = drm_dev_alloc(driver: &lima_drm_driver, parent: &pdev->dev); |
393 | if (IS_ERR(ptr: ddev)) { |
394 | err = PTR_ERR(ptr: ddev); |
395 | goto err_out0; |
396 | } |
397 | |
398 | ddev->dev_private = ldev; |
399 | ldev->ddev = ddev; |
400 | |
401 | err = lima_device_init(ldev); |
402 | if (err) |
403 | goto err_out1; |
404 | |
405 | err = lima_devfreq_init(ldev); |
406 | if (err) { |
407 | dev_err(&pdev->dev, "Fatal error during devfreq init\n" ); |
408 | goto err_out2; |
409 | } |
410 | |
411 | pm_runtime_set_active(dev: ldev->dev); |
412 | pm_runtime_mark_last_busy(dev: ldev->dev); |
413 | pm_runtime_set_autosuspend_delay(dev: ldev->dev, delay: 200); |
414 | pm_runtime_use_autosuspend(dev: ldev->dev); |
415 | pm_runtime_enable(dev: ldev->dev); |
416 | |
417 | /* |
418 | * Register the DRM device with the core and the connectors with |
419 | * sysfs. |
420 | */ |
421 | err = drm_dev_register(dev: ddev, flags: 0); |
422 | if (err < 0) |
423 | goto err_out3; |
424 | |
425 | if (sysfs_create_bin_file(kobj: &ldev->dev->kobj, attr: &lima_error_state_attr)) |
426 | dev_warn(ldev->dev, "fail to create error state sysfs\n" ); |
427 | |
428 | return 0; |
429 | |
430 | err_out3: |
431 | pm_runtime_disable(dev: ldev->dev); |
432 | lima_devfreq_fini(ldev); |
433 | err_out2: |
434 | lima_device_fini(ldev); |
435 | err_out1: |
436 | drm_dev_put(dev: ddev); |
437 | err_out0: |
438 | lima_sched_slab_fini(); |
439 | return err; |
440 | } |
441 | |
442 | static void lima_pdev_remove(struct platform_device *pdev) |
443 | { |
444 | struct lima_device *ldev = platform_get_drvdata(pdev); |
445 | struct drm_device *ddev = ldev->ddev; |
446 | |
447 | sysfs_remove_bin_file(kobj: &ldev->dev->kobj, attr: &lima_error_state_attr); |
448 | |
449 | drm_dev_unregister(dev: ddev); |
450 | |
451 | /* stop autosuspend to make sure device is in active state */ |
452 | pm_runtime_set_autosuspend_delay(dev: ldev->dev, delay: -1); |
453 | pm_runtime_disable(dev: ldev->dev); |
454 | |
455 | lima_devfreq_fini(ldev); |
456 | lima_device_fini(ldev); |
457 | |
458 | drm_dev_put(dev: ddev); |
459 | lima_sched_slab_fini(); |
460 | } |
461 | |
462 | static const struct of_device_id dt_match[] = { |
463 | { .compatible = "arm,mali-400" , .data = (void *)lima_gpu_mali400 }, |
464 | { .compatible = "arm,mali-450" , .data = (void *)lima_gpu_mali450 }, |
465 | {} |
466 | }; |
467 | MODULE_DEVICE_TABLE(of, dt_match); |
468 | |
469 | static const struct dev_pm_ops lima_pm_ops = { |
470 | SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) |
471 | SET_RUNTIME_PM_OPS(lima_device_suspend, lima_device_resume, NULL) |
472 | }; |
473 | |
474 | static struct platform_driver lima_platform_driver = { |
475 | .probe = lima_pdev_probe, |
476 | .remove_new = lima_pdev_remove, |
477 | .driver = { |
478 | .name = "lima" , |
479 | .pm = &lima_pm_ops, |
480 | .of_match_table = dt_match, |
481 | }, |
482 | }; |
483 | |
484 | module_platform_driver(lima_platform_driver); |
485 | |
486 | MODULE_AUTHOR("Lima Project Developers" ); |
487 | MODULE_DESCRIPTION("Lima DRM Driver" ); |
488 | MODULE_LICENSE("GPL v2" ); |
489 | |