1// SPDX-License-Identifier: GPL-2.0 or MIT
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4/* Copyright 2023 Collabora ltd. */
5
6#include <linux/clk.h>
7#include <linux/mm.h>
8#include <linux/platform_device.h>
9#include <linux/pm_domain.h>
10#include <linux/pm_runtime.h>
11#include <linux/regulator/consumer.h>
12#include <linux/reset.h>
13
14#include <drm/drm_drv.h>
15#include <drm/drm_managed.h>
16
17#include "panthor_devfreq.h"
18#include "panthor_device.h"
19#include "panthor_fw.h"
20#include "panthor_gpu.h"
21#include "panthor_mmu.h"
22#include "panthor_regs.h"
23#include "panthor_sched.h"
24
25static int panthor_gpu_coherency_init(struct panthor_device *ptdev)
26{
27 ptdev->coherent = device_get_dma_attr(dev: ptdev->base.dev) == DEV_DMA_COHERENT;
28
29 if (!ptdev->coherent)
30 return 0;
31
32 /* Check if the ACE-Lite coherency protocol is actually supported by the GPU.
33 * ACE protocol has never been supported for command stream frontend GPUs.
34 */
35 if ((gpu_read(ptdev, GPU_COHERENCY_FEATURES) &
36 GPU_COHERENCY_PROT_BIT(ACE_LITE)))
37 return 0;
38
39 drm_err(&ptdev->base, "Coherency not supported by the device");
40 return -ENOTSUPP;
41}
42
43static int panthor_clk_init(struct panthor_device *ptdev)
44{
45 ptdev->clks.core = devm_clk_get(dev: ptdev->base.dev, NULL);
46 if (IS_ERR(ptr: ptdev->clks.core))
47 return dev_err_probe(dev: ptdev->base.dev,
48 err: PTR_ERR(ptr: ptdev->clks.core),
49 fmt: "get 'core' clock failed");
50
51 ptdev->clks.stacks = devm_clk_get_optional(dev: ptdev->base.dev, id: "stacks");
52 if (IS_ERR(ptr: ptdev->clks.stacks))
53 return dev_err_probe(dev: ptdev->base.dev,
54 err: PTR_ERR(ptr: ptdev->clks.stacks),
55 fmt: "get 'stacks' clock failed");
56
57 ptdev->clks.coregroup = devm_clk_get_optional(dev: ptdev->base.dev, id: "coregroup");
58 if (IS_ERR(ptr: ptdev->clks.coregroup))
59 return dev_err_probe(dev: ptdev->base.dev,
60 err: PTR_ERR(ptr: ptdev->clks.coregroup),
61 fmt: "get 'coregroup' clock failed");
62
63 drm_info(&ptdev->base, "clock rate = %lu\n", clk_get_rate(ptdev->clks.core));
64 return 0;
65}
66
67void panthor_device_unplug(struct panthor_device *ptdev)
68{
69 /* This function can be called from two different path: the reset work
70 * and the platform device remove callback. drm_dev_unplug() doesn't
71 * deal with concurrent callers, so we have to protect drm_dev_unplug()
72 * calls with our own lock, and bail out if the device is already
73 * unplugged.
74 */
75 mutex_lock(&ptdev->unplug.lock);
76 if (drm_dev_is_unplugged(dev: &ptdev->base)) {
77 /* Someone beat us, release the lock and wait for the unplug
78 * operation to be reported as done.
79 **/
80 mutex_unlock(lock: &ptdev->unplug.lock);
81 wait_for_completion(&ptdev->unplug.done);
82 return;
83 }
84
85 /* Call drm_dev_unplug() so any access to HW blocks happening after
86 * that point get rejected.
87 */
88 drm_dev_unplug(dev: &ptdev->base);
89
90 /* We do the rest of the unplug with the unplug lock released,
91 * future callers will wait on ptdev->unplug.done anyway.
92 */
93 mutex_unlock(lock: &ptdev->unplug.lock);
94
95 drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0);
96
97 /* Now, try to cleanly shutdown the GPU before the device resources
98 * get reclaimed.
99 */
100 panthor_sched_unplug(ptdev);
101 panthor_fw_unplug(ptdev);
102 panthor_mmu_unplug(ptdev);
103 panthor_gpu_unplug(ptdev);
104
105 pm_runtime_dont_use_autosuspend(dev: ptdev->base.dev);
106 pm_runtime_put_sync_suspend(dev: ptdev->base.dev);
107
108 /* If PM is disabled, we need to call the suspend handler manually. */
109 if (!IS_ENABLED(CONFIG_PM))
110 panthor_device_suspend(dev: ptdev->base.dev);
111
112 /* Report the unplug operation as done to unblock concurrent
113 * panthor_device_unplug() callers.
114 */
115 complete_all(&ptdev->unplug.done);
116}
117
118static void panthor_device_reset_cleanup(struct drm_device *ddev, void *data)
119{
120 struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
121
122 cancel_work_sync(work: &ptdev->reset.work);
123 destroy_workqueue(wq: ptdev->reset.wq);
124}
125
126static void panthor_device_reset_work(struct work_struct *work)
127{
128 struct panthor_device *ptdev = container_of(work, struct panthor_device, reset.work);
129 int ret = 0, cookie;
130
131 /* If the device is entering suspend, we don't reset. A slow reset will
132 * be forced at resume time instead.
133 */
134 if (atomic_read(v: &ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)
135 return;
136
137 if (!drm_dev_enter(dev: &ptdev->base, idx: &cookie))
138 return;
139
140 panthor_sched_pre_reset(ptdev);
141 panthor_fw_pre_reset(ptdev, on_hang: true);
142 panthor_mmu_pre_reset(ptdev);
143 panthor_gpu_soft_reset(ptdev);
144 panthor_gpu_l2_power_on(ptdev);
145 panthor_mmu_post_reset(ptdev);
146 ret = panthor_fw_post_reset(ptdev);
147 atomic_set(v: &ptdev->reset.pending, i: 0);
148 panthor_sched_post_reset(ptdev, reset_failed: ret != 0);
149 drm_dev_exit(idx: cookie);
150
151 if (ret) {
152 panthor_device_unplug(ptdev);
153 drm_err(&ptdev->base, "Failed to boot MCU after reset, making device unusable.");
154 }
155}
156
157static bool panthor_device_is_initialized(struct panthor_device *ptdev)
158{
159 return !!ptdev->scheduler;
160}
161
162static void panthor_device_free_page(struct drm_device *ddev, void *data)
163{
164 __free_page(data);
165}
166
167int panthor_device_init(struct panthor_device *ptdev)
168{
169 u32 *dummy_page_virt;
170 struct resource *res;
171 struct page *p;
172 int ret;
173
174 init_completion(x: &ptdev->unplug.done);
175 ret = drmm_mutex_init(&ptdev->base, &ptdev->unplug.lock);
176 if (ret)
177 return ret;
178
179 ret = drmm_mutex_init(&ptdev->base, &ptdev->pm.mmio_lock);
180 if (ret)
181 return ret;
182
183#ifdef CONFIG_DEBUG_FS
184 drmm_mutex_init(&ptdev->base, &ptdev->gems.lock);
185 INIT_LIST_HEAD(list: &ptdev->gems.node);
186#endif
187
188 atomic_set(v: &ptdev->pm.state, i: PANTHOR_DEVICE_PM_STATE_SUSPENDED);
189 p = alloc_page(GFP_KERNEL | __GFP_ZERO);
190 if (!p)
191 return -ENOMEM;
192
193 ptdev->pm.dummy_latest_flush = p;
194 dummy_page_virt = page_address(p);
195 ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page,
196 ptdev->pm.dummy_latest_flush);
197 if (ret)
198 return ret;
199
200 /*
201 * Set the dummy page holding the latest flush to 1. This will cause the
202 * flush to avoided as we know it isn't necessary if the submission
203 * happens while the dummy page is mapped. Zero cannot be used because
204 * that means 'always flush'.
205 */
206 *dummy_page_virt = 1;
207
208 INIT_WORK(&ptdev->reset.work, panthor_device_reset_work);
209 ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0);
210 if (!ptdev->reset.wq)
211 return -ENOMEM;
212
213 ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_reset_cleanup, NULL);
214 if (ret)
215 return ret;
216
217 ret = panthor_clk_init(ptdev);
218 if (ret)
219 return ret;
220
221 ret = panthor_devfreq_init(ptdev);
222 if (ret)
223 return ret;
224
225 ptdev->iomem = devm_platform_get_and_ioremap_resource(to_platform_device(ptdev->base.dev),
226 index: 0, res: &res);
227 if (IS_ERR(ptr: ptdev->iomem))
228 return PTR_ERR(ptr: ptdev->iomem);
229
230 ptdev->phys_addr = res->start;
231
232 ret = devm_pm_runtime_enable(dev: ptdev->base.dev);
233 if (ret)
234 return ret;
235
236 ret = pm_runtime_resume_and_get(dev: ptdev->base.dev);
237 if (ret)
238 return ret;
239
240 /* If PM is disabled, we need to call panthor_device_resume() manually. */
241 if (!IS_ENABLED(CONFIG_PM)) {
242 ret = panthor_device_resume(dev: ptdev->base.dev);
243 if (ret)
244 return ret;
245 }
246
247 ret = panthor_gpu_init(ptdev);
248 if (ret)
249 goto err_rpm_put;
250
251 ret = panthor_gpu_coherency_init(ptdev);
252 if (ret)
253 goto err_unplug_gpu;
254
255 ret = panthor_mmu_init(ptdev);
256 if (ret)
257 goto err_unplug_gpu;
258
259 ret = panthor_fw_init(ptdev);
260 if (ret)
261 goto err_unplug_mmu;
262
263 ret = panthor_sched_init(ptdev);
264 if (ret)
265 goto err_unplug_fw;
266
267 /* ~3 frames */
268 pm_runtime_set_autosuspend_delay(dev: ptdev->base.dev, delay: 50);
269 pm_runtime_use_autosuspend(dev: ptdev->base.dev);
270
271 ret = drm_dev_register(dev: &ptdev->base, flags: 0);
272 if (ret)
273 goto err_disable_autosuspend;
274
275 pm_runtime_put_autosuspend(dev: ptdev->base.dev);
276 return 0;
277
278err_disable_autosuspend:
279 pm_runtime_dont_use_autosuspend(dev: ptdev->base.dev);
280 panthor_sched_unplug(ptdev);
281
282err_unplug_fw:
283 panthor_fw_unplug(ptdev);
284
285err_unplug_mmu:
286 panthor_mmu_unplug(ptdev);
287
288err_unplug_gpu:
289 panthor_gpu_unplug(ptdev);
290
291err_rpm_put:
292 pm_runtime_put_sync_suspend(dev: ptdev->base.dev);
293 return ret;
294}
295
296#define PANTHOR_EXCEPTION(id) \
297 [DRM_PANTHOR_EXCEPTION_ ## id] = { \
298 .name = #id, \
299 }
300
301struct panthor_exception_info {
302 const char *name;
303};
304
305static const struct panthor_exception_info panthor_exception_infos[] = {
306 PANTHOR_EXCEPTION(OK),
307 PANTHOR_EXCEPTION(TERMINATED),
308 PANTHOR_EXCEPTION(KABOOM),
309 PANTHOR_EXCEPTION(EUREKA),
310 PANTHOR_EXCEPTION(ACTIVE),
311 PANTHOR_EXCEPTION(CS_RES_TERM),
312 PANTHOR_EXCEPTION(CS_CONFIG_FAULT),
313 PANTHOR_EXCEPTION(CS_UNRECOVERABLE),
314 PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT),
315 PANTHOR_EXCEPTION(CS_BUS_FAULT),
316 PANTHOR_EXCEPTION(CS_INSTR_INVALID),
317 PANTHOR_EXCEPTION(CS_CALL_STACK_OVERFLOW),
318 PANTHOR_EXCEPTION(CS_INHERIT_FAULT),
319 PANTHOR_EXCEPTION(INSTR_INVALID_PC),
320 PANTHOR_EXCEPTION(INSTR_INVALID_ENC),
321 PANTHOR_EXCEPTION(INSTR_BARRIER_FAULT),
322 PANTHOR_EXCEPTION(DATA_INVALID_FAULT),
323 PANTHOR_EXCEPTION(TILE_RANGE_FAULT),
324 PANTHOR_EXCEPTION(ADDR_RANGE_FAULT),
325 PANTHOR_EXCEPTION(IMPRECISE_FAULT),
326 PANTHOR_EXCEPTION(OOM),
327 PANTHOR_EXCEPTION(CSF_FW_INTERNAL_ERROR),
328 PANTHOR_EXCEPTION(CSF_RES_EVICTION_TIMEOUT),
329 PANTHOR_EXCEPTION(GPU_BUS_FAULT),
330 PANTHOR_EXCEPTION(GPU_SHAREABILITY_FAULT),
331 PANTHOR_EXCEPTION(SYS_SHAREABILITY_FAULT),
332 PANTHOR_EXCEPTION(GPU_CACHEABILITY_FAULT),
333 PANTHOR_EXCEPTION(TRANSLATION_FAULT_0),
334 PANTHOR_EXCEPTION(TRANSLATION_FAULT_1),
335 PANTHOR_EXCEPTION(TRANSLATION_FAULT_2),
336 PANTHOR_EXCEPTION(TRANSLATION_FAULT_3),
337 PANTHOR_EXCEPTION(TRANSLATION_FAULT_4),
338 PANTHOR_EXCEPTION(PERM_FAULT_0),
339 PANTHOR_EXCEPTION(PERM_FAULT_1),
340 PANTHOR_EXCEPTION(PERM_FAULT_2),
341 PANTHOR_EXCEPTION(PERM_FAULT_3),
342 PANTHOR_EXCEPTION(ACCESS_FLAG_1),
343 PANTHOR_EXCEPTION(ACCESS_FLAG_2),
344 PANTHOR_EXCEPTION(ACCESS_FLAG_3),
345 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_IN),
346 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT0),
347 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT1),
348 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT2),
349 PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT3),
350 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_0),
351 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_1),
352 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_2),
353 PANTHOR_EXCEPTION(MEM_ATTR_FAULT_3),
354};
355
356const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code)
357{
358 if (exception_code >= ARRAY_SIZE(panthor_exception_infos) ||
359 !panthor_exception_infos[exception_code].name)
360 return "Unknown exception type";
361
362 return panthor_exception_infos[exception_code].name;
363}
364
365static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf)
366{
367 struct vm_area_struct *vma = vmf->vma;
368 struct panthor_device *ptdev = vma->vm_private_data;
369 u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
370 unsigned long pfn;
371 pgprot_t pgprot;
372 vm_fault_t ret;
373 bool active;
374 int cookie;
375
376 if (!drm_dev_enter(dev: &ptdev->base, idx: &cookie))
377 return VM_FAULT_SIGBUS;
378
379 mutex_lock(&ptdev->pm.mmio_lock);
380 active = atomic_read(v: &ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE;
381
382 switch (offset) {
383 case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
384 if (active)
385 pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID);
386 else
387 pfn = page_to_pfn(ptdev->pm.dummy_latest_flush);
388 break;
389
390 default:
391 ret = VM_FAULT_SIGBUS;
392 goto out_unlock;
393 }
394
395 pgprot = vma->vm_page_prot;
396 if (active)
397 pgprot = pgprot_noncached(pgprot);
398
399 ret = vmf_insert_pfn_prot(vma, addr: vmf->address, pfn, pgprot);
400
401out_unlock:
402 mutex_unlock(lock: &ptdev->pm.mmio_lock);
403 drm_dev_exit(idx: cookie);
404 return ret;
405}
406
407static const struct vm_operations_struct panthor_mmio_vm_ops = {
408 .fault = panthor_mmio_vm_fault,
409};
410
411int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma)
412{
413 u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
414
415 if ((vma->vm_flags & VM_SHARED) == 0)
416 return -EINVAL;
417
418 switch (offset) {
419 case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
420 if (vma->vm_end - vma->vm_start != PAGE_SIZE ||
421 (vma->vm_flags & (VM_WRITE | VM_EXEC)))
422 return -EINVAL;
423 vm_flags_clear(vma, VM_MAYWRITE);
424
425 break;
426
427 default:
428 return -EINVAL;
429 }
430
431 /* Defer actual mapping to the fault handler. */
432 vma->vm_private_data = ptdev;
433 vma->vm_ops = &panthor_mmio_vm_ops;
434 vm_flags_set(vma,
435 VM_IO | VM_DONTCOPY | VM_DONTEXPAND |
436 VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
437 return 0;
438}
439
440static int panthor_device_resume_hw_components(struct panthor_device *ptdev)
441{
442 int ret;
443
444 panthor_gpu_resume(ptdev);
445 panthor_mmu_resume(ptdev);
446
447 ret = panthor_fw_resume(ptdev);
448 if (!ret)
449 return 0;
450
451 panthor_mmu_suspend(ptdev);
452 panthor_gpu_suspend(ptdev);
453 return ret;
454}
455
456int panthor_device_resume(struct device *dev)
457{
458 struct panthor_device *ptdev = dev_get_drvdata(dev);
459 int ret, cookie;
460
461 if (atomic_read(v: &ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_SUSPENDED)
462 return -EINVAL;
463
464 atomic_set(v: &ptdev->pm.state, i: PANTHOR_DEVICE_PM_STATE_RESUMING);
465
466 ret = clk_prepare_enable(clk: ptdev->clks.core);
467 if (ret)
468 goto err_set_suspended;
469
470 ret = clk_prepare_enable(clk: ptdev->clks.stacks);
471 if (ret)
472 goto err_disable_core_clk;
473
474 ret = clk_prepare_enable(clk: ptdev->clks.coregroup);
475 if (ret)
476 goto err_disable_stacks_clk;
477
478 panthor_devfreq_resume(ptdev);
479
480 if (panthor_device_is_initialized(ptdev) &&
481 drm_dev_enter(dev: &ptdev->base, idx: &cookie)) {
482 /* If there was a reset pending at the time we suspended the
483 * device, we force a slow reset.
484 */
485 if (atomic_read(v: &ptdev->reset.pending)) {
486 ptdev->reset.fast = false;
487 atomic_set(v: &ptdev->reset.pending, i: 0);
488 }
489
490 ret = panthor_device_resume_hw_components(ptdev);
491 if (ret && ptdev->reset.fast) {
492 drm_err(&ptdev->base, "Fast reset failed, trying a slow reset");
493 ptdev->reset.fast = false;
494 ret = panthor_device_resume_hw_components(ptdev);
495 }
496
497 if (!ret)
498 panthor_sched_resume(ptdev);
499
500 drm_dev_exit(idx: cookie);
501
502 if (ret)
503 goto err_suspend_devfreq;
504 }
505
506 /* Clear all IOMEM mappings pointing to this device after we've
507 * resumed. This way the fake mappings pointing to the dummy pages
508 * are removed and the real iomem mapping will be restored on next
509 * access.
510 */
511 mutex_lock(&ptdev->pm.mmio_lock);
512 unmap_mapping_range(mapping: ptdev->base.anon_inode->i_mapping,
513 DRM_PANTHOR_USER_MMIO_OFFSET, holelen: 0, even_cows: 1);
514 atomic_set(v: &ptdev->pm.state, i: PANTHOR_DEVICE_PM_STATE_ACTIVE);
515 mutex_unlock(lock: &ptdev->pm.mmio_lock);
516 return 0;
517
518err_suspend_devfreq:
519 panthor_devfreq_suspend(ptdev);
520 clk_disable_unprepare(clk: ptdev->clks.coregroup);
521
522err_disable_stacks_clk:
523 clk_disable_unprepare(clk: ptdev->clks.stacks);
524
525err_disable_core_clk:
526 clk_disable_unprepare(clk: ptdev->clks.core);
527
528err_set_suspended:
529 atomic_set(v: &ptdev->pm.state, i: PANTHOR_DEVICE_PM_STATE_SUSPENDED);
530 atomic_set(v: &ptdev->pm.recovery_needed, i: 1);
531 return ret;
532}
533
534int panthor_device_suspend(struct device *dev)
535{
536 struct panthor_device *ptdev = dev_get_drvdata(dev);
537 int cookie;
538
539 if (atomic_read(v: &ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)
540 return -EINVAL;
541
542 /* Clear all IOMEM mappings pointing to this device before we
543 * shutdown the power-domain and clocks. Failing to do that results
544 * in external aborts when the process accesses the iomem region.
545 * We change the state and call unmap_mapping_range() with the
546 * mmio_lock held to make sure the vm_fault handler won't set up
547 * invalid mappings.
548 */
549 mutex_lock(&ptdev->pm.mmio_lock);
550 atomic_set(v: &ptdev->pm.state, i: PANTHOR_DEVICE_PM_STATE_SUSPENDING);
551 unmap_mapping_range(mapping: ptdev->base.anon_inode->i_mapping,
552 DRM_PANTHOR_USER_MMIO_OFFSET, holelen: 0, even_cows: 1);
553 mutex_unlock(lock: &ptdev->pm.mmio_lock);
554
555 if (panthor_device_is_initialized(ptdev) &&
556 drm_dev_enter(dev: &ptdev->base, idx: &cookie)) {
557 cancel_work_sync(work: &ptdev->reset.work);
558
559 /* We prepare everything as if we were resetting the GPU.
560 * The end of the reset will happen in the resume path though.
561 */
562 panthor_sched_suspend(ptdev);
563 panthor_fw_suspend(ptdev);
564 panthor_mmu_suspend(ptdev);
565 panthor_gpu_suspend(ptdev);
566 drm_dev_exit(idx: cookie);
567 }
568
569 panthor_devfreq_suspend(ptdev);
570
571 clk_disable_unprepare(clk: ptdev->clks.coregroup);
572 clk_disable_unprepare(clk: ptdev->clks.stacks);
573 clk_disable_unprepare(clk: ptdev->clks.core);
574 atomic_set(v: &ptdev->pm.state, i: PANTHOR_DEVICE_PM_STATE_SUSPENDED);
575 return 0;
576}
577

source code of linux/drivers/gpu/drm/panthor/panthor_device.c