1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2013 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7#include "drm/drm_drv.h"
8
9#include "msm_gpu.h"
10#include "msm_gem.h"
11#include "msm_mmu.h"
12#include "msm_fence.h"
13#include "msm_gpu_trace.h"
14//#include "adreno/adreno_gpu.h"
15
16#include <generated/utsrelease.h>
17#include <linux/string_helpers.h>
18#include <linux/devcoredump.h>
19#include <linux/sched/task.h>
20
21/*
22 * Power Management:
23 */
24
25static int enable_pwrrail(struct msm_gpu *gpu)
26{
27 struct drm_device *dev = gpu->dev;
28 int ret = 0;
29
30 if (gpu->gpu_reg) {
31 ret = regulator_enable(regulator: gpu->gpu_reg);
32 if (ret) {
33 DRM_DEV_ERROR(dev->dev, "failed to enable 'gpu_reg': %d\n", ret);
34 return ret;
35 }
36 }
37
38 if (gpu->gpu_cx) {
39 ret = regulator_enable(regulator: gpu->gpu_cx);
40 if (ret) {
41 DRM_DEV_ERROR(dev->dev, "failed to enable 'gpu_cx': %d\n", ret);
42 return ret;
43 }
44 }
45
46 return 0;
47}
48
49static int disable_pwrrail(struct msm_gpu *gpu)
50{
51 if (gpu->gpu_cx)
52 regulator_disable(regulator: gpu->gpu_cx);
53 if (gpu->gpu_reg)
54 regulator_disable(regulator: gpu->gpu_reg);
55 return 0;
56}
57
58static int enable_clk(struct msm_gpu *gpu)
59{
60 if (gpu->core_clk && gpu->fast_rate)
61 dev_pm_opp_set_rate(dev: &gpu->pdev->dev, target_freq: gpu->fast_rate);
62
63 /* Set the RBBM timer rate to 19.2Mhz */
64 if (gpu->rbbmtimer_clk)
65 clk_set_rate(clk: gpu->rbbmtimer_clk, rate: 19200000);
66
67 return clk_bulk_prepare_enable(num_clks: gpu->nr_clocks, clks: gpu->grp_clks);
68}
69
70static int disable_clk(struct msm_gpu *gpu)
71{
72 clk_bulk_disable_unprepare(num_clks: gpu->nr_clocks, clks: gpu->grp_clks);
73
74 /*
75 * Set the clock to a deliberately low rate. On older targets the clock
76 * speed had to be non zero to avoid problems. On newer targets this
77 * will be rounded down to zero anyway so it all works out.
78 */
79 if (gpu->core_clk)
80 dev_pm_opp_set_rate(dev: &gpu->pdev->dev, target_freq: 27000000);
81
82 if (gpu->rbbmtimer_clk)
83 clk_set_rate(clk: gpu->rbbmtimer_clk, rate: 0);
84
85 return 0;
86}
87
88static int enable_axi(struct msm_gpu *gpu)
89{
90 return clk_prepare_enable(clk: gpu->ebi1_clk);
91}
92
93static int disable_axi(struct msm_gpu *gpu)
94{
95 clk_disable_unprepare(clk: gpu->ebi1_clk);
96 return 0;
97}
98
99int msm_gpu_pm_resume(struct msm_gpu *gpu)
100{
101 int ret;
102
103 DBG("%s", gpu->name);
104 trace_msm_gpu_resume(dummy: 0);
105
106 ret = enable_pwrrail(gpu);
107 if (ret)
108 return ret;
109
110 ret = enable_clk(gpu);
111 if (ret)
112 return ret;
113
114 ret = enable_axi(gpu);
115 if (ret)
116 return ret;
117
118 msm_devfreq_resume(gpu);
119
120 gpu->needs_hw_init = true;
121
122 return 0;
123}
124
125int msm_gpu_pm_suspend(struct msm_gpu *gpu)
126{
127 int ret;
128
129 DBG("%s", gpu->name);
130 trace_msm_gpu_suspend(dummy: 0);
131
132 msm_devfreq_suspend(gpu);
133
134 ret = disable_axi(gpu);
135 if (ret)
136 return ret;
137
138 ret = disable_clk(gpu);
139 if (ret)
140 return ret;
141
142 ret = disable_pwrrail(gpu);
143 if (ret)
144 return ret;
145
146 gpu->suspend_count++;
147
148 return 0;
149}
150
151void msm_gpu_show_fdinfo(struct msm_gpu *gpu, struct msm_file_private *ctx,
152 struct drm_printer *p)
153{
154 drm_printf(p, f: "drm-engine-gpu:\t%llu ns\n", ctx->elapsed_ns);
155 drm_printf(p, f: "drm-cycles-gpu:\t%llu\n", ctx->cycles);
156 drm_printf(p, f: "drm-maxfreq-gpu:\t%u Hz\n", gpu->fast_rate);
157}
158
159int msm_gpu_hw_init(struct msm_gpu *gpu)
160{
161 int ret;
162
163 WARN_ON(!mutex_is_locked(&gpu->lock));
164
165 if (!gpu->needs_hw_init)
166 return 0;
167
168 disable_irq(irq: gpu->irq);
169 ret = gpu->funcs->hw_init(gpu);
170 if (!ret)
171 gpu->needs_hw_init = false;
172 enable_irq(irq: gpu->irq);
173
174 return ret;
175}
176
177#ifdef CONFIG_DEV_COREDUMP
178static ssize_t msm_gpu_devcoredump_read(char *buffer, loff_t offset,
179 size_t count, void *data, size_t datalen)
180{
181 struct msm_gpu *gpu = data;
182 struct drm_print_iterator iter;
183 struct drm_printer p;
184 struct msm_gpu_state *state;
185
186 state = msm_gpu_crashstate_get(gpu);
187 if (!state)
188 return 0;
189
190 iter.data = buffer;
191 iter.offset = 0;
192 iter.start = offset;
193 iter.remain = count;
194
195 p = drm_coredump_printer(iter: &iter);
196
197 drm_printf(p: &p, f: "---\n");
198 drm_printf(p: &p, f: "kernel: " UTS_RELEASE "\n");
199 drm_printf(p: &p, f: "module: " KBUILD_MODNAME "\n");
200 drm_printf(p: &p, f: "time: %lld.%09ld\n",
201 state->time.tv_sec, state->time.tv_nsec);
202 if (state->comm)
203 drm_printf(p: &p, f: "comm: %s\n", state->comm);
204 if (state->cmd)
205 drm_printf(p: &p, f: "cmdline: %s\n", state->cmd);
206
207 gpu->funcs->show(gpu, state, &p);
208
209 msm_gpu_crashstate_put(gpu);
210
211 return count - iter.remain;
212}
213
214static void msm_gpu_devcoredump_free(void *data)
215{
216 struct msm_gpu *gpu = data;
217
218 msm_gpu_crashstate_put(gpu);
219}
220
221static void msm_gpu_crashstate_get_bo(struct msm_gpu_state *state,
222 struct drm_gem_object *obj, u64 iova, bool full)
223{
224 struct msm_gpu_state_bo *state_bo = &state->bos[state->nr_bos];
225 struct msm_gem_object *msm_obj = to_msm_bo(obj);
226
227 /* Don't record write only objects */
228 state_bo->size = obj->size;
229 state_bo->flags = msm_obj->flags;
230 state_bo->iova = iova;
231
232 BUILD_BUG_ON(sizeof(state_bo->name) != sizeof(msm_obj->name));
233
234 memcpy(state_bo->name, msm_obj->name, sizeof(state_bo->name));
235
236 if (full) {
237 void *ptr;
238
239 state_bo->data = kvmalloc(obj->size, GFP_KERNEL);
240 if (!state_bo->data)
241 goto out;
242
243 msm_gem_lock(obj);
244 ptr = msm_gem_get_vaddr_active(obj);
245 msm_gem_unlock(obj);
246 if (IS_ERR(ptr)) {
247 kvfree(addr: state_bo->data);
248 state_bo->data = NULL;
249 goto out;
250 }
251
252 memcpy(state_bo->data, ptr, obj->size);
253 msm_gem_put_vaddr(obj);
254 }
255out:
256 state->nr_bos++;
257}
258
259static void msm_gpu_crashstate_capture(struct msm_gpu *gpu,
260 struct msm_gem_submit *submit, char *comm, char *cmd)
261{
262 struct msm_gpu_state *state;
263
264 /* Check if the target supports capturing crash state */
265 if (!gpu->funcs->gpu_state_get)
266 return;
267
268 /* Only save one crash state at a time */
269 if (gpu->crashstate)
270 return;
271
272 state = gpu->funcs->gpu_state_get(gpu);
273 if (IS_ERR_OR_NULL(ptr: state))
274 return;
275
276 /* Fill in the additional crash state information */
277 state->comm = kstrdup(s: comm, GFP_KERNEL);
278 state->cmd = kstrdup(s: cmd, GFP_KERNEL);
279 state->fault_info = gpu->fault_info;
280
281 if (submit) {
282 int i;
283
284 if (state->fault_info.ttbr0) {
285 struct msm_gpu_fault_info *info = &state->fault_info;
286 struct msm_mmu *mmu = submit->aspace->mmu;
287
288 msm_iommu_pagetable_params(mmu, ttbr: &info->pgtbl_ttbr0,
289 asid: &info->asid);
290 msm_iommu_pagetable_walk(mmu, iova: info->iova, ptes: info->ptes);
291 }
292
293 state->bos = kcalloc(submit->nr_bos,
294 sizeof(struct msm_gpu_state_bo), GFP_KERNEL);
295
296 for (i = 0; state->bos && i < submit->nr_bos; i++) {
297 msm_gpu_crashstate_get_bo(state, obj: submit->bos[i].obj,
298 iova: submit->bos[i].iova,
299 full: should_dump(submit, idx: i));
300 }
301 }
302
303 /* Set the active crash state to be dumped on failure */
304 gpu->crashstate = state;
305
306 dev_coredumpm(dev: &gpu->pdev->dev, THIS_MODULE, data: gpu, datalen: 0, GFP_KERNEL,
307 read: msm_gpu_devcoredump_read, free: msm_gpu_devcoredump_free);
308}
309#else
310static void msm_gpu_crashstate_capture(struct msm_gpu *gpu,
311 struct msm_gem_submit *submit, char *comm, char *cmd)
312{
313}
314#endif
315
316/*
317 * Hangcheck detection for locked gpu:
318 */
319
320static struct msm_gem_submit *
321find_submit(struct msm_ringbuffer *ring, uint32_t fence)
322{
323 struct msm_gem_submit *submit;
324 unsigned long flags;
325
326 spin_lock_irqsave(&ring->submit_lock, flags);
327 list_for_each_entry(submit, &ring->submits, node) {
328 if (submit->seqno == fence) {
329 spin_unlock_irqrestore(lock: &ring->submit_lock, flags);
330 return submit;
331 }
332 }
333 spin_unlock_irqrestore(lock: &ring->submit_lock, flags);
334
335 return NULL;
336}
337
338static void retire_submits(struct msm_gpu *gpu);
339
340static void get_comm_cmdline(struct msm_gem_submit *submit, char **comm, char **cmd)
341{
342 struct msm_file_private *ctx = submit->queue->ctx;
343 struct task_struct *task;
344
345 WARN_ON(!mutex_is_locked(&submit->gpu->lock));
346
347 /* Note that kstrdup will return NULL if argument is NULL: */
348 *comm = kstrdup(s: ctx->comm, GFP_KERNEL);
349 *cmd = kstrdup(s: ctx->cmdline, GFP_KERNEL);
350
351 task = get_pid_task(pid: submit->pid, PIDTYPE_PID);
352 if (!task)
353 return;
354
355 if (!*comm)
356 *comm = kstrdup(s: task->comm, GFP_KERNEL);
357
358 if (!*cmd)
359 *cmd = kstrdup_quotable_cmdline(task, GFP_KERNEL);
360
361 put_task_struct(t: task);
362}
363
364static void recover_worker(struct kthread_work *work)
365{
366 struct msm_gpu *gpu = container_of(work, struct msm_gpu, recover_work);
367 struct drm_device *dev = gpu->dev;
368 struct msm_drm_private *priv = dev->dev_private;
369 struct msm_gem_submit *submit;
370 struct msm_ringbuffer *cur_ring = gpu->funcs->active_ring(gpu);
371 char *comm = NULL, *cmd = NULL;
372 int i;
373
374 mutex_lock(&gpu->lock);
375
376 DRM_DEV_ERROR(dev->dev, "%s: hangcheck recover!\n", gpu->name);
377
378 submit = find_submit(ring: cur_ring, fence: cur_ring->memptrs->fence + 1);
379
380 /*
381 * If the submit retired while we were waiting for the worker to run,
382 * or waiting to acquire the gpu lock, then nothing more to do.
383 */
384 if (!submit)
385 goto out_unlock;
386
387 /* Increment the fault counts */
388 submit->queue->faults++;
389 if (submit->aspace)
390 submit->aspace->faults++;
391
392 get_comm_cmdline(submit, comm: &comm, cmd: &cmd);
393
394 if (comm && cmd) {
395 DRM_DEV_ERROR(dev->dev, "%s: offending task: %s (%s)\n",
396 gpu->name, comm, cmd);
397
398 msm_rd_dump_submit(rd: priv->hangrd, submit,
399 fmt: "offending task: %s (%s)", comm, cmd);
400 } else {
401 DRM_DEV_ERROR(dev->dev, "%s: offending task: unknown\n", gpu->name);
402
403 msm_rd_dump_submit(rd: priv->hangrd, submit, NULL);
404 }
405
406 /* Record the crash state */
407 pm_runtime_get_sync(dev: &gpu->pdev->dev);
408 msm_gpu_crashstate_capture(gpu, submit, comm, cmd);
409
410 kfree(objp: cmd);
411 kfree(objp: comm);
412
413 /*
414 * Update all the rings with the latest and greatest fence.. this
415 * needs to happen after msm_rd_dump_submit() to ensure that the
416 * bo's referenced by the offending submit are still around.
417 */
418 for (i = 0; i < gpu->nr_rings; i++) {
419 struct msm_ringbuffer *ring = gpu->rb[i];
420
421 uint32_t fence = ring->memptrs->fence;
422
423 /*
424 * For the current (faulting?) ring/submit advance the fence by
425 * one more to clear the faulting submit
426 */
427 if (ring == cur_ring)
428 ring->memptrs->fence = ++fence;
429
430 msm_update_fence(fctx: ring->fctx, fence);
431 }
432
433 if (msm_gpu_active(gpu)) {
434 /* retire completed submits, plus the one that hung: */
435 retire_submits(gpu);
436
437 gpu->funcs->recover(gpu);
438
439 /*
440 * Replay all remaining submits starting with highest priority
441 * ring
442 */
443 for (i = 0; i < gpu->nr_rings; i++) {
444 struct msm_ringbuffer *ring = gpu->rb[i];
445 unsigned long flags;
446
447 spin_lock_irqsave(&ring->submit_lock, flags);
448 list_for_each_entry(submit, &ring->submits, node)
449 gpu->funcs->submit(gpu, submit);
450 spin_unlock_irqrestore(lock: &ring->submit_lock, flags);
451 }
452 }
453
454 pm_runtime_put(dev: &gpu->pdev->dev);
455
456out_unlock:
457 mutex_unlock(lock: &gpu->lock);
458
459 msm_gpu_retire(gpu);
460}
461
462static void fault_worker(struct kthread_work *work)
463{
464 struct msm_gpu *gpu = container_of(work, struct msm_gpu, fault_work);
465 struct msm_gem_submit *submit;
466 struct msm_ringbuffer *cur_ring = gpu->funcs->active_ring(gpu);
467 char *comm = NULL, *cmd = NULL;
468
469 mutex_lock(&gpu->lock);
470
471 submit = find_submit(ring: cur_ring, fence: cur_ring->memptrs->fence + 1);
472 if (submit && submit->fault_dumped)
473 goto resume_smmu;
474
475 if (submit) {
476 get_comm_cmdline(submit, comm: &comm, cmd: &cmd);
477
478 /*
479 * When we get GPU iova faults, we can get 1000s of them,
480 * but we really only want to log the first one.
481 */
482 submit->fault_dumped = true;
483 }
484
485 /* Record the crash state */
486 pm_runtime_get_sync(dev: &gpu->pdev->dev);
487 msm_gpu_crashstate_capture(gpu, submit, comm, cmd);
488 pm_runtime_put_sync(dev: &gpu->pdev->dev);
489
490 kfree(objp: cmd);
491 kfree(objp: comm);
492
493resume_smmu:
494 memset(&gpu->fault_info, 0, sizeof(gpu->fault_info));
495 gpu->aspace->mmu->funcs->resume_translation(gpu->aspace->mmu);
496
497 mutex_unlock(lock: &gpu->lock);
498}
499
500static void hangcheck_timer_reset(struct msm_gpu *gpu)
501{
502 struct msm_drm_private *priv = gpu->dev->dev_private;
503 mod_timer(timer: &gpu->hangcheck_timer,
504 expires: round_jiffies_up(j: jiffies + msecs_to_jiffies(m: priv->hangcheck_period)));
505}
506
507static bool made_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
508{
509 if (ring->hangcheck_progress_retries >= DRM_MSM_HANGCHECK_PROGRESS_RETRIES)
510 return false;
511
512 if (!gpu->funcs->progress)
513 return false;
514
515 if (!gpu->funcs->progress(gpu, ring))
516 return false;
517
518 ring->hangcheck_progress_retries++;
519 return true;
520}
521
522static void hangcheck_handler(struct timer_list *t)
523{
524 struct msm_gpu *gpu = timer_container_of(gpu, t, hangcheck_timer);
525 struct drm_device *dev = gpu->dev;
526 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
527 uint32_t fence = ring->memptrs->fence;
528
529 if (fence != ring->hangcheck_fence) {
530 /* some progress has been made.. ya! */
531 ring->hangcheck_fence = fence;
532 ring->hangcheck_progress_retries = 0;
533 } else if (fence_before(a: fence, b: ring->fctx->last_fence) &&
534 !made_progress(gpu, ring)) {
535 /* no progress and not done.. hung! */
536 ring->hangcheck_fence = fence;
537 ring->hangcheck_progress_retries = 0;
538 DRM_DEV_ERROR(dev->dev, "%s: hangcheck detected gpu lockup rb %d!\n",
539 gpu->name, ring->id);
540 DRM_DEV_ERROR(dev->dev, "%s: completed fence: %u\n",
541 gpu->name, fence);
542 DRM_DEV_ERROR(dev->dev, "%s: submitted fence: %u\n",
543 gpu->name, ring->fctx->last_fence);
544
545 kthread_queue_work(worker: gpu->worker, work: &gpu->recover_work);
546 }
547
548 /* if still more pending work, reset the hangcheck timer: */
549 if (fence_after(a: ring->fctx->last_fence, b: ring->hangcheck_fence))
550 hangcheck_timer_reset(gpu);
551
552 /* workaround for missing irq: */
553 msm_gpu_retire(gpu);
554}
555
556/*
557 * Performance Counters:
558 */
559
560/* called under perf_lock */
561static int update_hw_cntrs(struct msm_gpu *gpu, uint32_t ncntrs, uint32_t *cntrs)
562{
563 uint32_t current_cntrs[ARRAY_SIZE(gpu->last_cntrs)];
564 int i, n = min(ncntrs, gpu->num_perfcntrs);
565
566 /* read current values: */
567 for (i = 0; i < gpu->num_perfcntrs; i++)
568 current_cntrs[i] = gpu_read(gpu, reg: gpu->perfcntrs[i].sample_reg);
569
570 /* update cntrs: */
571 for (i = 0; i < n; i++)
572 cntrs[i] = current_cntrs[i] - gpu->last_cntrs[i];
573
574 /* save current values: */
575 for (i = 0; i < gpu->num_perfcntrs; i++)
576 gpu->last_cntrs[i] = current_cntrs[i];
577
578 return n;
579}
580
581static void update_sw_cntrs(struct msm_gpu *gpu)
582{
583 ktime_t time;
584 uint32_t elapsed;
585 unsigned long flags;
586
587 spin_lock_irqsave(&gpu->perf_lock, flags);
588 if (!gpu->perfcntr_active)
589 goto out;
590
591 time = ktime_get();
592 elapsed = ktime_to_us(ktime_sub(time, gpu->last_sample.time));
593
594 gpu->totaltime += elapsed;
595 if (gpu->last_sample.active)
596 gpu->activetime += elapsed;
597
598 gpu->last_sample.active = msm_gpu_active(gpu);
599 gpu->last_sample.time = time;
600
601out:
602 spin_unlock_irqrestore(lock: &gpu->perf_lock, flags);
603}
604
605void msm_gpu_perfcntr_start(struct msm_gpu *gpu)
606{
607 unsigned long flags;
608
609 pm_runtime_get_sync(dev: &gpu->pdev->dev);
610
611 spin_lock_irqsave(&gpu->perf_lock, flags);
612 /* we could dynamically enable/disable perfcntr registers too.. */
613 gpu->last_sample.active = msm_gpu_active(gpu);
614 gpu->last_sample.time = ktime_get();
615 gpu->activetime = gpu->totaltime = 0;
616 gpu->perfcntr_active = true;
617 update_hw_cntrs(gpu, ncntrs: 0, NULL);
618 spin_unlock_irqrestore(lock: &gpu->perf_lock, flags);
619}
620
621void msm_gpu_perfcntr_stop(struct msm_gpu *gpu)
622{
623 gpu->perfcntr_active = false;
624 pm_runtime_put_sync(dev: &gpu->pdev->dev);
625}
626
627/* returns -errno or # of cntrs sampled */
628int msm_gpu_perfcntr_sample(struct msm_gpu *gpu, uint32_t *activetime,
629 uint32_t *totaltime, uint32_t ncntrs, uint32_t *cntrs)
630{
631 unsigned long flags;
632 int ret;
633
634 spin_lock_irqsave(&gpu->perf_lock, flags);
635
636 if (!gpu->perfcntr_active) {
637 ret = -EINVAL;
638 goto out;
639 }
640
641 *activetime = gpu->activetime;
642 *totaltime = gpu->totaltime;
643
644 gpu->activetime = gpu->totaltime = 0;
645
646 ret = update_hw_cntrs(gpu, ncntrs, cntrs);
647
648out:
649 spin_unlock_irqrestore(lock: &gpu->perf_lock, flags);
650
651 return ret;
652}
653
654/*
655 * Cmdstream submission/retirement:
656 */
657
658static void retire_submit(struct msm_gpu *gpu, struct msm_ringbuffer *ring,
659 struct msm_gem_submit *submit)
660{
661 int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT;
662 volatile struct msm_gpu_submit_stats *stats;
663 u64 elapsed, clock = 0, cycles;
664 unsigned long flags;
665
666 stats = &ring->memptrs->stats[index];
667 /* Convert 19.2Mhz alwayson ticks to nanoseconds for elapsed time */
668 elapsed = (stats->alwayson_end - stats->alwayson_start) * 10000;
669 do_div(elapsed, 192);
670
671 cycles = stats->cpcycles_end - stats->cpcycles_start;
672
673 /* Calculate the clock frequency from the number of CP cycles */
674 if (elapsed) {
675 clock = cycles * 1000;
676 do_div(clock, elapsed);
677 }
678
679 submit->queue->ctx->elapsed_ns += elapsed;
680 submit->queue->ctx->cycles += cycles;
681
682 trace_msm_gpu_submit_retired(submit, elapsed, clock,
683 start: stats->alwayson_start, end: stats->alwayson_end);
684
685 msm_submit_retire(submit);
686
687 pm_runtime_mark_last_busy(dev: &gpu->pdev->dev);
688
689 spin_lock_irqsave(&ring->submit_lock, flags);
690 list_del(entry: &submit->node);
691 spin_unlock_irqrestore(lock: &ring->submit_lock, flags);
692
693 /* Update devfreq on transition from active->idle: */
694 mutex_lock(&gpu->active_lock);
695 gpu->active_submits--;
696 WARN_ON(gpu->active_submits < 0);
697 if (!gpu->active_submits) {
698 msm_devfreq_idle(gpu);
699 pm_runtime_put_autosuspend(dev: &gpu->pdev->dev);
700 }
701
702 mutex_unlock(lock: &gpu->active_lock);
703
704 msm_gem_submit_put(submit);
705}
706
707static void retire_submits(struct msm_gpu *gpu)
708{
709 int i;
710
711 /* Retire the commits starting with highest priority */
712 for (i = 0; i < gpu->nr_rings; i++) {
713 struct msm_ringbuffer *ring = gpu->rb[i];
714
715 while (true) {
716 struct msm_gem_submit *submit = NULL;
717 unsigned long flags;
718
719 spin_lock_irqsave(&ring->submit_lock, flags);
720 submit = list_first_entry_or_null(&ring->submits,
721 struct msm_gem_submit, node);
722 spin_unlock_irqrestore(lock: &ring->submit_lock, flags);
723
724 /*
725 * If no submit, we are done. If submit->fence hasn't
726 * been signalled, then later submits are not signalled
727 * either, so we are also done.
728 */
729 if (submit && dma_fence_is_signaled(fence: submit->hw_fence)) {
730 retire_submit(gpu, ring, submit);
731 } else {
732 break;
733 }
734 }
735 }
736
737 wake_up_all(&gpu->retire_event);
738}
739
740static void retire_worker(struct kthread_work *work)
741{
742 struct msm_gpu *gpu = container_of(work, struct msm_gpu, retire_work);
743
744 retire_submits(gpu);
745}
746
747/* call from irq handler to schedule work to retire bo's */
748void msm_gpu_retire(struct msm_gpu *gpu)
749{
750 int i;
751
752 for (i = 0; i < gpu->nr_rings; i++)
753 msm_update_fence(fctx: gpu->rb[i]->fctx, fence: gpu->rb[i]->memptrs->fence);
754
755 kthread_queue_work(worker: gpu->worker, work: &gpu->retire_work);
756 update_sw_cntrs(gpu);
757}
758
759/* add bo's to gpu's ring, and kick gpu: */
760void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
761{
762 struct msm_ringbuffer *ring = submit->ring;
763 unsigned long flags;
764
765 WARN_ON(!mutex_is_locked(&gpu->lock));
766
767 pm_runtime_get_sync(dev: &gpu->pdev->dev);
768
769 msm_gpu_hw_init(gpu);
770
771 submit->seqno = submit->hw_fence->seqno;
772
773 update_sw_cntrs(gpu);
774
775 /*
776 * ring->submits holds a ref to the submit, to deal with the case
777 * that a submit completes before msm_ioctl_gem_submit() returns.
778 */
779 msm_gem_submit_get(submit);
780
781 spin_lock_irqsave(&ring->submit_lock, flags);
782 list_add_tail(new: &submit->node, head: &ring->submits);
783 spin_unlock_irqrestore(lock: &ring->submit_lock, flags);
784
785 /* Update devfreq on transition from idle->active: */
786 mutex_lock(&gpu->active_lock);
787 if (!gpu->active_submits) {
788 pm_runtime_get(dev: &gpu->pdev->dev);
789 msm_devfreq_active(gpu);
790 }
791 gpu->active_submits++;
792 mutex_unlock(lock: &gpu->active_lock);
793
794 gpu->funcs->submit(gpu, submit);
795 submit->ring->cur_ctx_seqno = submit->queue->ctx->seqno;
796
797 pm_runtime_put(dev: &gpu->pdev->dev);
798 hangcheck_timer_reset(gpu);
799}
800
801/*
802 * Init/Cleanup:
803 */
804
805static irqreturn_t irq_handler(int irq, void *data)
806{
807 struct msm_gpu *gpu = data;
808 return gpu->funcs->irq(gpu);
809}
810
811static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu)
812{
813 int ret = devm_clk_bulk_get_all(dev: &pdev->dev, clks: &gpu->grp_clks);
814
815 if (ret < 1) {
816 gpu->nr_clocks = 0;
817 return ret;
818 }
819
820 gpu->nr_clocks = ret;
821
822 gpu->core_clk = msm_clk_bulk_get_clock(bulk: gpu->grp_clks,
823 count: gpu->nr_clocks, name: "core");
824
825 gpu->rbbmtimer_clk = msm_clk_bulk_get_clock(bulk: gpu->grp_clks,
826 count: gpu->nr_clocks, name: "rbbmtimer");
827
828 return 0;
829}
830
831/* Return a new address space for a msm_drm_private instance */
832struct msm_gem_address_space *
833msm_gpu_create_private_address_space(struct msm_gpu *gpu, struct task_struct *task)
834{
835 struct msm_gem_address_space *aspace = NULL;
836 if (!gpu)
837 return NULL;
838
839 /*
840 * If the target doesn't support private address spaces then return
841 * the global one
842 */
843 if (gpu->funcs->create_private_address_space) {
844 aspace = gpu->funcs->create_private_address_space(gpu);
845 if (!IS_ERR(ptr: aspace))
846 aspace->pid = get_pid(pid: task_pid(task));
847 }
848
849 if (IS_ERR_OR_NULL(ptr: aspace))
850 aspace = msm_gem_address_space_get(aspace: gpu->aspace);
851
852 return aspace;
853}
854
855int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
856 struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs,
857 const char *name, struct msm_gpu_config *config)
858{
859 struct msm_drm_private *priv = drm->dev_private;
860 int i, ret, nr_rings = config->nr_rings;
861 void *memptrs;
862 uint64_t memptrs_iova;
863
864 if (WARN_ON(gpu->num_perfcntrs > ARRAY_SIZE(gpu->last_cntrs)))
865 gpu->num_perfcntrs = ARRAY_SIZE(gpu->last_cntrs);
866
867 gpu->dev = drm;
868 gpu->funcs = funcs;
869 gpu->name = name;
870
871 gpu->worker = kthread_run_worker(0, "gpu-worker");
872 if (IS_ERR(ptr: gpu->worker)) {
873 ret = PTR_ERR(ptr: gpu->worker);
874 gpu->worker = NULL;
875 goto fail;
876 }
877
878 sched_set_fifo_low(p: gpu->worker->task);
879
880 mutex_init(&gpu->active_lock);
881 mutex_init(&gpu->lock);
882 init_waitqueue_head(&gpu->retire_event);
883 kthread_init_work(&gpu->retire_work, retire_worker);
884 kthread_init_work(&gpu->recover_work, recover_worker);
885 kthread_init_work(&gpu->fault_work, fault_worker);
886
887 priv->hangcheck_period = DRM_MSM_HANGCHECK_DEFAULT_PERIOD;
888
889 /*
890 * If progress detection is supported, halve the hangcheck timer
891 * duration, as it takes two iterations of the hangcheck handler
892 * to detect a hang.
893 */
894 if (funcs->progress)
895 priv->hangcheck_period /= 2;
896
897 timer_setup(&gpu->hangcheck_timer, hangcheck_handler, 0);
898
899 spin_lock_init(&gpu->perf_lock);
900
901
902 /* Map registers: */
903 gpu->mmio = msm_ioremap(pdev, name: config->ioname);
904 if (IS_ERR(ptr: gpu->mmio)) {
905 ret = PTR_ERR(ptr: gpu->mmio);
906 goto fail;
907 }
908
909 /* Get Interrupt: */
910 gpu->irq = platform_get_irq(pdev, 0);
911 if (gpu->irq < 0) {
912 ret = gpu->irq;
913 goto fail;
914 }
915
916 ret = devm_request_irq(dev: &pdev->dev, irq: gpu->irq, handler: irq_handler,
917 IRQF_TRIGGER_HIGH, devname: "gpu-irq", dev_id: gpu);
918 if (ret) {
919 DRM_DEV_ERROR(drm->dev, "failed to request IRQ%u: %d\n", gpu->irq, ret);
920 goto fail;
921 }
922
923 ret = get_clocks(pdev, gpu);
924 if (ret)
925 goto fail;
926
927 gpu->ebi1_clk = msm_clk_get(pdev, name: "bus");
928 DBG("ebi1_clk: %p", gpu->ebi1_clk);
929 if (IS_ERR(ptr: gpu->ebi1_clk))
930 gpu->ebi1_clk = NULL;
931
932 /* Acquire regulators: */
933 gpu->gpu_reg = devm_regulator_get(dev: &pdev->dev, id: "vdd");
934 DBG("gpu_reg: %p", gpu->gpu_reg);
935 if (IS_ERR(ptr: gpu->gpu_reg))
936 gpu->gpu_reg = NULL;
937
938 gpu->gpu_cx = devm_regulator_get(dev: &pdev->dev, id: "vddcx");
939 DBG("gpu_cx: %p", gpu->gpu_cx);
940 if (IS_ERR(ptr: gpu->gpu_cx))
941 gpu->gpu_cx = NULL;
942
943 platform_set_drvdata(pdev, data: &gpu->adreno_smmu);
944
945 msm_devfreq_init(gpu);
946
947
948 gpu->aspace = gpu->funcs->create_address_space(gpu, pdev);
949
950 if (gpu->aspace == NULL)
951 DRM_DEV_INFO(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);
952 else if (IS_ERR(ptr: gpu->aspace)) {
953 ret = PTR_ERR(ptr: gpu->aspace);
954 goto fail;
955 }
956
957 memptrs = msm_gem_kernel_new(dev: drm,
958 size: sizeof(struct msm_rbmemptrs) * nr_rings,
959 check_apriv(gpu, MSM_BO_WC), aspace: gpu->aspace, bo: &gpu->memptrs_bo,
960 iova: &memptrs_iova);
961
962 if (IS_ERR(ptr: memptrs)) {
963 ret = PTR_ERR(ptr: memptrs);
964 DRM_DEV_ERROR(drm->dev, "could not allocate memptrs: %d\n", ret);
965 goto fail;
966 }
967
968 msm_gem_object_set_name(bo: gpu->memptrs_bo, fmt: "memptrs");
969
970 if (nr_rings > ARRAY_SIZE(gpu->rb)) {
971 DRM_DEV_INFO_ONCE(drm->dev, "Only creating %zu ringbuffers\n",
972 ARRAY_SIZE(gpu->rb));
973 nr_rings = ARRAY_SIZE(gpu->rb);
974 }
975
976 /* Create ringbuffer(s): */
977 for (i = 0; i < nr_rings; i++) {
978 gpu->rb[i] = msm_ringbuffer_new(gpu, id: i, memptrs, memptrs_iova);
979
980 if (IS_ERR(ptr: gpu->rb[i])) {
981 ret = PTR_ERR(ptr: gpu->rb[i]);
982 DRM_DEV_ERROR(drm->dev,
983 "could not create ringbuffer %d: %d\n", i, ret);
984 goto fail;
985 }
986
987 memptrs += sizeof(struct msm_rbmemptrs);
988 memptrs_iova += sizeof(struct msm_rbmemptrs);
989 }
990
991 gpu->nr_rings = nr_rings;
992
993 refcount_set(r: &gpu->sysprof_active, n: 1);
994
995 return 0;
996
997fail:
998 for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) {
999 msm_ringbuffer_destroy(ring: gpu->rb[i]);
1000 gpu->rb[i] = NULL;
1001 }
1002
1003 msm_gem_kernel_put(bo: gpu->memptrs_bo, aspace: gpu->aspace);
1004
1005 platform_set_drvdata(pdev, NULL);
1006 return ret;
1007}
1008
1009void msm_gpu_cleanup(struct msm_gpu *gpu)
1010{
1011 int i;
1012
1013 DBG("%s", gpu->name);
1014
1015 for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) {
1016 msm_ringbuffer_destroy(ring: gpu->rb[i]);
1017 gpu->rb[i] = NULL;
1018 }
1019
1020 msm_gem_kernel_put(bo: gpu->memptrs_bo, aspace: gpu->aspace);
1021
1022 if (!IS_ERR_OR_NULL(ptr: gpu->aspace)) {
1023 gpu->aspace->mmu->funcs->detach(gpu->aspace->mmu);
1024 msm_gem_address_space_put(aspace: gpu->aspace);
1025 }
1026
1027 if (gpu->worker) {
1028 kthread_destroy_worker(worker: gpu->worker);
1029 }
1030
1031 msm_devfreq_cleanup(gpu);
1032
1033 platform_set_drvdata(pdev: gpu->pdev, NULL);
1034}
1035

source code of linux/drivers/gpu/drm/msm/msm_gpu.c