1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2017 The Linux Foundation. All rights reserved.
3 */
4
5#include "msm_gem.h"
6#include "a5xx_gpu.h"
7
8/*
9 * Try to transition the preemption state from old to new. Return
10 * true on success or false if the original state wasn't 'old'
11 */
12static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
13 enum preempt_state old, enum preempt_state new)
14{
15 enum preempt_state cur = atomic_cmpxchg(v: &a5xx_gpu->preempt_state,
16 old, new);
17
18 return (cur == old);
19}
20
21/*
22 * Force the preemption state to the specified state. This is used in cases
23 * where the current state is known and won't change
24 */
25static inline void set_preempt_state(struct a5xx_gpu *gpu,
26 enum preempt_state new)
27{
28 /*
29 * preempt_state may be read by other cores trying to trigger a
30 * preemption or in the interrupt handler so barriers are needed
31 * before...
32 */
33 smp_mb__before_atomic();
34 atomic_set(v: &gpu->preempt_state, i: new);
35 /* ... and after*/
36 smp_mb__after_atomic();
37}
38
39/* Write the most recent wptr for the given ring into the hardware */
40static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
41{
42 unsigned long flags;
43 uint32_t wptr;
44
45 if (!ring)
46 return;
47
48 spin_lock_irqsave(&ring->preempt_lock, flags);
49 wptr = get_wptr(ring);
50 spin_unlock_irqrestore(lock: &ring->preempt_lock, flags);
51
52 gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
53}
54
55/* Return the highest priority ringbuffer with something in it */
56static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
57{
58 unsigned long flags;
59 int i;
60
61 for (i = 0; i < gpu->nr_rings; i++) {
62 bool empty;
63 struct msm_ringbuffer *ring = gpu->rb[i];
64
65 spin_lock_irqsave(&ring->preempt_lock, flags);
66 empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));
67 spin_unlock_irqrestore(lock: &ring->preempt_lock, flags);
68
69 if (!empty)
70 return ring;
71 }
72
73 return NULL;
74}
75
76static void a5xx_preempt_timer(struct timer_list *t)
77{
78 struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
79 struct msm_gpu *gpu = &a5xx_gpu->base.base;
80 struct drm_device *dev = gpu->dev;
81
82 if (!try_preempt_state(a5xx_gpu, old: PREEMPT_TRIGGERED, new: PREEMPT_FAULTED))
83 return;
84
85 DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
86 kthread_queue_work(gpu->worker, &gpu->recover_work);
87}
88
89/* Try to trigger a preemption switch */
90void a5xx_preempt_trigger(struct msm_gpu *gpu)
91{
92 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
93 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
94 unsigned long flags;
95 struct msm_ringbuffer *ring;
96
97 if (gpu->nr_rings == 1)
98 return;
99
100 /*
101 * Try to start preemption by moving from NONE to START. If
102 * unsuccessful, a preemption is already in flight
103 */
104 if (!try_preempt_state(a5xx_gpu, old: PREEMPT_NONE, new: PREEMPT_START))
105 return;
106
107 /* Get the next ring to preempt to */
108 ring = get_next_ring(gpu);
109
110 /*
111 * If no ring is populated or the highest priority ring is the current
112 * one do nothing except to update the wptr to the latest and greatest
113 */
114 if (!ring || (a5xx_gpu->cur_ring == ring)) {
115 /*
116 * Its possible that while a preemption request is in progress
117 * from an irq context, a user context trying to submit might
118 * fail to update the write pointer, because it determines
119 * that the preempt state is not PREEMPT_NONE.
120 *
121 * Close the race by introducing an intermediate
122 * state PREEMPT_ABORT to let the submit path
123 * know that the ringbuffer is not going to change
124 * and can safely update the write pointer.
125 */
126
127 set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_ABORT);
128 update_wptr(gpu, ring: a5xx_gpu->cur_ring);
129 set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_NONE);
130 return;
131 }
132
133 /* Make sure the wptr doesn't update while we're in motion */
134 spin_lock_irqsave(&ring->preempt_lock, flags);
135 a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
136 spin_unlock_irqrestore(lock: &ring->preempt_lock, flags);
137
138 /* Set the address of the incoming preemption record */
139 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
140 a5xx_gpu->preempt_iova[ring->id]);
141
142 a5xx_gpu->next_ring = ring;
143
144 /* Start a timer to catch a stuck preemption */
145 mod_timer(timer: &a5xx_gpu->preempt_timer, expires: jiffies + msecs_to_jiffies(m: 10000));
146
147 /* Set the preemption state to triggered */
148 set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_TRIGGERED);
149
150 /* Make sure everything is written before hitting the button */
151 wmb();
152
153 /* And actually start the preemption */
154 gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
155}
156
157void a5xx_preempt_irq(struct msm_gpu *gpu)
158{
159 uint32_t status;
160 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
161 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
162 struct drm_device *dev = gpu->dev;
163
164 if (!try_preempt_state(a5xx_gpu, old: PREEMPT_TRIGGERED, new: PREEMPT_PENDING))
165 return;
166
167 /* Delete the preemption watchdog timer */
168 del_timer(timer: &a5xx_gpu->preempt_timer);
169
170 /*
171 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
172 * firing the interrupt, but there is a non zero chance of a hardware
173 * condition or a software race that could set it again before we have a
174 * chance to finish. If that happens, log and go for recovery
175 */
176 status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
177 if (unlikely(status)) {
178 set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_FAULTED);
179 DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
180 gpu->name);
181 kthread_queue_work(gpu->worker, &gpu->recover_work);
182 return;
183 }
184
185 a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
186 a5xx_gpu->next_ring = NULL;
187
188 update_wptr(gpu, ring: a5xx_gpu->cur_ring);
189
190 set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_NONE);
191}
192
193void a5xx_preempt_hw_init(struct msm_gpu *gpu)
194{
195 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
196 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
197 int i;
198
199 /* Always come up on rb 0 */
200 a5xx_gpu->cur_ring = gpu->rb[0];
201
202 /* No preemption if we only have one ring */
203 if (gpu->nr_rings == 1)
204 return;
205
206 for (i = 0; i < gpu->nr_rings; i++) {
207 a5xx_gpu->preempt[i]->wptr = 0;
208 a5xx_gpu->preempt[i]->rptr = 0;
209 a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
210 a5xx_gpu->preempt[i]->rptr_addr = shadowptr(a5xx_gpu, gpu->rb[i]);
211 }
212
213 /* Write a 0 to signal that we aren't switching pagetables */
214 gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 0);
215
216 /* Reset the preemption state */
217 set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_NONE);
218}
219
220static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
221 struct msm_ringbuffer *ring)
222{
223 struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
224 struct msm_gpu *gpu = &adreno_gpu->base;
225 struct a5xx_preempt_record *ptr;
226 void *counters;
227 struct drm_gem_object *bo = NULL, *counters_bo = NULL;
228 u64 iova = 0, counters_iova = 0;
229
230 ptr = msm_gem_kernel_new(gpu->dev,
231 A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
232 MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova);
233
234 if (IS_ERR(ptr))
235 return PTR_ERR(ptr);
236
237 /* The buffer to store counters needs to be unprivileged */
238 counters = msm_gem_kernel_new(gpu->dev,
239 A5XX_PREEMPT_COUNTER_SIZE,
240 MSM_BO_WC, gpu->aspace, &counters_bo, &counters_iova);
241 if (IS_ERR(ptr: counters)) {
242 msm_gem_kernel_put(bo, gpu->aspace);
243 return PTR_ERR(ptr: counters);
244 }
245
246 msm_gem_object_set_name(bo, "preempt");
247 msm_gem_object_set_name(counters_bo, "preempt_counters");
248
249 a5xx_gpu->preempt_bo[ring->id] = bo;
250 a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo;
251 a5xx_gpu->preempt_iova[ring->id] = iova;
252 a5xx_gpu->preempt[ring->id] = ptr;
253
254 /* Set up the defaults on the preemption record */
255
256 ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
257 ptr->info = 0;
258 ptr->data = 0;
259 ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE;
260
261 ptr->counter = counters_iova;
262
263 return 0;
264}
265
266void a5xx_preempt_fini(struct msm_gpu *gpu)
267{
268 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
269 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
270 int i;
271
272 for (i = 0; i < gpu->nr_rings; i++) {
273 msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace);
274 msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], gpu->aspace);
275 }
276}
277
278void a5xx_preempt_init(struct msm_gpu *gpu)
279{
280 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
281 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
282 int i;
283
284 /* No preemption if we only have one ring */
285 if (gpu->nr_rings <= 1)
286 return;
287
288 for (i = 0; i < gpu->nr_rings; i++) {
289 if (preempt_init_ring(a5xx_gpu, ring: gpu->rb[i])) {
290 /*
291 * On any failure our adventure is over. Clean up and
292 * set nr_rings to 1 to force preemption off
293 */
294 a5xx_preempt_fini(gpu);
295 gpu->nr_rings = 1;
296
297 return;
298 }
299 }
300
301 timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
302}
303

source code of linux/drivers/gpu/drm/msm/adreno/a5xx_preempt.c