1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* Copyright (c) 2017 The Linux Foundation. All rights reserved. |
3 | */ |
4 | |
5 | #include "msm_gem.h" |
6 | #include "a5xx_gpu.h" |
7 | |
8 | /* |
9 | * Try to transition the preemption state from old to new. Return |
10 | * true on success or false if the original state wasn't 'old' |
11 | */ |
12 | static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu, |
13 | enum preempt_state old, enum preempt_state new) |
14 | { |
15 | enum preempt_state cur = atomic_cmpxchg(v: &a5xx_gpu->preempt_state, |
16 | old, new); |
17 | |
18 | return (cur == old); |
19 | } |
20 | |
21 | /* |
22 | * Force the preemption state to the specified state. This is used in cases |
23 | * where the current state is known and won't change |
24 | */ |
25 | static inline void set_preempt_state(struct a5xx_gpu *gpu, |
26 | enum preempt_state new) |
27 | { |
28 | /* |
29 | * preempt_state may be read by other cores trying to trigger a |
30 | * preemption or in the interrupt handler so barriers are needed |
31 | * before... |
32 | */ |
33 | smp_mb__before_atomic(); |
34 | atomic_set(v: &gpu->preempt_state, i: new); |
35 | /* ... and after*/ |
36 | smp_mb__after_atomic(); |
37 | } |
38 | |
39 | /* Write the most recent wptr for the given ring into the hardware */ |
40 | static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring) |
41 | { |
42 | unsigned long flags; |
43 | uint32_t wptr; |
44 | |
45 | if (!ring) |
46 | return; |
47 | |
48 | spin_lock_irqsave(&ring->preempt_lock, flags); |
49 | wptr = get_wptr(ring); |
50 | spin_unlock_irqrestore(lock: &ring->preempt_lock, flags); |
51 | |
52 | gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr); |
53 | } |
54 | |
55 | /* Return the highest priority ringbuffer with something in it */ |
56 | static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu) |
57 | { |
58 | unsigned long flags; |
59 | int i; |
60 | |
61 | for (i = 0; i < gpu->nr_rings; i++) { |
62 | bool empty; |
63 | struct msm_ringbuffer *ring = gpu->rb[i]; |
64 | |
65 | spin_lock_irqsave(&ring->preempt_lock, flags); |
66 | empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring)); |
67 | spin_unlock_irqrestore(lock: &ring->preempt_lock, flags); |
68 | |
69 | if (!empty) |
70 | return ring; |
71 | } |
72 | |
73 | return NULL; |
74 | } |
75 | |
76 | static void a5xx_preempt_timer(struct timer_list *t) |
77 | { |
78 | struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer); |
79 | struct msm_gpu *gpu = &a5xx_gpu->base.base; |
80 | struct drm_device *dev = gpu->dev; |
81 | |
82 | if (!try_preempt_state(a5xx_gpu, old: PREEMPT_TRIGGERED, new: PREEMPT_FAULTED)) |
83 | return; |
84 | |
85 | DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n" , gpu->name); |
86 | kthread_queue_work(gpu->worker, &gpu->recover_work); |
87 | } |
88 | |
89 | /* Try to trigger a preemption switch */ |
90 | void a5xx_preempt_trigger(struct msm_gpu *gpu) |
91 | { |
92 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); |
93 | struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); |
94 | unsigned long flags; |
95 | struct msm_ringbuffer *ring; |
96 | |
97 | if (gpu->nr_rings == 1) |
98 | return; |
99 | |
100 | /* |
101 | * Try to start preemption by moving from NONE to START. If |
102 | * unsuccessful, a preemption is already in flight |
103 | */ |
104 | if (!try_preempt_state(a5xx_gpu, old: PREEMPT_NONE, new: PREEMPT_START)) |
105 | return; |
106 | |
107 | /* Get the next ring to preempt to */ |
108 | ring = get_next_ring(gpu); |
109 | |
110 | /* |
111 | * If no ring is populated or the highest priority ring is the current |
112 | * one do nothing except to update the wptr to the latest and greatest |
113 | */ |
114 | if (!ring || (a5xx_gpu->cur_ring == ring)) { |
115 | /* |
116 | * Its possible that while a preemption request is in progress |
117 | * from an irq context, a user context trying to submit might |
118 | * fail to update the write pointer, because it determines |
119 | * that the preempt state is not PREEMPT_NONE. |
120 | * |
121 | * Close the race by introducing an intermediate |
122 | * state PREEMPT_ABORT to let the submit path |
123 | * know that the ringbuffer is not going to change |
124 | * and can safely update the write pointer. |
125 | */ |
126 | |
127 | set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_ABORT); |
128 | update_wptr(gpu, ring: a5xx_gpu->cur_ring); |
129 | set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_NONE); |
130 | return; |
131 | } |
132 | |
133 | /* Make sure the wptr doesn't update while we're in motion */ |
134 | spin_lock_irqsave(&ring->preempt_lock, flags); |
135 | a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring); |
136 | spin_unlock_irqrestore(lock: &ring->preempt_lock, flags); |
137 | |
138 | /* Set the address of the incoming preemption record */ |
139 | gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO, |
140 | a5xx_gpu->preempt_iova[ring->id]); |
141 | |
142 | a5xx_gpu->next_ring = ring; |
143 | |
144 | /* Start a timer to catch a stuck preemption */ |
145 | mod_timer(timer: &a5xx_gpu->preempt_timer, expires: jiffies + msecs_to_jiffies(m: 10000)); |
146 | |
147 | /* Set the preemption state to triggered */ |
148 | set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_TRIGGERED); |
149 | |
150 | /* Make sure everything is written before hitting the button */ |
151 | wmb(); |
152 | |
153 | /* And actually start the preemption */ |
154 | gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1); |
155 | } |
156 | |
157 | void a5xx_preempt_irq(struct msm_gpu *gpu) |
158 | { |
159 | uint32_t status; |
160 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); |
161 | struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); |
162 | struct drm_device *dev = gpu->dev; |
163 | |
164 | if (!try_preempt_state(a5xx_gpu, old: PREEMPT_TRIGGERED, new: PREEMPT_PENDING)) |
165 | return; |
166 | |
167 | /* Delete the preemption watchdog timer */ |
168 | del_timer(timer: &a5xx_gpu->preempt_timer); |
169 | |
170 | /* |
171 | * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before |
172 | * firing the interrupt, but there is a non zero chance of a hardware |
173 | * condition or a software race that could set it again before we have a |
174 | * chance to finish. If that happens, log and go for recovery |
175 | */ |
176 | status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL); |
177 | if (unlikely(status)) { |
178 | set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_FAULTED); |
179 | DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n" , |
180 | gpu->name); |
181 | kthread_queue_work(gpu->worker, &gpu->recover_work); |
182 | return; |
183 | } |
184 | |
185 | a5xx_gpu->cur_ring = a5xx_gpu->next_ring; |
186 | a5xx_gpu->next_ring = NULL; |
187 | |
188 | update_wptr(gpu, ring: a5xx_gpu->cur_ring); |
189 | |
190 | set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_NONE); |
191 | } |
192 | |
193 | void a5xx_preempt_hw_init(struct msm_gpu *gpu) |
194 | { |
195 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); |
196 | struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); |
197 | int i; |
198 | |
199 | /* Always come up on rb 0 */ |
200 | a5xx_gpu->cur_ring = gpu->rb[0]; |
201 | |
202 | /* No preemption if we only have one ring */ |
203 | if (gpu->nr_rings == 1) |
204 | return; |
205 | |
206 | for (i = 0; i < gpu->nr_rings; i++) { |
207 | a5xx_gpu->preempt[i]->wptr = 0; |
208 | a5xx_gpu->preempt[i]->rptr = 0; |
209 | a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova; |
210 | a5xx_gpu->preempt[i]->rptr_addr = shadowptr(a5xx_gpu, gpu->rb[i]); |
211 | } |
212 | |
213 | /* Write a 0 to signal that we aren't switching pagetables */ |
214 | gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 0); |
215 | |
216 | /* Reset the preemption state */ |
217 | set_preempt_state(gpu: a5xx_gpu, new: PREEMPT_NONE); |
218 | } |
219 | |
220 | static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu, |
221 | struct msm_ringbuffer *ring) |
222 | { |
223 | struct adreno_gpu *adreno_gpu = &a5xx_gpu->base; |
224 | struct msm_gpu *gpu = &adreno_gpu->base; |
225 | struct a5xx_preempt_record *ptr; |
226 | void *counters; |
227 | struct drm_gem_object *bo = NULL, *counters_bo = NULL; |
228 | u64 iova = 0, counters_iova = 0; |
229 | |
230 | ptr = msm_gem_kernel_new(gpu->dev, |
231 | A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE, |
232 | MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova); |
233 | |
234 | if (IS_ERR(ptr)) |
235 | return PTR_ERR(ptr); |
236 | |
237 | /* The buffer to store counters needs to be unprivileged */ |
238 | counters = msm_gem_kernel_new(gpu->dev, |
239 | A5XX_PREEMPT_COUNTER_SIZE, |
240 | MSM_BO_WC, gpu->aspace, &counters_bo, &counters_iova); |
241 | if (IS_ERR(ptr: counters)) { |
242 | msm_gem_kernel_put(bo, gpu->aspace); |
243 | return PTR_ERR(ptr: counters); |
244 | } |
245 | |
246 | msm_gem_object_set_name(bo, "preempt" ); |
247 | msm_gem_object_set_name(counters_bo, "preempt_counters" ); |
248 | |
249 | a5xx_gpu->preempt_bo[ring->id] = bo; |
250 | a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo; |
251 | a5xx_gpu->preempt_iova[ring->id] = iova; |
252 | a5xx_gpu->preempt[ring->id] = ptr; |
253 | |
254 | /* Set up the defaults on the preemption record */ |
255 | |
256 | ptr->magic = A5XX_PREEMPT_RECORD_MAGIC; |
257 | ptr->info = 0; |
258 | ptr->data = 0; |
259 | ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE; |
260 | |
261 | ptr->counter = counters_iova; |
262 | |
263 | return 0; |
264 | } |
265 | |
266 | void a5xx_preempt_fini(struct msm_gpu *gpu) |
267 | { |
268 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); |
269 | struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); |
270 | int i; |
271 | |
272 | for (i = 0; i < gpu->nr_rings; i++) { |
273 | msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace); |
274 | msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], gpu->aspace); |
275 | } |
276 | } |
277 | |
278 | void a5xx_preempt_init(struct msm_gpu *gpu) |
279 | { |
280 | struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); |
281 | struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu); |
282 | int i; |
283 | |
284 | /* No preemption if we only have one ring */ |
285 | if (gpu->nr_rings <= 1) |
286 | return; |
287 | |
288 | for (i = 0; i < gpu->nr_rings; i++) { |
289 | if (preempt_init_ring(a5xx_gpu, ring: gpu->rb[i])) { |
290 | /* |
291 | * On any failure our adventure is over. Clean up and |
292 | * set nr_rings to 1 to force preemption off |
293 | */ |
294 | a5xx_preempt_fini(gpu); |
295 | gpu->nr_rings = 1; |
296 | |
297 | return; |
298 | } |
299 | } |
300 | |
301 | timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0); |
302 | } |
303 | |