1 | /* |
2 | * Copyright © 2014 Broadcom |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice (including the next |
12 | * paragraph) shall be included in all copies or substantial portions of the |
13 | * Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
21 | * IN THE SOFTWARE. |
22 | */ |
23 | |
24 | /** |
25 | * DOC: Interrupt management for the V3D engine |
26 | * |
27 | * We have an interrupt status register (V3D_INTCTL) which reports |
28 | * interrupts, and where writing 1 bits clears those interrupts. |
29 | * There are also a pair of interrupt registers |
30 | * (V3D_INTENA/V3D_INTDIS) where writing a 1 to their bits enables or |
31 | * disables that specific interrupt, and 0s written are ignored |
32 | * (reading either one returns the set of enabled interrupts). |
33 | * |
34 | * When we take a binning flush done interrupt, we need to submit the |
35 | * next frame for binning and move the finished frame to the render |
36 | * thread. |
37 | * |
38 | * When we take a render frame interrupt, we need to wake the |
39 | * processes waiting for some frame to be done, and get the next frame |
40 | * submitted ASAP (so the hardware doesn't sit idle when there's work |
41 | * to do). |
42 | * |
43 | * When we take the binner out of memory interrupt, we need to |
44 | * allocate some new memory and pass it to the binner so that the |
45 | * current job can make progress. |
46 | */ |
47 | |
48 | #include <linux/platform_device.h> |
49 | |
50 | #include <drm/drm_drv.h> |
51 | |
52 | #include "vc4_drv.h" |
53 | #include "vc4_regs.h" |
54 | #include "vc4_trace.h" |
55 | |
56 | #define V3D_DRIVER_IRQS (V3D_INT_OUTOMEM | \ |
57 | V3D_INT_FLDONE | \ |
58 | V3D_INT_FRDONE) |
59 | |
60 | static void |
61 | vc4_overflow_mem_work(struct work_struct *work) |
62 | { |
63 | struct vc4_dev *vc4 = |
64 | container_of(work, struct vc4_dev, overflow_mem_work); |
65 | struct vc4_bo *bo; |
66 | int bin_bo_slot; |
67 | struct vc4_exec_info *exec; |
68 | unsigned long irqflags; |
69 | |
70 | mutex_lock(&vc4->bin_bo_lock); |
71 | |
72 | if (!vc4->bin_bo) |
73 | goto complete; |
74 | |
75 | bo = vc4->bin_bo; |
76 | |
77 | bin_bo_slot = vc4_v3d_get_bin_slot(vc4); |
78 | if (bin_bo_slot < 0) { |
79 | DRM_ERROR("Couldn't allocate binner overflow mem\n" ); |
80 | goto complete; |
81 | } |
82 | |
83 | spin_lock_irqsave(&vc4->job_lock, irqflags); |
84 | |
85 | if (vc4->bin_alloc_overflow) { |
86 | /* If we had overflow memory allocated previously, |
87 | * then that chunk will free when the current bin job |
88 | * is done. If we don't have a bin job running, then |
89 | * the chunk will be done whenever the list of render |
90 | * jobs has drained. |
91 | */ |
92 | exec = vc4_first_bin_job(vc4); |
93 | if (!exec) |
94 | exec = vc4_last_render_job(vc4); |
95 | if (exec) { |
96 | exec->bin_slots |= vc4->bin_alloc_overflow; |
97 | } else { |
98 | /* There's nothing queued in the hardware, so |
99 | * the old slot is free immediately. |
100 | */ |
101 | vc4->bin_alloc_used &= ~vc4->bin_alloc_overflow; |
102 | } |
103 | } |
104 | vc4->bin_alloc_overflow = BIT(bin_bo_slot); |
105 | |
106 | V3D_WRITE(V3D_BPOA, bo->base.dma_addr + bin_bo_slot * vc4->bin_alloc_size); |
107 | V3D_WRITE(V3D_BPOS, bo->base.base.size); |
108 | V3D_WRITE(V3D_INTCTL, V3D_INT_OUTOMEM); |
109 | V3D_WRITE(V3D_INTENA, V3D_INT_OUTOMEM); |
110 | spin_unlock_irqrestore(lock: &vc4->job_lock, flags: irqflags); |
111 | |
112 | complete: |
113 | mutex_unlock(lock: &vc4->bin_bo_lock); |
114 | } |
115 | |
116 | static void |
117 | vc4_irq_finish_bin_job(struct drm_device *dev) |
118 | { |
119 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
120 | struct vc4_exec_info *next, *exec = vc4_first_bin_job(vc4); |
121 | |
122 | if (!exec) |
123 | return; |
124 | |
125 | trace_vc4_bcl_end_irq(dev, seqno: exec->seqno); |
126 | |
127 | vc4_move_job_to_render(dev, exec); |
128 | next = vc4_first_bin_job(vc4); |
129 | |
130 | /* Only submit the next job in the bin list if it matches the perfmon |
131 | * attached to the one that just finished (or if both jobs don't have |
132 | * perfmon attached to them). |
133 | */ |
134 | if (next && next->perfmon == exec->perfmon) |
135 | vc4_submit_next_bin_job(dev); |
136 | } |
137 | |
138 | static void |
139 | vc4_cancel_bin_job(struct drm_device *dev) |
140 | { |
141 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
142 | struct vc4_exec_info *exec = vc4_first_bin_job(vc4); |
143 | |
144 | if (!exec) |
145 | return; |
146 | |
147 | /* Stop the perfmon so that the next bin job can be started. */ |
148 | if (exec->perfmon) |
149 | vc4_perfmon_stop(vc4, perfmon: exec->perfmon, capture: false); |
150 | |
151 | list_move_tail(list: &exec->head, head: &vc4->bin_job_list); |
152 | vc4_submit_next_bin_job(dev); |
153 | } |
154 | |
155 | static void |
156 | vc4_irq_finish_render_job(struct drm_device *dev) |
157 | { |
158 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
159 | struct vc4_exec_info *exec = vc4_first_render_job(vc4); |
160 | struct vc4_exec_info *nextbin, *nextrender; |
161 | |
162 | if (!exec) |
163 | return; |
164 | |
165 | trace_vc4_rcl_end_irq(dev, seqno: exec->seqno); |
166 | |
167 | vc4->finished_seqno++; |
168 | list_move_tail(list: &exec->head, head: &vc4->job_done_list); |
169 | |
170 | nextbin = vc4_first_bin_job(vc4); |
171 | nextrender = vc4_first_render_job(vc4); |
172 | |
173 | /* Only stop the perfmon if following jobs in the queue don't expect it |
174 | * to be enabled. |
175 | */ |
176 | if (exec->perfmon && !nextrender && |
177 | (!nextbin || nextbin->perfmon != exec->perfmon)) |
178 | vc4_perfmon_stop(vc4, perfmon: exec->perfmon, capture: true); |
179 | |
180 | /* If there's a render job waiting, start it. If this is not the case |
181 | * we may have to unblock the binner if it's been stalled because of |
182 | * perfmon (this can be checked by comparing the perfmon attached to |
183 | * the finished renderjob to the one attached to the next bin job: if |
184 | * they don't match, this means the binner is stalled and should be |
185 | * restarted). |
186 | */ |
187 | if (nextrender) |
188 | vc4_submit_next_render_job(dev); |
189 | else if (nextbin && nextbin->perfmon != exec->perfmon) |
190 | vc4_submit_next_bin_job(dev); |
191 | |
192 | if (exec->fence) { |
193 | dma_fence_signal_locked(fence: exec->fence); |
194 | dma_fence_put(fence: exec->fence); |
195 | exec->fence = NULL; |
196 | } |
197 | |
198 | wake_up_all(&vc4->job_wait_queue); |
199 | schedule_work(work: &vc4->job_done_work); |
200 | } |
201 | |
202 | static irqreturn_t |
203 | vc4_irq(int irq, void *arg) |
204 | { |
205 | struct drm_device *dev = arg; |
206 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
207 | uint32_t intctl; |
208 | irqreturn_t status = IRQ_NONE; |
209 | |
210 | barrier(); |
211 | intctl = V3D_READ(V3D_INTCTL); |
212 | |
213 | /* Acknowledge the interrupts we're handling here. The binner |
214 | * last flush / render frame done interrupt will be cleared, |
215 | * while OUTOMEM will stay high until the underlying cause is |
216 | * cleared. |
217 | */ |
218 | V3D_WRITE(V3D_INTCTL, intctl); |
219 | |
220 | if (intctl & V3D_INT_OUTOMEM) { |
221 | /* Disable OUTOMEM until the work is done. */ |
222 | V3D_WRITE(V3D_INTDIS, V3D_INT_OUTOMEM); |
223 | schedule_work(work: &vc4->overflow_mem_work); |
224 | status = IRQ_HANDLED; |
225 | } |
226 | |
227 | if (intctl & V3D_INT_FLDONE) { |
228 | spin_lock(lock: &vc4->job_lock); |
229 | vc4_irq_finish_bin_job(dev); |
230 | spin_unlock(lock: &vc4->job_lock); |
231 | status = IRQ_HANDLED; |
232 | } |
233 | |
234 | if (intctl & V3D_INT_FRDONE) { |
235 | spin_lock(lock: &vc4->job_lock); |
236 | vc4_irq_finish_render_job(dev); |
237 | spin_unlock(lock: &vc4->job_lock); |
238 | status = IRQ_HANDLED; |
239 | } |
240 | |
241 | return status; |
242 | } |
243 | |
244 | static void |
245 | vc4_irq_prepare(struct drm_device *dev) |
246 | { |
247 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
248 | |
249 | if (!vc4->v3d) |
250 | return; |
251 | |
252 | init_waitqueue_head(&vc4->job_wait_queue); |
253 | INIT_WORK(&vc4->overflow_mem_work, vc4_overflow_mem_work); |
254 | |
255 | /* Clear any pending interrupts someone might have left around |
256 | * for us. |
257 | */ |
258 | V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS); |
259 | } |
260 | |
261 | void |
262 | vc4_irq_enable(struct drm_device *dev) |
263 | { |
264 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
265 | |
266 | if (WARN_ON_ONCE(vc4->is_vc5)) |
267 | return; |
268 | |
269 | if (!vc4->v3d) |
270 | return; |
271 | |
272 | /* Enable the render done interrupts. The out-of-memory interrupt is |
273 | * enabled as soon as we have a binner BO allocated. |
274 | */ |
275 | V3D_WRITE(V3D_INTENA, V3D_INT_FLDONE | V3D_INT_FRDONE); |
276 | } |
277 | |
278 | void |
279 | vc4_irq_disable(struct drm_device *dev) |
280 | { |
281 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
282 | |
283 | if (WARN_ON_ONCE(vc4->is_vc5)) |
284 | return; |
285 | |
286 | if (!vc4->v3d) |
287 | return; |
288 | |
289 | /* Disable sending interrupts for our driver's IRQs. */ |
290 | V3D_WRITE(V3D_INTDIS, V3D_DRIVER_IRQS); |
291 | |
292 | /* Clear any pending interrupts we might have left. */ |
293 | V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS); |
294 | |
295 | /* Finish any interrupt handler still in flight. */ |
296 | synchronize_irq(irq: vc4->irq); |
297 | |
298 | cancel_work_sync(work: &vc4->overflow_mem_work); |
299 | } |
300 | |
301 | int vc4_irq_install(struct drm_device *dev, int irq) |
302 | { |
303 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
304 | int ret; |
305 | |
306 | if (WARN_ON_ONCE(vc4->is_vc5)) |
307 | return -ENODEV; |
308 | |
309 | if (irq == IRQ_NOTCONNECTED) |
310 | return -ENOTCONN; |
311 | |
312 | vc4_irq_prepare(dev); |
313 | |
314 | ret = request_irq(irq, handler: vc4_irq, flags: 0, name: dev->driver->name, dev); |
315 | if (ret) |
316 | return ret; |
317 | |
318 | vc4_irq_enable(dev); |
319 | |
320 | return 0; |
321 | } |
322 | |
323 | void vc4_irq_uninstall(struct drm_device *dev) |
324 | { |
325 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
326 | |
327 | if (WARN_ON_ONCE(vc4->is_vc5)) |
328 | return; |
329 | |
330 | vc4_irq_disable(dev); |
331 | free_irq(vc4->irq, dev); |
332 | } |
333 | |
334 | /** Reinitializes interrupt registers when a GPU reset is performed. */ |
335 | void vc4_irq_reset(struct drm_device *dev) |
336 | { |
337 | struct vc4_dev *vc4 = to_vc4_dev(dev); |
338 | unsigned long irqflags; |
339 | |
340 | if (WARN_ON_ONCE(vc4->is_vc5)) |
341 | return; |
342 | |
343 | /* Acknowledge any stale IRQs. */ |
344 | V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS); |
345 | |
346 | /* |
347 | * Turn all our interrupts on. Binner out of memory is the |
348 | * only one we expect to trigger at this point, since we've |
349 | * just come from poweron and haven't supplied any overflow |
350 | * memory yet. |
351 | */ |
352 | V3D_WRITE(V3D_INTENA, V3D_DRIVER_IRQS); |
353 | |
354 | spin_lock_irqsave(&vc4->job_lock, irqflags); |
355 | vc4_cancel_bin_job(dev); |
356 | vc4_irq_finish_render_job(dev); |
357 | spin_unlock_irqrestore(lock: &vc4->job_lock, flags: irqflags); |
358 | } |
359 | |