1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ |
3 | |
4 | #include <linux/interrupt.h> |
5 | #include <linux/io.h> |
6 | #include <linux/device.h> |
7 | #include <linux/slab.h> |
8 | |
9 | #include <drm/lima_drm.h> |
10 | |
11 | #include "lima_device.h" |
12 | #include "lima_pp.h" |
13 | #include "lima_dlbu.h" |
14 | #include "lima_bcast.h" |
15 | #include "lima_vm.h" |
16 | #include "lima_regs.h" |
17 | |
18 | #define pp_write(reg, data) writel(data, ip->iomem + reg) |
19 | #define pp_read(reg) readl(ip->iomem + reg) |
20 | |
21 | static void lima_pp_handle_irq(struct lima_ip *ip, u32 state) |
22 | { |
23 | struct lima_device *dev = ip->dev; |
24 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
25 | |
26 | if (state & LIMA_PP_IRQ_MASK_ERROR) { |
27 | u32 status = pp_read(LIMA_PP_STATUS); |
28 | |
29 | dev_err(dev->dev, "pp error irq state=%x status=%x\n" , |
30 | state, status); |
31 | |
32 | pipe->error = true; |
33 | |
34 | /* mask all interrupts before hard reset */ |
35 | pp_write(LIMA_PP_INT_MASK, 0); |
36 | } |
37 | |
38 | pp_write(LIMA_PP_INT_CLEAR, state); |
39 | } |
40 | |
41 | static irqreturn_t lima_pp_irq_handler(int irq, void *data) |
42 | { |
43 | struct lima_ip *ip = data; |
44 | struct lima_device *dev = ip->dev; |
45 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
46 | u32 state = pp_read(LIMA_PP_INT_STATUS); |
47 | |
48 | /* for shared irq case */ |
49 | if (!state) |
50 | return IRQ_NONE; |
51 | |
52 | lima_pp_handle_irq(ip, state); |
53 | |
54 | if (atomic_dec_and_test(v: &pipe->task)) |
55 | lima_sched_pipe_task_done(pipe); |
56 | |
57 | return IRQ_HANDLED; |
58 | } |
59 | |
60 | static irqreturn_t lima_pp_bcast_irq_handler(int irq, void *data) |
61 | { |
62 | int i; |
63 | irqreturn_t ret = IRQ_NONE; |
64 | struct lima_ip *pp_bcast = data; |
65 | struct lima_device *dev = pp_bcast->dev; |
66 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
67 | struct drm_lima_m450_pp_frame *frame; |
68 | |
69 | /* for shared irq case */ |
70 | if (!pipe->current_task) |
71 | return IRQ_NONE; |
72 | |
73 | frame = pipe->current_task->frame; |
74 | |
75 | for (i = 0; i < frame->num_pp; i++) { |
76 | struct lima_ip *ip = pipe->processor[i]; |
77 | u32 status, state; |
78 | |
79 | if (pipe->done & (1 << i)) |
80 | continue; |
81 | |
82 | /* status read first in case int state change in the middle |
83 | * which may miss the interrupt handling |
84 | */ |
85 | status = pp_read(LIMA_PP_STATUS); |
86 | state = pp_read(LIMA_PP_INT_STATUS); |
87 | |
88 | if (state) { |
89 | lima_pp_handle_irq(ip, state); |
90 | ret = IRQ_HANDLED; |
91 | } else { |
92 | if (status & LIMA_PP_STATUS_RENDERING_ACTIVE) |
93 | continue; |
94 | } |
95 | |
96 | pipe->done |= (1 << i); |
97 | if (atomic_dec_and_test(v: &pipe->task)) |
98 | lima_sched_pipe_task_done(pipe); |
99 | } |
100 | |
101 | return ret; |
102 | } |
103 | |
104 | static void lima_pp_soft_reset_async(struct lima_ip *ip) |
105 | { |
106 | if (ip->data.async_reset) |
107 | return; |
108 | |
109 | pp_write(LIMA_PP_INT_MASK, 0); |
110 | pp_write(LIMA_PP_INT_RAWSTAT, LIMA_PP_IRQ_MASK_ALL); |
111 | pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_SOFT_RESET); |
112 | ip->data.async_reset = true; |
113 | } |
114 | |
115 | static int lima_pp_soft_reset_poll(struct lima_ip *ip) |
116 | { |
117 | return !(pp_read(LIMA_PP_STATUS) & LIMA_PP_STATUS_RENDERING_ACTIVE) && |
118 | pp_read(LIMA_PP_INT_RAWSTAT) == LIMA_PP_IRQ_RESET_COMPLETED; |
119 | } |
120 | |
121 | static int lima_pp_soft_reset_async_wait_one(struct lima_ip *ip) |
122 | { |
123 | struct lima_device *dev = ip->dev; |
124 | int ret; |
125 | |
126 | ret = lima_poll_timeout(ip, func: lima_pp_soft_reset_poll, sleep_us: 0, timeout_us: 100); |
127 | if (ret) { |
128 | dev_err(dev->dev, "pp %s reset time out\n" , lima_ip_name(ip)); |
129 | return ret; |
130 | } |
131 | |
132 | pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL); |
133 | pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED); |
134 | return 0; |
135 | } |
136 | |
137 | static int lima_pp_soft_reset_async_wait(struct lima_ip *ip) |
138 | { |
139 | int i, err = 0; |
140 | |
141 | if (!ip->data.async_reset) |
142 | return 0; |
143 | |
144 | if (ip->id == lima_ip_pp_bcast) { |
145 | struct lima_device *dev = ip->dev; |
146 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
147 | struct drm_lima_m450_pp_frame *frame = pipe->current_task->frame; |
148 | |
149 | for (i = 0; i < frame->num_pp; i++) |
150 | err |= lima_pp_soft_reset_async_wait_one(ip: pipe->processor[i]); |
151 | } else |
152 | err = lima_pp_soft_reset_async_wait_one(ip); |
153 | |
154 | ip->data.async_reset = false; |
155 | return err; |
156 | } |
157 | |
158 | static void lima_pp_write_frame(struct lima_ip *ip, u32 *frame, u32 *wb) |
159 | { |
160 | int i, j, n = 0; |
161 | |
162 | for (i = 0; i < LIMA_PP_FRAME_REG_NUM; i++) |
163 | writel(val: frame[i], addr: ip->iomem + LIMA_PP_FRAME + i * 4); |
164 | |
165 | for (i = 0; i < 3; i++) { |
166 | for (j = 0; j < LIMA_PP_WB_REG_NUM; j++) |
167 | writel(val: wb[n++], addr: ip->iomem + LIMA_PP_WB(i) + j * 4); |
168 | } |
169 | } |
170 | |
171 | static int lima_pp_hard_reset_poll(struct lima_ip *ip) |
172 | { |
173 | pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC01A0000); |
174 | return pp_read(LIMA_PP_PERF_CNT_0_LIMIT) == 0xC01A0000; |
175 | } |
176 | |
177 | static int lima_pp_hard_reset(struct lima_ip *ip) |
178 | { |
179 | struct lima_device *dev = ip->dev; |
180 | int ret; |
181 | |
182 | pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC0FFE000); |
183 | pp_write(LIMA_PP_INT_MASK, 0); |
184 | pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_FORCE_RESET); |
185 | ret = lima_poll_timeout(ip, func: lima_pp_hard_reset_poll, sleep_us: 10, timeout_us: 100); |
186 | if (ret) { |
187 | dev_err(dev->dev, "pp hard reset timeout\n" ); |
188 | return ret; |
189 | } |
190 | |
191 | pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0); |
192 | pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL); |
193 | pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED); |
194 | return 0; |
195 | } |
196 | |
197 | static void lima_pp_print_version(struct lima_ip *ip) |
198 | { |
199 | u32 version, major, minor; |
200 | char *name; |
201 | |
202 | version = pp_read(LIMA_PP_VERSION); |
203 | major = (version >> 8) & 0xFF; |
204 | minor = version & 0xFF; |
205 | switch (version >> 16) { |
206 | case 0xC807: |
207 | name = "mali200" ; |
208 | break; |
209 | case 0xCE07: |
210 | name = "mali300" ; |
211 | break; |
212 | case 0xCD07: |
213 | name = "mali400" ; |
214 | break; |
215 | case 0xCF07: |
216 | name = "mali450" ; |
217 | break; |
218 | default: |
219 | name = "unknown" ; |
220 | break; |
221 | } |
222 | dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n" , |
223 | lima_ip_name(ip), name, major, minor); |
224 | } |
225 | |
226 | static int lima_pp_hw_init(struct lima_ip *ip) |
227 | { |
228 | ip->data.async_reset = false; |
229 | lima_pp_soft_reset_async(ip); |
230 | return lima_pp_soft_reset_async_wait(ip); |
231 | } |
232 | |
233 | int lima_pp_resume(struct lima_ip *ip) |
234 | { |
235 | return lima_pp_hw_init(ip); |
236 | } |
237 | |
238 | void lima_pp_suspend(struct lima_ip *ip) |
239 | { |
240 | |
241 | } |
242 | |
243 | int lima_pp_init(struct lima_ip *ip) |
244 | { |
245 | struct lima_device *dev = ip->dev; |
246 | int err; |
247 | |
248 | lima_pp_print_version(ip); |
249 | |
250 | err = lima_pp_hw_init(ip); |
251 | if (err) |
252 | return err; |
253 | |
254 | err = devm_request_irq(dev: dev->dev, irq: ip->irq, handler: lima_pp_irq_handler, |
255 | IRQF_SHARED, devname: lima_ip_name(ip), dev_id: ip); |
256 | if (err) { |
257 | dev_err(dev->dev, "pp %s fail to request irq\n" , |
258 | lima_ip_name(ip)); |
259 | return err; |
260 | } |
261 | |
262 | dev->pp_version = pp_read(LIMA_PP_VERSION); |
263 | |
264 | return 0; |
265 | } |
266 | |
267 | void lima_pp_fini(struct lima_ip *ip) |
268 | { |
269 | |
270 | } |
271 | |
272 | int lima_pp_bcast_resume(struct lima_ip *ip) |
273 | { |
274 | /* PP has been reset by individual PP resume */ |
275 | ip->data.async_reset = false; |
276 | return 0; |
277 | } |
278 | |
279 | void lima_pp_bcast_suspend(struct lima_ip *ip) |
280 | { |
281 | |
282 | } |
283 | |
284 | int lima_pp_bcast_init(struct lima_ip *ip) |
285 | { |
286 | struct lima_device *dev = ip->dev; |
287 | int err; |
288 | |
289 | err = devm_request_irq(dev: dev->dev, irq: ip->irq, handler: lima_pp_bcast_irq_handler, |
290 | IRQF_SHARED, devname: lima_ip_name(ip), dev_id: ip); |
291 | if (err) { |
292 | dev_err(dev->dev, "pp %s fail to request irq\n" , |
293 | lima_ip_name(ip)); |
294 | return err; |
295 | } |
296 | |
297 | return 0; |
298 | } |
299 | |
300 | void lima_pp_bcast_fini(struct lima_ip *ip) |
301 | { |
302 | |
303 | } |
304 | |
305 | static int lima_pp_task_validate(struct lima_sched_pipe *pipe, |
306 | struct lima_sched_task *task) |
307 | { |
308 | u32 num_pp; |
309 | |
310 | if (pipe->bcast_processor) { |
311 | struct drm_lima_m450_pp_frame *f = task->frame; |
312 | |
313 | num_pp = f->num_pp; |
314 | |
315 | if (f->_pad) |
316 | return -EINVAL; |
317 | } else { |
318 | struct drm_lima_m400_pp_frame *f = task->frame; |
319 | |
320 | num_pp = f->num_pp; |
321 | } |
322 | |
323 | if (num_pp == 0 || num_pp > pipe->num_processor) |
324 | return -EINVAL; |
325 | |
326 | return 0; |
327 | } |
328 | |
329 | static void lima_pp_task_run(struct lima_sched_pipe *pipe, |
330 | struct lima_sched_task *task) |
331 | { |
332 | if (pipe->bcast_processor) { |
333 | struct drm_lima_m450_pp_frame *frame = task->frame; |
334 | struct lima_device *dev = pipe->bcast_processor->dev; |
335 | struct lima_ip *ip = pipe->bcast_processor; |
336 | int i; |
337 | |
338 | pipe->done = 0; |
339 | atomic_set(v: &pipe->task, i: frame->num_pp); |
340 | |
341 | if (frame->use_dlbu) { |
342 | lima_dlbu_enable(dev, num_pp: frame->num_pp); |
343 | |
344 | frame->frame[LIMA_PP_FRAME >> 2] = LIMA_VA_RESERVE_DLBU; |
345 | lima_dlbu_set_reg(ip: dev->ip + lima_ip_dlbu, reg: frame->dlbu_regs); |
346 | } else |
347 | lima_dlbu_disable(dev); |
348 | |
349 | lima_bcast_enable(dev, num_pp: frame->num_pp); |
350 | |
351 | lima_pp_soft_reset_async_wait(ip); |
352 | |
353 | lima_pp_write_frame(ip, frame: frame->frame, wb: frame->wb); |
354 | |
355 | for (i = 0; i < frame->num_pp; i++) { |
356 | struct lima_ip *ip = pipe->processor[i]; |
357 | |
358 | pp_write(LIMA_PP_STACK, frame->fragment_stack_address[i]); |
359 | if (!frame->use_dlbu) |
360 | pp_write(LIMA_PP_FRAME, frame->plbu_array_address[i]); |
361 | } |
362 | |
363 | pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING); |
364 | } else { |
365 | struct drm_lima_m400_pp_frame *frame = task->frame; |
366 | int i; |
367 | |
368 | atomic_set(v: &pipe->task, i: frame->num_pp); |
369 | |
370 | for (i = 0; i < frame->num_pp; i++) { |
371 | struct lima_ip *ip = pipe->processor[i]; |
372 | |
373 | frame->frame[LIMA_PP_FRAME >> 2] = |
374 | frame->plbu_array_address[i]; |
375 | frame->frame[LIMA_PP_STACK >> 2] = |
376 | frame->fragment_stack_address[i]; |
377 | |
378 | lima_pp_soft_reset_async_wait(ip); |
379 | |
380 | lima_pp_write_frame(ip, frame: frame->frame, wb: frame->wb); |
381 | |
382 | pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING); |
383 | } |
384 | } |
385 | } |
386 | |
387 | static void lima_pp_task_fini(struct lima_sched_pipe *pipe) |
388 | { |
389 | if (pipe->bcast_processor) |
390 | lima_pp_soft_reset_async(ip: pipe->bcast_processor); |
391 | else { |
392 | int i; |
393 | |
394 | for (i = 0; i < pipe->num_processor; i++) |
395 | lima_pp_soft_reset_async(ip: pipe->processor[i]); |
396 | } |
397 | } |
398 | |
399 | static void lima_pp_task_error(struct lima_sched_pipe *pipe) |
400 | { |
401 | int i; |
402 | |
403 | for (i = 0; i < pipe->num_processor; i++) { |
404 | struct lima_ip *ip = pipe->processor[i]; |
405 | |
406 | dev_err(ip->dev->dev, "pp task error %d int_state=%x status=%x\n" , |
407 | i, pp_read(LIMA_PP_INT_STATUS), pp_read(LIMA_PP_STATUS)); |
408 | |
409 | lima_pp_hard_reset(ip); |
410 | } |
411 | } |
412 | |
413 | static void lima_pp_task_mmu_error(struct lima_sched_pipe *pipe) |
414 | { |
415 | if (atomic_dec_and_test(v: &pipe->task)) |
416 | lima_sched_pipe_task_done(pipe); |
417 | } |
418 | |
419 | static struct kmem_cache *lima_pp_task_slab; |
420 | static int lima_pp_task_slab_refcnt; |
421 | |
422 | int lima_pp_pipe_init(struct lima_device *dev) |
423 | { |
424 | int frame_size; |
425 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp; |
426 | |
427 | if (dev->id == lima_gpu_mali400) |
428 | frame_size = sizeof(struct drm_lima_m400_pp_frame); |
429 | else |
430 | frame_size = sizeof(struct drm_lima_m450_pp_frame); |
431 | |
432 | if (!lima_pp_task_slab) { |
433 | lima_pp_task_slab = kmem_cache_create_usercopy( |
434 | name: "lima_pp_task" , size: sizeof(struct lima_sched_task) + frame_size, |
435 | align: 0, SLAB_HWCACHE_ALIGN, useroffset: sizeof(struct lima_sched_task), |
436 | usersize: frame_size, NULL); |
437 | if (!lima_pp_task_slab) |
438 | return -ENOMEM; |
439 | } |
440 | lima_pp_task_slab_refcnt++; |
441 | |
442 | pipe->frame_size = frame_size; |
443 | pipe->task_slab = lima_pp_task_slab; |
444 | |
445 | pipe->task_validate = lima_pp_task_validate; |
446 | pipe->task_run = lima_pp_task_run; |
447 | pipe->task_fini = lima_pp_task_fini; |
448 | pipe->task_error = lima_pp_task_error; |
449 | pipe->task_mmu_error = lima_pp_task_mmu_error; |
450 | |
451 | return 0; |
452 | } |
453 | |
454 | void lima_pp_pipe_fini(struct lima_device *dev) |
455 | { |
456 | if (!--lima_pp_task_slab_refcnt) { |
457 | kmem_cache_destroy(s: lima_pp_task_slab); |
458 | lima_pp_task_slab = NULL; |
459 | } |
460 | } |
461 | |