1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ |
3 | |
4 | #include <linux/interrupt.h> |
5 | #include <linux/iopoll.h> |
6 | #include <linux/device.h> |
7 | #include <linux/slab.h> |
8 | |
9 | #include <drm/lima_drm.h> |
10 | |
11 | #include "lima_device.h" |
12 | #include "lima_gp.h" |
13 | #include "lima_regs.h" |
14 | #include "lima_gem.h" |
15 | #include "lima_vm.h" |
16 | |
17 | #define gp_write(reg, data) writel(data, ip->iomem + reg) |
18 | #define gp_read(reg) readl(ip->iomem + reg) |
19 | |
20 | static irqreturn_t lima_gp_irq_handler(int irq, void *data) |
21 | { |
22 | struct lima_ip *ip = data; |
23 | struct lima_device *dev = ip->dev; |
24 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; |
25 | struct lima_sched_task *task = pipe->current_task; |
26 | u32 state = gp_read(LIMA_GP_INT_STAT); |
27 | u32 status = gp_read(LIMA_GP_STATUS); |
28 | bool done = false; |
29 | |
30 | /* for shared irq case */ |
31 | if (!state) |
32 | return IRQ_NONE; |
33 | |
34 | if (state & LIMA_GP_IRQ_MASK_ERROR) { |
35 | if ((state & LIMA_GP_IRQ_MASK_ERROR) == |
36 | LIMA_GP_IRQ_PLBU_OUT_OF_MEM) { |
37 | dev_dbg(dev->dev, "gp out of heap irq status=%x\n" , |
38 | status); |
39 | } else { |
40 | dev_err(dev->dev, "gp error irq state=%x status=%x\n" , |
41 | state, status); |
42 | if (task) |
43 | task->recoverable = false; |
44 | } |
45 | |
46 | /* mask all interrupts before hard reset */ |
47 | gp_write(LIMA_GP_INT_MASK, 0); |
48 | |
49 | pipe->error = true; |
50 | done = true; |
51 | } else { |
52 | bool valid = state & (LIMA_GP_IRQ_VS_END_CMD_LST | |
53 | LIMA_GP_IRQ_PLBU_END_CMD_LST); |
54 | bool active = status & (LIMA_GP_STATUS_VS_ACTIVE | |
55 | LIMA_GP_STATUS_PLBU_ACTIVE); |
56 | done = valid && !active; |
57 | pipe->error = false; |
58 | } |
59 | |
60 | gp_write(LIMA_GP_INT_CLEAR, state); |
61 | |
62 | if (done) |
63 | lima_sched_pipe_task_done(pipe); |
64 | |
65 | return IRQ_HANDLED; |
66 | } |
67 | |
68 | static void lima_gp_soft_reset_async(struct lima_ip *ip) |
69 | { |
70 | if (ip->data.async_reset) |
71 | return; |
72 | |
73 | gp_write(LIMA_GP_INT_MASK, 0); |
74 | gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_RESET_COMPLETED); |
75 | gp_write(LIMA_GP_CMD, LIMA_GP_CMD_SOFT_RESET); |
76 | ip->data.async_reset = true; |
77 | } |
78 | |
79 | static int lima_gp_soft_reset_async_wait(struct lima_ip *ip) |
80 | { |
81 | struct lima_device *dev = ip->dev; |
82 | int err; |
83 | u32 v; |
84 | |
85 | if (!ip->data.async_reset) |
86 | return 0; |
87 | |
88 | err = readl_poll_timeout(ip->iomem + LIMA_GP_INT_RAWSTAT, v, |
89 | v & LIMA_GP_IRQ_RESET_COMPLETED, |
90 | 0, 100); |
91 | if (err) { |
92 | dev_err(dev->dev, "gp soft reset time out\n" ); |
93 | return err; |
94 | } |
95 | |
96 | gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL); |
97 | gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED); |
98 | |
99 | ip->data.async_reset = false; |
100 | return 0; |
101 | } |
102 | |
103 | static int lima_gp_task_validate(struct lima_sched_pipe *pipe, |
104 | struct lima_sched_task *task) |
105 | { |
106 | struct drm_lima_gp_frame *frame = task->frame; |
107 | u32 *f = frame->frame; |
108 | (void)pipe; |
109 | |
110 | if (f[LIMA_GP_VSCL_START_ADDR >> 2] > |
111 | f[LIMA_GP_VSCL_END_ADDR >> 2] || |
112 | f[LIMA_GP_PLBUCL_START_ADDR >> 2] > |
113 | f[LIMA_GP_PLBUCL_END_ADDR >> 2] || |
114 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] > |
115 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]) |
116 | return -EINVAL; |
117 | |
118 | if (f[LIMA_GP_VSCL_START_ADDR >> 2] == |
119 | f[LIMA_GP_VSCL_END_ADDR >> 2] && |
120 | f[LIMA_GP_PLBUCL_START_ADDR >> 2] == |
121 | f[LIMA_GP_PLBUCL_END_ADDR >> 2]) |
122 | return -EINVAL; |
123 | |
124 | return 0; |
125 | } |
126 | |
127 | static void lima_gp_task_run(struct lima_sched_pipe *pipe, |
128 | struct lima_sched_task *task) |
129 | { |
130 | struct lima_ip *ip = pipe->processor[0]; |
131 | struct drm_lima_gp_frame *frame = task->frame; |
132 | u32 *f = frame->frame; |
133 | u32 cmd = 0; |
134 | int i; |
135 | |
136 | /* update real heap buffer size for GP */ |
137 | for (i = 0; i < task->num_bos; i++) { |
138 | struct lima_bo *bo = task->bos[i]; |
139 | |
140 | if (bo->heap_size && |
141 | lima_vm_get_va(vm: task->vm, bo) == |
142 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2]) { |
143 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] = |
144 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] + |
145 | bo->heap_size; |
146 | task->recoverable = true; |
147 | task->heap = bo; |
148 | break; |
149 | } |
150 | } |
151 | |
152 | if (f[LIMA_GP_VSCL_START_ADDR >> 2] != |
153 | f[LIMA_GP_VSCL_END_ADDR >> 2]) |
154 | cmd |= LIMA_GP_CMD_START_VS; |
155 | if (f[LIMA_GP_PLBUCL_START_ADDR >> 2] != |
156 | f[LIMA_GP_PLBUCL_END_ADDR >> 2]) |
157 | cmd |= LIMA_GP_CMD_START_PLBU; |
158 | |
159 | /* before any hw ops, wait last success task async soft reset */ |
160 | lima_gp_soft_reset_async_wait(ip); |
161 | |
162 | for (i = 0; i < LIMA_GP_FRAME_REG_NUM; i++) |
163 | writel(val: f[i], addr: ip->iomem + LIMA_GP_VSCL_START_ADDR + i * 4); |
164 | |
165 | gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC); |
166 | gp_write(LIMA_GP_CMD, cmd); |
167 | } |
168 | |
169 | static int lima_gp_hard_reset_poll(struct lima_ip *ip) |
170 | { |
171 | gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC01A0000); |
172 | return gp_read(LIMA_GP_PERF_CNT_0_LIMIT) == 0xC01A0000; |
173 | } |
174 | |
175 | static int lima_gp_hard_reset(struct lima_ip *ip) |
176 | { |
177 | struct lima_device *dev = ip->dev; |
178 | int ret; |
179 | |
180 | gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC0FFE000); |
181 | gp_write(LIMA_GP_INT_MASK, 0); |
182 | gp_write(LIMA_GP_CMD, LIMA_GP_CMD_RESET); |
183 | ret = lima_poll_timeout(ip, func: lima_gp_hard_reset_poll, sleep_us: 10, timeout_us: 100); |
184 | if (ret) { |
185 | dev_err(dev->dev, "gp hard reset timeout\n" ); |
186 | return ret; |
187 | } |
188 | |
189 | gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0); |
190 | gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL); |
191 | gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED); |
192 | return 0; |
193 | } |
194 | |
195 | static void lima_gp_task_fini(struct lima_sched_pipe *pipe) |
196 | { |
197 | lima_gp_soft_reset_async(ip: pipe->processor[0]); |
198 | } |
199 | |
200 | static void lima_gp_task_error(struct lima_sched_pipe *pipe) |
201 | { |
202 | struct lima_ip *ip = pipe->processor[0]; |
203 | |
204 | dev_err(ip->dev->dev, "gp task error int_state=%x status=%x\n" , |
205 | gp_read(LIMA_GP_INT_STAT), gp_read(LIMA_GP_STATUS)); |
206 | |
207 | lima_gp_hard_reset(ip); |
208 | } |
209 | |
210 | static void lima_gp_task_mmu_error(struct lima_sched_pipe *pipe) |
211 | { |
212 | lima_sched_pipe_task_done(pipe); |
213 | } |
214 | |
215 | static int lima_gp_task_recover(struct lima_sched_pipe *pipe) |
216 | { |
217 | struct lima_ip *ip = pipe->processor[0]; |
218 | struct lima_sched_task *task = pipe->current_task; |
219 | struct drm_lima_gp_frame *frame = task->frame; |
220 | u32 *f = frame->frame; |
221 | size_t fail_size = |
222 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] - |
223 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2]; |
224 | |
225 | if (fail_size == task->heap->heap_size) { |
226 | int ret; |
227 | |
228 | ret = lima_heap_alloc(bo: task->heap, vm: task->vm); |
229 | if (ret < 0) |
230 | return ret; |
231 | } |
232 | |
233 | gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED); |
234 | /* Resume from where we stopped, i.e. new start is old end */ |
235 | gp_write(LIMA_GP_PLBU_ALLOC_START_ADDR, |
236 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]); |
237 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] = |
238 | f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] + task->heap->heap_size; |
239 | gp_write(LIMA_GP_PLBU_ALLOC_END_ADDR, |
240 | f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]); |
241 | gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC); |
242 | return 0; |
243 | } |
244 | |
245 | static void lima_gp_print_version(struct lima_ip *ip) |
246 | { |
247 | u32 version, major, minor; |
248 | char *name; |
249 | |
250 | version = gp_read(LIMA_GP_VERSION); |
251 | major = (version >> 8) & 0xFF; |
252 | minor = version & 0xFF; |
253 | switch (version >> 16) { |
254 | case 0xA07: |
255 | name = "mali200" ; |
256 | break; |
257 | case 0xC07: |
258 | name = "mali300" ; |
259 | break; |
260 | case 0xB07: |
261 | name = "mali400" ; |
262 | break; |
263 | case 0xD07: |
264 | name = "mali450" ; |
265 | break; |
266 | default: |
267 | name = "unknown" ; |
268 | break; |
269 | } |
270 | dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n" , |
271 | lima_ip_name(ip), name, major, minor); |
272 | } |
273 | |
274 | static struct kmem_cache *lima_gp_task_slab; |
275 | static int lima_gp_task_slab_refcnt; |
276 | |
277 | static int lima_gp_hw_init(struct lima_ip *ip) |
278 | { |
279 | ip->data.async_reset = false; |
280 | lima_gp_soft_reset_async(ip); |
281 | return lima_gp_soft_reset_async_wait(ip); |
282 | } |
283 | |
284 | int lima_gp_resume(struct lima_ip *ip) |
285 | { |
286 | return lima_gp_hw_init(ip); |
287 | } |
288 | |
289 | void lima_gp_suspend(struct lima_ip *ip) |
290 | { |
291 | |
292 | } |
293 | |
294 | int lima_gp_init(struct lima_ip *ip) |
295 | { |
296 | struct lima_device *dev = ip->dev; |
297 | int err; |
298 | |
299 | lima_gp_print_version(ip); |
300 | |
301 | err = lima_gp_hw_init(ip); |
302 | if (err) |
303 | return err; |
304 | |
305 | err = devm_request_irq(dev: dev->dev, irq: ip->irq, handler: lima_gp_irq_handler, |
306 | IRQF_SHARED, devname: lima_ip_name(ip), dev_id: ip); |
307 | if (err) { |
308 | dev_err(dev->dev, "gp %s fail to request irq\n" , |
309 | lima_ip_name(ip)); |
310 | return err; |
311 | } |
312 | |
313 | dev->gp_version = gp_read(LIMA_GP_VERSION); |
314 | |
315 | return 0; |
316 | } |
317 | |
318 | void lima_gp_fini(struct lima_ip *ip) |
319 | { |
320 | |
321 | } |
322 | |
323 | int lima_gp_pipe_init(struct lima_device *dev) |
324 | { |
325 | int frame_size = sizeof(struct drm_lima_gp_frame); |
326 | struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp; |
327 | |
328 | if (!lima_gp_task_slab) { |
329 | lima_gp_task_slab = kmem_cache_create_usercopy( |
330 | name: "lima_gp_task" , size: sizeof(struct lima_sched_task) + frame_size, |
331 | align: 0, SLAB_HWCACHE_ALIGN, useroffset: sizeof(struct lima_sched_task), |
332 | usersize: frame_size, NULL); |
333 | if (!lima_gp_task_slab) |
334 | return -ENOMEM; |
335 | } |
336 | lima_gp_task_slab_refcnt++; |
337 | |
338 | pipe->frame_size = frame_size; |
339 | pipe->task_slab = lima_gp_task_slab; |
340 | |
341 | pipe->task_validate = lima_gp_task_validate; |
342 | pipe->task_run = lima_gp_task_run; |
343 | pipe->task_fini = lima_gp_task_fini; |
344 | pipe->task_error = lima_gp_task_error; |
345 | pipe->task_mmu_error = lima_gp_task_mmu_error; |
346 | pipe->task_recover = lima_gp_task_recover; |
347 | |
348 | return 0; |
349 | } |
350 | |
351 | void lima_gp_pipe_fini(struct lima_device *dev) |
352 | { |
353 | if (!--lima_gp_task_slab_refcnt) { |
354 | kmem_cache_destroy(s: lima_gp_task_slab); |
355 | lima_gp_task_slab = NULL; |
356 | } |
357 | } |
358 | |