1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */ |
3 | /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ |
4 | |
5 | #include <linux/clk.h> |
6 | #include <linux/reset.h> |
7 | #include <linux/platform_device.h> |
8 | #include <linux/pm_domain.h> |
9 | #include <linux/pm_runtime.h> |
10 | #include <linux/regulator/consumer.h> |
11 | |
12 | #include "panfrost_device.h" |
13 | #include "panfrost_devfreq.h" |
14 | #include "panfrost_features.h" |
15 | #include "panfrost_issues.h" |
16 | #include "panfrost_gpu.h" |
17 | #include "panfrost_job.h" |
18 | #include "panfrost_mmu.h" |
19 | #include "panfrost_perfcnt.h" |
20 | |
21 | static int panfrost_reset_init(struct panfrost_device *pfdev) |
22 | { |
23 | pfdev->rstc = devm_reset_control_array_get_optional_exclusive(dev: pfdev->dev); |
24 | if (IS_ERR(ptr: pfdev->rstc)) { |
25 | dev_err(pfdev->dev, "get reset failed %ld\n" , PTR_ERR(pfdev->rstc)); |
26 | return PTR_ERR(ptr: pfdev->rstc); |
27 | } |
28 | |
29 | return reset_control_deassert(rstc: pfdev->rstc); |
30 | } |
31 | |
32 | static void panfrost_reset_fini(struct panfrost_device *pfdev) |
33 | { |
34 | reset_control_assert(rstc: pfdev->rstc); |
35 | } |
36 | |
37 | static int panfrost_clk_init(struct panfrost_device *pfdev) |
38 | { |
39 | int err; |
40 | unsigned long rate; |
41 | |
42 | pfdev->clock = devm_clk_get(dev: pfdev->dev, NULL); |
43 | if (IS_ERR(ptr: pfdev->clock)) { |
44 | dev_err(pfdev->dev, "get clock failed %ld\n" , PTR_ERR(pfdev->clock)); |
45 | return PTR_ERR(ptr: pfdev->clock); |
46 | } |
47 | |
48 | rate = clk_get_rate(clk: pfdev->clock); |
49 | dev_info(pfdev->dev, "clock rate = %lu\n" , rate); |
50 | |
51 | err = clk_prepare_enable(clk: pfdev->clock); |
52 | if (err) |
53 | return err; |
54 | |
55 | pfdev->bus_clock = devm_clk_get_optional(dev: pfdev->dev, id: "bus" ); |
56 | if (IS_ERR(ptr: pfdev->bus_clock)) { |
57 | dev_err(pfdev->dev, "get bus_clock failed %ld\n" , |
58 | PTR_ERR(pfdev->bus_clock)); |
59 | err = PTR_ERR(ptr: pfdev->bus_clock); |
60 | goto disable_clock; |
61 | } |
62 | |
63 | if (pfdev->bus_clock) { |
64 | rate = clk_get_rate(clk: pfdev->bus_clock); |
65 | dev_info(pfdev->dev, "bus_clock rate = %lu\n" , rate); |
66 | |
67 | err = clk_prepare_enable(clk: pfdev->bus_clock); |
68 | if (err) |
69 | goto disable_clock; |
70 | } |
71 | |
72 | return 0; |
73 | |
74 | disable_clock: |
75 | clk_disable_unprepare(clk: pfdev->clock); |
76 | |
77 | return err; |
78 | } |
79 | |
80 | static void panfrost_clk_fini(struct panfrost_device *pfdev) |
81 | { |
82 | clk_disable_unprepare(clk: pfdev->bus_clock); |
83 | clk_disable_unprepare(clk: pfdev->clock); |
84 | } |
85 | |
86 | static int panfrost_regulator_init(struct panfrost_device *pfdev) |
87 | { |
88 | int ret, i; |
89 | |
90 | pfdev->regulators = devm_kcalloc(dev: pfdev->dev, n: pfdev->comp->num_supplies, |
91 | size: sizeof(*pfdev->regulators), |
92 | GFP_KERNEL); |
93 | if (!pfdev->regulators) |
94 | return -ENOMEM; |
95 | |
96 | for (i = 0; i < pfdev->comp->num_supplies; i++) |
97 | pfdev->regulators[i].supply = pfdev->comp->supply_names[i]; |
98 | |
99 | ret = devm_regulator_bulk_get(dev: pfdev->dev, |
100 | num_consumers: pfdev->comp->num_supplies, |
101 | consumers: pfdev->regulators); |
102 | if (ret < 0) { |
103 | if (ret != -EPROBE_DEFER) |
104 | dev_err(pfdev->dev, "failed to get regulators: %d\n" , |
105 | ret); |
106 | return ret; |
107 | } |
108 | |
109 | ret = regulator_bulk_enable(num_consumers: pfdev->comp->num_supplies, |
110 | consumers: pfdev->regulators); |
111 | if (ret < 0) { |
112 | dev_err(pfdev->dev, "failed to enable regulators: %d\n" , ret); |
113 | return ret; |
114 | } |
115 | |
116 | return 0; |
117 | } |
118 | |
119 | static void panfrost_regulator_fini(struct panfrost_device *pfdev) |
120 | { |
121 | if (!pfdev->regulators) |
122 | return; |
123 | |
124 | regulator_bulk_disable(num_consumers: pfdev->comp->num_supplies, consumers: pfdev->regulators); |
125 | } |
126 | |
127 | static void panfrost_pm_domain_fini(struct panfrost_device *pfdev) |
128 | { |
129 | int i; |
130 | |
131 | for (i = 0; i < ARRAY_SIZE(pfdev->pm_domain_devs); i++) { |
132 | if (!pfdev->pm_domain_devs[i]) |
133 | break; |
134 | |
135 | if (pfdev->pm_domain_links[i]) |
136 | device_link_del(link: pfdev->pm_domain_links[i]); |
137 | |
138 | dev_pm_domain_detach(dev: pfdev->pm_domain_devs[i], power_off: true); |
139 | } |
140 | } |
141 | |
142 | static int panfrost_pm_domain_init(struct panfrost_device *pfdev) |
143 | { |
144 | int err; |
145 | int i, num_domains; |
146 | |
147 | num_domains = of_count_phandle_with_args(np: pfdev->dev->of_node, |
148 | list_name: "power-domains" , |
149 | cells_name: "#power-domain-cells" ); |
150 | |
151 | /* |
152 | * Single domain is handled by the core, and, if only a single power |
153 | * the power domain is requested, the property is optional. |
154 | */ |
155 | if (num_domains < 2 && pfdev->comp->num_pm_domains < 2) |
156 | return 0; |
157 | |
158 | if (num_domains != pfdev->comp->num_pm_domains) { |
159 | dev_err(pfdev->dev, |
160 | "Incorrect number of power domains: %d provided, %d needed\n" , |
161 | num_domains, pfdev->comp->num_pm_domains); |
162 | return -EINVAL; |
163 | } |
164 | |
165 | if (WARN(num_domains > ARRAY_SIZE(pfdev->pm_domain_devs), |
166 | "Too many supplies in compatible structure.\n" )) |
167 | return -EINVAL; |
168 | |
169 | for (i = 0; i < num_domains; i++) { |
170 | pfdev->pm_domain_devs[i] = |
171 | dev_pm_domain_attach_by_name(dev: pfdev->dev, |
172 | name: pfdev->comp->pm_domain_names[i]); |
173 | if (IS_ERR_OR_NULL(ptr: pfdev->pm_domain_devs[i])) { |
174 | err = PTR_ERR(ptr: pfdev->pm_domain_devs[i]) ? : -ENODATA; |
175 | pfdev->pm_domain_devs[i] = NULL; |
176 | dev_err(pfdev->dev, |
177 | "failed to get pm-domain %s(%d): %d\n" , |
178 | pfdev->comp->pm_domain_names[i], i, err); |
179 | goto err; |
180 | } |
181 | |
182 | pfdev->pm_domain_links[i] = device_link_add(consumer: pfdev->dev, |
183 | supplier: pfdev->pm_domain_devs[i], DL_FLAG_PM_RUNTIME | |
184 | DL_FLAG_STATELESS | DL_FLAG_RPM_ACTIVE); |
185 | if (!pfdev->pm_domain_links[i]) { |
186 | dev_err(pfdev->pm_domain_devs[i], |
187 | "adding device link failed!\n" ); |
188 | err = -ENODEV; |
189 | goto err; |
190 | } |
191 | } |
192 | |
193 | return 0; |
194 | |
195 | err: |
196 | panfrost_pm_domain_fini(pfdev); |
197 | return err; |
198 | } |
199 | |
200 | int panfrost_device_init(struct panfrost_device *pfdev) |
201 | { |
202 | int err; |
203 | |
204 | mutex_init(&pfdev->sched_lock); |
205 | INIT_LIST_HEAD(list: &pfdev->scheduled_jobs); |
206 | INIT_LIST_HEAD(list: &pfdev->as_lru_list); |
207 | |
208 | spin_lock_init(&pfdev->as_lock); |
209 | |
210 | spin_lock_init(&pfdev->cycle_counter.lock); |
211 | |
212 | err = panfrost_clk_init(pfdev); |
213 | if (err) { |
214 | dev_err(pfdev->dev, "clk init failed %d\n" , err); |
215 | return err; |
216 | } |
217 | |
218 | err = panfrost_devfreq_init(pfdev); |
219 | if (err) { |
220 | if (err != -EPROBE_DEFER) |
221 | dev_err(pfdev->dev, "devfreq init failed %d\n" , err); |
222 | goto out_clk; |
223 | } |
224 | |
225 | /* OPP will handle regulators */ |
226 | if (!pfdev->pfdevfreq.opp_of_table_added) { |
227 | err = panfrost_regulator_init(pfdev); |
228 | if (err) |
229 | goto out_devfreq; |
230 | } |
231 | |
232 | err = panfrost_reset_init(pfdev); |
233 | if (err) { |
234 | dev_err(pfdev->dev, "reset init failed %d\n" , err); |
235 | goto out_regulator; |
236 | } |
237 | |
238 | err = panfrost_pm_domain_init(pfdev); |
239 | if (err) |
240 | goto out_reset; |
241 | |
242 | pfdev->iomem = devm_platform_ioremap_resource(pdev: pfdev->pdev, index: 0); |
243 | if (IS_ERR(ptr: pfdev->iomem)) { |
244 | err = PTR_ERR(ptr: pfdev->iomem); |
245 | goto out_pm_domain; |
246 | } |
247 | |
248 | err = panfrost_gpu_init(pfdev); |
249 | if (err) |
250 | goto out_pm_domain; |
251 | |
252 | err = panfrost_mmu_init(pfdev); |
253 | if (err) |
254 | goto out_gpu; |
255 | |
256 | err = panfrost_job_init(pfdev); |
257 | if (err) |
258 | goto out_mmu; |
259 | |
260 | err = panfrost_perfcnt_init(pfdev); |
261 | if (err) |
262 | goto out_job; |
263 | |
264 | return 0; |
265 | out_job: |
266 | panfrost_job_fini(pfdev); |
267 | out_mmu: |
268 | panfrost_mmu_fini(pfdev); |
269 | out_gpu: |
270 | panfrost_gpu_fini(pfdev); |
271 | out_pm_domain: |
272 | panfrost_pm_domain_fini(pfdev); |
273 | out_reset: |
274 | panfrost_reset_fini(pfdev); |
275 | out_regulator: |
276 | panfrost_regulator_fini(pfdev); |
277 | out_devfreq: |
278 | panfrost_devfreq_fini(pfdev); |
279 | out_clk: |
280 | panfrost_clk_fini(pfdev); |
281 | return err; |
282 | } |
283 | |
284 | void panfrost_device_fini(struct panfrost_device *pfdev) |
285 | { |
286 | panfrost_perfcnt_fini(pfdev); |
287 | panfrost_job_fini(pfdev); |
288 | panfrost_mmu_fini(pfdev); |
289 | panfrost_gpu_fini(pfdev); |
290 | panfrost_pm_domain_fini(pfdev); |
291 | panfrost_reset_fini(pfdev); |
292 | panfrost_devfreq_fini(pfdev); |
293 | panfrost_regulator_fini(pfdev); |
294 | panfrost_clk_fini(pfdev); |
295 | } |
296 | |
297 | #define PANFROST_EXCEPTION(id) \ |
298 | [DRM_PANFROST_EXCEPTION_ ## id] = { \ |
299 | .name = #id, \ |
300 | } |
301 | |
302 | struct panfrost_exception_info { |
303 | const char *name; |
304 | }; |
305 | |
306 | static const struct panfrost_exception_info panfrost_exception_infos[] = { |
307 | PANFROST_EXCEPTION(OK), |
308 | PANFROST_EXCEPTION(DONE), |
309 | PANFROST_EXCEPTION(INTERRUPTED), |
310 | PANFROST_EXCEPTION(STOPPED), |
311 | PANFROST_EXCEPTION(TERMINATED), |
312 | PANFROST_EXCEPTION(KABOOM), |
313 | PANFROST_EXCEPTION(EUREKA), |
314 | PANFROST_EXCEPTION(ACTIVE), |
315 | PANFROST_EXCEPTION(JOB_CONFIG_FAULT), |
316 | PANFROST_EXCEPTION(JOB_POWER_FAULT), |
317 | PANFROST_EXCEPTION(JOB_READ_FAULT), |
318 | PANFROST_EXCEPTION(JOB_WRITE_FAULT), |
319 | PANFROST_EXCEPTION(JOB_AFFINITY_FAULT), |
320 | PANFROST_EXCEPTION(JOB_BUS_FAULT), |
321 | PANFROST_EXCEPTION(INSTR_INVALID_PC), |
322 | PANFROST_EXCEPTION(INSTR_INVALID_ENC), |
323 | PANFROST_EXCEPTION(INSTR_TYPE_MISMATCH), |
324 | PANFROST_EXCEPTION(INSTR_OPERAND_FAULT), |
325 | PANFROST_EXCEPTION(INSTR_TLS_FAULT), |
326 | PANFROST_EXCEPTION(INSTR_BARRIER_FAULT), |
327 | PANFROST_EXCEPTION(INSTR_ALIGN_FAULT), |
328 | PANFROST_EXCEPTION(DATA_INVALID_FAULT), |
329 | PANFROST_EXCEPTION(TILE_RANGE_FAULT), |
330 | PANFROST_EXCEPTION(ADDR_RANGE_FAULT), |
331 | PANFROST_EXCEPTION(IMPRECISE_FAULT), |
332 | PANFROST_EXCEPTION(OOM), |
333 | PANFROST_EXCEPTION(OOM_AFBC), |
334 | PANFROST_EXCEPTION(UNKNOWN), |
335 | PANFROST_EXCEPTION(DELAYED_BUS_FAULT), |
336 | PANFROST_EXCEPTION(GPU_SHAREABILITY_FAULT), |
337 | PANFROST_EXCEPTION(SYS_SHAREABILITY_FAULT), |
338 | PANFROST_EXCEPTION(GPU_CACHEABILITY_FAULT), |
339 | PANFROST_EXCEPTION(TRANSLATION_FAULT_0), |
340 | PANFROST_EXCEPTION(TRANSLATION_FAULT_1), |
341 | PANFROST_EXCEPTION(TRANSLATION_FAULT_2), |
342 | PANFROST_EXCEPTION(TRANSLATION_FAULT_3), |
343 | PANFROST_EXCEPTION(TRANSLATION_FAULT_4), |
344 | PANFROST_EXCEPTION(TRANSLATION_FAULT_IDENTITY), |
345 | PANFROST_EXCEPTION(PERM_FAULT_0), |
346 | PANFROST_EXCEPTION(PERM_FAULT_1), |
347 | PANFROST_EXCEPTION(PERM_FAULT_2), |
348 | PANFROST_EXCEPTION(PERM_FAULT_3), |
349 | PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_0), |
350 | PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_1), |
351 | PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_2), |
352 | PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_3), |
353 | PANFROST_EXCEPTION(ACCESS_FLAG_0), |
354 | PANFROST_EXCEPTION(ACCESS_FLAG_1), |
355 | PANFROST_EXCEPTION(ACCESS_FLAG_2), |
356 | PANFROST_EXCEPTION(ACCESS_FLAG_3), |
357 | PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN0), |
358 | PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN1), |
359 | PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN2), |
360 | PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN3), |
361 | PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT0), |
362 | PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT1), |
363 | PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT2), |
364 | PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT3), |
365 | PANFROST_EXCEPTION(MEM_ATTR_FAULT_0), |
366 | PANFROST_EXCEPTION(MEM_ATTR_FAULT_1), |
367 | PANFROST_EXCEPTION(MEM_ATTR_FAULT_2), |
368 | PANFROST_EXCEPTION(MEM_ATTR_FAULT_3), |
369 | PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_0), |
370 | PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_1), |
371 | PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_2), |
372 | PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_3), |
373 | }; |
374 | |
375 | const char *panfrost_exception_name(u32 exception_code) |
376 | { |
377 | if (WARN_ON(exception_code >= ARRAY_SIZE(panfrost_exception_infos) || |
378 | !panfrost_exception_infos[exception_code].name)) |
379 | return "Unknown exception type" ; |
380 | |
381 | return panfrost_exception_infos[exception_code].name; |
382 | } |
383 | |
384 | bool panfrost_exception_needs_reset(const struct panfrost_device *pfdev, |
385 | u32 exception_code) |
386 | { |
387 | /* If an occlusion query write causes a bus fault on affected GPUs, |
388 | * future fragment jobs may hang. Reset to workaround. |
389 | */ |
390 | if (exception_code == DRM_PANFROST_EXCEPTION_JOB_BUS_FAULT) |
391 | return panfrost_has_hw_issue(pfdev, issue: HW_ISSUE_TTRX_3076); |
392 | |
393 | /* No other GPUs we support need a reset */ |
394 | return false; |
395 | } |
396 | |
397 | void panfrost_device_reset(struct panfrost_device *pfdev) |
398 | { |
399 | panfrost_gpu_soft_reset(pfdev); |
400 | |
401 | panfrost_gpu_power_on(pfdev); |
402 | panfrost_mmu_reset(pfdev); |
403 | panfrost_job_enable_interrupts(pfdev); |
404 | } |
405 | |
406 | static int panfrost_device_resume(struct device *dev) |
407 | { |
408 | struct panfrost_device *pfdev = dev_get_drvdata(dev); |
409 | |
410 | panfrost_device_reset(pfdev); |
411 | panfrost_devfreq_resume(pfdev); |
412 | |
413 | return 0; |
414 | } |
415 | |
416 | static int panfrost_device_suspend(struct device *dev) |
417 | { |
418 | struct panfrost_device *pfdev = dev_get_drvdata(dev); |
419 | |
420 | if (!panfrost_job_is_idle(pfdev)) |
421 | return -EBUSY; |
422 | |
423 | panfrost_devfreq_suspend(pfdev); |
424 | panfrost_gpu_power_off(pfdev); |
425 | |
426 | return 0; |
427 | } |
428 | |
429 | EXPORT_GPL_RUNTIME_DEV_PM_OPS(panfrost_pm_ops, panfrost_device_suspend, |
430 | panfrost_device_resume, NULL); |
431 | |