1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * CPU idle driver for Tegra CPUs |
4 | * |
5 | * Copyright (c) 2010-2013, NVIDIA Corporation. |
6 | * Copyright (c) 2011 Google, Inc. |
7 | * Author: Colin Cross <ccross@android.com> |
8 | * Gary King <gking@nvidia.com> |
9 | * |
10 | * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com> |
11 | * |
12 | * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com> |
13 | */ |
14 | |
15 | #define pr_fmt(fmt) "tegra-cpuidle: " fmt |
16 | |
17 | #include <linux/atomic.h> |
18 | #include <linux/cpuidle.h> |
19 | #include <linux/cpumask.h> |
20 | #include <linux/cpu_pm.h> |
21 | #include <linux/delay.h> |
22 | #include <linux/errno.h> |
23 | #include <linux/platform_device.h> |
24 | #include <linux/types.h> |
25 | |
26 | #include <linux/clk/tegra.h> |
27 | #include <linux/firmware/trusted_foundations.h> |
28 | |
29 | #include <soc/tegra/cpuidle.h> |
30 | #include <soc/tegra/flowctrl.h> |
31 | #include <soc/tegra/fuse.h> |
32 | #include <soc/tegra/irq.h> |
33 | #include <soc/tegra/pm.h> |
34 | #include <soc/tegra/pmc.h> |
35 | |
36 | #include <asm/cpuidle.h> |
37 | #include <asm/firmware.h> |
38 | #include <asm/smp_plat.h> |
39 | #include <asm/suspend.h> |
40 | |
41 | enum tegra_state { |
42 | TEGRA_C1, |
43 | TEGRA_C7, |
44 | TEGRA_CC6, |
45 | TEGRA_STATE_COUNT, |
46 | }; |
47 | |
48 | static atomic_t tegra_idle_barrier; |
49 | static atomic_t tegra_abort_flag; |
50 | |
51 | static void tegra_cpuidle_report_cpus_state(void) |
52 | { |
53 | unsigned long cpu, lcpu, csr; |
54 | |
55 | for_each_cpu(lcpu, cpu_possible_mask) { |
56 | cpu = cpu_logical_map(lcpu); |
57 | csr = flowctrl_read_cpu_csr(cpuid: cpu); |
58 | |
59 | pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n" , |
60 | cpu, cpu_online(lcpu), csr); |
61 | } |
62 | } |
63 | |
64 | static int tegra_cpuidle_wait_for_secondary_cpus_parking(void) |
65 | { |
66 | unsigned int retries = 3; |
67 | |
68 | while (retries--) { |
69 | unsigned int delay_us = 10; |
70 | unsigned int timeout_us = 500 * 1000 / delay_us; |
71 | |
72 | /* |
73 | * The primary CPU0 core shall wait for the secondaries |
74 | * shutdown in order to power-off CPU's cluster safely. |
75 | * The timeout value depends on the current CPU frequency, |
76 | * it takes about 40-150us in average and over 1000us in |
77 | * a worst case scenario. |
78 | */ |
79 | do { |
80 | if (tegra_cpu_rail_off_ready()) |
81 | return 0; |
82 | |
83 | udelay(delay_us); |
84 | |
85 | } while (timeout_us--); |
86 | |
87 | pr_err("secondary CPU taking too long to park\n" ); |
88 | |
89 | tegra_cpuidle_report_cpus_state(); |
90 | } |
91 | |
92 | pr_err("timed out waiting secondaries to park\n" ); |
93 | |
94 | return -ETIMEDOUT; |
95 | } |
96 | |
97 | static void tegra_cpuidle_unpark_secondary_cpus(void) |
98 | { |
99 | unsigned int cpu, lcpu; |
100 | |
101 | for_each_cpu(lcpu, cpu_online_mask) { |
102 | cpu = cpu_logical_map(lcpu); |
103 | |
104 | if (cpu > 0) { |
105 | tegra_enable_cpu_clock(cpu); |
106 | tegra_cpu_out_of_reset(cpu); |
107 | flowctrl_write_cpu_halt(cpuid: cpu, value: 0); |
108 | } |
109 | } |
110 | } |
111 | |
112 | static int tegra_cpuidle_cc6_enter(unsigned int cpu) |
113 | { |
114 | int ret; |
115 | |
116 | if (cpu > 0) { |
117 | ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu); |
118 | } else { |
119 | ret = tegra_cpuidle_wait_for_secondary_cpus_parking(); |
120 | if (!ret) |
121 | ret = tegra_pm_enter_lp2(); |
122 | |
123 | tegra_cpuidle_unpark_secondary_cpus(); |
124 | } |
125 | |
126 | return ret; |
127 | } |
128 | |
129 | static int tegra_cpuidle_c7_enter(void) |
130 | { |
131 | int err; |
132 | |
133 | err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2); |
134 | if (err && err != -ENOSYS) |
135 | return err; |
136 | |
137 | return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend); |
138 | } |
139 | |
140 | static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev) |
141 | { |
142 | if (tegra_pending_sgi()) { |
143 | /* |
144 | * CPU got local interrupt that will be lost after GIC's |
145 | * shutdown because GIC driver doesn't save/restore the |
146 | * pending SGI state across CPU cluster PM. Abort and retry |
147 | * next time. |
148 | */ |
149 | atomic_set(v: &tegra_abort_flag, i: 1); |
150 | } |
151 | |
152 | cpuidle_coupled_parallel_barrier(dev, a: &tegra_idle_barrier); |
153 | |
154 | if (atomic_read(v: &tegra_abort_flag)) { |
155 | cpuidle_coupled_parallel_barrier(dev, a: &tegra_idle_barrier); |
156 | atomic_set(v: &tegra_abort_flag, i: 0); |
157 | return -EINTR; |
158 | } |
159 | |
160 | return 0; |
161 | } |
162 | |
163 | static __cpuidle int tegra_cpuidle_state_enter(struct cpuidle_device *dev, |
164 | int index, unsigned int cpu) |
165 | { |
166 | int err; |
167 | |
168 | /* |
169 | * CC6 state is the "CPU cluster power-off" state. In order to |
170 | * enter this state, at first the secondary CPU cores need to be |
171 | * parked into offline mode, then the last CPU should clean out |
172 | * remaining dirty cache lines into DRAM and trigger Flow Controller |
173 | * logic that turns off the cluster's power domain (which includes |
174 | * CPU cores, GIC and L2 cache). |
175 | */ |
176 | if (index == TEGRA_CC6) { |
177 | err = tegra_cpuidle_coupled_barrier(dev); |
178 | if (err) |
179 | return err; |
180 | } |
181 | |
182 | local_fiq_disable(); |
183 | tegra_pm_set_cpu_in_lp2(); |
184 | cpu_pm_enter(); |
185 | |
186 | ct_cpuidle_enter(); |
187 | |
188 | switch (index) { |
189 | case TEGRA_C7: |
190 | err = tegra_cpuidle_c7_enter(); |
191 | break; |
192 | |
193 | case TEGRA_CC6: |
194 | err = tegra_cpuidle_cc6_enter(cpu); |
195 | break; |
196 | |
197 | default: |
198 | err = -EINVAL; |
199 | break; |
200 | } |
201 | |
202 | ct_cpuidle_exit(); |
203 | |
204 | cpu_pm_exit(); |
205 | tegra_pm_clear_cpu_in_lp2(); |
206 | local_fiq_enable(); |
207 | |
208 | return err ?: index; |
209 | } |
210 | |
211 | static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu) |
212 | { |
213 | /* |
214 | * On Tegra30 CPU0 can't be power-gated separately from secondary |
215 | * cores because it gates the whole CPU cluster. |
216 | */ |
217 | if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30) |
218 | return index; |
219 | |
220 | /* put CPU0 into C1 if C7 is requested and secondaries are online */ |
221 | if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1) |
222 | index = TEGRA_C1; |
223 | else |
224 | index = TEGRA_CC6; |
225 | |
226 | return index; |
227 | } |
228 | |
229 | static __cpuidle int tegra_cpuidle_enter(struct cpuidle_device *dev, |
230 | struct cpuidle_driver *drv, |
231 | int index) |
232 | { |
233 | bool do_rcu = drv->states[index].flags & CPUIDLE_FLAG_RCU_IDLE; |
234 | unsigned int cpu = cpu_logical_map(dev->cpu); |
235 | int ret; |
236 | |
237 | index = tegra_cpuidle_adjust_state_index(index, cpu); |
238 | if (dev->states_usage[index].disable) |
239 | return -1; |
240 | |
241 | if (index == TEGRA_C1) { |
242 | if (do_rcu) |
243 | ct_cpuidle_enter(); |
244 | ret = arm_cpuidle_simple_enter(dev, drv, index); |
245 | if (do_rcu) |
246 | ct_cpuidle_exit(); |
247 | } else |
248 | ret = tegra_cpuidle_state_enter(dev, index, cpu); |
249 | |
250 | if (ret < 0) { |
251 | if (ret != -EINTR || index != TEGRA_CC6) |
252 | pr_err_once("failed to enter state %d err: %d\n" , |
253 | index, ret); |
254 | index = -1; |
255 | } else { |
256 | index = ret; |
257 | } |
258 | |
259 | return index; |
260 | } |
261 | |
262 | static int tegra114_enter_s2idle(struct cpuidle_device *dev, |
263 | struct cpuidle_driver *drv, |
264 | int index) |
265 | { |
266 | tegra_cpuidle_enter(dev, drv, index); |
267 | |
268 | return 0; |
269 | } |
270 | |
271 | /* |
272 | * The previous versions of Tegra CPUIDLE driver used a different "legacy" |
273 | * terminology for naming of the idling states, while this driver uses the |
274 | * new terminology. |
275 | * |
276 | * Mapping of the old terms into the new ones: |
277 | * |
278 | * Old | New |
279 | * --------- |
280 | * LP3 | C1 (CPU core clock gating) |
281 | * LP2 | C7 (CPU core power gating) |
282 | * LP2 | CC6 (CPU cluster power gating) |
283 | * |
284 | * Note that that the older CPUIDLE driver versions didn't explicitly |
285 | * differentiate the LP2 states because these states either used the same |
286 | * code path or because CC6 wasn't supported. |
287 | */ |
288 | static struct cpuidle_driver tegra_idle_driver = { |
289 | .name = "tegra_idle" , |
290 | .states = { |
291 | [TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600), |
292 | [TEGRA_C7] = { |
293 | .enter = tegra_cpuidle_enter, |
294 | .exit_latency = 2000, |
295 | .target_residency = 2200, |
296 | .power_usage = 100, |
297 | .flags = CPUIDLE_FLAG_TIMER_STOP | |
298 | CPUIDLE_FLAG_RCU_IDLE, |
299 | .name = "C7" , |
300 | .desc = "CPU core powered off" , |
301 | }, |
302 | [TEGRA_CC6] = { |
303 | .enter = tegra_cpuidle_enter, |
304 | .exit_latency = 5000, |
305 | .target_residency = 10000, |
306 | .power_usage = 0, |
307 | .flags = CPUIDLE_FLAG_TIMER_STOP | |
308 | CPUIDLE_FLAG_RCU_IDLE | |
309 | CPUIDLE_FLAG_COUPLED, |
310 | .name = "CC6" , |
311 | .desc = "CPU cluster powered off" , |
312 | }, |
313 | }, |
314 | .state_count = TEGRA_STATE_COUNT, |
315 | .safe_state_index = TEGRA_C1, |
316 | }; |
317 | |
318 | static inline void tegra_cpuidle_disable_state(enum tegra_state state) |
319 | { |
320 | cpuidle_driver_state_disabled(drv: &tegra_idle_driver, idx: state, disable: true); |
321 | } |
322 | |
323 | /* |
324 | * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether |
325 | * they are legacy IRQs or MSI, are lost when CC6 is enabled. To work around |
326 | * this, simply disable CC6 if the PCI driver and DT node are both enabled. |
327 | */ |
328 | void tegra_cpuidle_pcie_irqs_in_use(void) |
329 | { |
330 | struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6]; |
331 | |
332 | if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) || |
333 | tegra_get_chip_id() != TEGRA20) |
334 | return; |
335 | |
336 | pr_info("disabling CC6 state, since PCIe IRQs are in use\n" ); |
337 | tegra_cpuidle_disable_state(state: TEGRA_CC6); |
338 | } |
339 | |
340 | static void tegra_cpuidle_setup_tegra114_c7_state(void) |
341 | { |
342 | struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7]; |
343 | |
344 | s->enter_s2idle = tegra114_enter_s2idle; |
345 | s->target_residency = 1000; |
346 | s->exit_latency = 500; |
347 | } |
348 | |
349 | static int tegra_cpuidle_probe(struct platform_device *pdev) |
350 | { |
351 | if (tegra_pmc_get_suspend_mode() == TEGRA_SUSPEND_NOT_READY) |
352 | return -EPROBE_DEFER; |
353 | |
354 | /* LP2 could be disabled in device-tree */ |
355 | if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2) |
356 | tegra_cpuidle_disable_state(state: TEGRA_CC6); |
357 | |
358 | /* |
359 | * Required suspend-resume functionality, which is provided by the |
360 | * Tegra-arch core and PMC driver, is unavailable if PM-sleep option |
361 | * is disabled. |
362 | */ |
363 | if (!IS_ENABLED(CONFIG_PM_SLEEP)) { |
364 | tegra_cpuidle_disable_state(state: TEGRA_C7); |
365 | tegra_cpuidle_disable_state(state: TEGRA_CC6); |
366 | } |
367 | |
368 | /* |
369 | * Generic WFI state (also known as C1 or LP3) and the coupled CPU |
370 | * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs. |
371 | */ |
372 | switch (tegra_get_chip_id()) { |
373 | case TEGRA20: |
374 | /* Tegra20 isn't capable to power-off individual CPU cores */ |
375 | tegra_cpuidle_disable_state(state: TEGRA_C7); |
376 | break; |
377 | |
378 | case TEGRA30: |
379 | break; |
380 | |
381 | case TEGRA114: |
382 | case TEGRA124: |
383 | tegra_cpuidle_setup_tegra114_c7_state(); |
384 | |
385 | /* coupled CC6 (LP2) state isn't implemented yet */ |
386 | tegra_cpuidle_disable_state(state: TEGRA_CC6); |
387 | break; |
388 | |
389 | default: |
390 | return -EINVAL; |
391 | } |
392 | |
393 | return cpuidle_register(drv: &tegra_idle_driver, cpu_possible_mask); |
394 | } |
395 | |
396 | static struct platform_driver tegra_cpuidle_driver = { |
397 | .probe = tegra_cpuidle_probe, |
398 | .driver = { |
399 | .name = "tegra-cpuidle" , |
400 | }, |
401 | }; |
402 | builtin_platform_driver(tegra_cpuidle_driver); |
403 | |