1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * PSCI CPU idle driver.
4 *
5 * Copyright (C) 2019 ARM Ltd.
6 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
7 */
8
9#define pr_fmt(fmt) "CPUidle PSCI: " fmt
10
11#include <linux/cpuhotplug.h>
12#include <linux/cpu_cooling.h>
13#include <linux/cpuidle.h>
14#include <linux/cpumask.h>
15#include <linux/cpu_pm.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/of.h>
19#include <linux/device/faux.h>
20#include <linux/psci.h>
21#include <linux/pm_domain.h>
22#include <linux/pm_runtime.h>
23#include <linux/slab.h>
24#include <linux/string.h>
25#include <linux/syscore_ops.h>
26
27#include <asm/cpuidle.h>
28#include <trace/events/power.h>
29
30#include "cpuidle-psci.h"
31#include "dt_idle_states.h"
32#include "dt_idle_genpd.h"
33
34struct psci_cpuidle_data {
35 u32 *psci_states;
36 struct device *dev;
37};
38
39struct psci_cpuidle_domain_state {
40 struct generic_pm_domain *pd;
41 unsigned int state_idx;
42 u32 state;
43};
44
45static DEFINE_PER_CPU_READ_MOSTLY(struct psci_cpuidle_data, psci_cpuidle_data);
46static DEFINE_PER_CPU(struct psci_cpuidle_domain_state, psci_domain_state);
47static bool psci_cpuidle_use_syscore;
48
49void psci_set_domain_state(struct generic_pm_domain *pd, unsigned int state_idx,
50 u32 state)
51{
52 struct psci_cpuidle_domain_state *ds = this_cpu_ptr(&psci_domain_state);
53
54 ds->pd = pd;
55 ds->state_idx = state_idx;
56 ds->state = state;
57}
58
59static inline void psci_clear_domain_state(void)
60{
61 __this_cpu_write(psci_domain_state.state, 0);
62}
63
64static __cpuidle int __psci_enter_domain_idle_state(struct cpuidle_device *dev,
65 struct cpuidle_driver *drv, int idx,
66 bool s2idle)
67{
68 struct psci_cpuidle_data *data = this_cpu_ptr(&psci_cpuidle_data);
69 u32 *states = data->psci_states;
70 struct device *pd_dev = data->dev;
71 struct psci_cpuidle_domain_state *ds;
72 u32 state = states[idx];
73 int ret;
74
75 ret = cpu_pm_enter();
76 if (ret)
77 return -1;
78
79 /* Do runtime PM to manage a hierarchical CPU toplogy. */
80 if (s2idle)
81 dev_pm_genpd_suspend(dev: pd_dev);
82 else
83 pm_runtime_put_sync_suspend(dev: pd_dev);
84
85 ds = this_cpu_ptr(&psci_domain_state);
86 if (ds->state)
87 state = ds->state;
88
89 trace_psci_domain_idle_enter(dev->cpu, state, s2idle);
90 ret = psci_cpu_suspend_enter(state) ? -1 : idx;
91 trace_psci_domain_idle_exit(dev->cpu, state, s2idle);
92
93 if (s2idle)
94 dev_pm_genpd_resume(dev: pd_dev);
95 else
96 pm_runtime_get_sync(dev: pd_dev);
97
98 cpu_pm_exit();
99
100 /* Correct domain-idlestate statistics if we failed to enter. */
101 if (ret == -1 && ds->state)
102 pm_genpd_inc_rejected(genpd: ds->pd, state_idx: ds->state_idx);
103
104 /* Clear the domain state to start fresh when back from idle. */
105 psci_clear_domain_state();
106 return ret;
107}
108
109static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
110 struct cpuidle_driver *drv, int idx)
111{
112 return __psci_enter_domain_idle_state(dev, drv, idx, s2idle: false);
113}
114
115static int psci_enter_s2idle_domain_idle_state(struct cpuidle_device *dev,
116 struct cpuidle_driver *drv,
117 int idx)
118{
119 return __psci_enter_domain_idle_state(dev, drv, idx, s2idle: true);
120}
121
122static int psci_idle_cpuhp_up(unsigned int cpu)
123{
124 struct device *pd_dev = __this_cpu_read(psci_cpuidle_data.dev);
125
126 if (pd_dev) {
127 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
128 pm_runtime_get_sync(dev: pd_dev);
129 else
130 dev_pm_genpd_resume(dev: pd_dev);
131 }
132
133 return 0;
134}
135
136static int psci_idle_cpuhp_down(unsigned int cpu)
137{
138 struct device *pd_dev = __this_cpu_read(psci_cpuidle_data.dev);
139
140 if (pd_dev) {
141 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
142 pm_runtime_put_sync(dev: pd_dev);
143 else
144 dev_pm_genpd_suspend(dev: pd_dev);
145
146 /* Clear domain state to start fresh at next online. */
147 psci_clear_domain_state();
148 }
149
150 return 0;
151}
152
153static void psci_idle_syscore_switch(bool suspend)
154{
155 bool cleared = false;
156 struct device *dev;
157 int cpu;
158
159 for_each_possible_cpu(cpu) {
160 dev = per_cpu_ptr(&psci_cpuidle_data, cpu)->dev;
161
162 if (dev && suspend) {
163 dev_pm_genpd_suspend(dev);
164 } else if (dev) {
165 dev_pm_genpd_resume(dev);
166
167 /* Account for userspace having offlined a CPU. */
168 if (pm_runtime_status_suspended(dev))
169 pm_runtime_set_active(dev);
170
171 /* Clear domain state to re-start fresh. */
172 if (!cleared) {
173 psci_clear_domain_state();
174 cleared = true;
175 }
176 }
177 }
178}
179
180static int psci_idle_syscore_suspend(void *data)
181{
182 psci_idle_syscore_switch(suspend: true);
183 return 0;
184}
185
186static void psci_idle_syscore_resume(void *data)
187{
188 psci_idle_syscore_switch(suspend: false);
189}
190
191static const struct syscore_ops psci_idle_syscore_ops = {
192 .suspend = psci_idle_syscore_suspend,
193 .resume = psci_idle_syscore_resume,
194};
195
196static struct syscore psci_idle_syscore = {
197 .ops = &psci_idle_syscore_ops,
198};
199
200static void psci_idle_init_syscore(void)
201{
202 if (psci_cpuidle_use_syscore)
203 register_syscore(syscore: &psci_idle_syscore);
204}
205
206static void psci_idle_init_cpuhp(void)
207{
208 int err;
209
210 err = cpuhp_setup_state_nocalls(state: CPUHP_AP_CPU_PM_STARTING,
211 name: "cpuidle/psci:online",
212 startup: psci_idle_cpuhp_up,
213 teardown: psci_idle_cpuhp_down);
214 if (err)
215 pr_warn("Failed %d while setup cpuhp state\n", err);
216}
217
218static __cpuidle int psci_enter_idle_state(struct cpuidle_device *dev,
219 struct cpuidle_driver *drv, int idx)
220{
221 u32 *state = __this_cpu_read(psci_cpuidle_data.psci_states);
222
223 return CPU_PM_CPU_IDLE_ENTER_PARAM_RCU(psci_cpu_suspend_enter, idx, state[idx]);
224}
225
226static const struct of_device_id psci_idle_state_match[] = {
227 { .compatible = "arm,idle-state",
228 .data = psci_enter_idle_state },
229 { },
230};
231
232int psci_dt_parse_state_node(struct device_node *np, u32 *state)
233{
234 int err = of_property_read_u32(np, propname: "arm,psci-suspend-param", out_value: state);
235
236 if (err) {
237 pr_warn("%pOF missing arm,psci-suspend-param property\n", np);
238 return err;
239 }
240
241 if (!psci_power_state_is_valid(state: *state)) {
242 pr_warn("Invalid PSCI power state %#x\n", *state);
243 return -EINVAL;
244 }
245
246 return 0;
247}
248
249static int psci_dt_cpu_init_topology(struct cpuidle_driver *drv,
250 struct psci_cpuidle_data *data,
251 unsigned int state_count, int cpu)
252{
253 /* Currently limit the hierarchical topology to be used in OSI mode. */
254 if (!psci_has_osi_support())
255 return 0;
256
257 data->dev = dt_idle_attach_cpu(cpu, name: "psci");
258 if (IS_ERR_OR_NULL(ptr: data->dev))
259 return PTR_ERR_OR_ZERO(ptr: data->dev);
260
261 psci_cpuidle_use_syscore = true;
262
263 /*
264 * Using the deepest state for the CPU to trigger a potential selection
265 * of a shared state for the domain, assumes the domain states are all
266 * deeper states. On PREEMPT_RT the hierarchical topology is limited to
267 * s2ram and s2idle.
268 */
269 drv->states[state_count - 1].enter_s2idle = psci_enter_s2idle_domain_idle_state;
270 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
271 drv->states[state_count - 1].enter = psci_enter_domain_idle_state;
272
273 return 0;
274}
275
276static int psci_dt_cpu_init_idle(struct device *dev, struct cpuidle_driver *drv,
277 struct device_node *cpu_node,
278 unsigned int state_count, int cpu)
279{
280 int i, ret = 0;
281 u32 *psci_states;
282 struct device_node *state_node;
283 struct psci_cpuidle_data *data = per_cpu_ptr(&psci_cpuidle_data, cpu);
284
285 state_count++; /* Add WFI state too */
286 psci_states = devm_kcalloc(dev, n: state_count, size: sizeof(*psci_states),
287 GFP_KERNEL);
288 if (!psci_states)
289 return -ENOMEM;
290
291 for (i = 1; i < state_count; i++) {
292 state_node = of_get_cpu_state_node(cpu_node, index: i - 1);
293 if (!state_node)
294 break;
295
296 ret = psci_dt_parse_state_node(np: state_node, state: &psci_states[i]);
297 of_node_put(node: state_node);
298
299 if (ret)
300 return ret;
301
302 pr_debug("psci-power-state %#x index %d\n", psci_states[i], i);
303 }
304
305 if (i != state_count)
306 return -ENODEV;
307
308 /* Initialize optional data, used for the hierarchical topology. */
309 ret = psci_dt_cpu_init_topology(drv, data, state_count, cpu);
310 if (ret < 0)
311 return ret;
312
313 /* Idle states parsed correctly, store them in the per-cpu struct. */
314 data->psci_states = psci_states;
315 return 0;
316}
317
318static int psci_cpu_init_idle(struct device *dev, struct cpuidle_driver *drv,
319 unsigned int cpu, unsigned int state_count)
320{
321 struct device_node *cpu_node;
322 int ret;
323
324 /*
325 * If the PSCI cpu_suspend function hook has not been initialized
326 * idle states must not be enabled, so bail out
327 */
328 if (!psci_ops.cpu_suspend)
329 return -EOPNOTSUPP;
330
331 cpu_node = of_cpu_device_node_get(cpu);
332 if (!cpu_node)
333 return -ENODEV;
334
335 ret = psci_dt_cpu_init_idle(dev, drv, cpu_node, state_count, cpu);
336
337 of_node_put(node: cpu_node);
338
339 return ret;
340}
341
342static void psci_cpu_deinit_idle(int cpu)
343{
344 struct psci_cpuidle_data *data = per_cpu_ptr(&psci_cpuidle_data, cpu);
345
346 dt_idle_detach_cpu(dev: data->dev);
347 psci_cpuidle_use_syscore = false;
348}
349
350static int psci_idle_init_cpu(struct device *dev, int cpu)
351{
352 struct cpuidle_driver *drv;
353 struct device_node *cpu_node;
354 const char *enable_method;
355 int ret = 0;
356
357 cpu_node = of_cpu_device_node_get(cpu);
358 if (!cpu_node)
359 return -ENODEV;
360
361 /*
362 * Check whether the enable-method for the cpu is PSCI, fail
363 * if it is not.
364 */
365 enable_method = of_get_property(node: cpu_node, name: "enable-method", NULL);
366 if (!enable_method || (strcmp(enable_method, "psci")))
367 ret = -ENODEV;
368
369 of_node_put(node: cpu_node);
370 if (ret)
371 return ret;
372
373 drv = devm_kzalloc(dev, size: sizeof(*drv), GFP_KERNEL);
374 if (!drv)
375 return -ENOMEM;
376
377 drv->name = "psci_idle";
378 drv->owner = THIS_MODULE;
379 drv->cpumask = (struct cpumask *)cpumask_of(cpu);
380
381 /*
382 * PSCI idle states relies on architectural WFI to be represented as
383 * state index 0.
384 */
385 drv->states[0].enter = psci_enter_idle_state;
386 drv->states[0].exit_latency = 1;
387 drv->states[0].target_residency = 1;
388 drv->states[0].power_usage = UINT_MAX;
389 strscpy(drv->states[0].name, "WFI");
390 strscpy(drv->states[0].desc, "ARM WFI");
391
392 /*
393 * If no DT idle states are detected (ret == 0) let the driver
394 * initialization fail accordingly since there is no reason to
395 * initialize the idle driver if only wfi is supported, the
396 * default archictectural back-end already executes wfi
397 * on idle entry.
398 */
399 ret = dt_init_idle_driver(drv, matches: psci_idle_state_match, start_idx: 1);
400 if (ret <= 0)
401 return ret ? : -ENODEV;
402
403 /*
404 * Initialize PSCI idle states.
405 */
406 ret = psci_cpu_init_idle(dev, drv, cpu, state_count: ret);
407 if (ret) {
408 pr_err("CPU %d failed to PSCI idle\n", cpu);
409 return ret;
410 }
411
412 ret = cpuidle_register(drv, NULL);
413 if (ret)
414 goto deinit;
415
416 cpuidle_cooling_register(drv);
417
418 return 0;
419deinit:
420 psci_cpu_deinit_idle(cpu);
421 return ret;
422}
423
424/*
425 * psci_idle_probe - Initializes PSCI cpuidle driver
426 *
427 * Initializes PSCI cpuidle driver for all present CPUs, if any CPU fails
428 * to register cpuidle driver then rollback to cancel all CPUs
429 * registration.
430 */
431static int psci_cpuidle_probe(struct faux_device *fdev)
432{
433 int cpu, ret;
434 struct cpuidle_driver *drv;
435 struct cpuidle_device *dev;
436
437 for_each_present_cpu(cpu) {
438 ret = psci_idle_init_cpu(dev: &fdev->dev, cpu);
439 if (ret)
440 goto out_fail;
441 }
442
443 psci_idle_init_syscore();
444 psci_idle_init_cpuhp();
445 return 0;
446
447out_fail:
448 while (--cpu >= 0) {
449 dev = per_cpu(cpuidle_devices, cpu);
450 drv = cpuidle_get_cpu_driver(dev);
451 cpuidle_unregister(drv);
452 psci_cpu_deinit_idle(cpu);
453 }
454
455 return ret;
456}
457
458static struct faux_device_ops psci_cpuidle_ops = {
459 .probe = psci_cpuidle_probe,
460};
461
462static bool __init dt_idle_state_present(void)
463{
464 struct device_node *cpu_node __free(device_node) =
465 of_cpu_device_node_get(cpu: cpumask_first(cpu_possible_mask));
466 if (!cpu_node)
467 return false;
468
469 struct device_node *state_node __free(device_node) =
470 of_get_cpu_state_node(cpu_node, index: 0);
471 if (!state_node)
472 return false;
473
474 return !!of_match_node(matches: psci_idle_state_match, node: state_node);
475}
476
477static int __init psci_idle_init(void)
478{
479 struct faux_device *fdev;
480
481 if (!dt_idle_state_present())
482 return 0;
483
484 fdev = faux_device_create(name: "psci-cpuidle", NULL, faux_ops: &psci_cpuidle_ops);
485 if (!fdev) {
486 pr_err("Failed to create psci-cpuidle device\n");
487 return -ENODEV;
488 }
489
490 return 0;
491}
492device_initcall(psci_idle_init);
493

source code of linux/drivers/cpuidle/cpuidle-psci.c