1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * System Control and Power Interface (SCMI) based CPUFreq Interface driver |
4 | * |
5 | * Copyright (C) 2018-2021 ARM Ltd. |
6 | * Sudeep Holla <sudeep.holla@arm.com> |
7 | */ |
8 | |
9 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
10 | |
11 | #include <linux/clk-provider.h> |
12 | #include <linux/cpu.h> |
13 | #include <linux/cpufreq.h> |
14 | #include <linux/cpumask.h> |
15 | #include <linux/energy_model.h> |
16 | #include <linux/export.h> |
17 | #include <linux/module.h> |
18 | #include <linux/pm_opp.h> |
19 | #include <linux/slab.h> |
20 | #include <linux/scmi_protocol.h> |
21 | #include <linux/types.h> |
22 | #include <linux/units.h> |
23 | |
24 | struct scmi_data { |
25 | int domain_id; |
26 | int nr_opp; |
27 | struct device *cpu_dev; |
28 | cpumask_var_t opp_shared_cpus; |
29 | }; |
30 | |
31 | static struct scmi_protocol_handle *ph; |
32 | static const struct scmi_perf_proto_ops *perf_ops; |
33 | static struct cpufreq_driver scmi_cpufreq_driver; |
34 | |
35 | static unsigned int scmi_cpufreq_get_rate(unsigned int cpu) |
36 | { |
37 | struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); |
38 | struct scmi_data *priv = policy->driver_data; |
39 | unsigned long rate; |
40 | int ret; |
41 | |
42 | ret = perf_ops->freq_get(ph, priv->domain_id, &rate, false); |
43 | if (ret) |
44 | return 0; |
45 | return rate / 1000; |
46 | } |
47 | |
48 | /* |
49 | * perf_ops->freq_set is not a synchronous, the actual OPP change will |
50 | * happen asynchronously and can get notified if the events are |
51 | * subscribed for by the SCMI firmware |
52 | */ |
53 | static int |
54 | scmi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index) |
55 | { |
56 | struct scmi_data *priv = policy->driver_data; |
57 | u64 freq = policy->freq_table[index].frequency; |
58 | |
59 | return perf_ops->freq_set(ph, priv->domain_id, freq * 1000, false); |
60 | } |
61 | |
62 | static unsigned int scmi_cpufreq_fast_switch(struct cpufreq_policy *policy, |
63 | unsigned int target_freq) |
64 | { |
65 | struct scmi_data *priv = policy->driver_data; |
66 | |
67 | if (!perf_ops->freq_set(ph, priv->domain_id, |
68 | target_freq * 1000, true)) |
69 | return target_freq; |
70 | |
71 | return 0; |
72 | } |
73 | |
74 | static int scmi_cpu_domain_id(struct device *cpu_dev) |
75 | { |
76 | struct device_node *np = cpu_dev->of_node; |
77 | struct of_phandle_args domain_id; |
78 | int index; |
79 | |
80 | if (of_parse_phandle_with_args(np, list_name: "clocks" , cells_name: "#clock-cells" , index: 0, |
81 | out_args: &domain_id)) { |
82 | /* Find the corresponding index for power-domain "perf". */ |
83 | index = of_property_match_string(np, propname: "power-domain-names" , |
84 | string: "perf" ); |
85 | if (index < 0) |
86 | return -EINVAL; |
87 | |
88 | if (of_parse_phandle_with_args(np, list_name: "power-domains" , |
89 | cells_name: "#power-domain-cells" , index, |
90 | out_args: &domain_id)) |
91 | return -EINVAL; |
92 | } |
93 | |
94 | return domain_id.args[0]; |
95 | } |
96 | |
97 | static int |
98 | scmi_get_sharing_cpus(struct device *cpu_dev, int domain, |
99 | struct cpumask *cpumask) |
100 | { |
101 | int cpu, tdomain; |
102 | struct device *tcpu_dev; |
103 | |
104 | for_each_possible_cpu(cpu) { |
105 | if (cpu == cpu_dev->id) |
106 | continue; |
107 | |
108 | tcpu_dev = get_cpu_device(cpu); |
109 | if (!tcpu_dev) |
110 | continue; |
111 | |
112 | tdomain = scmi_cpu_domain_id(cpu_dev: tcpu_dev); |
113 | if (tdomain == domain) |
114 | cpumask_set_cpu(cpu, dstp: cpumask); |
115 | } |
116 | |
117 | return 0; |
118 | } |
119 | |
120 | static int __maybe_unused |
121 | scmi_get_cpu_power(struct device *cpu_dev, unsigned long *power, |
122 | unsigned long *KHz) |
123 | { |
124 | enum scmi_power_scale power_scale = perf_ops->power_scale_get(ph); |
125 | unsigned long Hz; |
126 | int ret, domain; |
127 | |
128 | domain = scmi_cpu_domain_id(cpu_dev); |
129 | if (domain < 0) |
130 | return domain; |
131 | |
132 | /* Get the power cost of the performance domain. */ |
133 | Hz = *KHz * 1000; |
134 | ret = perf_ops->est_power_get(ph, domain, &Hz, power); |
135 | if (ret) |
136 | return ret; |
137 | |
138 | /* Convert the power to uW if it is mW (ignore bogoW) */ |
139 | if (power_scale == SCMI_POWER_MILLIWATTS) |
140 | *power *= MICROWATT_PER_MILLIWATT; |
141 | |
142 | /* The EM framework specifies the frequency in KHz. */ |
143 | *KHz = Hz / 1000; |
144 | |
145 | return 0; |
146 | } |
147 | |
148 | static int |
149 | scmi_get_rate_limit(u32 domain, bool has_fast_switch) |
150 | { |
151 | int ret, rate_limit; |
152 | |
153 | if (has_fast_switch) { |
154 | /* |
155 | * Fast channels are used whenever available, |
156 | * so use their rate_limit value if populated. |
157 | */ |
158 | ret = perf_ops->fast_switch_rate_limit(ph, domain, |
159 | &rate_limit); |
160 | if (!ret && rate_limit) |
161 | return rate_limit; |
162 | } |
163 | |
164 | ret = perf_ops->rate_limit_get(ph, domain, &rate_limit); |
165 | if (ret) |
166 | return 0; |
167 | |
168 | return rate_limit; |
169 | } |
170 | |
171 | static struct freq_attr *scmi_cpufreq_hw_attr[] = { |
172 | &cpufreq_freq_attr_scaling_available_freqs, |
173 | NULL, |
174 | NULL, |
175 | }; |
176 | |
177 | static int scmi_cpufreq_init(struct cpufreq_policy *policy) |
178 | { |
179 | int ret, nr_opp, domain; |
180 | unsigned int latency; |
181 | struct device *cpu_dev; |
182 | struct scmi_data *priv; |
183 | struct cpufreq_frequency_table *freq_table; |
184 | |
185 | cpu_dev = get_cpu_device(cpu: policy->cpu); |
186 | if (!cpu_dev) { |
187 | pr_err("failed to get cpu%d device\n" , policy->cpu); |
188 | return -ENODEV; |
189 | } |
190 | |
191 | domain = scmi_cpu_domain_id(cpu_dev); |
192 | if (domain < 0) |
193 | return domain; |
194 | |
195 | priv = kzalloc(size: sizeof(*priv), GFP_KERNEL); |
196 | if (!priv) |
197 | return -ENOMEM; |
198 | |
199 | if (!zalloc_cpumask_var(mask: &priv->opp_shared_cpus, GFP_KERNEL)) { |
200 | ret = -ENOMEM; |
201 | goto out_free_priv; |
202 | } |
203 | |
204 | /* Obtain CPUs that share SCMI performance controls */ |
205 | ret = scmi_get_sharing_cpus(cpu_dev, domain, cpumask: policy->cpus); |
206 | if (ret) { |
207 | dev_warn(cpu_dev, "failed to get sharing cpumask\n" ); |
208 | goto out_free_cpumask; |
209 | } |
210 | |
211 | /* |
212 | * Obtain CPUs that share performance levels. |
213 | * The OPP 'sharing cpus' info may come from DT through an empty opp |
214 | * table and opp-shared. |
215 | */ |
216 | ret = dev_pm_opp_of_get_sharing_cpus(cpu_dev, cpumask: priv->opp_shared_cpus); |
217 | if (ret || cpumask_empty(srcp: priv->opp_shared_cpus)) { |
218 | /* |
219 | * Either opp-table is not set or no opp-shared was found. |
220 | * Use the CPU mask from SCMI to designate CPUs sharing an OPP |
221 | * table. |
222 | */ |
223 | cpumask_copy(dstp: priv->opp_shared_cpus, srcp: policy->cpus); |
224 | } |
225 | |
226 | /* |
227 | * A previous CPU may have marked OPPs as shared for a few CPUs, based on |
228 | * what OPP core provided. If the current CPU is part of those few, then |
229 | * there is no need to add OPPs again. |
230 | */ |
231 | nr_opp = dev_pm_opp_get_opp_count(dev: cpu_dev); |
232 | if (nr_opp <= 0) { |
233 | ret = perf_ops->device_opps_add(ph, cpu_dev, domain); |
234 | if (ret) { |
235 | dev_warn(cpu_dev, "failed to add opps to the device\n" ); |
236 | goto out_free_cpumask; |
237 | } |
238 | |
239 | nr_opp = dev_pm_opp_get_opp_count(dev: cpu_dev); |
240 | if (nr_opp <= 0) { |
241 | dev_err(cpu_dev, "%s: No OPPs for this device: %d\n" , |
242 | __func__, nr_opp); |
243 | |
244 | ret = -ENODEV; |
245 | goto out_free_opp; |
246 | } |
247 | |
248 | ret = dev_pm_opp_set_sharing_cpus(cpu_dev, cpumask: priv->opp_shared_cpus); |
249 | if (ret) { |
250 | dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n" , |
251 | __func__, ret); |
252 | |
253 | goto out_free_opp; |
254 | } |
255 | |
256 | priv->nr_opp = nr_opp; |
257 | } |
258 | |
259 | ret = dev_pm_opp_init_cpufreq_table(dev: cpu_dev, table: &freq_table); |
260 | if (ret) { |
261 | dev_err(cpu_dev, "failed to init cpufreq table: %d\n" , ret); |
262 | goto out_free_opp; |
263 | } |
264 | |
265 | priv->cpu_dev = cpu_dev; |
266 | priv->domain_id = domain; |
267 | |
268 | policy->driver_data = priv; |
269 | policy->freq_table = freq_table; |
270 | |
271 | /* SCMI allows DVFS request for any domain from any CPU */ |
272 | policy->dvfs_possible_from_any_cpu = true; |
273 | |
274 | latency = perf_ops->transition_latency_get(ph, domain); |
275 | if (!latency) |
276 | latency = CPUFREQ_ETERNAL; |
277 | |
278 | policy->cpuinfo.transition_latency = latency; |
279 | |
280 | policy->fast_switch_possible = |
281 | perf_ops->fast_switch_possible(ph, domain); |
282 | |
283 | policy->transition_delay_us = |
284 | scmi_get_rate_limit(domain, has_fast_switch: policy->fast_switch_possible); |
285 | |
286 | if (policy_has_boost_freq(policy)) { |
287 | ret = cpufreq_enable_boost_support(); |
288 | if (ret) { |
289 | dev_warn(cpu_dev, "failed to enable boost: %d\n" , ret); |
290 | goto out_free_opp; |
291 | } else { |
292 | scmi_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; |
293 | scmi_cpufreq_driver.boost_enabled = true; |
294 | } |
295 | } |
296 | |
297 | return 0; |
298 | |
299 | out_free_opp: |
300 | dev_pm_opp_remove_all_dynamic(dev: cpu_dev); |
301 | |
302 | out_free_cpumask: |
303 | free_cpumask_var(mask: priv->opp_shared_cpus); |
304 | |
305 | out_free_priv: |
306 | kfree(objp: priv); |
307 | |
308 | return ret; |
309 | } |
310 | |
311 | static int scmi_cpufreq_exit(struct cpufreq_policy *policy) |
312 | { |
313 | struct scmi_data *priv = policy->driver_data; |
314 | |
315 | dev_pm_opp_free_cpufreq_table(dev: priv->cpu_dev, table: &policy->freq_table); |
316 | dev_pm_opp_remove_all_dynamic(dev: priv->cpu_dev); |
317 | free_cpumask_var(mask: priv->opp_shared_cpus); |
318 | kfree(objp: priv); |
319 | |
320 | return 0; |
321 | } |
322 | |
323 | static void scmi_cpufreq_register_em(struct cpufreq_policy *policy) |
324 | { |
325 | struct em_data_callback em_cb = EM_DATA_CB(scmi_get_cpu_power); |
326 | enum scmi_power_scale power_scale = perf_ops->power_scale_get(ph); |
327 | struct scmi_data *priv = policy->driver_data; |
328 | bool em_power_scale = false; |
329 | |
330 | /* |
331 | * This callback will be called for each policy, but we don't need to |
332 | * register with EM every time. Despite not being part of the same |
333 | * policy, some CPUs may still share their perf-domains, and a CPU from |
334 | * another policy may already have registered with EM on behalf of CPUs |
335 | * of this policy. |
336 | */ |
337 | if (!priv->nr_opp) |
338 | return; |
339 | |
340 | if (power_scale == SCMI_POWER_MILLIWATTS |
341 | || power_scale == SCMI_POWER_MICROWATTS) |
342 | em_power_scale = true; |
343 | |
344 | em_dev_register_perf_domain(dev: get_cpu_device(cpu: policy->cpu), nr_states: priv->nr_opp, |
345 | cb: &em_cb, span: priv->opp_shared_cpus, |
346 | microwatts: em_power_scale); |
347 | } |
348 | |
349 | static struct cpufreq_driver scmi_cpufreq_driver = { |
350 | .name = "scmi" , |
351 | .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | |
352 | CPUFREQ_NEED_INITIAL_FREQ_CHECK | |
353 | CPUFREQ_IS_COOLING_DEV, |
354 | .verify = cpufreq_generic_frequency_table_verify, |
355 | .attr = scmi_cpufreq_hw_attr, |
356 | .target_index = scmi_cpufreq_set_target, |
357 | .fast_switch = scmi_cpufreq_fast_switch, |
358 | .get = scmi_cpufreq_get_rate, |
359 | .init = scmi_cpufreq_init, |
360 | .exit = scmi_cpufreq_exit, |
361 | .register_em = scmi_cpufreq_register_em, |
362 | }; |
363 | |
364 | static int scmi_cpufreq_probe(struct scmi_device *sdev) |
365 | { |
366 | int ret; |
367 | struct device *dev = &sdev->dev; |
368 | const struct scmi_handle *handle; |
369 | |
370 | handle = sdev->handle; |
371 | |
372 | if (!handle) |
373 | return -ENODEV; |
374 | |
375 | perf_ops = handle->devm_protocol_get(sdev, SCMI_PROTOCOL_PERF, &ph); |
376 | if (IS_ERR(ptr: perf_ops)) |
377 | return PTR_ERR(ptr: perf_ops); |
378 | |
379 | #ifdef CONFIG_COMMON_CLK |
380 | /* dummy clock provider as needed by OPP if clocks property is used */ |
381 | if (of_property_present(np: dev->of_node, propname: "#clock-cells" )) { |
382 | ret = devm_of_clk_add_hw_provider(dev, get: of_clk_hw_simple_get, NULL); |
383 | if (ret) |
384 | return dev_err_probe(dev, err: ret, fmt: "%s: registering clock provider failed\n" , __func__); |
385 | } |
386 | #endif |
387 | |
388 | ret = cpufreq_register_driver(driver_data: &scmi_cpufreq_driver); |
389 | if (ret) { |
390 | dev_err(dev, "%s: registering cpufreq failed, err: %d\n" , |
391 | __func__, ret); |
392 | } |
393 | |
394 | return ret; |
395 | } |
396 | |
397 | static void scmi_cpufreq_remove(struct scmi_device *sdev) |
398 | { |
399 | cpufreq_unregister_driver(driver_data: &scmi_cpufreq_driver); |
400 | } |
401 | |
402 | static const struct scmi_device_id scmi_id_table[] = { |
403 | { SCMI_PROTOCOL_PERF, "cpufreq" }, |
404 | { }, |
405 | }; |
406 | MODULE_DEVICE_TABLE(scmi, scmi_id_table); |
407 | |
408 | static struct scmi_driver scmi_cpufreq_drv = { |
409 | .name = "scmi-cpufreq" , |
410 | .probe = scmi_cpufreq_probe, |
411 | .remove = scmi_cpufreq_remove, |
412 | .id_table = scmi_id_table, |
413 | }; |
414 | module_scmi_driver(scmi_cpufreq_drv); |
415 | |
416 | MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>" ); |
417 | MODULE_DESCRIPTION("ARM SCMI CPUFreq interface driver" ); |
418 | MODULE_LICENSE("GPL v2" ); |
419 | |