1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Apple SoC CPU cluster performance state driver |
4 | * |
5 | * Copyright The Asahi Linux Contributors |
6 | * |
7 | * Based on scpi-cpufreq.c |
8 | */ |
9 | |
10 | #include <linux/bitfield.h> |
11 | #include <linux/bitops.h> |
12 | #include <linux/cpu.h> |
13 | #include <linux/cpufreq.h> |
14 | #include <linux/cpumask.h> |
15 | #include <linux/delay.h> |
16 | #include <linux/err.h> |
17 | #include <linux/io.h> |
18 | #include <linux/iopoll.h> |
19 | #include <linux/module.h> |
20 | #include <linux/of.h> |
21 | #include <linux/of_address.h> |
22 | #include <linux/pm_opp.h> |
23 | #include <linux/slab.h> |
24 | |
25 | #define APPLE_DVFS_CMD 0x20 |
26 | #define APPLE_DVFS_CMD_BUSY BIT(31) |
27 | #define APPLE_DVFS_CMD_SET BIT(25) |
28 | #define APPLE_DVFS_CMD_PS2 GENMASK(16, 12) |
29 | #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0) |
30 | |
31 | /* Same timebase as CPU counter (24MHz) */ |
32 | #define APPLE_DVFS_LAST_CHG_TIME 0x38 |
33 | |
34 | /* |
35 | * Apple ran out of bits and had to shift this in T8112... |
36 | */ |
37 | #define APPLE_DVFS_STATUS 0x50 |
38 | #define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4) |
39 | #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4 |
40 | #define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0) |
41 | #define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5) |
42 | #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5 |
43 | #define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0) |
44 | |
45 | /* |
46 | * Div is +1, base clock is 12MHz on existing SoCs. |
47 | * For documentation purposes. We use the OPP table to |
48 | * get the frequency. |
49 | */ |
50 | #define APPLE_DVFS_PLL_STATUS 0xc0 |
51 | #define APPLE_DVFS_PLL_FACTOR 0xc8 |
52 | #define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16) |
53 | #define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0) |
54 | |
55 | #define APPLE_DVFS_TRANSITION_TIMEOUT 100 |
56 | |
57 | struct apple_soc_cpufreq_info { |
58 | u64 max_pstate; |
59 | u64 cur_pstate_mask; |
60 | u64 cur_pstate_shift; |
61 | }; |
62 | |
63 | struct apple_cpu_priv { |
64 | struct device *cpu_dev; |
65 | void __iomem *reg_base; |
66 | const struct apple_soc_cpufreq_info *info; |
67 | }; |
68 | |
69 | static struct cpufreq_driver apple_soc_cpufreq_driver; |
70 | |
71 | static const struct apple_soc_cpufreq_info soc_t8103_info = { |
72 | .max_pstate = 15, |
73 | .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103, |
74 | .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103, |
75 | }; |
76 | |
77 | static const struct apple_soc_cpufreq_info soc_t8112_info = { |
78 | .max_pstate = 31, |
79 | .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112, |
80 | .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112, |
81 | }; |
82 | |
83 | static const struct apple_soc_cpufreq_info soc_default_info = { |
84 | .max_pstate = 15, |
85 | .cur_pstate_mask = 0, /* fallback */ |
86 | }; |
87 | |
88 | static const struct of_device_id apple_soc_cpufreq_of_match[] = { |
89 | { |
90 | .compatible = "apple,t8103-cluster-cpufreq" , |
91 | .data = &soc_t8103_info, |
92 | }, |
93 | { |
94 | .compatible = "apple,t8112-cluster-cpufreq" , |
95 | .data = &soc_t8112_info, |
96 | }, |
97 | { |
98 | .compatible = "apple,cluster-cpufreq" , |
99 | .data = &soc_default_info, |
100 | }, |
101 | {} |
102 | }; |
103 | |
104 | static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) |
105 | { |
106 | struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); |
107 | struct apple_cpu_priv *priv = policy->driver_data; |
108 | struct cpufreq_frequency_table *p; |
109 | unsigned int pstate; |
110 | |
111 | if (priv->info->cur_pstate_mask) { |
112 | u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS); |
113 | |
114 | pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift; |
115 | } else { |
116 | /* |
117 | * For the fallback case we might not know the layout of DVFS_STATUS, |
118 | * so just use the command register value (which ignores boost limitations). |
119 | */ |
120 | u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); |
121 | |
122 | pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); |
123 | } |
124 | |
125 | cpufreq_for_each_valid_entry(p, policy->freq_table) |
126 | if (p->driver_data == pstate) |
127 | return p->frequency; |
128 | |
129 | dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n" , |
130 | pstate); |
131 | return 0; |
132 | } |
133 | |
134 | static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, |
135 | unsigned int index) |
136 | { |
137 | struct apple_cpu_priv *priv = policy->driver_data; |
138 | unsigned int pstate = policy->freq_table[index].driver_data; |
139 | u64 reg; |
140 | |
141 | /* Fallback for newer SoCs */ |
142 | if (index > priv->info->max_pstate) |
143 | index = priv->info->max_pstate; |
144 | |
145 | if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, |
146 | !(reg & APPLE_DVFS_CMD_BUSY), 2, |
147 | APPLE_DVFS_TRANSITION_TIMEOUT)) { |
148 | return -EIO; |
149 | } |
150 | |
151 | reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2); |
152 | reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate); |
153 | reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate); |
154 | reg |= APPLE_DVFS_CMD_SET; |
155 | |
156 | writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); |
157 | |
158 | return 0; |
159 | } |
160 | |
161 | static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, |
162 | unsigned int target_freq) |
163 | { |
164 | if (apple_soc_cpufreq_set_target(policy, index: policy->cached_resolved_idx) < 0) |
165 | return 0; |
166 | |
167 | return policy->freq_table[policy->cached_resolved_idx].frequency; |
168 | } |
169 | |
170 | static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, |
171 | void __iomem **reg_base, |
172 | const struct apple_soc_cpufreq_info **info) |
173 | { |
174 | struct of_phandle_args args; |
175 | const struct of_device_id *match; |
176 | int ret = 0; |
177 | |
178 | ret = of_perf_domain_get_sharing_cpumask(pcpu: policy->cpu, list_name: "performance-domains" , |
179 | cell_name: "#performance-domain-cells" , |
180 | cpumask: policy->cpus, pargs: &args); |
181 | if (ret < 0) |
182 | return ret; |
183 | |
184 | match = of_match_node(matches: apple_soc_cpufreq_of_match, node: args.np); |
185 | of_node_put(node: args.np); |
186 | if (!match) |
187 | return -ENODEV; |
188 | |
189 | *info = match->data; |
190 | |
191 | *reg_base = of_iomap(node: args.np, index: 0); |
192 | if (!*reg_base) |
193 | return -ENOMEM; |
194 | |
195 | return 0; |
196 | } |
197 | |
198 | static struct freq_attr *apple_soc_cpufreq_hw_attr[] = { |
199 | &cpufreq_freq_attr_scaling_available_freqs, |
200 | NULL, /* Filled in below if boost is enabled */ |
201 | NULL, |
202 | }; |
203 | |
204 | static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) |
205 | { |
206 | int ret, i; |
207 | unsigned int transition_latency; |
208 | void __iomem *reg_base; |
209 | struct device *cpu_dev; |
210 | struct apple_cpu_priv *priv; |
211 | const struct apple_soc_cpufreq_info *info; |
212 | struct cpufreq_frequency_table *freq_table; |
213 | |
214 | cpu_dev = get_cpu_device(cpu: policy->cpu); |
215 | if (!cpu_dev) { |
216 | pr_err("failed to get cpu%d device\n" , policy->cpu); |
217 | return -ENODEV; |
218 | } |
219 | |
220 | ret = dev_pm_opp_of_add_table(dev: cpu_dev); |
221 | if (ret < 0) { |
222 | dev_err(cpu_dev, "%s: failed to add OPP table: %d\n" , __func__, ret); |
223 | return ret; |
224 | } |
225 | |
226 | ret = apple_soc_cpufreq_find_cluster(policy, reg_base: ®_base, info: &info); |
227 | if (ret) { |
228 | dev_err(cpu_dev, "%s: failed to get cluster info: %d\n" , __func__, ret); |
229 | return ret; |
230 | } |
231 | |
232 | ret = dev_pm_opp_set_sharing_cpus(cpu_dev, cpumask: policy->cpus); |
233 | if (ret) { |
234 | dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n" , __func__, ret); |
235 | goto out_iounmap; |
236 | } |
237 | |
238 | ret = dev_pm_opp_get_opp_count(dev: cpu_dev); |
239 | if (ret <= 0) { |
240 | dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n" ); |
241 | ret = -EPROBE_DEFER; |
242 | goto out_free_opp; |
243 | } |
244 | |
245 | priv = kzalloc(size: sizeof(*priv), GFP_KERNEL); |
246 | if (!priv) { |
247 | ret = -ENOMEM; |
248 | goto out_free_opp; |
249 | } |
250 | |
251 | ret = dev_pm_opp_init_cpufreq_table(dev: cpu_dev, table: &freq_table); |
252 | if (ret) { |
253 | dev_err(cpu_dev, "failed to init cpufreq table: %d\n" , ret); |
254 | goto out_free_priv; |
255 | } |
256 | |
257 | /* Get OPP levels (p-state indexes) and stash them in driver_data */ |
258 | for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { |
259 | unsigned long rate = freq_table[i].frequency * 1000 + 999; |
260 | struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(dev: cpu_dev, freq: &rate); |
261 | |
262 | if (IS_ERR(ptr: opp)) { |
263 | ret = PTR_ERR(ptr: opp); |
264 | goto out_free_cpufreq_table; |
265 | } |
266 | freq_table[i].driver_data = dev_pm_opp_get_level(opp); |
267 | dev_pm_opp_put(opp); |
268 | } |
269 | |
270 | priv->cpu_dev = cpu_dev; |
271 | priv->reg_base = reg_base; |
272 | priv->info = info; |
273 | policy->driver_data = priv; |
274 | policy->freq_table = freq_table; |
275 | |
276 | transition_latency = dev_pm_opp_get_max_transition_latency(dev: cpu_dev); |
277 | if (!transition_latency) |
278 | transition_latency = CPUFREQ_ETERNAL; |
279 | |
280 | policy->cpuinfo.transition_latency = transition_latency; |
281 | policy->dvfs_possible_from_any_cpu = true; |
282 | policy->fast_switch_possible = true; |
283 | policy->suspend_freq = freq_table[0].frequency; |
284 | |
285 | if (policy_has_boost_freq(policy)) { |
286 | ret = cpufreq_enable_boost_support(); |
287 | if (ret) { |
288 | dev_warn(cpu_dev, "failed to enable boost: %d\n" , ret); |
289 | } else { |
290 | apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; |
291 | apple_soc_cpufreq_driver.boost_enabled = true; |
292 | } |
293 | } |
294 | |
295 | return 0; |
296 | |
297 | out_free_cpufreq_table: |
298 | dev_pm_opp_free_cpufreq_table(dev: cpu_dev, table: &freq_table); |
299 | out_free_priv: |
300 | kfree(objp: priv); |
301 | out_free_opp: |
302 | dev_pm_opp_remove_all_dynamic(dev: cpu_dev); |
303 | out_iounmap: |
304 | iounmap(addr: reg_base); |
305 | return ret; |
306 | } |
307 | |
308 | static int apple_soc_cpufreq_exit(struct cpufreq_policy *policy) |
309 | { |
310 | struct apple_cpu_priv *priv = policy->driver_data; |
311 | |
312 | dev_pm_opp_free_cpufreq_table(dev: priv->cpu_dev, table: &policy->freq_table); |
313 | dev_pm_opp_remove_all_dynamic(dev: priv->cpu_dev); |
314 | iounmap(addr: priv->reg_base); |
315 | kfree(objp: priv); |
316 | |
317 | return 0; |
318 | } |
319 | |
320 | static struct cpufreq_driver apple_soc_cpufreq_driver = { |
321 | .name = "apple-cpufreq" , |
322 | .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | |
323 | CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, |
324 | .verify = cpufreq_generic_frequency_table_verify, |
325 | .get = apple_soc_cpufreq_get_rate, |
326 | .init = apple_soc_cpufreq_init, |
327 | .exit = apple_soc_cpufreq_exit, |
328 | .target_index = apple_soc_cpufreq_set_target, |
329 | .fast_switch = apple_soc_cpufreq_fast_switch, |
330 | .register_em = cpufreq_register_em_with_opp, |
331 | .attr = apple_soc_cpufreq_hw_attr, |
332 | .suspend = cpufreq_generic_suspend, |
333 | }; |
334 | |
335 | static int __init apple_soc_cpufreq_module_init(void) |
336 | { |
337 | if (!of_machine_is_compatible(compat: "apple,arm-platform" )) |
338 | return -ENODEV; |
339 | |
340 | return cpufreq_register_driver(driver_data: &apple_soc_cpufreq_driver); |
341 | } |
342 | module_init(apple_soc_cpufreq_module_init); |
343 | |
344 | static void __exit apple_soc_cpufreq_module_exit(void) |
345 | { |
346 | cpufreq_unregister_driver(driver_data: &apple_soc_cpufreq_driver); |
347 | } |
348 | module_exit(apple_soc_cpufreq_module_exit); |
349 | |
350 | MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match); |
351 | MODULE_AUTHOR("Hector Martin <marcan@marcan.st>" ); |
352 | MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver" ); |
353 | MODULE_LICENSE("GPL" ); |
354 | |