1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2018 Chen-Yu Tsai |
4 | * |
5 | * Chen-Yu Tsai <wens@csie.org> |
6 | * |
7 | * arch/arm/mach-sunxi/mc_smp.c |
8 | * |
9 | * Based on Allwinner code, arch/arm/mach-exynos/mcpm-exynos.c, and |
10 | * arch/arm/mach-hisi/platmcpm.c |
11 | * Cluster cache enable trampoline code adapted from MCPM framework |
12 | */ |
13 | |
14 | #include <linux/arm-cci.h> |
15 | #include <linux/cpu_pm.h> |
16 | #include <linux/delay.h> |
17 | #include <linux/io.h> |
18 | #include <linux/iopoll.h> |
19 | #include <linux/irqchip/arm-gic.h> |
20 | #include <linux/of.h> |
21 | #include <linux/of_address.h> |
22 | #include <linux/smp.h> |
23 | |
24 | #include <asm/cacheflush.h> |
25 | #include <asm/cp15.h> |
26 | #include <asm/cputype.h> |
27 | #include <asm/idmap.h> |
28 | #include <asm/smp_plat.h> |
29 | #include <asm/suspend.h> |
30 | |
31 | #define SUNXI_CPUS_PER_CLUSTER 4 |
32 | #define SUNXI_NR_CLUSTERS 2 |
33 | |
34 | #define POLL_USEC 100 |
35 | #define TIMEOUT_USEC 100000 |
36 | |
37 | #define CPUCFG_CX_CTRL_REG0(c) (0x10 * (c)) |
38 | #define CPUCFG_CX_CTRL_REG0_L1_RST_DISABLE(n) BIT(n) |
39 | #define CPUCFG_CX_CTRL_REG0_L1_RST_DISABLE_ALL 0xf |
40 | #define CPUCFG_CX_CTRL_REG0_L2_RST_DISABLE_A7 BIT(4) |
41 | #define CPUCFG_CX_CTRL_REG0_L2_RST_DISABLE_A15 BIT(0) |
42 | #define CPUCFG_CX_CTRL_REG1(c) (0x10 * (c) + 0x4) |
43 | #define CPUCFG_CX_CTRL_REG1_ACINACTM BIT(0) |
44 | #define CPUCFG_CX_STATUS(c) (0x30 + 0x4 * (c)) |
45 | #define CPUCFG_CX_STATUS_STANDBYWFI(n) BIT(16 + (n)) |
46 | #define CPUCFG_CX_STATUS_STANDBYWFIL2 BIT(0) |
47 | #define CPUCFG_CX_RST_CTRL(c) (0x80 + 0x4 * (c)) |
48 | #define CPUCFG_CX_RST_CTRL_DBG_SOC_RST BIT(24) |
49 | #define CPUCFG_CX_RST_CTRL_ETM_RST(n) BIT(20 + (n)) |
50 | #define CPUCFG_CX_RST_CTRL_ETM_RST_ALL (0xf << 20) |
51 | #define CPUCFG_CX_RST_CTRL_DBG_RST(n) BIT(16 + (n)) |
52 | #define CPUCFG_CX_RST_CTRL_DBG_RST_ALL (0xf << 16) |
53 | #define CPUCFG_CX_RST_CTRL_H_RST BIT(12) |
54 | #define CPUCFG_CX_RST_CTRL_L2_RST BIT(8) |
55 | #define CPUCFG_CX_RST_CTRL_CX_RST(n) BIT(4 + (n)) |
56 | #define CPUCFG_CX_RST_CTRL_CORE_RST(n) BIT(n) |
57 | #define CPUCFG_CX_RST_CTRL_CORE_RST_ALL (0xf << 0) |
58 | |
59 | #define PRCM_CPU_PO_RST_CTRL(c) (0x4 + 0x4 * (c)) |
60 | #define PRCM_CPU_PO_RST_CTRL_CORE(n) BIT(n) |
61 | #define PRCM_CPU_PO_RST_CTRL_CORE_ALL 0xf |
62 | #define PRCM_PWROFF_GATING_REG(c) (0x100 + 0x4 * (c)) |
63 | /* The power off register for clusters are different from a80 and a83t */ |
64 | #define PRCM_PWROFF_GATING_REG_CLUSTER_SUN8I BIT(0) |
65 | #define PRCM_PWROFF_GATING_REG_CLUSTER_SUN9I BIT(4) |
66 | #define PRCM_PWROFF_GATING_REG_CORE(n) BIT(n) |
67 | #define PRCM_PWR_SWITCH_REG(c, cpu) (0x140 + 0x10 * (c) + 0x4 * (cpu)) |
68 | #define PRCM_CPU_SOFT_ENTRY_REG 0x164 |
69 | |
70 | /* R_CPUCFG registers, specific to sun8i-a83t */ |
71 | #define R_CPUCFG_CLUSTER_PO_RST_CTRL(c) (0x30 + (c) * 0x4) |
72 | #define R_CPUCFG_CLUSTER_PO_RST_CTRL_CORE(n) BIT(n) |
73 | #define R_CPUCFG_CPU_SOFT_ENTRY_REG 0x01a4 |
74 | |
75 | #define CPU0_SUPPORT_HOTPLUG_MAGIC0 0xFA50392F |
76 | #define CPU0_SUPPORT_HOTPLUG_MAGIC1 0x790DCA3A |
77 | |
78 | static void __iomem *cpucfg_base; |
79 | static void __iomem *prcm_base; |
80 | static void __iomem *sram_b_smp_base; |
81 | static void __iomem *r_cpucfg_base; |
82 | |
83 | extern void sunxi_mc_smp_secondary_startup(void); |
84 | extern void sunxi_mc_smp_resume(void); |
85 | static bool is_a83t; |
86 | |
87 | static bool sunxi_core_is_cortex_a15(unsigned int core, unsigned int cluster) |
88 | { |
89 | struct device_node *node; |
90 | int cpu = cluster * SUNXI_CPUS_PER_CLUSTER + core; |
91 | bool is_compatible; |
92 | |
93 | node = of_cpu_device_node_get(cpu); |
94 | |
95 | /* In case of_cpu_device_node_get fails */ |
96 | if (!node) |
97 | node = of_get_cpu_node(cpu, NULL); |
98 | |
99 | if (!node) { |
100 | /* |
101 | * There's no point in returning an error, since we |
102 | * would be mid way in a core or cluster power sequence. |
103 | */ |
104 | pr_err("%s: Couldn't get CPU cluster %u core %u device node\n" , |
105 | __func__, cluster, core); |
106 | |
107 | return false; |
108 | } |
109 | |
110 | is_compatible = of_device_is_compatible(device: node, "arm,cortex-a15" ); |
111 | of_node_put(node); |
112 | return is_compatible; |
113 | } |
114 | |
115 | static int sunxi_cpu_power_switch_set(unsigned int cpu, unsigned int cluster, |
116 | bool enable) |
117 | { |
118 | u32 reg; |
119 | |
120 | /* control sequence from Allwinner A80 user manual v1.2 PRCM section */ |
121 | reg = readl(addr: prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); |
122 | if (enable) { |
123 | if (reg == 0x00) { |
124 | pr_debug("power clamp for cluster %u cpu %u already open\n" , |
125 | cluster, cpu); |
126 | return 0; |
127 | } |
128 | |
129 | writel(val: 0xff, addr: prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); |
130 | udelay(10); |
131 | writel(val: 0xfe, addr: prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); |
132 | udelay(10); |
133 | writel(val: 0xf8, addr: prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); |
134 | udelay(10); |
135 | writel(val: 0xf0, addr: prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); |
136 | udelay(10); |
137 | writel(val: 0x00, addr: prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); |
138 | udelay(10); |
139 | } else { |
140 | writel(val: 0xff, addr: prcm_base + PRCM_PWR_SWITCH_REG(cluster, cpu)); |
141 | udelay(10); |
142 | } |
143 | |
144 | return 0; |
145 | } |
146 | |
147 | static void sunxi_cpu0_hotplug_support_set(bool enable) |
148 | { |
149 | if (enable) { |
150 | writel(CPU0_SUPPORT_HOTPLUG_MAGIC0, addr: sram_b_smp_base); |
151 | writel(CPU0_SUPPORT_HOTPLUG_MAGIC1, addr: sram_b_smp_base + 0x4); |
152 | } else { |
153 | writel(val: 0x0, addr: sram_b_smp_base); |
154 | writel(val: 0x0, addr: sram_b_smp_base + 0x4); |
155 | } |
156 | } |
157 | |
158 | static int sunxi_cpu_powerup(unsigned int cpu, unsigned int cluster) |
159 | { |
160 | u32 reg; |
161 | |
162 | pr_debug("%s: cluster %u cpu %u\n" , __func__, cluster, cpu); |
163 | if (cpu >= SUNXI_CPUS_PER_CLUSTER || cluster >= SUNXI_NR_CLUSTERS) |
164 | return -EINVAL; |
165 | |
166 | /* Set hotplug support magic flags for cpu0 */ |
167 | if (cluster == 0 && cpu == 0) |
168 | sunxi_cpu0_hotplug_support_set(enable: true); |
169 | |
170 | /* assert processor power-on reset */ |
171 | reg = readl(addr: prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); |
172 | reg &= ~PRCM_CPU_PO_RST_CTRL_CORE(cpu); |
173 | writel(val: reg, addr: prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); |
174 | |
175 | if (is_a83t) { |
176 | /* assert cpu power-on reset */ |
177 | reg = readl(addr: r_cpucfg_base + |
178 | R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster)); |
179 | reg &= ~(R_CPUCFG_CLUSTER_PO_RST_CTRL_CORE(cpu)); |
180 | writel(val: reg, addr: r_cpucfg_base + |
181 | R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster)); |
182 | udelay(10); |
183 | } |
184 | |
185 | /* Cortex-A7: hold L1 reset disable signal low */ |
186 | if (!sunxi_core_is_cortex_a15(core: cpu, cluster)) { |
187 | reg = readl(addr: cpucfg_base + CPUCFG_CX_CTRL_REG0(cluster)); |
188 | reg &= ~CPUCFG_CX_CTRL_REG0_L1_RST_DISABLE(cpu); |
189 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_CTRL_REG0(cluster)); |
190 | } |
191 | |
192 | /* assert processor related resets */ |
193 | reg = readl(addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
194 | reg &= ~CPUCFG_CX_RST_CTRL_DBG_RST(cpu); |
195 | |
196 | /* |
197 | * Allwinner code also asserts resets for NEON on A15. According |
198 | * to ARM manuals, asserting power-on reset is sufficient. |
199 | */ |
200 | if (!sunxi_core_is_cortex_a15(core: cpu, cluster)) |
201 | reg &= ~CPUCFG_CX_RST_CTRL_ETM_RST(cpu); |
202 | |
203 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
204 | |
205 | /* open power switch */ |
206 | sunxi_cpu_power_switch_set(cpu, cluster, enable: true); |
207 | |
208 | /* Handle A83T bit swap */ |
209 | if (is_a83t) { |
210 | if (cpu == 0) |
211 | cpu = 4; |
212 | } |
213 | |
214 | /* clear processor power gate */ |
215 | reg = readl(addr: prcm_base + PRCM_PWROFF_GATING_REG(cluster)); |
216 | reg &= ~PRCM_PWROFF_GATING_REG_CORE(cpu); |
217 | writel(val: reg, addr: prcm_base + PRCM_PWROFF_GATING_REG(cluster)); |
218 | udelay(20); |
219 | |
220 | /* Handle A83T bit swap */ |
221 | if (is_a83t) { |
222 | if (cpu == 4) |
223 | cpu = 0; |
224 | } |
225 | |
226 | /* de-assert processor power-on reset */ |
227 | reg = readl(addr: prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); |
228 | reg |= PRCM_CPU_PO_RST_CTRL_CORE(cpu); |
229 | writel(val: reg, addr: prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); |
230 | |
231 | if (is_a83t) { |
232 | reg = readl(addr: r_cpucfg_base + |
233 | R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster)); |
234 | reg |= R_CPUCFG_CLUSTER_PO_RST_CTRL_CORE(cpu); |
235 | writel(val: reg, addr: r_cpucfg_base + |
236 | R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster)); |
237 | udelay(10); |
238 | } |
239 | |
240 | /* de-assert all processor resets */ |
241 | reg = readl(addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
242 | reg |= CPUCFG_CX_RST_CTRL_DBG_RST(cpu); |
243 | reg |= CPUCFG_CX_RST_CTRL_CORE_RST(cpu); |
244 | if (!sunxi_core_is_cortex_a15(core: cpu, cluster)) |
245 | reg |= CPUCFG_CX_RST_CTRL_ETM_RST(cpu); |
246 | else |
247 | reg |= CPUCFG_CX_RST_CTRL_CX_RST(cpu); /* NEON */ |
248 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
249 | |
250 | return 0; |
251 | } |
252 | |
253 | static int sunxi_cluster_powerup(unsigned int cluster) |
254 | { |
255 | u32 reg; |
256 | |
257 | pr_debug("%s: cluster %u\n" , __func__, cluster); |
258 | if (cluster >= SUNXI_NR_CLUSTERS) |
259 | return -EINVAL; |
260 | |
261 | /* For A83T, assert cluster cores resets */ |
262 | if (is_a83t) { |
263 | reg = readl(addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
264 | reg &= ~CPUCFG_CX_RST_CTRL_CORE_RST_ALL; /* Core Reset */ |
265 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
266 | udelay(10); |
267 | } |
268 | |
269 | /* assert ACINACTM */ |
270 | reg = readl(addr: cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); |
271 | reg |= CPUCFG_CX_CTRL_REG1_ACINACTM; |
272 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); |
273 | |
274 | /* assert cluster processor power-on resets */ |
275 | reg = readl(addr: prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); |
276 | reg &= ~PRCM_CPU_PO_RST_CTRL_CORE_ALL; |
277 | writel(val: reg, addr: prcm_base + PRCM_CPU_PO_RST_CTRL(cluster)); |
278 | |
279 | /* assert cluster cores resets */ |
280 | if (is_a83t) { |
281 | reg = readl(addr: r_cpucfg_base + |
282 | R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster)); |
283 | reg &= ~CPUCFG_CX_RST_CTRL_CORE_RST_ALL; |
284 | writel(val: reg, addr: r_cpucfg_base + |
285 | R_CPUCFG_CLUSTER_PO_RST_CTRL(cluster)); |
286 | udelay(10); |
287 | } |
288 | |
289 | /* assert cluster resets */ |
290 | reg = readl(addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
291 | reg &= ~CPUCFG_CX_RST_CTRL_DBG_SOC_RST; |
292 | reg &= ~CPUCFG_CX_RST_CTRL_DBG_RST_ALL; |
293 | reg &= ~CPUCFG_CX_RST_CTRL_H_RST; |
294 | reg &= ~CPUCFG_CX_RST_CTRL_L2_RST; |
295 | |
296 | /* |
297 | * Allwinner code also asserts resets for NEON on A15. According |
298 | * to ARM manuals, asserting power-on reset is sufficient. |
299 | */ |
300 | if (!sunxi_core_is_cortex_a15(core: 0, cluster)) |
301 | reg &= ~CPUCFG_CX_RST_CTRL_ETM_RST_ALL; |
302 | |
303 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
304 | |
305 | /* hold L1/L2 reset disable signals low */ |
306 | reg = readl(addr: cpucfg_base + CPUCFG_CX_CTRL_REG0(cluster)); |
307 | if (sunxi_core_is_cortex_a15(core: 0, cluster)) { |
308 | /* Cortex-A15: hold L2RSTDISABLE low */ |
309 | reg &= ~CPUCFG_CX_CTRL_REG0_L2_RST_DISABLE_A15; |
310 | } else { |
311 | /* Cortex-A7: hold L1RSTDISABLE and L2RSTDISABLE low */ |
312 | reg &= ~CPUCFG_CX_CTRL_REG0_L1_RST_DISABLE_ALL; |
313 | reg &= ~CPUCFG_CX_CTRL_REG0_L2_RST_DISABLE_A7; |
314 | } |
315 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_CTRL_REG0(cluster)); |
316 | |
317 | /* clear cluster power gate */ |
318 | reg = readl(addr: prcm_base + PRCM_PWROFF_GATING_REG(cluster)); |
319 | if (is_a83t) |
320 | reg &= ~PRCM_PWROFF_GATING_REG_CLUSTER_SUN8I; |
321 | else |
322 | reg &= ~PRCM_PWROFF_GATING_REG_CLUSTER_SUN9I; |
323 | writel(val: reg, addr: prcm_base + PRCM_PWROFF_GATING_REG(cluster)); |
324 | udelay(20); |
325 | |
326 | /* de-assert cluster resets */ |
327 | reg = readl(addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
328 | reg |= CPUCFG_CX_RST_CTRL_DBG_SOC_RST; |
329 | reg |= CPUCFG_CX_RST_CTRL_H_RST; |
330 | reg |= CPUCFG_CX_RST_CTRL_L2_RST; |
331 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
332 | |
333 | /* de-assert ACINACTM */ |
334 | reg = readl(addr: cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); |
335 | reg &= ~CPUCFG_CX_CTRL_REG1_ACINACTM; |
336 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); |
337 | |
338 | return 0; |
339 | } |
340 | |
341 | /* |
342 | * This bit is shared between the initial nocache_trampoline call to |
343 | * enable CCI-400 and proper cluster cache disable before power down. |
344 | */ |
345 | static void sunxi_cluster_cache_disable_without_axi(void) |
346 | { |
347 | if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) { |
348 | /* |
349 | * On the Cortex-A15 we need to disable |
350 | * L2 prefetching before flushing the cache. |
351 | */ |
352 | asm volatile( |
353 | "mcr p15, 1, %0, c15, c0, 3\n" |
354 | "isb\n" |
355 | "dsb" |
356 | : : "r" (0x400)); |
357 | } |
358 | |
359 | /* Flush all cache levels for this cluster. */ |
360 | v7_exit_coherency_flush(all); |
361 | |
362 | /* |
363 | * Disable cluster-level coherency by masking |
364 | * incoming snoops and DVM messages: |
365 | */ |
366 | cci_disable_port_by_cpu(mpidr: read_cpuid_mpidr()); |
367 | } |
368 | |
369 | static int sunxi_mc_smp_cpu_table[SUNXI_NR_CLUSTERS][SUNXI_CPUS_PER_CLUSTER]; |
370 | int sunxi_mc_smp_first_comer; |
371 | |
372 | static DEFINE_SPINLOCK(boot_lock); |
373 | |
374 | static bool sunxi_mc_smp_cluster_is_down(unsigned int cluster) |
375 | { |
376 | int i; |
377 | |
378 | for (i = 0; i < SUNXI_CPUS_PER_CLUSTER; i++) |
379 | if (sunxi_mc_smp_cpu_table[cluster][i]) |
380 | return false; |
381 | return true; |
382 | } |
383 | |
384 | static void sunxi_mc_smp_secondary_init(unsigned int cpu) |
385 | { |
386 | /* Clear hotplug support magic flags for cpu0 */ |
387 | if (cpu == 0) |
388 | sunxi_cpu0_hotplug_support_set(enable: false); |
389 | } |
390 | |
391 | static int sunxi_mc_smp_boot_secondary(unsigned int l_cpu, struct task_struct *idle) |
392 | { |
393 | unsigned int mpidr, cpu, cluster; |
394 | |
395 | mpidr = cpu_logical_map(l_cpu); |
396 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
397 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
398 | |
399 | if (!cpucfg_base) |
400 | return -ENODEV; |
401 | if (cluster >= SUNXI_NR_CLUSTERS || cpu >= SUNXI_CPUS_PER_CLUSTER) |
402 | return -EINVAL; |
403 | |
404 | spin_lock_irq(lock: &boot_lock); |
405 | |
406 | if (sunxi_mc_smp_cpu_table[cluster][cpu]) |
407 | goto out; |
408 | |
409 | if (sunxi_mc_smp_cluster_is_down(cluster)) { |
410 | sunxi_mc_smp_first_comer = true; |
411 | sunxi_cluster_powerup(cluster); |
412 | } else { |
413 | sunxi_mc_smp_first_comer = false; |
414 | } |
415 | |
416 | /* This is read by incoming CPUs with their cache and MMU disabled */ |
417 | sync_cache_w(&sunxi_mc_smp_first_comer); |
418 | sunxi_cpu_powerup(cpu, cluster); |
419 | |
420 | out: |
421 | sunxi_mc_smp_cpu_table[cluster][cpu]++; |
422 | spin_unlock_irq(lock: &boot_lock); |
423 | |
424 | return 0; |
425 | } |
426 | |
427 | #ifdef CONFIG_HOTPLUG_CPU |
428 | static void sunxi_cluster_cache_disable(void) |
429 | { |
430 | unsigned int cluster = MPIDR_AFFINITY_LEVEL(read_cpuid_mpidr(), 1); |
431 | u32 reg; |
432 | |
433 | pr_debug("%s: cluster %u\n" , __func__, cluster); |
434 | |
435 | sunxi_cluster_cache_disable_without_axi(); |
436 | |
437 | /* last man standing, assert ACINACTM */ |
438 | reg = readl(addr: cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); |
439 | reg |= CPUCFG_CX_CTRL_REG1_ACINACTM; |
440 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_CTRL_REG1(cluster)); |
441 | } |
442 | |
443 | static void sunxi_mc_smp_cpu_die(unsigned int l_cpu) |
444 | { |
445 | unsigned int mpidr, cpu, cluster; |
446 | bool last_man; |
447 | |
448 | mpidr = cpu_logical_map(l_cpu); |
449 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
450 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
451 | pr_debug("%s: cluster %u cpu %u\n" , __func__, cluster, cpu); |
452 | |
453 | spin_lock(lock: &boot_lock); |
454 | sunxi_mc_smp_cpu_table[cluster][cpu]--; |
455 | if (sunxi_mc_smp_cpu_table[cluster][cpu] == 1) { |
456 | /* A power_up request went ahead of us. */ |
457 | pr_debug("%s: aborting due to a power up request\n" , |
458 | __func__); |
459 | spin_unlock(lock: &boot_lock); |
460 | return; |
461 | } else if (sunxi_mc_smp_cpu_table[cluster][cpu] > 1) { |
462 | pr_err("Cluster %d CPU%d boots multiple times\n" , |
463 | cluster, cpu); |
464 | BUG(); |
465 | } |
466 | |
467 | last_man = sunxi_mc_smp_cluster_is_down(cluster); |
468 | spin_unlock(lock: &boot_lock); |
469 | |
470 | gic_cpu_if_down(gic_nr: 0); |
471 | if (last_man) |
472 | sunxi_cluster_cache_disable(); |
473 | else |
474 | v7_exit_coherency_flush(louis); |
475 | |
476 | for (;;) |
477 | wfi(); |
478 | } |
479 | |
480 | static int sunxi_cpu_powerdown(unsigned int cpu, unsigned int cluster) |
481 | { |
482 | u32 reg; |
483 | int gating_bit = cpu; |
484 | |
485 | pr_debug("%s: cluster %u cpu %u\n" , __func__, cluster, cpu); |
486 | if (cpu >= SUNXI_CPUS_PER_CLUSTER || cluster >= SUNXI_NR_CLUSTERS) |
487 | return -EINVAL; |
488 | |
489 | if (is_a83t && cpu == 0) |
490 | gating_bit = 4; |
491 | |
492 | /* gate processor power */ |
493 | reg = readl(addr: prcm_base + PRCM_PWROFF_GATING_REG(cluster)); |
494 | reg |= PRCM_PWROFF_GATING_REG_CORE(gating_bit); |
495 | writel(val: reg, addr: prcm_base + PRCM_PWROFF_GATING_REG(cluster)); |
496 | udelay(20); |
497 | |
498 | /* close power switch */ |
499 | sunxi_cpu_power_switch_set(cpu, cluster, enable: false); |
500 | |
501 | return 0; |
502 | } |
503 | |
504 | static int sunxi_cluster_powerdown(unsigned int cluster) |
505 | { |
506 | u32 reg; |
507 | |
508 | pr_debug("%s: cluster %u\n" , __func__, cluster); |
509 | if (cluster >= SUNXI_NR_CLUSTERS) |
510 | return -EINVAL; |
511 | |
512 | /* assert cluster resets or system will hang */ |
513 | pr_debug("%s: assert cluster reset\n" , __func__); |
514 | reg = readl(addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
515 | reg &= ~CPUCFG_CX_RST_CTRL_DBG_SOC_RST; |
516 | reg &= ~CPUCFG_CX_RST_CTRL_H_RST; |
517 | reg &= ~CPUCFG_CX_RST_CTRL_L2_RST; |
518 | writel(val: reg, addr: cpucfg_base + CPUCFG_CX_RST_CTRL(cluster)); |
519 | |
520 | /* gate cluster power */ |
521 | pr_debug("%s: gate cluster power\n" , __func__); |
522 | reg = readl(addr: prcm_base + PRCM_PWROFF_GATING_REG(cluster)); |
523 | if (is_a83t) |
524 | reg |= PRCM_PWROFF_GATING_REG_CLUSTER_SUN8I; |
525 | else |
526 | reg |= PRCM_PWROFF_GATING_REG_CLUSTER_SUN9I; |
527 | writel(val: reg, addr: prcm_base + PRCM_PWROFF_GATING_REG(cluster)); |
528 | udelay(20); |
529 | |
530 | return 0; |
531 | } |
532 | |
533 | static int sunxi_mc_smp_cpu_kill(unsigned int l_cpu) |
534 | { |
535 | unsigned int mpidr, cpu, cluster; |
536 | unsigned int tries, count; |
537 | int ret = 0; |
538 | u32 reg; |
539 | |
540 | mpidr = cpu_logical_map(l_cpu); |
541 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
542 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
543 | |
544 | /* This should never happen */ |
545 | if (WARN_ON(cluster >= SUNXI_NR_CLUSTERS || |
546 | cpu >= SUNXI_CPUS_PER_CLUSTER)) |
547 | return 0; |
548 | |
549 | /* wait for CPU core to die and enter WFI */ |
550 | count = TIMEOUT_USEC / POLL_USEC; |
551 | spin_lock_irq(lock: &boot_lock); |
552 | for (tries = 0; tries < count; tries++) { |
553 | spin_unlock_irq(lock: &boot_lock); |
554 | usleep_range(POLL_USEC / 2, POLL_USEC); |
555 | spin_lock_irq(lock: &boot_lock); |
556 | |
557 | /* |
558 | * If the user turns off a bunch of cores at the same |
559 | * time, the kernel might call cpu_kill before some of |
560 | * them are ready. This is because boot_lock serializes |
561 | * both cpu_die and cpu_kill callbacks. Either one could |
562 | * run first. We should wait for cpu_die to complete. |
563 | */ |
564 | if (sunxi_mc_smp_cpu_table[cluster][cpu]) |
565 | continue; |
566 | |
567 | reg = readl(addr: cpucfg_base + CPUCFG_CX_STATUS(cluster)); |
568 | if (reg & CPUCFG_CX_STATUS_STANDBYWFI(cpu)) |
569 | break; |
570 | } |
571 | |
572 | if (tries >= count) { |
573 | ret = ETIMEDOUT; |
574 | goto out; |
575 | } |
576 | |
577 | /* power down CPU core */ |
578 | sunxi_cpu_powerdown(cpu, cluster); |
579 | |
580 | if (!sunxi_mc_smp_cluster_is_down(cluster)) |
581 | goto out; |
582 | |
583 | /* wait for cluster L2 WFI */ |
584 | ret = readl_poll_timeout(cpucfg_base + CPUCFG_CX_STATUS(cluster), reg, |
585 | reg & CPUCFG_CX_STATUS_STANDBYWFIL2, |
586 | POLL_USEC, TIMEOUT_USEC); |
587 | if (ret) { |
588 | /* |
589 | * Ignore timeout on the cluster. Leaving the cluster on |
590 | * will not affect system execution, just use a bit more |
591 | * power. But returning an error here will only confuse |
592 | * the user as the CPU has already been shutdown. |
593 | */ |
594 | ret = 0; |
595 | goto out; |
596 | } |
597 | |
598 | /* Power down cluster */ |
599 | sunxi_cluster_powerdown(cluster); |
600 | |
601 | out: |
602 | spin_unlock_irq(lock: &boot_lock); |
603 | pr_debug("%s: cluster %u cpu %u powerdown: %d\n" , |
604 | __func__, cluster, cpu, ret); |
605 | return !ret; |
606 | } |
607 | |
608 | static bool sunxi_mc_smp_cpu_can_disable(unsigned int cpu) |
609 | { |
610 | /* CPU0 hotplug not handled for sun8i-a83t */ |
611 | if (is_a83t) |
612 | if (cpu == 0) |
613 | return false; |
614 | return true; |
615 | } |
616 | #endif |
617 | |
618 | static const struct smp_operations sunxi_mc_smp_smp_ops __initconst = { |
619 | .smp_secondary_init = sunxi_mc_smp_secondary_init, |
620 | .smp_boot_secondary = sunxi_mc_smp_boot_secondary, |
621 | #ifdef CONFIG_HOTPLUG_CPU |
622 | .cpu_die = sunxi_mc_smp_cpu_die, |
623 | .cpu_kill = sunxi_mc_smp_cpu_kill, |
624 | .cpu_can_disable = sunxi_mc_smp_cpu_can_disable, |
625 | #endif |
626 | }; |
627 | |
628 | static bool __init sunxi_mc_smp_cpu_table_init(void) |
629 | { |
630 | unsigned int mpidr, cpu, cluster; |
631 | |
632 | mpidr = read_cpuid_mpidr(); |
633 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); |
634 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); |
635 | |
636 | if (cluster >= SUNXI_NR_CLUSTERS || cpu >= SUNXI_CPUS_PER_CLUSTER) { |
637 | pr_err("%s: boot CPU is out of bounds!\n" , __func__); |
638 | return false; |
639 | } |
640 | sunxi_mc_smp_cpu_table[cluster][cpu] = 1; |
641 | return true; |
642 | } |
643 | |
644 | /* |
645 | * Adapted from arch/arm/common/mc_smp_entry.c |
646 | * |
647 | * We need the trampoline code to enable CCI-400 on the first cluster |
648 | */ |
649 | typedef typeof(cpu_reset) phys_reset_t; |
650 | |
651 | static int __init nocache_trampoline(unsigned long __unused) |
652 | { |
653 | phys_reset_t phys_reset; |
654 | |
655 | setup_mm_for_reboot(); |
656 | sunxi_cluster_cache_disable_without_axi(); |
657 | |
658 | phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset); |
659 | phys_reset(__pa_symbol(sunxi_mc_smp_resume), false); |
660 | BUG(); |
661 | } |
662 | |
663 | static int __init sunxi_mc_smp_loopback(void) |
664 | { |
665 | int ret; |
666 | |
667 | /* |
668 | * We're going to soft-restart the current CPU through the |
669 | * low-level MCPM code by leveraging the suspend/resume |
670 | * infrastructure. Let's play it safe by using cpu_pm_enter() |
671 | * in case the CPU init code path resets the VFP or similar. |
672 | */ |
673 | sunxi_mc_smp_first_comer = true; |
674 | local_irq_disable(); |
675 | local_fiq_disable(); |
676 | ret = cpu_pm_enter(); |
677 | if (!ret) { |
678 | ret = cpu_suspend(0, nocache_trampoline); |
679 | cpu_pm_exit(); |
680 | } |
681 | local_fiq_enable(); |
682 | local_irq_enable(); |
683 | sunxi_mc_smp_first_comer = false; |
684 | |
685 | return ret; |
686 | } |
687 | |
688 | /* |
689 | * This holds any device nodes that we requested resources for, |
690 | * so that we may easily release resources in the error path. |
691 | */ |
692 | struct sunxi_mc_smp_nodes { |
693 | struct device_node *prcm_node; |
694 | struct device_node *cpucfg_node; |
695 | struct device_node *sram_node; |
696 | struct device_node *r_cpucfg_node; |
697 | }; |
698 | |
699 | /* This structure holds SoC-specific bits tied to an enable-method string. */ |
700 | struct sunxi_mc_smp_data { |
701 | const char *enable_method; |
702 | int (*get_smp_nodes)(struct sunxi_mc_smp_nodes *nodes); |
703 | bool is_a83t; |
704 | }; |
705 | |
706 | static void __init sunxi_mc_smp_put_nodes(struct sunxi_mc_smp_nodes *nodes) |
707 | { |
708 | of_node_put(node: nodes->prcm_node); |
709 | of_node_put(node: nodes->cpucfg_node); |
710 | of_node_put(node: nodes->sram_node); |
711 | of_node_put(node: nodes->r_cpucfg_node); |
712 | memset(nodes, 0, sizeof(*nodes)); |
713 | } |
714 | |
715 | static int __init sun9i_a80_get_smp_nodes(struct sunxi_mc_smp_nodes *nodes) |
716 | { |
717 | nodes->prcm_node = of_find_compatible_node(NULL, NULL, |
718 | compat: "allwinner,sun9i-a80-prcm" ); |
719 | if (!nodes->prcm_node) { |
720 | pr_err("%s: PRCM not available\n" , __func__); |
721 | return -ENODEV; |
722 | } |
723 | |
724 | nodes->cpucfg_node = of_find_compatible_node(NULL, NULL, |
725 | compat: "allwinner,sun9i-a80-cpucfg" ); |
726 | if (!nodes->cpucfg_node) { |
727 | pr_err("%s: CPUCFG not available\n" , __func__); |
728 | return -ENODEV; |
729 | } |
730 | |
731 | nodes->sram_node = of_find_compatible_node(NULL, NULL, |
732 | compat: "allwinner,sun9i-a80-smp-sram" ); |
733 | if (!nodes->sram_node) { |
734 | pr_err("%s: Secure SRAM not available\n" , __func__); |
735 | return -ENODEV; |
736 | } |
737 | |
738 | return 0; |
739 | } |
740 | |
741 | static int __init sun8i_a83t_get_smp_nodes(struct sunxi_mc_smp_nodes *nodes) |
742 | { |
743 | nodes->prcm_node = of_find_compatible_node(NULL, NULL, |
744 | compat: "allwinner,sun8i-a83t-r-ccu" ); |
745 | if (!nodes->prcm_node) { |
746 | pr_err("%s: PRCM not available\n" , __func__); |
747 | return -ENODEV; |
748 | } |
749 | |
750 | nodes->cpucfg_node = of_find_compatible_node(NULL, NULL, |
751 | compat: "allwinner,sun8i-a83t-cpucfg" ); |
752 | if (!nodes->cpucfg_node) { |
753 | pr_err("%s: CPUCFG not available\n" , __func__); |
754 | return -ENODEV; |
755 | } |
756 | |
757 | nodes->r_cpucfg_node = of_find_compatible_node(NULL, NULL, |
758 | compat: "allwinner,sun8i-a83t-r-cpucfg" ); |
759 | if (!nodes->r_cpucfg_node) { |
760 | pr_err("%s: RCPUCFG not available\n" , __func__); |
761 | return -ENODEV; |
762 | } |
763 | |
764 | return 0; |
765 | } |
766 | |
767 | static const struct sunxi_mc_smp_data sunxi_mc_smp_data[] __initconst = { |
768 | { |
769 | .enable_method = "allwinner,sun9i-a80-smp" , |
770 | .get_smp_nodes = sun9i_a80_get_smp_nodes, |
771 | }, |
772 | { |
773 | .enable_method = "allwinner,sun8i-a83t-smp" , |
774 | .get_smp_nodes = sun8i_a83t_get_smp_nodes, |
775 | .is_a83t = true, |
776 | }, |
777 | }; |
778 | |
779 | static int __init sunxi_mc_smp_init(void) |
780 | { |
781 | struct sunxi_mc_smp_nodes nodes = { 0 }; |
782 | struct device_node *node; |
783 | struct resource res; |
784 | void __iomem *addr; |
785 | int i, ret; |
786 | |
787 | /* |
788 | * Don't bother checking the "cpus" node, as an enable-method |
789 | * property in that node is undocumented. |
790 | */ |
791 | node = of_cpu_device_node_get(cpu: 0); |
792 | if (!node) |
793 | return -ENODEV; |
794 | |
795 | /* |
796 | * We can't actually use the enable-method magic in the kernel. |
797 | * Our loopback / trampoline code uses the CPU suspend framework, |
798 | * which requires the identity mapping be available. It would not |
799 | * yet be available if we used the .init_cpus or .prepare_cpus |
800 | * callbacks in smp_operations, which we would use if we were to |
801 | * use CPU_METHOD_OF_DECLARE |
802 | */ |
803 | for (i = 0; i < ARRAY_SIZE(sunxi_mc_smp_data); i++) { |
804 | ret = of_property_match_string(np: node, propname: "enable-method" , |
805 | string: sunxi_mc_smp_data[i].enable_method); |
806 | if (ret >= 0) |
807 | break; |
808 | } |
809 | |
810 | of_node_put(node); |
811 | if (ret < 0) |
812 | return -ENODEV; |
813 | |
814 | is_a83t = sunxi_mc_smp_data[i].is_a83t; |
815 | |
816 | if (!sunxi_mc_smp_cpu_table_init()) |
817 | return -EINVAL; |
818 | |
819 | if (!cci_probed()) { |
820 | pr_err("%s: CCI-400 not available\n" , __func__); |
821 | return -ENODEV; |
822 | } |
823 | |
824 | /* Get needed device tree nodes */ |
825 | ret = sunxi_mc_smp_data[i].get_smp_nodes(&nodes); |
826 | if (ret) |
827 | goto err_put_nodes; |
828 | |
829 | /* |
830 | * Unfortunately we can not request the I/O region for the PRCM. |
831 | * It is shared with the PRCM clock. |
832 | */ |
833 | prcm_base = of_iomap(node: nodes.prcm_node, index: 0); |
834 | if (!prcm_base) { |
835 | pr_err("%s: failed to map PRCM registers\n" , __func__); |
836 | ret = -ENOMEM; |
837 | goto err_put_nodes; |
838 | } |
839 | |
840 | cpucfg_base = of_io_request_and_map(device: nodes.cpucfg_node, index: 0, |
841 | name: "sunxi-mc-smp" ); |
842 | if (IS_ERR(ptr: cpucfg_base)) { |
843 | ret = PTR_ERR(ptr: cpucfg_base); |
844 | pr_err("%s: failed to map CPUCFG registers: %d\n" , |
845 | __func__, ret); |
846 | goto err_unmap_prcm; |
847 | } |
848 | |
849 | if (is_a83t) { |
850 | r_cpucfg_base = of_io_request_and_map(device: nodes.r_cpucfg_node, |
851 | index: 0, name: "sunxi-mc-smp" ); |
852 | if (IS_ERR(ptr: r_cpucfg_base)) { |
853 | ret = PTR_ERR(ptr: r_cpucfg_base); |
854 | pr_err("%s: failed to map R-CPUCFG registers\n" , |
855 | __func__); |
856 | goto err_unmap_release_cpucfg; |
857 | } |
858 | } else { |
859 | sram_b_smp_base = of_io_request_and_map(device: nodes.sram_node, index: 0, |
860 | name: "sunxi-mc-smp" ); |
861 | if (IS_ERR(ptr: sram_b_smp_base)) { |
862 | ret = PTR_ERR(ptr: sram_b_smp_base); |
863 | pr_err("%s: failed to map secure SRAM\n" , __func__); |
864 | goto err_unmap_release_cpucfg; |
865 | } |
866 | } |
867 | |
868 | /* Configure CCI-400 for boot cluster */ |
869 | ret = sunxi_mc_smp_loopback(); |
870 | if (ret) { |
871 | pr_err("%s: failed to configure boot cluster: %d\n" , |
872 | __func__, ret); |
873 | goto err_unmap_release_sram_rcpucfg; |
874 | } |
875 | |
876 | /* We don't need the device nodes anymore */ |
877 | sunxi_mc_smp_put_nodes(nodes: &nodes); |
878 | |
879 | /* Set the hardware entry point address */ |
880 | if (is_a83t) |
881 | addr = r_cpucfg_base + R_CPUCFG_CPU_SOFT_ENTRY_REG; |
882 | else |
883 | addr = prcm_base + PRCM_CPU_SOFT_ENTRY_REG; |
884 | writel(__pa_symbol(sunxi_mc_smp_secondary_startup), addr); |
885 | |
886 | /* Actually enable multi cluster SMP */ |
887 | smp_set_ops(&sunxi_mc_smp_smp_ops); |
888 | |
889 | pr_info("sunxi multi cluster SMP support installed\n" ); |
890 | |
891 | return 0; |
892 | |
893 | err_unmap_release_sram_rcpucfg: |
894 | if (is_a83t) { |
895 | iounmap(addr: r_cpucfg_base); |
896 | of_address_to_resource(dev: nodes.r_cpucfg_node, index: 0, r: &res); |
897 | } else { |
898 | iounmap(addr: sram_b_smp_base); |
899 | of_address_to_resource(dev: nodes.sram_node, index: 0, r: &res); |
900 | } |
901 | release_mem_region(res.start, resource_size(&res)); |
902 | err_unmap_release_cpucfg: |
903 | iounmap(addr: cpucfg_base); |
904 | of_address_to_resource(dev: nodes.cpucfg_node, index: 0, r: &res); |
905 | release_mem_region(res.start, resource_size(&res)); |
906 | err_unmap_prcm: |
907 | iounmap(addr: prcm_base); |
908 | err_put_nodes: |
909 | sunxi_mc_smp_put_nodes(nodes: &nodes); |
910 | return ret; |
911 | } |
912 | |
913 | early_initcall(sunxi_mc_smp_init); |
914 | |