1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * SMP support for pSeries machines. |
4 | * |
5 | * Dave Engebretsen, Peter Bergner, and |
6 | * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com |
7 | * |
8 | * Plus various changes from other IBM teams... |
9 | */ |
10 | |
11 | |
12 | #include <linux/kernel.h> |
13 | #include <linux/sched.h> |
14 | #include <linux/smp.h> |
15 | #include <linux/interrupt.h> |
16 | #include <linux/delay.h> |
17 | #include <linux/init.h> |
18 | #include <linux/spinlock.h> |
19 | #include <linux/cache.h> |
20 | #include <linux/err.h> |
21 | #include <linux/device.h> |
22 | #include <linux/cpu.h> |
23 | #include <linux/pgtable.h> |
24 | |
25 | #include <asm/ptrace.h> |
26 | #include <linux/atomic.h> |
27 | #include <asm/irq.h> |
28 | #include <asm/page.h> |
29 | #include <asm/io.h> |
30 | #include <asm/smp.h> |
31 | #include <asm/paca.h> |
32 | #include <asm/machdep.h> |
33 | #include <asm/cputable.h> |
34 | #include <asm/firmware.h> |
35 | #include <asm/rtas.h> |
36 | #include <asm/vdso_datapage.h> |
37 | #include <asm/cputhreads.h> |
38 | #include <asm/xics.h> |
39 | #include <asm/xive.h> |
40 | #include <asm/dbell.h> |
41 | #include <asm/plpar_wrappers.h> |
42 | #include <asm/code-patching.h> |
43 | #include <asm/svm.h> |
44 | #include <asm/kvm_guest.h> |
45 | |
46 | #include "pseries.h" |
47 | |
48 | /* |
49 | * The Primary thread of each non-boot processor was started from the OF client |
50 | * interface by prom_hold_cpus and is spinning on secondary_hold_spinloop. |
51 | */ |
52 | static cpumask_var_t of_spin_mask; |
53 | |
54 | /* Query where a cpu is now. Return codes #defined in plpar_wrappers.h */ |
55 | int smp_query_cpu_stopped(unsigned int pcpu) |
56 | { |
57 | int cpu_status, status; |
58 | int qcss_tok = rtas_function_token(RTAS_FN_QUERY_CPU_STOPPED_STATE); |
59 | |
60 | if (qcss_tok == RTAS_UNKNOWN_SERVICE) { |
61 | printk_once(KERN_INFO |
62 | "Firmware doesn't support query-cpu-stopped-state\n" ); |
63 | return QCSS_HARDWARE_ERROR; |
64 | } |
65 | |
66 | status = rtas_call(qcss_tok, 1, 2, &cpu_status, pcpu); |
67 | if (status != 0) { |
68 | printk(KERN_ERR |
69 | "RTAS query-cpu-stopped-state failed: %i\n" , status); |
70 | return status; |
71 | } |
72 | |
73 | return cpu_status; |
74 | } |
75 | |
76 | /** |
77 | * smp_startup_cpu() - start the given cpu |
78 | * |
79 | * At boot time, there is nothing to do for primary threads which were |
80 | * started from Open Firmware. For anything else, call RTAS with the |
81 | * appropriate start location. |
82 | * |
83 | * Returns: |
84 | * 0 - failure |
85 | * 1 - success |
86 | */ |
87 | static inline int smp_startup_cpu(unsigned int lcpu) |
88 | { |
89 | int status; |
90 | unsigned long start_here = |
91 | __pa(ppc_function_entry(generic_secondary_smp_init)); |
92 | unsigned int pcpu; |
93 | int start_cpu; |
94 | |
95 | if (cpumask_test_cpu(cpu: lcpu, cpumask: of_spin_mask)) |
96 | /* Already started by OF and sitting in spin loop */ |
97 | return 1; |
98 | |
99 | pcpu = get_hard_smp_processor_id(lcpu); |
100 | |
101 | /* Check to see if the CPU out of FW already for kexec */ |
102 | if (smp_query_cpu_stopped(pcpu) == QCSS_NOT_STOPPED){ |
103 | cpumask_set_cpu(cpu: lcpu, dstp: of_spin_mask); |
104 | return 1; |
105 | } |
106 | |
107 | /* |
108 | * If the RTAS start-cpu token does not exist then presume the |
109 | * cpu is already spinning. |
110 | */ |
111 | start_cpu = rtas_function_token(RTAS_FN_START_CPU); |
112 | if (start_cpu == RTAS_UNKNOWN_SERVICE) |
113 | return 1; |
114 | |
115 | status = rtas_call(start_cpu, 3, 1, NULL, pcpu, start_here, pcpu); |
116 | if (status != 0) { |
117 | printk(KERN_ERR "start-cpu failed: %i\n" , status); |
118 | return 0; |
119 | } |
120 | |
121 | return 1; |
122 | } |
123 | |
124 | static void smp_setup_cpu(int cpu) |
125 | { |
126 | if (xive_enabled()) |
127 | xive_smp_setup_cpu(); |
128 | else if (cpu != boot_cpuid) |
129 | xics_setup_cpu(); |
130 | |
131 | if (firmware_has_feature(FW_FEATURE_SPLPAR)) |
132 | vpa_init(cpu); |
133 | |
134 | cpumask_clear_cpu(cpu, dstp: of_spin_mask); |
135 | } |
136 | |
137 | static int smp_pSeries_kick_cpu(int nr) |
138 | { |
139 | if (nr < 0 || nr >= nr_cpu_ids) |
140 | return -EINVAL; |
141 | |
142 | if (!smp_startup_cpu(lcpu: nr)) |
143 | return -ENOENT; |
144 | |
145 | /* |
146 | * The processor is currently spinning, waiting for the |
147 | * cpu_start field to become non-zero After we set cpu_start, |
148 | * the processor will continue on to secondary_start |
149 | */ |
150 | paca_ptrs[nr]->cpu_start = 1; |
151 | |
152 | return 0; |
153 | } |
154 | |
155 | static int pseries_smp_prepare_cpu(int cpu) |
156 | { |
157 | if (xive_enabled()) |
158 | return xive_smp_prepare_cpu(cpu); |
159 | return 0; |
160 | } |
161 | |
162 | /* Cause IPI as setup by the interrupt controller (xics or xive) */ |
163 | static void (*ic_cause_ipi)(int cpu) __ro_after_init; |
164 | |
165 | /* Use msgsndp doorbells target is a sibling, else use interrupt controller */ |
166 | static void dbell_or_ic_cause_ipi(int cpu) |
167 | { |
168 | if (doorbell_try_core_ipi(cpu)) |
169 | return; |
170 | |
171 | ic_cause_ipi(cpu); |
172 | } |
173 | |
174 | static int pseries_cause_nmi_ipi(int cpu) |
175 | { |
176 | int hwcpu; |
177 | |
178 | if (cpu == NMI_IPI_ALL_OTHERS) { |
179 | hwcpu = H_SIGNAL_SYS_RESET_ALL_OTHERS; |
180 | } else { |
181 | if (cpu < 0) { |
182 | WARN_ONCE(true, "incorrect cpu parameter %d" , cpu); |
183 | return 0; |
184 | } |
185 | |
186 | hwcpu = get_hard_smp_processor_id(cpu); |
187 | } |
188 | |
189 | if (plpar_signal_sys_reset(hwcpu) == H_SUCCESS) |
190 | return 1; |
191 | |
192 | return 0; |
193 | } |
194 | |
195 | static __init void pSeries_smp_probe(void) |
196 | { |
197 | if (xive_enabled()) |
198 | xive_smp_probe(); |
199 | else |
200 | xics_smp_probe(); |
201 | |
202 | /* No doorbell facility, must use the interrupt controller for IPIs */ |
203 | if (!cpu_has_feature(CPU_FTR_DBELL)) |
204 | return; |
205 | |
206 | /* Doorbells can only be used for IPIs between SMT siblings */ |
207 | if (!cpu_has_feature(CPU_FTR_SMT)) |
208 | return; |
209 | |
210 | check_kvm_guest(); |
211 | |
212 | if (is_kvm_guest()) { |
213 | /* |
214 | * KVM emulates doorbells by disabling FSCR[MSGP] so msgsndp |
215 | * faults to the hypervisor which then reads the instruction |
216 | * from guest memory, which tends to be slower than using XIVE. |
217 | */ |
218 | if (xive_enabled()) |
219 | return; |
220 | |
221 | /* |
222 | * XICS hcalls aren't as fast, so we can use msgsndp (which |
223 | * also helps exercise KVM emulation), however KVM can't |
224 | * emulate secure guests because it can't read the instruction |
225 | * out of their memory. |
226 | */ |
227 | if (is_secure_guest()) |
228 | return; |
229 | } |
230 | |
231 | /* |
232 | * Under PowerVM, FSCR[MSGP] is enabled as guest vCPU siblings are |
233 | * gang scheduled on the same physical core, so doorbells are always |
234 | * faster than the interrupt controller, and they can be used by |
235 | * secure guests. |
236 | */ |
237 | |
238 | ic_cause_ipi = smp_ops->cause_ipi; |
239 | smp_ops->cause_ipi = dbell_or_ic_cause_ipi; |
240 | } |
241 | |
242 | static struct smp_ops_t pseries_smp_ops = { |
243 | .message_pass = NULL, /* Use smp_muxed_ipi_message_pass */ |
244 | .cause_ipi = NULL, /* Filled at runtime by pSeries_smp_probe() */ |
245 | .cause_nmi_ipi = pseries_cause_nmi_ipi, |
246 | .probe = pSeries_smp_probe, |
247 | .prepare_cpu = pseries_smp_prepare_cpu, |
248 | .kick_cpu = smp_pSeries_kick_cpu, |
249 | .setup_cpu = smp_setup_cpu, |
250 | .cpu_bootable = smp_generic_cpu_bootable, |
251 | }; |
252 | |
253 | /* This is called very early */ |
254 | void __init smp_init_pseries(void) |
255 | { |
256 | int i; |
257 | |
258 | pr_debug(" -> smp_init_pSeries()\n" ); |
259 | smp_ops = &pseries_smp_ops; |
260 | |
261 | alloc_bootmem_cpumask_var(mask: &of_spin_mask); |
262 | |
263 | /* |
264 | * Mark threads which are still spinning in hold loops |
265 | * |
266 | * We know prom_init will not have started them if RTAS supports |
267 | * query-cpu-stopped-state. |
268 | */ |
269 | if (rtas_function_token(RTAS_FN_QUERY_CPU_STOPPED_STATE) == RTAS_UNKNOWN_SERVICE) { |
270 | if (cpu_has_feature(CPU_FTR_SMT)) { |
271 | for_each_present_cpu(i) { |
272 | if (cpu_thread_in_core(i) == 0) |
273 | cpumask_set_cpu(cpu: i, dstp: of_spin_mask); |
274 | } |
275 | } else |
276 | cpumask_copy(dstp: of_spin_mask, cpu_present_mask); |
277 | |
278 | cpumask_clear_cpu(cpu: boot_cpuid, dstp: of_spin_mask); |
279 | } |
280 | |
281 | pr_debug(" <- smp_init_pSeries()\n" ); |
282 | } |
283 | |