1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * SMP related functions |
4 | * |
5 | * Copyright IBM Corp. 1999, 2012 |
6 | * Author(s): Denis Joseph Barrow, |
7 | * Martin Schwidefsky <schwidefsky@de.ibm.com>, |
8 | * |
9 | * based on other smp stuff by |
10 | * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net> |
11 | * (c) 1998 Ingo Molnar |
12 | * |
13 | * The code outside of smp.c uses logical cpu numbers, only smp.c does |
14 | * the translation of logical to physical cpu ids. All new code that |
15 | * operates on physical cpu numbers needs to go into smp.c. |
16 | */ |
17 | |
18 | #define KMSG_COMPONENT "cpu" |
19 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt |
20 | |
21 | #include <linux/workqueue.h> |
22 | #include <linux/memblock.h> |
23 | #include <linux/export.h> |
24 | #include <linux/init.h> |
25 | #include <linux/mm.h> |
26 | #include <linux/err.h> |
27 | #include <linux/spinlock.h> |
28 | #include <linux/kernel_stat.h> |
29 | #include <linux/delay.h> |
30 | #include <linux/interrupt.h> |
31 | #include <linux/irqflags.h> |
32 | #include <linux/irq_work.h> |
33 | #include <linux/cpu.h> |
34 | #include <linux/slab.h> |
35 | #include <linux/sched/hotplug.h> |
36 | #include <linux/sched/task_stack.h> |
37 | #include <linux/crash_dump.h> |
38 | #include <linux/kprobes.h> |
39 | #include <asm/access-regs.h> |
40 | #include <asm/asm-offsets.h> |
41 | #include <asm/ctlreg.h> |
42 | #include <asm/pfault.h> |
43 | #include <asm/diag.h> |
44 | #include <asm/facility.h> |
45 | #include <asm/fpu.h> |
46 | #include <asm/ipl.h> |
47 | #include <asm/setup.h> |
48 | #include <asm/irq.h> |
49 | #include <asm/tlbflush.h> |
50 | #include <asm/vtimer.h> |
51 | #include <asm/abs_lowcore.h> |
52 | #include <asm/sclp.h> |
53 | #include <asm/debug.h> |
54 | #include <asm/os_info.h> |
55 | #include <asm/sigp.h> |
56 | #include <asm/idle.h> |
57 | #include <asm/nmi.h> |
58 | #include <asm/stacktrace.h> |
59 | #include <asm/topology.h> |
60 | #include <asm/vdso.h> |
61 | #include <asm/maccess.h> |
62 | #include "entry.h" |
63 | |
64 | enum { |
65 | ec_schedule = 0, |
66 | ec_call_function_single, |
67 | ec_stop_cpu, |
68 | ec_mcck_pending, |
69 | ec_irq_work, |
70 | }; |
71 | |
72 | enum { |
73 | CPU_STATE_STANDBY, |
74 | CPU_STATE_CONFIGURED, |
75 | }; |
76 | |
77 | static DEFINE_PER_CPU(struct cpu *, cpu_device); |
78 | |
79 | struct pcpu { |
80 | unsigned long ec_mask; /* bit mask for ec_xxx functions */ |
81 | unsigned long ec_clk; /* sigp timestamp for ec_xxx */ |
82 | signed char state; /* physical cpu state */ |
83 | signed char polarization; /* physical polarization */ |
84 | u16 address; /* physical cpu address */ |
85 | }; |
86 | |
87 | static u8 boot_core_type; |
88 | static struct pcpu pcpu_devices[NR_CPUS]; |
89 | |
90 | unsigned int smp_cpu_mt_shift; |
91 | EXPORT_SYMBOL(smp_cpu_mt_shift); |
92 | |
93 | unsigned int smp_cpu_mtid; |
94 | EXPORT_SYMBOL(smp_cpu_mtid); |
95 | |
96 | #ifdef CONFIG_CRASH_DUMP |
97 | __vector128 __initdata boot_cpu_vector_save_area[__NUM_VXRS]; |
98 | #endif |
99 | |
100 | static unsigned int smp_max_threads __initdata = -1U; |
101 | cpumask_t cpu_setup_mask; |
102 | |
103 | static int __init early_nosmt(char *s) |
104 | { |
105 | smp_max_threads = 1; |
106 | return 0; |
107 | } |
108 | early_param("nosmt" , early_nosmt); |
109 | |
110 | static int __init early_smt(char *s) |
111 | { |
112 | get_option(str: &s, pint: &smp_max_threads); |
113 | return 0; |
114 | } |
115 | early_param("smt" , early_smt); |
116 | |
117 | /* |
118 | * The smp_cpu_state_mutex must be held when changing the state or polarization |
119 | * member of a pcpu data structure within the pcpu_devices array. |
120 | */ |
121 | DEFINE_MUTEX(smp_cpu_state_mutex); |
122 | |
123 | /* |
124 | * Signal processor helper functions. |
125 | */ |
126 | static inline int __pcpu_sigp_relax(u16 addr, u8 order, unsigned long parm) |
127 | { |
128 | int cc; |
129 | |
130 | while (1) { |
131 | cc = __pcpu_sigp(addr, order, parm, NULL); |
132 | if (cc != SIGP_CC_BUSY) |
133 | return cc; |
134 | cpu_relax(); |
135 | } |
136 | } |
137 | |
138 | static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm) |
139 | { |
140 | int cc, retry; |
141 | |
142 | for (retry = 0; ; retry++) { |
143 | cc = __pcpu_sigp(pcpu->address, order, parm, NULL); |
144 | if (cc != SIGP_CC_BUSY) |
145 | break; |
146 | if (retry >= 3) |
147 | udelay(10); |
148 | } |
149 | return cc; |
150 | } |
151 | |
152 | static inline int pcpu_stopped(struct pcpu *pcpu) |
153 | { |
154 | u32 status; |
155 | |
156 | if (__pcpu_sigp(pcpu->address, SIGP_SENSE, |
157 | 0, &status) != SIGP_CC_STATUS_STORED) |
158 | return 0; |
159 | return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED)); |
160 | } |
161 | |
162 | static inline int pcpu_running(struct pcpu *pcpu) |
163 | { |
164 | if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING, |
165 | 0, NULL) != SIGP_CC_STATUS_STORED) |
166 | return 1; |
167 | /* Status stored condition code is equivalent to cpu not running. */ |
168 | return 0; |
169 | } |
170 | |
171 | /* |
172 | * Find struct pcpu by cpu address. |
173 | */ |
174 | static struct pcpu *pcpu_find_address(const struct cpumask *mask, u16 address) |
175 | { |
176 | int cpu; |
177 | |
178 | for_each_cpu(cpu, mask) |
179 | if (pcpu_devices[cpu].address == address) |
180 | return pcpu_devices + cpu; |
181 | return NULL; |
182 | } |
183 | |
184 | static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit) |
185 | { |
186 | int order; |
187 | |
188 | if (test_and_set_bit(nr: ec_bit, addr: &pcpu->ec_mask)) |
189 | return; |
190 | order = pcpu_running(pcpu) ? SIGP_EXTERNAL_CALL : SIGP_EMERGENCY_SIGNAL; |
191 | pcpu->ec_clk = get_tod_clock_fast(); |
192 | pcpu_sigp_retry(pcpu, order, parm: 0); |
193 | } |
194 | |
195 | static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) |
196 | { |
197 | unsigned long async_stack, nodat_stack, mcck_stack; |
198 | struct lowcore *lc; |
199 | |
200 | lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER); |
201 | nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER); |
202 | async_stack = stack_alloc(); |
203 | mcck_stack = stack_alloc(); |
204 | if (!lc || !nodat_stack || !async_stack || !mcck_stack) |
205 | goto out; |
206 | memcpy(lc, &S390_lowcore, 512); |
207 | memset((char *) lc + 512, 0, sizeof(*lc) - 512); |
208 | lc->async_stack = async_stack + STACK_INIT_OFFSET; |
209 | lc->nodat_stack = nodat_stack + STACK_INIT_OFFSET; |
210 | lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET; |
211 | lc->cpu_nr = cpu; |
212 | lc->spinlock_lockval = arch_spin_lockval(cpu); |
213 | lc->spinlock_index = 0; |
214 | lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW); |
215 | lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW); |
216 | lc->preempt_count = PREEMPT_DISABLED; |
217 | if (nmi_alloc_mcesa(&lc->mcesad)) |
218 | goto out; |
219 | if (abs_lowcore_map(cpu, lc, true)) |
220 | goto out_mcesa; |
221 | lowcore_ptr[cpu] = lc; |
222 | pcpu_sigp_retry(pcpu, order: SIGP_SET_PREFIX, __pa(lc)); |
223 | return 0; |
224 | |
225 | out_mcesa: |
226 | nmi_free_mcesa(&lc->mcesad); |
227 | out: |
228 | stack_free(stack: mcck_stack); |
229 | stack_free(stack: async_stack); |
230 | free_pages(addr: nodat_stack, THREAD_SIZE_ORDER); |
231 | free_pages(addr: (unsigned long) lc, order: LC_ORDER); |
232 | return -ENOMEM; |
233 | } |
234 | |
235 | static void pcpu_free_lowcore(struct pcpu *pcpu) |
236 | { |
237 | unsigned long async_stack, nodat_stack, mcck_stack; |
238 | struct lowcore *lc; |
239 | int cpu; |
240 | |
241 | cpu = pcpu - pcpu_devices; |
242 | lc = lowcore_ptr[cpu]; |
243 | nodat_stack = lc->nodat_stack - STACK_INIT_OFFSET; |
244 | async_stack = lc->async_stack - STACK_INIT_OFFSET; |
245 | mcck_stack = lc->mcck_stack - STACK_INIT_OFFSET; |
246 | pcpu_sigp_retry(pcpu, order: SIGP_SET_PREFIX, parm: 0); |
247 | lowcore_ptr[cpu] = NULL; |
248 | abs_lowcore_unmap(cpu); |
249 | nmi_free_mcesa(&lc->mcesad); |
250 | stack_free(stack: async_stack); |
251 | stack_free(stack: mcck_stack); |
252 | free_pages(addr: nodat_stack, THREAD_SIZE_ORDER); |
253 | free_pages(addr: (unsigned long) lc, order: LC_ORDER); |
254 | } |
255 | |
256 | static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) |
257 | { |
258 | struct lowcore *lc, *abs_lc; |
259 | |
260 | lc = lowcore_ptr[cpu]; |
261 | cpumask_set_cpu(cpu, dstp: &init_mm.context.cpu_attach_mask); |
262 | cpumask_set_cpu(cpu, dstp: mm_cpumask(mm: &init_mm)); |
263 | lc->cpu_nr = cpu; |
264 | lc->restart_flags = RESTART_FLAG_CTLREGS; |
265 | lc->spinlock_lockval = arch_spin_lockval(cpu); |
266 | lc->spinlock_index = 0; |
267 | lc->percpu_offset = __per_cpu_offset[cpu]; |
268 | lc->kernel_asce = S390_lowcore.kernel_asce; |
269 | lc->user_asce = s390_invalid_asce; |
270 | lc->machine_flags = S390_lowcore.machine_flags; |
271 | lc->user_timer = lc->system_timer = |
272 | lc->steal_timer = lc->avg_steal_timer = 0; |
273 | abs_lc = get_abs_lowcore(); |
274 | memcpy(lc->cregs_save_area, abs_lc->cregs_save_area, sizeof(lc->cregs_save_area)); |
275 | put_abs_lowcore(abs_lc); |
276 | lc->cregs_save_area[1] = lc->kernel_asce; |
277 | lc->cregs_save_area[7] = lc->user_asce; |
278 | save_access_regs((unsigned int *) lc->access_regs_save_area); |
279 | arch_spin_lock_setup(cpu); |
280 | } |
281 | |
282 | static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk) |
283 | { |
284 | struct lowcore *lc; |
285 | int cpu; |
286 | |
287 | cpu = pcpu - pcpu_devices; |
288 | lc = lowcore_ptr[cpu]; |
289 | lc->kernel_stack = (unsigned long)task_stack_page(tsk) + STACK_INIT_OFFSET; |
290 | lc->current_task = (unsigned long)tsk; |
291 | lc->lpp = LPP_MAGIC; |
292 | lc->current_pid = tsk->pid; |
293 | lc->user_timer = tsk->thread.user_timer; |
294 | lc->guest_timer = tsk->thread.guest_timer; |
295 | lc->system_timer = tsk->thread.system_timer; |
296 | lc->hardirq_timer = tsk->thread.hardirq_timer; |
297 | lc->softirq_timer = tsk->thread.softirq_timer; |
298 | lc->steal_timer = 0; |
299 | } |
300 | |
301 | static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data) |
302 | { |
303 | struct lowcore *lc; |
304 | int cpu; |
305 | |
306 | cpu = pcpu - pcpu_devices; |
307 | lc = lowcore_ptr[cpu]; |
308 | lc->restart_stack = lc->kernel_stack; |
309 | lc->restart_fn = (unsigned long) func; |
310 | lc->restart_data = (unsigned long) data; |
311 | lc->restart_source = -1U; |
312 | pcpu_sigp_retry(pcpu, SIGP_RESTART, 0); |
313 | } |
314 | |
315 | typedef void (pcpu_delegate_fn)(void *); |
316 | |
317 | /* |
318 | * Call function via PSW restart on pcpu and stop the current cpu. |
319 | */ |
320 | static void __pcpu_delegate(pcpu_delegate_fn *func, void *data) |
321 | { |
322 | func(data); /* should not return */ |
323 | } |
324 | |
325 | static void pcpu_delegate(struct pcpu *pcpu, |
326 | pcpu_delegate_fn *func, |
327 | void *data, unsigned long stack) |
328 | { |
329 | struct lowcore *lc, *abs_lc; |
330 | unsigned int source_cpu; |
331 | |
332 | lc = lowcore_ptr[pcpu - pcpu_devices]; |
333 | source_cpu = stap(); |
334 | |
335 | if (pcpu->address == source_cpu) { |
336 | call_on_stack(2, stack, void, __pcpu_delegate, |
337 | pcpu_delegate_fn *, func, void *, data); |
338 | } |
339 | /* Stop target cpu (if func returns this stops the current cpu). */ |
340 | pcpu_sigp_retry(pcpu, SIGP_STOP, 0); |
341 | pcpu_sigp_retry(pcpu, SIGP_CPU_RESET, 0); |
342 | /* Restart func on the target cpu and stop the current cpu. */ |
343 | if (lc) { |
344 | lc->restart_stack = stack; |
345 | lc->restart_fn = (unsigned long)func; |
346 | lc->restart_data = (unsigned long)data; |
347 | lc->restart_source = source_cpu; |
348 | } else { |
349 | abs_lc = get_abs_lowcore(); |
350 | abs_lc->restart_stack = stack; |
351 | abs_lc->restart_fn = (unsigned long)func; |
352 | abs_lc->restart_data = (unsigned long)data; |
353 | abs_lc->restart_source = source_cpu; |
354 | put_abs_lowcore(abs_lc); |
355 | } |
356 | asm volatile( |
357 | "0: sigp 0,%0,%2 # sigp restart to target cpu\n" |
358 | " brc 2,0b # busy, try again\n" |
359 | "1: sigp 0,%1,%3 # sigp stop to current cpu\n" |
360 | " brc 2,1b # busy, try again\n" |
361 | : : "d" (pcpu->address), "d" (source_cpu), |
362 | "K" (SIGP_RESTART), "K" (SIGP_STOP) |
363 | : "0" , "1" , "cc" ); |
364 | for (;;) ; |
365 | } |
366 | |
367 | /* |
368 | * Enable additional logical cpus for multi-threading. |
369 | */ |
370 | static int pcpu_set_smt(unsigned int mtid) |
371 | { |
372 | int cc; |
373 | |
374 | if (smp_cpu_mtid == mtid) |
375 | return 0; |
376 | cc = __pcpu_sigp(0, SIGP_SET_MULTI_THREADING, mtid, NULL); |
377 | if (cc == 0) { |
378 | smp_cpu_mtid = mtid; |
379 | smp_cpu_mt_shift = 0; |
380 | while (smp_cpu_mtid >= (1U << smp_cpu_mt_shift)) |
381 | smp_cpu_mt_shift++; |
382 | pcpu_devices[0].address = stap(); |
383 | } |
384 | return cc; |
385 | } |
386 | |
387 | /* |
388 | * Call function on an online CPU. |
389 | */ |
390 | void smp_call_online_cpu(void (*func)(void *), void *data) |
391 | { |
392 | struct pcpu *pcpu; |
393 | |
394 | /* Use the current cpu if it is online. */ |
395 | pcpu = pcpu_find_address(cpu_online_mask, address: stap()); |
396 | if (!pcpu) |
397 | /* Use the first online cpu. */ |
398 | pcpu = pcpu_devices + cpumask_first(cpu_online_mask); |
399 | pcpu_delegate(pcpu, func, data, stack: (unsigned long) restart_stack); |
400 | } |
401 | |
402 | /* |
403 | * Call function on the ipl CPU. |
404 | */ |
405 | void smp_call_ipl_cpu(void (*func)(void *), void *data) |
406 | { |
407 | struct lowcore *lc = lowcore_ptr[0]; |
408 | |
409 | if (pcpu_devices[0].address == stap()) |
410 | lc = &S390_lowcore; |
411 | |
412 | pcpu_delegate(pcpu: &pcpu_devices[0], func, data, |
413 | stack: lc->nodat_stack); |
414 | } |
415 | |
416 | int smp_find_processor_id(u16 address) |
417 | { |
418 | int cpu; |
419 | |
420 | for_each_present_cpu(cpu) |
421 | if (pcpu_devices[cpu].address == address) |
422 | return cpu; |
423 | return -1; |
424 | } |
425 | |
426 | void schedule_mcck_handler(void) |
427 | { |
428 | pcpu_ec_call(pcpu: pcpu_devices + smp_processor_id(), ec_bit: ec_mcck_pending); |
429 | } |
430 | |
431 | bool notrace arch_vcpu_is_preempted(int cpu) |
432 | { |
433 | if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu)) |
434 | return false; |
435 | if (pcpu_running(pcpu: pcpu_devices + cpu)) |
436 | return false; |
437 | return true; |
438 | } |
439 | EXPORT_SYMBOL(arch_vcpu_is_preempted); |
440 | |
441 | void notrace smp_yield_cpu(int cpu) |
442 | { |
443 | if (!MACHINE_HAS_DIAG9C) |
444 | return; |
445 | diag_stat_inc_norecursion(DIAG_STAT_X09C); |
446 | asm volatile("diag %0,0,0x9c" |
447 | : : "d" (pcpu_devices[cpu].address)); |
448 | } |
449 | EXPORT_SYMBOL_GPL(smp_yield_cpu); |
450 | |
451 | /* |
452 | * Send cpus emergency shutdown signal. This gives the cpus the |
453 | * opportunity to complete outstanding interrupts. |
454 | */ |
455 | void notrace smp_emergency_stop(void) |
456 | { |
457 | static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED; |
458 | static cpumask_t cpumask; |
459 | u64 end; |
460 | int cpu; |
461 | |
462 | arch_spin_lock(&lock); |
463 | cpumask_copy(dstp: &cpumask, cpu_online_mask); |
464 | cpumask_clear_cpu(smp_processor_id(), dstp: &cpumask); |
465 | |
466 | end = get_tod_clock() + (1000000UL << 12); |
467 | for_each_cpu(cpu, &cpumask) { |
468 | struct pcpu *pcpu = pcpu_devices + cpu; |
469 | set_bit(nr: ec_stop_cpu, addr: &pcpu->ec_mask); |
470 | while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL, |
471 | 0, NULL) == SIGP_CC_BUSY && |
472 | get_tod_clock() < end) |
473 | cpu_relax(); |
474 | } |
475 | while (get_tod_clock() < end) { |
476 | for_each_cpu(cpu, &cpumask) |
477 | if (pcpu_stopped(pcpu: pcpu_devices + cpu)) |
478 | cpumask_clear_cpu(cpu, dstp: &cpumask); |
479 | if (cpumask_empty(srcp: &cpumask)) |
480 | break; |
481 | cpu_relax(); |
482 | } |
483 | arch_spin_unlock(&lock); |
484 | } |
485 | NOKPROBE_SYMBOL(smp_emergency_stop); |
486 | |
487 | /* |
488 | * Stop all cpus but the current one. |
489 | */ |
490 | void smp_send_stop(void) |
491 | { |
492 | int cpu; |
493 | |
494 | /* Disable all interrupts/machine checks */ |
495 | __load_psw_mask(PSW_KERNEL_BITS); |
496 | trace_hardirqs_off(); |
497 | |
498 | debug_set_critical(); |
499 | |
500 | if (oops_in_progress) |
501 | smp_emergency_stop(); |
502 | |
503 | /* stop all processors */ |
504 | for_each_online_cpu(cpu) { |
505 | if (cpu == smp_processor_id()) |
506 | continue; |
507 | pcpu_sigp_retry(pcpu_devices + cpu, SIGP_STOP, 0); |
508 | while (!pcpu_stopped(pcpu: pcpu_devices + cpu)) |
509 | cpu_relax(); |
510 | } |
511 | } |
512 | |
513 | /* |
514 | * This is the main routine where commands issued by other |
515 | * cpus are handled. |
516 | */ |
517 | static void smp_handle_ext_call(void) |
518 | { |
519 | unsigned long bits; |
520 | |
521 | /* handle bit signal external calls */ |
522 | bits = xchg(&pcpu_devices[smp_processor_id()].ec_mask, 0); |
523 | if (test_bit(ec_stop_cpu, &bits)) |
524 | smp_stop_cpu(); |
525 | if (test_bit(ec_schedule, &bits)) |
526 | scheduler_ipi(); |
527 | if (test_bit(ec_call_function_single, &bits)) |
528 | generic_smp_call_function_single_interrupt(); |
529 | if (test_bit(ec_mcck_pending, &bits)) |
530 | s390_handle_mcck(); |
531 | if (test_bit(ec_irq_work, &bits)) |
532 | irq_work_run(); |
533 | } |
534 | |
535 | static void do_ext_call_interrupt(struct ext_code ext_code, |
536 | unsigned int param32, unsigned long param64) |
537 | { |
538 | inc_irq_stat(ext_code.code == 0x1202 ? IRQEXT_EXC : IRQEXT_EMS); |
539 | smp_handle_ext_call(); |
540 | } |
541 | |
542 | void arch_send_call_function_ipi_mask(const struct cpumask *mask) |
543 | { |
544 | int cpu; |
545 | |
546 | for_each_cpu(cpu, mask) |
547 | pcpu_ec_call(pcpu: pcpu_devices + cpu, ec_bit: ec_call_function_single); |
548 | } |
549 | |
550 | void arch_send_call_function_single_ipi(int cpu) |
551 | { |
552 | pcpu_ec_call(pcpu: pcpu_devices + cpu, ec_bit: ec_call_function_single); |
553 | } |
554 | |
555 | /* |
556 | * this function sends a 'reschedule' IPI to another CPU. |
557 | * it goes straight through and wastes no time serializing |
558 | * anything. Worst case is that we lose a reschedule ... |
559 | */ |
560 | void arch_smp_send_reschedule(int cpu) |
561 | { |
562 | pcpu_ec_call(pcpu: pcpu_devices + cpu, ec_bit: ec_schedule); |
563 | } |
564 | |
565 | #ifdef CONFIG_IRQ_WORK |
566 | void arch_irq_work_raise(void) |
567 | { |
568 | pcpu_ec_call(pcpu: pcpu_devices + smp_processor_id(), ec_bit: ec_irq_work); |
569 | } |
570 | #endif |
571 | |
572 | #ifdef CONFIG_CRASH_DUMP |
573 | |
574 | int smp_store_status(int cpu) |
575 | { |
576 | struct lowcore *lc; |
577 | struct pcpu *pcpu; |
578 | unsigned long pa; |
579 | |
580 | pcpu = pcpu_devices + cpu; |
581 | lc = lowcore_ptr[cpu]; |
582 | pa = __pa(&lc->floating_pt_save_area); |
583 | if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_STATUS_AT_ADDRESS, |
584 | pa) != SIGP_CC_ORDER_CODE_ACCEPTED) |
585 | return -EIO; |
586 | if (!cpu_has_vx() && !MACHINE_HAS_GS) |
587 | return 0; |
588 | pa = lc->mcesad & MCESA_ORIGIN_MASK; |
589 | if (MACHINE_HAS_GS) |
590 | pa |= lc->mcesad & MCESA_LC_MASK; |
591 | if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_ADDITIONAL_STATUS, |
592 | pa) != SIGP_CC_ORDER_CODE_ACCEPTED) |
593 | return -EIO; |
594 | return 0; |
595 | } |
596 | |
597 | /* |
598 | * Collect CPU state of the previous, crashed system. |
599 | * There are four cases: |
600 | * 1) standard zfcp/nvme dump |
601 | * condition: OLDMEM_BASE == NULL && is_ipl_type_dump() == true |
602 | * The state for all CPUs except the boot CPU needs to be collected |
603 | * with sigp stop-and-store-status. The boot CPU state is located in |
604 | * the absolute lowcore of the memory stored in the HSA. The zcore code |
605 | * will copy the boot CPU state from the HSA. |
606 | * 2) stand-alone kdump for SCSI/NVMe (zfcp/nvme dump with swapped memory) |
607 | * condition: OLDMEM_BASE != NULL && is_ipl_type_dump() == true |
608 | * The state for all CPUs except the boot CPU needs to be collected |
609 | * with sigp stop-and-store-status. The firmware or the boot-loader |
610 | * stored the registers of the boot CPU in the absolute lowcore in the |
611 | * memory of the old system. |
612 | * 3) kdump and the old kernel did not store the CPU state, |
613 | * or stand-alone kdump for DASD |
614 | * condition: OLDMEM_BASE != NULL && !is_kdump_kernel() |
615 | * The state for all CPUs except the boot CPU needs to be collected |
616 | * with sigp stop-and-store-status. The kexec code or the boot-loader |
617 | * stored the registers of the boot CPU in the memory of the old system. |
618 | * 4) kdump and the old kernel stored the CPU state |
619 | * condition: OLDMEM_BASE != NULL && is_kdump_kernel() |
620 | * This case does not exist for s390 anymore, setup_arch explicitly |
621 | * deactivates the elfcorehdr= kernel parameter |
622 | */ |
623 | static bool dump_available(void) |
624 | { |
625 | return oldmem_data.start || is_ipl_type_dump(); |
626 | } |
627 | |
628 | void __init smp_save_dump_ipl_cpu(void) |
629 | { |
630 | struct save_area *sa; |
631 | void *regs; |
632 | |
633 | if (!dump_available()) |
634 | return; |
635 | sa = save_area_alloc(true); |
636 | regs = memblock_alloc(size: 512, align: 8); |
637 | if (!sa || !regs) |
638 | panic(fmt: "could not allocate memory for boot CPU save area\n" ); |
639 | copy_oldmem_kernel(regs, __LC_FPREGS_SAVE_AREA, 512); |
640 | save_area_add_regs(sa, regs); |
641 | memblock_free(ptr: regs, size: 512); |
642 | if (cpu_has_vx()) |
643 | save_area_add_vxrs(sa, boot_cpu_vector_save_area); |
644 | } |
645 | |
646 | void __init smp_save_dump_secondary_cpus(void) |
647 | { |
648 | int addr, boot_cpu_addr, max_cpu_addr; |
649 | struct save_area *sa; |
650 | void *page; |
651 | |
652 | if (!dump_available()) |
653 | return; |
654 | /* Allocate a page as dumping area for the store status sigps */ |
655 | page = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); |
656 | if (!page) |
657 | panic(fmt: "ERROR: Failed to allocate %lx bytes below %lx\n" , |
658 | PAGE_SIZE, 1UL << 31); |
659 | |
660 | /* Set multi-threading state to the previous system. */ |
661 | pcpu_set_smt(sclp.mtid_prev); |
662 | boot_cpu_addr = stap(); |
663 | max_cpu_addr = SCLP_MAX_CORES << sclp.mtid_prev; |
664 | for (addr = 0; addr <= max_cpu_addr; addr++) { |
665 | if (addr == boot_cpu_addr) |
666 | continue; |
667 | if (__pcpu_sigp_relax(addr, SIGP_SENSE, 0) == |
668 | SIGP_CC_NOT_OPERATIONAL) |
669 | continue; |
670 | sa = save_area_alloc(false); |
671 | if (!sa) |
672 | panic(fmt: "could not allocate memory for save area\n" ); |
673 | __pcpu_sigp_relax(addr, SIGP_STORE_STATUS_AT_ADDRESS, __pa(page)); |
674 | save_area_add_regs(sa, page); |
675 | if (cpu_has_vx()) { |
676 | __pcpu_sigp_relax(addr, SIGP_STORE_ADDITIONAL_STATUS, __pa(page)); |
677 | save_area_add_vxrs(sa, page); |
678 | } |
679 | } |
680 | memblock_free(ptr: page, PAGE_SIZE); |
681 | diag_amode31_ops.diag308_reset(); |
682 | pcpu_set_smt(mtid: 0); |
683 | } |
684 | #endif /* CONFIG_CRASH_DUMP */ |
685 | |
686 | void smp_cpu_set_polarization(int cpu, int val) |
687 | { |
688 | pcpu_devices[cpu].polarization = val; |
689 | } |
690 | |
691 | int smp_cpu_get_polarization(int cpu) |
692 | { |
693 | return pcpu_devices[cpu].polarization; |
694 | } |
695 | |
696 | int smp_cpu_get_cpu_address(int cpu) |
697 | { |
698 | return pcpu_devices[cpu].address; |
699 | } |
700 | |
701 | static void __ref smp_get_core_info(struct sclp_core_info *info, int early) |
702 | { |
703 | static int use_sigp_detection; |
704 | int address; |
705 | |
706 | if (use_sigp_detection || sclp_get_core_info(info, early)) { |
707 | use_sigp_detection = 1; |
708 | for (address = 0; |
709 | address < (SCLP_MAX_CORES << smp_cpu_mt_shift); |
710 | address += (1U << smp_cpu_mt_shift)) { |
711 | if (__pcpu_sigp_relax(address, SIGP_SENSE, 0) == |
712 | SIGP_CC_NOT_OPERATIONAL) |
713 | continue; |
714 | info->core[info->configured].core_id = |
715 | address >> smp_cpu_mt_shift; |
716 | info->configured++; |
717 | } |
718 | info->combined = info->configured; |
719 | } |
720 | } |
721 | |
722 | static int smp_add_present_cpu(int cpu); |
723 | |
724 | static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail, |
725 | bool configured, bool early) |
726 | { |
727 | struct pcpu *pcpu; |
728 | int cpu, nr, i; |
729 | u16 address; |
730 | |
731 | nr = 0; |
732 | if (sclp.has_core_type && core->type != boot_core_type) |
733 | return nr; |
734 | cpu = cpumask_first(srcp: avail); |
735 | address = core->core_id << smp_cpu_mt_shift; |
736 | for (i = 0; (i <= smp_cpu_mtid) && (cpu < nr_cpu_ids); i++) { |
737 | if (pcpu_find_address(cpu_present_mask, address: address + i)) |
738 | continue; |
739 | pcpu = pcpu_devices + cpu; |
740 | pcpu->address = address + i; |
741 | if (configured) |
742 | pcpu->state = CPU_STATE_CONFIGURED; |
743 | else |
744 | pcpu->state = CPU_STATE_STANDBY; |
745 | smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); |
746 | set_cpu_present(cpu, present: true); |
747 | if (!early && smp_add_present_cpu(cpu) != 0) |
748 | set_cpu_present(cpu, present: false); |
749 | else |
750 | nr++; |
751 | cpumask_clear_cpu(cpu, dstp: avail); |
752 | cpu = cpumask_next(n: cpu, srcp: avail); |
753 | } |
754 | return nr; |
755 | } |
756 | |
757 | static int __smp_rescan_cpus(struct sclp_core_info *info, bool early) |
758 | { |
759 | struct sclp_core_entry *core; |
760 | static cpumask_t avail; |
761 | bool configured; |
762 | u16 core_id; |
763 | int nr, i; |
764 | |
765 | cpus_read_lock(); |
766 | mutex_lock(&smp_cpu_state_mutex); |
767 | nr = 0; |
768 | cpumask_xor(dstp: &avail, cpu_possible_mask, cpu_present_mask); |
769 | /* |
770 | * Add IPL core first (which got logical CPU number 0) to make sure |
771 | * that all SMT threads get subsequent logical CPU numbers. |
772 | */ |
773 | if (early) { |
774 | core_id = pcpu_devices[0].address >> smp_cpu_mt_shift; |
775 | for (i = 0; i < info->configured; i++) { |
776 | core = &info->core[i]; |
777 | if (core->core_id == core_id) { |
778 | nr += smp_add_core(core, avail: &avail, configured: true, early); |
779 | break; |
780 | } |
781 | } |
782 | } |
783 | for (i = 0; i < info->combined; i++) { |
784 | configured = i < info->configured; |
785 | nr += smp_add_core(core: &info->core[i], avail: &avail, configured, early); |
786 | } |
787 | mutex_unlock(lock: &smp_cpu_state_mutex); |
788 | cpus_read_unlock(); |
789 | return nr; |
790 | } |
791 | |
792 | void __init smp_detect_cpus(void) |
793 | { |
794 | unsigned int cpu, mtid, c_cpus, s_cpus; |
795 | struct sclp_core_info *info; |
796 | u16 address; |
797 | |
798 | /* Get CPU information */ |
799 | info = memblock_alloc(sizeof(*info), 8); |
800 | if (!info) |
801 | panic("%s: Failed to allocate %zu bytes align=0x%x\n" , |
802 | __func__, sizeof(*info), 8); |
803 | smp_get_core_info(info, early: 1); |
804 | /* Find boot CPU type */ |
805 | if (sclp.has_core_type) { |
806 | address = stap(); |
807 | for (cpu = 0; cpu < info->combined; cpu++) |
808 | if (info->core[cpu].core_id == address) { |
809 | /* The boot cpu dictates the cpu type. */ |
810 | boot_core_type = info->core[cpu].type; |
811 | break; |
812 | } |
813 | if (cpu >= info->combined) |
814 | panic(fmt: "Could not find boot CPU type" ); |
815 | } |
816 | |
817 | /* Set multi-threading state for the current system */ |
818 | mtid = boot_core_type ? sclp.mtid : sclp.mtid_cp; |
819 | mtid = (mtid < smp_max_threads) ? mtid : smp_max_threads - 1; |
820 | pcpu_set_smt(mtid); |
821 | |
822 | /* Print number of CPUs */ |
823 | c_cpus = s_cpus = 0; |
824 | for (cpu = 0; cpu < info->combined; cpu++) { |
825 | if (sclp.has_core_type && |
826 | info->core[cpu].type != boot_core_type) |
827 | continue; |
828 | if (cpu < info->configured) |
829 | c_cpus += smp_cpu_mtid + 1; |
830 | else |
831 | s_cpus += smp_cpu_mtid + 1; |
832 | } |
833 | pr_info("%d configured CPUs, %d standby CPUs\n" , c_cpus, s_cpus); |
834 | |
835 | /* Add CPUs present at boot */ |
836 | __smp_rescan_cpus(info, early: true); |
837 | memblock_free(info, sizeof(*info)); |
838 | } |
839 | |
840 | /* |
841 | * Activate a secondary processor. |
842 | */ |
843 | static void smp_start_secondary(void *cpuvoid) |
844 | { |
845 | int cpu = raw_smp_processor_id(); |
846 | |
847 | S390_lowcore.last_update_clock = get_tod_clock(); |
848 | S390_lowcore.restart_stack = (unsigned long)restart_stack; |
849 | S390_lowcore.restart_fn = (unsigned long)do_restart; |
850 | S390_lowcore.restart_data = 0; |
851 | S390_lowcore.restart_source = -1U; |
852 | S390_lowcore.restart_flags = 0; |
853 | restore_access_regs(S390_lowcore.access_regs_save_area); |
854 | cpu_init(); |
855 | rcutree_report_cpu_starting(cpu); |
856 | init_cpu_timer(); |
857 | vtime_init(); |
858 | vdso_getcpu_init(); |
859 | pfault_init(); |
860 | cpumask_set_cpu(cpu, dstp: &cpu_setup_mask); |
861 | update_cpu_masks(); |
862 | notify_cpu_starting(cpu); |
863 | if (topology_cpu_dedicated(cpu)) |
864 | set_cpu_flag(CIF_DEDICATED_CPU); |
865 | else |
866 | clear_cpu_flag(CIF_DEDICATED_CPU); |
867 | set_cpu_online(cpu, online: true); |
868 | inc_irq_stat(CPU_RST); |
869 | local_irq_enable(); |
870 | cpu_startup_entry(state: CPUHP_AP_ONLINE_IDLE); |
871 | } |
872 | |
873 | /* Upping and downing of CPUs */ |
874 | int __cpu_up(unsigned int cpu, struct task_struct *tidle) |
875 | { |
876 | struct pcpu *pcpu = pcpu_devices + cpu; |
877 | int rc; |
878 | |
879 | if (pcpu->state != CPU_STATE_CONFIGURED) |
880 | return -EIO; |
881 | if (pcpu_sigp_retry(pcpu, SIGP_INITIAL_CPU_RESET, 0) != |
882 | SIGP_CC_ORDER_CODE_ACCEPTED) |
883 | return -EIO; |
884 | |
885 | rc = pcpu_alloc_lowcore(pcpu, cpu); |
886 | if (rc) |
887 | return rc; |
888 | /* |
889 | * Make sure global control register contents do not change |
890 | * until new CPU has initialized control registers. |
891 | */ |
892 | system_ctlreg_lock(); |
893 | pcpu_prepare_secondary(pcpu, cpu); |
894 | pcpu_attach_task(pcpu, tsk: tidle); |
895 | pcpu_start_fn(pcpu, func: smp_start_secondary, NULL); |
896 | /* Wait until cpu puts itself in the online & active maps */ |
897 | while (!cpu_online(cpu)) |
898 | cpu_relax(); |
899 | system_ctlreg_unlock(); |
900 | return 0; |
901 | } |
902 | |
903 | static unsigned int setup_possible_cpus __initdata; |
904 | |
905 | static int __init _setup_possible_cpus(char *s) |
906 | { |
907 | get_option(str: &s, pint: &setup_possible_cpus); |
908 | return 0; |
909 | } |
910 | early_param("possible_cpus" , _setup_possible_cpus); |
911 | |
912 | int __cpu_disable(void) |
913 | { |
914 | struct ctlreg cregs[16]; |
915 | int cpu; |
916 | |
917 | /* Handle possible pending IPIs */ |
918 | smp_handle_ext_call(); |
919 | cpu = smp_processor_id(); |
920 | set_cpu_online(cpu, online: false); |
921 | cpumask_clear_cpu(cpu, dstp: &cpu_setup_mask); |
922 | update_cpu_masks(); |
923 | /* Disable pseudo page faults on this cpu. */ |
924 | pfault_fini(); |
925 | /* Disable interrupt sources via control register. */ |
926 | __local_ctl_store(0, 15, cregs); |
927 | cregs[0].val &= ~0x0000ee70UL; /* disable all external interrupts */ |
928 | cregs[6].val &= ~0xff000000UL; /* disable all I/O interrupts */ |
929 | cregs[14].val &= ~0x1f000000UL; /* disable most machine checks */ |
930 | __local_ctl_load(0, 15, cregs); |
931 | clear_cpu_flag(CIF_NOHZ_DELAY); |
932 | return 0; |
933 | } |
934 | |
935 | void __cpu_die(unsigned int cpu) |
936 | { |
937 | struct pcpu *pcpu; |
938 | |
939 | /* Wait until target cpu is down */ |
940 | pcpu = pcpu_devices + cpu; |
941 | while (!pcpu_stopped(pcpu)) |
942 | cpu_relax(); |
943 | pcpu_free_lowcore(pcpu); |
944 | cpumask_clear_cpu(cpu, dstp: mm_cpumask(mm: &init_mm)); |
945 | cpumask_clear_cpu(cpu, dstp: &init_mm.context.cpu_attach_mask); |
946 | } |
947 | |
948 | void __noreturn cpu_die(void) |
949 | { |
950 | idle_task_exit(); |
951 | pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0); |
952 | for (;;) ; |
953 | } |
954 | |
955 | void __init smp_fill_possible_mask(void) |
956 | { |
957 | unsigned int possible, sclp_max, cpu; |
958 | |
959 | sclp_max = max(sclp.mtid, sclp.mtid_cp) + 1; |
960 | sclp_max = min(smp_max_threads, sclp_max); |
961 | sclp_max = (sclp.max_cores * sclp_max) ?: nr_cpu_ids; |
962 | possible = setup_possible_cpus ?: nr_cpu_ids; |
963 | possible = min(possible, sclp_max); |
964 | for (cpu = 0; cpu < possible && cpu < nr_cpu_ids; cpu++) |
965 | set_cpu_possible(cpu, possible: true); |
966 | } |
967 | |
968 | void __init smp_prepare_cpus(unsigned int max_cpus) |
969 | { |
970 | if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt)) |
971 | panic(fmt: "Couldn't request external interrupt 0x1201" ); |
972 | system_ctl_set_bit(0, 14); |
973 | if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt)) |
974 | panic(fmt: "Couldn't request external interrupt 0x1202" ); |
975 | system_ctl_set_bit(0, 13); |
976 | } |
977 | |
978 | void __init smp_prepare_boot_cpu(void) |
979 | { |
980 | struct pcpu *pcpu = pcpu_devices; |
981 | |
982 | WARN_ON(!cpu_present(0) || !cpu_online(0)); |
983 | pcpu->state = CPU_STATE_CONFIGURED; |
984 | S390_lowcore.percpu_offset = __per_cpu_offset[0]; |
985 | smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN); |
986 | } |
987 | |
988 | void __init smp_setup_processor_id(void) |
989 | { |
990 | pcpu_devices[0].address = stap(); |
991 | S390_lowcore.cpu_nr = 0; |
992 | S390_lowcore.spinlock_lockval = arch_spin_lockval(0); |
993 | S390_lowcore.spinlock_index = 0; |
994 | } |
995 | |
996 | /* |
997 | * the frequency of the profiling timer can be changed |
998 | * by writing a multiplier value into /proc/profile. |
999 | * |
1000 | * usually you want to run this on all CPUs ;) |
1001 | */ |
1002 | int setup_profiling_timer(unsigned int multiplier) |
1003 | { |
1004 | return 0; |
1005 | } |
1006 | |
1007 | static ssize_t cpu_configure_show(struct device *dev, |
1008 | struct device_attribute *attr, char *buf) |
1009 | { |
1010 | ssize_t count; |
1011 | |
1012 | mutex_lock(&smp_cpu_state_mutex); |
1013 | count = sprintf(buf, fmt: "%d\n" , pcpu_devices[dev->id].state); |
1014 | mutex_unlock(lock: &smp_cpu_state_mutex); |
1015 | return count; |
1016 | } |
1017 | |
1018 | static ssize_t cpu_configure_store(struct device *dev, |
1019 | struct device_attribute *attr, |
1020 | const char *buf, size_t count) |
1021 | { |
1022 | struct pcpu *pcpu; |
1023 | int cpu, val, rc, i; |
1024 | char delim; |
1025 | |
1026 | if (sscanf(buf, "%d %c" , &val, &delim) != 1) |
1027 | return -EINVAL; |
1028 | if (val != 0 && val != 1) |
1029 | return -EINVAL; |
1030 | cpus_read_lock(); |
1031 | mutex_lock(&smp_cpu_state_mutex); |
1032 | rc = -EBUSY; |
1033 | /* disallow configuration changes of online cpus */ |
1034 | cpu = dev->id; |
1035 | cpu = smp_get_base_cpu(cpu); |
1036 | for (i = 0; i <= smp_cpu_mtid; i++) |
1037 | if (cpu_online(cpu: cpu + i)) |
1038 | goto out; |
1039 | pcpu = pcpu_devices + cpu; |
1040 | rc = 0; |
1041 | switch (val) { |
1042 | case 0: |
1043 | if (pcpu->state != CPU_STATE_CONFIGURED) |
1044 | break; |
1045 | rc = sclp_core_deconfigure(pcpu->address >> smp_cpu_mt_shift); |
1046 | if (rc) |
1047 | break; |
1048 | for (i = 0; i <= smp_cpu_mtid; i++) { |
1049 | if (cpu + i >= nr_cpu_ids || !cpu_present(cpu: cpu + i)) |
1050 | continue; |
1051 | pcpu[i].state = CPU_STATE_STANDBY; |
1052 | smp_cpu_set_polarization(cpu + i, |
1053 | POLARIZATION_UNKNOWN); |
1054 | } |
1055 | topology_expect_change(); |
1056 | break; |
1057 | case 1: |
1058 | if (pcpu->state != CPU_STATE_STANDBY) |
1059 | break; |
1060 | rc = sclp_core_configure(pcpu->address >> smp_cpu_mt_shift); |
1061 | if (rc) |
1062 | break; |
1063 | for (i = 0; i <= smp_cpu_mtid; i++) { |
1064 | if (cpu + i >= nr_cpu_ids || !cpu_present(cpu: cpu + i)) |
1065 | continue; |
1066 | pcpu[i].state = CPU_STATE_CONFIGURED; |
1067 | smp_cpu_set_polarization(cpu + i, |
1068 | POLARIZATION_UNKNOWN); |
1069 | } |
1070 | topology_expect_change(); |
1071 | break; |
1072 | default: |
1073 | break; |
1074 | } |
1075 | out: |
1076 | mutex_unlock(lock: &smp_cpu_state_mutex); |
1077 | cpus_read_unlock(); |
1078 | return rc ? rc : count; |
1079 | } |
1080 | static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store); |
1081 | |
1082 | static ssize_t show_cpu_address(struct device *dev, |
1083 | struct device_attribute *attr, char *buf) |
1084 | { |
1085 | return sprintf(buf, fmt: "%d\n" , pcpu_devices[dev->id].address); |
1086 | } |
1087 | static DEVICE_ATTR(address, 0444, show_cpu_address, NULL); |
1088 | |
1089 | static struct attribute *cpu_common_attrs[] = { |
1090 | &dev_attr_configure.attr, |
1091 | &dev_attr_address.attr, |
1092 | NULL, |
1093 | }; |
1094 | |
1095 | static struct attribute_group cpu_common_attr_group = { |
1096 | .attrs = cpu_common_attrs, |
1097 | }; |
1098 | |
1099 | static struct attribute *cpu_online_attrs[] = { |
1100 | &dev_attr_idle_count.attr, |
1101 | &dev_attr_idle_time_us.attr, |
1102 | NULL, |
1103 | }; |
1104 | |
1105 | static struct attribute_group cpu_online_attr_group = { |
1106 | .attrs = cpu_online_attrs, |
1107 | }; |
1108 | |
1109 | static int smp_cpu_online(unsigned int cpu) |
1110 | { |
1111 | struct device *s = &per_cpu(cpu_device, cpu)->dev; |
1112 | |
1113 | return sysfs_create_group(kobj: &s->kobj, grp: &cpu_online_attr_group); |
1114 | } |
1115 | |
1116 | static int smp_cpu_pre_down(unsigned int cpu) |
1117 | { |
1118 | struct device *s = &per_cpu(cpu_device, cpu)->dev; |
1119 | |
1120 | sysfs_remove_group(kobj: &s->kobj, grp: &cpu_online_attr_group); |
1121 | return 0; |
1122 | } |
1123 | |
1124 | static int smp_add_present_cpu(int cpu) |
1125 | { |
1126 | struct device *s; |
1127 | struct cpu *c; |
1128 | int rc; |
1129 | |
1130 | c = kzalloc(size: sizeof(*c), GFP_KERNEL); |
1131 | if (!c) |
1132 | return -ENOMEM; |
1133 | per_cpu(cpu_device, cpu) = c; |
1134 | s = &c->dev; |
1135 | c->hotpluggable = !!cpu; |
1136 | rc = register_cpu(cpu: c, num: cpu); |
1137 | if (rc) |
1138 | goto out; |
1139 | rc = sysfs_create_group(kobj: &s->kobj, grp: &cpu_common_attr_group); |
1140 | if (rc) |
1141 | goto out_cpu; |
1142 | rc = topology_cpu_init(c); |
1143 | if (rc) |
1144 | goto out_topology; |
1145 | return 0; |
1146 | |
1147 | out_topology: |
1148 | sysfs_remove_group(kobj: &s->kobj, grp: &cpu_common_attr_group); |
1149 | out_cpu: |
1150 | unregister_cpu(cpu: c); |
1151 | out: |
1152 | return rc; |
1153 | } |
1154 | |
1155 | int __ref smp_rescan_cpus(void) |
1156 | { |
1157 | struct sclp_core_info *info; |
1158 | int nr; |
1159 | |
1160 | info = kzalloc(sizeof(*info), GFP_KERNEL); |
1161 | if (!info) |
1162 | return -ENOMEM; |
1163 | smp_get_core_info(info, early: 0); |
1164 | nr = __smp_rescan_cpus(info, early: false); |
1165 | kfree(objp: info); |
1166 | if (nr) |
1167 | topology_schedule_update(); |
1168 | return 0; |
1169 | } |
1170 | |
1171 | static ssize_t __ref rescan_store(struct device *dev, |
1172 | struct device_attribute *attr, |
1173 | const char *buf, |
1174 | size_t count) |
1175 | { |
1176 | int rc; |
1177 | |
1178 | rc = lock_device_hotplug_sysfs(); |
1179 | if (rc) |
1180 | return rc; |
1181 | rc = smp_rescan_cpus(); |
1182 | unlock_device_hotplug(); |
1183 | return rc ? rc : count; |
1184 | } |
1185 | static DEVICE_ATTR_WO(rescan); |
1186 | |
1187 | static int __init s390_smp_init(void) |
1188 | { |
1189 | struct device *dev_root; |
1190 | int cpu, rc = 0; |
1191 | |
1192 | dev_root = bus_get_dev_root(bus: &cpu_subsys); |
1193 | if (dev_root) { |
1194 | rc = device_create_file(device: dev_root, entry: &dev_attr_rescan); |
1195 | put_device(dev: dev_root); |
1196 | if (rc) |
1197 | return rc; |
1198 | } |
1199 | |
1200 | for_each_present_cpu(cpu) { |
1201 | rc = smp_add_present_cpu(cpu); |
1202 | if (rc) |
1203 | goto out; |
1204 | } |
1205 | |
1206 | rc = cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, name: "s390/smp:online" , |
1207 | startup: smp_cpu_online, teardown: smp_cpu_pre_down); |
1208 | rc = rc <= 0 ? rc : 0; |
1209 | out: |
1210 | return rc; |
1211 | } |
1212 | subsys_initcall(s390_smp_init); |
1213 | |