1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Machine check handler |
4 | * |
5 | * Copyright IBM Corp. 2000, 2009 |
6 | * Author(s): Ingo Adlung <adlung@de.ibm.com>, |
7 | * Martin Schwidefsky <schwidefsky@de.ibm.com>, |
8 | * Cornelia Huck <cornelia.huck@de.ibm.com>, |
9 | */ |
10 | |
11 | #include <linux/kernel_stat.h> |
12 | #include <linux/init.h> |
13 | #include <linux/errno.h> |
14 | #include <linux/entry-common.h> |
15 | #include <linux/hardirq.h> |
16 | #include <linux/log2.h> |
17 | #include <linux/kprobes.h> |
18 | #include <linux/kmemleak.h> |
19 | #include <linux/time.h> |
20 | #include <linux/module.h> |
21 | #include <linux/sched/signal.h> |
22 | #include <linux/kvm_host.h> |
23 | #include <linux/export.h> |
24 | #include <asm/lowcore.h> |
25 | #include <asm/ctlreg.h> |
26 | #include <asm/fpu.h> |
27 | #include <asm/smp.h> |
28 | #include <asm/stp.h> |
29 | #include <asm/cputime.h> |
30 | #include <asm/nmi.h> |
31 | #include <asm/crw.h> |
32 | #include <asm/asm-offsets.h> |
33 | #include <asm/pai.h> |
34 | |
35 | struct mcck_struct { |
36 | unsigned int kill_task : 1; |
37 | unsigned int channel_report : 1; |
38 | unsigned int warning : 1; |
39 | unsigned int stp_queue : 1; |
40 | unsigned long mcck_code; |
41 | }; |
42 | |
43 | static DEFINE_PER_CPU(struct mcck_struct, cpu_mcck); |
44 | |
45 | static inline int nmi_needs_mcesa(void) |
46 | { |
47 | return cpu_has_vx() || MACHINE_HAS_GS; |
48 | } |
49 | |
50 | /* |
51 | * The initial machine check extended save area for the boot CPU. |
52 | * It will be replaced on the boot CPU reinit with an allocated |
53 | * structure. The structure is required for machine check happening |
54 | * early in the boot process. |
55 | */ |
56 | static struct mcesa boot_mcesa __aligned(MCESA_MAX_SIZE); |
57 | |
58 | void __init nmi_alloc_mcesa_early(u64 *mcesad) |
59 | { |
60 | if (!nmi_needs_mcesa()) |
61 | return; |
62 | *mcesad = __pa(&boot_mcesa); |
63 | if (MACHINE_HAS_GS) |
64 | *mcesad |= ilog2(MCESA_MAX_SIZE); |
65 | } |
66 | |
67 | int nmi_alloc_mcesa(u64 *mcesad) |
68 | { |
69 | unsigned long size; |
70 | void *origin; |
71 | |
72 | *mcesad = 0; |
73 | if (!nmi_needs_mcesa()) |
74 | return 0; |
75 | size = MACHINE_HAS_GS ? MCESA_MAX_SIZE : MCESA_MIN_SIZE; |
76 | origin = kmalloc(size, GFP_KERNEL); |
77 | if (!origin) |
78 | return -ENOMEM; |
79 | /* The pointer is stored with mcesa_bits ORed in */ |
80 | kmemleak_not_leak(ptr: origin); |
81 | *mcesad = __pa(origin); |
82 | if (MACHINE_HAS_GS) |
83 | *mcesad |= ilog2(MCESA_MAX_SIZE); |
84 | return 0; |
85 | } |
86 | |
87 | void nmi_free_mcesa(u64 *mcesad) |
88 | { |
89 | if (!nmi_needs_mcesa()) |
90 | return; |
91 | kfree(__va(*mcesad & MCESA_ORIGIN_MASK)); |
92 | } |
93 | |
94 | static __always_inline char *nmi_puts(char *dest, const char *src) |
95 | { |
96 | while (*src) |
97 | *dest++ = *src++; |
98 | *dest = 0; |
99 | return dest; |
100 | } |
101 | |
102 | static __always_inline char *u64_to_hex(char *dest, u64 val) |
103 | { |
104 | int i, num; |
105 | |
106 | for (i = 1; i <= 16; i++) { |
107 | num = (val >> (64 - 4 * i)) & 0xf; |
108 | if (num >= 10) |
109 | *dest++ = 'A' + num - 10; |
110 | else |
111 | *dest++ = '0' + num; |
112 | } |
113 | *dest = 0; |
114 | return dest; |
115 | } |
116 | |
117 | static notrace void s390_handle_damage(void) |
118 | { |
119 | union ctlreg0 cr0, cr0_new; |
120 | char message[100]; |
121 | psw_t psw_save; |
122 | char *ptr; |
123 | |
124 | smp_emergency_stop(); |
125 | diag_amode31_ops.diag308_reset(); |
126 | ptr = nmi_puts(dest: message, src: "System stopped due to unrecoverable machine check, code: 0x" ); |
127 | u64_to_hex(dest: ptr, val: S390_lowcore.mcck_interruption_code); |
128 | |
129 | /* |
130 | * Disable low address protection and make machine check new PSW a |
131 | * disabled wait PSW. Any additional machine check cannot be handled. |
132 | */ |
133 | local_ctl_store(0, &cr0.reg); |
134 | cr0_new = cr0; |
135 | cr0_new.lap = 0; |
136 | local_ctl_load(0, &cr0_new.reg); |
137 | psw_save = S390_lowcore.mcck_new_psw; |
138 | psw_bits(S390_lowcore.mcck_new_psw).io = 0; |
139 | psw_bits(S390_lowcore.mcck_new_psw).ext = 0; |
140 | psw_bits(S390_lowcore.mcck_new_psw).wait = 1; |
141 | sclp_emergency_printk(message); |
142 | |
143 | /* |
144 | * Restore machine check new PSW and control register 0 to original |
145 | * values. This makes possible system dump analysis easier. |
146 | */ |
147 | S390_lowcore.mcck_new_psw = psw_save; |
148 | local_ctl_load(0, &cr0.reg); |
149 | disabled_wait(); |
150 | while (1); |
151 | } |
152 | NOKPROBE_SYMBOL(s390_handle_damage); |
153 | |
154 | /* |
155 | * Main machine check handler function. Will be called with interrupts disabled |
156 | * and machine checks enabled. |
157 | */ |
158 | void s390_handle_mcck(void) |
159 | { |
160 | struct mcck_struct mcck; |
161 | unsigned long mflags; |
162 | |
163 | /* |
164 | * Disable machine checks and get the current state of accumulated |
165 | * machine checks. Afterwards delete the old state and enable machine |
166 | * checks again. |
167 | */ |
168 | local_mcck_save(mflags); |
169 | mcck = *this_cpu_ptr(&cpu_mcck); |
170 | memset(this_cpu_ptr(&cpu_mcck), 0, sizeof(mcck)); |
171 | local_mcck_restore(mflags); |
172 | |
173 | if (mcck.channel_report) |
174 | crw_handle_channel_report(); |
175 | /* |
176 | * A warning may remain for a prolonged period on the bare iron. |
177 | * (actually until the machine is powered off, or the problem is gone) |
178 | * So we just stop listening for the WARNING MCH and avoid continuously |
179 | * being interrupted. One caveat is however, that we must do this per |
180 | * processor and cannot use the smp version of ctl_clear_bit(). |
181 | * On VM we only get one interrupt per virtally presented machinecheck. |
182 | * Though one suffices, we may get one interrupt per (virtual) cpu. |
183 | */ |
184 | if (mcck.warning) { /* WARNING pending ? */ |
185 | static int mchchk_wng_posted = 0; |
186 | |
187 | /* Use single cpu clear, as we cannot handle smp here. */ |
188 | local_ctl_clear_bit(14, CR14_WARNING_SUBMASK_BIT); |
189 | if (xchg(&mchchk_wng_posted, 1) == 0) |
190 | kill_cad_pid(SIGPWR, priv: 1); |
191 | } |
192 | if (mcck.stp_queue) |
193 | stp_queue_work(); |
194 | if (mcck.kill_task) { |
195 | printk(KERN_EMERG "mcck: Terminating task because of machine " |
196 | "malfunction (code 0x%016lx).\n" , mcck.mcck_code); |
197 | printk(KERN_EMERG "mcck: task: %s, pid: %d.\n" , |
198 | current->comm, current->pid); |
199 | if (is_global_init(current)) |
200 | panic(fmt: "mcck: Attempting to kill init!\n" ); |
201 | do_send_sig_info(SIGKILL, SEND_SIG_PRIV, current, type: PIDTYPE_PID); |
202 | } |
203 | } |
204 | |
205 | /** |
206 | * nmi_registers_valid - verify if registers are valid |
207 | * @mci: machine check interruption code |
208 | * |
209 | * Inspect a machine check interruption code and verify if all required |
210 | * registers are valid. For some registers the corresponding validity bit is |
211 | * ignored and the registers are set to the expected value. |
212 | * Returns true if all registers are valid, otherwise false. |
213 | */ |
214 | static bool notrace nmi_registers_valid(union mci mci) |
215 | { |
216 | union ctlreg2 cr2; |
217 | |
218 | /* |
219 | * The getcpu vdso syscall reads the CPU number from the programmable |
220 | * field of the TOD clock. Disregard the TOD programmable register |
221 | * validity bit and load the CPU number into the TOD programmable field |
222 | * unconditionally. |
223 | */ |
224 | set_tod_programmable_field(raw_smp_processor_id()); |
225 | /* |
226 | * Set the clock comparator register to the next expected value. |
227 | */ |
228 | set_clock_comparator(S390_lowcore.clock_comparator); |
229 | if (!mci.gr || !mci.fp || !mci.fc) |
230 | return false; |
231 | /* |
232 | * The vector validity must only be checked if not running a |
233 | * KVM guest. For KVM guests the machine check is forwarded by |
234 | * KVM and it is the responsibility of the guest to take |
235 | * appropriate actions. The host vector or FPU values have been |
236 | * saved by KVM and will be restored by KVM. |
237 | */ |
238 | if (!mci.vr && !test_cpu_flag(CIF_MCCK_GUEST)) |
239 | return false; |
240 | if (!mci.ar) |
241 | return false; |
242 | /* |
243 | * Two cases for guarded storage registers: |
244 | * - machine check in kernel or userspace |
245 | * - machine check while running SIE (KVM guest) |
246 | * For kernel or userspace the userspace values of guarded storage |
247 | * control can not be recreated, the process must be terminated. |
248 | * For SIE the guest values of guarded storage can not be recreated. |
249 | * This is either due to a bug or due to GS being disabled in the |
250 | * guest. The guest will be notified by KVM code and the guests machine |
251 | * check handling must take care of this. The host values are saved by |
252 | * KVM and are not affected. |
253 | */ |
254 | cr2.reg = S390_lowcore.cregs_save_area[2]; |
255 | if (cr2.gse && !mci.gs && !test_cpu_flag(CIF_MCCK_GUEST)) |
256 | return false; |
257 | if (!mci.ms || !mci.pm || !mci.ia) |
258 | return false; |
259 | return true; |
260 | } |
261 | NOKPROBE_SYMBOL(nmi_registers_valid); |
262 | |
263 | /* |
264 | * Backup the guest's machine check info to its description block |
265 | */ |
266 | static void notrace s390_backup_mcck_info(struct pt_regs *regs) |
267 | { |
268 | struct mcck_volatile_info *mcck_backup; |
269 | struct sie_page *sie_page; |
270 | |
271 | /* r14 contains the sie block, which was set in sie64a */ |
272 | struct kvm_s390_sie_block *sie_block = phys_to_virt(address: regs->gprs[14]); |
273 | |
274 | if (sie_block == NULL) |
275 | /* Something's seriously wrong, stop system. */ |
276 | s390_handle_damage(); |
277 | |
278 | sie_page = container_of(sie_block, struct sie_page, sie_block); |
279 | mcck_backup = &sie_page->mcck_info; |
280 | mcck_backup->mcic = S390_lowcore.mcck_interruption_code & |
281 | ~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE); |
282 | mcck_backup->ext_damage_code = S390_lowcore.external_damage_code; |
283 | mcck_backup->failing_storage_address |
284 | = S390_lowcore.failing_storage_address; |
285 | } |
286 | NOKPROBE_SYMBOL(s390_backup_mcck_info); |
287 | |
288 | #define MAX_IPD_COUNT 29 |
289 | #define MAX_IPD_TIME (5 * 60 * USEC_PER_SEC) /* 5 minutes */ |
290 | |
291 | #define ED_STP_ISLAND 6 /* External damage STP island check */ |
292 | #define ED_STP_SYNC 7 /* External damage STP sync check */ |
293 | |
294 | #define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE) |
295 | |
296 | /* |
297 | * machine check handler. |
298 | */ |
299 | void notrace s390_do_machine_check(struct pt_regs *regs) |
300 | { |
301 | static int ipd_count; |
302 | static DEFINE_SPINLOCK(ipd_lock); |
303 | static unsigned long long last_ipd; |
304 | struct mcck_struct *mcck; |
305 | unsigned long long tmp; |
306 | irqentry_state_t irq_state; |
307 | union mci mci; |
308 | unsigned long mcck_dam_code; |
309 | int mcck_pending = 0; |
310 | |
311 | irq_state = irqentry_nmi_enter(regs); |
312 | |
313 | if (user_mode(regs)) |
314 | update_timer_mcck(); |
315 | inc_irq_stat(NMI_NMI); |
316 | mci.val = S390_lowcore.mcck_interruption_code; |
317 | mcck = this_cpu_ptr(&cpu_mcck); |
318 | |
319 | /* |
320 | * Reinject the instruction processing damages' machine checks |
321 | * including Delayed Access Exception into the guest |
322 | * instead of damaging the host if they happen in the guest. |
323 | */ |
324 | if (mci.pd && !test_cpu_flag(CIF_MCCK_GUEST)) { |
325 | if (mci.b) { |
326 | /* Processing backup -> verify if we can survive this */ |
327 | u64 z_mcic, o_mcic, t_mcic; |
328 | z_mcic = (1ULL<<63 | 1ULL<<59 | 1ULL<<29); |
329 | o_mcic = (1ULL<<43 | 1ULL<<42 | 1ULL<<41 | 1ULL<<40 | |
330 | 1ULL<<36 | 1ULL<<35 | 1ULL<<34 | 1ULL<<32 | |
331 | 1ULL<<30 | 1ULL<<21 | 1ULL<<20 | 1ULL<<17 | |
332 | 1ULL<<16); |
333 | t_mcic = mci.val; |
334 | |
335 | if (((t_mcic & z_mcic) != 0) || |
336 | ((t_mcic & o_mcic) != o_mcic)) { |
337 | s390_handle_damage(); |
338 | } |
339 | |
340 | /* |
341 | * Nullifying exigent condition, therefore we might |
342 | * retry this instruction. |
343 | */ |
344 | spin_lock(lock: &ipd_lock); |
345 | tmp = get_tod_clock(); |
346 | if (((tmp - last_ipd) >> 12) < MAX_IPD_TIME) |
347 | ipd_count++; |
348 | else |
349 | ipd_count = 1; |
350 | last_ipd = tmp; |
351 | if (ipd_count == MAX_IPD_COUNT) |
352 | s390_handle_damage(); |
353 | spin_unlock(lock: &ipd_lock); |
354 | } else { |
355 | /* Processing damage -> stopping machine */ |
356 | s390_handle_damage(); |
357 | } |
358 | } |
359 | if (!nmi_registers_valid(mci: mci)) { |
360 | if (!user_mode(regs)) |
361 | s390_handle_damage(); |
362 | /* |
363 | * Couldn't restore all register contents for the |
364 | * user space process -> mark task for termination. |
365 | */ |
366 | mcck->kill_task = 1; |
367 | mcck->mcck_code = mci.val; |
368 | mcck_pending = 1; |
369 | } |
370 | |
371 | /* |
372 | * Backup the machine check's info if it happens when the guest |
373 | * is running. |
374 | */ |
375 | if (test_cpu_flag(CIF_MCCK_GUEST)) |
376 | s390_backup_mcck_info(regs); |
377 | |
378 | if (mci.cd) { |
379 | /* Timing facility damage */ |
380 | s390_handle_damage(); |
381 | } |
382 | if (mci.ed && mci.ec) { |
383 | /* External damage */ |
384 | if (S390_lowcore.external_damage_code & (1U << ED_STP_SYNC)) |
385 | mcck->stp_queue |= stp_sync_check(); |
386 | if (S390_lowcore.external_damage_code & (1U << ED_STP_ISLAND)) |
387 | mcck->stp_queue |= stp_island_check(); |
388 | mcck_pending = 1; |
389 | } |
390 | /* |
391 | * Reinject storage related machine checks into the guest if they |
392 | * happen when the guest is running. |
393 | */ |
394 | if (!test_cpu_flag(CIF_MCCK_GUEST)) { |
395 | /* Storage error uncorrected */ |
396 | if (mci.se) |
397 | s390_handle_damage(); |
398 | /* Storage key-error uncorrected */ |
399 | if (mci.ke) |
400 | s390_handle_damage(); |
401 | /* Storage degradation */ |
402 | if (mci.ds && mci.fa) |
403 | s390_handle_damage(); |
404 | } |
405 | if (mci.cp) { |
406 | /* Channel report word pending */ |
407 | mcck->channel_report = 1; |
408 | mcck_pending = 1; |
409 | } |
410 | if (mci.w) { |
411 | /* Warning pending */ |
412 | mcck->warning = 1; |
413 | mcck_pending = 1; |
414 | } |
415 | |
416 | /* |
417 | * If there are only Channel Report Pending and External Damage |
418 | * machine checks, they will not be reinjected into the guest |
419 | * because they refer to host conditions only. |
420 | */ |
421 | mcck_dam_code = (mci.val & MCIC_SUBCLASS_MASK); |
422 | if (test_cpu_flag(CIF_MCCK_GUEST) && |
423 | (mcck_dam_code & MCCK_CODE_NO_GUEST) != mcck_dam_code) { |
424 | /* Set exit reason code for host's later handling */ |
425 | *((long *)(regs->gprs[15] + __SF_SIE_REASON)) = -EINTR; |
426 | } |
427 | clear_cpu_flag(CIF_MCCK_GUEST); |
428 | |
429 | if (mcck_pending) |
430 | schedule_mcck_handler(); |
431 | |
432 | irqentry_nmi_exit(regs, irq_state); |
433 | } |
434 | NOKPROBE_SYMBOL(s390_do_machine_check); |
435 | |
436 | static int __init machine_check_init(void) |
437 | { |
438 | system_ctl_set_bit(14, CR14_EXTERNAL_DAMAGE_SUBMASK_BIT); |
439 | system_ctl_set_bit(14, CR14_RECOVERY_SUBMASK_BIT); |
440 | system_ctl_set_bit(14, CR14_WARNING_SUBMASK_BIT); |
441 | return 0; |
442 | } |
443 | early_initcall(machine_check_init); |
444 | |