1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Exception handling code
4 *
5 * Copyright (C) 2019 ARM Ltd.
6 */
7
8#include <linux/context_tracking.h>
9#include <linux/irq-entry-common.h>
10#include <linux/kasan.h>
11#include <linux/linkage.h>
12#include <linux/livepatch.h>
13#include <linux/lockdep.h>
14#include <linux/ptrace.h>
15#include <linux/resume_user_mode.h>
16#include <linux/sched.h>
17#include <linux/sched/debug.h>
18#include <linux/thread_info.h>
19
20#include <asm/cpufeature.h>
21#include <asm/daifflags.h>
22#include <asm/esr.h>
23#include <asm/exception.h>
24#include <asm/irq_regs.h>
25#include <asm/kprobes.h>
26#include <asm/mmu.h>
27#include <asm/processor.h>
28#include <asm/sdei.h>
29#include <asm/stacktrace.h>
30#include <asm/sysreg.h>
31#include <asm/system_misc.h>
32
33/*
34 * Handle IRQ/context state management when entering from kernel mode.
35 * Before this function is called it is not safe to call regular kernel code,
36 * instrumentable code, or any code which may trigger an exception.
37 */
38static noinstr irqentry_state_t enter_from_kernel_mode(struct pt_regs *regs)
39{
40 irqentry_state_t state;
41
42 state = irqentry_enter(regs);
43 mte_check_tfsr_entry();
44 mte_disable_tco_entry(current);
45
46 return state;
47}
48
49/*
50 * Handle IRQ/context state management when exiting to kernel mode.
51 * After this function returns it is not safe to call regular kernel code,
52 * instrumentable code, or any code which may trigger an exception.
53 */
54static void noinstr exit_to_kernel_mode(struct pt_regs *regs,
55 irqentry_state_t state)
56{
57 mte_check_tfsr_exit();
58 irqentry_exit(regs, state);
59}
60
61/*
62 * Handle IRQ/context state management when entering from user mode.
63 * Before this function is called it is not safe to call regular kernel code,
64 * instrumentable code, or any code which may trigger an exception.
65 */
66static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs)
67{
68 enter_from_user_mode(regs);
69 mte_disable_tco_entry(current);
70}
71
72/*
73 * Handle IRQ/context state management when exiting to user mode.
74 * After this function returns it is not safe to call regular kernel code,
75 * instrumentable code, or any code which may trigger an exception.
76 */
77
78static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs)
79{
80 local_irq_disable();
81 exit_to_user_mode_prepare_legacy(regs);
82 local_daif_mask();
83 mte_check_tfsr_exit();
84 exit_to_user_mode();
85}
86
87asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs)
88{
89 arm64_exit_to_user_mode(regs);
90}
91
92/*
93 * Handle IRQ/context state management when entering a debug exception from
94 * kernel mode. Before this function is called it is not safe to call regular
95 * kernel code, instrumentable code, or any code which may trigger an exception.
96 */
97static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs)
98{
99 irqentry_state_t state;
100
101 state.lockdep = lockdep_hardirqs_enabled();
102
103 lockdep_hardirqs_off(CALLER_ADDR0);
104 ct_nmi_enter();
105
106 trace_hardirqs_off_finish();
107
108 return state;
109}
110
111/*
112 * Handle IRQ/context state management when exiting a debug exception from
113 * kernel mode. After this function returns it is not safe to call regular
114 * kernel code, instrumentable code, or any code which may trigger an exception.
115 */
116static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs,
117 irqentry_state_t state)
118{
119 if (state.lockdep) {
120 trace_hardirqs_on_prepare();
121 lockdep_hardirqs_on_prepare();
122 }
123
124 ct_nmi_exit();
125 if (state.lockdep)
126 lockdep_hardirqs_on(CALLER_ADDR0);
127}
128
129static void do_interrupt_handler(struct pt_regs *regs,
130 void (*handler)(struct pt_regs *))
131{
132 struct pt_regs *old_regs = set_irq_regs(regs);
133
134 if (on_thread_stack())
135 call_on_irq_stack(regs, handler);
136 else
137 handler(regs);
138
139 set_irq_regs(old_regs);
140}
141
142extern void (*handle_arch_irq)(struct pt_regs *);
143extern void (*handle_arch_fiq)(struct pt_regs *);
144
145static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
146 unsigned long esr)
147{
148 irqentry_nmi_enter(regs);
149
150 console_verbose();
151
152 pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n",
153 vector, smp_processor_id(), esr,
154 esr_get_class_string(esr));
155
156 __show_regs(regs);
157 panic(fmt: "Unhandled exception");
158}
159
160#define UNHANDLED(el, regsize, vector) \
161asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
162{ \
163 const char *desc = #regsize "-bit " #el " " #vector; \
164 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
165}
166
167#ifdef CONFIG_ARM64_ERRATUM_1463225
168static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
169
170static void cortex_a76_erratum_1463225_svc_handler(void)
171{
172 u64 reg, val;
173
174 if (!unlikely(test_thread_flag(TIF_SINGLESTEP)))
175 return;
176
177 if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225)))
178 return;
179
180 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1);
181 reg = read_sysreg(mdscr_el1);
182 val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE;
183 write_sysreg(val, mdscr_el1);
184 asm volatile("msr daifclr, #8");
185 isb();
186
187 /* We will have taken a single-step exception by this point */
188
189 write_sysreg(reg, mdscr_el1);
190 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0);
191}
192
193static __always_inline bool
194cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
195{
196 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa))
197 return false;
198
199 /*
200 * We've taken a dummy step exception from the kernel to ensure
201 * that interrupts are re-enabled on the syscall path. Return back
202 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions
203 * masked so that we can safely restore the mdscr and get on with
204 * handling the syscall.
205 */
206 regs->pstate |= PSR_D_BIT;
207 return true;
208}
209#else /* CONFIG_ARM64_ERRATUM_1463225 */
210static void cortex_a76_erratum_1463225_svc_handler(void) { }
211static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
212{
213 return false;
214}
215#endif /* CONFIG_ARM64_ERRATUM_1463225 */
216
217/*
218 * As per the ABI exit SME streaming mode and clear the SVE state not
219 * shared with FPSIMD on syscall entry.
220 */
221static inline void fpsimd_syscall_enter(void)
222{
223 /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */
224 if (system_supports_sme())
225 sme_smstop_sm();
226
227 /*
228 * The CPU is not in streaming mode. If non-streaming SVE is not
229 * supported, there is no SVE state that needs to be discarded.
230 */
231 if (!system_supports_sve())
232 return;
233
234 if (test_thread_flag(TIF_SVE)) {
235 unsigned int sve_vq_minus_one;
236
237 sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1;
238 sve_flush_live(true, sve_vq_minus_one);
239 }
240
241 /*
242 * Any live non-FPSIMD SVE state has been zeroed. Allow
243 * fpsimd_save_user_state() to lazily discard SVE state until either
244 * the live state is unbound or fpsimd_syscall_exit() is called.
245 */
246 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD);
247}
248
249static __always_inline void fpsimd_syscall_exit(void)
250{
251 if (!system_supports_sve())
252 return;
253
254 /*
255 * The current task's user FPSIMD/SVE/SME state is now bound to this
256 * CPU. The fpsimd_last_state.to_save value is either:
257 *
258 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU
259 * since fpsimd_syscall_enter().
260 *
261 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at
262 * any point.
263 *
264 * Reset this to FP_STATE_CURRENT to stop lazy discarding.
265 */
266 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT);
267}
268
269/*
270 * In debug exception context, we explicitly disable preemption despite
271 * having interrupts disabled.
272 * This serves two purposes: it makes it much less likely that we would
273 * accidentally schedule in exception context and it will force a warning
274 * if we somehow manage to schedule by accident.
275 */
276static void debug_exception_enter(struct pt_regs *regs)
277{
278 preempt_disable();
279
280 /* This code is a bit fragile. Test it. */
281 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work");
282}
283NOKPROBE_SYMBOL(debug_exception_enter);
284
285static void debug_exception_exit(struct pt_regs *regs)
286{
287 preempt_enable_no_resched();
288}
289NOKPROBE_SYMBOL(debug_exception_exit);
290
291UNHANDLED(el1t, 64, sync)
292UNHANDLED(el1t, 64, irq)
293UNHANDLED(el1t, 64, fiq)
294UNHANDLED(el1t, 64, error)
295
296static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
297{
298 unsigned long far = read_sysreg(far_el1);
299 irqentry_state_t state;
300
301 state = enter_from_kernel_mode(regs);
302 local_daif_inherit(regs);
303 do_mem_abort(far, esr, regs);
304 local_daif_mask();
305 exit_to_kernel_mode(regs, state);
306}
307
308static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
309{
310 unsigned long far = read_sysreg(far_el1);
311 irqentry_state_t state;
312
313 state = enter_from_kernel_mode(regs);
314 local_daif_inherit(regs);
315 do_sp_pc_abort(far, esr, regs);
316 local_daif_mask();
317 exit_to_kernel_mode(regs, state);
318}
319
320static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
321{
322 irqentry_state_t state;
323
324 state = enter_from_kernel_mode(regs);
325 local_daif_inherit(regs);
326 do_el1_undef(regs, esr);
327 local_daif_mask();
328 exit_to_kernel_mode(regs, state);
329}
330
331static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
332{
333 irqentry_state_t state;
334
335 state = enter_from_kernel_mode(regs);
336 local_daif_inherit(regs);
337 do_el1_bti(regs, esr);
338 local_daif_mask();
339 exit_to_kernel_mode(regs, state);
340}
341
342static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr)
343{
344 irqentry_state_t state;
345
346 state = enter_from_kernel_mode(regs);
347 local_daif_inherit(regs);
348 do_el1_gcs(regs, esr);
349 local_daif_mask();
350 exit_to_kernel_mode(regs, state);
351}
352
353static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr)
354{
355 irqentry_state_t state;
356
357 state = enter_from_kernel_mode(regs);
358 local_daif_inherit(regs);
359 do_el1_mops(regs, esr);
360 local_daif_mask();
361 exit_to_kernel_mode(regs, state);
362}
363
364static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr)
365{
366 irqentry_state_t state;
367
368 state = arm64_enter_el1_dbg(regs);
369 debug_exception_enter(regs);
370 do_breakpoint(esr, regs);
371 debug_exception_exit(regs);
372 arm64_exit_el1_dbg(regs, state);
373}
374
375static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr)
376{
377 irqentry_state_t state;
378
379 state = arm64_enter_el1_dbg(regs);
380 if (!cortex_a76_erratum_1463225_debug_handler(regs)) {
381 debug_exception_enter(regs);
382 /*
383 * After handling a breakpoint, we suspend the breakpoint
384 * and use single-step to move to the next instruction.
385 * If we are stepping a suspended breakpoint there's nothing more to do:
386 * the single-step is complete.
387 */
388 if (!try_step_suspended_breakpoints(regs))
389 do_el1_softstep(esr, regs);
390 debug_exception_exit(regs);
391 }
392 arm64_exit_el1_dbg(regs, state);
393}
394
395static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr)
396{
397 /* Watchpoints are the only debug exception to write FAR_EL1 */
398 unsigned long far = read_sysreg(far_el1);
399 irqentry_state_t state;
400
401 state = arm64_enter_el1_dbg(regs);
402 debug_exception_enter(regs);
403 do_watchpoint(far, esr, regs);
404 debug_exception_exit(regs);
405 arm64_exit_el1_dbg(regs, state);
406}
407
408static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr)
409{
410 irqentry_state_t state;
411
412 state = arm64_enter_el1_dbg(regs);
413 debug_exception_enter(regs);
414 do_el1_brk64(esr, regs);
415 debug_exception_exit(regs);
416 arm64_exit_el1_dbg(regs, state);
417}
418
419static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
420{
421 irqentry_state_t state;
422
423 state = enter_from_kernel_mode(regs);
424 local_daif_inherit(regs);
425 do_el1_fpac(regs, esr);
426 local_daif_mask();
427 exit_to_kernel_mode(regs, state);
428}
429
430asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
431{
432 unsigned long esr = read_sysreg(esr_el1);
433
434 switch (ESR_ELx_EC(esr)) {
435 case ESR_ELx_EC_DABT_CUR:
436 case ESR_ELx_EC_IABT_CUR:
437 el1_abort(regs, esr);
438 break;
439 /*
440 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a
441 * recursive exception when trying to push the initial pt_regs.
442 */
443 case ESR_ELx_EC_PC_ALIGN:
444 el1_pc(regs, esr);
445 break;
446 case ESR_ELx_EC_SYS64:
447 case ESR_ELx_EC_UNKNOWN:
448 el1_undef(regs, esr);
449 break;
450 case ESR_ELx_EC_BTI:
451 el1_bti(regs, esr);
452 break;
453 case ESR_ELx_EC_GCS:
454 el1_gcs(regs, esr);
455 break;
456 case ESR_ELx_EC_MOPS:
457 el1_mops(regs, esr);
458 break;
459 case ESR_ELx_EC_BREAKPT_CUR:
460 el1_breakpt(regs, esr);
461 break;
462 case ESR_ELx_EC_SOFTSTP_CUR:
463 el1_softstp(regs, esr);
464 break;
465 case ESR_ELx_EC_WATCHPT_CUR:
466 el1_watchpt(regs, esr);
467 break;
468 case ESR_ELx_EC_BRK64:
469 el1_brk64(regs, esr);
470 break;
471 case ESR_ELx_EC_FPAC:
472 el1_fpac(regs, esr);
473 break;
474 default:
475 __panic_unhandled(regs, vector: "64-bit el1h sync", esr);
476 }
477}
478
479static __always_inline void __el1_pnmi(struct pt_regs *regs,
480 void (*handler)(struct pt_regs *))
481{
482 irqentry_state_t state;
483
484 state = irqentry_nmi_enter(regs);
485 do_interrupt_handler(regs, handler);
486 irqentry_nmi_exit(regs, irq_state: state);
487}
488
489static __always_inline void __el1_irq(struct pt_regs *regs,
490 void (*handler)(struct pt_regs *))
491{
492 irqentry_state_t state;
493
494 state = enter_from_kernel_mode(regs);
495
496 irq_enter_rcu();
497 do_interrupt_handler(regs, handler);
498 irq_exit_rcu();
499
500 exit_to_kernel_mode(regs, state);
501}
502static void noinstr el1_interrupt(struct pt_regs *regs,
503 void (*handler)(struct pt_regs *))
504{
505 write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
506
507 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs))
508 __el1_pnmi(regs, handler);
509 else
510 __el1_irq(regs, handler);
511}
512
513asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
514{
515 el1_interrupt(regs, handler: handle_arch_irq);
516}
517
518asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
519{
520 el1_interrupt(regs, handler: handle_arch_fiq);
521}
522
523asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
524{
525 unsigned long esr = read_sysreg(esr_el1);
526 irqentry_state_t state;
527
528 local_daif_restore(DAIF_ERRCTX);
529 state = irqentry_nmi_enter(regs);
530 do_serror(regs, esr);
531 irqentry_nmi_exit(regs, irq_state: state);
532}
533
534static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
535{
536 unsigned long far = read_sysreg(far_el1);
537
538 arm64_enter_from_user_mode(regs);
539 local_daif_restore(DAIF_PROCCTX);
540 do_mem_abort(far, esr, regs);
541 arm64_exit_to_user_mode(regs);
542}
543
544static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
545{
546 unsigned long far = read_sysreg(far_el1);
547
548 /*
549 * We've taken an instruction abort from userspace and not yet
550 * re-enabled IRQs. If the address is a kernel address, apply
551 * BP hardening prior to enabling IRQs and pre-emption.
552 */
553 if (!is_ttbr0_addr(far))
554 arm64_apply_bp_hardening();
555
556 arm64_enter_from_user_mode(regs);
557 local_daif_restore(DAIF_PROCCTX);
558 do_mem_abort(far, esr, regs);
559 arm64_exit_to_user_mode(regs);
560}
561
562static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
563{
564 arm64_enter_from_user_mode(regs);
565 local_daif_restore(DAIF_PROCCTX);
566 do_fpsimd_acc(esr, regs);
567 arm64_exit_to_user_mode(regs);
568}
569
570static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
571{
572 arm64_enter_from_user_mode(regs);
573 local_daif_restore(DAIF_PROCCTX);
574 do_sve_acc(esr, regs);
575 arm64_exit_to_user_mode(regs);
576}
577
578static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr)
579{
580 arm64_enter_from_user_mode(regs);
581 local_daif_restore(DAIF_PROCCTX);
582 do_sme_acc(esr, regs);
583 arm64_exit_to_user_mode(regs);
584}
585
586static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
587{
588 arm64_enter_from_user_mode(regs);
589 local_daif_restore(DAIF_PROCCTX);
590 do_fpsimd_exc(esr, regs);
591 arm64_exit_to_user_mode(regs);
592}
593
594static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
595{
596 arm64_enter_from_user_mode(regs);
597 local_daif_restore(DAIF_PROCCTX);
598 do_el0_sys(esr, regs);
599 arm64_exit_to_user_mode(regs);
600}
601
602static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
603{
604 unsigned long far = read_sysreg(far_el1);
605
606 if (!is_ttbr0_addr(instruction_pointer(regs)))
607 arm64_apply_bp_hardening();
608
609 arm64_enter_from_user_mode(regs);
610 local_daif_restore(DAIF_PROCCTX);
611 do_sp_pc_abort(far, esr, regs);
612 arm64_exit_to_user_mode(regs);
613}
614
615static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
616{
617 arm64_enter_from_user_mode(regs);
618 local_daif_restore(DAIF_PROCCTX);
619 do_sp_pc_abort(regs->sp, esr, regs);
620 arm64_exit_to_user_mode(regs);
621}
622
623static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
624{
625 arm64_enter_from_user_mode(regs);
626 local_daif_restore(DAIF_PROCCTX);
627 do_el0_undef(regs, esr);
628 arm64_exit_to_user_mode(regs);
629}
630
631static void noinstr el0_bti(struct pt_regs *regs)
632{
633 arm64_enter_from_user_mode(regs);
634 local_daif_restore(DAIF_PROCCTX);
635 do_el0_bti(regs);
636 arm64_exit_to_user_mode(regs);
637}
638
639static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr)
640{
641 arm64_enter_from_user_mode(regs);
642 local_daif_restore(DAIF_PROCCTX);
643 do_el0_mops(regs, esr);
644 arm64_exit_to_user_mode(regs);
645}
646
647static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr)
648{
649 arm64_enter_from_user_mode(regs);
650 local_daif_restore(DAIF_PROCCTX);
651 do_el0_gcs(regs, esr);
652 arm64_exit_to_user_mode(regs);
653}
654
655static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
656{
657 arm64_enter_from_user_mode(regs);
658 local_daif_restore(DAIF_PROCCTX);
659 bad_el0_sync(regs, 0, esr);
660 arm64_exit_to_user_mode(regs);
661}
662
663static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr)
664{
665 if (!is_ttbr0_addr(regs->pc))
666 arm64_apply_bp_hardening();
667
668 arm64_enter_from_user_mode(regs);
669 debug_exception_enter(regs);
670 do_breakpoint(esr, regs);
671 debug_exception_exit(regs);
672 local_daif_restore(DAIF_PROCCTX);
673 arm64_exit_to_user_mode(regs);
674}
675
676static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr)
677{
678 bool step_done;
679
680 if (!is_ttbr0_addr(regs->pc))
681 arm64_apply_bp_hardening();
682
683 arm64_enter_from_user_mode(regs);
684 /*
685 * After handling a breakpoint, we suspend the breakpoint
686 * and use single-step to move to the next instruction.
687 * If we are stepping a suspended breakpoint there's nothing more to do:
688 * the single-step is complete.
689 */
690 step_done = try_step_suspended_breakpoints(regs);
691 local_daif_restore(DAIF_PROCCTX);
692 if (!step_done)
693 do_el0_softstep(esr, regs);
694 arm64_exit_to_user_mode(regs);
695}
696
697static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr)
698{
699 /* Watchpoints are the only debug exception to write FAR_EL1 */
700 unsigned long far = read_sysreg(far_el1);
701
702 arm64_enter_from_user_mode(regs);
703 debug_exception_enter(regs);
704 do_watchpoint(far, esr, regs);
705 debug_exception_exit(regs);
706 local_daif_restore(DAIF_PROCCTX);
707 arm64_exit_to_user_mode(regs);
708}
709
710static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr)
711{
712 arm64_enter_from_user_mode(regs);
713 local_daif_restore(DAIF_PROCCTX);
714 do_el0_brk64(esr, regs);
715 arm64_exit_to_user_mode(regs);
716}
717
718static void noinstr el0_svc(struct pt_regs *regs)
719{
720 arm64_enter_from_user_mode(regs);
721 cortex_a76_erratum_1463225_svc_handler();
722 fpsimd_syscall_enter();
723 local_daif_restore(DAIF_PROCCTX);
724 do_el0_svc(regs);
725 arm64_exit_to_user_mode(regs);
726 fpsimd_syscall_exit();
727}
728
729static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
730{
731 arm64_enter_from_user_mode(regs);
732 local_daif_restore(DAIF_PROCCTX);
733 do_el0_fpac(regs, esr);
734 arm64_exit_to_user_mode(regs);
735}
736
737asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
738{
739 unsigned long esr = read_sysreg(esr_el1);
740
741 switch (ESR_ELx_EC(esr)) {
742 case ESR_ELx_EC_SVC64:
743 el0_svc(regs);
744 break;
745 case ESR_ELx_EC_DABT_LOW:
746 el0_da(regs, esr);
747 break;
748 case ESR_ELx_EC_IABT_LOW:
749 el0_ia(regs, esr);
750 break;
751 case ESR_ELx_EC_FP_ASIMD:
752 el0_fpsimd_acc(regs, esr);
753 break;
754 case ESR_ELx_EC_SVE:
755 el0_sve_acc(regs, esr);
756 break;
757 case ESR_ELx_EC_SME:
758 el0_sme_acc(regs, esr);
759 break;
760 case ESR_ELx_EC_FP_EXC64:
761 el0_fpsimd_exc(regs, esr);
762 break;
763 case ESR_ELx_EC_SYS64:
764 case ESR_ELx_EC_WFx:
765 el0_sys(regs, esr);
766 break;
767 case ESR_ELx_EC_SP_ALIGN:
768 el0_sp(regs, esr);
769 break;
770 case ESR_ELx_EC_PC_ALIGN:
771 el0_pc(regs, esr);
772 break;
773 case ESR_ELx_EC_UNKNOWN:
774 el0_undef(regs, esr);
775 break;
776 case ESR_ELx_EC_BTI:
777 el0_bti(regs);
778 break;
779 case ESR_ELx_EC_MOPS:
780 el0_mops(regs, esr);
781 break;
782 case ESR_ELx_EC_GCS:
783 el0_gcs(regs, esr);
784 break;
785 case ESR_ELx_EC_BREAKPT_LOW:
786 el0_breakpt(regs, esr);
787 break;
788 case ESR_ELx_EC_SOFTSTP_LOW:
789 el0_softstp(regs, esr);
790 break;
791 case ESR_ELx_EC_WATCHPT_LOW:
792 el0_watchpt(regs, esr);
793 break;
794 case ESR_ELx_EC_BRK64:
795 el0_brk64(regs, esr);
796 break;
797 case ESR_ELx_EC_FPAC:
798 el0_fpac(regs, esr);
799 break;
800 default:
801 el0_inv(regs, esr);
802 }
803}
804
805static void noinstr el0_interrupt(struct pt_regs *regs,
806 void (*handler)(struct pt_regs *))
807{
808 arm64_enter_from_user_mode(regs);
809
810 write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
811
812 if (regs->pc & BIT(55))
813 arm64_apply_bp_hardening();
814
815 irq_enter_rcu();
816 do_interrupt_handler(regs, handler);
817 irq_exit_rcu();
818
819 arm64_exit_to_user_mode(regs);
820}
821
822static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
823{
824 el0_interrupt(regs, handler: handle_arch_irq);
825}
826
827asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
828{
829 __el0_irq_handler_common(regs);
830}
831
832static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
833{
834 el0_interrupt(regs, handler: handle_arch_fiq);
835}
836
837asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
838{
839 __el0_fiq_handler_common(regs);
840}
841
842static void noinstr __el0_error_handler_common(struct pt_regs *regs)
843{
844 unsigned long esr = read_sysreg(esr_el1);
845 irqentry_state_t state;
846
847 arm64_enter_from_user_mode(regs);
848 local_daif_restore(DAIF_ERRCTX);
849 state = irqentry_nmi_enter(regs);
850 do_serror(regs, esr);
851 irqentry_nmi_exit(regs, irq_state: state);
852 local_daif_restore(DAIF_PROCCTX);
853 arm64_exit_to_user_mode(regs);
854}
855
856asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
857{
858 __el0_error_handler_common(regs);
859}
860
861#ifdef CONFIG_COMPAT
862static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
863{
864 arm64_enter_from_user_mode(regs);
865 local_daif_restore(DAIF_PROCCTX);
866 do_el0_cp15(esr, regs);
867 arm64_exit_to_user_mode(regs);
868}
869
870static void noinstr el0_svc_compat(struct pt_regs *regs)
871{
872 arm64_enter_from_user_mode(regs);
873 cortex_a76_erratum_1463225_svc_handler();
874 local_daif_restore(DAIF_PROCCTX);
875 do_el0_svc_compat(regs);
876 arm64_exit_to_user_mode(regs);
877}
878
879static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr)
880{
881 arm64_enter_from_user_mode(regs);
882 local_daif_restore(DAIF_PROCCTX);
883 do_bkpt32(esr, regs);
884 arm64_exit_to_user_mode(regs);
885}
886
887asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
888{
889 unsigned long esr = read_sysreg(esr_el1);
890
891 switch (ESR_ELx_EC(esr)) {
892 case ESR_ELx_EC_SVC32:
893 el0_svc_compat(regs);
894 break;
895 case ESR_ELx_EC_DABT_LOW:
896 el0_da(regs, esr);
897 break;
898 case ESR_ELx_EC_IABT_LOW:
899 el0_ia(regs, esr);
900 break;
901 case ESR_ELx_EC_FP_ASIMD:
902 el0_fpsimd_acc(regs, esr);
903 break;
904 case ESR_ELx_EC_FP_EXC32:
905 el0_fpsimd_exc(regs, esr);
906 break;
907 case ESR_ELx_EC_PC_ALIGN:
908 el0_pc(regs, esr);
909 break;
910 case ESR_ELx_EC_UNKNOWN:
911 case ESR_ELx_EC_CP14_MR:
912 case ESR_ELx_EC_CP14_LS:
913 case ESR_ELx_EC_CP14_64:
914 el0_undef(regs, esr);
915 break;
916 case ESR_ELx_EC_CP15_32:
917 case ESR_ELx_EC_CP15_64:
918 el0_cp15(regs, esr);
919 break;
920 case ESR_ELx_EC_BREAKPT_LOW:
921 el0_breakpt(regs, esr);
922 break;
923 case ESR_ELx_EC_SOFTSTP_LOW:
924 el0_softstp(regs, esr);
925 break;
926 case ESR_ELx_EC_WATCHPT_LOW:
927 el0_watchpt(regs, esr);
928 break;
929 case ESR_ELx_EC_BKPT32:
930 el0_bkpt32(regs, esr);
931 break;
932 default:
933 el0_inv(regs, esr);
934 }
935}
936
937asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)
938{
939 __el0_irq_handler_common(regs);
940}
941
942asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)
943{
944 __el0_fiq_handler_common(regs);
945}
946
947asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)
948{
949 __el0_error_handler_common(regs);
950}
951#else /* CONFIG_COMPAT */
952UNHANDLED(el0t, 32, sync)
953UNHANDLED(el0t, 32, irq)
954UNHANDLED(el0t, 32, fiq)
955UNHANDLED(el0t, 32, error)
956#endif /* CONFIG_COMPAT */
957
958asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs)
959{
960 unsigned long esr = read_sysreg(esr_el1);
961 unsigned long far = read_sysreg(far_el1);
962
963 irqentry_nmi_enter(regs);
964 panic_bad_stack(regs, esr, far);
965}
966
967#ifdef CONFIG_ARM_SDE_INTERFACE
968asmlinkage noinstr unsigned long
969__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
970{
971 irqentry_state_t state;
972 unsigned long ret;
973
974 /*
975 * We didn't take an exception to get here, so the HW hasn't
976 * set/cleared bits in PSTATE that we may rely on.
977 *
978 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to
979 * whether PSTATE bits are inherited unchanged or generated from
980 * scratch, and the TF-A implementation always clears PAN and always
981 * clears UAO. There are no other known implementations.
982 *
983 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how
984 * PSTATE is modified upon architectural exceptions, and so PAN is
985 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always
986 * cleared.
987 *
988 * We must explicitly reset PAN to the expected state, including
989 * clearing it when the host isn't using it, in case a VM had it set.
990 */
991 if (system_uses_hw_pan())
992 set_pstate_pan(1);
993 else if (cpu_has_pan())
994 set_pstate_pan(0);
995
996 state = irqentry_nmi_enter(regs);
997 ret = do_sdei_event(regs, arg);
998 irqentry_nmi_exit(regs, state);
999
1000 return ret;
1001}
1002#endif /* CONFIG_ARM_SDE_INTERFACE */
1003

source code of linux/arch/arm64/kernel/entry-common.c