| 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
|---|---|
| 2 | #ifndef _ASM_X86_ENTRY_COMMON_H |
| 3 | #define _ASM_X86_ENTRY_COMMON_H |
| 4 | |
| 5 | #include <linux/randomize_kstack.h> |
| 6 | #include <linux/user-return-notifier.h> |
| 7 | |
| 8 | #include <asm/nospec-branch.h> |
| 9 | #include <asm/io_bitmap.h> |
| 10 | #include <asm/fpu/api.h> |
| 11 | |
| 12 | /* Check that the stack and regs on entry from user mode are sane. */ |
| 13 | static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs) |
| 14 | { |
| 15 | if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) { |
| 16 | /* |
| 17 | * Make sure that the entry code gave us a sensible EFLAGS |
| 18 | * register. Native because we want to check the actual CPU |
| 19 | * state, not the interrupt state as imagined by Xen. |
| 20 | */ |
| 21 | unsigned long flags = native_save_fl(); |
| 22 | unsigned long mask = X86_EFLAGS_DF | X86_EFLAGS_NT; |
| 23 | |
| 24 | /* |
| 25 | * For !SMAP hardware we patch out CLAC on entry. |
| 26 | */ |
| 27 | if (cpu_feature_enabled(X86_FEATURE_SMAP) || |
| 28 | cpu_feature_enabled(X86_FEATURE_XENPV)) |
| 29 | mask |= X86_EFLAGS_AC; |
| 30 | |
| 31 | WARN_ON_ONCE(flags & mask); |
| 32 | |
| 33 | /* We think we came from user mode. Make sure pt_regs agrees. */ |
| 34 | WARN_ON_ONCE(!user_mode(regs)); |
| 35 | |
| 36 | /* |
| 37 | * All entries from user mode (except #DF) should be on the |
| 38 | * normal thread stack and should have user pt_regs in the |
| 39 | * correct location. |
| 40 | */ |
| 41 | WARN_ON_ONCE(!on_thread_stack()); |
| 42 | WARN_ON_ONCE(regs != task_pt_regs(current)); |
| 43 | } |
| 44 | } |
| 45 | #define arch_enter_from_user_mode arch_enter_from_user_mode |
| 46 | |
| 47 | static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, |
| 48 | unsigned long ti_work) |
| 49 | { |
| 50 | if (ti_work & _TIF_USER_RETURN_NOTIFY) |
| 51 | fire_user_return_notifiers(); |
| 52 | |
| 53 | if (unlikely(ti_work & _TIF_IO_BITMAP)) |
| 54 | tss_update_io_bitmap(); |
| 55 | |
| 56 | fpregs_assert_state_consistent(); |
| 57 | if (unlikely(ti_work & _TIF_NEED_FPU_LOAD)) |
| 58 | switch_fpu_return(); |
| 59 | |
| 60 | #ifdef CONFIG_COMPAT |
| 61 | /* |
| 62 | * Compat syscalls set TS_COMPAT. Make sure we clear it before |
| 63 | * returning to user mode. We need to clear it *after* signal |
| 64 | * handling, because syscall restart has a fixup for compat |
| 65 | * syscalls. The fixup is exercised by the ptrace_syscall_32 |
| 66 | * selftest. |
| 67 | * |
| 68 | * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer |
| 69 | * special case only applies after poking regs and before the |
| 70 | * very next return to user mode. |
| 71 | */ |
| 72 | current_thread_info()->status &= ~(TS_COMPAT | TS_I386_REGS_POKED); |
| 73 | #endif |
| 74 | |
| 75 | /* |
| 76 | * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(), |
| 77 | * but not enough for x86 stack utilization comfort. To keep |
| 78 | * reasonable stack head room, reduce the maximum offset to 8 bits. |
| 79 | * |
| 80 | * The actual entropy will be further reduced by the compiler when |
| 81 | * applying stack alignment constraints (see cc_stack_align4/8 in |
| 82 | * arch/x86/Makefile), which will remove the 3 (x86_64) or 2 (ia32) |
| 83 | * low bits from any entropy chosen here. |
| 84 | * |
| 85 | * Therefore, final stack offset entropy will be 5 (x86_64) or |
| 86 | * 6 (ia32) bits. |
| 87 | */ |
| 88 | choose_random_kstack_offset(rdtsc() & 0xFF); |
| 89 | } |
| 90 | #define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare |
| 91 | |
| 92 | static __always_inline void arch_exit_to_user_mode(void) |
| 93 | { |
| 94 | mds_user_clear_cpu_buffers(); |
| 95 | amd_clear_divider(); |
| 96 | } |
| 97 | #define arch_exit_to_user_mode arch_exit_to_user_mode |
| 98 | |
| 99 | #endif |
| 100 |