1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ASM_X86_SYNC_CORE_H |
3 | #define _ASM_X86_SYNC_CORE_H |
4 | |
5 | #include <linux/preempt.h> |
6 | #include <asm/processor.h> |
7 | #include <asm/cpufeature.h> |
8 | #include <asm/special_insns.h> |
9 | |
10 | #ifdef CONFIG_X86_32 |
11 | static inline void iret_to_self(void) |
12 | { |
13 | asm volatile ( |
14 | "pushfl\n\t" |
15 | "pushl %%cs\n\t" |
16 | "pushl $1f\n\t" |
17 | "iret\n\t" |
18 | "1:" |
19 | : ASM_CALL_CONSTRAINT : : "memory" ); |
20 | } |
21 | #else |
22 | static inline void iret_to_self(void) |
23 | { |
24 | unsigned int tmp; |
25 | |
26 | asm volatile ( |
27 | "mov %%ss, %0\n\t" |
28 | "pushq %q0\n\t" |
29 | "pushq %%rsp\n\t" |
30 | "addq $8, (%%rsp)\n\t" |
31 | "pushfq\n\t" |
32 | "mov %%cs, %0\n\t" |
33 | "pushq %q0\n\t" |
34 | "pushq $1f\n\t" |
35 | "iretq\n\t" |
36 | "1:" |
37 | : "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc" , "memory" ); |
38 | } |
39 | #endif /* CONFIG_X86_32 */ |
40 | |
41 | /* |
42 | * This function forces the icache and prefetched instruction stream to |
43 | * catch up with reality in two very specific cases: |
44 | * |
45 | * a) Text was modified using one virtual address and is about to be executed |
46 | * from the same physical page at a different virtual address. |
47 | * |
48 | * b) Text was modified on a different CPU, may subsequently be |
49 | * executed on this CPU, and you want to make sure the new version |
50 | * gets executed. This generally means you're calling this in an IPI. |
51 | * |
52 | * If you're calling this for a different reason, you're probably doing |
53 | * it wrong. |
54 | * |
55 | * Like all of Linux's memory ordering operations, this is a |
56 | * compiler barrier as well. |
57 | */ |
58 | static inline void sync_core(void) |
59 | { |
60 | /* |
61 | * The SERIALIZE instruction is the most straightforward way to |
62 | * do this, but it is not universally available. |
63 | */ |
64 | if (static_cpu_has(X86_FEATURE_SERIALIZE)) { |
65 | serialize(); |
66 | return; |
67 | } |
68 | |
69 | /* |
70 | * For all other processors, there are quite a few ways to do this. |
71 | * IRET-to-self is nice because it works on every CPU, at any CPL |
72 | * (so it's compatible with paravirtualization), and it never exits |
73 | * to a hypervisor. The only downsides are that it's a bit slow |
74 | * (it seems to be a bit more than 2x slower than the fastest |
75 | * options) and that it unmasks NMIs. The "push %cs" is needed, |
76 | * because in paravirtual environments __KERNEL_CS may not be a |
77 | * valid CS value when we do IRET directly. |
78 | * |
79 | * In case NMI unmasking or performance ever becomes a problem, |
80 | * the next best option appears to be MOV-to-CR2 and an |
81 | * unconditional jump. That sequence also works on all CPUs, |
82 | * but it will fault at CPL3 (i.e. Xen PV). |
83 | * |
84 | * CPUID is the conventional way, but it's nasty: it doesn't |
85 | * exist on some 486-like CPUs, and it usually exits to a |
86 | * hypervisor. |
87 | */ |
88 | iret_to_self(); |
89 | } |
90 | |
91 | /* |
92 | * Ensure that a core serializing instruction is issued before returning |
93 | * to user-mode. x86 implements return to user-space through sysexit, |
94 | * sysrel, and sysretq, which are not core serializing. |
95 | */ |
96 | static inline void sync_core_before_usermode(void) |
97 | { |
98 | /* With PTI, we unconditionally serialize before running user code. */ |
99 | if (static_cpu_has(X86_FEATURE_PTI)) |
100 | return; |
101 | |
102 | /* |
103 | * Even if we're in an interrupt, we might reschedule before returning, |
104 | * in which case we could switch to a different thread in the same mm |
105 | * and return using SYSRET or SYSEXIT. Instead of trying to keep |
106 | * track of our need to sync the core, just sync right away. |
107 | */ |
108 | sync_core(); |
109 | } |
110 | |
111 | #endif /* _ASM_X86_SYNC_CORE_H */ |
112 | |