1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/mm.h> |
3 | #include <linux/sched.h> |
4 | #include <linux/sched/debug.h> |
5 | #include <linux/init_task.h> |
6 | #include <linux/fs.h> |
7 | |
8 | #include <linux/uaccess.h> |
9 | #include <asm/processor.h> |
10 | #include <asm/desc.h> |
11 | #include <asm/traps.h> |
12 | #include <asm/doublefault.h> |
13 | |
14 | #define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM) |
15 | |
16 | #define TSS(x) this_cpu_read(cpu_tss_rw.x86_tss.x) |
17 | |
18 | static void set_df_gdt_entry(unsigned int cpu); |
19 | |
20 | /* |
21 | * Called by double_fault with CR0.TS and EFLAGS.NT cleared. The CPU thinks |
22 | * we're running the doublefault task. Cannot return. |
23 | */ |
24 | asmlinkage noinstr void __noreturn doublefault_shim(void) |
25 | { |
26 | unsigned long cr2; |
27 | struct pt_regs regs; |
28 | |
29 | BUILD_BUG_ON(sizeof(struct doublefault_stack) != PAGE_SIZE); |
30 | |
31 | cr2 = native_read_cr2(); |
32 | |
33 | /* Reset back to the normal kernel task. */ |
34 | force_reload_TR(); |
35 | set_df_gdt_entry(smp_processor_id()); |
36 | |
37 | trace_hardirqs_off(); |
38 | |
39 | /* |
40 | * Fill in pt_regs. A downside of doing this in C is that the unwinder |
41 | * won't see it (no ENCODE_FRAME_POINTER), so a nested stack dump |
42 | * won't successfully unwind to the source of the double fault. |
43 | * The main dump from exc_double_fault() is fine, though, since it |
44 | * uses these regs directly. |
45 | * |
46 | * If anyone ever cares, this could be moved to asm. |
47 | */ |
48 | regs.ss = TSS(ss); |
49 | regs.__ssh = 0; |
50 | regs.sp = TSS(sp); |
51 | regs.flags = TSS(flags); |
52 | regs.cs = TSS(cs); |
53 | /* We won't go through the entry asm, so we can leave __csh as 0. */ |
54 | regs.__csh = 0; |
55 | regs.ip = TSS(ip); |
56 | regs.orig_ax = 0; |
57 | regs.gs = TSS(gs); |
58 | regs.__gsh = 0; |
59 | regs.fs = TSS(fs); |
60 | regs.__fsh = 0; |
61 | regs.es = TSS(es); |
62 | regs.__esh = 0; |
63 | regs.ds = TSS(ds); |
64 | regs.__dsh = 0; |
65 | regs.ax = TSS(ax); |
66 | regs.bp = TSS(bp); |
67 | regs.di = TSS(di); |
68 | regs.si = TSS(si); |
69 | regs.dx = TSS(dx); |
70 | regs.cx = TSS(cx); |
71 | regs.bx = TSS(bx); |
72 | |
73 | exc_double_fault(®s, 0, cr2); |
74 | |
75 | /* |
76 | * x86_32 does not save the original CR3 anywhere on a task switch. |
77 | * This means that, even if we wanted to return, we would need to find |
78 | * some way to reconstruct CR3. We could make a credible guess based |
79 | * on cpu_tlbstate, but that would be racy and would not account for |
80 | * PTI. |
81 | */ |
82 | panic(fmt: "cannot return from double fault\n" ); |
83 | } |
84 | |
85 | DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = { |
86 | .tss = { |
87 | /* |
88 | * No sp0 or ss0 -- we never run CPL != 0 with this TSS |
89 | * active. sp is filled in later. |
90 | */ |
91 | .ldt = 0, |
92 | .io_bitmap_base = IO_BITMAP_OFFSET_INVALID, |
93 | |
94 | .ip = (unsigned long) asm_exc_double_fault, |
95 | .flags = X86_EFLAGS_FIXED, |
96 | .es = __USER_DS, |
97 | .cs = __KERNEL_CS, |
98 | .ss = __KERNEL_DS, |
99 | .ds = __USER_DS, |
100 | .fs = __KERNEL_PERCPU, |
101 | .gs = 0, |
102 | |
103 | .__cr3 = __pa_nodebug(swapper_pg_dir), |
104 | }, |
105 | }; |
106 | |
107 | static void set_df_gdt_entry(unsigned int cpu) |
108 | { |
109 | /* Set up doublefault TSS pointer in the GDT */ |
110 | __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, |
111 | &get_cpu_entry_area(cpu)->doublefault_stack.tss); |
112 | |
113 | } |
114 | |
115 | void doublefault_init_cpu_tss(void) |
116 | { |
117 | unsigned int cpu = smp_processor_id(); |
118 | struct cpu_entry_area *cea = get_cpu_entry_area(cpu); |
119 | |
120 | /* |
121 | * The linker isn't smart enough to initialize percpu variables that |
122 | * point to other places in percpu space. |
123 | */ |
124 | this_cpu_write(doublefault_stack.tss.sp, |
125 | (unsigned long)&cea->doublefault_stack.stack + |
126 | sizeof(doublefault_stack.stack)); |
127 | |
128 | set_df_gdt_entry(cpu); |
129 | } |
130 | |