1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * relocate_kernel.S - put the kernel image in place to boot |
4 | * Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com> |
5 | */ |
6 | |
7 | #include <linux/linkage.h> |
8 | #include <asm/page_types.h> |
9 | #include <asm/kexec.h> |
10 | #include <asm/nospec-branch.h> |
11 | #include <asm/processor-flags.h> |
12 | |
13 | /* |
14 | * Must be relocatable PIC code callable as a C function, in particular |
15 | * there must be a plain RET and not jump to return thunk. |
16 | */ |
17 | |
18 | #define PTR(x) (x << 2) |
19 | |
20 | /* |
21 | * control_page + KEXEC_CONTROL_CODE_MAX_SIZE |
22 | * ~ control_page + PAGE_SIZE are used as data storage and stack for |
23 | * jumping back |
24 | */ |
25 | #define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset)) |
26 | |
27 | /* Minimal CPU state */ |
28 | #define ESP DATA(0x0) |
29 | #define CR0 DATA(0x4) |
30 | #define CR3 DATA(0x8) |
31 | #define CR4 DATA(0xc) |
32 | |
33 | /* other data */ |
34 | #define CP_VA_CONTROL_PAGE DATA(0x10) |
35 | #define CP_PA_PGD DATA(0x14) |
36 | #define CP_PA_SWAP_PAGE DATA(0x18) |
37 | #define CP_PA_BACKUP_PAGES_MAP DATA(0x1c) |
38 | |
39 | .text |
40 | SYM_CODE_START_NOALIGN(relocate_kernel) |
41 | /* Save the CPU context, used for jumping back */ |
42 | |
43 | pushl %ebx |
44 | pushl %esi |
45 | pushl %edi |
46 | pushl %ebp |
47 | pushf |
48 | |
49 | movl 20+8(%esp), %ebp /* list of pages */ |
50 | movl PTR(VA_CONTROL_PAGE)(%ebp), %edi |
51 | movl %esp, ESP(%edi) |
52 | movl %cr0, %eax |
53 | movl %eax, CR0(%edi) |
54 | movl %cr3, %eax |
55 | movl %eax, CR3(%edi) |
56 | movl %cr4, %eax |
57 | movl %eax, CR4(%edi) |
58 | |
59 | /* read the arguments and say goodbye to the stack */ |
60 | movl 20+4(%esp), %ebx /* page_list */ |
61 | movl 20+8(%esp), %ebp /* list of pages */ |
62 | movl 20+12(%esp), %edx /* start address */ |
63 | movl 20+16(%esp), %ecx /* cpu_has_pae */ |
64 | movl 20+20(%esp), %esi /* preserve_context */ |
65 | |
66 | /* zero out flags, and disable interrupts */ |
67 | pushl $0 |
68 | popfl |
69 | |
70 | /* save some information for jumping back */ |
71 | movl PTR(VA_CONTROL_PAGE)(%ebp), %edi |
72 | movl %edi, CP_VA_CONTROL_PAGE(%edi) |
73 | movl PTR(PA_PGD)(%ebp), %eax |
74 | movl %eax, CP_PA_PGD(%edi) |
75 | movl PTR(PA_SWAP_PAGE)(%ebp), %eax |
76 | movl %eax, CP_PA_SWAP_PAGE(%edi) |
77 | movl %ebx, CP_PA_BACKUP_PAGES_MAP(%edi) |
78 | |
79 | /* |
80 | * get physical address of control page now |
81 | * this is impossible after page table switch |
82 | */ |
83 | movl PTR(PA_CONTROL_PAGE)(%ebp), %edi |
84 | |
85 | /* switch to new set of page tables */ |
86 | movl PTR(PA_PGD)(%ebp), %eax |
87 | movl %eax, %cr3 |
88 | |
89 | /* setup a new stack at the end of the physical control page */ |
90 | lea PAGE_SIZE(%edi), %esp |
91 | |
92 | /* jump to identity mapped page */ |
93 | movl %edi, %eax |
94 | addl $(identity_mapped - relocate_kernel), %eax |
95 | pushl %eax |
96 | ANNOTATE_UNRET_SAFE |
97 | ret |
98 | int3 |
99 | SYM_CODE_END(relocate_kernel) |
100 | |
101 | SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) |
102 | /* set return address to 0 if not preserving context */ |
103 | pushl $0 |
104 | /* store the start address on the stack */ |
105 | pushl %edx |
106 | |
107 | /* |
108 | * Set cr0 to a known state: |
109 | * - Paging disabled |
110 | * - Alignment check disabled |
111 | * - Write protect disabled |
112 | * - No task switch |
113 | * - Don't do FP software emulation. |
114 | * - Protected mode enabled |
115 | */ |
116 | movl %cr0, %eax |
117 | andl $~(X86_CR0_PG | X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %eax |
118 | orl $(X86_CR0_PE), %eax |
119 | movl %eax, %cr0 |
120 | |
121 | /* clear cr4 if applicable */ |
122 | testl %ecx, %ecx |
123 | jz 1f |
124 | /* |
125 | * Set cr4 to a known state: |
126 | * Setting everything to zero seems safe. |
127 | */ |
128 | xorl %eax, %eax |
129 | movl %eax, %cr4 |
130 | |
131 | jmp 1f |
132 | 1: |
133 | |
134 | /* Flush the TLB (needed?) */ |
135 | xorl %eax, %eax |
136 | movl %eax, %cr3 |
137 | |
138 | movl CP_PA_SWAP_PAGE(%edi), %eax |
139 | pushl %eax |
140 | pushl %ebx |
141 | call swap_pages |
142 | addl $8, %esp |
143 | |
144 | /* |
145 | * To be certain of avoiding problems with self-modifying code |
146 | * I need to execute a serializing instruction here. |
147 | * So I flush the TLB, it's handy, and not processor dependent. |
148 | */ |
149 | xorl %eax, %eax |
150 | movl %eax, %cr3 |
151 | |
152 | /* |
153 | * set all of the registers to known values |
154 | * leave %esp alone |
155 | */ |
156 | |
157 | testl %esi, %esi |
158 | jnz 1f |
159 | xorl %edi, %edi |
160 | xorl %eax, %eax |
161 | xorl %ebx, %ebx |
162 | xorl %ecx, %ecx |
163 | xorl %edx, %edx |
164 | xorl %esi, %esi |
165 | xorl %ebp, %ebp |
166 | ANNOTATE_UNRET_SAFE |
167 | ret |
168 | int3 |
169 | 1: |
170 | popl %edx |
171 | movl CP_PA_SWAP_PAGE(%edi), %esp |
172 | addl $PAGE_SIZE, %esp |
173 | 2: |
174 | ANNOTATE_RETPOLINE_SAFE |
175 | call *%edx |
176 | |
177 | /* get the re-entry point of the peer system */ |
178 | movl 0(%esp), %ebp |
179 | call 1f |
180 | 1: |
181 | popl %ebx |
182 | subl $(1b - relocate_kernel), %ebx |
183 | movl CP_VA_CONTROL_PAGE(%ebx), %edi |
184 | lea PAGE_SIZE(%ebx), %esp |
185 | movl CP_PA_SWAP_PAGE(%ebx), %eax |
186 | movl CP_PA_BACKUP_PAGES_MAP(%ebx), %edx |
187 | pushl %eax |
188 | pushl %edx |
189 | call swap_pages |
190 | addl $8, %esp |
191 | movl CP_PA_PGD(%ebx), %eax |
192 | movl %eax, %cr3 |
193 | movl %cr0, %eax |
194 | orl $X86_CR0_PG, %eax |
195 | movl %eax, %cr0 |
196 | lea PAGE_SIZE(%edi), %esp |
197 | movl %edi, %eax |
198 | addl $(virtual_mapped - relocate_kernel), %eax |
199 | pushl %eax |
200 | ANNOTATE_UNRET_SAFE |
201 | ret |
202 | int3 |
203 | SYM_CODE_END(identity_mapped) |
204 | |
205 | SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) |
206 | movl CR4(%edi), %eax |
207 | movl %eax, %cr4 |
208 | movl CR3(%edi), %eax |
209 | movl %eax, %cr3 |
210 | movl CR0(%edi), %eax |
211 | movl %eax, %cr0 |
212 | movl ESP(%edi), %esp |
213 | movl %ebp, %eax |
214 | |
215 | popf |
216 | popl %ebp |
217 | popl %edi |
218 | popl %esi |
219 | popl %ebx |
220 | ANNOTATE_UNRET_SAFE |
221 | ret |
222 | int3 |
223 | SYM_CODE_END(virtual_mapped) |
224 | |
225 | /* Do the copies */ |
226 | SYM_CODE_START_LOCAL_NOALIGN(swap_pages) |
227 | movl 8(%esp), %edx |
228 | movl 4(%esp), %ecx |
229 | pushl %ebp |
230 | pushl %ebx |
231 | pushl %edi |
232 | pushl %esi |
233 | movl %ecx, %ebx |
234 | jmp 1f |
235 | |
236 | 0: /* top, read another word from the indirection page */ |
237 | movl (%ebx), %ecx |
238 | addl $4, %ebx |
239 | 1: |
240 | testb $0x1, %cl /* is it a destination page */ |
241 | jz 2f |
242 | movl %ecx, %edi |
243 | andl $0xfffff000, %edi |
244 | jmp 0b |
245 | 2: |
246 | testb $0x2, %cl /* is it an indirection page */ |
247 | jz 2f |
248 | movl %ecx, %ebx |
249 | andl $0xfffff000, %ebx |
250 | jmp 0b |
251 | 2: |
252 | testb $0x4, %cl /* is it the done indicator */ |
253 | jz 2f |
254 | jmp 3f |
255 | 2: |
256 | testb $0x8, %cl /* is it the source indicator */ |
257 | jz 0b /* Ignore it otherwise */ |
258 | movl %ecx, %esi /* For every source page do a copy */ |
259 | andl $0xfffff000, %esi |
260 | |
261 | movl %edi, %eax |
262 | movl %esi, %ebp |
263 | |
264 | movl %edx, %edi |
265 | movl $1024, %ecx |
266 | rep ; movsl |
267 | |
268 | movl %ebp, %edi |
269 | movl %eax, %esi |
270 | movl $1024, %ecx |
271 | rep ; movsl |
272 | |
273 | movl %eax, %edi |
274 | movl %edx, %esi |
275 | movl $1024, %ecx |
276 | rep ; movsl |
277 | |
278 | lea PAGE_SIZE(%ebp), %esi |
279 | jmp 0b |
280 | 3: |
281 | popl %esi |
282 | popl %edi |
283 | popl %ebx |
284 | popl %ebp |
285 | ANNOTATE_UNRET_SAFE |
286 | ret |
287 | int3 |
288 | SYM_CODE_END(swap_pages) |
289 | |
290 | .globl kexec_control_code_size |
291 | .set kexec_control_code_size, . - relocate_kernel |
292 | |