1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * misc.c |
4 | * |
5 | * This is a collection of several routines used to extract the kernel |
6 | * which includes KASLR relocation, decompression, ELF parsing, and |
7 | * relocation processing. Additionally included are the screen and serial |
8 | * output functions and related debugging support functions. |
9 | * |
10 | * malloc by Hannu Savolainen 1993 and Matthias Urlichs 1994 |
11 | * puts by Nick Holloway 1993, better puts by Martin Mares 1995 |
12 | * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996 |
13 | */ |
14 | |
15 | #include "misc.h" |
16 | #include "error.h" |
17 | #include "pgtable.h" |
18 | #include "../string.h" |
19 | #include "../voffset.h" |
20 | #include <asm/bootparam_utils.h> |
21 | |
22 | /* |
23 | * WARNING!! |
24 | * This code is compiled with -fPIC and it is relocated dynamically at |
25 | * run time, but no relocation processing is performed. This means that |
26 | * it is not safe to place pointers in static structures. |
27 | */ |
28 | |
29 | /* Macros used by the included decompressor code below. */ |
30 | #define STATIC static |
31 | /* Define an externally visible malloc()/free(). */ |
32 | #define MALLOC_VISIBLE |
33 | #include <linux/decompress/mm.h> |
34 | |
35 | /* |
36 | * Provide definitions of memzero and memmove as some of the decompressors will |
37 | * try to define their own functions if these are not defined as macros. |
38 | */ |
39 | #define memzero(s, n) memset((s), 0, (n)) |
40 | #ifndef memmove |
41 | #define memmove memmove |
42 | /* Functions used by the included decompressor code below. */ |
43 | void *memmove(void *dest, const void *src, size_t n); |
44 | #endif |
45 | |
46 | /* |
47 | * This is set up by the setup-routine at boot-time |
48 | */ |
49 | struct boot_params *boot_params_ptr; |
50 | |
51 | struct port_io_ops pio_ops; |
52 | |
53 | memptr free_mem_ptr; |
54 | memptr free_mem_end_ptr; |
55 | int spurious_nmi_count; |
56 | |
57 | static char *vidmem; |
58 | static int vidport; |
59 | |
60 | /* These might be accessed before .bss is cleared, so use .data instead. */ |
61 | static int lines __section(".data" ); |
62 | static int cols __section(".data" ); |
63 | |
64 | #ifdef CONFIG_KERNEL_GZIP |
65 | #include "../../../../lib/decompress_inflate.c" |
66 | #endif |
67 | |
68 | #ifdef CONFIG_KERNEL_BZIP2 |
69 | #include "../../../../lib/decompress_bunzip2.c" |
70 | #endif |
71 | |
72 | #ifdef CONFIG_KERNEL_LZMA |
73 | #include "../../../../lib/decompress_unlzma.c" |
74 | #endif |
75 | |
76 | #ifdef CONFIG_KERNEL_XZ |
77 | #include "../../../../lib/decompress_unxz.c" |
78 | #endif |
79 | |
80 | #ifdef CONFIG_KERNEL_LZO |
81 | #include "../../../../lib/decompress_unlzo.c" |
82 | #endif |
83 | |
84 | #ifdef CONFIG_KERNEL_LZ4 |
85 | #include "../../../../lib/decompress_unlz4.c" |
86 | #endif |
87 | |
88 | #ifdef CONFIG_KERNEL_ZSTD |
89 | #include "../../../../lib/decompress_unzstd.c" |
90 | #endif |
91 | /* |
92 | * NOTE: When adding a new decompressor, please update the analysis in |
93 | * ../header.S. |
94 | */ |
95 | |
96 | static void scroll(void) |
97 | { |
98 | int i; |
99 | |
100 | memmove(dest: vidmem, src: vidmem + cols * 2, n: (lines - 1) * cols * 2); |
101 | for (i = (lines - 1) * cols * 2; i < lines * cols * 2; i += 2) |
102 | vidmem[i] = ' '; |
103 | } |
104 | |
105 | #define XMTRDY 0x20 |
106 | |
107 | #define TXR 0 /* Transmit register (WRITE) */ |
108 | #define LSR 5 /* Line Status */ |
109 | static void serial_putchar(int ch) |
110 | { |
111 | unsigned timeout = 0xffff; |
112 | |
113 | while ((inb(early_serial_base + LSR) & XMTRDY) == 0 && --timeout) |
114 | cpu_relax(); |
115 | |
116 | outb(ch, early_serial_base + TXR); |
117 | } |
118 | |
119 | void __putstr(const char *s) |
120 | { |
121 | int x, y, pos; |
122 | char c; |
123 | |
124 | if (early_serial_base) { |
125 | const char *str = s; |
126 | while (*str) { |
127 | if (*str == '\n') |
128 | serial_putchar(ch: '\r'); |
129 | serial_putchar(ch: *str++); |
130 | } |
131 | } |
132 | |
133 | if (lines == 0 || cols == 0) |
134 | return; |
135 | |
136 | x = boot_params_ptr->screen_info.orig_x; |
137 | y = boot_params_ptr->screen_info.orig_y; |
138 | |
139 | while ((c = *s++) != '\0') { |
140 | if (c == '\n') { |
141 | x = 0; |
142 | if (++y >= lines) { |
143 | scroll(); |
144 | y--; |
145 | } |
146 | } else { |
147 | vidmem[(x + cols * y) * 2] = c; |
148 | if (++x >= cols) { |
149 | x = 0; |
150 | if (++y >= lines) { |
151 | scroll(); |
152 | y--; |
153 | } |
154 | } |
155 | } |
156 | } |
157 | |
158 | boot_params_ptr->screen_info.orig_x = x; |
159 | boot_params_ptr->screen_info.orig_y = y; |
160 | |
161 | pos = (x + cols * y) * 2; /* Update cursor position */ |
162 | outb(14, vidport); |
163 | outb(0xff & (pos >> 9), vidport+1); |
164 | outb(15, vidport); |
165 | outb(0xff & (pos >> 1), vidport+1); |
166 | } |
167 | |
168 | static noinline void __putnum(unsigned long value, unsigned int base, |
169 | int mindig) |
170 | { |
171 | char buf[8*sizeof(value)+1]; |
172 | char *p; |
173 | |
174 | p = buf + sizeof(buf); |
175 | *--p = '\0'; |
176 | |
177 | while (mindig-- > 0 || value) { |
178 | unsigned char digit = value % base; |
179 | digit += (digit >= 10) ? ('a'-10) : '0'; |
180 | *--p = digit; |
181 | |
182 | value /= base; |
183 | } |
184 | |
185 | __putstr(s: p); |
186 | } |
187 | |
188 | void __puthex(unsigned long value) |
189 | { |
190 | __putnum(value, base: 16, mindig: sizeof(value)*2); |
191 | } |
192 | |
193 | void __putdec(unsigned long value) |
194 | { |
195 | __putnum(value, base: 10, mindig: 1); |
196 | } |
197 | |
198 | #ifdef CONFIG_X86_NEED_RELOCS |
199 | static void handle_relocations(void *output, unsigned long output_len, |
200 | unsigned long virt_addr) |
201 | { |
202 | int *reloc; |
203 | unsigned long delta, map, ptr; |
204 | unsigned long min_addr = (unsigned long)output; |
205 | unsigned long max_addr = min_addr + (VO___bss_start - VO__text); |
206 | |
207 | /* |
208 | * Calculate the delta between where vmlinux was linked to load |
209 | * and where it was actually loaded. |
210 | */ |
211 | delta = min_addr - LOAD_PHYSICAL_ADDR; |
212 | |
213 | /* |
214 | * The kernel contains a table of relocation addresses. Those |
215 | * addresses have the final load address of the kernel in virtual |
216 | * memory. We are currently working in the self map. So we need to |
217 | * create an adjustment for kernel memory addresses to the self map. |
218 | * This will involve subtracting out the base address of the kernel. |
219 | */ |
220 | map = delta - __START_KERNEL_map; |
221 | |
222 | /* |
223 | * 32-bit always performs relocations. 64-bit relocations are only |
224 | * needed if KASLR has chosen a different starting address offset |
225 | * from __START_KERNEL_map. |
226 | */ |
227 | if (IS_ENABLED(CONFIG_X86_64)) |
228 | delta = virt_addr - LOAD_PHYSICAL_ADDR; |
229 | |
230 | if (!delta) { |
231 | debug_putstr("No relocation needed... " ); |
232 | return; |
233 | } |
234 | debug_putstr("Performing relocations... " ); |
235 | |
236 | /* |
237 | * Process relocations: 32 bit relocations first then 64 bit after. |
238 | * Three sets of binary relocations are added to the end of the kernel |
239 | * before compression. Each relocation table entry is the kernel |
240 | * address of the location which needs to be updated stored as a |
241 | * 32-bit value which is sign extended to 64 bits. |
242 | * |
243 | * Format is: |
244 | * |
245 | * kernel bits... |
246 | * 0 - zero terminator for 64 bit relocations |
247 | * 64 bit relocation repeated |
248 | * 0 - zero terminator for inverse 32 bit relocations |
249 | * 32 bit inverse relocation repeated |
250 | * 0 - zero terminator for 32 bit relocations |
251 | * 32 bit relocation repeated |
252 | * |
253 | * So we work backwards from the end of the decompressed image. |
254 | */ |
255 | for (reloc = output + output_len - sizeof(*reloc); *reloc; reloc--) { |
256 | long extended = *reloc; |
257 | extended += map; |
258 | |
259 | ptr = (unsigned long)extended; |
260 | if (ptr < min_addr || ptr > max_addr) |
261 | error(m: "32-bit relocation outside of kernel!\n" ); |
262 | |
263 | *(uint32_t *)ptr += delta; |
264 | } |
265 | #ifdef CONFIG_X86_64 |
266 | while (*--reloc) { |
267 | long extended = *reloc; |
268 | extended += map; |
269 | |
270 | ptr = (unsigned long)extended; |
271 | if (ptr < min_addr || ptr > max_addr) |
272 | error(m: "inverse 32-bit relocation outside of kernel!\n" ); |
273 | |
274 | *(int32_t *)ptr -= delta; |
275 | } |
276 | for (reloc--; *reloc; reloc--) { |
277 | long extended = *reloc; |
278 | extended += map; |
279 | |
280 | ptr = (unsigned long)extended; |
281 | if (ptr < min_addr || ptr > max_addr) |
282 | error(m: "64-bit relocation outside of kernel!\n" ); |
283 | |
284 | *(uint64_t *)ptr += delta; |
285 | } |
286 | #endif |
287 | } |
288 | #else |
289 | static inline void handle_relocations(void *output, unsigned long output_len, |
290 | unsigned long virt_addr) |
291 | { } |
292 | #endif |
293 | |
294 | static size_t parse_elf(void *output) |
295 | { |
296 | #ifdef CONFIG_X86_64 |
297 | Elf64_Ehdr ehdr; |
298 | Elf64_Phdr *phdrs, *phdr; |
299 | #else |
300 | Elf32_Ehdr ehdr; |
301 | Elf32_Phdr *phdrs, *phdr; |
302 | #endif |
303 | void *dest; |
304 | int i; |
305 | |
306 | memcpy(&ehdr, output, sizeof(ehdr)); |
307 | if (ehdr.e_ident[EI_MAG0] != ELFMAG0 || |
308 | ehdr.e_ident[EI_MAG1] != ELFMAG1 || |
309 | ehdr.e_ident[EI_MAG2] != ELFMAG2 || |
310 | ehdr.e_ident[EI_MAG3] != ELFMAG3) |
311 | error(m: "Kernel is not a valid ELF file" ); |
312 | |
313 | debug_putstr("Parsing ELF... " ); |
314 | |
315 | phdrs = malloc(size: sizeof(*phdrs) * ehdr.e_phnum); |
316 | if (!phdrs) |
317 | error(m: "Failed to allocate space for phdrs" ); |
318 | |
319 | memcpy(phdrs, output + ehdr.e_phoff, sizeof(*phdrs) * ehdr.e_phnum); |
320 | |
321 | for (i = 0; i < ehdr.e_phnum; i++) { |
322 | phdr = &phdrs[i]; |
323 | |
324 | switch (phdr->p_type) { |
325 | case PT_LOAD: |
326 | #ifdef CONFIG_X86_64 |
327 | if ((phdr->p_align % 0x200000) != 0) |
328 | error(m: "Alignment of LOAD segment isn't multiple of 2MB" ); |
329 | #endif |
330 | #ifdef CONFIG_RELOCATABLE |
331 | dest = output; |
332 | dest += (phdr->p_paddr - LOAD_PHYSICAL_ADDR); |
333 | #else |
334 | dest = (void *)(phdr->p_paddr); |
335 | #endif |
336 | memmove(dest, src: output + phdr->p_offset, n: phdr->p_filesz); |
337 | break; |
338 | default: /* Ignore other PT_* */ break; |
339 | } |
340 | } |
341 | |
342 | free(where: phdrs); |
343 | |
344 | return ehdr.e_entry - LOAD_PHYSICAL_ADDR; |
345 | } |
346 | |
347 | const unsigned long kernel_text_size = VO___start_rodata - VO__text; |
348 | const unsigned long kernel_total_size = VO__end - VO__text; |
349 | |
350 | static u8 boot_heap[BOOT_HEAP_SIZE] __aligned(4); |
351 | |
352 | extern unsigned char input_data[]; |
353 | extern unsigned int input_len, output_len; |
354 | |
355 | unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr, |
356 | void (*error)(char *x)) |
357 | { |
358 | unsigned long entry; |
359 | |
360 | if (!free_mem_ptr) { |
361 | free_mem_ptr = (unsigned long)boot_heap; |
362 | free_mem_end_ptr = (unsigned long)boot_heap + sizeof(boot_heap); |
363 | } |
364 | |
365 | if (__decompress(buf: input_data, len: input_len, NULL, NULL, out_buf: outbuf, out_len: output_len, |
366 | NULL, error) < 0) |
367 | return ULONG_MAX; |
368 | |
369 | entry = parse_elf(output: outbuf); |
370 | handle_relocations(output: outbuf, output_len, virt_addr); |
371 | |
372 | return entry; |
373 | } |
374 | |
375 | /* |
376 | * Set the memory encryption xloadflag based on the mem_encrypt= command line |
377 | * parameter, if provided. |
378 | */ |
379 | static void parse_mem_encrypt(struct setup_header *hdr) |
380 | { |
381 | int on = cmdline_find_option_bool(option: "mem_encrypt=on" ); |
382 | int off = cmdline_find_option_bool(option: "mem_encrypt=off" ); |
383 | |
384 | if (on > off) |
385 | hdr->xloadflags |= XLF_MEM_ENCRYPTION; |
386 | } |
387 | |
388 | /* |
389 | * The compressed kernel image (ZO), has been moved so that its position |
390 | * is against the end of the buffer used to hold the uncompressed kernel |
391 | * image (VO) and the execution environment (.bss, .brk), which makes sure |
392 | * there is room to do the in-place decompression. (See header.S for the |
393 | * calculations.) |
394 | * |
395 | * |-----compressed kernel image------| |
396 | * V V |
397 | * 0 extract_offset +INIT_SIZE |
398 | * |-----------|---------------|-------------------------|--------| |
399 | * | | | | |
400 | * VO__text startup_32 of ZO VO__end ZO__end |
401 | * ^ ^ |
402 | * |-------uncompressed kernel image---------| |
403 | * |
404 | */ |
405 | asmlinkage __visible void *(void *rmode, unsigned char *output) |
406 | { |
407 | unsigned long virt_addr = LOAD_PHYSICAL_ADDR; |
408 | memptr heap = (memptr)boot_heap; |
409 | unsigned long needed_size; |
410 | size_t entry_offset; |
411 | |
412 | /* Retain x86 boot parameters pointer passed from startup_32/64. */ |
413 | boot_params_ptr = rmode; |
414 | |
415 | /* Clear flags intended for solely in-kernel use. */ |
416 | boot_params_ptr->hdr.loadflags &= ~KASLR_FLAG; |
417 | |
418 | parse_mem_encrypt(hdr: &boot_params_ptr->hdr); |
419 | |
420 | sanitize_boot_params(boot_params: boot_params_ptr); |
421 | |
422 | if (boot_params_ptr->screen_info.orig_video_mode == 7) { |
423 | vidmem = (char *) 0xb0000; |
424 | vidport = 0x3b4; |
425 | } else { |
426 | vidmem = (char *) 0xb8000; |
427 | vidport = 0x3d4; |
428 | } |
429 | |
430 | lines = boot_params_ptr->screen_info.orig_video_lines; |
431 | cols = boot_params_ptr->screen_info.orig_video_cols; |
432 | |
433 | init_default_io_ops(); |
434 | |
435 | /* |
436 | * Detect TDX guest environment. |
437 | * |
438 | * It has to be done before console_init() in order to use |
439 | * paravirtualized port I/O operations if needed. |
440 | */ |
441 | early_tdx_detect(); |
442 | |
443 | console_init(); |
444 | |
445 | /* |
446 | * Save RSDP address for later use. Have this after console_init() |
447 | * so that early debugging output from the RSDP parsing code can be |
448 | * collected. |
449 | */ |
450 | boot_params_ptr->acpi_rsdp_addr = get_rsdp_addr(); |
451 | |
452 | debug_putstr("early console in extract_kernel\n" ); |
453 | |
454 | free_mem_ptr = heap; /* Heap */ |
455 | free_mem_end_ptr = heap + BOOT_HEAP_SIZE; |
456 | |
457 | /* |
458 | * The memory hole needed for the kernel is the larger of either |
459 | * the entire decompressed kernel plus relocation table, or the |
460 | * entire decompressed kernel plus .bss and .brk sections. |
461 | * |
462 | * On X86_64, the memory is mapped with PMD pages. Round the |
463 | * size up so that the full extent of PMD pages mapped is |
464 | * included in the check against the valid memory table |
465 | * entries. This ensures the full mapped area is usable RAM |
466 | * and doesn't include any reserved areas. |
467 | */ |
468 | needed_size = max_t(unsigned long, output_len, kernel_total_size); |
469 | #ifdef CONFIG_X86_64 |
470 | needed_size = ALIGN(needed_size, MIN_KERNEL_ALIGN); |
471 | #endif |
472 | |
473 | /* Report initial kernel position details. */ |
474 | debug_putaddr(input_data); |
475 | debug_putaddr(input_len); |
476 | debug_putaddr(output); |
477 | debug_putaddr(output_len); |
478 | debug_putaddr(kernel_total_size); |
479 | debug_putaddr(needed_size); |
480 | |
481 | #ifdef CONFIG_X86_64 |
482 | /* Report address of 32-bit trampoline */ |
483 | debug_putaddr(trampoline_32bit); |
484 | #endif |
485 | |
486 | choose_random_location(input: (unsigned long)input_data, input_size: input_len, |
487 | output: (unsigned long *)&output, |
488 | output_size: needed_size, |
489 | virt_addr: &virt_addr); |
490 | |
491 | /* Validate memory location choices. */ |
492 | if ((unsigned long)output & (MIN_KERNEL_ALIGN - 1)) |
493 | error(m: "Destination physical address inappropriately aligned" ); |
494 | if (virt_addr & (MIN_KERNEL_ALIGN - 1)) |
495 | error(m: "Destination virtual address inappropriately aligned" ); |
496 | #ifdef CONFIG_X86_64 |
497 | if (heap > 0x3fffffffffffUL) |
498 | error(m: "Destination address too large" ); |
499 | if (virt_addr + needed_size > KERNEL_IMAGE_SIZE) |
500 | error(m: "Destination virtual address is beyond the kernel mapping area" ); |
501 | #else |
502 | if (heap > ((-__PAGE_OFFSET-(128<<20)-1) & 0x7fffffff)) |
503 | error("Destination address too large" ); |
504 | #endif |
505 | #ifndef CONFIG_RELOCATABLE |
506 | if (virt_addr != LOAD_PHYSICAL_ADDR) |
507 | error("Destination virtual address changed when not relocatable" ); |
508 | #endif |
509 | |
510 | debug_putstr("\nDecompressing Linux... " ); |
511 | |
512 | if (init_unaccepted_memory()) { |
513 | debug_putstr("Accepting memory... " ); |
514 | accept_memory(__pa(output), __pa(output) + needed_size); |
515 | } |
516 | |
517 | entry_offset = decompress_kernel(outbuf: output, virt_addr, error); |
518 | |
519 | debug_putstr("done.\nBooting the kernel (entry_offset: 0x" ); |
520 | debug_puthex(entry_offset); |
521 | debug_putstr(").\n" ); |
522 | |
523 | /* Disable exception handling before booting the kernel */ |
524 | cleanup_exception_handling(); |
525 | |
526 | if (spurious_nmi_count) { |
527 | error_putstr("Spurious early NMIs ignored: " ); |
528 | error_putdec(spurious_nmi_count); |
529 | error_putstr("\n" ); |
530 | } |
531 | |
532 | return output + entry_offset; |
533 | } |
534 | |
535 | void __fortify_panic(const u8 reason, size_t avail, size_t size) |
536 | { |
537 | error(m: "detected buffer overflow" ); |
538 | } |
539 | |