1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ASM_X86_PAGE_64_H |
3 | #define _ASM_X86_PAGE_64_H |
4 | |
5 | #include <asm/page_64_types.h> |
6 | |
7 | #ifndef __ASSEMBLY__ |
8 | #include <asm/cpufeatures.h> |
9 | #include <asm/alternative.h> |
10 | |
11 | #include <linux/kmsan-checks.h> |
12 | |
13 | /* duplicated to the one in bootmem.h */ |
14 | extern unsigned long max_pfn; |
15 | extern unsigned long phys_base; |
16 | |
17 | extern unsigned long page_offset_base; |
18 | extern unsigned long vmalloc_base; |
19 | extern unsigned long vmemmap_base; |
20 | |
21 | static __always_inline unsigned long __phys_addr_nodebug(unsigned long x) |
22 | { |
23 | unsigned long y = x - __START_KERNEL_map; |
24 | |
25 | /* use the carry flag to determine if x was < __START_KERNEL_map */ |
26 | x = y + ((x > y) ? phys_base : (__START_KERNEL_map - PAGE_OFFSET)); |
27 | |
28 | return x; |
29 | } |
30 | |
31 | #ifdef CONFIG_DEBUG_VIRTUAL |
32 | extern unsigned long __phys_addr(unsigned long); |
33 | extern unsigned long __phys_addr_symbol(unsigned long); |
34 | #else |
35 | #define __phys_addr(x) __phys_addr_nodebug(x) |
36 | #define __phys_addr_symbol(x) \ |
37 | ((unsigned long)(x) - __START_KERNEL_map + phys_base) |
38 | #endif |
39 | |
40 | #define __phys_reloc_hide(x) (x) |
41 | |
42 | void clear_page_orig(void *page); |
43 | void clear_page_rep(void *page); |
44 | void clear_page_erms(void *page); |
45 | |
46 | static inline void clear_page(void *page) |
47 | { |
48 | /* |
49 | * Clean up KMSAN metadata for the page being cleared. The assembly call |
50 | * below clobbers @page, so we perform unpoisoning before it. |
51 | */ |
52 | kmsan_unpoison_memory(address: page, PAGE_SIZE); |
53 | alternative_call_2(clear_page_orig, |
54 | clear_page_rep, X86_FEATURE_REP_GOOD, |
55 | clear_page_erms, X86_FEATURE_ERMS, |
56 | "=D" (page), |
57 | "0" (page) |
58 | : "cc" , "memory" , "rax" , "rcx" ); |
59 | } |
60 | |
61 | void copy_page(void *to, void *from); |
62 | |
63 | #ifdef CONFIG_X86_5LEVEL |
64 | /* |
65 | * User space process size. This is the first address outside the user range. |
66 | * There are a few constraints that determine this: |
67 | * |
68 | * On Intel CPUs, if a SYSCALL instruction is at the highest canonical |
69 | * address, then that syscall will enter the kernel with a |
70 | * non-canonical return address, and SYSRET will explode dangerously. |
71 | * We avoid this particular problem by preventing anything |
72 | * from being mapped at the maximum canonical address. |
73 | * |
74 | * On AMD CPUs in the Ryzen family, there's a nasty bug in which the |
75 | * CPUs malfunction if they execute code from the highest canonical page. |
76 | * They'll speculate right off the end of the canonical space, and |
77 | * bad things happen. This is worked around in the same way as the |
78 | * Intel problem. |
79 | * |
80 | * With page table isolation enabled, we map the LDT in ... [stay tuned] |
81 | */ |
82 | static __always_inline unsigned long task_size_max(void) |
83 | { |
84 | unsigned long ret; |
85 | |
86 | alternative_io("movq %[small],%0" ,"movq %[large],%0" , |
87 | X86_FEATURE_LA57, |
88 | "=r" (ret), |
89 | [small] "i" ((1ul << 47)-PAGE_SIZE), |
90 | [large] "i" ((1ul << 56)-PAGE_SIZE)); |
91 | |
92 | return ret; |
93 | } |
94 | #endif /* CONFIG_X86_5LEVEL */ |
95 | |
96 | #endif /* !__ASSEMBLY__ */ |
97 | |
98 | #ifdef CONFIG_X86_VSYSCALL_EMULATION |
99 | # define __HAVE_ARCH_GATE_AREA 1 |
100 | #endif |
101 | |
102 | #endif /* _ASM_X86_PAGE_64_H */ |
103 | |