1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | |
3 | #include <linux/efi.h> |
4 | #include <linux/memblock.h> |
5 | #include <linux/spinlock.h> |
6 | #include <linux/crash_dump.h> |
7 | #include <asm/unaccepted_memory.h> |
8 | |
9 | /* Protects unaccepted memory bitmap and accepting_list */ |
10 | static DEFINE_SPINLOCK(unaccepted_memory_lock); |
11 | |
12 | struct accept_range { |
13 | struct list_head list; |
14 | unsigned long start; |
15 | unsigned long end; |
16 | }; |
17 | |
18 | static LIST_HEAD(accepting_list); |
19 | |
20 | /* |
21 | * accept_memory() -- Consult bitmap and accept the memory if needed. |
22 | * |
23 | * Only memory that is explicitly marked as unaccepted in the bitmap requires |
24 | * an action. All the remaining memory is implicitly accepted and doesn't need |
25 | * acceptance. |
26 | * |
27 | * No need to accept: |
28 | * - anything if the system has no unaccepted table; |
29 | * - memory that is below phys_base; |
30 | * - memory that is above the memory that addressable by the bitmap; |
31 | */ |
32 | void accept_memory(phys_addr_t start, phys_addr_t end) |
33 | { |
34 | struct efi_unaccepted_memory *unaccepted; |
35 | unsigned long range_start, range_end; |
36 | struct accept_range range, *entry; |
37 | unsigned long flags; |
38 | u64 unit_size; |
39 | |
40 | unaccepted = efi_get_unaccepted_table(); |
41 | if (!unaccepted) |
42 | return; |
43 | |
44 | unit_size = unaccepted->unit_size; |
45 | |
46 | /* |
47 | * Only care for the part of the range that is represented |
48 | * in the bitmap. |
49 | */ |
50 | if (start < unaccepted->phys_base) |
51 | start = unaccepted->phys_base; |
52 | if (end < unaccepted->phys_base) |
53 | return; |
54 | |
55 | /* Translate to offsets from the beginning of the bitmap */ |
56 | start -= unaccepted->phys_base; |
57 | end -= unaccepted->phys_base; |
58 | |
59 | /* |
60 | * load_unaligned_zeropad() can lead to unwanted loads across page |
61 | * boundaries. The unwanted loads are typically harmless. But, they |
62 | * might be made to totally unrelated or even unmapped memory. |
63 | * load_unaligned_zeropad() relies on exception fixup (#PF, #GP and now |
64 | * #VE) to recover from these unwanted loads. |
65 | * |
66 | * But, this approach does not work for unaccepted memory. For TDX, a |
67 | * load from unaccepted memory will not lead to a recoverable exception |
68 | * within the guest. The guest will exit to the VMM where the only |
69 | * recourse is to terminate the guest. |
70 | * |
71 | * There are two parts to fix this issue and comprehensively avoid |
72 | * access to unaccepted memory. Together these ensure that an extra |
73 | * "guard" page is accepted in addition to the memory that needs to be |
74 | * used: |
75 | * |
76 | * 1. Implicitly extend the range_contains_unaccepted_memory(start, end) |
77 | * checks up to end+unit_size if 'end' is aligned on a unit_size |
78 | * boundary. |
79 | * |
80 | * 2. Implicitly extend accept_memory(start, end) to end+unit_size if |
81 | * 'end' is aligned on a unit_size boundary. (immediately following |
82 | * this comment) |
83 | */ |
84 | if (!(end % unit_size)) |
85 | end += unit_size; |
86 | |
87 | /* Make sure not to overrun the bitmap */ |
88 | if (end > unaccepted->size * unit_size * BITS_PER_BYTE) |
89 | end = unaccepted->size * unit_size * BITS_PER_BYTE; |
90 | |
91 | range.start = start / unit_size; |
92 | range.end = DIV_ROUND_UP(end, unit_size); |
93 | retry: |
94 | spin_lock_irqsave(&unaccepted_memory_lock, flags); |
95 | |
96 | /* |
97 | * Check if anybody works on accepting the same range of the memory. |
98 | * |
99 | * The check is done with unit_size granularity. It is crucial to catch |
100 | * all accept requests to the same unit_size block, even if they don't |
101 | * overlap on physical address level. |
102 | */ |
103 | list_for_each_entry(entry, &accepting_list, list) { |
104 | if (entry->end <= range.start) |
105 | continue; |
106 | if (entry->start >= range.end) |
107 | continue; |
108 | |
109 | /* |
110 | * Somebody else accepting the range. Or at least part of it. |
111 | * |
112 | * Drop the lock and retry until it is complete. |
113 | */ |
114 | spin_unlock_irqrestore(lock: &unaccepted_memory_lock, flags); |
115 | goto retry; |
116 | } |
117 | |
118 | /* |
119 | * Register that the range is about to be accepted. |
120 | * Make sure nobody else will accept it. |
121 | */ |
122 | list_add(new: &range.list, head: &accepting_list); |
123 | |
124 | range_start = range.start; |
125 | for_each_set_bitrange_from(range_start, range_end, unaccepted->bitmap, |
126 | range.end) { |
127 | unsigned long phys_start, phys_end; |
128 | unsigned long len = range_end - range_start; |
129 | |
130 | phys_start = range_start * unit_size + unaccepted->phys_base; |
131 | phys_end = range_end * unit_size + unaccepted->phys_base; |
132 | |
133 | /* |
134 | * Keep interrupts disabled until the accept operation is |
135 | * complete in order to prevent deadlocks. |
136 | * |
137 | * Enabling interrupts before calling arch_accept_memory() |
138 | * creates an opportunity for an interrupt handler to request |
139 | * acceptance for the same memory. The handler will continuously |
140 | * spin with interrupts disabled, preventing other task from |
141 | * making progress with the acceptance process. |
142 | */ |
143 | spin_unlock(lock: &unaccepted_memory_lock); |
144 | |
145 | arch_accept_memory(start: phys_start, end: phys_end); |
146 | |
147 | spin_lock(lock: &unaccepted_memory_lock); |
148 | bitmap_clear(map: unaccepted->bitmap, start: range_start, nbits: len); |
149 | } |
150 | |
151 | list_del(entry: &range.list); |
152 | spin_unlock_irqrestore(lock: &unaccepted_memory_lock, flags); |
153 | } |
154 | |
155 | bool range_contains_unaccepted_memory(phys_addr_t start, phys_addr_t end) |
156 | { |
157 | struct efi_unaccepted_memory *unaccepted; |
158 | unsigned long flags; |
159 | bool ret = false; |
160 | u64 unit_size; |
161 | |
162 | unaccepted = efi_get_unaccepted_table(); |
163 | if (!unaccepted) |
164 | return false; |
165 | |
166 | unit_size = unaccepted->unit_size; |
167 | |
168 | /* |
169 | * Only care for the part of the range that is represented |
170 | * in the bitmap. |
171 | */ |
172 | if (start < unaccepted->phys_base) |
173 | start = unaccepted->phys_base; |
174 | if (end < unaccepted->phys_base) |
175 | return false; |
176 | |
177 | /* Translate to offsets from the beginning of the bitmap */ |
178 | start -= unaccepted->phys_base; |
179 | end -= unaccepted->phys_base; |
180 | |
181 | /* |
182 | * Also consider the unaccepted state of the *next* page. See fix #1 in |
183 | * the comment on load_unaligned_zeropad() in accept_memory(). |
184 | */ |
185 | if (!(end % unit_size)) |
186 | end += unit_size; |
187 | |
188 | /* Make sure not to overrun the bitmap */ |
189 | if (end > unaccepted->size * unit_size * BITS_PER_BYTE) |
190 | end = unaccepted->size * unit_size * BITS_PER_BYTE; |
191 | |
192 | spin_lock_irqsave(&unaccepted_memory_lock, flags); |
193 | while (start < end) { |
194 | if (test_bit(start / unit_size, unaccepted->bitmap)) { |
195 | ret = true; |
196 | break; |
197 | } |
198 | |
199 | start += unit_size; |
200 | } |
201 | spin_unlock_irqrestore(lock: &unaccepted_memory_lock, flags); |
202 | |
203 | return ret; |
204 | } |
205 | |
206 | #ifdef CONFIG_PROC_VMCORE |
207 | static bool unaccepted_memory_vmcore_pfn_is_ram(struct vmcore_cb *cb, |
208 | unsigned long pfn) |
209 | { |
210 | return !pfn_is_unaccepted_memory(pfn); |
211 | } |
212 | |
213 | static struct vmcore_cb vmcore_cb = { |
214 | .pfn_is_ram = unaccepted_memory_vmcore_pfn_is_ram, |
215 | }; |
216 | |
217 | static int __init unaccepted_memory_init_kdump(void) |
218 | { |
219 | register_vmcore_cb(cb: &vmcore_cb); |
220 | return 0; |
221 | } |
222 | core_initcall(unaccepted_memory_init_kdump); |
223 | #endif /* CONFIG_PROC_VMCORE */ |
224 | |