1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * KVM nVHE hypervisor stack tracing support. |
4 | * |
5 | * The unwinder implementation depends on the nVHE mode: |
6 | * |
7 | * 1) Non-protected nVHE mode - the host can directly access the |
8 | * HYP stack pages and unwind the HYP stack in EL1. This saves having |
9 | * to allocate shared buffers for the host to read the unwinded |
10 | * stacktrace. |
11 | * |
12 | * 2) pKVM (protected nVHE) mode - the host cannot directly access |
13 | * the HYP memory. The stack is unwinded in EL2 and dumped to a shared |
14 | * buffer where the host can read and print the stacktrace. |
15 | * |
16 | * Copyright (C) 2022 Google LLC |
17 | */ |
18 | |
19 | #include <linux/kvm.h> |
20 | #include <linux/kvm_host.h> |
21 | |
22 | #include <asm/stacktrace/nvhe.h> |
23 | |
24 | static struct stack_info stackinfo_get_overflow(void) |
25 | { |
26 | struct kvm_nvhe_stacktrace_info *stacktrace_info |
27 | = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); |
28 | unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base; |
29 | unsigned long high = low + OVERFLOW_STACK_SIZE; |
30 | |
31 | return (struct stack_info) { |
32 | .low = low, |
33 | .high = high, |
34 | }; |
35 | } |
36 | |
37 | static struct stack_info stackinfo_get_overflow_kern_va(void) |
38 | { |
39 | unsigned long low = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack); |
40 | unsigned long high = low + OVERFLOW_STACK_SIZE; |
41 | |
42 | return (struct stack_info) { |
43 | .low = low, |
44 | .high = high, |
45 | }; |
46 | } |
47 | |
48 | static struct stack_info stackinfo_get_hyp(void) |
49 | { |
50 | struct kvm_nvhe_stacktrace_info *stacktrace_info |
51 | = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); |
52 | unsigned long low = (unsigned long)stacktrace_info->stack_base; |
53 | unsigned long high = low + PAGE_SIZE; |
54 | |
55 | return (struct stack_info) { |
56 | .low = low, |
57 | .high = high, |
58 | }; |
59 | } |
60 | |
61 | static struct stack_info stackinfo_get_hyp_kern_va(void) |
62 | { |
63 | unsigned long low = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_page); |
64 | unsigned long high = low + PAGE_SIZE; |
65 | |
66 | return (struct stack_info) { |
67 | .low = low, |
68 | .high = high, |
69 | }; |
70 | } |
71 | |
72 | /* |
73 | * kvm_nvhe_stack_kern_va - Convert KVM nVHE HYP stack addresses to a kernel VAs |
74 | * |
75 | * The nVHE hypervisor stack is mapped in the flexible 'private' VA range, to |
76 | * allow for guard pages below the stack. Consequently, the fixed offset address |
77 | * translation macros won't work here. |
78 | * |
79 | * The kernel VA is calculated as an offset from the kernel VA of the hypervisor |
80 | * stack base. |
81 | * |
82 | * Returns true on success and updates @addr to its corresponding kernel VA; |
83 | * otherwise returns false. |
84 | */ |
85 | static bool kvm_nvhe_stack_kern_va(unsigned long *addr, unsigned long size) |
86 | { |
87 | struct stack_info stack_hyp, stack_kern; |
88 | |
89 | stack_hyp = stackinfo_get_hyp(); |
90 | stack_kern = stackinfo_get_hyp_kern_va(); |
91 | if (stackinfo_on_stack(&stack_hyp, *addr, size)) |
92 | goto found; |
93 | |
94 | stack_hyp = stackinfo_get_overflow(); |
95 | stack_kern = stackinfo_get_overflow_kern_va(); |
96 | if (stackinfo_on_stack(&stack_hyp, *addr, size)) |
97 | goto found; |
98 | |
99 | return false; |
100 | |
101 | found: |
102 | *addr = *addr - stack_hyp.low + stack_kern.low; |
103 | return true; |
104 | } |
105 | |
106 | /* |
107 | * Convert a KVN nVHE HYP frame record address to a kernel VA |
108 | */ |
109 | static bool kvm_nvhe_stack_kern_record_va(unsigned long *addr) |
110 | { |
111 | return kvm_nvhe_stack_kern_va(addr, size: 16); |
112 | } |
113 | |
114 | static int unwind_next(struct unwind_state *state) |
115 | { |
116 | /* |
117 | * The FP is in the hypervisor VA space. Convert it to the kernel VA |
118 | * space so it can be unwound by the regular unwind functions. |
119 | */ |
120 | if (!kvm_nvhe_stack_kern_record_va(addr: &state->fp)) |
121 | return -EINVAL; |
122 | |
123 | return unwind_next_frame_record(state); |
124 | } |
125 | |
126 | static void unwind(struct unwind_state *state, |
127 | stack_trace_consume_fn consume_entry, void *cookie) |
128 | { |
129 | while (1) { |
130 | int ret; |
131 | |
132 | if (!consume_entry(cookie, state->pc)) |
133 | break; |
134 | ret = unwind_next(state); |
135 | if (ret < 0) |
136 | break; |
137 | } |
138 | } |
139 | |
140 | /* |
141 | * kvm_nvhe_dump_backtrace_entry - Symbolize and print an nVHE backtrace entry |
142 | * |
143 | * @arg : the hypervisor offset, used for address translation |
144 | * @where : the program counter corresponding to the stack frame |
145 | */ |
146 | static bool kvm_nvhe_dump_backtrace_entry(void *arg, unsigned long where) |
147 | { |
148 | unsigned long va_mask = GENMASK_ULL(vabits_actual - 1, 0); |
149 | unsigned long hyp_offset = (unsigned long)arg; |
150 | |
151 | /* Mask tags and convert to kern addr */ |
152 | where = (where & va_mask) + hyp_offset; |
153 | kvm_err(" [<%016lx>] %pB\n" , where, (void *)(where + kaslr_offset())); |
154 | |
155 | return true; |
156 | } |
157 | |
158 | static void kvm_nvhe_dump_backtrace_start(void) |
159 | { |
160 | kvm_err("nVHE call trace:\n" ); |
161 | } |
162 | |
163 | static void kvm_nvhe_dump_backtrace_end(void) |
164 | { |
165 | kvm_err("---[ end nVHE call trace ]---\n" ); |
166 | } |
167 | |
168 | /* |
169 | * hyp_dump_backtrace - Dump the non-protected nVHE backtrace. |
170 | * |
171 | * @hyp_offset: hypervisor offset, used for address translation. |
172 | * |
173 | * The host can directly access HYP stack pages in non-protected |
174 | * mode, so the unwinding is done directly from EL1. This removes |
175 | * the need for shared buffers between host and hypervisor for |
176 | * the stacktrace. |
177 | */ |
178 | static void hyp_dump_backtrace(unsigned long hyp_offset) |
179 | { |
180 | struct kvm_nvhe_stacktrace_info *stacktrace_info; |
181 | struct stack_info stacks[] = { |
182 | stackinfo_get_overflow_kern_va(), |
183 | stackinfo_get_hyp_kern_va(), |
184 | }; |
185 | struct unwind_state state = { |
186 | .stacks = stacks, |
187 | .nr_stacks = ARRAY_SIZE(stacks), |
188 | }; |
189 | |
190 | stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); |
191 | |
192 | kvm_nvhe_unwind_init(&state, stacktrace_info->fp, stacktrace_info->pc); |
193 | |
194 | kvm_nvhe_dump_backtrace_start(); |
195 | unwind(state: &state, consume_entry: kvm_nvhe_dump_backtrace_entry, cookie: (void *)hyp_offset); |
196 | kvm_nvhe_dump_backtrace_end(); |
197 | } |
198 | |
199 | #ifdef CONFIG_PROTECTED_NVHE_STACKTRACE |
200 | DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], |
201 | pkvm_stacktrace); |
202 | |
203 | /* |
204 | * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace. |
205 | * |
206 | * @hyp_offset: hypervisor offset, used for address translation. |
207 | * |
208 | * Dumping of the pKVM HYP backtrace is done by reading the |
209 | * stack addresses from the shared stacktrace buffer, since the |
210 | * host cannot directly access hypervisor memory in protected |
211 | * mode. |
212 | */ |
213 | static void pkvm_dump_backtrace(unsigned long hyp_offset) |
214 | { |
215 | unsigned long *stacktrace |
216 | = (unsigned long *) this_cpu_ptr_nvhe_sym(pkvm_stacktrace); |
217 | int i; |
218 | |
219 | kvm_nvhe_dump_backtrace_start(); |
220 | /* The saved stacktrace is terminated by a null entry */ |
221 | for (i = 0; |
222 | i < ARRAY_SIZE(kvm_nvhe_sym(pkvm_stacktrace)) && stacktrace[i]; |
223 | i++) |
224 | kvm_nvhe_dump_backtrace_entry((void *)hyp_offset, stacktrace[i]); |
225 | kvm_nvhe_dump_backtrace_end(); |
226 | } |
227 | #else /* !CONFIG_PROTECTED_NVHE_STACKTRACE */ |
228 | static void pkvm_dump_backtrace(unsigned long hyp_offset) |
229 | { |
230 | kvm_err("Cannot dump pKVM nVHE stacktrace: !CONFIG_PROTECTED_NVHE_STACKTRACE\n" ); |
231 | } |
232 | #endif /* CONFIG_PROTECTED_NVHE_STACKTRACE */ |
233 | |
234 | /* |
235 | * kvm_nvhe_dump_backtrace - Dump KVM nVHE hypervisor backtrace. |
236 | * |
237 | * @hyp_offset: hypervisor offset, used for address translation. |
238 | */ |
239 | void kvm_nvhe_dump_backtrace(unsigned long hyp_offset) |
240 | { |
241 | if (is_protected_kvm_enabled()) |
242 | pkvm_dump_backtrace(hyp_offset); |
243 | else |
244 | hyp_dump_backtrace(hyp_offset); |
245 | } |
246 | |