1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * KVM nVHE hypervisor stack tracing support.
4 *
5 * The unwinder implementation depends on the nVHE mode:
6 *
7 * 1) Non-protected nVHE mode - the host can directly access the
8 * HYP stack pages and unwind the HYP stack in EL1. This saves having
9 * to allocate shared buffers for the host to read the unwinded
10 * stacktrace.
11 *
12 * 2) pKVM (protected nVHE) mode - the host cannot directly access
13 * the HYP memory. The stack is unwinded in EL2 and dumped to a shared
14 * buffer where the host can read and print the stacktrace.
15 *
16 * Copyright (C) 2022 Google LLC
17 */
18
19#include <linux/kvm.h>
20#include <linux/kvm_host.h>
21
22#include <asm/stacktrace/nvhe.h>
23
24static struct stack_info stackinfo_get_overflow(void)
25{
26 struct kvm_nvhe_stacktrace_info *stacktrace_info
27 = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
28 unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base;
29 unsigned long high = low + OVERFLOW_STACK_SIZE;
30
31 return (struct stack_info) {
32 .low = low,
33 .high = high,
34 };
35}
36
37static struct stack_info stackinfo_get_overflow_kern_va(void)
38{
39 unsigned long low = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack);
40 unsigned long high = low + OVERFLOW_STACK_SIZE;
41
42 return (struct stack_info) {
43 .low = low,
44 .high = high,
45 };
46}
47
48static struct stack_info stackinfo_get_hyp(void)
49{
50 struct kvm_nvhe_stacktrace_info *stacktrace_info
51 = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
52 unsigned long low = (unsigned long)stacktrace_info->stack_base;
53 unsigned long high = low + PAGE_SIZE;
54
55 return (struct stack_info) {
56 .low = low,
57 .high = high,
58 };
59}
60
61static struct stack_info stackinfo_get_hyp_kern_va(void)
62{
63 unsigned long low = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_page);
64 unsigned long high = low + PAGE_SIZE;
65
66 return (struct stack_info) {
67 .low = low,
68 .high = high,
69 };
70}
71
72/*
73 * kvm_nvhe_stack_kern_va - Convert KVM nVHE HYP stack addresses to a kernel VAs
74 *
75 * The nVHE hypervisor stack is mapped in the flexible 'private' VA range, to
76 * allow for guard pages below the stack. Consequently, the fixed offset address
77 * translation macros won't work here.
78 *
79 * The kernel VA is calculated as an offset from the kernel VA of the hypervisor
80 * stack base.
81 *
82 * Returns true on success and updates @addr to its corresponding kernel VA;
83 * otherwise returns false.
84 */
85static bool kvm_nvhe_stack_kern_va(unsigned long *addr, unsigned long size)
86{
87 struct stack_info stack_hyp, stack_kern;
88
89 stack_hyp = stackinfo_get_hyp();
90 stack_kern = stackinfo_get_hyp_kern_va();
91 if (stackinfo_on_stack(&stack_hyp, *addr, size))
92 goto found;
93
94 stack_hyp = stackinfo_get_overflow();
95 stack_kern = stackinfo_get_overflow_kern_va();
96 if (stackinfo_on_stack(&stack_hyp, *addr, size))
97 goto found;
98
99 return false;
100
101found:
102 *addr = *addr - stack_hyp.low + stack_kern.low;
103 return true;
104}
105
106/*
107 * Convert a KVN nVHE HYP frame record address to a kernel VA
108 */
109static bool kvm_nvhe_stack_kern_record_va(unsigned long *addr)
110{
111 return kvm_nvhe_stack_kern_va(addr, size: 16);
112}
113
114static int unwind_next(struct unwind_state *state)
115{
116 /*
117 * The FP is in the hypervisor VA space. Convert it to the kernel VA
118 * space so it can be unwound by the regular unwind functions.
119 */
120 if (!kvm_nvhe_stack_kern_record_va(addr: &state->fp))
121 return -EINVAL;
122
123 return unwind_next_frame_record(state);
124}
125
126static void unwind(struct unwind_state *state,
127 stack_trace_consume_fn consume_entry, void *cookie)
128{
129 while (1) {
130 int ret;
131
132 if (!consume_entry(cookie, state->pc))
133 break;
134 ret = unwind_next(state);
135 if (ret < 0)
136 break;
137 }
138}
139
140/*
141 * kvm_nvhe_dump_backtrace_entry - Symbolize and print an nVHE backtrace entry
142 *
143 * @arg : the hypervisor offset, used for address translation
144 * @where : the program counter corresponding to the stack frame
145 */
146static bool kvm_nvhe_dump_backtrace_entry(void *arg, unsigned long where)
147{
148 unsigned long va_mask = GENMASK_ULL(vabits_actual - 1, 0);
149 unsigned long hyp_offset = (unsigned long)arg;
150
151 /* Mask tags and convert to kern addr */
152 where = (where & va_mask) + hyp_offset;
153 kvm_err(" [<%016lx>] %pB\n", where, (void *)(where + kaslr_offset()));
154
155 return true;
156}
157
158static void kvm_nvhe_dump_backtrace_start(void)
159{
160 kvm_err("nVHE call trace:\n");
161}
162
163static void kvm_nvhe_dump_backtrace_end(void)
164{
165 kvm_err("---[ end nVHE call trace ]---\n");
166}
167
168/*
169 * hyp_dump_backtrace - Dump the non-protected nVHE backtrace.
170 *
171 * @hyp_offset: hypervisor offset, used for address translation.
172 *
173 * The host can directly access HYP stack pages in non-protected
174 * mode, so the unwinding is done directly from EL1. This removes
175 * the need for shared buffers between host and hypervisor for
176 * the stacktrace.
177 */
178static void hyp_dump_backtrace(unsigned long hyp_offset)
179{
180 struct kvm_nvhe_stacktrace_info *stacktrace_info;
181 struct stack_info stacks[] = {
182 stackinfo_get_overflow_kern_va(),
183 stackinfo_get_hyp_kern_va(),
184 };
185 struct unwind_state state = {
186 .stacks = stacks,
187 .nr_stacks = ARRAY_SIZE(stacks),
188 };
189
190 stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
191
192 kvm_nvhe_unwind_init(&state, stacktrace_info->fp, stacktrace_info->pc);
193
194 kvm_nvhe_dump_backtrace_start();
195 unwind(state: &state, consume_entry: kvm_nvhe_dump_backtrace_entry, cookie: (void *)hyp_offset);
196 kvm_nvhe_dump_backtrace_end();
197}
198
199#ifdef CONFIG_PROTECTED_NVHE_STACKTRACE
200DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
201 pkvm_stacktrace);
202
203/*
204 * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace.
205 *
206 * @hyp_offset: hypervisor offset, used for address translation.
207 *
208 * Dumping of the pKVM HYP backtrace is done by reading the
209 * stack addresses from the shared stacktrace buffer, since the
210 * host cannot directly access hypervisor memory in protected
211 * mode.
212 */
213static void pkvm_dump_backtrace(unsigned long hyp_offset)
214{
215 unsigned long *stacktrace
216 = (unsigned long *) this_cpu_ptr_nvhe_sym(pkvm_stacktrace);
217 int i;
218
219 kvm_nvhe_dump_backtrace_start();
220 /* The saved stacktrace is terminated by a null entry */
221 for (i = 0;
222 i < ARRAY_SIZE(kvm_nvhe_sym(pkvm_stacktrace)) && stacktrace[i];
223 i++)
224 kvm_nvhe_dump_backtrace_entry((void *)hyp_offset, stacktrace[i]);
225 kvm_nvhe_dump_backtrace_end();
226}
227#else /* !CONFIG_PROTECTED_NVHE_STACKTRACE */
228static void pkvm_dump_backtrace(unsigned long hyp_offset)
229{
230 kvm_err("Cannot dump pKVM nVHE stacktrace: !CONFIG_PROTECTED_NVHE_STACKTRACE\n");
231}
232#endif /* CONFIG_PROTECTED_NVHE_STACKTRACE */
233
234/*
235 * kvm_nvhe_dump_backtrace - Dump KVM nVHE hypervisor backtrace.
236 *
237 * @hyp_offset: hypervisor offset, used for address translation.
238 */
239void kvm_nvhe_dump_backtrace(unsigned long hyp_offset)
240{
241 if (is_protected_kvm_enabled())
242 pkvm_dump_backtrace(hyp_offset);
243 else
244 hyp_dump_backtrace(hyp_offset);
245}
246

source code of linux/arch/arm64/kvm/stacktrace.c