1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Stack tracing support |
4 | * |
5 | * Copyright (C) 2012 ARM Ltd. |
6 | */ |
7 | #include <linux/kernel.h> |
8 | #include <linux/efi.h> |
9 | #include <linux/export.h> |
10 | #include <linux/filter.h> |
11 | #include <linux/ftrace.h> |
12 | #include <linux/kprobes.h> |
13 | #include <linux/sched.h> |
14 | #include <linux/sched/debug.h> |
15 | #include <linux/sched/task_stack.h> |
16 | #include <linux/stacktrace.h> |
17 | |
18 | #include <asm/efi.h> |
19 | #include <asm/irq.h> |
20 | #include <asm/stack_pointer.h> |
21 | #include <asm/stacktrace.h> |
22 | |
23 | /* |
24 | * Kernel unwind state |
25 | * |
26 | * @common: Common unwind state. |
27 | * @task: The task being unwound. |
28 | * @kr_cur: When KRETPROBES is selected, holds the kretprobe instance |
29 | * associated with the most recently encountered replacement lr |
30 | * value. |
31 | */ |
32 | struct kunwind_state { |
33 | struct unwind_state common; |
34 | struct task_struct *task; |
35 | #ifdef CONFIG_KRETPROBES |
36 | struct llist_node *kr_cur; |
37 | #endif |
38 | }; |
39 | |
40 | static __always_inline void |
41 | kunwind_init(struct kunwind_state *state, |
42 | struct task_struct *task) |
43 | { |
44 | unwind_init_common(&state->common); |
45 | state->task = task; |
46 | } |
47 | |
48 | /* |
49 | * Start an unwind from a pt_regs. |
50 | * |
51 | * The unwind will begin at the PC within the regs. |
52 | * |
53 | * The regs must be on a stack currently owned by the calling task. |
54 | */ |
55 | static __always_inline void |
56 | kunwind_init_from_regs(struct kunwind_state *state, |
57 | struct pt_regs *regs) |
58 | { |
59 | kunwind_init(state, current); |
60 | |
61 | state->common.fp = regs->regs[29]; |
62 | state->common.pc = regs->pc; |
63 | } |
64 | |
65 | /* |
66 | * Start an unwind from a caller. |
67 | * |
68 | * The unwind will begin at the caller of whichever function this is inlined |
69 | * into. |
70 | * |
71 | * The function which invokes this must be noinline. |
72 | */ |
73 | static __always_inline void |
74 | kunwind_init_from_caller(struct kunwind_state *state) |
75 | { |
76 | kunwind_init(state, current); |
77 | |
78 | state->common.fp = (unsigned long)__builtin_frame_address(1); |
79 | state->common.pc = (unsigned long)__builtin_return_address(0); |
80 | } |
81 | |
82 | /* |
83 | * Start an unwind from a blocked task. |
84 | * |
85 | * The unwind will begin at the blocked tasks saved PC (i.e. the caller of |
86 | * cpu_switch_to()). |
87 | * |
88 | * The caller should ensure the task is blocked in cpu_switch_to() for the |
89 | * duration of the unwind, or the unwind will be bogus. It is never valid to |
90 | * call this for the current task. |
91 | */ |
92 | static __always_inline void |
93 | kunwind_init_from_task(struct kunwind_state *state, |
94 | struct task_struct *task) |
95 | { |
96 | kunwind_init(state, task); |
97 | |
98 | state->common.fp = thread_saved_fp(task); |
99 | state->common.pc = thread_saved_pc(task); |
100 | } |
101 | |
102 | static __always_inline int |
103 | kunwind_recover_return_address(struct kunwind_state *state) |
104 | { |
105 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
106 | if (state->task->ret_stack && |
107 | (state->common.pc == (unsigned long)return_to_handler)) { |
108 | unsigned long orig_pc; |
109 | orig_pc = ftrace_graph_ret_addr(task: state->task, NULL, |
110 | ret: state->common.pc, |
111 | retp: (void *)state->common.fp); |
112 | if (WARN_ON_ONCE(state->common.pc == orig_pc)) |
113 | return -EINVAL; |
114 | state->common.pc = orig_pc; |
115 | } |
116 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
117 | |
118 | #ifdef CONFIG_KRETPROBES |
119 | if (is_kretprobe_trampoline(addr: state->common.pc)) { |
120 | unsigned long orig_pc; |
121 | orig_pc = kretprobe_find_ret_addr(tsk: state->task, |
122 | fp: (void *)state->common.fp, |
123 | cur: &state->kr_cur); |
124 | state->common.pc = orig_pc; |
125 | } |
126 | #endif /* CONFIG_KRETPROBES */ |
127 | |
128 | return 0; |
129 | } |
130 | |
131 | /* |
132 | * Unwind from one frame record (A) to the next frame record (B). |
133 | * |
134 | * We terminate early if the location of B indicates a malformed chain of frame |
135 | * records (e.g. a cycle), determined based on the location and fp value of A |
136 | * and the location (but not the fp value) of B. |
137 | */ |
138 | static __always_inline int |
139 | kunwind_next(struct kunwind_state *state) |
140 | { |
141 | struct task_struct *tsk = state->task; |
142 | unsigned long fp = state->common.fp; |
143 | int err; |
144 | |
145 | /* Final frame; nothing to unwind */ |
146 | if (fp == (unsigned long)task_pt_regs(tsk)->stackframe) |
147 | return -ENOENT; |
148 | |
149 | err = unwind_next_frame_record(&state->common); |
150 | if (err) |
151 | return err; |
152 | |
153 | state->common.pc = ptrauth_strip_kernel_insn_pac(state->common.pc); |
154 | |
155 | return kunwind_recover_return_address(state); |
156 | } |
157 | |
158 | typedef bool (*kunwind_consume_fn)(const struct kunwind_state *state, void *cookie); |
159 | |
160 | static __always_inline void |
161 | do_kunwind(struct kunwind_state *state, kunwind_consume_fn consume_state, |
162 | void *cookie) |
163 | { |
164 | if (kunwind_recover_return_address(state)) |
165 | return; |
166 | |
167 | while (1) { |
168 | int ret; |
169 | |
170 | if (!consume_state(state, cookie)) |
171 | break; |
172 | ret = kunwind_next(state); |
173 | if (ret < 0) |
174 | break; |
175 | } |
176 | } |
177 | |
178 | /* |
179 | * Per-cpu stacks are only accessible when unwinding the current task in a |
180 | * non-preemptible context. |
181 | */ |
182 | #define STACKINFO_CPU(name) \ |
183 | ({ \ |
184 | ((task == current) && !preemptible()) \ |
185 | ? stackinfo_get_##name() \ |
186 | : stackinfo_get_unknown(); \ |
187 | }) |
188 | |
189 | /* |
190 | * SDEI stacks are only accessible when unwinding the current task in an NMI |
191 | * context. |
192 | */ |
193 | #define STACKINFO_SDEI(name) \ |
194 | ({ \ |
195 | ((task == current) && in_nmi()) \ |
196 | ? stackinfo_get_sdei_##name() \ |
197 | : stackinfo_get_unknown(); \ |
198 | }) |
199 | |
200 | #define STACKINFO_EFI \ |
201 | ({ \ |
202 | ((task == current) && current_in_efi()) \ |
203 | ? stackinfo_get_efi() \ |
204 | : stackinfo_get_unknown(); \ |
205 | }) |
206 | |
207 | static __always_inline void |
208 | kunwind_stack_walk(kunwind_consume_fn consume_state, |
209 | void *cookie, struct task_struct *task, |
210 | struct pt_regs *regs) |
211 | { |
212 | struct stack_info stacks[] = { |
213 | stackinfo_get_task(task), |
214 | STACKINFO_CPU(irq), |
215 | #if defined(CONFIG_VMAP_STACK) |
216 | STACKINFO_CPU(overflow), |
217 | #endif |
218 | #if defined(CONFIG_VMAP_STACK) && defined(CONFIG_ARM_SDE_INTERFACE) |
219 | STACKINFO_SDEI(normal), |
220 | STACKINFO_SDEI(critical), |
221 | #endif |
222 | #ifdef CONFIG_EFI |
223 | STACKINFO_EFI, |
224 | #endif |
225 | }; |
226 | struct kunwind_state state = { |
227 | .common = { |
228 | .stacks = stacks, |
229 | .nr_stacks = ARRAY_SIZE(stacks), |
230 | }, |
231 | }; |
232 | |
233 | if (regs) { |
234 | if (task != current) |
235 | return; |
236 | kunwind_init_from_regs(state: &state, regs); |
237 | } else if (task == current) { |
238 | kunwind_init_from_caller(state: &state); |
239 | } else { |
240 | kunwind_init_from_task(state: &state, task); |
241 | } |
242 | |
243 | do_kunwind(state: &state, consume_state, cookie); |
244 | } |
245 | |
246 | struct kunwind_consume_entry_data { |
247 | stack_trace_consume_fn consume_entry; |
248 | void *cookie; |
249 | }; |
250 | |
251 | static __always_inline bool |
252 | arch_kunwind_consume_entry(const struct kunwind_state *state, void *cookie) |
253 | { |
254 | struct kunwind_consume_entry_data *data = cookie; |
255 | return data->consume_entry(data->cookie, state->common.pc); |
256 | } |
257 | |
258 | noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry, |
259 | void *cookie, struct task_struct *task, |
260 | struct pt_regs *regs) |
261 | { |
262 | struct kunwind_consume_entry_data data = { |
263 | .consume_entry = consume_entry, |
264 | .cookie = cookie, |
265 | }; |
266 | |
267 | kunwind_stack_walk(consume_state: arch_kunwind_consume_entry, cookie: &data, task, regs); |
268 | } |
269 | |
270 | struct bpf_unwind_consume_entry_data { |
271 | bool (*consume_entry)(void *cookie, u64 ip, u64 sp, u64 fp); |
272 | void *cookie; |
273 | }; |
274 | |
275 | static bool |
276 | arch_bpf_unwind_consume_entry(const struct kunwind_state *state, void *cookie) |
277 | { |
278 | struct bpf_unwind_consume_entry_data *data = cookie; |
279 | |
280 | return data->consume_entry(data->cookie, state->common.pc, 0, |
281 | state->common.fp); |
282 | } |
283 | |
284 | noinline noinstr void arch_bpf_stack_walk(bool (*consume_entry)(void *cookie, u64 ip, u64 sp, |
285 | u64 fp), void *cookie) |
286 | { |
287 | struct bpf_unwind_consume_entry_data data = { |
288 | .consume_entry = consume_entry, |
289 | .cookie = cookie, |
290 | }; |
291 | |
292 | kunwind_stack_walk(consume_state: arch_bpf_unwind_consume_entry, cookie: &data, current, NULL); |
293 | } |
294 | |
295 | static bool dump_backtrace_entry(void *arg, unsigned long where) |
296 | { |
297 | char *loglvl = arg; |
298 | printk("%s %pSb\n" , loglvl, (void *)where); |
299 | return true; |
300 | } |
301 | |
302 | void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, |
303 | const char *loglvl) |
304 | { |
305 | pr_debug("%s(regs = %p tsk = %p)\n" , __func__, regs, tsk); |
306 | |
307 | if (regs && user_mode(regs)) |
308 | return; |
309 | |
310 | if (!tsk) |
311 | tsk = current; |
312 | |
313 | if (!try_get_task_stack(tsk)) |
314 | return; |
315 | |
316 | printk("%sCall trace:\n" , loglvl); |
317 | arch_stack_walk(consume_entry: dump_backtrace_entry, cookie: (void *)loglvl, task: tsk, regs); |
318 | |
319 | put_task_stack(tsk); |
320 | } |
321 | |
322 | void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) |
323 | { |
324 | dump_backtrace(NULL, tsk, loglvl); |
325 | barrier(); |
326 | } |
327 | |