1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Stack tracing support
4 *
5 * Copyright (C) 2012 ARM Ltd.
6 */
7#include <linux/kernel.h>
8#include <linux/efi.h>
9#include <linux/export.h>
10#include <linux/filter.h>
11#include <linux/ftrace.h>
12#include <linux/kprobes.h>
13#include <linux/sched.h>
14#include <linux/sched/debug.h>
15#include <linux/sched/task_stack.h>
16#include <linux/stacktrace.h>
17
18#include <asm/efi.h>
19#include <asm/irq.h>
20#include <asm/stack_pointer.h>
21#include <asm/stacktrace.h>
22
23enum kunwind_source {
24 KUNWIND_SOURCE_UNKNOWN,
25 KUNWIND_SOURCE_FRAME,
26 KUNWIND_SOURCE_CALLER,
27 KUNWIND_SOURCE_TASK,
28 KUNWIND_SOURCE_REGS_PC,
29};
30
31union unwind_flags {
32 unsigned long all;
33 struct {
34 unsigned long fgraph : 1,
35 kretprobe : 1;
36 };
37};
38
39/*
40 * Kernel unwind state
41 *
42 * @common: Common unwind state.
43 * @task: The task being unwound.
44 * @graph_idx: Used by ftrace_graph_ret_addr() for optimized stack unwinding.
45 * @kr_cur: When KRETPROBES is selected, holds the kretprobe instance
46 * associated with the most recently encountered replacement lr
47 * value.
48 */
49struct kunwind_state {
50 struct unwind_state common;
51 struct task_struct *task;
52 int graph_idx;
53#ifdef CONFIG_KRETPROBES
54 struct llist_node *kr_cur;
55#endif
56 enum kunwind_source source;
57 union unwind_flags flags;
58 struct pt_regs *regs;
59};
60
61static __always_inline void
62kunwind_init(struct kunwind_state *state,
63 struct task_struct *task)
64{
65 unwind_init_common(&state->common);
66 state->task = task;
67 state->source = KUNWIND_SOURCE_UNKNOWN;
68 state->flags.all = 0;
69 state->regs = NULL;
70}
71
72/*
73 * Start an unwind from a pt_regs.
74 *
75 * The unwind will begin at the PC within the regs.
76 *
77 * The regs must be on a stack currently owned by the calling task.
78 */
79static __always_inline void
80kunwind_init_from_regs(struct kunwind_state *state,
81 struct pt_regs *regs)
82{
83 kunwind_init(state, current);
84
85 state->regs = regs;
86 state->common.fp = regs->regs[29];
87 state->common.pc = regs->pc;
88 state->source = KUNWIND_SOURCE_REGS_PC;
89}
90
91/*
92 * Start an unwind from a caller.
93 *
94 * The unwind will begin at the caller of whichever function this is inlined
95 * into.
96 *
97 * The function which invokes this must be noinline.
98 */
99static __always_inline void
100kunwind_init_from_caller(struct kunwind_state *state)
101{
102 kunwind_init(state, current);
103
104 state->common.fp = (unsigned long)__builtin_frame_address(1);
105 state->common.pc = (unsigned long)__builtin_return_address(0);
106 state->source = KUNWIND_SOURCE_CALLER;
107}
108
109/*
110 * Start an unwind from a blocked task.
111 *
112 * The unwind will begin at the blocked tasks saved PC (i.e. the caller of
113 * cpu_switch_to()).
114 *
115 * The caller should ensure the task is blocked in cpu_switch_to() for the
116 * duration of the unwind, or the unwind will be bogus. It is never valid to
117 * call this for the current task.
118 */
119static __always_inline void
120kunwind_init_from_task(struct kunwind_state *state,
121 struct task_struct *task)
122{
123 kunwind_init(state, task);
124
125 state->common.fp = thread_saved_fp(task);
126 state->common.pc = thread_saved_pc(task);
127 state->source = KUNWIND_SOURCE_TASK;
128}
129
130static __always_inline int
131kunwind_recover_return_address(struct kunwind_state *state)
132{
133#ifdef CONFIG_FUNCTION_GRAPH_TRACER
134 if (state->task->ret_stack &&
135 (state->common.pc == (unsigned long)return_to_handler)) {
136 unsigned long orig_pc;
137 orig_pc = ftrace_graph_ret_addr(task: state->task, idx: &state->graph_idx,
138 ret: state->common.pc,
139 retp: (void *)state->common.fp);
140 if (state->common.pc == orig_pc) {
141 WARN_ON_ONCE(state->task == current);
142 return -EINVAL;
143 }
144 state->common.pc = orig_pc;
145 state->flags.fgraph = 1;
146 }
147#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
148
149#ifdef CONFIG_KRETPROBES
150 if (is_kretprobe_trampoline(addr: state->common.pc)) {
151 unsigned long orig_pc;
152 orig_pc = kretprobe_find_ret_addr(tsk: state->task,
153 fp: (void *)state->common.fp,
154 cur: &state->kr_cur);
155 if (!orig_pc)
156 return -EINVAL;
157 state->common.pc = orig_pc;
158 state->flags.kretprobe = 1;
159 }
160#endif /* CONFIG_KRETPROBES */
161
162 return 0;
163}
164
165static __always_inline
166int kunwind_next_regs_pc(struct kunwind_state *state)
167{
168 struct stack_info *info;
169 unsigned long fp = state->common.fp;
170 struct pt_regs *regs;
171
172 regs = container_of((u64 *)fp, struct pt_regs, stackframe.record.fp);
173
174 info = unwind_find_stack(&state->common, (unsigned long)regs, sizeof(*regs));
175 if (!info)
176 return -EINVAL;
177
178 unwind_consume_stack(&state->common, info, (unsigned long)regs,
179 sizeof(*regs));
180
181 state->regs = regs;
182 state->common.pc = regs->pc;
183 state->common.fp = regs->regs[29];
184 state->regs = NULL;
185 state->source = KUNWIND_SOURCE_REGS_PC;
186 return 0;
187}
188
189static __always_inline int
190kunwind_next_frame_record_meta(struct kunwind_state *state)
191{
192 struct task_struct *tsk = state->task;
193 unsigned long fp = state->common.fp;
194 struct frame_record_meta *meta;
195 struct stack_info *info;
196
197 info = unwind_find_stack(&state->common, fp, sizeof(*meta));
198 if (!info)
199 return -EINVAL;
200
201 meta = (struct frame_record_meta *)fp;
202 switch (READ_ONCE(meta->type)) {
203 case FRAME_META_TYPE_FINAL:
204 if (meta == &task_pt_regs(tsk)->stackframe)
205 return -ENOENT;
206 WARN_ON_ONCE(tsk == current);
207 return -EINVAL;
208 case FRAME_META_TYPE_PT_REGS:
209 return kunwind_next_regs_pc(state);
210 default:
211 WARN_ON_ONCE(tsk == current);
212 return -EINVAL;
213 }
214}
215
216static __always_inline int
217kunwind_next_frame_record(struct kunwind_state *state)
218{
219 unsigned long fp = state->common.fp;
220 struct frame_record *record;
221 struct stack_info *info;
222 unsigned long new_fp, new_pc;
223
224 if (fp & 0x7)
225 return -EINVAL;
226
227 info = unwind_find_stack(&state->common, fp, sizeof(*record));
228 if (!info)
229 return -EINVAL;
230
231 record = (struct frame_record *)fp;
232 new_fp = READ_ONCE(record->fp);
233 new_pc = READ_ONCE(record->lr);
234
235 if (!new_fp && !new_pc)
236 return kunwind_next_frame_record_meta(state);
237
238 unwind_consume_stack(&state->common, info, fp, sizeof(*record));
239
240 state->common.fp = new_fp;
241 state->common.pc = new_pc;
242 state->source = KUNWIND_SOURCE_FRAME;
243
244 return 0;
245}
246
247/*
248 * Unwind from one frame record (A) to the next frame record (B).
249 *
250 * We terminate early if the location of B indicates a malformed chain of frame
251 * records (e.g. a cycle), determined based on the location and fp value of A
252 * and the location (but not the fp value) of B.
253 */
254static __always_inline int
255kunwind_next(struct kunwind_state *state)
256{
257 int err;
258
259 state->flags.all = 0;
260
261 switch (state->source) {
262 case KUNWIND_SOURCE_FRAME:
263 case KUNWIND_SOURCE_CALLER:
264 case KUNWIND_SOURCE_TASK:
265 case KUNWIND_SOURCE_REGS_PC:
266 err = kunwind_next_frame_record(state);
267 break;
268 default:
269 err = -EINVAL;
270 }
271
272 if (err)
273 return err;
274
275 state->common.pc = ptrauth_strip_kernel_insn_pac(state->common.pc);
276
277 return kunwind_recover_return_address(state);
278}
279
280typedef bool (*kunwind_consume_fn)(const struct kunwind_state *state, void *cookie);
281
282static __always_inline int
283do_kunwind(struct kunwind_state *state, kunwind_consume_fn consume_state,
284 void *cookie)
285{
286 int ret;
287
288 ret = kunwind_recover_return_address(state);
289 if (ret)
290 return ret;
291
292 while (1) {
293 if (!consume_state(state, cookie))
294 return -EINVAL;
295 ret = kunwind_next(state);
296 if (ret == -ENOENT)
297 return 0;
298 if (ret < 0)
299 return ret;
300 }
301}
302
303/*
304 * Per-cpu stacks are only accessible when unwinding the current task in a
305 * non-preemptible context.
306 */
307#define STACKINFO_CPU(name) \
308 ({ \
309 ((task == current) && !preemptible()) \
310 ? stackinfo_get_##name() \
311 : stackinfo_get_unknown(); \
312 })
313
314/*
315 * SDEI stacks are only accessible when unwinding the current task in an NMI
316 * context.
317 */
318#define STACKINFO_SDEI(name) \
319 ({ \
320 ((task == current) && in_nmi()) \
321 ? stackinfo_get_sdei_##name() \
322 : stackinfo_get_unknown(); \
323 })
324
325#define STACKINFO_EFI \
326 ({ \
327 ((task == current) && current_in_efi()) \
328 ? stackinfo_get_efi() \
329 : stackinfo_get_unknown(); \
330 })
331
332static __always_inline int
333kunwind_stack_walk(kunwind_consume_fn consume_state,
334 void *cookie, struct task_struct *task,
335 struct pt_regs *regs)
336{
337 struct stack_info stacks[] = {
338 stackinfo_get_task(task),
339 STACKINFO_CPU(irq),
340 STACKINFO_CPU(overflow),
341#if defined(CONFIG_ARM_SDE_INTERFACE)
342 STACKINFO_SDEI(normal),
343 STACKINFO_SDEI(critical),
344#endif
345#ifdef CONFIG_EFI
346 STACKINFO_EFI,
347#endif
348 };
349 struct kunwind_state state = {
350 .common = {
351 .stacks = stacks,
352 .nr_stacks = ARRAY_SIZE(stacks),
353 },
354 };
355
356 if (regs) {
357 if (task != current)
358 return -EINVAL;
359 kunwind_init_from_regs(state: &state, regs);
360 } else if (task == current) {
361 kunwind_init_from_caller(state: &state);
362 } else {
363 kunwind_init_from_task(state: &state, task);
364 }
365
366 return do_kunwind(state: &state, consume_state, cookie);
367}
368
369struct kunwind_consume_entry_data {
370 stack_trace_consume_fn consume_entry;
371 void *cookie;
372};
373
374static __always_inline bool
375arch_kunwind_consume_entry(const struct kunwind_state *state, void *cookie)
376{
377 struct kunwind_consume_entry_data *data = cookie;
378 return data->consume_entry(data->cookie, state->common.pc);
379}
380
381noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry,
382 void *cookie, struct task_struct *task,
383 struct pt_regs *regs)
384{
385 struct kunwind_consume_entry_data data = {
386 .consume_entry = consume_entry,
387 .cookie = cookie,
388 };
389
390 kunwind_stack_walk(consume_state: arch_kunwind_consume_entry, cookie: &data, task, regs);
391}
392
393static __always_inline bool
394arch_reliable_kunwind_consume_entry(const struct kunwind_state *state, void *cookie)
395{
396 /*
397 * At an exception boundary we can reliably consume the saved PC. We do
398 * not know whether the LR was live when the exception was taken, and
399 * so we cannot perform the next unwind step reliably.
400 *
401 * All that matters is whether the *entire* unwind is reliable, so give
402 * up as soon as we hit an exception boundary.
403 */
404 if (state->source == KUNWIND_SOURCE_REGS_PC)
405 return false;
406
407 return arch_kunwind_consume_entry(state, cookie);
408}
409
410noinline noinstr int arch_stack_walk_reliable(stack_trace_consume_fn consume_entry,
411 void *cookie,
412 struct task_struct *task)
413{
414 struct kunwind_consume_entry_data data = {
415 .consume_entry = consume_entry,
416 .cookie = cookie,
417 };
418
419 return kunwind_stack_walk(consume_state: arch_reliable_kunwind_consume_entry, cookie: &data,
420 task, NULL);
421}
422
423struct bpf_unwind_consume_entry_data {
424 bool (*consume_entry)(void *cookie, u64 ip, u64 sp, u64 fp);
425 void *cookie;
426};
427
428static bool
429arch_bpf_unwind_consume_entry(const struct kunwind_state *state, void *cookie)
430{
431 struct bpf_unwind_consume_entry_data *data = cookie;
432
433 return data->consume_entry(data->cookie, state->common.pc, 0,
434 state->common.fp);
435}
436
437noinline noinstr void arch_bpf_stack_walk(bool (*consume_entry)(void *cookie, u64 ip, u64 sp,
438 u64 fp), void *cookie)
439{
440 struct bpf_unwind_consume_entry_data data = {
441 .consume_entry = consume_entry,
442 .cookie = cookie,
443 };
444
445 kunwind_stack_walk(consume_state: arch_bpf_unwind_consume_entry, cookie: &data, current, NULL);
446}
447
448static const char *state_source_string(const struct kunwind_state *state)
449{
450 switch (state->source) {
451 case KUNWIND_SOURCE_FRAME: return NULL;
452 case KUNWIND_SOURCE_CALLER: return "C";
453 case KUNWIND_SOURCE_TASK: return "T";
454 case KUNWIND_SOURCE_REGS_PC: return "P";
455 default: return "U";
456 }
457}
458
459static bool dump_backtrace_entry(const struct kunwind_state *state, void *arg)
460{
461 const char *source = state_source_string(state);
462 union unwind_flags flags = state->flags;
463 bool has_info = source || flags.all;
464 char *loglvl = arg;
465
466 printk("%s %pSb%s%s%s%s%s\n", loglvl,
467 (void *)state->common.pc,
468 has_info ? " (" : "",
469 source ? source : "",
470 flags.fgraph ? "F" : "",
471 flags.kretprobe ? "K" : "",
472 has_info ? ")" : "");
473
474 return true;
475}
476
477void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
478 const char *loglvl)
479{
480 pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
481
482 if (regs && user_mode(regs))
483 return;
484
485 if (!tsk)
486 tsk = current;
487
488 if (!try_get_task_stack(tsk))
489 return;
490
491 printk("%sCall trace:\n", loglvl);
492 kunwind_stack_walk(consume_state: dump_backtrace_entry, cookie: (void *)loglvl, task: tsk, regs);
493
494 put_task_stack(tsk);
495}
496
497void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
498{
499 dump_backtrace(NULL, tsk, loglvl);
500 barrier();
501}
502
503/*
504 * The struct defined for userspace stack frame in AARCH64 mode.
505 */
506struct frame_tail {
507 struct frame_tail __user *fp;
508 unsigned long lr;
509} __attribute__((packed));
510
511/*
512 * Get the return address for a single stackframe and return a pointer to the
513 * next frame tail.
514 */
515static struct frame_tail __user *
516unwind_user_frame(struct frame_tail __user *tail, void *cookie,
517 stack_trace_consume_fn consume_entry)
518{
519 struct frame_tail buftail;
520 unsigned long err;
521 unsigned long lr;
522
523 /* Also check accessibility of one struct frame_tail beyond */
524 if (!access_ok(tail, sizeof(buftail)))
525 return NULL;
526
527 pagefault_disable();
528 err = __copy_from_user_inatomic(to: &buftail, from: tail, n: sizeof(buftail));
529 pagefault_enable();
530
531 if (err)
532 return NULL;
533
534 lr = ptrauth_strip_user_insn_pac(buftail.lr);
535
536 if (!consume_entry(cookie, lr))
537 return NULL;
538
539 /*
540 * Frame pointers should strictly progress back up the stack
541 * (towards higher addresses).
542 */
543 if (tail >= buftail.fp)
544 return NULL;
545
546 return buftail.fp;
547}
548
549#ifdef CONFIG_COMPAT
550/*
551 * The registers we're interested in are at the end of the variable
552 * length saved register structure. The fp points at the end of this
553 * structure so the address of this struct is:
554 * (struct compat_frame_tail *)(xxx->fp)-1
555 *
556 * This code has been adapted from the ARM OProfile support.
557 */
558struct compat_frame_tail {
559 compat_uptr_t fp; /* a (struct compat_frame_tail *) in compat mode */
560 u32 sp;
561 u32 lr;
562} __attribute__((packed));
563
564static struct compat_frame_tail __user *
565unwind_compat_user_frame(struct compat_frame_tail __user *tail, void *cookie,
566 stack_trace_consume_fn consume_entry)
567{
568 struct compat_frame_tail buftail;
569 unsigned long err;
570
571 /* Also check accessibility of one struct frame_tail beyond */
572 if (!access_ok(tail, sizeof(buftail)))
573 return NULL;
574
575 pagefault_disable();
576 err = __copy_from_user_inatomic(to: &buftail, from: tail, n: sizeof(buftail));
577 pagefault_enable();
578
579 if (err)
580 return NULL;
581
582 if (!consume_entry(cookie, buftail.lr))
583 return NULL;
584
585 /*
586 * Frame pointers should strictly progress back up the stack
587 * (towards higher addresses).
588 */
589 if (tail + 1 >= (struct compat_frame_tail __user *)
590 compat_ptr(uptr: buftail.fp))
591 return NULL;
592
593 return (struct compat_frame_tail __user *)compat_ptr(uptr: buftail.fp) - 1;
594}
595#endif /* CONFIG_COMPAT */
596
597
598void arch_stack_walk_user(stack_trace_consume_fn consume_entry, void *cookie,
599 const struct pt_regs *regs)
600{
601 if (!consume_entry(cookie, regs->pc))
602 return;
603
604 if (!compat_user_mode(regs)) {
605 /* AARCH64 mode */
606 struct frame_tail __user *tail;
607
608 tail = (struct frame_tail __user *)regs->regs[29];
609 while (tail && !((unsigned long)tail & 0x7))
610 tail = unwind_user_frame(tail, cookie, consume_entry);
611 } else {
612#ifdef CONFIG_COMPAT
613 /* AARCH32 compat mode */
614 struct compat_frame_tail __user *tail;
615
616 tail = (struct compat_frame_tail __user *)regs->compat_fp - 1;
617 while (tail && !((unsigned long)tail & 0x3))
618 tail = unwind_compat_user_frame(tail, cookie, consume_entry);
619#endif
620 }
621}
622

source code of linux/arch/arm64/kernel/stacktrace.c