1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Shadow Call Stack support. |
4 | * |
5 | * Copyright (C) 2019 Google LLC |
6 | */ |
7 | |
8 | #include <linux/cpuhotplug.h> |
9 | #include <linux/kasan.h> |
10 | #include <linux/mm.h> |
11 | #include <linux/scs.h> |
12 | #include <linux/vmalloc.h> |
13 | #include <linux/vmstat.h> |
14 | |
15 | #ifdef CONFIG_DYNAMIC_SCS |
16 | DEFINE_STATIC_KEY_FALSE(dynamic_scs_enabled); |
17 | #endif |
18 | |
19 | static void __scs_account(void *s, int account) |
20 | { |
21 | struct page *scs_page = vmalloc_to_page(addr: s); |
22 | |
23 | mod_node_page_state(page_pgdat(scs_page), NR_KERNEL_SCS_KB, |
24 | account * (SCS_SIZE / SZ_1K)); |
25 | } |
26 | |
27 | /* Matches NR_CACHED_STACKS for VMAP_STACK */ |
28 | #define NR_CACHED_SCS 2 |
29 | static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]); |
30 | |
31 | static void *__scs_alloc(int node) |
32 | { |
33 | int i; |
34 | void *s; |
35 | |
36 | for (i = 0; i < NR_CACHED_SCS; i++) { |
37 | s = this_cpu_xchg(scs_cache[i], NULL); |
38 | if (s) { |
39 | s = kasan_unpoison_vmalloc(start: s, size: SCS_SIZE, |
40 | KASAN_VMALLOC_PROT_NORMAL); |
41 | memset(s, 0, SCS_SIZE); |
42 | goto out; |
43 | } |
44 | } |
45 | |
46 | s = __vmalloc_node_range(size: SCS_SIZE, align: 1, VMALLOC_START, VMALLOC_END, |
47 | gfp_mask: GFP_SCS, PAGE_KERNEL, vm_flags: 0, node, |
48 | caller: __builtin_return_address(0)); |
49 | |
50 | out: |
51 | return kasan_reset_tag(addr: s); |
52 | } |
53 | |
54 | void *scs_alloc(int node) |
55 | { |
56 | void *s; |
57 | |
58 | s = __scs_alloc(node); |
59 | if (!s) |
60 | return NULL; |
61 | |
62 | *__scs_magic(s) = SCS_END_MAGIC; |
63 | |
64 | /* |
65 | * Poison the allocation to catch unintentional accesses to |
66 | * the shadow stack when KASAN is enabled. |
67 | */ |
68 | kasan_poison_vmalloc(start: s, size: SCS_SIZE); |
69 | __scs_account(s, account: 1); |
70 | return s; |
71 | } |
72 | |
73 | void scs_free(void *s) |
74 | { |
75 | int i; |
76 | |
77 | __scs_account(s, account: -1); |
78 | |
79 | /* |
80 | * We cannot sleep as this can be called in interrupt context, |
81 | * so use this_cpu_cmpxchg to update the cache, and vfree_atomic |
82 | * to free the stack. |
83 | */ |
84 | |
85 | for (i = 0; i < NR_CACHED_SCS; i++) |
86 | if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL) |
87 | return; |
88 | |
89 | kasan_unpoison_vmalloc(start: s, size: SCS_SIZE, KASAN_VMALLOC_PROT_NORMAL); |
90 | vfree_atomic(addr: s); |
91 | } |
92 | |
93 | static int scs_cleanup(unsigned int cpu) |
94 | { |
95 | int i; |
96 | void **cache = per_cpu_ptr(scs_cache, cpu); |
97 | |
98 | for (i = 0; i < NR_CACHED_SCS; i++) { |
99 | vfree(addr: cache[i]); |
100 | cache[i] = NULL; |
101 | } |
102 | |
103 | return 0; |
104 | } |
105 | |
106 | void __init scs_init(void) |
107 | { |
108 | if (!scs_is_enabled()) |
109 | return; |
110 | cpuhp_setup_state(state: CPUHP_BP_PREPARE_DYN, name: "scs:scs_cache" , NULL, |
111 | teardown: scs_cleanup); |
112 | } |
113 | |
114 | int scs_prepare(struct task_struct *tsk, int node) |
115 | { |
116 | void *s; |
117 | |
118 | if (!scs_is_enabled()) |
119 | return 0; |
120 | |
121 | s = scs_alloc(node); |
122 | if (!s) |
123 | return -ENOMEM; |
124 | |
125 | task_scs(tsk) = task_scs_sp(tsk) = s; |
126 | return 0; |
127 | } |
128 | |
129 | static void scs_check_usage(struct task_struct *tsk) |
130 | { |
131 | static unsigned long highest; |
132 | |
133 | unsigned long *p, prev, curr = highest, used = 0; |
134 | |
135 | if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE)) |
136 | return; |
137 | |
138 | for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) { |
139 | if (!READ_ONCE_NOCHECK(*p)) |
140 | break; |
141 | used += sizeof(*p); |
142 | } |
143 | |
144 | while (used > curr) { |
145 | prev = cmpxchg_relaxed(&highest, curr, used); |
146 | |
147 | if (prev == curr) { |
148 | pr_info("%s (%d): highest shadow stack usage: %lu bytes\n" , |
149 | tsk->comm, task_pid_nr(tsk), used); |
150 | break; |
151 | } |
152 | |
153 | curr = prev; |
154 | } |
155 | } |
156 | |
157 | void scs_release(struct task_struct *tsk) |
158 | { |
159 | void *s = task_scs(tsk); |
160 | |
161 | if (!scs_is_enabled() || !s) |
162 | return; |
163 | |
164 | WARN(task_scs_end_corrupted(tsk), |
165 | "corrupted shadow stack detected when freeing task\n" ); |
166 | scs_check_usage(tsk); |
167 | scs_free(s); |
168 | } |
169 | |