1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2020 SiFive |
4 | */ |
5 | |
6 | #include <linux/spinlock.h> |
7 | #include <linux/mm.h> |
8 | #include <linux/memory.h> |
9 | #include <linux/string.h> |
10 | #include <linux/uaccess.h> |
11 | #include <linux/stop_machine.h> |
12 | #include <asm/kprobes.h> |
13 | #include <asm/cacheflush.h> |
14 | #include <asm/fixmap.h> |
15 | #include <asm/ftrace.h> |
16 | #include <asm/patch.h> |
17 | #include <asm/sections.h> |
18 | |
19 | struct patch_insn { |
20 | void *addr; |
21 | u32 *insns; |
22 | int ninsns; |
23 | atomic_t cpu_count; |
24 | }; |
25 | |
26 | int riscv_patch_in_stop_machine = false; |
27 | |
28 | #ifdef CONFIG_MMU |
29 | |
30 | static inline bool is_kernel_exittext(uintptr_t addr) |
31 | { |
32 | return system_state < SYSTEM_RUNNING && |
33 | addr >= (uintptr_t)__exittext_begin && |
34 | addr < (uintptr_t)__exittext_end; |
35 | } |
36 | |
37 | /* |
38 | * The fix_to_virt(, idx) needs a const value (not a dynamic variable of |
39 | * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses". |
40 | * So use '__always_inline' and 'const unsigned int fixmap' here. |
41 | */ |
42 | static __always_inline void *patch_map(void *addr, const unsigned int fixmap) |
43 | { |
44 | uintptr_t uintaddr = (uintptr_t) addr; |
45 | struct page *page; |
46 | |
47 | if (core_kernel_text(addr: uintaddr) || is_kernel_exittext(addr: uintaddr)) |
48 | page = phys_to_page(__pa_symbol(addr)); |
49 | else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) |
50 | page = vmalloc_to_page(addr); |
51 | else |
52 | return addr; |
53 | |
54 | BUG_ON(!page); |
55 | |
56 | return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + |
57 | (uintaddr & ~PAGE_MASK)); |
58 | } |
59 | |
60 | static void patch_unmap(int fixmap) |
61 | { |
62 | clear_fixmap(fixmap); |
63 | } |
64 | NOKPROBE_SYMBOL(patch_unmap); |
65 | |
66 | static int __patch_insn_set(void *addr, u8 c, size_t len) |
67 | { |
68 | void *waddr = addr; |
69 | bool across_pages = (((uintptr_t)addr & ~PAGE_MASK) + len) > PAGE_SIZE; |
70 | |
71 | /* |
72 | * Only two pages can be mapped at a time for writing. |
73 | */ |
74 | if (len + offset_in_page(addr) > 2 * PAGE_SIZE) |
75 | return -EINVAL; |
76 | /* |
77 | * Before reaching here, it was expected to lock the text_mutex |
78 | * already, so we don't need to give another lock here and could |
79 | * ensure that it was safe between each cores. |
80 | */ |
81 | lockdep_assert_held(&text_mutex); |
82 | |
83 | preempt_disable(); |
84 | |
85 | if (across_pages) |
86 | patch_map(addr: addr + PAGE_SIZE, fixmap: FIX_TEXT_POKE1); |
87 | |
88 | waddr = patch_map(addr, fixmap: FIX_TEXT_POKE0); |
89 | |
90 | memset(waddr, c, len); |
91 | |
92 | patch_unmap(fixmap: FIX_TEXT_POKE0); |
93 | |
94 | if (across_pages) |
95 | patch_unmap(fixmap: FIX_TEXT_POKE1); |
96 | |
97 | preempt_enable(); |
98 | |
99 | return 0; |
100 | } |
101 | NOKPROBE_SYMBOL(__patch_insn_set); |
102 | |
103 | static int __patch_insn_write(void *addr, const void *insn, size_t len) |
104 | { |
105 | void *waddr = addr; |
106 | bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE; |
107 | int ret; |
108 | |
109 | /* |
110 | * Only two pages can be mapped at a time for writing. |
111 | */ |
112 | if (len + offset_in_page(addr) > 2 * PAGE_SIZE) |
113 | return -EINVAL; |
114 | |
115 | /* |
116 | * Before reaching here, it was expected to lock the text_mutex |
117 | * already, so we don't need to give another lock here and could |
118 | * ensure that it was safe between each cores. |
119 | * |
120 | * We're currently using stop_machine() for ftrace & kprobes, and while |
121 | * that ensures text_mutex is held before installing the mappings it |
122 | * does not ensure text_mutex is held by the calling thread. That's |
123 | * safe but triggers a lockdep failure, so just elide it for that |
124 | * specific case. |
125 | */ |
126 | if (!riscv_patch_in_stop_machine) |
127 | lockdep_assert_held(&text_mutex); |
128 | |
129 | preempt_disable(); |
130 | |
131 | if (across_pages) |
132 | patch_map(addr: addr + PAGE_SIZE, fixmap: FIX_TEXT_POKE1); |
133 | |
134 | waddr = patch_map(addr, fixmap: FIX_TEXT_POKE0); |
135 | |
136 | ret = copy_to_kernel_nofault(dst: waddr, src: insn, size: len); |
137 | |
138 | patch_unmap(fixmap: FIX_TEXT_POKE0); |
139 | |
140 | if (across_pages) |
141 | patch_unmap(fixmap: FIX_TEXT_POKE1); |
142 | |
143 | preempt_enable(); |
144 | |
145 | return ret; |
146 | } |
147 | NOKPROBE_SYMBOL(__patch_insn_write); |
148 | #else |
149 | static int __patch_insn_set(void *addr, u8 c, size_t len) |
150 | { |
151 | memset(addr, c, len); |
152 | |
153 | return 0; |
154 | } |
155 | NOKPROBE_SYMBOL(__patch_insn_set); |
156 | |
157 | static int __patch_insn_write(void *addr, const void *insn, size_t len) |
158 | { |
159 | return copy_to_kernel_nofault(addr, insn, len); |
160 | } |
161 | NOKPROBE_SYMBOL(__patch_insn_write); |
162 | #endif /* CONFIG_MMU */ |
163 | |
164 | static int patch_insn_set(void *addr, u8 c, size_t len) |
165 | { |
166 | size_t patched = 0; |
167 | size_t size; |
168 | int ret = 0; |
169 | |
170 | /* |
171 | * __patch_insn_set() can only work on 2 pages at a time so call it in a |
172 | * loop with len <= 2 * PAGE_SIZE. |
173 | */ |
174 | while (patched < len && !ret) { |
175 | size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched); |
176 | ret = __patch_insn_set(addr: addr + patched, c, len: size); |
177 | |
178 | patched += size; |
179 | } |
180 | |
181 | return ret; |
182 | } |
183 | NOKPROBE_SYMBOL(patch_insn_set); |
184 | |
185 | int patch_text_set_nosync(void *addr, u8 c, size_t len) |
186 | { |
187 | u32 *tp = addr; |
188 | int ret; |
189 | |
190 | ret = patch_insn_set(addr: tp, c, len); |
191 | |
192 | if (!ret) |
193 | flush_icache_range(start: (uintptr_t)tp, end: (uintptr_t)tp + len); |
194 | |
195 | return ret; |
196 | } |
197 | NOKPROBE_SYMBOL(patch_text_set_nosync); |
198 | |
199 | static int patch_insn_write(void *addr, const void *insn, size_t len) |
200 | { |
201 | size_t patched = 0; |
202 | size_t size; |
203 | int ret = 0; |
204 | |
205 | /* |
206 | * Copy the instructions to the destination address, two pages at a time |
207 | * because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE. |
208 | */ |
209 | while (patched < len && !ret) { |
210 | size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched); |
211 | ret = __patch_insn_write(addr: addr + patched, insn: insn + patched, len: size); |
212 | |
213 | patched += size; |
214 | } |
215 | |
216 | return ret; |
217 | } |
218 | NOKPROBE_SYMBOL(patch_insn_write); |
219 | |
220 | int patch_text_nosync(void *addr, const void *insns, size_t len) |
221 | { |
222 | u32 *tp = addr; |
223 | int ret; |
224 | |
225 | ret = patch_insn_write(addr: tp, insn: insns, len); |
226 | |
227 | if (!ret) |
228 | flush_icache_range(start: (uintptr_t) tp, end: (uintptr_t) tp + len); |
229 | |
230 | return ret; |
231 | } |
232 | NOKPROBE_SYMBOL(patch_text_nosync); |
233 | |
234 | static int patch_text_cb(void *data) |
235 | { |
236 | struct patch_insn *patch = data; |
237 | unsigned long len; |
238 | int i, ret = 0; |
239 | |
240 | if (atomic_inc_return(v: &patch->cpu_count) == num_online_cpus()) { |
241 | for (i = 0; ret == 0 && i < patch->ninsns; i++) { |
242 | len = GET_INSN_LENGTH(patch->insns[i]); |
243 | ret = patch_text_nosync(addr: patch->addr + i * len, |
244 | insns: &patch->insns[i], len); |
245 | } |
246 | atomic_inc(v: &patch->cpu_count); |
247 | } else { |
248 | while (atomic_read(v: &patch->cpu_count) <= num_online_cpus()) |
249 | cpu_relax(); |
250 | smp_mb(); |
251 | } |
252 | |
253 | return ret; |
254 | } |
255 | NOKPROBE_SYMBOL(patch_text_cb); |
256 | |
257 | int patch_text(void *addr, u32 *insns, int ninsns) |
258 | { |
259 | int ret; |
260 | struct patch_insn patch = { |
261 | .addr = addr, |
262 | .insns = insns, |
263 | .ninsns = ninsns, |
264 | .cpu_count = ATOMIC_INIT(0), |
265 | }; |
266 | |
267 | /* |
268 | * kprobes takes text_mutex, before calling patch_text(), but as we call |
269 | * calls stop_machine(), the lockdep assertion in patch_insn_write() |
270 | * gets confused by the context in which the lock is taken. |
271 | * Instead, ensure the lock is held before calling stop_machine(), and |
272 | * set riscv_patch_in_stop_machine to skip the check in |
273 | * patch_insn_write(). |
274 | */ |
275 | lockdep_assert_held(&text_mutex); |
276 | riscv_patch_in_stop_machine = true; |
277 | ret = stop_machine_cpuslocked(fn: patch_text_cb, data: &patch, cpu_online_mask); |
278 | riscv_patch_in_stop_machine = false; |
279 | return ret; |
280 | } |
281 | NOKPROBE_SYMBOL(patch_text); |
282 | |