1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Dynamic function tracer architecture backend. |
4 | * |
5 | * Copyright IBM Corp. 2009,2014 |
6 | * |
7 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> |
8 | */ |
9 | |
10 | #include <linux/moduleloader.h> |
11 | #include <linux/hardirq.h> |
12 | #include <linux/uaccess.h> |
13 | #include <linux/ftrace.h> |
14 | #include <linux/kernel.h> |
15 | #include <linux/types.h> |
16 | #include <linux/kprobes.h> |
17 | #include <trace/syscall.h> |
18 | #include <asm/asm-offsets.h> |
19 | #include <asm/text-patching.h> |
20 | #include <asm/cacheflush.h> |
21 | #include <asm/ftrace.lds.h> |
22 | #include <asm/nospec-branch.h> |
23 | #include <asm/set_memory.h> |
24 | #include "entry.h" |
25 | #include "ftrace.h" |
26 | |
27 | /* |
28 | * To generate function prologue either gcc's hotpatch feature (since gcc 4.8) |
29 | * or a combination of -pg -mrecord-mcount -mnop-mcount -mfentry flags |
30 | * (since gcc 9 / clang 10) is used. |
31 | * In both cases the original and also the disabled function prologue contains |
32 | * only a single six byte instruction and looks like this: |
33 | * > brcl 0,0 # offset 0 |
34 | * To enable ftrace the code gets patched like above and afterwards looks |
35 | * like this: |
36 | * > brasl %r0,ftrace_caller # offset 0 |
37 | * |
38 | * The instruction will be patched by ftrace_make_call / ftrace_make_nop. |
39 | * The ftrace function gets called with a non-standard C function call ABI |
40 | * where r0 contains the return address. It is also expected that the called |
41 | * function only clobbers r0 and r1, but restores r2-r15. |
42 | * For module code we can't directly jump to ftrace caller, but need a |
43 | * trampoline (ftrace_plt), which clobbers also r1. |
44 | */ |
45 | |
46 | void *ftrace_func __read_mostly = ftrace_stub; |
47 | struct ftrace_insn { |
48 | u16 opc; |
49 | s32 disp; |
50 | } __packed; |
51 | |
52 | #ifdef CONFIG_MODULES |
53 | static char *ftrace_plt; |
54 | #endif /* CONFIG_MODULES */ |
55 | |
56 | static const char *ftrace_shared_hotpatch_trampoline(const char **end) |
57 | { |
58 | const char *tstart, *tend; |
59 | |
60 | tstart = ftrace_shared_hotpatch_trampoline_br; |
61 | tend = ftrace_shared_hotpatch_trampoline_br_end; |
62 | #ifdef CONFIG_EXPOLINE |
63 | if (!nospec_disable) { |
64 | tstart = ftrace_shared_hotpatch_trampoline_exrl; |
65 | tend = ftrace_shared_hotpatch_trampoline_exrl_end; |
66 | } |
67 | #endif /* CONFIG_EXPOLINE */ |
68 | if (end) |
69 | *end = tend; |
70 | return tstart; |
71 | } |
72 | |
73 | bool ftrace_need_init_nop(void) |
74 | { |
75 | return true; |
76 | } |
77 | |
78 | int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) |
79 | { |
80 | static struct ftrace_hotpatch_trampoline *next_vmlinux_trampoline = |
81 | __ftrace_hotpatch_trampolines_start; |
82 | static const char orig[6] = { 0xc0, 0x04, 0x00, 0x00, 0x00, 0x00 }; |
83 | static struct ftrace_hotpatch_trampoline *trampoline; |
84 | struct ftrace_hotpatch_trampoline **next_trampoline; |
85 | struct ftrace_hotpatch_trampoline *trampolines_end; |
86 | struct ftrace_hotpatch_trampoline tmp; |
87 | struct ftrace_insn *insn; |
88 | const char *shared; |
89 | s32 disp; |
90 | |
91 | BUILD_BUG_ON(sizeof(struct ftrace_hotpatch_trampoline) != |
92 | SIZEOF_FTRACE_HOTPATCH_TRAMPOLINE); |
93 | |
94 | next_trampoline = &next_vmlinux_trampoline; |
95 | trampolines_end = __ftrace_hotpatch_trampolines_end; |
96 | shared = ftrace_shared_hotpatch_trampoline(NULL); |
97 | #ifdef CONFIG_MODULES |
98 | if (mod) { |
99 | next_trampoline = &mod->arch.next_trampoline; |
100 | trampolines_end = mod->arch.trampolines_end; |
101 | shared = ftrace_plt; |
102 | } |
103 | #endif |
104 | |
105 | if (WARN_ON_ONCE(*next_trampoline >= trampolines_end)) |
106 | return -ENOMEM; |
107 | trampoline = (*next_trampoline)++; |
108 | |
109 | /* Check for the compiler-generated fentry nop (brcl 0, .). */ |
110 | if (WARN_ON_ONCE(memcmp((const void *)rec->ip, &orig, sizeof(orig)))) |
111 | return -EINVAL; |
112 | |
113 | /* Generate the trampoline. */ |
114 | tmp.brasl_opc = 0xc015; /* brasl %r1, shared */ |
115 | tmp.brasl_disp = (shared - (const char *)&trampoline->brasl_opc) / 2; |
116 | tmp.interceptor = FTRACE_ADDR; |
117 | tmp.rest_of_intercepted_function = rec->ip + sizeof(struct ftrace_insn); |
118 | s390_kernel_write(trampoline, &tmp, sizeof(tmp)); |
119 | |
120 | /* Generate a jump to the trampoline. */ |
121 | disp = ((char *)trampoline - (char *)rec->ip) / 2; |
122 | insn = (struct ftrace_insn *)rec->ip; |
123 | s390_kernel_write(&insn->disp, &disp, sizeof(disp)); |
124 | |
125 | return 0; |
126 | } |
127 | |
128 | static struct ftrace_hotpatch_trampoline *ftrace_get_trampoline(struct dyn_ftrace *rec) |
129 | { |
130 | struct ftrace_hotpatch_trampoline *trampoline; |
131 | struct ftrace_insn insn; |
132 | s64 disp; |
133 | u16 opc; |
134 | |
135 | if (copy_from_kernel_nofault(dst: &insn, src: (void *)rec->ip, size: sizeof(insn))) |
136 | return ERR_PTR(error: -EFAULT); |
137 | disp = (s64)insn.disp * 2; |
138 | trampoline = (void *)(rec->ip + disp); |
139 | if (get_kernel_nofault(opc, &trampoline->brasl_opc)) |
140 | return ERR_PTR(error: -EFAULT); |
141 | if (opc != 0xc015) |
142 | return ERR_PTR(error: -EINVAL); |
143 | return trampoline; |
144 | } |
145 | |
146 | int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, |
147 | unsigned long addr) |
148 | { |
149 | struct ftrace_hotpatch_trampoline *trampoline; |
150 | u64 old; |
151 | |
152 | trampoline = ftrace_get_trampoline(rec); |
153 | if (IS_ERR(ptr: trampoline)) |
154 | return PTR_ERR(ptr: trampoline); |
155 | if (get_kernel_nofault(old, &trampoline->interceptor)) |
156 | return -EFAULT; |
157 | if (old != old_addr) |
158 | return -EINVAL; |
159 | s390_kernel_write(&trampoline->interceptor, &addr, sizeof(addr)); |
160 | return 0; |
161 | } |
162 | |
163 | static int ftrace_patch_branch_mask(void *addr, u16 expected, bool enable) |
164 | { |
165 | u16 old; |
166 | u8 op; |
167 | |
168 | if (get_kernel_nofault(old, addr)) |
169 | return -EFAULT; |
170 | if (old != expected) |
171 | return -EINVAL; |
172 | /* set mask field to all ones or zeroes */ |
173 | op = enable ? 0xf4 : 0x04; |
174 | s390_kernel_write((char *)addr + 1, &op, sizeof(op)); |
175 | return 0; |
176 | } |
177 | |
178 | int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, |
179 | unsigned long addr) |
180 | { |
181 | /* Expect brcl 0xf,... */ |
182 | return ftrace_patch_branch_mask(addr: (void *)rec->ip, expected: 0xc0f4, enable: false); |
183 | } |
184 | |
185 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
186 | { |
187 | struct ftrace_hotpatch_trampoline *trampoline; |
188 | |
189 | trampoline = ftrace_get_trampoline(rec); |
190 | if (IS_ERR(ptr: trampoline)) |
191 | return PTR_ERR(ptr: trampoline); |
192 | s390_kernel_write(&trampoline->interceptor, &addr, sizeof(addr)); |
193 | /* Expect brcl 0x0,... */ |
194 | return ftrace_patch_branch_mask(addr: (void *)rec->ip, expected: 0xc004, enable: true); |
195 | } |
196 | |
197 | int ftrace_update_ftrace_func(ftrace_func_t func) |
198 | { |
199 | ftrace_func = func; |
200 | return 0; |
201 | } |
202 | |
203 | void arch_ftrace_update_code(int command) |
204 | { |
205 | ftrace_modify_all_code(command); |
206 | } |
207 | |
208 | void ftrace_arch_code_modify_post_process(void) |
209 | { |
210 | /* |
211 | * Flush any pre-fetched instructions on all |
212 | * CPUs to make the new code visible. |
213 | */ |
214 | text_poke_sync_lock(); |
215 | } |
216 | |
217 | #ifdef CONFIG_MODULES |
218 | |
219 | static int __init ftrace_plt_init(void) |
220 | { |
221 | const char *start, *end; |
222 | |
223 | ftrace_plt = module_alloc(PAGE_SIZE); |
224 | if (!ftrace_plt) |
225 | panic(fmt: "cannot allocate ftrace plt\n" ); |
226 | |
227 | start = ftrace_shared_hotpatch_trampoline(end: &end); |
228 | memcpy(ftrace_plt, start, end - start); |
229 | set_memory_rox(addr: (unsigned long)ftrace_plt, numpages: 1); |
230 | return 0; |
231 | } |
232 | device_initcall(ftrace_plt_init); |
233 | |
234 | #endif /* CONFIG_MODULES */ |
235 | |
236 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
237 | /* |
238 | * Hook the return address and push it in the stack of return addresses |
239 | * in current thread info. |
240 | */ |
241 | unsigned long prepare_ftrace_return(unsigned long ra, unsigned long sp, |
242 | unsigned long ip) |
243 | { |
244 | if (unlikely(ftrace_graph_is_dead())) |
245 | goto out; |
246 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
247 | goto out; |
248 | ip -= MCOUNT_INSN_SIZE; |
249 | if (!function_graph_enter(ret: ra, func: ip, frame_pointer: 0, retp: (void *) sp)) |
250 | ra = (unsigned long) return_to_handler; |
251 | out: |
252 | return ra; |
253 | } |
254 | NOKPROBE_SYMBOL(prepare_ftrace_return); |
255 | |
256 | /* |
257 | * Patch the kernel code at ftrace_graph_caller location. The instruction |
258 | * there is branch relative on condition. To enable the ftrace graph code |
259 | * block, we simply patch the mask field of the instruction to zero and |
260 | * turn the instruction into a nop. |
261 | * To disable the ftrace graph code the mask field will be patched to |
262 | * all ones, which turns the instruction into an unconditional branch. |
263 | */ |
264 | int ftrace_enable_ftrace_graph_caller(void) |
265 | { |
266 | int rc; |
267 | |
268 | /* Expect brc 0xf,... */ |
269 | rc = ftrace_patch_branch_mask(addr: ftrace_graph_caller, expected: 0xa7f4, enable: false); |
270 | if (rc) |
271 | return rc; |
272 | text_poke_sync_lock(); |
273 | return 0; |
274 | } |
275 | |
276 | int ftrace_disable_ftrace_graph_caller(void) |
277 | { |
278 | int rc; |
279 | |
280 | /* Expect brc 0x0,... */ |
281 | rc = ftrace_patch_branch_mask(addr: ftrace_graph_caller, expected: 0xa704, enable: true); |
282 | if (rc) |
283 | return rc; |
284 | text_poke_sync_lock(); |
285 | return 0; |
286 | } |
287 | |
288 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
289 | |
290 | #ifdef CONFIG_KPROBES_ON_FTRACE |
291 | void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, |
292 | struct ftrace_ops *ops, struct ftrace_regs *fregs) |
293 | { |
294 | struct kprobe_ctlblk *kcb; |
295 | struct pt_regs *regs; |
296 | struct kprobe *p; |
297 | int bit; |
298 | |
299 | bit = ftrace_test_recursion_trylock(ip, parent_ip); |
300 | if (bit < 0) |
301 | return; |
302 | |
303 | regs = ftrace_get_regs(fregs); |
304 | p = get_kprobe(addr: (kprobe_opcode_t *)ip); |
305 | if (!regs || unlikely(!p) || kprobe_disabled(p)) |
306 | goto out; |
307 | |
308 | if (kprobe_running()) { |
309 | kprobes_inc_nmissed_count(p); |
310 | goto out; |
311 | } |
312 | |
313 | __this_cpu_write(current_kprobe, p); |
314 | |
315 | kcb = get_kprobe_ctlblk(); |
316 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
317 | |
318 | instruction_pointer_set(regs, val: ip); |
319 | |
320 | if (!p->pre_handler || !p->pre_handler(p, regs)) { |
321 | |
322 | instruction_pointer_set(regs, val: ip + MCOUNT_INSN_SIZE); |
323 | |
324 | if (unlikely(p->post_handler)) { |
325 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
326 | p->post_handler(p, regs, 0); |
327 | } |
328 | } |
329 | __this_cpu_write(current_kprobe, NULL); |
330 | out: |
331 | ftrace_test_recursion_unlock(bit); |
332 | } |
333 | NOKPROBE_SYMBOL(kprobe_ftrace_handler); |
334 | |
335 | int arch_prepare_kprobe_ftrace(struct kprobe *p) |
336 | { |
337 | p->ainsn.insn = NULL; |
338 | return 0; |
339 | } |
340 | #endif |
341 | |