1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Code for replacing ftrace calls with jumps. |
4 | * |
5 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> |
6 | * |
7 | * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. |
8 | * |
9 | * Added function graph tracer code, taken from x86 that was written |
10 | * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. |
11 | * |
12 | */ |
13 | |
14 | #define pr_fmt(fmt) "ftrace-powerpc: " fmt |
15 | |
16 | #include <linux/spinlock.h> |
17 | #include <linux/hardirq.h> |
18 | #include <linux/uaccess.h> |
19 | #include <linux/module.h> |
20 | #include <linux/ftrace.h> |
21 | #include <linux/percpu.h> |
22 | #include <linux/init.h> |
23 | #include <linux/list.h> |
24 | |
25 | #include <asm/cacheflush.h> |
26 | #include <asm/code-patching.h> |
27 | #include <asm/ftrace.h> |
28 | #include <asm/syscall.h> |
29 | #include <asm/inst.h> |
30 | |
31 | /* |
32 | * We generally only have a single long_branch tramp and at most 2 or 3 plt |
33 | * tramps generated. But, we don't use the plt tramps currently. We also allot |
34 | * 2 tramps after .text and .init.text. So, we only end up with around 3 usable |
35 | * tramps in total. Set aside 8 just to be sure. |
36 | */ |
37 | #define NUM_FTRACE_TRAMPS 8 |
38 | static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS]; |
39 | |
40 | unsigned long ftrace_call_adjust(unsigned long addr) |
41 | { |
42 | return addr; |
43 | } |
44 | |
45 | static ppc_inst_t |
46 | ftrace_call_replace(unsigned long ip, unsigned long addr, int link) |
47 | { |
48 | ppc_inst_t op; |
49 | |
50 | addr = ppc_function_entry((void *)addr); |
51 | |
52 | /* if (link) set op to 'bl' else 'b' */ |
53 | create_branch(&op, (u32 *)ip, addr, link ? BRANCH_SET_LINK : 0); |
54 | |
55 | return op; |
56 | } |
57 | |
58 | static inline int |
59 | ftrace_modify_code(unsigned long ip, ppc_inst_t old, ppc_inst_t new) |
60 | { |
61 | ppc_inst_t replaced; |
62 | |
63 | /* |
64 | * Note: |
65 | * We are paranoid about modifying text, as if a bug was to happen, it |
66 | * could cause us to read or write to someplace that could cause harm. |
67 | * Carefully read and modify the code with probe_kernel_*(), and make |
68 | * sure what we read is what we expected it to be before modifying it. |
69 | */ |
70 | |
71 | /* read the text we want to modify */ |
72 | if (copy_inst_from_kernel_nofault(&replaced, (void *)ip)) |
73 | return -EFAULT; |
74 | |
75 | /* Make sure it is what we expect it to be */ |
76 | if (!ppc_inst_equal(replaced, old)) { |
77 | pr_err("%p: replaced (%08lx) != old (%08lx)" , (void *)ip, |
78 | ppc_inst_as_ulong(replaced), ppc_inst_as_ulong(old)); |
79 | return -EINVAL; |
80 | } |
81 | |
82 | /* replace the text with the new text */ |
83 | return patch_instruction((u32 *)ip, new); |
84 | } |
85 | |
86 | /* |
87 | * Helper functions that are the same for both PPC64 and PPC32. |
88 | */ |
89 | static int test_24bit_addr(unsigned long ip, unsigned long addr) |
90 | { |
91 | addr = ppc_function_entry((void *)addr); |
92 | |
93 | return is_offset_in_branch_range(addr - ip); |
94 | } |
95 | |
96 | static int is_bl_op(ppc_inst_t op) |
97 | { |
98 | return (ppc_inst_val(op) & ~PPC_LI_MASK) == PPC_RAW_BL(0); |
99 | } |
100 | |
101 | static int is_b_op(ppc_inst_t op) |
102 | { |
103 | return (ppc_inst_val(op) & ~PPC_LI_MASK) == PPC_RAW_BRANCH(0); |
104 | } |
105 | |
106 | static unsigned long find_bl_target(unsigned long ip, ppc_inst_t op) |
107 | { |
108 | int offset; |
109 | |
110 | offset = PPC_LI(ppc_inst_val(op)); |
111 | /* make it signed */ |
112 | if (offset & 0x02000000) |
113 | offset |= 0xfe000000; |
114 | |
115 | return ip + (long)offset; |
116 | } |
117 | |
118 | #ifdef CONFIG_MODULES |
119 | static int |
120 | __ftrace_make_nop(struct module *mod, |
121 | struct dyn_ftrace *rec, unsigned long addr) |
122 | { |
123 | unsigned long entry, ptr, tramp; |
124 | unsigned long ip = rec->ip; |
125 | ppc_inst_t op, pop; |
126 | |
127 | /* read where this goes */ |
128 | if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { |
129 | pr_err("Fetching opcode failed.\n" ); |
130 | return -EFAULT; |
131 | } |
132 | |
133 | /* Make sure that this is still a 24bit jump */ |
134 | if (!is_bl_op(op)) { |
135 | pr_err("Not expected bl: opcode is %08lx\n" , ppc_inst_as_ulong(op)); |
136 | return -EINVAL; |
137 | } |
138 | |
139 | /* lets find where the pointer goes */ |
140 | tramp = find_bl_target(ip, op); |
141 | |
142 | pr_devel("ip:%lx jumps to %lx" , ip, tramp); |
143 | |
144 | if (module_trampoline_target(mod, tramp, &ptr)) { |
145 | pr_err("Failed to get trampoline target\n" ); |
146 | return -EFAULT; |
147 | } |
148 | |
149 | pr_devel("trampoline target %lx" , ptr); |
150 | |
151 | entry = ppc_global_function_entry((void *)addr); |
152 | /* This should match what was called */ |
153 | if (ptr != entry) { |
154 | pr_err("addr %lx does not match expected %lx\n" , ptr, entry); |
155 | return -EINVAL; |
156 | } |
157 | |
158 | if (IS_ENABLED(CONFIG_MPROFILE_KERNEL)) { |
159 | if (copy_inst_from_kernel_nofault(&op, (void *)(ip - 4))) { |
160 | pr_err("Fetching instruction at %lx failed.\n" , ip - 4); |
161 | return -EFAULT; |
162 | } |
163 | |
164 | /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */ |
165 | if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_MFLR(_R0))) && |
166 | !ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) { |
167 | pr_err("Unexpected instruction %08lx around bl _mcount\n" , |
168 | ppc_inst_as_ulong(op)); |
169 | return -EINVAL; |
170 | } |
171 | } else if (IS_ENABLED(CONFIG_PPC64)) { |
172 | /* |
173 | * Check what is in the next instruction. We can see ld r2,40(r1), but |
174 | * on first pass after boot we will see mflr r0. |
175 | */ |
176 | if (copy_inst_from_kernel_nofault(&op, (void *)(ip + 4))) { |
177 | pr_err("Fetching op failed.\n" ); |
178 | return -EFAULT; |
179 | } |
180 | |
181 | if (!ppc_inst_equal(op, ppc_inst(PPC_INST_LD_TOC))) { |
182 | pr_err("Expected %08lx found %08lx\n" , PPC_INST_LD_TOC, |
183 | ppc_inst_as_ulong(op)); |
184 | return -EINVAL; |
185 | } |
186 | } |
187 | |
188 | /* |
189 | * When using -mprofile-kernel or PPC32 there is no load to jump over. |
190 | * |
191 | * Otherwise our original call site looks like: |
192 | * |
193 | * bl <tramp> |
194 | * ld r2,XX(r1) |
195 | * |
196 | * Milton Miller pointed out that we can not simply nop the branch. |
197 | * If a task was preempted when calling a trace function, the nops |
198 | * will remove the way to restore the TOC in r2 and the r2 TOC will |
199 | * get corrupted. |
200 | * |
201 | * Use a b +8 to jump over the load. |
202 | */ |
203 | if (IS_ENABLED(CONFIG_MPROFILE_KERNEL) || IS_ENABLED(CONFIG_PPC32)) |
204 | pop = ppc_inst(PPC_RAW_NOP()); |
205 | else |
206 | pop = ppc_inst(PPC_RAW_BRANCH(8)); /* b +8 */ |
207 | |
208 | if (patch_instruction((u32 *)ip, pop)) { |
209 | pr_err("Patching NOP failed.\n" ); |
210 | return -EPERM; |
211 | } |
212 | |
213 | return 0; |
214 | } |
215 | #else |
216 | static int __ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) |
217 | { |
218 | return 0; |
219 | } |
220 | #endif /* CONFIG_MODULES */ |
221 | |
222 | static unsigned long find_ftrace_tramp(unsigned long ip) |
223 | { |
224 | int i; |
225 | |
226 | /* |
227 | * We have the compiler generated long_branch tramps at the end |
228 | * and we prefer those |
229 | */ |
230 | for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--) |
231 | if (!ftrace_tramps[i]) |
232 | continue; |
233 | else if (is_offset_in_branch_range(ftrace_tramps[i] - ip)) |
234 | return ftrace_tramps[i]; |
235 | |
236 | return 0; |
237 | } |
238 | |
239 | static int add_ftrace_tramp(unsigned long tramp) |
240 | { |
241 | int i; |
242 | |
243 | for (i = 0; i < NUM_FTRACE_TRAMPS; i++) |
244 | if (!ftrace_tramps[i]) { |
245 | ftrace_tramps[i] = tramp; |
246 | return 0; |
247 | } |
248 | |
249 | return -1; |
250 | } |
251 | |
252 | /* |
253 | * If this is a compiler generated long_branch trampoline (essentially, a |
254 | * trampoline that has a branch to _mcount()), we re-write the branch to |
255 | * instead go to ftrace_[regs_]caller() and note down the location of this |
256 | * trampoline. |
257 | */ |
258 | static int setup_mcount_compiler_tramp(unsigned long tramp) |
259 | { |
260 | int i; |
261 | ppc_inst_t op; |
262 | unsigned long ptr; |
263 | |
264 | /* Is this a known long jump tramp? */ |
265 | for (i = 0; i < NUM_FTRACE_TRAMPS; i++) |
266 | if (ftrace_tramps[i] == tramp) |
267 | return 0; |
268 | |
269 | /* New trampoline -- read where this goes */ |
270 | if (copy_inst_from_kernel_nofault(&op, (void *)tramp)) { |
271 | pr_debug("Fetching opcode failed.\n" ); |
272 | return -1; |
273 | } |
274 | |
275 | /* Is this a 24 bit branch? */ |
276 | if (!is_b_op(op)) { |
277 | pr_debug("Trampoline is not a long branch tramp.\n" ); |
278 | return -1; |
279 | } |
280 | |
281 | /* lets find where the pointer goes */ |
282 | ptr = find_bl_target(tramp, op); |
283 | |
284 | if (ptr != ppc_global_function_entry((void *)_mcount)) { |
285 | pr_debug("Trampoline target %p is not _mcount\n" , (void *)ptr); |
286 | return -1; |
287 | } |
288 | |
289 | /* Let's re-write the tramp to go to ftrace_[regs_]caller */ |
290 | if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) |
291 | ptr = ppc_global_function_entry((void *)ftrace_regs_caller); |
292 | else |
293 | ptr = ppc_global_function_entry((void *)ftrace_caller); |
294 | |
295 | if (patch_branch((u32 *)tramp, ptr, 0)) { |
296 | pr_debug("REL24 out of range!\n" ); |
297 | return -1; |
298 | } |
299 | |
300 | if (add_ftrace_tramp(tramp)) { |
301 | pr_debug("No tramp locations left\n" ); |
302 | return -1; |
303 | } |
304 | |
305 | return 0; |
306 | } |
307 | |
308 | static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) |
309 | { |
310 | unsigned long tramp, ip = rec->ip; |
311 | ppc_inst_t op; |
312 | |
313 | /* Read where this goes */ |
314 | if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { |
315 | pr_err("Fetching opcode failed.\n" ); |
316 | return -EFAULT; |
317 | } |
318 | |
319 | /* Make sure that this is still a 24bit jump */ |
320 | if (!is_bl_op(op)) { |
321 | pr_err("Not expected bl: opcode is %08lx\n" , ppc_inst_as_ulong(op)); |
322 | return -EINVAL; |
323 | } |
324 | |
325 | /* Let's find where the pointer goes */ |
326 | tramp = find_bl_target(ip, op); |
327 | |
328 | pr_devel("ip:%lx jumps to %lx" , ip, tramp); |
329 | |
330 | if (setup_mcount_compiler_tramp(tramp)) { |
331 | /* Are other trampolines reachable? */ |
332 | if (!find_ftrace_tramp(ip)) { |
333 | pr_err("No ftrace trampolines reachable from %ps\n" , |
334 | (void *)ip); |
335 | return -EINVAL; |
336 | } |
337 | } |
338 | |
339 | if (patch_instruction((u32 *)ip, ppc_inst(PPC_RAW_NOP()))) { |
340 | pr_err("Patching NOP failed.\n" ); |
341 | return -EPERM; |
342 | } |
343 | |
344 | return 0; |
345 | } |
346 | |
347 | int ftrace_make_nop(struct module *mod, |
348 | struct dyn_ftrace *rec, unsigned long addr) |
349 | { |
350 | unsigned long ip = rec->ip; |
351 | ppc_inst_t old, new; |
352 | |
353 | /* |
354 | * If the calling address is more that 24 bits away, |
355 | * then we had to use a trampoline to make the call. |
356 | * Otherwise just update the call site. |
357 | */ |
358 | if (test_24bit_addr(ip, addr)) { |
359 | /* within range */ |
360 | old = ftrace_call_replace(ip, addr, 1); |
361 | new = ppc_inst(PPC_RAW_NOP()); |
362 | return ftrace_modify_code(ip, old, new); |
363 | } else if (core_kernel_text(addr: ip)) { |
364 | return __ftrace_make_nop_kernel(rec, addr); |
365 | } else if (!IS_ENABLED(CONFIG_MODULES)) { |
366 | return -EINVAL; |
367 | } |
368 | |
369 | /* |
370 | * Out of range jumps are called from modules. |
371 | * We should either already have a pointer to the module |
372 | * or it has been passed in. |
373 | */ |
374 | if (!rec->arch.mod) { |
375 | if (!mod) { |
376 | pr_err("No module loaded addr=%lx\n" , addr); |
377 | return -EFAULT; |
378 | } |
379 | rec->arch.mod = mod; |
380 | } else if (mod) { |
381 | if (mod != rec->arch.mod) { |
382 | pr_err("Record mod %p not equal to passed in mod %p\n" , |
383 | rec->arch.mod, mod); |
384 | return -EINVAL; |
385 | } |
386 | /* nothing to do if mod == rec->arch.mod */ |
387 | } else |
388 | mod = rec->arch.mod; |
389 | |
390 | return __ftrace_make_nop(mod, rec, addr); |
391 | } |
392 | |
393 | #ifdef CONFIG_MODULES |
394 | /* |
395 | * Examine the existing instructions for __ftrace_make_call. |
396 | * They should effectively be a NOP, and follow formal constraints, |
397 | * depending on the ABI. Return false if they don't. |
398 | */ |
399 | static bool expected_nop_sequence(void *ip, ppc_inst_t op0, ppc_inst_t op1) |
400 | { |
401 | if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) |
402 | return ppc_inst_equal(op0, ppc_inst(PPC_RAW_NOP())); |
403 | else |
404 | return ppc_inst_equal(op0, ppc_inst(PPC_RAW_BRANCH(8))) && |
405 | ppc_inst_equal(op1, ppc_inst(PPC_INST_LD_TOC)); |
406 | } |
407 | |
408 | static int |
409 | __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
410 | { |
411 | ppc_inst_t op[2]; |
412 | void *ip = (void *)rec->ip; |
413 | unsigned long entry, ptr, tramp; |
414 | struct module *mod = rec->arch.mod; |
415 | |
416 | /* read where this goes */ |
417 | if (copy_inst_from_kernel_nofault(op, ip)) |
418 | return -EFAULT; |
419 | |
420 | if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && |
421 | copy_inst_from_kernel_nofault(op + 1, ip + 4)) |
422 | return -EFAULT; |
423 | |
424 | if (!expected_nop_sequence(ip, op[0], op[1])) { |
425 | pr_err("Unexpected call sequence at %p: %08lx %08lx\n" , ip, |
426 | ppc_inst_as_ulong(op[0]), ppc_inst_as_ulong(op[1])); |
427 | return -EINVAL; |
428 | } |
429 | |
430 | /* If we never set up ftrace trampoline(s), then bail */ |
431 | if (!mod->arch.tramp || |
432 | (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && !mod->arch.tramp_regs)) { |
433 | pr_err("No ftrace trampoline\n" ); |
434 | return -EINVAL; |
435 | } |
436 | |
437 | if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && rec->flags & FTRACE_FL_REGS) |
438 | tramp = mod->arch.tramp_regs; |
439 | else |
440 | tramp = mod->arch.tramp; |
441 | |
442 | if (module_trampoline_target(mod, tramp, &ptr)) { |
443 | pr_err("Failed to get trampoline target\n" ); |
444 | return -EFAULT; |
445 | } |
446 | |
447 | pr_devel("trampoline target %lx" , ptr); |
448 | |
449 | entry = ppc_global_function_entry((void *)addr); |
450 | /* This should match what was called */ |
451 | if (ptr != entry) { |
452 | pr_err("addr %lx does not match expected %lx\n" , ptr, entry); |
453 | return -EINVAL; |
454 | } |
455 | |
456 | if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { |
457 | pr_err("REL24 out of range!\n" ); |
458 | return -EINVAL; |
459 | } |
460 | |
461 | return 0; |
462 | } |
463 | #else |
464 | static int __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
465 | { |
466 | return 0; |
467 | } |
468 | #endif /* CONFIG_MODULES */ |
469 | |
470 | static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) |
471 | { |
472 | ppc_inst_t op; |
473 | void *ip = (void *)rec->ip; |
474 | unsigned long tramp, entry, ptr; |
475 | |
476 | /* Make sure we're being asked to patch branch to a known ftrace addr */ |
477 | entry = ppc_global_function_entry((void *)ftrace_caller); |
478 | ptr = ppc_global_function_entry((void *)addr); |
479 | |
480 | if (ptr != entry && IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) |
481 | entry = ppc_global_function_entry((void *)ftrace_regs_caller); |
482 | |
483 | if (ptr != entry) { |
484 | pr_err("Unknown ftrace addr to patch: %ps\n" , (void *)ptr); |
485 | return -EINVAL; |
486 | } |
487 | |
488 | /* Make sure we have a nop */ |
489 | if (copy_inst_from_kernel_nofault(&op, ip)) { |
490 | pr_err("Unable to read ftrace location %p\n" , ip); |
491 | return -EFAULT; |
492 | } |
493 | |
494 | if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_NOP()))) { |
495 | pr_err("Unexpected call sequence at %p: %08lx\n" , |
496 | ip, ppc_inst_as_ulong(op)); |
497 | return -EINVAL; |
498 | } |
499 | |
500 | tramp = find_ftrace_tramp(ip: (unsigned long)ip); |
501 | if (!tramp) { |
502 | pr_err("No ftrace trampolines reachable from %ps\n" , ip); |
503 | return -EINVAL; |
504 | } |
505 | |
506 | if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { |
507 | pr_err("Error patching branch to ftrace tramp!\n" ); |
508 | return -EINVAL; |
509 | } |
510 | |
511 | return 0; |
512 | } |
513 | |
514 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
515 | { |
516 | unsigned long ip = rec->ip; |
517 | ppc_inst_t old, new; |
518 | |
519 | /* |
520 | * If the calling address is more that 24 bits away, |
521 | * then we had to use a trampoline to make the call. |
522 | * Otherwise just update the call site. |
523 | */ |
524 | if (test_24bit_addr(ip, addr)) { |
525 | /* within range */ |
526 | old = ppc_inst(PPC_RAW_NOP()); |
527 | new = ftrace_call_replace(ip, addr, 1); |
528 | return ftrace_modify_code(ip, old, new); |
529 | } else if (core_kernel_text(addr: ip)) { |
530 | return __ftrace_make_call_kernel(rec, addr); |
531 | } else if (!IS_ENABLED(CONFIG_MODULES)) { |
532 | /* We should not get here without modules */ |
533 | return -EINVAL; |
534 | } |
535 | |
536 | /* |
537 | * Out of range jumps are called from modules. |
538 | * Being that we are converting from nop, it had better |
539 | * already have a module defined. |
540 | */ |
541 | if (!rec->arch.mod) { |
542 | pr_err("No module loaded\n" ); |
543 | return -EINVAL; |
544 | } |
545 | |
546 | return __ftrace_make_call(rec, addr); |
547 | } |
548 | |
549 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
550 | #ifdef CONFIG_MODULES |
551 | static int |
552 | __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, |
553 | unsigned long addr) |
554 | { |
555 | ppc_inst_t op; |
556 | unsigned long ip = rec->ip; |
557 | unsigned long entry, ptr, tramp; |
558 | struct module *mod = rec->arch.mod; |
559 | |
560 | /* If we never set up ftrace trampolines, then bail */ |
561 | if (!mod->arch.tramp || !mod->arch.tramp_regs) { |
562 | pr_err("No ftrace trampoline\n" ); |
563 | return -EINVAL; |
564 | } |
565 | |
566 | /* read where this goes */ |
567 | if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { |
568 | pr_err("Fetching opcode failed.\n" ); |
569 | return -EFAULT; |
570 | } |
571 | |
572 | /* Make sure that this is still a 24bit jump */ |
573 | if (!is_bl_op(op)) { |
574 | pr_err("Not expected bl: opcode is %08lx\n" , ppc_inst_as_ulong(op)); |
575 | return -EINVAL; |
576 | } |
577 | |
578 | /* lets find where the pointer goes */ |
579 | tramp = find_bl_target(ip, op); |
580 | entry = ppc_global_function_entry((void *)old_addr); |
581 | |
582 | pr_devel("ip:%lx jumps to %lx" , ip, tramp); |
583 | |
584 | if (tramp != entry) { |
585 | /* old_addr is not within range, so we must have used a trampoline */ |
586 | if (module_trampoline_target(mod, tramp, &ptr)) { |
587 | pr_err("Failed to get trampoline target\n" ); |
588 | return -EFAULT; |
589 | } |
590 | |
591 | pr_devel("trampoline target %lx" , ptr); |
592 | |
593 | /* This should match what was called */ |
594 | if (ptr != entry) { |
595 | pr_err("addr %lx does not match expected %lx\n" , ptr, entry); |
596 | return -EINVAL; |
597 | } |
598 | } |
599 | |
600 | /* The new target may be within range */ |
601 | if (test_24bit_addr(ip, addr)) { |
602 | /* within range */ |
603 | if (patch_branch((u32 *)ip, addr, BRANCH_SET_LINK)) { |
604 | pr_err("REL24 out of range!\n" ); |
605 | return -EINVAL; |
606 | } |
607 | |
608 | return 0; |
609 | } |
610 | |
611 | if (rec->flags & FTRACE_FL_REGS) |
612 | tramp = mod->arch.tramp_regs; |
613 | else |
614 | tramp = mod->arch.tramp; |
615 | |
616 | if (module_trampoline_target(mod, tramp, &ptr)) { |
617 | pr_err("Failed to get trampoline target\n" ); |
618 | return -EFAULT; |
619 | } |
620 | |
621 | pr_devel("trampoline target %lx" , ptr); |
622 | |
623 | entry = ppc_global_function_entry((void *)addr); |
624 | /* This should match what was called */ |
625 | if (ptr != entry) { |
626 | pr_err("addr %lx does not match expected %lx\n" , ptr, entry); |
627 | return -EINVAL; |
628 | } |
629 | |
630 | if (patch_branch((u32 *)ip, tramp, BRANCH_SET_LINK)) { |
631 | pr_err("REL24 out of range!\n" ); |
632 | return -EINVAL; |
633 | } |
634 | |
635 | return 0; |
636 | } |
637 | #else |
638 | static int __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) |
639 | { |
640 | return 0; |
641 | } |
642 | #endif |
643 | |
644 | int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, |
645 | unsigned long addr) |
646 | { |
647 | unsigned long ip = rec->ip; |
648 | ppc_inst_t old, new; |
649 | |
650 | /* |
651 | * If the calling address is more that 24 bits away, |
652 | * then we had to use a trampoline to make the call. |
653 | * Otherwise just update the call site. |
654 | */ |
655 | if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, addr: old_addr)) { |
656 | /* within range */ |
657 | old = ftrace_call_replace(ip, old_addr, 1); |
658 | new = ftrace_call_replace(ip, addr, 1); |
659 | return ftrace_modify_code(ip, old, new); |
660 | } else if (core_kernel_text(addr: ip)) { |
661 | /* |
662 | * We always patch out of range locations to go to the regs |
663 | * variant, so there is nothing to do here |
664 | */ |
665 | return 0; |
666 | } else if (!IS_ENABLED(CONFIG_MODULES)) { |
667 | /* We should not get here without modules */ |
668 | return -EINVAL; |
669 | } |
670 | |
671 | /* |
672 | * Out of range jumps are called from modules. |
673 | */ |
674 | if (!rec->arch.mod) { |
675 | pr_err("No module loaded\n" ); |
676 | return -EINVAL; |
677 | } |
678 | |
679 | return __ftrace_modify_call(rec, old_addr, addr); |
680 | } |
681 | #endif |
682 | |
683 | int ftrace_update_ftrace_func(ftrace_func_t func) |
684 | { |
685 | unsigned long ip = (unsigned long)(&ftrace_call); |
686 | ppc_inst_t old, new; |
687 | int ret; |
688 | |
689 | old = ppc_inst_read((u32 *)&ftrace_call); |
690 | new = ftrace_call_replace(ip, (unsigned long)func, 1); |
691 | ret = ftrace_modify_code(ip, old, new); |
692 | |
693 | /* Also update the regs callback function */ |
694 | if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && !ret) { |
695 | ip = (unsigned long)(&ftrace_regs_call); |
696 | old = ppc_inst_read((u32 *)&ftrace_regs_call); |
697 | new = ftrace_call_replace(ip, (unsigned long)func, 1); |
698 | ret = ftrace_modify_code(ip, old, new); |
699 | } |
700 | |
701 | return ret; |
702 | } |
703 | |
704 | /* |
705 | * Use the default ftrace_modify_all_code, but without |
706 | * stop_machine(). |
707 | */ |
708 | void arch_ftrace_update_code(int command) |
709 | { |
710 | ftrace_modify_all_code(command); |
711 | } |
712 | |
713 | #ifdef CONFIG_PPC64 |
714 | #define PACATOC offsetof(struct paca_struct, kernel_toc) |
715 | |
716 | extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; |
717 | |
718 | void ftrace_free_init_tramp(void) |
719 | { |
720 | int i; |
721 | |
722 | for (i = 0; i < NUM_FTRACE_TRAMPS && ftrace_tramps[i]; i++) |
723 | if (ftrace_tramps[i] == (unsigned long)ftrace_tramp_init) { |
724 | ftrace_tramps[i] = 0; |
725 | return; |
726 | } |
727 | } |
728 | |
729 | int __init ftrace_dyn_arch_init(void) |
730 | { |
731 | int i; |
732 | unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init }; |
733 | u32 stub_insns[] = { |
734 | PPC_RAW_LD(_R12, _R13, PACATOC), |
735 | PPC_RAW_ADDIS(_R12, _R12, 0), |
736 | PPC_RAW_ADDI(_R12, _R12, 0), |
737 | PPC_RAW_MTCTR(_R12), |
738 | PPC_RAW_BCTR() |
739 | }; |
740 | unsigned long addr; |
741 | long reladdr; |
742 | |
743 | if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) |
744 | addr = ppc_global_function_entry((void *)ftrace_regs_caller); |
745 | else |
746 | addr = ppc_global_function_entry((void *)ftrace_caller); |
747 | |
748 | reladdr = addr - kernel_toc_addr(); |
749 | |
750 | if (reladdr >= SZ_2G || reladdr < -(long)SZ_2G) { |
751 | pr_err("Address of %ps out of range of kernel_toc.\n" , |
752 | (void *)addr); |
753 | return -1; |
754 | } |
755 | |
756 | for (i = 0; i < 2; i++) { |
757 | memcpy(tramp[i], stub_insns, sizeof(stub_insns)); |
758 | tramp[i][1] |= PPC_HA(reladdr); |
759 | tramp[i][2] |= PPC_LO(reladdr); |
760 | add_ftrace_tramp((unsigned long)tramp[i]); |
761 | } |
762 | |
763 | return 0; |
764 | } |
765 | #endif |
766 | |
767 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
768 | |
769 | extern void ftrace_graph_call(void); |
770 | extern void ftrace_graph_stub(void); |
771 | |
772 | static int ftrace_modify_ftrace_graph_caller(bool enable) |
773 | { |
774 | unsigned long ip = (unsigned long)(&ftrace_graph_call); |
775 | unsigned long addr = (unsigned long)(&ftrace_graph_caller); |
776 | unsigned long stub = (unsigned long)(&ftrace_graph_stub); |
777 | ppc_inst_t old, new; |
778 | |
779 | if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS)) |
780 | return 0; |
781 | |
782 | old = ftrace_call_replace(ip, enable ? stub : addr, 0); |
783 | new = ftrace_call_replace(ip, enable ? addr : stub, 0); |
784 | |
785 | return ftrace_modify_code(ip, old, new); |
786 | } |
787 | |
788 | int ftrace_enable_ftrace_graph_caller(void) |
789 | { |
790 | return ftrace_modify_ftrace_graph_caller(enable: true); |
791 | } |
792 | |
793 | int ftrace_disable_ftrace_graph_caller(void) |
794 | { |
795 | return ftrace_modify_ftrace_graph_caller(enable: false); |
796 | } |
797 | |
798 | /* |
799 | * Hook the return address and push it in the stack of return addrs |
800 | * in current thread info. Return the address we want to divert to. |
801 | */ |
802 | static unsigned long |
803 | __prepare_ftrace_return(unsigned long parent, unsigned long ip, unsigned long sp) |
804 | { |
805 | unsigned long return_hooker; |
806 | int bit; |
807 | |
808 | if (unlikely(ftrace_graph_is_dead())) |
809 | goto out; |
810 | |
811 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
812 | goto out; |
813 | |
814 | bit = ftrace_test_recursion_trylock(ip, parent_ip: parent); |
815 | if (bit < 0) |
816 | goto out; |
817 | |
818 | return_hooker = ppc_function_entry(return_to_handler); |
819 | |
820 | if (!function_graph_enter(ret: parent, func: ip, frame_pointer: 0, retp: (unsigned long *)sp)) |
821 | parent = return_hooker; |
822 | |
823 | ftrace_test_recursion_unlock(bit); |
824 | out: |
825 | return parent; |
826 | } |
827 | |
828 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS |
829 | void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, |
830 | struct ftrace_ops *op, struct ftrace_regs *fregs) |
831 | { |
832 | fregs->regs.link = __prepare_ftrace_return(parent: parent_ip, ip, sp: fregs->regs.gpr[1]); |
833 | } |
834 | #else |
835 | unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip, |
836 | unsigned long sp) |
837 | { |
838 | return __prepare_ftrace_return(parent, ip, sp); |
839 | } |
840 | #endif |
841 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
842 | |
843 | #ifdef CONFIG_PPC64_ELF_ABI_V1 |
844 | char *arch_ftrace_match_adjust(char *str, const char *search) |
845 | { |
846 | if (str[0] == '.' && search[0] != '.') |
847 | return str + 1; |
848 | else |
849 | return str; |
850 | } |
851 | #endif /* CONFIG_PPC64_ELF_ABI_V1 */ |
852 | |