1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Code for replacing ftrace calls with jumps. |
4 | * |
5 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> |
6 | * Copyright (C) 2009, 2010 DSLab, Lanzhou University, China |
7 | * Author: Wu Zhangjin <wuzhangjin@gmail.com> |
8 | * |
9 | * Thanks goes to Steven Rostedt for writing the original x86 version. |
10 | */ |
11 | |
12 | #include <linux/uaccess.h> |
13 | #include <linux/init.h> |
14 | #include <linux/ftrace.h> |
15 | #include <linux/syscalls.h> |
16 | |
17 | #include <asm/asm.h> |
18 | #include <asm/asm-offsets.h> |
19 | #include <asm/cacheflush.h> |
20 | #include <asm/syscall.h> |
21 | #include <asm/uasm.h> |
22 | #include <asm/unistd.h> |
23 | |
24 | #include <asm-generic/sections.h> |
25 | |
26 | #if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) |
27 | #define MCOUNT_OFFSET_INSNS 5 |
28 | #else |
29 | #define MCOUNT_OFFSET_INSNS 4 |
30 | #endif |
31 | |
32 | #ifdef CONFIG_DYNAMIC_FTRACE |
33 | |
34 | /* Arch override because MIPS doesn't need to run this from stop_machine() */ |
35 | void arch_ftrace_update_code(int command) |
36 | { |
37 | ftrace_modify_all_code(command); |
38 | } |
39 | |
40 | #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ |
41 | #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ |
42 | #define JUMP_RANGE_MASK ((1UL << 28) - 1) |
43 | |
44 | #define INSN_NOP 0x00000000 /* nop */ |
45 | #define INSN_JAL(addr) \ |
46 | ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK))) |
47 | |
48 | static unsigned int insn_jal_ftrace_caller __read_mostly; |
49 | static unsigned int insn_la_mcount[2] __read_mostly; |
50 | static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly; |
51 | |
52 | static inline void ftrace_dyn_arch_init_insns(void) |
53 | { |
54 | u32 *buf; |
55 | unsigned int v1; |
56 | |
57 | /* la v1, _mcount */ |
58 | v1 = 3; |
59 | buf = (u32 *)&insn_la_mcount[0]; |
60 | UASM_i_LA(&buf, v1, MCOUNT_ADDR); |
61 | |
62 | /* jal (ftrace_caller + 8), jump over the first two instruction */ |
63 | buf = (u32 *)&insn_jal_ftrace_caller; |
64 | uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK); |
65 | |
66 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
67 | /* j ftrace_graph_caller */ |
68 | buf = (u32 *)&insn_j_ftrace_graph_caller; |
69 | uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK); |
70 | #endif |
71 | } |
72 | |
73 | static int ftrace_modify_code(unsigned long ip, unsigned int new_code) |
74 | { |
75 | int faulted; |
76 | |
77 | /* *(unsigned int *)ip = new_code; */ |
78 | safe_store_code(new_code, ip, faulted); |
79 | |
80 | if (unlikely(faulted)) |
81 | return -EFAULT; |
82 | |
83 | flush_icache_range(start: ip, end: ip + 8); |
84 | |
85 | return 0; |
86 | } |
87 | |
88 | #ifndef CONFIG_64BIT |
89 | static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1, |
90 | unsigned int new_code2) |
91 | { |
92 | int faulted; |
93 | |
94 | safe_store_code(new_code1, ip, faulted); |
95 | if (unlikely(faulted)) |
96 | return -EFAULT; |
97 | |
98 | ip += 4; |
99 | safe_store_code(new_code2, ip, faulted); |
100 | if (unlikely(faulted)) |
101 | return -EFAULT; |
102 | |
103 | ip -= 4; |
104 | flush_icache_range(ip, ip + 8); |
105 | |
106 | return 0; |
107 | } |
108 | |
109 | static int ftrace_modify_code_2r(unsigned long ip, unsigned int new_code1, |
110 | unsigned int new_code2) |
111 | { |
112 | int faulted; |
113 | |
114 | ip += 4; |
115 | safe_store_code(new_code2, ip, faulted); |
116 | if (unlikely(faulted)) |
117 | return -EFAULT; |
118 | |
119 | ip -= 4; |
120 | safe_store_code(new_code1, ip, faulted); |
121 | if (unlikely(faulted)) |
122 | return -EFAULT; |
123 | |
124 | flush_icache_range(ip, ip + 8); |
125 | |
126 | return 0; |
127 | } |
128 | #endif |
129 | |
130 | /* |
131 | * The details about the calling site of mcount on MIPS |
132 | * |
133 | * 1. For kernel: |
134 | * |
135 | * move at, ra |
136 | * jal _mcount --> nop |
137 | * sub sp, sp, 8 --> nop (CONFIG_32BIT) |
138 | * |
139 | * 2. For modules: |
140 | * |
141 | * 2.1 For KBUILD_MCOUNT_RA_ADDRESS and CONFIG_32BIT |
142 | * |
143 | * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005) |
144 | * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) |
145 | * move at, ra |
146 | * move $12, ra_address |
147 | * jalr v1 |
148 | * sub sp, sp, 8 |
149 | * 1: offset = 5 instructions |
150 | * 2.2 For the Other situations |
151 | * |
152 | * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) |
153 | * addiu v1, v1, low_16bit_of_mcount --> nop (CONFIG_32BIT) |
154 | * move at, ra |
155 | * jalr v1 |
156 | * nop | move $12, ra_address | sub sp, sp, 8 |
157 | * 1: offset = 4 instructions |
158 | */ |
159 | |
160 | #define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS) |
161 | |
162 | int ftrace_make_nop(struct module *mod, |
163 | struct dyn_ftrace *rec, unsigned long addr) |
164 | { |
165 | unsigned int new; |
166 | unsigned long ip = rec->ip; |
167 | |
168 | /* |
169 | * If ip is in kernel space, no long call, otherwise, long call is |
170 | * needed. |
171 | */ |
172 | new = core_kernel_text(addr: ip) ? INSN_NOP : INSN_B_1F; |
173 | #ifdef CONFIG_64BIT |
174 | return ftrace_modify_code(ip, new_code: new); |
175 | #else |
176 | /* |
177 | * On 32 bit MIPS platforms, gcc adds a stack adjust |
178 | * instruction in the delay slot after the branch to |
179 | * mcount and expects mcount to restore the sp on return. |
180 | * This is based on a legacy API and does nothing but |
181 | * waste instructions so it's being removed at runtime. |
182 | */ |
183 | return ftrace_modify_code_2(ip, new, INSN_NOP); |
184 | #endif |
185 | } |
186 | |
187 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) |
188 | { |
189 | unsigned int new; |
190 | unsigned long ip = rec->ip; |
191 | |
192 | new = core_kernel_text(addr: ip) ? insn_jal_ftrace_caller : insn_la_mcount[0]; |
193 | |
194 | #ifdef CONFIG_64BIT |
195 | return ftrace_modify_code(ip, new_code: new); |
196 | #else |
197 | return ftrace_modify_code_2r(ip, new, core_kernel_text(ip) ? |
198 | INSN_NOP : insn_la_mcount[1]); |
199 | #endif |
200 | } |
201 | |
202 | #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call)) |
203 | |
204 | int ftrace_update_ftrace_func(ftrace_func_t func) |
205 | { |
206 | unsigned int new; |
207 | |
208 | new = INSN_JAL((unsigned long)func); |
209 | |
210 | return ftrace_modify_code(FTRACE_CALL_IP, new_code: new); |
211 | } |
212 | |
213 | int __init ftrace_dyn_arch_init(void) |
214 | { |
215 | /* Encode the instructions when booting */ |
216 | ftrace_dyn_arch_init_insns(); |
217 | |
218 | /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */ |
219 | ftrace_modify_code(MCOUNT_ADDR, INSN_NOP); |
220 | |
221 | return 0; |
222 | } |
223 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
224 | |
225 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
226 | |
227 | #ifdef CONFIG_DYNAMIC_FTRACE |
228 | |
229 | extern void ftrace_graph_call(void); |
230 | #define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call)) |
231 | |
232 | int ftrace_enable_ftrace_graph_caller(void) |
233 | { |
234 | return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, |
235 | new_code: insn_j_ftrace_graph_caller); |
236 | } |
237 | |
238 | int ftrace_disable_ftrace_graph_caller(void) |
239 | { |
240 | return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, INSN_NOP); |
241 | } |
242 | |
243 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
244 | |
245 | #ifndef KBUILD_MCOUNT_RA_ADDRESS |
246 | |
247 | #define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ |
248 | #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ |
249 | #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ |
250 | |
251 | unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long |
252 | old_parent_ra, unsigned long parent_ra_addr, unsigned long fp) |
253 | { |
254 | unsigned long sp, ip, tmp; |
255 | unsigned int code; |
256 | int faulted; |
257 | |
258 | /* |
259 | * For module, move the ip from the return address after the |
260 | * instruction "lui v1, hi_16bit_of_mcount"(offset is 24), but for |
261 | * kernel, move after the instruction "move ra, at"(offset is 16) |
262 | */ |
263 | ip = self_ra - (core_kernel_text(addr: self_ra) ? 16 : 24); |
264 | |
265 | /* |
266 | * search the text until finding the non-store instruction or "s{d,w} |
267 | * ra, offset(sp)" instruction |
268 | */ |
269 | do { |
270 | /* get the code at "ip": code = *(unsigned int *)ip; */ |
271 | safe_load_code(code, ip, faulted); |
272 | |
273 | if (unlikely(faulted)) |
274 | return 0; |
275 | /* |
276 | * If we hit the non-store instruction before finding where the |
277 | * ra is stored, then this is a leaf function and it does not |
278 | * store the ra on the stack |
279 | */ |
280 | if ((code & S_R_SP) != S_R_SP) |
281 | return parent_ra_addr; |
282 | |
283 | /* Move to the next instruction */ |
284 | ip -= 4; |
285 | } while ((code & S_RA_SP) != S_RA_SP); |
286 | |
287 | sp = fp + (code & OFFSET_MASK); |
288 | |
289 | /* tmp = *(unsigned long *)sp; */ |
290 | safe_load_stack(tmp, sp, faulted); |
291 | if (unlikely(faulted)) |
292 | return 0; |
293 | |
294 | if (tmp == old_parent_ra) |
295 | return sp; |
296 | return 0; |
297 | } |
298 | |
299 | #endif /* !KBUILD_MCOUNT_RA_ADDRESS */ |
300 | |
301 | /* |
302 | * Hook the return address and push it in the stack of return addrs |
303 | * in current thread info. |
304 | */ |
305 | void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, |
306 | unsigned long fp) |
307 | { |
308 | unsigned long old_parent_ra; |
309 | unsigned long return_hooker = (unsigned long) |
310 | &return_to_handler; |
311 | int faulted, insns; |
312 | |
313 | if (unlikely(ftrace_graph_is_dead())) |
314 | return; |
315 | |
316 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
317 | return; |
318 | |
319 | /* |
320 | * "parent_ra_addr" is the stack address where the return address of |
321 | * the caller of _mcount is saved. |
322 | * |
323 | * If gcc < 4.5, a leaf function does not save the return address |
324 | * in the stack address, so we "emulate" one in _mcount's stack space, |
325 | * and hijack it directly. |
326 | * For a non-leaf function, it does save the return address to its own |
327 | * stack space, so we can not hijack it directly, but need to find the |
328 | * real stack address, which is done by ftrace_get_parent_addr(). |
329 | * |
330 | * If gcc >= 4.5, with the new -mmcount-ra-address option, for a |
331 | * non-leaf function, the location of the return address will be saved |
332 | * to $12 for us. |
333 | * For a leaf function, it just puts a zero into $12, so we handle |
334 | * it in ftrace_graph_caller() of mcount.S. |
335 | */ |
336 | |
337 | /* old_parent_ra = *parent_ra_addr; */ |
338 | safe_load_stack(old_parent_ra, parent_ra_addr, faulted); |
339 | if (unlikely(faulted)) |
340 | goto out; |
341 | #ifndef KBUILD_MCOUNT_RA_ADDRESS |
342 | parent_ra_addr = (unsigned long *)ftrace_get_parent_ra_addr(self_ra, |
343 | old_parent_ra, parent_ra_addr: (unsigned long)parent_ra_addr, fp); |
344 | /* |
345 | * If fails when getting the stack address of the non-leaf function's |
346 | * ra, stop function graph tracer and return |
347 | */ |
348 | if (parent_ra_addr == NULL) |
349 | goto out; |
350 | #endif |
351 | /* *parent_ra_addr = return_hooker; */ |
352 | safe_store_stack(return_hooker, parent_ra_addr, faulted); |
353 | if (unlikely(faulted)) |
354 | goto out; |
355 | |
356 | /* |
357 | * Get the recorded ip of the current mcount calling site in the |
358 | * __mcount_loc section, which will be used to filter the function |
359 | * entries configured through the tracing/set_graph_function interface. |
360 | */ |
361 | |
362 | insns = core_kernel_text(addr: self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1; |
363 | self_ra -= (MCOUNT_INSN_SIZE * insns); |
364 | |
365 | if (function_graph_enter(ret: old_parent_ra, func: self_ra, frame_pointer: fp, NULL)) |
366 | *parent_ra_addr = old_parent_ra; |
367 | return; |
368 | out: |
369 | ftrace_graph_stop(); |
370 | WARN_ON(1); |
371 | } |
372 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
373 | |
374 | #ifdef CONFIG_FTRACE_SYSCALLS |
375 | |
376 | #ifdef CONFIG_32BIT |
377 | unsigned long __init arch_syscall_addr(int nr) |
378 | { |
379 | return (unsigned long)sys_call_table[nr - __NR_O32_Linux]; |
380 | } |
381 | #endif |
382 | |
383 | #ifdef CONFIG_64BIT |
384 | |
385 | unsigned long __init arch_syscall_addr(int nr) |
386 | { |
387 | #ifdef CONFIG_MIPS32_N32 |
388 | if (nr >= __NR_N32_Linux && nr < __NR_N32_Linux + __NR_N32_Linux_syscalls) |
389 | return (unsigned long)sysn32_call_table[nr - __NR_N32_Linux]; |
390 | #endif |
391 | if (nr >= __NR_64_Linux && nr < __NR_64_Linux + __NR_64_Linux_syscalls) |
392 | return (unsigned long)sys_call_table[nr - __NR_64_Linux]; |
393 | #ifdef CONFIG_MIPS32_O32 |
394 | if (nr >= __NR_O32_Linux && nr < __NR_O32_Linux + __NR_O32_Linux_syscalls) |
395 | return (unsigned long)sys32_call_table[nr - __NR_O32_Linux]; |
396 | #endif |
397 | |
398 | return (unsigned long) &sys_ni_syscall; |
399 | } |
400 | #endif |
401 | |
402 | #endif /* CONFIG_FTRACE_SYSCALLS */ |
403 | |