1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | |
3 | #include <asm/assembler.h> |
4 | #include <asm/ftrace.h> |
5 | #include <asm/unwind.h> |
6 | |
7 | #include "entry-header.S" |
8 | |
9 | /* |
10 | * When compiling with -pg, gcc inserts a call to the mcount routine at the |
11 | * start of every function. In mcount, apart from the function's address (in |
12 | * lr), we need to get hold of the function's caller's address. |
13 | * |
14 | * Newer GCCs (4.4+) solve this problem by using a version of mcount with call |
15 | * sites like: |
16 | * |
17 | * push {lr} |
18 | * bl __gnu_mcount_nc |
19 | * |
20 | * With these compilers, frame pointers are not necessary. |
21 | * |
22 | * mcount can be thought of as a function called in the middle of a subroutine |
23 | * call. As such, it needs to be transparent for both the caller and the |
24 | * callee: the original lr needs to be restored when leaving mcount, and no |
25 | * registers should be clobbered. |
26 | * |
27 | * When using dynamic ftrace, we patch out the mcount call by a "add sp, #4" |
28 | * instead of the __gnu_mcount_nc call (see arch/arm/kernel/ftrace.c). |
29 | */ |
30 | |
31 | .macro mcount_adjust_addr rd, rn |
32 | bic \rd, \rn, #1 @ clear the Thumb bit if present |
33 | sub \rd, \rd, #MCOUNT_INSN_SIZE |
34 | .endm |
35 | |
36 | .macro __mcount suffix |
37 | mcount_enter |
38 | ldr_va r2, ftrace_trace_function |
39 | badr r0, .Lftrace_stub |
40 | cmp r0, r2 |
41 | bne 1f |
42 | |
43 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
44 | ldr_va r2, ftrace_graph_return |
45 | cmp r0, r2 |
46 | bne ftrace_graph_caller\suffix |
47 | |
48 | ldr_va r2, ftrace_graph_entry |
49 | mov_l r0, ftrace_graph_entry_stub |
50 | cmp r0, r2 |
51 | bne ftrace_graph_caller\suffix |
52 | #endif |
53 | |
54 | mcount_exit |
55 | |
56 | 1: mcount_get_lr r1 @ lr of instrumented func |
57 | mcount_adjust_addr r0, lr @ instrumented function |
58 | badr lr, 2f |
59 | mov pc, r2 |
60 | 2: mcount_exit |
61 | .endm |
62 | |
63 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
64 | |
65 | .macro __ftrace_regs_caller |
66 | |
67 | str lr, [sp, #-8]! @ store LR as PC and make space for CPSR/OLD_R0, |
68 | @ OLD_R0 will overwrite previous LR |
69 | |
70 | ldr lr, [sp, #8] @ get previous LR |
71 | |
72 | str r0, [sp, #8] @ write r0 as OLD_R0 over previous LR |
73 | |
74 | str lr, [sp, #-4]! @ store previous LR as LR |
75 | |
76 | add lr, sp, #16 @ move in LR the value of SP as it was |
77 | @ before the push {lr} of the mcount mechanism |
78 | |
79 | push {r0-r11, ip, lr} |
80 | |
81 | @ stack content at this point: |
82 | @ 0 4 48 52 56 60 64 68 72 |
83 | @ R0 | R1 | ... | IP | SP + 4 | previous LR | LR | PSR | OLD_R0 | |
84 | |
85 | mov r3, sp @ struct pt_regs* |
86 | |
87 | ldr_va r2, function_trace_op @ pointer to the current |
88 | @ function tracing op |
89 | |
90 | ldr r1, [sp, #S_LR] @ lr of instrumented func |
91 | |
92 | ldr lr, [sp, #S_PC] @ get LR |
93 | |
94 | mcount_adjust_addr r0, lr @ instrumented function |
95 | |
96 | .globl ftrace_regs_call |
97 | ftrace_regs_call: |
98 | bl ftrace_stub |
99 | |
100 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
101 | .globl ftrace_graph_regs_call |
102 | ftrace_graph_regs_call: |
103 | ARM( mov r0, r0 ) |
104 | THUMB( nop.w ) |
105 | #endif |
106 | |
107 | @ pop saved regs |
108 | pop {r0-r11, ip, lr} @ restore r0 through r12 |
109 | ldr lr, [sp], #4 @ restore LR |
110 | ldr pc, [sp], #12 |
111 | .endm |
112 | |
113 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
114 | .macro __ftrace_graph_regs_caller |
115 | |
116 | #ifdef CONFIG_UNWINDER_FRAME_POINTER |
117 | sub r0, fp, #4 @ lr of instrumented routine (parent) |
118 | #else |
119 | add r0, sp, #S_LR |
120 | #endif |
121 | |
122 | @ called from __ftrace_regs_caller |
123 | ldr r1, [sp, #S_PC] @ instrumented routine (func) |
124 | mcount_adjust_addr r1, r1 |
125 | |
126 | mov r2, fpreg @ frame pointer |
127 | add r3, sp, #PT_REGS_SIZE |
128 | bl prepare_ftrace_return |
129 | |
130 | @ pop registers saved in ftrace_regs_caller |
131 | pop {r0-r11, ip, lr} @ restore r0 through r12 |
132 | ldr lr, [sp], #4 @ restore LR |
133 | ldr pc, [sp], #12 |
134 | |
135 | .endm |
136 | #endif |
137 | #endif |
138 | |
139 | .macro __ftrace_caller suffix |
140 | mcount_enter |
141 | |
142 | mcount_get_lr r1 @ lr of instrumented func |
143 | mcount_adjust_addr r0, lr @ instrumented function |
144 | |
145 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
146 | ldr_va r2, function_trace_op @ pointer to the current |
147 | @ function tracing op |
148 | mov r3, #0 @ regs is NULL |
149 | #endif |
150 | |
151 | .globl ftrace_call\suffix |
152 | ftrace_call\suffix: |
153 | bl ftrace_stub |
154 | |
155 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
156 | .globl ftrace_graph_call\suffix |
157 | ftrace_graph_call\suffix: |
158 | ARM( mov r0, r0 ) |
159 | THUMB( nop.w ) |
160 | #endif |
161 | |
162 | mcount_exit |
163 | .endm |
164 | |
165 | .macro __ftrace_graph_caller |
166 | #ifdef CONFIG_UNWINDER_FRAME_POINTER |
167 | sub r0, fp, #4 @ &lr of instrumented routine (&parent) |
168 | #else |
169 | add r0, sp, #20 |
170 | #endif |
171 | #ifdef CONFIG_DYNAMIC_FTRACE |
172 | @ called from __ftrace_caller, saved in mcount_enter |
173 | ldr r1, [sp, #16] @ instrumented routine (func) |
174 | mcount_adjust_addr r1, r1 |
175 | #else |
176 | @ called from __mcount, untouched in lr |
177 | mcount_adjust_addr r1, lr @ instrumented routine (func) |
178 | #endif |
179 | mov r2, fpreg @ frame pointer |
180 | add r3, sp, #24 |
181 | bl prepare_ftrace_return |
182 | mcount_exit |
183 | .endm |
184 | |
185 | /* |
186 | * __gnu_mcount_nc |
187 | */ |
188 | |
189 | .macro mcount_enter |
190 | /* |
191 | * This pad compensates for the push {lr} at the call site. Note that we are |
192 | * unable to unwind through a function which does not otherwise save its lr. |
193 | */ |
194 | UNWIND(.pad #4) |
195 | stmdb sp!, {r0-r3, lr} |
196 | UNWIND(.save {r0-r3, lr}) |
197 | .endm |
198 | |
199 | .macro mcount_get_lr reg |
200 | ldr \reg, [sp, #20] |
201 | .endm |
202 | |
203 | .macro mcount_exit |
204 | ldmia sp!, {r0-r3} |
205 | ldr lr, [sp, #4] |
206 | ldr pc, [sp], #8 |
207 | .endm |
208 | |
209 | ENTRY(__gnu_mcount_nc) |
210 | UNWIND(.fnstart) |
211 | #ifdef CONFIG_DYNAMIC_FTRACE |
212 | push {lr} |
213 | ldr lr, [sp, #4] |
214 | ldr pc, [sp], #8 |
215 | #else |
216 | __mcount |
217 | #endif |
218 | UNWIND(.fnend) |
219 | ENDPROC(__gnu_mcount_nc) |
220 | |
221 | #ifdef CONFIG_DYNAMIC_FTRACE |
222 | ENTRY(ftrace_caller) |
223 | UNWIND(.fnstart) |
224 | __ftrace_caller |
225 | UNWIND(.fnend) |
226 | ENDPROC(ftrace_caller) |
227 | |
228 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
229 | ENTRY(ftrace_regs_caller) |
230 | UNWIND(.fnstart) |
231 | __ftrace_regs_caller |
232 | UNWIND(.fnend) |
233 | ENDPROC(ftrace_regs_caller) |
234 | #endif |
235 | |
236 | #endif |
237 | |
238 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
239 | ENTRY(ftrace_graph_caller) |
240 | UNWIND(.fnstart) |
241 | __ftrace_graph_caller |
242 | UNWIND(.fnend) |
243 | ENDPROC(ftrace_graph_caller) |
244 | |
245 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
246 | ENTRY(ftrace_graph_regs_caller) |
247 | UNWIND(.fnstart) |
248 | __ftrace_graph_regs_caller |
249 | UNWIND(.fnend) |
250 | ENDPROC(ftrace_graph_regs_caller) |
251 | #endif |
252 | #endif |
253 | |
254 | .purgem mcount_enter |
255 | .purgem mcount_get_lr |
256 | .purgem mcount_exit |
257 | |
258 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
259 | ENTRY(return_to_handler) |
260 | stmdb sp!, {r0-r3} |
261 | add r0, sp, #16 @ sp at exit of instrumented routine |
262 | bl ftrace_return_to_handler |
263 | mov lr, r0 @ r0 has real ret addr |
264 | ldmia sp!, {r0-r3} |
265 | ret lr |
266 | ENDPROC(return_to_handler) |
267 | #endif |
268 | |
269 | ENTRY(ftrace_stub) |
270 | .Lftrace_stub: |
271 | ret lr |
272 | ENDPROC(ftrace_stub) |
273 | |
274 | #ifdef CONFIG_DYNAMIC_FTRACE |
275 | |
276 | __INIT |
277 | |
278 | .macro init_tramp, dst:req |
279 | ENTRY(\dst\()_from_init) |
280 | ldr pc, =\dst |
281 | ENDPROC(\dst\()_from_init) |
282 | .endm |
283 | |
284 | init_tramp ftrace_caller |
285 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
286 | init_tramp ftrace_regs_caller |
287 | #endif |
288 | #endif |
289 | |