1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* |
3 | * Copyright IBM Corp. 2008, 2009 |
4 | * |
5 | */ |
6 | |
7 | #include <linux/linkage.h> |
8 | #include <asm/asm-offsets.h> |
9 | #include <asm/ftrace.h> |
10 | #include <asm/nospec-insn.h> |
11 | #include <asm/ptrace.h> |
12 | |
13 | #define STACK_FRAME_SIZE_PTREGS (STACK_FRAME_OVERHEAD + __PT_SIZE) |
14 | #define STACK_PTREGS (STACK_FRAME_OVERHEAD) |
15 | #define STACK_PTREGS_GPRS (STACK_PTREGS + __PT_GPRS) |
16 | #define STACK_PTREGS_PSW (STACK_PTREGS + __PT_PSW) |
17 | |
18 | #define STACK_FRAME_SIZE_FREGS (STACK_FRAME_OVERHEAD + __FTRACE_REGS_SIZE) |
19 | #define STACK_FREGS (STACK_FRAME_OVERHEAD) |
20 | #define STACK_FREGS_PTREGS (STACK_FRAME_OVERHEAD + __FTRACE_REGS_PT_REGS) |
21 | #define STACK_FREGS_PTREGS_GPRS (STACK_FREGS_PTREGS + __PT_GPRS) |
22 | #define STACK_FREGS_PTREGS_PSW (STACK_FREGS_PTREGS + __PT_PSW) |
23 | #define STACK_FREGS_PTREGS_ORIG_GPR2 (STACK_FREGS_PTREGS + __PT_ORIG_GPR2) |
24 | #define STACK_FREGS_PTREGS_FLAGS (STACK_FREGS_PTREGS + __PT_FLAGS) |
25 | |
26 | /* packed stack: allocate just enough for r14, r15 and backchain */ |
27 | #define TRACED_FUNC_FRAME_SIZE 24 |
28 | |
29 | #ifdef CONFIG_FUNCTION_TRACER |
30 | |
31 | GEN_BR_THUNK %r1 |
32 | GEN_BR_THUNK %r14 |
33 | |
34 | .section .kprobes.text, "ax" |
35 | |
36 | SYM_FUNC_START(ftrace_stub) |
37 | BR_EX %r14 |
38 | SYM_FUNC_END(ftrace_stub) |
39 | |
40 | SYM_CODE_START(ftrace_stub_direct_tramp) |
41 | lgr %r1, %r0 |
42 | BR_EX %r1 |
43 | SYM_CODE_END(ftrace_stub_direct_tramp) |
44 | |
45 | .macro ftrace_regs_entry, allregs=0 |
46 | stg %r14,(__SF_GPRS+8*8)(%r15) # save traced function caller |
47 | |
48 | .if \allregs == 1 |
49 | # save psw mask |
50 | # don't put any instructions clobbering CC before this point |
51 | epsw %r1,%r14 |
52 | risbg %r14,%r1,0,31,32 |
53 | .endif |
54 | |
55 | lgr %r1,%r15 |
56 | # allocate stack frame for ftrace_caller to contain traced function |
57 | aghi %r15,-TRACED_FUNC_FRAME_SIZE |
58 | stg %r1,__SF_BACKCHAIN(%r15) |
59 | stg %r0,(__SF_GPRS+8*8)(%r15) |
60 | stg %r15,(__SF_GPRS+9*8)(%r15) |
61 | # allocate ftrace_regs and stack frame for ftrace_trace_function |
62 | aghi %r15,-STACK_FRAME_SIZE_FREGS |
63 | stg %r1,(STACK_FREGS_PTREGS_GPRS+15*8)(%r15) |
64 | xc STACK_FREGS_PTREGS_ORIG_GPR2(8,%r15),STACK_FREGS_PTREGS_ORIG_GPR2(%r15) |
65 | |
66 | .if \allregs == 1 |
67 | stg %r14,(STACK_FREGS_PTREGS_PSW)(%r15) |
68 | mvghi STACK_FREGS_PTREGS_FLAGS(%r15),_PIF_FTRACE_FULL_REGS |
69 | .else |
70 | xc STACK_FREGS_PTREGS_FLAGS(8,%r15),STACK_FREGS_PTREGS_FLAGS(%r15) |
71 | .endif |
72 | |
73 | lg %r14,(__SF_GPRS+8*8)(%r1) # restore original return address |
74 | aghi %r1,-TRACED_FUNC_FRAME_SIZE |
75 | stg %r1,__SF_BACKCHAIN(%r15) |
76 | stg %r0,(STACK_FREGS_PTREGS_PSW+8)(%r15) |
77 | stmg %r2,%r14,(STACK_FREGS_PTREGS_GPRS+2*8)(%r15) |
78 | .endm |
79 | |
80 | SYM_CODE_START(ftrace_regs_caller) |
81 | ftrace_regs_entry 1 |
82 | j ftrace_common |
83 | SYM_CODE_END(ftrace_regs_caller) |
84 | |
85 | SYM_CODE_START(ftrace_caller) |
86 | ftrace_regs_entry 0 |
87 | j ftrace_common |
88 | SYM_CODE_END(ftrace_caller) |
89 | |
90 | SYM_CODE_START(ftrace_common) |
91 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
92 | aghik %r2,%r0,-MCOUNT_INSN_SIZE |
93 | lgrl %r4,function_trace_op |
94 | lgrl %r1,ftrace_func |
95 | #else |
96 | lgr %r2,%r0 |
97 | aghi %r2,-MCOUNT_INSN_SIZE |
98 | larl %r4,function_trace_op |
99 | lg %r4,0(%r4) |
100 | larl %r1,ftrace_func |
101 | lg %r1,0(%r1) |
102 | #endif |
103 | lgr %r3,%r14 |
104 | la %r5,STACK_FREGS(%r15) |
105 | BASR_EX %r14,%r1 |
106 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
107 | # The j instruction gets runtime patched to a nop instruction. |
108 | # See ftrace_enable_ftrace_graph_caller. |
109 | SYM_INNER_LABEL(ftrace_graph_caller, SYM_L_GLOBAL) |
110 | j .Lftrace_graph_caller_end |
111 | lmg %r2,%r3,(STACK_FREGS_PTREGS_GPRS+14*8)(%r15) |
112 | lg %r4,(STACK_FREGS_PTREGS_PSW+8)(%r15) |
113 | brasl %r14,prepare_ftrace_return |
114 | stg %r2,(STACK_FREGS_PTREGS_GPRS+14*8)(%r15) |
115 | .Lftrace_graph_caller_end: |
116 | #endif |
117 | lg %r0,(STACK_FREGS_PTREGS_PSW+8)(%r15) |
118 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
119 | ltg %r1,STACK_FREGS_PTREGS_ORIG_GPR2(%r15) |
120 | locgrz %r1,%r0 |
121 | #else |
122 | lg %r1,STACK_FREGS_PTREGS_ORIG_GPR2(%r15) |
123 | ltgr %r1,%r1 |
124 | jnz 0f |
125 | lgr %r1,%r0 |
126 | #endif |
127 | 0: lmg %r2,%r15,(STACK_FREGS_PTREGS_GPRS+2*8)(%r15) |
128 | BR_EX %r1 |
129 | SYM_CODE_END(ftrace_common) |
130 | |
131 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
132 | |
133 | SYM_FUNC_START(return_to_handler) |
134 | stmg %r2,%r5,32(%r15) |
135 | lgr %r1,%r15 |
136 | aghi %r15,-(STACK_FRAME_OVERHEAD+__FGRAPH_RET_SIZE) |
137 | stg %r1,__SF_BACKCHAIN(%r15) |
138 | la %r3,STACK_FRAME_OVERHEAD(%r15) |
139 | stg %r1,__FGRAPH_RET_FP(%r3) |
140 | stg %r2,__FGRAPH_RET_GPR2(%r3) |
141 | lgr %r2,%r3 |
142 | brasl %r14,ftrace_return_to_handler |
143 | aghi %r15,STACK_FRAME_OVERHEAD+__FGRAPH_RET_SIZE |
144 | lgr %r14,%r2 |
145 | lmg %r2,%r5,32(%r15) |
146 | BR_EX %r14 |
147 | SYM_FUNC_END(return_to_handler) |
148 | |
149 | #endif |
150 | #endif /* CONFIG_FUNCTION_TRACER */ |
151 | |
152 | SYM_CODE_START(ftrace_shared_hotpatch_trampoline_br) |
153 | lmg %r0,%r1,2(%r1) |
154 | br %r1 |
155 | SYM_INNER_LABEL(ftrace_shared_hotpatch_trampoline_br_end, SYM_L_GLOBAL) |
156 | SYM_CODE_END(ftrace_shared_hotpatch_trampoline_br) |
157 | |
158 | #ifdef CONFIG_EXPOLINE |
159 | SYM_CODE_START(ftrace_shared_hotpatch_trampoline_exrl) |
160 | lmg %r0,%r1,2(%r1) |
161 | exrl %r0,0f |
162 | j . |
163 | 0: br %r1 |
164 | SYM_INNER_LABEL(ftrace_shared_hotpatch_trampoline_exrl_end, SYM_L_GLOBAL) |
165 | SYM_CODE_END(ftrace_shared_hotpatch_trampoline_exrl) |
166 | #endif /* CONFIG_EXPOLINE */ |
167 | |
168 | #ifdef CONFIG_RETHOOK |
169 | |
170 | SYM_CODE_START(arch_rethook_trampoline) |
171 | stg %r14,(__SF_GPRS+8*8)(%r15) |
172 | lay %r15,-STACK_FRAME_SIZE_PTREGS(%r15) |
173 | stmg %r0,%r14,STACK_PTREGS_GPRS(%r15) |
174 | |
175 | # store original stack pointer in backchain and pt_regs |
176 | lay %r7,STACK_FRAME_SIZE_PTREGS(%r15) |
177 | stg %r7,__SF_BACKCHAIN(%r15) |
178 | stg %r7,STACK_PTREGS_GPRS+(15*8)(%r15) |
179 | |
180 | # store full psw |
181 | epsw %r2,%r3 |
182 | risbg %r3,%r2,0,31,32 |
183 | stg %r3,STACK_PTREGS_PSW(%r15) |
184 | larl %r1,arch_rethook_trampoline |
185 | stg %r1,STACK_PTREGS_PSW+8(%r15) |
186 | |
187 | lay %r2,STACK_PTREGS(%r15) |
188 | brasl %r14,arch_rethook_trampoline_callback |
189 | |
190 | mvc __SF_EMPTY(16,%r7),STACK_PTREGS_PSW(%r15) |
191 | lmg %r0,%r15,STACK_PTREGS_GPRS(%r15) |
192 | lpswe __SF_EMPTY(%r15) |
193 | SYM_CODE_END(arch_rethook_trampoline) |
194 | |
195 | #endif /* CONFIG_RETHOOK */ |
196 | |