1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/kdebug.h> |
3 | #include <linux/kprobes.h> |
4 | #include <linux/preempt.h> |
5 | #include <asm/break.h> |
6 | |
7 | #define KPROBE_BP_INSN larch_insn_gen_break(BRK_KPROBE_BP) |
8 | #define KPROBE_SSTEPBP_INSN larch_insn_gen_break(BRK_KPROBE_SSTEPBP) |
9 | |
10 | DEFINE_PER_CPU(struct kprobe *, current_kprobe); |
11 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
12 | |
13 | static void arch_prepare_ss_slot(struct kprobe *p) |
14 | { |
15 | p->ainsn.insn[0] = *p->addr; |
16 | p->ainsn.insn[1] = KPROBE_SSTEPBP_INSN; |
17 | p->ainsn.restore = (unsigned long)p->addr + LOONGARCH_INSN_SIZE; |
18 | } |
19 | NOKPROBE_SYMBOL(arch_prepare_ss_slot); |
20 | |
21 | static void arch_prepare_simulate(struct kprobe *p) |
22 | { |
23 | p->ainsn.restore = 0; |
24 | } |
25 | NOKPROBE_SYMBOL(arch_prepare_simulate); |
26 | |
27 | int arch_prepare_kprobe(struct kprobe *p) |
28 | { |
29 | union loongarch_instruction insn; |
30 | |
31 | if ((unsigned long)p->addr & 0x3) |
32 | return -EILSEQ; |
33 | |
34 | /* copy instruction */ |
35 | p->opcode = *p->addr; |
36 | insn.word = p->opcode; |
37 | |
38 | /* decode instruction */ |
39 | if (insns_not_supported(insn)) |
40 | return -EINVAL; |
41 | |
42 | if (insns_need_simulation(insn)) { |
43 | p->ainsn.insn = NULL; |
44 | } else { |
45 | p->ainsn.insn = get_insn_slot(); |
46 | if (!p->ainsn.insn) |
47 | return -ENOMEM; |
48 | } |
49 | |
50 | /* prepare the instruction */ |
51 | if (p->ainsn.insn) |
52 | arch_prepare_ss_slot(p); |
53 | else |
54 | arch_prepare_simulate(p); |
55 | |
56 | return 0; |
57 | } |
58 | NOKPROBE_SYMBOL(arch_prepare_kprobe); |
59 | |
60 | /* Install breakpoint in text */ |
61 | void arch_arm_kprobe(struct kprobe *p) |
62 | { |
63 | *p->addr = KPROBE_BP_INSN; |
64 | flush_insn_slot(p); |
65 | } |
66 | NOKPROBE_SYMBOL(arch_arm_kprobe); |
67 | |
68 | /* Remove breakpoint from text */ |
69 | void arch_disarm_kprobe(struct kprobe *p) |
70 | { |
71 | *p->addr = p->opcode; |
72 | flush_insn_slot(p); |
73 | } |
74 | NOKPROBE_SYMBOL(arch_disarm_kprobe); |
75 | |
76 | void arch_remove_kprobe(struct kprobe *p) |
77 | { |
78 | if (p->ainsn.insn) { |
79 | free_insn_slot(slot: p->ainsn.insn, dirty: 0); |
80 | p->ainsn.insn = NULL; |
81 | } |
82 | } |
83 | NOKPROBE_SYMBOL(arch_remove_kprobe); |
84 | |
85 | static void save_previous_kprobe(struct kprobe_ctlblk *kcb) |
86 | { |
87 | kcb->prev_kprobe.kp = kprobe_running(); |
88 | kcb->prev_kprobe.status = kcb->kprobe_status; |
89 | } |
90 | NOKPROBE_SYMBOL(save_previous_kprobe); |
91 | |
92 | static void restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
93 | { |
94 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
95 | kcb->kprobe_status = kcb->prev_kprobe.status; |
96 | } |
97 | NOKPROBE_SYMBOL(restore_previous_kprobe); |
98 | |
99 | static void set_current_kprobe(struct kprobe *p) |
100 | { |
101 | __this_cpu_write(current_kprobe, p); |
102 | } |
103 | NOKPROBE_SYMBOL(set_current_kprobe); |
104 | |
105 | /* |
106 | * Interrupts need to be disabled before single-step mode is set, |
107 | * and not reenabled until after single-step mode ends. |
108 | * Without disabling interrupt on local CPU, there is a chance of |
109 | * interrupt occurrence in the period of exception return and start |
110 | * of out-of-line single-step, that result in wrongly single stepping |
111 | * into the interrupt handler. |
112 | */ |
113 | static void save_local_irqflag(struct kprobe_ctlblk *kcb, |
114 | struct pt_regs *regs) |
115 | { |
116 | kcb->saved_status = regs->csr_prmd; |
117 | regs->csr_prmd &= ~CSR_PRMD_PIE; |
118 | } |
119 | NOKPROBE_SYMBOL(save_local_irqflag); |
120 | |
121 | static void restore_local_irqflag(struct kprobe_ctlblk *kcb, |
122 | struct pt_regs *regs) |
123 | { |
124 | regs->csr_prmd = kcb->saved_status; |
125 | } |
126 | NOKPROBE_SYMBOL(restore_local_irqflag); |
127 | |
128 | static void post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, |
129 | struct pt_regs *regs) |
130 | { |
131 | /* return addr restore if non-branching insn */ |
132 | if (cur->ainsn.restore != 0) |
133 | instruction_pointer_set(regs, val: cur->ainsn.restore); |
134 | |
135 | /* restore back original saved kprobe variables and continue */ |
136 | if (kcb->kprobe_status == KPROBE_REENTER) { |
137 | restore_previous_kprobe(kcb); |
138 | preempt_enable_no_resched(); |
139 | return; |
140 | } |
141 | |
142 | /* |
143 | * update the kcb status even if the cur->post_handler is |
144 | * not set because reset_curent_kprobe() doesn't update kcb. |
145 | */ |
146 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
147 | if (cur->post_handler) |
148 | cur->post_handler(cur, regs, 0); |
149 | |
150 | reset_current_kprobe(); |
151 | preempt_enable_no_resched(); |
152 | } |
153 | NOKPROBE_SYMBOL(post_kprobe_handler); |
154 | |
155 | static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, |
156 | struct kprobe_ctlblk *kcb, int reenter) |
157 | { |
158 | union loongarch_instruction insn; |
159 | |
160 | if (reenter) { |
161 | save_previous_kprobe(kcb); |
162 | set_current_kprobe(p); |
163 | kcb->kprobe_status = KPROBE_REENTER; |
164 | } else { |
165 | kcb->kprobe_status = KPROBE_HIT_SS; |
166 | } |
167 | |
168 | if (p->ainsn.insn) { |
169 | /* IRQs and single stepping do not mix well */ |
170 | save_local_irqflag(kcb, regs); |
171 | /* set ip register to prepare for single stepping */ |
172 | regs->csr_era = (unsigned long)p->ainsn.insn; |
173 | } else { |
174 | /* simulate single steping */ |
175 | insn.word = p->opcode; |
176 | arch_simulate_insn(insn, regs); |
177 | /* now go for post processing */ |
178 | post_kprobe_handler(cur: p, kcb, regs); |
179 | } |
180 | } |
181 | NOKPROBE_SYMBOL(setup_singlestep); |
182 | |
183 | static bool reenter_kprobe(struct kprobe *p, struct pt_regs *regs, |
184 | struct kprobe_ctlblk *kcb) |
185 | { |
186 | switch (kcb->kprobe_status) { |
187 | case KPROBE_HIT_SS: |
188 | case KPROBE_HIT_SSDONE: |
189 | case KPROBE_HIT_ACTIVE: |
190 | kprobes_inc_nmissed_count(p); |
191 | setup_singlestep(p, regs, kcb, reenter: 1); |
192 | break; |
193 | case KPROBE_REENTER: |
194 | pr_warn("Failed to recover from reentered kprobes.\n" ); |
195 | dump_kprobe(kp: p); |
196 | WARN_ON_ONCE(1); |
197 | break; |
198 | default: |
199 | WARN_ON(1); |
200 | return false; |
201 | } |
202 | |
203 | return true; |
204 | } |
205 | NOKPROBE_SYMBOL(reenter_kprobe); |
206 | |
207 | bool kprobe_breakpoint_handler(struct pt_regs *regs) |
208 | { |
209 | struct kprobe_ctlblk *kcb; |
210 | struct kprobe *p, *cur_kprobe; |
211 | kprobe_opcode_t *addr = (kprobe_opcode_t *)regs->csr_era; |
212 | |
213 | /* |
214 | * We don't want to be preempted for the entire |
215 | * duration of kprobe processing. |
216 | */ |
217 | preempt_disable(); |
218 | kcb = get_kprobe_ctlblk(); |
219 | cur_kprobe = kprobe_running(); |
220 | |
221 | p = get_kprobe(addr); |
222 | if (p) { |
223 | if (cur_kprobe) { |
224 | if (reenter_kprobe(p, regs, kcb)) |
225 | return true; |
226 | } else { |
227 | /* Probe hit */ |
228 | set_current_kprobe(p); |
229 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
230 | |
231 | /* |
232 | * If we have no pre-handler or it returned 0, we |
233 | * continue with normal processing. If we have a |
234 | * pre-handler and it returned non-zero, it will |
235 | * modify the execution path and no need to single |
236 | * stepping. Let's just reset current kprobe and exit. |
237 | * |
238 | * pre_handler can hit a breakpoint and can step thru |
239 | * before return. |
240 | */ |
241 | if (!p->pre_handler || !p->pre_handler(p, regs)) { |
242 | setup_singlestep(p, regs, kcb, reenter: 0); |
243 | } else { |
244 | reset_current_kprobe(); |
245 | preempt_enable_no_resched(); |
246 | } |
247 | return true; |
248 | } |
249 | } |
250 | |
251 | if (*addr != KPROBE_BP_INSN) { |
252 | /* |
253 | * The breakpoint instruction was removed right |
254 | * after we hit it. Another cpu has removed |
255 | * either a probepoint or a debugger breakpoint |
256 | * at this address. In either case, no further |
257 | * handling of this interrupt is appropriate. |
258 | * Return back to original instruction, and continue. |
259 | */ |
260 | regs->csr_era = (unsigned long)addr; |
261 | preempt_enable_no_resched(); |
262 | return true; |
263 | } |
264 | |
265 | preempt_enable_no_resched(); |
266 | return false; |
267 | } |
268 | NOKPROBE_SYMBOL(kprobe_breakpoint_handler); |
269 | |
270 | bool kprobe_singlestep_handler(struct pt_regs *regs) |
271 | { |
272 | struct kprobe *cur = kprobe_running(); |
273 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
274 | unsigned long addr = instruction_pointer(regs); |
275 | |
276 | if (cur && (kcb->kprobe_status & (KPROBE_HIT_SS | KPROBE_REENTER)) && |
277 | ((unsigned long)&cur->ainsn.insn[1] == addr)) { |
278 | restore_local_irqflag(kcb, regs); |
279 | post_kprobe_handler(cur, kcb, regs); |
280 | return true; |
281 | } |
282 | |
283 | preempt_enable_no_resched(); |
284 | return false; |
285 | } |
286 | NOKPROBE_SYMBOL(kprobe_singlestep_handler); |
287 | |
288 | bool kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
289 | { |
290 | struct kprobe *cur = kprobe_running(); |
291 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
292 | |
293 | switch (kcb->kprobe_status) { |
294 | case KPROBE_HIT_SS: |
295 | case KPROBE_REENTER: |
296 | /* |
297 | * We are here because the instruction being single |
298 | * stepped caused a page fault. We reset the current |
299 | * kprobe and the ip points back to the probe address |
300 | * and allow the page fault handler to continue as a |
301 | * normal page fault. |
302 | */ |
303 | regs->csr_era = (unsigned long)cur->addr; |
304 | WARN_ON_ONCE(!instruction_pointer(regs)); |
305 | |
306 | if (kcb->kprobe_status == KPROBE_REENTER) { |
307 | restore_previous_kprobe(kcb); |
308 | } else { |
309 | restore_local_irqflag(kcb, regs); |
310 | reset_current_kprobe(); |
311 | } |
312 | preempt_enable_no_resched(); |
313 | break; |
314 | } |
315 | return false; |
316 | } |
317 | NOKPROBE_SYMBOL(kprobe_fault_handler); |
318 | |
319 | /* |
320 | * Provide a blacklist of symbols identifying ranges which cannot be kprobed. |
321 | * This blacklist is exposed to userspace via debugfs (kprobes/blacklist). |
322 | */ |
323 | int __init arch_populate_kprobe_blacklist(void) |
324 | { |
325 | return kprobe_add_area_blacklist(start: (unsigned long)__irqentry_text_start, |
326 | end: (unsigned long)__irqentry_text_end); |
327 | } |
328 | |
329 | int __init arch_init_kprobes(void) |
330 | { |
331 | return 0; |
332 | } |
333 | |
334 | int arch_trampoline_kprobe(struct kprobe *p) |
335 | { |
336 | return 0; |
337 | } |
338 | NOKPROBE_SYMBOL(arch_trampoline_kprobe); |
339 | |