1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) |
4 | */ |
5 | |
6 | #include <linux/types.h> |
7 | #include <linux/kprobes.h> |
8 | #include <linux/slab.h> |
9 | #include <linux/module.h> |
10 | #include <linux/kdebug.h> |
11 | #include <linux/sched.h> |
12 | #include <linux/uaccess.h> |
13 | #include <asm/cacheflush.h> |
14 | #include <asm/current.h> |
15 | #include <asm/disasm.h> |
16 | |
17 | #define MIN_STACK_SIZE(addr) min((unsigned long)MAX_STACK_SIZE, \ |
18 | (unsigned long)current_thread_info() + THREAD_SIZE - (addr)) |
19 | |
20 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
21 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
22 | |
23 | int __kprobes arch_prepare_kprobe(struct kprobe *p) |
24 | { |
25 | /* Attempt to probe at unaligned address */ |
26 | if ((unsigned long)p->addr & 0x01) |
27 | return -EINVAL; |
28 | |
29 | /* Address should not be in exception handling code */ |
30 | |
31 | p->ainsn.is_short = is_short_instr((unsigned long)p->addr); |
32 | p->opcode = *p->addr; |
33 | |
34 | return 0; |
35 | } |
36 | |
37 | void __kprobes arch_arm_kprobe(struct kprobe *p) |
38 | { |
39 | *p->addr = UNIMP_S_INSTRUCTION; |
40 | |
41 | flush_icache_range(start: (unsigned long)p->addr, |
42 | end: (unsigned long)p->addr + sizeof(kprobe_opcode_t)); |
43 | } |
44 | |
45 | void __kprobes arch_disarm_kprobe(struct kprobe *p) |
46 | { |
47 | *p->addr = p->opcode; |
48 | |
49 | flush_icache_range(start: (unsigned long)p->addr, |
50 | end: (unsigned long)p->addr + sizeof(kprobe_opcode_t)); |
51 | } |
52 | |
53 | void __kprobes arch_remove_kprobe(struct kprobe *p) |
54 | { |
55 | arch_disarm_kprobe(p); |
56 | |
57 | /* Can we remove the kprobe in the middle of kprobe handling? */ |
58 | if (p->ainsn.t1_addr) { |
59 | *(p->ainsn.t1_addr) = p->ainsn.t1_opcode; |
60 | |
61 | flush_icache_range(start: (unsigned long)p->ainsn.t1_addr, |
62 | end: (unsigned long)p->ainsn.t1_addr + |
63 | sizeof(kprobe_opcode_t)); |
64 | |
65 | p->ainsn.t1_addr = NULL; |
66 | } |
67 | |
68 | if (p->ainsn.t2_addr) { |
69 | *(p->ainsn.t2_addr) = p->ainsn.t2_opcode; |
70 | |
71 | flush_icache_range(start: (unsigned long)p->ainsn.t2_addr, |
72 | end: (unsigned long)p->ainsn.t2_addr + |
73 | sizeof(kprobe_opcode_t)); |
74 | |
75 | p->ainsn.t2_addr = NULL; |
76 | } |
77 | } |
78 | |
79 | static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) |
80 | { |
81 | kcb->prev_kprobe.kp = kprobe_running(); |
82 | kcb->prev_kprobe.status = kcb->kprobe_status; |
83 | } |
84 | |
85 | static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
86 | { |
87 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
88 | kcb->kprobe_status = kcb->prev_kprobe.status; |
89 | } |
90 | |
91 | static inline void __kprobes set_current_kprobe(struct kprobe *p) |
92 | { |
93 | __this_cpu_write(current_kprobe, p); |
94 | } |
95 | |
96 | static void __kprobes resume_execution(struct kprobe *p, unsigned long addr, |
97 | struct pt_regs *regs) |
98 | { |
99 | /* Remove the trap instructions inserted for single step and |
100 | * restore the original instructions |
101 | */ |
102 | if (p->ainsn.t1_addr) { |
103 | *(p->ainsn.t1_addr) = p->ainsn.t1_opcode; |
104 | |
105 | flush_icache_range(start: (unsigned long)p->ainsn.t1_addr, |
106 | end: (unsigned long)p->ainsn.t1_addr + |
107 | sizeof(kprobe_opcode_t)); |
108 | |
109 | p->ainsn.t1_addr = NULL; |
110 | } |
111 | |
112 | if (p->ainsn.t2_addr) { |
113 | *(p->ainsn.t2_addr) = p->ainsn.t2_opcode; |
114 | |
115 | flush_icache_range(start: (unsigned long)p->ainsn.t2_addr, |
116 | end: (unsigned long)p->ainsn.t2_addr + |
117 | sizeof(kprobe_opcode_t)); |
118 | |
119 | p->ainsn.t2_addr = NULL; |
120 | } |
121 | |
122 | return; |
123 | } |
124 | |
125 | static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs) |
126 | { |
127 | unsigned long next_pc; |
128 | unsigned long tgt_if_br = 0; |
129 | int is_branch; |
130 | unsigned long bta; |
131 | |
132 | /* Copy the opcode back to the kprobe location and execute the |
133 | * instruction. Because of this we will not be able to get into the |
134 | * same kprobe until this kprobe is done |
135 | */ |
136 | *(p->addr) = p->opcode; |
137 | |
138 | flush_icache_range(start: (unsigned long)p->addr, |
139 | end: (unsigned long)p->addr + sizeof(kprobe_opcode_t)); |
140 | |
141 | /* Now we insert the trap at the next location after this instruction to |
142 | * single step. If it is a branch we insert the trap at possible branch |
143 | * targets |
144 | */ |
145 | |
146 | bta = regs->bta; |
147 | |
148 | if (regs->status32 & 0x40) { |
149 | /* We are in a delay slot with the branch taken */ |
150 | |
151 | next_pc = bta & ~0x01; |
152 | |
153 | if (!p->ainsn.is_short) { |
154 | if (bta & 0x01) |
155 | regs->blink += 2; |
156 | else { |
157 | /* Branch not taken */ |
158 | next_pc += 2; |
159 | |
160 | /* next pc is taken from bta after executing the |
161 | * delay slot instruction |
162 | */ |
163 | regs->bta += 2; |
164 | } |
165 | } |
166 | |
167 | is_branch = 0; |
168 | } else |
169 | is_branch = |
170 | disasm_next_pc((unsigned long)p->addr, regs, |
171 | (struct callee_regs *) current->thread.callee_reg, |
172 | &next_pc, &tgt_if_br); |
173 | |
174 | p->ainsn.t1_addr = (kprobe_opcode_t *) next_pc; |
175 | p->ainsn.t1_opcode = *(p->ainsn.t1_addr); |
176 | *(p->ainsn.t1_addr) = TRAP_S_2_INSTRUCTION; |
177 | |
178 | flush_icache_range(start: (unsigned long)p->ainsn.t1_addr, |
179 | end: (unsigned long)p->ainsn.t1_addr + |
180 | sizeof(kprobe_opcode_t)); |
181 | |
182 | if (is_branch) { |
183 | p->ainsn.t2_addr = (kprobe_opcode_t *) tgt_if_br; |
184 | p->ainsn.t2_opcode = *(p->ainsn.t2_addr); |
185 | *(p->ainsn.t2_addr) = TRAP_S_2_INSTRUCTION; |
186 | |
187 | flush_icache_range(start: (unsigned long)p->ainsn.t2_addr, |
188 | end: (unsigned long)p->ainsn.t2_addr + |
189 | sizeof(kprobe_opcode_t)); |
190 | } |
191 | } |
192 | |
193 | int __kprobes arc_kprobe_handler(unsigned long addr, struct pt_regs *regs) |
194 | { |
195 | struct kprobe *p; |
196 | struct kprobe_ctlblk *kcb; |
197 | |
198 | preempt_disable(); |
199 | |
200 | kcb = get_kprobe_ctlblk(); |
201 | p = get_kprobe(addr: (unsigned long *)addr); |
202 | |
203 | if (p) { |
204 | /* |
205 | * We have reentered the kprobe_handler, since another kprobe |
206 | * was hit while within the handler, we save the original |
207 | * kprobes and single step on the instruction of the new probe |
208 | * without calling any user handlers to avoid recursive |
209 | * kprobes. |
210 | */ |
211 | if (kprobe_running()) { |
212 | save_previous_kprobe(kcb); |
213 | set_current_kprobe(p); |
214 | kprobes_inc_nmissed_count(p); |
215 | setup_singlestep(p, regs); |
216 | kcb->kprobe_status = KPROBE_REENTER; |
217 | return 1; |
218 | } |
219 | |
220 | set_current_kprobe(p); |
221 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
222 | |
223 | /* If we have no pre-handler or it returned 0, we continue with |
224 | * normal processing. If we have a pre-handler and it returned |
225 | * non-zero - which means user handler setup registers to exit |
226 | * to another instruction, we must skip the single stepping. |
227 | */ |
228 | if (!p->pre_handler || !p->pre_handler(p, regs)) { |
229 | setup_singlestep(p, regs); |
230 | kcb->kprobe_status = KPROBE_HIT_SS; |
231 | } else { |
232 | reset_current_kprobe(); |
233 | preempt_enable_no_resched(); |
234 | } |
235 | |
236 | return 1; |
237 | } |
238 | |
239 | /* no_kprobe: */ |
240 | preempt_enable_no_resched(); |
241 | return 0; |
242 | } |
243 | |
244 | static int __kprobes arc_post_kprobe_handler(unsigned long addr, |
245 | struct pt_regs *regs) |
246 | { |
247 | struct kprobe *cur = kprobe_running(); |
248 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
249 | |
250 | if (!cur) |
251 | return 0; |
252 | |
253 | resume_execution(p: cur, addr, regs); |
254 | |
255 | /* Rearm the kprobe */ |
256 | arch_arm_kprobe(p: cur); |
257 | |
258 | /* |
259 | * When we return from trap instruction we go to the next instruction |
260 | * We restored the actual instruction in resume_exectuiont and we to |
261 | * return to the same address and execute it |
262 | */ |
263 | regs->ret = addr; |
264 | |
265 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
266 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
267 | cur->post_handler(cur, regs, 0); |
268 | } |
269 | |
270 | if (kcb->kprobe_status == KPROBE_REENTER) { |
271 | restore_previous_kprobe(kcb); |
272 | goto out; |
273 | } |
274 | |
275 | reset_current_kprobe(); |
276 | |
277 | out: |
278 | preempt_enable_no_resched(); |
279 | return 1; |
280 | } |
281 | |
282 | /* |
283 | * Fault can be for the instruction being single stepped or for the |
284 | * pre/post handlers in the module. |
285 | * This is applicable for applications like user probes, where we have the |
286 | * probe in user space and the handlers in the kernel |
287 | */ |
288 | |
289 | int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned long trapnr) |
290 | { |
291 | struct kprobe *cur = kprobe_running(); |
292 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
293 | |
294 | switch (kcb->kprobe_status) { |
295 | case KPROBE_HIT_SS: |
296 | case KPROBE_REENTER: |
297 | /* |
298 | * We are here because the instruction being single stepped |
299 | * caused the fault. We reset the current kprobe and allow the |
300 | * exception handler as if it is regular exception. In our |
301 | * case it doesn't matter because the system will be halted |
302 | */ |
303 | resume_execution(p: cur, addr: (unsigned long)cur->addr, regs); |
304 | |
305 | if (kcb->kprobe_status == KPROBE_REENTER) |
306 | restore_previous_kprobe(kcb); |
307 | else |
308 | reset_current_kprobe(); |
309 | |
310 | preempt_enable_no_resched(); |
311 | break; |
312 | |
313 | case KPROBE_HIT_ACTIVE: |
314 | case KPROBE_HIT_SSDONE: |
315 | /* |
316 | * We are here because the instructions in the pre/post handler |
317 | * caused the fault. |
318 | */ |
319 | |
320 | /* |
321 | * In case the user-specified fault handler returned zero, |
322 | * try to fix up. |
323 | */ |
324 | if (fixup_exception(regs)) |
325 | return 1; |
326 | |
327 | /* |
328 | * fixup_exception() could not handle it, |
329 | * Let do_page_fault() fix it. |
330 | */ |
331 | break; |
332 | |
333 | default: |
334 | break; |
335 | } |
336 | return 0; |
337 | } |
338 | |
339 | int __kprobes kprobe_exceptions_notify(struct notifier_block *self, |
340 | unsigned long val, void *data) |
341 | { |
342 | struct die_args *args = data; |
343 | unsigned long addr = args->err; |
344 | int ret = NOTIFY_DONE; |
345 | |
346 | switch (val) { |
347 | case DIE_IERR: |
348 | if (arc_kprobe_handler(addr, regs: args->regs)) |
349 | return NOTIFY_STOP; |
350 | break; |
351 | |
352 | case DIE_TRAP: |
353 | if (arc_post_kprobe_handler(addr, regs: args->regs)) |
354 | return NOTIFY_STOP; |
355 | break; |
356 | |
357 | default: |
358 | break; |
359 | } |
360 | |
361 | return ret; |
362 | } |
363 | |
364 | static void __used kretprobe_trampoline_holder(void) |
365 | { |
366 | __asm__ __volatile__(".global __kretprobe_trampoline\n" |
367 | "__kretprobe_trampoline:\n" |
368 | "nop\n" ); |
369 | } |
370 | |
371 | void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, |
372 | struct pt_regs *regs) |
373 | { |
374 | |
375 | ri->ret_addr = (kprobe_opcode_t *) regs->blink; |
376 | ri->fp = NULL; |
377 | |
378 | /* Replace the return addr with trampoline addr */ |
379 | regs->blink = (unsigned long)&__kretprobe_trampoline; |
380 | } |
381 | |
382 | static int __kprobes trampoline_probe_handler(struct kprobe *p, |
383 | struct pt_regs *regs) |
384 | { |
385 | regs->ret = __kretprobe_trampoline_handler(regs, NULL); |
386 | |
387 | /* By returning a non zero value, we are telling the kprobe handler |
388 | * that we don't want the post_handler to run |
389 | */ |
390 | return 1; |
391 | } |
392 | |
393 | static struct kprobe trampoline_p = { |
394 | .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, |
395 | .pre_handler = trampoline_probe_handler |
396 | }; |
397 | |
398 | int __init arch_init_kprobes(void) |
399 | { |
400 | /* Registering the trampoline code for the kret probe */ |
401 | return register_kprobe(p: &trampoline_p); |
402 | } |
403 | |
404 | int __kprobes arch_trampoline_kprobe(struct kprobe *p) |
405 | { |
406 | if (p->addr == (kprobe_opcode_t *) &__kretprobe_trampoline) |
407 | return 1; |
408 | |
409 | return 0; |
410 | } |
411 | |
412 | void trap_is_kprobe(unsigned long address, struct pt_regs *regs) |
413 | { |
414 | notify_die(val: DIE_TRAP, str: "kprobe_trap" , regs, err: address, trap: 0, SIGTRAP); |
415 | } |
416 | |