1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* arch/sparc64/kernel/kprobes.c |
3 | * |
4 | * Copyright (C) 2004 David S. Miller <davem@davemloft.net> |
5 | */ |
6 | |
7 | #include <linux/kernel.h> |
8 | #include <linux/kprobes.h> |
9 | #include <linux/extable.h> |
10 | #include <linux/kdebug.h> |
11 | #include <linux/slab.h> |
12 | #include <linux/context_tracking.h> |
13 | #include <asm/signal.h> |
14 | #include <asm/cacheflush.h> |
15 | #include <linux/uaccess.h> |
16 | |
17 | /* We do not have hardware single-stepping on sparc64. |
18 | * So we implement software single-stepping with breakpoint |
19 | * traps. The top-level scheme is similar to that used |
20 | * in the x86 kprobes implementation. |
21 | * |
22 | * In the kprobe->ainsn.insn[] array we store the original |
23 | * instruction at index zero and a break instruction at |
24 | * index one. |
25 | * |
26 | * When we hit a kprobe we: |
27 | * - Run the pre-handler |
28 | * - Remember "regs->tnpc" and interrupt level stored in |
29 | * "regs->tstate" so we can restore them later |
30 | * - Disable PIL interrupts |
31 | * - Set regs->tpc to point to kprobe->ainsn.insn[0] |
32 | * - Set regs->tnpc to point to kprobe->ainsn.insn[1] |
33 | * - Mark that we are actively in a kprobe |
34 | * |
35 | * At this point we wait for the second breakpoint at |
36 | * kprobe->ainsn.insn[1] to hit. When it does we: |
37 | * - Run the post-handler |
38 | * - Set regs->tpc to "remembered" regs->tnpc stored above, |
39 | * restore the PIL interrupt level in "regs->tstate" as well |
40 | * - Make any adjustments necessary to regs->tnpc in order |
41 | * to handle relative branches correctly. See below. |
42 | * - Mark that we are no longer actively in a kprobe. |
43 | */ |
44 | |
45 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
46 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
47 | |
48 | struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; |
49 | |
50 | int __kprobes arch_prepare_kprobe(struct kprobe *p) |
51 | { |
52 | if ((unsigned long) p->addr & 0x3UL) |
53 | return -EILSEQ; |
54 | |
55 | p->ainsn.insn[0] = *p->addr; |
56 | flushi(&p->ainsn.insn[0]); |
57 | |
58 | p->ainsn.insn[1] = BREAKPOINT_INSTRUCTION_2; |
59 | flushi(&p->ainsn.insn[1]); |
60 | |
61 | p->opcode = *p->addr; |
62 | return 0; |
63 | } |
64 | |
65 | void __kprobes arch_arm_kprobe(struct kprobe *p) |
66 | { |
67 | *p->addr = BREAKPOINT_INSTRUCTION; |
68 | flushi(p->addr); |
69 | } |
70 | |
71 | void __kprobes arch_disarm_kprobe(struct kprobe *p) |
72 | { |
73 | *p->addr = p->opcode; |
74 | flushi(p->addr); |
75 | } |
76 | |
77 | static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) |
78 | { |
79 | kcb->prev_kprobe.kp = kprobe_running(); |
80 | kcb->prev_kprobe.status = kcb->kprobe_status; |
81 | kcb->prev_kprobe.orig_tnpc = kcb->kprobe_orig_tnpc; |
82 | kcb->prev_kprobe.orig_tstate_pil = kcb->kprobe_orig_tstate_pil; |
83 | } |
84 | |
85 | static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
86 | { |
87 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
88 | kcb->kprobe_status = kcb->prev_kprobe.status; |
89 | kcb->kprobe_orig_tnpc = kcb->prev_kprobe.orig_tnpc; |
90 | kcb->kprobe_orig_tstate_pil = kcb->prev_kprobe.orig_tstate_pil; |
91 | } |
92 | |
93 | static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, |
94 | struct kprobe_ctlblk *kcb) |
95 | { |
96 | __this_cpu_write(current_kprobe, p); |
97 | kcb->kprobe_orig_tnpc = regs->tnpc; |
98 | kcb->kprobe_orig_tstate_pil = (regs->tstate & TSTATE_PIL); |
99 | } |
100 | |
101 | static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs, |
102 | struct kprobe_ctlblk *kcb) |
103 | { |
104 | regs->tstate |= TSTATE_PIL; |
105 | |
106 | /*single step inline, if it a breakpoint instruction*/ |
107 | if (p->opcode == BREAKPOINT_INSTRUCTION) { |
108 | regs->tpc = (unsigned long) p->addr; |
109 | regs->tnpc = kcb->kprobe_orig_tnpc; |
110 | } else { |
111 | regs->tpc = (unsigned long) &p->ainsn.insn[0]; |
112 | regs->tnpc = (unsigned long) &p->ainsn.insn[1]; |
113 | } |
114 | } |
115 | |
116 | static int __kprobes kprobe_handler(struct pt_regs *regs) |
117 | { |
118 | struct kprobe *p; |
119 | void *addr = (void *) regs->tpc; |
120 | int ret = 0; |
121 | struct kprobe_ctlblk *kcb; |
122 | |
123 | /* |
124 | * We don't want to be preempted for the entire |
125 | * duration of kprobe processing |
126 | */ |
127 | preempt_disable(); |
128 | kcb = get_kprobe_ctlblk(); |
129 | |
130 | if (kprobe_running()) { |
131 | p = get_kprobe(addr); |
132 | if (p) { |
133 | if (kcb->kprobe_status == KPROBE_HIT_SS) { |
134 | regs->tstate = ((regs->tstate & ~TSTATE_PIL) | |
135 | kcb->kprobe_orig_tstate_pil); |
136 | goto no_kprobe; |
137 | } |
138 | /* We have reentered the kprobe_handler(), since |
139 | * another probe was hit while within the handler. |
140 | * We here save the original kprobes variables and |
141 | * just single step on the instruction of the new probe |
142 | * without calling any user handlers. |
143 | */ |
144 | save_previous_kprobe(kcb); |
145 | set_current_kprobe(p, regs, kcb); |
146 | kprobes_inc_nmissed_count(p); |
147 | kcb->kprobe_status = KPROBE_REENTER; |
148 | prepare_singlestep(p, regs, kcb); |
149 | return 1; |
150 | } else if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { |
151 | /* The breakpoint instruction was removed by |
152 | * another cpu right after we hit, no further |
153 | * handling of this interrupt is appropriate |
154 | */ |
155 | ret = 1; |
156 | } |
157 | goto no_kprobe; |
158 | } |
159 | |
160 | p = get_kprobe(addr); |
161 | if (!p) { |
162 | if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { |
163 | /* |
164 | * The breakpoint instruction was removed right |
165 | * after we hit it. Another cpu has removed |
166 | * either a probepoint or a debugger breakpoint |
167 | * at this address. In either case, no further |
168 | * handling of this interrupt is appropriate. |
169 | */ |
170 | ret = 1; |
171 | } |
172 | /* Not one of ours: let kernel handle it */ |
173 | goto no_kprobe; |
174 | } |
175 | |
176 | set_current_kprobe(p, regs, kcb); |
177 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
178 | if (p->pre_handler && p->pre_handler(p, regs)) { |
179 | reset_current_kprobe(); |
180 | preempt_enable_no_resched(); |
181 | return 1; |
182 | } |
183 | |
184 | prepare_singlestep(p, regs, kcb); |
185 | kcb->kprobe_status = KPROBE_HIT_SS; |
186 | return 1; |
187 | |
188 | no_kprobe: |
189 | preempt_enable_no_resched(); |
190 | return ret; |
191 | } |
192 | |
193 | /* If INSN is a relative control transfer instruction, |
194 | * return the corrected branch destination value. |
195 | * |
196 | * regs->tpc and regs->tnpc still hold the values of the |
197 | * program counters at the time of trap due to the execution |
198 | * of the BREAKPOINT_INSTRUCTION_2 at p->ainsn.insn[1] |
199 | * |
200 | */ |
201 | static unsigned long __kprobes relbranch_fixup(u32 insn, struct kprobe *p, |
202 | struct pt_regs *regs) |
203 | { |
204 | unsigned long real_pc = (unsigned long) p->addr; |
205 | |
206 | /* Branch not taken, no mods necessary. */ |
207 | if (regs->tnpc == regs->tpc + 0x4UL) |
208 | return real_pc + 0x8UL; |
209 | |
210 | /* The three cases are call, branch w/prediction, |
211 | * and traditional branch. |
212 | */ |
213 | if ((insn & 0xc0000000) == 0x40000000 || |
214 | (insn & 0xc1c00000) == 0x00400000 || |
215 | (insn & 0xc1c00000) == 0x00800000) { |
216 | unsigned long ainsn_addr; |
217 | |
218 | ainsn_addr = (unsigned long) &p->ainsn.insn[0]; |
219 | |
220 | /* The instruction did all the work for us |
221 | * already, just apply the offset to the correct |
222 | * instruction location. |
223 | */ |
224 | return (real_pc + (regs->tnpc - ainsn_addr)); |
225 | } |
226 | |
227 | /* It is jmpl or some other absolute PC modification instruction, |
228 | * leave NPC as-is. |
229 | */ |
230 | return regs->tnpc; |
231 | } |
232 | |
233 | /* If INSN is an instruction which writes its PC location |
234 | * into a destination register, fix that up. |
235 | */ |
236 | static void __kprobes retpc_fixup(struct pt_regs *regs, u32 insn, |
237 | unsigned long real_pc) |
238 | { |
239 | unsigned long *slot = NULL; |
240 | |
241 | /* Simplest case is 'call', which always uses %o7 */ |
242 | if ((insn & 0xc0000000) == 0x40000000) { |
243 | slot = ®s->u_regs[UREG_I7]; |
244 | } |
245 | |
246 | /* 'jmpl' encodes the register inside of the opcode */ |
247 | if ((insn & 0xc1f80000) == 0x81c00000) { |
248 | unsigned long rd = ((insn >> 25) & 0x1f); |
249 | |
250 | if (rd <= 15) { |
251 | slot = ®s->u_regs[rd]; |
252 | } else { |
253 | /* Hard case, it goes onto the stack. */ |
254 | flushw_all(); |
255 | |
256 | rd -= 16; |
257 | slot = (unsigned long *) |
258 | (regs->u_regs[UREG_FP] + STACK_BIAS); |
259 | slot += rd; |
260 | } |
261 | } |
262 | if (slot != NULL) |
263 | *slot = real_pc; |
264 | } |
265 | |
266 | /* |
267 | * Called after single-stepping. p->addr is the address of the |
268 | * instruction which has been replaced by the breakpoint |
269 | * instruction. To avoid the SMP problems that can occur when we |
270 | * temporarily put back the original opcode to single-step, we |
271 | * single-stepped a copy of the instruction. The address of this |
272 | * copy is &p->ainsn.insn[0]. |
273 | * |
274 | * This function prepares to return from the post-single-step |
275 | * breakpoint trap. |
276 | */ |
277 | static void __kprobes resume_execution(struct kprobe *p, |
278 | struct pt_regs *regs, struct kprobe_ctlblk *kcb) |
279 | { |
280 | u32 insn = p->ainsn.insn[0]; |
281 | |
282 | regs->tnpc = relbranch_fixup(insn, p, regs); |
283 | |
284 | /* This assignment must occur after relbranch_fixup() */ |
285 | regs->tpc = kcb->kprobe_orig_tnpc; |
286 | |
287 | retpc_fixup(regs, insn, real_pc: (unsigned long) p->addr); |
288 | |
289 | regs->tstate = ((regs->tstate & ~TSTATE_PIL) | |
290 | kcb->kprobe_orig_tstate_pil); |
291 | } |
292 | |
293 | static int __kprobes post_kprobe_handler(struct pt_regs *regs) |
294 | { |
295 | struct kprobe *cur = kprobe_running(); |
296 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
297 | |
298 | if (!cur) |
299 | return 0; |
300 | |
301 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
302 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
303 | cur->post_handler(cur, regs, 0); |
304 | } |
305 | |
306 | resume_execution(p: cur, regs, kcb); |
307 | |
308 | /*Restore back the original saved kprobes variables and continue. */ |
309 | if (kcb->kprobe_status == KPROBE_REENTER) { |
310 | restore_previous_kprobe(kcb); |
311 | goto out; |
312 | } |
313 | reset_current_kprobe(); |
314 | out: |
315 | preempt_enable_no_resched(); |
316 | |
317 | return 1; |
318 | } |
319 | |
320 | int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
321 | { |
322 | struct kprobe *cur = kprobe_running(); |
323 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
324 | const struct exception_table_entry *entry; |
325 | |
326 | switch(kcb->kprobe_status) { |
327 | case KPROBE_HIT_SS: |
328 | case KPROBE_REENTER: |
329 | /* |
330 | * We are here because the instruction being single |
331 | * stepped caused a page fault. We reset the current |
332 | * kprobe and the tpc points back to the probe address |
333 | * and allow the page fault handler to continue as a |
334 | * normal page fault. |
335 | */ |
336 | regs->tpc = (unsigned long)cur->addr; |
337 | regs->tnpc = kcb->kprobe_orig_tnpc; |
338 | regs->tstate = ((regs->tstate & ~TSTATE_PIL) | |
339 | kcb->kprobe_orig_tstate_pil); |
340 | if (kcb->kprobe_status == KPROBE_REENTER) |
341 | restore_previous_kprobe(kcb); |
342 | else |
343 | reset_current_kprobe(); |
344 | preempt_enable_no_resched(); |
345 | break; |
346 | case KPROBE_HIT_ACTIVE: |
347 | case KPROBE_HIT_SSDONE: |
348 | /* |
349 | * In case the user-specified fault handler returned |
350 | * zero, try to fix up. |
351 | */ |
352 | |
353 | entry = search_exception_tables(add: regs->tpc); |
354 | if (entry) { |
355 | regs->tpc = entry->fixup; |
356 | regs->tnpc = regs->tpc + 4; |
357 | return 1; |
358 | } |
359 | |
360 | /* |
361 | * fixup_exception() could not handle it, |
362 | * Let do_page_fault() fix it. |
363 | */ |
364 | break; |
365 | default: |
366 | break; |
367 | } |
368 | |
369 | return 0; |
370 | } |
371 | |
372 | /* |
373 | * Wrapper routine to for handling exceptions. |
374 | */ |
375 | int __kprobes kprobe_exceptions_notify(struct notifier_block *self, |
376 | unsigned long val, void *data) |
377 | { |
378 | struct die_args *args = (struct die_args *)data; |
379 | int ret = NOTIFY_DONE; |
380 | |
381 | if (args->regs && user_mode(regs: args->regs)) |
382 | return ret; |
383 | |
384 | switch (val) { |
385 | case DIE_DEBUG: |
386 | if (kprobe_handler(regs: args->regs)) |
387 | ret = NOTIFY_STOP; |
388 | break; |
389 | case DIE_DEBUG_2: |
390 | if (post_kprobe_handler(regs: args->regs)) |
391 | ret = NOTIFY_STOP; |
392 | break; |
393 | default: |
394 | break; |
395 | } |
396 | return ret; |
397 | } |
398 | |
399 | asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, |
400 | struct pt_regs *regs) |
401 | { |
402 | enum ctx_state prev_state = exception_enter(); |
403 | |
404 | BUG_ON(trap_level != 0x170 && trap_level != 0x171); |
405 | |
406 | if (user_mode(regs)) { |
407 | local_irq_enable(); |
408 | bad_trap(regs, trap_level); |
409 | goto out; |
410 | } |
411 | |
412 | /* trap_level == 0x170 --> ta 0x70 |
413 | * trap_level == 0x171 --> ta 0x71 |
414 | */ |
415 | if (notify_die((trap_level == 0x170) ? DIE_DEBUG : DIE_DEBUG_2, |
416 | (trap_level == 0x170) ? "debug" : "debug_2" , |
417 | regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) |
418 | bad_trap(regs, trap_level); |
419 | out: |
420 | exception_exit(prev_ctx: prev_state); |
421 | } |
422 | |
423 | /* The value stored in the return address register is actually 2 |
424 | * instructions before where the callee will return to. |
425 | * Sequences usually look something like this |
426 | * |
427 | * call some_function <--- return register points here |
428 | * nop <--- call delay slot |
429 | * whatever <--- where callee returns to |
430 | * |
431 | * To keep trampoline_probe_handler logic simpler, we normalize the |
432 | * value kept in ri->ret_addr so we don't need to keep adjusting it |
433 | * back and forth. |
434 | */ |
435 | void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, |
436 | struct pt_regs *regs) |
437 | { |
438 | ri->ret_addr = (kprobe_opcode_t *)(regs->u_regs[UREG_RETPC] + 8); |
439 | ri->fp = NULL; |
440 | |
441 | /* Replace the return addr with trampoline addr */ |
442 | regs->u_regs[UREG_RETPC] = |
443 | ((unsigned long)__kretprobe_trampoline) - 8; |
444 | } |
445 | |
446 | /* |
447 | * Called when the probe at kretprobe trampoline is hit |
448 | */ |
449 | static int __kprobes trampoline_probe_handler(struct kprobe *p, |
450 | struct pt_regs *regs) |
451 | { |
452 | unsigned long orig_ret_address = 0; |
453 | |
454 | orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); |
455 | regs->tpc = orig_ret_address; |
456 | regs->tnpc = orig_ret_address + 4; |
457 | |
458 | /* |
459 | * By returning a non-zero value, we are telling |
460 | * kprobe_handler() that we don't want the post_handler |
461 | * to run (and have re-enabled preemption) |
462 | */ |
463 | return 1; |
464 | } |
465 | |
466 | static void __used kretprobe_trampoline_holder(void) |
467 | { |
468 | asm volatile(".global __kretprobe_trampoline\n" |
469 | "__kretprobe_trampoline:\n" |
470 | "\tnop\n" |
471 | "\tnop\n" ); |
472 | } |
473 | static struct kprobe trampoline_p = { |
474 | .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, |
475 | .pre_handler = trampoline_probe_handler |
476 | }; |
477 | |
478 | int __init arch_init_kprobes(void) |
479 | { |
480 | return register_kprobe(p: &trampoline_p); |
481 | } |
482 | |
483 | int __kprobes arch_trampoline_kprobe(struct kprobe *p) |
484 | { |
485 | if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) |
486 | return 1; |
487 | |
488 | return 0; |
489 | } |
490 | |