1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Kernel Probes (KProbes) |
4 | * |
5 | * Copyright (C) IBM Corporation, 2002, 2004 |
6 | * |
7 | * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel |
8 | * Probes initial implementation ( includes contributions from |
9 | * Rusty Russell). |
10 | * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes |
11 | * interface to access function arguments. |
12 | * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port |
13 | * for PPC64 |
14 | */ |
15 | |
16 | #include <linux/kprobes.h> |
17 | #include <linux/ptrace.h> |
18 | #include <linux/preempt.h> |
19 | #include <linux/extable.h> |
20 | #include <linux/kdebug.h> |
21 | #include <linux/slab.h> |
22 | #include <linux/moduleloader.h> |
23 | #include <linux/set_memory.h> |
24 | #include <asm/code-patching.h> |
25 | #include <asm/cacheflush.h> |
26 | #include <asm/sstep.h> |
27 | #include <asm/sections.h> |
28 | #include <asm/inst.h> |
29 | #include <linux/uaccess.h> |
30 | |
31 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
32 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
33 | |
34 | struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; |
35 | |
36 | bool arch_within_kprobe_blacklist(unsigned long addr) |
37 | { |
38 | return (addr >= (unsigned long)__kprobes_text_start && |
39 | addr < (unsigned long)__kprobes_text_end) || |
40 | (addr >= (unsigned long)_stext && |
41 | addr < (unsigned long)__head_end); |
42 | } |
43 | |
44 | kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset) |
45 | { |
46 | kprobe_opcode_t *addr = NULL; |
47 | |
48 | #ifdef CONFIG_PPC64_ELF_ABI_V2 |
49 | /* PPC64 ABIv2 needs local entry point */ |
50 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); |
51 | if (addr && !offset) { |
52 | #ifdef CONFIG_KPROBES_ON_FTRACE |
53 | unsigned long faddr; |
54 | /* |
55 | * Per livepatch.h, ftrace location is always within the first |
56 | * 16 bytes of a function on powerpc with -mprofile-kernel. |
57 | */ |
58 | faddr = ftrace_location_range((unsigned long)addr, |
59 | (unsigned long)addr + 16); |
60 | if (faddr) |
61 | addr = (kprobe_opcode_t *)faddr; |
62 | else |
63 | #endif |
64 | addr = (kprobe_opcode_t *)ppc_function_entry(addr); |
65 | } |
66 | #elif defined(CONFIG_PPC64_ELF_ABI_V1) |
67 | /* |
68 | * 64bit powerpc ABIv1 uses function descriptors: |
69 | * - Check for the dot variant of the symbol first. |
70 | * - If that fails, try looking up the symbol provided. |
71 | * |
72 | * This ensures we always get to the actual symbol and not |
73 | * the descriptor. |
74 | * |
75 | * Also handle <module:symbol> format. |
76 | */ |
77 | char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN]; |
78 | bool dot_appended = false; |
79 | const char *c; |
80 | ssize_t ret = 0; |
81 | int len = 0; |
82 | |
83 | if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) { |
84 | c++; |
85 | len = c - name; |
86 | memcpy(dot_name, name, len); |
87 | } else |
88 | c = name; |
89 | |
90 | if (*c != '\0' && *c != '.') { |
91 | dot_name[len++] = '.'; |
92 | dot_appended = true; |
93 | } |
94 | ret = strscpy(dot_name + len, c, KSYM_NAME_LEN); |
95 | if (ret > 0) |
96 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name); |
97 | |
98 | /* Fallback to the original non-dot symbol lookup */ |
99 | if (!addr && dot_appended) |
100 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); |
101 | #else |
102 | addr = (kprobe_opcode_t *)kallsyms_lookup_name(name); |
103 | #endif |
104 | |
105 | return addr; |
106 | } |
107 | |
108 | static bool arch_kprobe_on_func_entry(unsigned long offset) |
109 | { |
110 | #ifdef CONFIG_PPC64_ELF_ABI_V2 |
111 | #ifdef CONFIG_KPROBES_ON_FTRACE |
112 | return offset <= 16; |
113 | #else |
114 | return offset <= 8; |
115 | #endif |
116 | #else |
117 | return !offset; |
118 | #endif |
119 | } |
120 | |
121 | /* XXX try and fold the magic of kprobe_lookup_name() in this */ |
122 | kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset, |
123 | bool *on_func_entry) |
124 | { |
125 | *on_func_entry = arch_kprobe_on_func_entry(offset); |
126 | return (kprobe_opcode_t *)(addr + offset); |
127 | } |
128 | |
129 | void *alloc_insn_page(void) |
130 | { |
131 | void *page; |
132 | |
133 | page = module_alloc(PAGE_SIZE); |
134 | if (!page) |
135 | return NULL; |
136 | |
137 | if (strict_module_rwx_enabled()) { |
138 | int err = set_memory_rox(addr: (unsigned long)page, numpages: 1); |
139 | |
140 | if (err) |
141 | goto error; |
142 | } |
143 | return page; |
144 | error: |
145 | module_memfree(module_region: page); |
146 | return NULL; |
147 | } |
148 | |
149 | int arch_prepare_kprobe(struct kprobe *p) |
150 | { |
151 | int ret = 0; |
152 | struct kprobe *prev; |
153 | ppc_inst_t insn = ppc_inst_read(p->addr); |
154 | |
155 | if ((unsigned long)p->addr & 0x03) { |
156 | printk("Attempt to register kprobe at an unaligned address\n" ); |
157 | ret = -EINVAL; |
158 | } else if (!can_single_step(ppc_inst_val(insn))) { |
159 | printk("Cannot register a kprobe on instructions that can't be single stepped\n" ); |
160 | ret = -EINVAL; |
161 | } else if ((unsigned long)p->addr & ~PAGE_MASK && |
162 | ppc_inst_prefixed(ppc_inst_read(p->addr - 1))) { |
163 | printk("Cannot register a kprobe on the second word of prefixed instruction\n" ); |
164 | ret = -EINVAL; |
165 | } |
166 | prev = get_kprobe(addr: p->addr - 1); |
167 | |
168 | /* |
169 | * When prev is a ftrace-based kprobe, we don't have an insn, and it |
170 | * doesn't probe for prefixed instruction. |
171 | */ |
172 | if (prev && !kprobe_ftrace(p: prev) && |
173 | ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) { |
174 | printk("Cannot register a kprobe on the second word of prefixed instruction\n" ); |
175 | ret = -EINVAL; |
176 | } |
177 | |
178 | /* insn must be on a special executable page on ppc64. This is |
179 | * not explicitly required on ppc32 (right now), but it doesn't hurt */ |
180 | if (!ret) { |
181 | p->ainsn.insn = get_insn_slot(); |
182 | if (!p->ainsn.insn) |
183 | ret = -ENOMEM; |
184 | } |
185 | |
186 | if (!ret) { |
187 | patch_instruction(p->ainsn.insn, insn); |
188 | p->opcode = ppc_inst_val(insn); |
189 | } |
190 | |
191 | p->ainsn.boostable = 0; |
192 | return ret; |
193 | } |
194 | NOKPROBE_SYMBOL(arch_prepare_kprobe); |
195 | |
196 | void arch_arm_kprobe(struct kprobe *p) |
197 | { |
198 | WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(BREAKPOINT_INSTRUCTION))); |
199 | } |
200 | NOKPROBE_SYMBOL(arch_arm_kprobe); |
201 | |
202 | void arch_disarm_kprobe(struct kprobe *p) |
203 | { |
204 | WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(p->opcode))); |
205 | } |
206 | NOKPROBE_SYMBOL(arch_disarm_kprobe); |
207 | |
208 | void arch_remove_kprobe(struct kprobe *p) |
209 | { |
210 | if (p->ainsn.insn) { |
211 | free_insn_slot(slot: p->ainsn.insn, dirty: 0); |
212 | p->ainsn.insn = NULL; |
213 | } |
214 | } |
215 | NOKPROBE_SYMBOL(arch_remove_kprobe); |
216 | |
217 | static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs) |
218 | { |
219 | enable_single_step(regs); |
220 | |
221 | /* |
222 | * On powerpc we should single step on the original |
223 | * instruction even if the probed insn is a trap |
224 | * variant as values in regs could play a part in |
225 | * if the trap is taken or not |
226 | */ |
227 | regs_set_return_ip(regs, (unsigned long)p->ainsn.insn); |
228 | } |
229 | |
230 | static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb) |
231 | { |
232 | kcb->prev_kprobe.kp = kprobe_running(); |
233 | kcb->prev_kprobe.status = kcb->kprobe_status; |
234 | kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr; |
235 | } |
236 | |
237 | static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
238 | { |
239 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
240 | kcb->kprobe_status = kcb->prev_kprobe.status; |
241 | kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr; |
242 | } |
243 | |
244 | static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs, |
245 | struct kprobe_ctlblk *kcb) |
246 | { |
247 | __this_cpu_write(current_kprobe, p); |
248 | kcb->kprobe_saved_msr = regs->msr; |
249 | } |
250 | |
251 | void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) |
252 | { |
253 | ri->ret_addr = (kprobe_opcode_t *)regs->link; |
254 | ri->fp = NULL; |
255 | |
256 | /* Replace the return addr with trampoline addr */ |
257 | regs->link = (unsigned long)__kretprobe_trampoline; |
258 | } |
259 | NOKPROBE_SYMBOL(arch_prepare_kretprobe); |
260 | |
261 | static int try_to_emulate(struct kprobe *p, struct pt_regs *regs) |
262 | { |
263 | int ret; |
264 | ppc_inst_t insn = ppc_inst_read(p->ainsn.insn); |
265 | |
266 | /* regs->nip is also adjusted if emulate_step returns 1 */ |
267 | ret = emulate_step(regs, insn); |
268 | if (ret > 0) { |
269 | /* |
270 | * Once this instruction has been boosted |
271 | * successfully, set the boostable flag |
272 | */ |
273 | if (unlikely(p->ainsn.boostable == 0)) |
274 | p->ainsn.boostable = 1; |
275 | } else if (ret < 0) { |
276 | /* |
277 | * We don't allow kprobes on mtmsr(d)/rfi(d), etc. |
278 | * So, we should never get here... but, its still |
279 | * good to catch them, just in case... |
280 | */ |
281 | printk("Can't step on instruction %08lx\n" , ppc_inst_as_ulong(insn)); |
282 | BUG(); |
283 | } else { |
284 | /* |
285 | * If we haven't previously emulated this instruction, then it |
286 | * can't be boosted. Note it down so we don't try to do so again. |
287 | * |
288 | * If, however, we had emulated this instruction in the past, |
289 | * then this is just an error with the current run (for |
290 | * instance, exceptions due to a load/store). We return 0 so |
291 | * that this is now single-stepped, but continue to try |
292 | * emulating it in subsequent probe hits. |
293 | */ |
294 | if (unlikely(p->ainsn.boostable != 1)) |
295 | p->ainsn.boostable = -1; |
296 | } |
297 | |
298 | return ret; |
299 | } |
300 | NOKPROBE_SYMBOL(try_to_emulate); |
301 | |
302 | int kprobe_handler(struct pt_regs *regs) |
303 | { |
304 | struct kprobe *p; |
305 | int ret = 0; |
306 | unsigned int *addr = (unsigned int *)regs->nip; |
307 | struct kprobe_ctlblk *kcb; |
308 | |
309 | if (user_mode(regs)) |
310 | return 0; |
311 | |
312 | if (!IS_ENABLED(CONFIG_BOOKE) && |
313 | (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR))) |
314 | return 0; |
315 | |
316 | /* |
317 | * We don't want to be preempted for the entire |
318 | * duration of kprobe processing |
319 | */ |
320 | preempt_disable(); |
321 | kcb = get_kprobe_ctlblk(); |
322 | |
323 | p = get_kprobe(addr); |
324 | if (!p) { |
325 | unsigned int instr; |
326 | |
327 | if (get_kernel_nofault(instr, addr)) |
328 | goto no_kprobe; |
329 | |
330 | if (instr != BREAKPOINT_INSTRUCTION) { |
331 | /* |
332 | * PowerPC has multiple variants of the "trap" |
333 | * instruction. If the current instruction is a |
334 | * trap variant, it could belong to someone else |
335 | */ |
336 | if (is_trap(instr)) |
337 | goto no_kprobe; |
338 | /* |
339 | * The breakpoint instruction was removed right |
340 | * after we hit it. Another cpu has removed |
341 | * either a probepoint or a debugger breakpoint |
342 | * at this address. In either case, no further |
343 | * handling of this interrupt is appropriate. |
344 | */ |
345 | ret = 1; |
346 | } |
347 | /* Not one of ours: let kernel handle it */ |
348 | goto no_kprobe; |
349 | } |
350 | |
351 | /* Check we're not actually recursing */ |
352 | if (kprobe_running()) { |
353 | kprobe_opcode_t insn = *p->ainsn.insn; |
354 | if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) { |
355 | /* Turn off 'trace' bits */ |
356 | regs_set_return_msr(regs, |
357 | (regs->msr & ~MSR_SINGLESTEP) | |
358 | kcb->kprobe_saved_msr); |
359 | goto no_kprobe; |
360 | } |
361 | |
362 | /* |
363 | * We have reentered the kprobe_handler(), since another probe |
364 | * was hit while within the handler. We here save the original |
365 | * kprobes variables and just single step on the instruction of |
366 | * the new probe without calling any user handlers. |
367 | */ |
368 | save_previous_kprobe(kcb); |
369 | set_current_kprobe(p, regs, kcb); |
370 | kprobes_inc_nmissed_count(p); |
371 | kcb->kprobe_status = KPROBE_REENTER; |
372 | if (p->ainsn.boostable >= 0) { |
373 | ret = try_to_emulate(p, regs); |
374 | |
375 | if (ret > 0) { |
376 | restore_previous_kprobe(kcb); |
377 | preempt_enable(); |
378 | return 1; |
379 | } |
380 | } |
381 | prepare_singlestep(p, regs); |
382 | return 1; |
383 | } |
384 | |
385 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
386 | set_current_kprobe(p, regs, kcb); |
387 | if (p->pre_handler && p->pre_handler(p, regs)) { |
388 | /* handler changed execution path, so skip ss setup */ |
389 | reset_current_kprobe(); |
390 | preempt_enable(); |
391 | return 1; |
392 | } |
393 | |
394 | if (p->ainsn.boostable >= 0) { |
395 | ret = try_to_emulate(p, regs); |
396 | |
397 | if (ret > 0) { |
398 | if (p->post_handler) |
399 | p->post_handler(p, regs, 0); |
400 | |
401 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
402 | reset_current_kprobe(); |
403 | preempt_enable(); |
404 | return 1; |
405 | } |
406 | } |
407 | prepare_singlestep(p, regs); |
408 | kcb->kprobe_status = KPROBE_HIT_SS; |
409 | return 1; |
410 | |
411 | no_kprobe: |
412 | preempt_enable(); |
413 | return ret; |
414 | } |
415 | NOKPROBE_SYMBOL(kprobe_handler); |
416 | |
417 | /* |
418 | * Function return probe trampoline: |
419 | * - init_kprobes() establishes a probepoint here |
420 | * - When the probed function returns, this probe |
421 | * causes the handlers to fire |
422 | */ |
423 | asm(".global __kretprobe_trampoline\n" |
424 | ".type __kretprobe_trampoline, @function\n" |
425 | "__kretprobe_trampoline:\n" |
426 | "nop\n" |
427 | "blr\n" |
428 | ".size __kretprobe_trampoline, .-__kretprobe_trampoline\n" ); |
429 | |
430 | /* |
431 | * Called when the probe at kretprobe trampoline is hit |
432 | */ |
433 | static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) |
434 | { |
435 | unsigned long orig_ret_address; |
436 | |
437 | orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); |
438 | /* |
439 | * We get here through one of two paths: |
440 | * 1. by taking a trap -> kprobe_handler() -> here |
441 | * 2. by optprobe branch -> optimized_callback() -> opt_pre_handler() -> here |
442 | * |
443 | * When going back through (1), we need regs->nip to be setup properly |
444 | * as it is used to determine the return address from the trap. |
445 | * For (2), since nip is not honoured with optprobes, we instead setup |
446 | * the link register properly so that the subsequent 'blr' in |
447 | * __kretprobe_trampoline jumps back to the right instruction. |
448 | * |
449 | * For nip, we should set the address to the previous instruction since |
450 | * we end up emulating it in kprobe_handler(), which increments the nip |
451 | * again. |
452 | */ |
453 | regs_set_return_ip(regs, orig_ret_address - 4); |
454 | regs->link = orig_ret_address; |
455 | |
456 | return 0; |
457 | } |
458 | NOKPROBE_SYMBOL(trampoline_probe_handler); |
459 | |
460 | /* |
461 | * Called after single-stepping. p->addr is the address of the |
462 | * instruction whose first byte has been replaced by the "breakpoint" |
463 | * instruction. To avoid the SMP problems that can occur when we |
464 | * temporarily put back the original opcode to single-step, we |
465 | * single-stepped a copy of the instruction. The address of this |
466 | * copy is p->ainsn.insn. |
467 | */ |
468 | int kprobe_post_handler(struct pt_regs *regs) |
469 | { |
470 | int len; |
471 | struct kprobe *cur = kprobe_running(); |
472 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
473 | |
474 | if (!cur || user_mode(regs)) |
475 | return 0; |
476 | |
477 | len = ppc_inst_len(ppc_inst_read(cur->ainsn.insn)); |
478 | /* make sure we got here for instruction we have a kprobe on */ |
479 | if (((unsigned long)cur->ainsn.insn + len) != regs->nip) |
480 | return 0; |
481 | |
482 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
483 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
484 | cur->post_handler(cur, regs, 0); |
485 | } |
486 | |
487 | /* Adjust nip to after the single-stepped instruction */ |
488 | regs_set_return_ip(regs, (unsigned long)cur->addr + len); |
489 | regs_set_return_msr(regs, regs->msr | kcb->kprobe_saved_msr); |
490 | |
491 | /*Restore back the original saved kprobes variables and continue. */ |
492 | if (kcb->kprobe_status == KPROBE_REENTER) { |
493 | restore_previous_kprobe(kcb); |
494 | goto out; |
495 | } |
496 | reset_current_kprobe(); |
497 | out: |
498 | preempt_enable(); |
499 | |
500 | /* |
501 | * if somebody else is singlestepping across a probe point, msr |
502 | * will have DE/SE set, in which case, continue the remaining processing |
503 | * of do_debug, as if this is not a probe hit. |
504 | */ |
505 | if (regs->msr & MSR_SINGLESTEP) |
506 | return 0; |
507 | |
508 | return 1; |
509 | } |
510 | NOKPROBE_SYMBOL(kprobe_post_handler); |
511 | |
512 | int kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
513 | { |
514 | struct kprobe *cur = kprobe_running(); |
515 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
516 | const struct exception_table_entry *entry; |
517 | |
518 | switch(kcb->kprobe_status) { |
519 | case KPROBE_HIT_SS: |
520 | case KPROBE_REENTER: |
521 | /* |
522 | * We are here because the instruction being single |
523 | * stepped caused a page fault. We reset the current |
524 | * kprobe and the nip points back to the probe address |
525 | * and allow the page fault handler to continue as a |
526 | * normal page fault. |
527 | */ |
528 | regs_set_return_ip(regs, (unsigned long)cur->addr); |
529 | /* Turn off 'trace' bits */ |
530 | regs_set_return_msr(regs, |
531 | (regs->msr & ~MSR_SINGLESTEP) | |
532 | kcb->kprobe_saved_msr); |
533 | if (kcb->kprobe_status == KPROBE_REENTER) |
534 | restore_previous_kprobe(kcb); |
535 | else |
536 | reset_current_kprobe(); |
537 | preempt_enable(); |
538 | break; |
539 | case KPROBE_HIT_ACTIVE: |
540 | case KPROBE_HIT_SSDONE: |
541 | /* |
542 | * In case the user-specified fault handler returned |
543 | * zero, try to fix up. |
544 | */ |
545 | if ((entry = search_exception_tables(add: regs->nip)) != NULL) { |
546 | regs_set_return_ip(regs, extable_fixup(entry)); |
547 | return 1; |
548 | } |
549 | |
550 | /* |
551 | * fixup_exception() could not handle it, |
552 | * Let do_page_fault() fix it. |
553 | */ |
554 | break; |
555 | default: |
556 | break; |
557 | } |
558 | return 0; |
559 | } |
560 | NOKPROBE_SYMBOL(kprobe_fault_handler); |
561 | |
562 | static struct kprobe trampoline_p = { |
563 | .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, |
564 | .pre_handler = trampoline_probe_handler |
565 | }; |
566 | |
567 | int __init arch_init_kprobes(void) |
568 | { |
569 | return register_kprobe(p: &trampoline_p); |
570 | } |
571 | |
572 | int arch_trampoline_kprobe(struct kprobe *p) |
573 | { |
574 | if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) |
575 | return 1; |
576 | |
577 | return 0; |
578 | } |
579 | NOKPROBE_SYMBOL(arch_trampoline_kprobe); |
580 | |