1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* |
3 | * Kernel Probes (KProbes) |
4 | * |
5 | * Copyright IBM Corp. 2002, 2006 |
6 | * |
7 | * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com> |
8 | */ |
9 | |
10 | #define pr_fmt(fmt) "kprobes: " fmt |
11 | |
12 | #include <linux/moduleloader.h> |
13 | #include <linux/kprobes.h> |
14 | #include <linux/ptrace.h> |
15 | #include <linux/preempt.h> |
16 | #include <linux/stop_machine.h> |
17 | #include <linux/kdebug.h> |
18 | #include <linux/uaccess.h> |
19 | #include <linux/extable.h> |
20 | #include <linux/module.h> |
21 | #include <linux/slab.h> |
22 | #include <linux/hardirq.h> |
23 | #include <linux/ftrace.h> |
24 | #include <asm/set_memory.h> |
25 | #include <asm/sections.h> |
26 | #include <asm/dis.h> |
27 | #include "kprobes.h" |
28 | #include "entry.h" |
29 | |
30 | DEFINE_PER_CPU(struct kprobe *, current_kprobe); |
31 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
32 | |
33 | struct kretprobe_blackpoint kretprobe_blacklist[] = { }; |
34 | |
35 | static int insn_page_in_use; |
36 | |
37 | void *alloc_insn_page(void) |
38 | { |
39 | void *page; |
40 | |
41 | page = module_alloc(PAGE_SIZE); |
42 | if (!page) |
43 | return NULL; |
44 | set_memory_rox(addr: (unsigned long)page, numpages: 1); |
45 | return page; |
46 | } |
47 | |
48 | static void *alloc_s390_insn_page(void) |
49 | { |
50 | if (xchg(&insn_page_in_use, 1) == 1) |
51 | return NULL; |
52 | return &kprobes_insn_page; |
53 | } |
54 | |
55 | static void free_s390_insn_page(void *page) |
56 | { |
57 | xchg(&insn_page_in_use, 0); |
58 | } |
59 | |
60 | struct kprobe_insn_cache kprobe_s390_insn_slots = { |
61 | .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex), |
62 | .alloc = alloc_s390_insn_page, |
63 | .free = free_s390_insn_page, |
64 | .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages), |
65 | .insn_size = MAX_INSN_SIZE, |
66 | }; |
67 | |
68 | static void copy_instruction(struct kprobe *p) |
69 | { |
70 | kprobe_opcode_t insn[MAX_INSN_SIZE]; |
71 | s64 disp, new_disp; |
72 | u64 addr, new_addr; |
73 | unsigned int len; |
74 | |
75 | len = insn_length(*p->addr >> 8); |
76 | memcpy(&insn, p->addr, len); |
77 | p->opcode = insn[0]; |
78 | if (probe_is_insn_relative_long(&insn[0])) { |
79 | /* |
80 | * For pc-relative instructions in RIL-b or RIL-c format patch |
81 | * the RI2 displacement field. We have already made sure that |
82 | * the insn slot for the patched instruction is within the same |
83 | * 2GB area as the original instruction (either kernel image or |
84 | * module area). Therefore the new displacement will always fit. |
85 | */ |
86 | disp = *(s32 *)&insn[1]; |
87 | addr = (u64)(unsigned long)p->addr; |
88 | new_addr = (u64)(unsigned long)p->ainsn.insn; |
89 | new_disp = ((addr + (disp * 2)) - new_addr) / 2; |
90 | *(s32 *)&insn[1] = new_disp; |
91 | } |
92 | s390_kernel_write(p->ainsn.insn, &insn, len); |
93 | } |
94 | NOKPROBE_SYMBOL(copy_instruction); |
95 | |
96 | static int s390_get_insn_slot(struct kprobe *p) |
97 | { |
98 | /* |
99 | * Get an insn slot that is within the same 2GB area like the original |
100 | * instruction. That way instructions with a 32bit signed displacement |
101 | * field can be patched and executed within the insn slot. |
102 | */ |
103 | p->ainsn.insn = NULL; |
104 | if (is_kernel(addr: (unsigned long)p->addr)) |
105 | p->ainsn.insn = get_s390_insn_slot(); |
106 | else if (is_module_addr(p->addr)) |
107 | p->ainsn.insn = get_insn_slot(); |
108 | return p->ainsn.insn ? 0 : -ENOMEM; |
109 | } |
110 | NOKPROBE_SYMBOL(s390_get_insn_slot); |
111 | |
112 | static void s390_free_insn_slot(struct kprobe *p) |
113 | { |
114 | if (!p->ainsn.insn) |
115 | return; |
116 | if (is_kernel(addr: (unsigned long)p->addr)) |
117 | free_s390_insn_slot(slot: p->ainsn.insn, dirty: 0); |
118 | else |
119 | free_insn_slot(slot: p->ainsn.insn, dirty: 0); |
120 | p->ainsn.insn = NULL; |
121 | } |
122 | NOKPROBE_SYMBOL(s390_free_insn_slot); |
123 | |
124 | /* Check if paddr is at an instruction boundary */ |
125 | static bool can_probe(unsigned long paddr) |
126 | { |
127 | unsigned long addr, offset = 0; |
128 | kprobe_opcode_t insn; |
129 | struct kprobe *kp; |
130 | |
131 | if (paddr & 0x01) |
132 | return false; |
133 | |
134 | if (!kallsyms_lookup_size_offset(addr: paddr, NULL, offset: &offset)) |
135 | return false; |
136 | |
137 | /* Decode instructions */ |
138 | addr = paddr - offset; |
139 | while (addr < paddr) { |
140 | if (copy_from_kernel_nofault(dst: &insn, src: (void *)addr, size: sizeof(insn))) |
141 | return false; |
142 | |
143 | if (insn >> 8 == 0) { |
144 | if (insn != BREAKPOINT_INSTRUCTION) { |
145 | /* |
146 | * Note that QEMU inserts opcode 0x0000 to implement |
147 | * software breakpoints for guests. Since the size of |
148 | * the original instruction is unknown, stop following |
149 | * instructions and prevent setting a kprobe. |
150 | */ |
151 | return false; |
152 | } |
153 | /* |
154 | * Check if the instruction has been modified by another |
155 | * kprobe, in which case the original instruction is |
156 | * decoded. |
157 | */ |
158 | kp = get_kprobe(addr: (void *)addr); |
159 | if (!kp) { |
160 | /* not a kprobe */ |
161 | return false; |
162 | } |
163 | insn = kp->opcode; |
164 | } |
165 | addr += insn_length(insn >> 8); |
166 | } |
167 | return addr == paddr; |
168 | } |
169 | |
170 | int arch_prepare_kprobe(struct kprobe *p) |
171 | { |
172 | if (!can_probe(paddr: (unsigned long)p->addr)) |
173 | return -EINVAL; |
174 | /* Make sure the probe isn't going on a difficult instruction */ |
175 | if (probe_is_prohibited_opcode(p->addr)) |
176 | return -EINVAL; |
177 | if (s390_get_insn_slot(p)) |
178 | return -ENOMEM; |
179 | copy_instruction(p); |
180 | return 0; |
181 | } |
182 | NOKPROBE_SYMBOL(arch_prepare_kprobe); |
183 | |
184 | struct swap_insn_args { |
185 | struct kprobe *p; |
186 | unsigned int arm_kprobe : 1; |
187 | }; |
188 | |
189 | static int swap_instruction(void *data) |
190 | { |
191 | struct swap_insn_args *args = data; |
192 | struct kprobe *p = args->p; |
193 | u16 opc; |
194 | |
195 | opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode; |
196 | s390_kernel_write(p->addr, &opc, sizeof(opc)); |
197 | return 0; |
198 | } |
199 | NOKPROBE_SYMBOL(swap_instruction); |
200 | |
201 | void arch_arm_kprobe(struct kprobe *p) |
202 | { |
203 | struct swap_insn_args args = {.p = p, .arm_kprobe = 1}; |
204 | |
205 | stop_machine_cpuslocked(fn: swap_instruction, data: &args, NULL); |
206 | } |
207 | NOKPROBE_SYMBOL(arch_arm_kprobe); |
208 | |
209 | void arch_disarm_kprobe(struct kprobe *p) |
210 | { |
211 | struct swap_insn_args args = {.p = p, .arm_kprobe = 0}; |
212 | |
213 | stop_machine_cpuslocked(fn: swap_instruction, data: &args, NULL); |
214 | } |
215 | NOKPROBE_SYMBOL(arch_disarm_kprobe); |
216 | |
217 | void arch_remove_kprobe(struct kprobe *p) |
218 | { |
219 | s390_free_insn_slot(p); |
220 | } |
221 | NOKPROBE_SYMBOL(arch_remove_kprobe); |
222 | |
223 | static void enable_singlestep(struct kprobe_ctlblk *kcb, |
224 | struct pt_regs *regs, |
225 | unsigned long ip) |
226 | { |
227 | union { |
228 | struct ctlreg regs[3]; |
229 | struct { |
230 | struct ctlreg control; |
231 | struct ctlreg start; |
232 | struct ctlreg end; |
233 | }; |
234 | } per_kprobe; |
235 | |
236 | /* Set up the PER control registers %cr9-%cr11 */ |
237 | per_kprobe.control.val = PER_EVENT_IFETCH; |
238 | per_kprobe.start.val = ip; |
239 | per_kprobe.end.val = ip; |
240 | |
241 | /* Save control regs and psw mask */ |
242 | __local_ctl_store(9, 11, kcb->kprobe_saved_ctl); |
243 | kcb->kprobe_saved_imask = regs->psw.mask & |
244 | (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT); |
245 | |
246 | /* Set PER control regs, turns on single step for the given address */ |
247 | __local_ctl_load(9, 11, per_kprobe.regs); |
248 | regs->psw.mask |= PSW_MASK_PER; |
249 | regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT); |
250 | regs->psw.addr = ip; |
251 | } |
252 | NOKPROBE_SYMBOL(enable_singlestep); |
253 | |
254 | static void disable_singlestep(struct kprobe_ctlblk *kcb, |
255 | struct pt_regs *regs, |
256 | unsigned long ip) |
257 | { |
258 | /* Restore control regs and psw mask, set new psw address */ |
259 | __local_ctl_load(9, 11, kcb->kprobe_saved_ctl); |
260 | regs->psw.mask &= ~PSW_MASK_PER; |
261 | regs->psw.mask |= kcb->kprobe_saved_imask; |
262 | regs->psw.addr = ip; |
263 | } |
264 | NOKPROBE_SYMBOL(disable_singlestep); |
265 | |
266 | /* |
267 | * Activate a kprobe by storing its pointer to current_kprobe. The |
268 | * previous kprobe is stored in kcb->prev_kprobe. A stack of up to |
269 | * two kprobes can be active, see KPROBE_REENTER. |
270 | */ |
271 | static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p) |
272 | { |
273 | kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe); |
274 | kcb->prev_kprobe.status = kcb->kprobe_status; |
275 | __this_cpu_write(current_kprobe, p); |
276 | } |
277 | NOKPROBE_SYMBOL(push_kprobe); |
278 | |
279 | /* |
280 | * Deactivate a kprobe by backing up to the previous state. If the |
281 | * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL, |
282 | * for any other state prev_kprobe.kp will be NULL. |
283 | */ |
284 | static void pop_kprobe(struct kprobe_ctlblk *kcb) |
285 | { |
286 | __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); |
287 | kcb->kprobe_status = kcb->prev_kprobe.status; |
288 | kcb->prev_kprobe.kp = NULL; |
289 | } |
290 | NOKPROBE_SYMBOL(pop_kprobe); |
291 | |
292 | static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p) |
293 | { |
294 | switch (kcb->kprobe_status) { |
295 | case KPROBE_HIT_SSDONE: |
296 | case KPROBE_HIT_ACTIVE: |
297 | kprobes_inc_nmissed_count(p); |
298 | break; |
299 | case KPROBE_HIT_SS: |
300 | case KPROBE_REENTER: |
301 | default: |
302 | /* |
303 | * A kprobe on the code path to single step an instruction |
304 | * is a BUG. The code path resides in the .kprobes.text |
305 | * section and is executed with interrupts disabled. |
306 | */ |
307 | pr_err("Failed to recover from reentered kprobes.\n" ); |
308 | dump_kprobe(kp: p); |
309 | BUG(); |
310 | } |
311 | } |
312 | NOKPROBE_SYMBOL(kprobe_reenter_check); |
313 | |
314 | static int kprobe_handler(struct pt_regs *regs) |
315 | { |
316 | struct kprobe_ctlblk *kcb; |
317 | struct kprobe *p; |
318 | |
319 | /* |
320 | * We want to disable preemption for the entire duration of kprobe |
321 | * processing. That includes the calls to the pre/post handlers |
322 | * and single stepping the kprobe instruction. |
323 | */ |
324 | preempt_disable(); |
325 | kcb = get_kprobe_ctlblk(); |
326 | p = get_kprobe(addr: (void *)(regs->psw.addr - 2)); |
327 | |
328 | if (p) { |
329 | if (kprobe_running()) { |
330 | /* |
331 | * We have hit a kprobe while another is still |
332 | * active. This can happen in the pre and post |
333 | * handler. Single step the instruction of the |
334 | * new probe but do not call any handler function |
335 | * of this secondary kprobe. |
336 | * push_kprobe and pop_kprobe saves and restores |
337 | * the currently active kprobe. |
338 | */ |
339 | kprobe_reenter_check(kcb, p); |
340 | push_kprobe(kcb, p); |
341 | kcb->kprobe_status = KPROBE_REENTER; |
342 | } else { |
343 | /* |
344 | * If we have no pre-handler or it returned 0, we |
345 | * continue with single stepping. If we have a |
346 | * pre-handler and it returned non-zero, it prepped |
347 | * for changing execution path, so get out doing |
348 | * nothing more here. |
349 | */ |
350 | push_kprobe(kcb, p); |
351 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
352 | if (p->pre_handler && p->pre_handler(p, regs)) { |
353 | pop_kprobe(kcb); |
354 | preempt_enable_no_resched(); |
355 | return 1; |
356 | } |
357 | kcb->kprobe_status = KPROBE_HIT_SS; |
358 | } |
359 | enable_singlestep(kcb, regs, ip: (unsigned long) p->ainsn.insn); |
360 | return 1; |
361 | } /* else: |
362 | * No kprobe at this address and no active kprobe. The trap has |
363 | * not been caused by a kprobe breakpoint. The race of breakpoint |
364 | * vs. kprobe remove does not exist because on s390 as we use |
365 | * stop_machine to arm/disarm the breakpoints. |
366 | */ |
367 | preempt_enable_no_resched(); |
368 | return 0; |
369 | } |
370 | NOKPROBE_SYMBOL(kprobe_handler); |
371 | |
372 | /* |
373 | * Called after single-stepping. p->addr is the address of the |
374 | * instruction whose first byte has been replaced by the "breakpoint" |
375 | * instruction. To avoid the SMP problems that can occur when we |
376 | * temporarily put back the original opcode to single-step, we |
377 | * single-stepped a copy of the instruction. The address of this |
378 | * copy is p->ainsn.insn. |
379 | */ |
380 | static void resume_execution(struct kprobe *p, struct pt_regs *regs) |
381 | { |
382 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
383 | unsigned long ip = regs->psw.addr; |
384 | int fixup = probe_get_fixup_type(p->ainsn.insn); |
385 | |
386 | if (fixup & FIXUP_PSW_NORMAL) |
387 | ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn; |
388 | |
389 | if (fixup & FIXUP_BRANCH_NOT_TAKEN) { |
390 | int ilen = insn_length(p->ainsn.insn[0] >> 8); |
391 | if (ip - (unsigned long) p->ainsn.insn == ilen) |
392 | ip = (unsigned long) p->addr + ilen; |
393 | } |
394 | |
395 | if (fixup & FIXUP_RETURN_REGISTER) { |
396 | int reg = (p->ainsn.insn[0] & 0xf0) >> 4; |
397 | regs->gprs[reg] += (unsigned long) p->addr - |
398 | (unsigned long) p->ainsn.insn; |
399 | } |
400 | |
401 | disable_singlestep(kcb, regs, ip); |
402 | } |
403 | NOKPROBE_SYMBOL(resume_execution); |
404 | |
405 | static int post_kprobe_handler(struct pt_regs *regs) |
406 | { |
407 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
408 | struct kprobe *p = kprobe_running(); |
409 | |
410 | if (!p) |
411 | return 0; |
412 | |
413 | resume_execution(p, regs); |
414 | if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) { |
415 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
416 | p->post_handler(p, regs, 0); |
417 | } |
418 | pop_kprobe(kcb); |
419 | preempt_enable_no_resched(); |
420 | |
421 | /* |
422 | * if somebody else is singlestepping across a probe point, psw mask |
423 | * will have PER set, in which case, continue the remaining processing |
424 | * of do_single_step, as if this is not a probe hit. |
425 | */ |
426 | if (regs->psw.mask & PSW_MASK_PER) |
427 | return 0; |
428 | |
429 | return 1; |
430 | } |
431 | NOKPROBE_SYMBOL(post_kprobe_handler); |
432 | |
433 | static int kprobe_trap_handler(struct pt_regs *regs, int trapnr) |
434 | { |
435 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
436 | struct kprobe *p = kprobe_running(); |
437 | |
438 | switch(kcb->kprobe_status) { |
439 | case KPROBE_HIT_SS: |
440 | case KPROBE_REENTER: |
441 | /* |
442 | * We are here because the instruction being single |
443 | * stepped caused a page fault. We reset the current |
444 | * kprobe and the nip points back to the probe address |
445 | * and allow the page fault handler to continue as a |
446 | * normal page fault. |
447 | */ |
448 | disable_singlestep(kcb, regs, ip: (unsigned long) p->addr); |
449 | pop_kprobe(kcb); |
450 | preempt_enable_no_resched(); |
451 | break; |
452 | case KPROBE_HIT_ACTIVE: |
453 | case KPROBE_HIT_SSDONE: |
454 | /* |
455 | * In case the user-specified fault handler returned |
456 | * zero, try to fix up. |
457 | */ |
458 | if (fixup_exception(regs)) |
459 | return 1; |
460 | /* |
461 | * fixup_exception() could not handle it, |
462 | * Let do_page_fault() fix it. |
463 | */ |
464 | break; |
465 | default: |
466 | break; |
467 | } |
468 | return 0; |
469 | } |
470 | NOKPROBE_SYMBOL(kprobe_trap_handler); |
471 | |
472 | int kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
473 | { |
474 | int ret; |
475 | |
476 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) |
477 | local_irq_disable(); |
478 | ret = kprobe_trap_handler(regs, trapnr); |
479 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) |
480 | local_irq_restore(regs->psw.mask & ~PSW_MASK_PER); |
481 | return ret; |
482 | } |
483 | NOKPROBE_SYMBOL(kprobe_fault_handler); |
484 | |
485 | /* |
486 | * Wrapper routine to for handling exceptions. |
487 | */ |
488 | int kprobe_exceptions_notify(struct notifier_block *self, |
489 | unsigned long val, void *data) |
490 | { |
491 | struct die_args *args = (struct die_args *) data; |
492 | struct pt_regs *regs = args->regs; |
493 | int ret = NOTIFY_DONE; |
494 | |
495 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) |
496 | local_irq_disable(); |
497 | |
498 | switch (val) { |
499 | case DIE_BPT: |
500 | if (kprobe_handler(regs)) |
501 | ret = NOTIFY_STOP; |
502 | break; |
503 | case DIE_SSTEP: |
504 | if (post_kprobe_handler(regs)) |
505 | ret = NOTIFY_STOP; |
506 | break; |
507 | case DIE_TRAP: |
508 | if (!preemptible() && kprobe_running() && |
509 | kprobe_trap_handler(regs, trapnr: args->trapnr)) |
510 | ret = NOTIFY_STOP; |
511 | break; |
512 | default: |
513 | break; |
514 | } |
515 | |
516 | if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT)) |
517 | local_irq_restore(regs->psw.mask & ~PSW_MASK_PER); |
518 | |
519 | return ret; |
520 | } |
521 | NOKPROBE_SYMBOL(kprobe_exceptions_notify); |
522 | |
523 | int __init arch_init_kprobes(void) |
524 | { |
525 | return 0; |
526 | } |
527 | |
528 | int arch_trampoline_kprobe(struct kprobe *p) |
529 | { |
530 | return 0; |
531 | } |
532 | NOKPROBE_SYMBOL(arch_trampoline_kprobe); |
533 | |