1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
4 * using the CPU's debug registers.
5 *
6 * Copyright (C) 2012 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
8 */
9
10#define pr_fmt(fmt) "hw-breakpoint: " fmt
11
12#include <linux/compat.h>
13#include <linux/cpu_pm.h>
14#include <linux/errno.h>
15#include <linux/hw_breakpoint.h>
16#include <linux/kprobes.h>
17#include <linux/perf_event.h>
18#include <linux/ptrace.h>
19#include <linux/smp.h>
20#include <linux/uaccess.h>
21
22#include <asm/current.h>
23#include <asm/debug-monitors.h>
24#include <asm/esr.h>
25#include <asm/hw_breakpoint.h>
26#include <asm/traps.h>
27#include <asm/cputype.h>
28#include <asm/system_misc.h>
29
30/* Breakpoint currently in use for each BRP. */
31static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
32
33/* Watchpoint currently in use for each WRP. */
34static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
35
36/* Currently stepping a per-CPU kernel breakpoint. */
37static DEFINE_PER_CPU(int, stepping_kernel_bp);
38
39/* Number of BRP/WRP registers on this CPU. */
40static int core_num_brps;
41static int core_num_wrps;
42
43int hw_breakpoint_slots(int type)
44{
45 /*
46 * We can be called early, so don't rely on
47 * our static variables being initialised.
48 */
49 switch (type) {
50 case TYPE_INST:
51 return get_num_brps();
52 case TYPE_DATA:
53 return get_num_wrps();
54 default:
55 pr_warn("unknown slot type: %d\n", type);
56 return 0;
57 }
58}
59
60#define READ_WB_REG_CASE(OFF, N, REG, VAL) \
61 case (OFF + N): \
62 AARCH64_DBG_READ(N, REG, VAL); \
63 break
64
65#define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \
66 case (OFF + N): \
67 AARCH64_DBG_WRITE(N, REG, VAL); \
68 break
69
70#define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \
71 READ_WB_REG_CASE(OFF, 0, REG, VAL); \
72 READ_WB_REG_CASE(OFF, 1, REG, VAL); \
73 READ_WB_REG_CASE(OFF, 2, REG, VAL); \
74 READ_WB_REG_CASE(OFF, 3, REG, VAL); \
75 READ_WB_REG_CASE(OFF, 4, REG, VAL); \
76 READ_WB_REG_CASE(OFF, 5, REG, VAL); \
77 READ_WB_REG_CASE(OFF, 6, REG, VAL); \
78 READ_WB_REG_CASE(OFF, 7, REG, VAL); \
79 READ_WB_REG_CASE(OFF, 8, REG, VAL); \
80 READ_WB_REG_CASE(OFF, 9, REG, VAL); \
81 READ_WB_REG_CASE(OFF, 10, REG, VAL); \
82 READ_WB_REG_CASE(OFF, 11, REG, VAL); \
83 READ_WB_REG_CASE(OFF, 12, REG, VAL); \
84 READ_WB_REG_CASE(OFF, 13, REG, VAL); \
85 READ_WB_REG_CASE(OFF, 14, REG, VAL); \
86 READ_WB_REG_CASE(OFF, 15, REG, VAL)
87
88#define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \
89 WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \
90 WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \
91 WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \
92 WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \
93 WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \
94 WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \
95 WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \
96 WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \
97 WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \
98 WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \
99 WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \
100 WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \
101 WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \
102 WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \
103 WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \
104 WRITE_WB_REG_CASE(OFF, 15, REG, VAL)
105
106static u64 read_wb_reg(int reg, int n)
107{
108 u64 val = 0;
109
110 switch (reg + n) {
111 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
112 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
113 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
114 GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
115 default:
116 pr_warn("attempt to read from unknown breakpoint register %d\n", n);
117 }
118
119 return val;
120}
121NOKPROBE_SYMBOL(read_wb_reg);
122
123static void write_wb_reg(int reg, int n, u64 val)
124{
125 switch (reg + n) {
126 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val);
127 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val);
128 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val);
129 GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val);
130 default:
131 pr_warn("attempt to write to unknown breakpoint register %d\n", n);
132 }
133 isb();
134}
135NOKPROBE_SYMBOL(write_wb_reg);
136
137/*
138 * Convert a breakpoint privilege level to the corresponding exception
139 * level.
140 */
141static enum dbg_active_el debug_exception_level(int privilege)
142{
143 switch (privilege) {
144 case AARCH64_BREAKPOINT_EL0:
145 return DBG_ACTIVE_EL0;
146 case AARCH64_BREAKPOINT_EL1:
147 return DBG_ACTIVE_EL1;
148 default:
149 pr_warn("invalid breakpoint privilege level %d\n", privilege);
150 return -EINVAL;
151 }
152}
153NOKPROBE_SYMBOL(debug_exception_level);
154
155enum hw_breakpoint_ops {
156 HW_BREAKPOINT_INSTALL,
157 HW_BREAKPOINT_UNINSTALL,
158 HW_BREAKPOINT_RESTORE
159};
160
161static int is_compat_bp(struct perf_event *bp)
162{
163 struct task_struct *tsk = bp->hw.target;
164
165 /*
166 * tsk can be NULL for per-cpu (non-ptrace) breakpoints.
167 * In this case, use the native interface, since we don't have
168 * the notion of a "compat CPU" and could end up relying on
169 * deprecated behaviour if we use unaligned watchpoints in
170 * AArch64 state.
171 */
172 return tsk && is_compat_thread(task_thread_info(tsk));
173}
174
175/**
176 * hw_breakpoint_slot_setup - Find and setup a perf slot according to
177 * operations
178 *
179 * @slots: pointer to array of slots
180 * @max_slots: max number of slots
181 * @bp: perf_event to setup
182 * @ops: operation to be carried out on the slot
183 *
184 * Return:
185 * slot index on success
186 * -ENOSPC if no slot is available/matches
187 * -EINVAL on wrong operations parameter
188 */
189static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
190 struct perf_event *bp,
191 enum hw_breakpoint_ops ops)
192{
193 int i;
194 struct perf_event **slot;
195
196 for (i = 0; i < max_slots; ++i) {
197 slot = &slots[i];
198 switch (ops) {
199 case HW_BREAKPOINT_INSTALL:
200 if (!*slot) {
201 *slot = bp;
202 return i;
203 }
204 break;
205 case HW_BREAKPOINT_UNINSTALL:
206 if (*slot == bp) {
207 *slot = NULL;
208 return i;
209 }
210 break;
211 case HW_BREAKPOINT_RESTORE:
212 if (*slot == bp)
213 return i;
214 break;
215 default:
216 pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
217 return -EINVAL;
218 }
219 }
220 return -ENOSPC;
221}
222
223static int hw_breakpoint_control(struct perf_event *bp,
224 enum hw_breakpoint_ops ops)
225{
226 struct arch_hw_breakpoint *info = counter_arch_bp(bp);
227 struct perf_event **slots;
228 struct debug_info *debug_info = &current->thread.debug;
229 int i, max_slots, ctrl_reg, val_reg, reg_enable;
230 enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege);
231 u32 ctrl;
232
233 if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
234 /* Breakpoint */
235 ctrl_reg = AARCH64_DBG_REG_BCR;
236 val_reg = AARCH64_DBG_REG_BVR;
237 slots = this_cpu_ptr(bp_on_reg);
238 max_slots = core_num_brps;
239 reg_enable = !debug_info->bps_disabled;
240 } else {
241 /* Watchpoint */
242 ctrl_reg = AARCH64_DBG_REG_WCR;
243 val_reg = AARCH64_DBG_REG_WVR;
244 slots = this_cpu_ptr(wp_on_reg);
245 max_slots = core_num_wrps;
246 reg_enable = !debug_info->wps_disabled;
247 }
248
249 i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops);
250
251 if (WARN_ONCE(i < 0, "Can't find any breakpoint slot"))
252 return i;
253
254 switch (ops) {
255 case HW_BREAKPOINT_INSTALL:
256 /*
257 * Ensure debug monitors are enabled at the correct exception
258 * level.
259 */
260 enable_debug_monitors(dbg_el);
261 fallthrough;
262 case HW_BREAKPOINT_RESTORE:
263 /* Setup the address register. */
264 write_wb_reg(reg: val_reg, n: i, val: info->address);
265
266 /* Setup the control register. */
267 ctrl = encode_ctrl_reg(info->ctrl);
268 write_wb_reg(reg: ctrl_reg, n: i,
269 val: reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
270 break;
271 case HW_BREAKPOINT_UNINSTALL:
272 /* Reset the control register. */
273 write_wb_reg(reg: ctrl_reg, n: i, val: 0);
274
275 /*
276 * Release the debug monitors for the correct exception
277 * level.
278 */
279 disable_debug_monitors(dbg_el);
280 break;
281 }
282
283 return 0;
284}
285
286/*
287 * Install a perf counter breakpoint.
288 */
289int arch_install_hw_breakpoint(struct perf_event *bp)
290{
291 return hw_breakpoint_control(bp, ops: HW_BREAKPOINT_INSTALL);
292}
293
294void arch_uninstall_hw_breakpoint(struct perf_event *bp)
295{
296 hw_breakpoint_control(bp, ops: HW_BREAKPOINT_UNINSTALL);
297}
298
299static int get_hbp_len(u8 hbp_len)
300{
301 unsigned int len_in_bytes = 0;
302
303 switch (hbp_len) {
304 case ARM_BREAKPOINT_LEN_1:
305 len_in_bytes = 1;
306 break;
307 case ARM_BREAKPOINT_LEN_2:
308 len_in_bytes = 2;
309 break;
310 case ARM_BREAKPOINT_LEN_3:
311 len_in_bytes = 3;
312 break;
313 case ARM_BREAKPOINT_LEN_4:
314 len_in_bytes = 4;
315 break;
316 case ARM_BREAKPOINT_LEN_5:
317 len_in_bytes = 5;
318 break;
319 case ARM_BREAKPOINT_LEN_6:
320 len_in_bytes = 6;
321 break;
322 case ARM_BREAKPOINT_LEN_7:
323 len_in_bytes = 7;
324 break;
325 case ARM_BREAKPOINT_LEN_8:
326 len_in_bytes = 8;
327 break;
328 }
329
330 return len_in_bytes;
331}
332
333/*
334 * Check whether bp virtual address is in kernel space.
335 */
336int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
337{
338 unsigned int len;
339 unsigned long va;
340
341 va = hw->address;
342 len = get_hbp_len(hbp_len: hw->ctrl.len);
343
344 return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
345}
346
347/*
348 * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
349 * Hopefully this will disappear when ptrace can bypass the conversion
350 * to generic breakpoint descriptions.
351 */
352int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
353 int *gen_len, int *gen_type, int *offset)
354{
355 /* Type */
356 switch (ctrl.type) {
357 case ARM_BREAKPOINT_EXECUTE:
358 *gen_type = HW_BREAKPOINT_X;
359 break;
360 case ARM_BREAKPOINT_LOAD:
361 *gen_type = HW_BREAKPOINT_R;
362 break;
363 case ARM_BREAKPOINT_STORE:
364 *gen_type = HW_BREAKPOINT_W;
365 break;
366 case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
367 *gen_type = HW_BREAKPOINT_RW;
368 break;
369 default:
370 return -EINVAL;
371 }
372
373 if (!ctrl.len)
374 return -EINVAL;
375 *offset = __ffs(ctrl.len);
376
377 /* Len */
378 switch (ctrl.len >> *offset) {
379 case ARM_BREAKPOINT_LEN_1:
380 *gen_len = HW_BREAKPOINT_LEN_1;
381 break;
382 case ARM_BREAKPOINT_LEN_2:
383 *gen_len = HW_BREAKPOINT_LEN_2;
384 break;
385 case ARM_BREAKPOINT_LEN_3:
386 *gen_len = HW_BREAKPOINT_LEN_3;
387 break;
388 case ARM_BREAKPOINT_LEN_4:
389 *gen_len = HW_BREAKPOINT_LEN_4;
390 break;
391 case ARM_BREAKPOINT_LEN_5:
392 *gen_len = HW_BREAKPOINT_LEN_5;
393 break;
394 case ARM_BREAKPOINT_LEN_6:
395 *gen_len = HW_BREAKPOINT_LEN_6;
396 break;
397 case ARM_BREAKPOINT_LEN_7:
398 *gen_len = HW_BREAKPOINT_LEN_7;
399 break;
400 case ARM_BREAKPOINT_LEN_8:
401 *gen_len = HW_BREAKPOINT_LEN_8;
402 break;
403 default:
404 return -EINVAL;
405 }
406
407 return 0;
408}
409
410/*
411 * Construct an arch_hw_breakpoint from a perf_event.
412 */
413static int arch_build_bp_info(struct perf_event *bp,
414 const struct perf_event_attr *attr,
415 struct arch_hw_breakpoint *hw)
416{
417 /* Type */
418 switch (attr->bp_type) {
419 case HW_BREAKPOINT_X:
420 hw->ctrl.type = ARM_BREAKPOINT_EXECUTE;
421 break;
422 case HW_BREAKPOINT_R:
423 hw->ctrl.type = ARM_BREAKPOINT_LOAD;
424 break;
425 case HW_BREAKPOINT_W:
426 hw->ctrl.type = ARM_BREAKPOINT_STORE;
427 break;
428 case HW_BREAKPOINT_RW:
429 hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
430 break;
431 default:
432 return -EINVAL;
433 }
434
435 /* Len */
436 switch (attr->bp_len) {
437 case HW_BREAKPOINT_LEN_1:
438 hw->ctrl.len = ARM_BREAKPOINT_LEN_1;
439 break;
440 case HW_BREAKPOINT_LEN_2:
441 hw->ctrl.len = ARM_BREAKPOINT_LEN_2;
442 break;
443 case HW_BREAKPOINT_LEN_3:
444 hw->ctrl.len = ARM_BREAKPOINT_LEN_3;
445 break;
446 case HW_BREAKPOINT_LEN_4:
447 hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
448 break;
449 case HW_BREAKPOINT_LEN_5:
450 hw->ctrl.len = ARM_BREAKPOINT_LEN_5;
451 break;
452 case HW_BREAKPOINT_LEN_6:
453 hw->ctrl.len = ARM_BREAKPOINT_LEN_6;
454 break;
455 case HW_BREAKPOINT_LEN_7:
456 hw->ctrl.len = ARM_BREAKPOINT_LEN_7;
457 break;
458 case HW_BREAKPOINT_LEN_8:
459 hw->ctrl.len = ARM_BREAKPOINT_LEN_8;
460 break;
461 default:
462 return -EINVAL;
463 }
464
465 /*
466 * On AArch64, we only permit breakpoints of length 4, whereas
467 * AArch32 also requires breakpoints of length 2 for Thumb.
468 * Watchpoints can be of length 1, 2, 4 or 8 bytes.
469 */
470 if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
471 if (is_compat_bp(bp)) {
472 if (hw->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
473 hw->ctrl.len != ARM_BREAKPOINT_LEN_4)
474 return -EINVAL;
475 } else if (hw->ctrl.len != ARM_BREAKPOINT_LEN_4) {
476 /*
477 * FIXME: Some tools (I'm looking at you perf) assume
478 * that breakpoints should be sizeof(long). This
479 * is nonsense. For now, we fix up the parameter
480 * but we should probably return -EINVAL instead.
481 */
482 hw->ctrl.len = ARM_BREAKPOINT_LEN_4;
483 }
484 }
485
486 /* Address */
487 hw->address = attr->bp_addr;
488
489 /*
490 * Privilege
491 * Note that we disallow combined EL0/EL1 breakpoints because
492 * that would complicate the stepping code.
493 */
494 if (arch_check_bp_in_kernelspace(hw))
495 hw->ctrl.privilege = AARCH64_BREAKPOINT_EL1;
496 else
497 hw->ctrl.privilege = AARCH64_BREAKPOINT_EL0;
498
499 /* Enabled? */
500 hw->ctrl.enabled = !attr->disabled;
501
502 return 0;
503}
504
505/*
506 * Validate the arch-specific HW Breakpoint register settings.
507 */
508int hw_breakpoint_arch_parse(struct perf_event *bp,
509 const struct perf_event_attr *attr,
510 struct arch_hw_breakpoint *hw)
511{
512 int ret;
513 u64 alignment_mask, offset;
514
515 /* Build the arch_hw_breakpoint. */
516 ret = arch_build_bp_info(bp, attr, hw);
517 if (ret)
518 return ret;
519
520 /*
521 * Check address alignment.
522 * We don't do any clever alignment correction for watchpoints
523 * because using 64-bit unaligned addresses is deprecated for
524 * AArch64.
525 *
526 * AArch32 tasks expect some simple alignment fixups, so emulate
527 * that here.
528 */
529 if (is_compat_bp(bp)) {
530 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8)
531 alignment_mask = 0x7;
532 else
533 alignment_mask = 0x3;
534 offset = hw->address & alignment_mask;
535 switch (offset) {
536 case 0:
537 /* Aligned */
538 break;
539 case 1:
540 case 2:
541 /* Allow halfword watchpoints and breakpoints. */
542 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2)
543 break;
544
545 fallthrough;
546 case 3:
547 /* Allow single byte watchpoint. */
548 if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1)
549 break;
550
551 fallthrough;
552 default:
553 return -EINVAL;
554 }
555 } else {
556 if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE)
557 alignment_mask = 0x3;
558 else
559 alignment_mask = 0x7;
560 offset = hw->address & alignment_mask;
561 }
562
563 hw->address &= ~alignment_mask;
564 hw->ctrl.len <<= offset;
565
566 /*
567 * Disallow per-task kernel breakpoints since these would
568 * complicate the stepping code.
569 */
570 if (hw->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target)
571 return -EINVAL;
572
573 return 0;
574}
575
576/*
577 * Enable/disable all of the breakpoints active at the specified
578 * exception level at the register level.
579 * This is used when single-stepping after a breakpoint exception.
580 */
581static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable)
582{
583 int i, max_slots, privilege;
584 u32 ctrl;
585 struct perf_event **slots;
586
587 switch (reg) {
588 case AARCH64_DBG_REG_BCR:
589 slots = this_cpu_ptr(bp_on_reg);
590 max_slots = core_num_brps;
591 break;
592 case AARCH64_DBG_REG_WCR:
593 slots = this_cpu_ptr(wp_on_reg);
594 max_slots = core_num_wrps;
595 break;
596 default:
597 return;
598 }
599
600 for (i = 0; i < max_slots; ++i) {
601 if (!slots[i])
602 continue;
603
604 privilege = counter_arch_bp(bp: slots[i])->ctrl.privilege;
605 if (debug_exception_level(privilege) != el)
606 continue;
607
608 ctrl = read_wb_reg(reg, n: i);
609 if (enable)
610 ctrl |= 0x1;
611 else
612 ctrl &= ~0x1;
613 write_wb_reg(reg, n: i, val: ctrl);
614 }
615}
616NOKPROBE_SYMBOL(toggle_bp_registers);
617
618/*
619 * Debug exception handlers.
620 */
621static int breakpoint_handler(unsigned long unused, unsigned long esr,
622 struct pt_regs *regs)
623{
624 int i, step = 0, *kernel_step;
625 u32 ctrl_reg;
626 u64 addr, val;
627 struct perf_event *bp, **slots;
628 struct debug_info *debug_info;
629 struct arch_hw_breakpoint_ctrl ctrl;
630
631 slots = this_cpu_ptr(bp_on_reg);
632 addr = instruction_pointer(regs);
633 debug_info = &current->thread.debug;
634
635 for (i = 0; i < core_num_brps; ++i) {
636 rcu_read_lock();
637
638 bp = slots[i];
639
640 if (bp == NULL)
641 goto unlock;
642
643 /* Check if the breakpoint value matches. */
644 val = read_wb_reg(AARCH64_DBG_REG_BVR, i);
645 if (val != (addr & ~0x3))
646 goto unlock;
647
648 /* Possible match, check the byte address select to confirm. */
649 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i);
650 decode_ctrl_reg(ctrl_reg, &ctrl);
651 if (!((1 << (addr & 0x3)) & ctrl.len))
652 goto unlock;
653
654 counter_arch_bp(bp)->trigger = addr;
655 perf_bp_event(event: bp, data: regs);
656
657 /* Do we need to handle the stepping? */
658 if (uses_default_overflow_handler(event: bp))
659 step = 1;
660unlock:
661 rcu_read_unlock();
662 }
663
664 if (!step)
665 return 0;
666
667 if (user_mode(regs)) {
668 debug_info->bps_disabled = 1;
669 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0);
670
671 /* If we're already stepping a watchpoint, just return. */
672 if (debug_info->wps_disabled)
673 return 0;
674
675 if (test_thread_flag(TIF_SINGLESTEP))
676 debug_info->suspended_step = 1;
677 else
678 user_enable_single_step(current);
679 } else {
680 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0);
681 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
682
683 if (*kernel_step != ARM_KERNEL_STEP_NONE)
684 return 0;
685
686 if (kernel_active_single_step()) {
687 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
688 } else {
689 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
690 kernel_enable_single_step(regs);
691 }
692 }
693
694 return 0;
695}
696NOKPROBE_SYMBOL(breakpoint_handler);
697
698/*
699 * Arm64 hardware does not always report a watchpoint hit address that matches
700 * one of the watchpoints set. It can also report an address "near" the
701 * watchpoint if a single instruction access both watched and unwatched
702 * addresses. There is no straight-forward way, short of disassembling the
703 * offending instruction, to map that address back to the watchpoint. This
704 * function computes the distance of the memory access from the watchpoint as a
705 * heuristic for the likelihood that a given access triggered the watchpoint.
706 *
707 * See Section D2.10.5 "Determining the memory location that caused a Watchpoint
708 * exception" of ARMv8 Architecture Reference Manual for details.
709 *
710 * The function returns the distance of the address from the bytes watched by
711 * the watchpoint. In case of an exact match, it returns 0.
712 */
713static u64 get_distance_from_watchpoint(unsigned long addr, u64 val,
714 struct arch_hw_breakpoint_ctrl *ctrl)
715{
716 u64 wp_low, wp_high;
717 u32 lens, lene;
718
719 addr = untagged_addr(addr);
720
721 lens = __ffs(ctrl->len);
722 lene = __fls(word: ctrl->len);
723
724 wp_low = val + lens;
725 wp_high = val + lene;
726 if (addr < wp_low)
727 return wp_low - addr;
728 else if (addr > wp_high)
729 return addr - wp_high;
730 else
731 return 0;
732}
733
734static int watchpoint_report(struct perf_event *wp, unsigned long addr,
735 struct pt_regs *regs)
736{
737 int step = uses_default_overflow_handler(event: wp);
738 struct arch_hw_breakpoint *info = counter_arch_bp(bp: wp);
739
740 info->trigger = addr;
741
742 /*
743 * If we triggered a user watchpoint from a uaccess routine, then
744 * handle the stepping ourselves since userspace really can't help
745 * us with this.
746 */
747 if (!user_mode(regs) && info->ctrl.privilege == AARCH64_BREAKPOINT_EL0)
748 step = 1;
749 else
750 perf_bp_event(event: wp, data: regs);
751
752 return step;
753}
754
755static int watchpoint_handler(unsigned long addr, unsigned long esr,
756 struct pt_regs *regs)
757{
758 int i, step = 0, *kernel_step, access, closest_match = 0;
759 u64 min_dist = -1, dist;
760 u32 ctrl_reg;
761 u64 val;
762 struct perf_event *wp, **slots;
763 struct debug_info *debug_info;
764 struct arch_hw_breakpoint_ctrl ctrl;
765
766 slots = this_cpu_ptr(wp_on_reg);
767 debug_info = &current->thread.debug;
768
769 /*
770 * Find all watchpoints that match the reported address. If no exact
771 * match is found. Attribute the hit to the closest watchpoint.
772 */
773 rcu_read_lock();
774 for (i = 0; i < core_num_wrps; ++i) {
775 wp = slots[i];
776 if (wp == NULL)
777 continue;
778
779 /*
780 * Check that the access type matches.
781 * 0 => load, otherwise => store
782 */
783 access = (esr & ESR_ELx_WNR) ? HW_BREAKPOINT_W :
784 HW_BREAKPOINT_R;
785 if (!(access & hw_breakpoint_type(bp: wp)))
786 continue;
787
788 /* Check if the watchpoint value and byte select match. */
789 val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
790 ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
791 decode_ctrl_reg(ctrl_reg, &ctrl);
792 dist = get_distance_from_watchpoint(addr, val, ctrl: &ctrl);
793 if (dist < min_dist) {
794 min_dist = dist;
795 closest_match = i;
796 }
797 /* Is this an exact match? */
798 if (dist != 0)
799 continue;
800
801 step = watchpoint_report(wp, addr, regs);
802 }
803
804 /* No exact match found? */
805 if (min_dist > 0 && min_dist != -1)
806 step = watchpoint_report(wp: slots[closest_match], addr, regs);
807
808 rcu_read_unlock();
809
810 if (!step)
811 return 0;
812
813 /*
814 * We always disable EL0 watchpoints because the kernel can
815 * cause these to fire via an unprivileged access.
816 */
817 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0);
818
819 if (user_mode(regs)) {
820 debug_info->wps_disabled = 1;
821
822 /* If we're already stepping a breakpoint, just return. */
823 if (debug_info->bps_disabled)
824 return 0;
825
826 if (test_thread_flag(TIF_SINGLESTEP))
827 debug_info->suspended_step = 1;
828 else
829 user_enable_single_step(current);
830 } else {
831 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0);
832 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
833
834 if (*kernel_step != ARM_KERNEL_STEP_NONE)
835 return 0;
836
837 if (kernel_active_single_step()) {
838 *kernel_step = ARM_KERNEL_STEP_SUSPEND;
839 } else {
840 *kernel_step = ARM_KERNEL_STEP_ACTIVE;
841 kernel_enable_single_step(regs);
842 }
843 }
844
845 return 0;
846}
847NOKPROBE_SYMBOL(watchpoint_handler);
848
849/*
850 * Handle single-step exception.
851 */
852int reinstall_suspended_bps(struct pt_regs *regs)
853{
854 struct debug_info *debug_info = &current->thread.debug;
855 int handled_exception = 0, *kernel_step;
856
857 kernel_step = this_cpu_ptr(&stepping_kernel_bp);
858
859 /*
860 * Called from single-step exception handler.
861 * Return 0 if execution can resume, 1 if a SIGTRAP should be
862 * reported.
863 */
864 if (user_mode(regs)) {
865 if (debug_info->bps_disabled) {
866 debug_info->bps_disabled = 0;
867 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1);
868 handled_exception = 1;
869 }
870
871 if (debug_info->wps_disabled) {
872 debug_info->wps_disabled = 0;
873 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
874 handled_exception = 1;
875 }
876
877 if (handled_exception) {
878 if (debug_info->suspended_step) {
879 debug_info->suspended_step = 0;
880 /* Allow exception handling to fall-through. */
881 handled_exception = 0;
882 } else {
883 user_disable_single_step(current);
884 }
885 }
886 } else if (*kernel_step != ARM_KERNEL_STEP_NONE) {
887 toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1);
888 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1);
889
890 if (!debug_info->wps_disabled)
891 toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1);
892
893 if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) {
894 kernel_disable_single_step();
895 handled_exception = 1;
896 } else {
897 handled_exception = 0;
898 }
899
900 *kernel_step = ARM_KERNEL_STEP_NONE;
901 }
902
903 return !handled_exception;
904}
905NOKPROBE_SYMBOL(reinstall_suspended_bps);
906
907/*
908 * Context-switcher for restoring suspended breakpoints.
909 */
910void hw_breakpoint_thread_switch(struct task_struct *next)
911{
912 /*
913 * current next
914 * disabled: 0 0 => The usual case, NOTIFY_DONE
915 * 0 1 => Disable the registers
916 * 1 0 => Enable the registers
917 * 1 1 => NOTIFY_DONE. per-task bps will
918 * get taken care of by perf.
919 */
920
921 struct debug_info *current_debug_info, *next_debug_info;
922
923 current_debug_info = &current->thread.debug;
924 next_debug_info = &next->thread.debug;
925
926 /* Update breakpoints. */
927 if (current_debug_info->bps_disabled != next_debug_info->bps_disabled)
928 toggle_bp_registers(AARCH64_DBG_REG_BCR,
929 DBG_ACTIVE_EL0,
930 !next_debug_info->bps_disabled);
931
932 /* Update watchpoints. */
933 if (current_debug_info->wps_disabled != next_debug_info->wps_disabled)
934 toggle_bp_registers(AARCH64_DBG_REG_WCR,
935 DBG_ACTIVE_EL0,
936 !next_debug_info->wps_disabled);
937}
938
939/*
940 * CPU initialisation.
941 */
942static int hw_breakpoint_reset(unsigned int cpu)
943{
944 int i;
945 struct perf_event **slots;
946 /*
947 * When a CPU goes through cold-boot, it does not have any installed
948 * slot, so it is safe to share the same function for restoring and
949 * resetting breakpoints; when a CPU is hotplugged in, it goes
950 * through the slots, which are all empty, hence it just resets control
951 * and value for debug registers.
952 * When this function is triggered on warm-boot through a CPU PM
953 * notifier some slots might be initialized; if so they are
954 * reprogrammed according to the debug slots content.
955 */
956 for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) {
957 if (slots[i]) {
958 hw_breakpoint_control(bp: slots[i], ops: HW_BREAKPOINT_RESTORE);
959 } else {
960 write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
961 write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
962 }
963 }
964
965 for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) {
966 if (slots[i]) {
967 hw_breakpoint_control(bp: slots[i], ops: HW_BREAKPOINT_RESTORE);
968 } else {
969 write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
970 write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
971 }
972 }
973
974 return 0;
975}
976
977/*
978 * One-time initialisation.
979 */
980static int __init arch_hw_breakpoint_init(void)
981{
982 int ret;
983
984 core_num_brps = get_num_brps();
985 core_num_wrps = get_num_wrps();
986
987 pr_info("found %d breakpoint and %d watchpoint registers.\n",
988 core_num_brps, core_num_wrps);
989
990 /* Register debug fault handlers. */
991 hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP,
992 TRAP_HWBKPT, "hw-breakpoint handler");
993 hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP,
994 TRAP_HWBKPT, "hw-watchpoint handler");
995
996 /*
997 * Reset the breakpoint resources. We assume that a halting
998 * debugger will leave the world in a nice state for us.
999 */
1000 ret = cpuhp_setup_state(state: CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING,
1001 name: "perf/arm64/hw_breakpoint:starting",
1002 startup: hw_breakpoint_reset, NULL);
1003 if (ret)
1004 pr_err("failed to register CPU hotplug notifier: %d\n", ret);
1005
1006 /* Register cpu_suspend hw breakpoint restore hook */
1007 cpu_suspend_set_dbg_restorer(hw_breakpoint_reset);
1008
1009 return ret;
1010}
1011arch_initcall(arch_hw_breakpoint_init);
1012
1013void hw_breakpoint_pmu_read(struct perf_event *bp)
1014{
1015}
1016
1017/*
1018 * Dummy function to register with die_notifier.
1019 */
1020int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
1021 unsigned long val, void *data)
1022{
1023 return NOTIFY_DONE;
1024}
1025

source code of linux/arch/arm64/kernel/hw_breakpoint.c