1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, |
4 | * using the CPU's debug registers. |
5 | * |
6 | * Copyright (C) 2012 ARM Limited |
7 | * Author: Will Deacon <will.deacon@arm.com> |
8 | */ |
9 | |
10 | #define pr_fmt(fmt) "hw-breakpoint: " fmt |
11 | |
12 | #include <linux/compat.h> |
13 | #include <linux/cpu_pm.h> |
14 | #include <linux/errno.h> |
15 | #include <linux/hw_breakpoint.h> |
16 | #include <linux/kprobes.h> |
17 | #include <linux/perf_event.h> |
18 | #include <linux/ptrace.h> |
19 | #include <linux/smp.h> |
20 | #include <linux/uaccess.h> |
21 | |
22 | #include <asm/current.h> |
23 | #include <asm/debug-monitors.h> |
24 | #include <asm/esr.h> |
25 | #include <asm/hw_breakpoint.h> |
26 | #include <asm/traps.h> |
27 | #include <asm/cputype.h> |
28 | #include <asm/system_misc.h> |
29 | |
30 | /* Breakpoint currently in use for each BRP. */ |
31 | static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); |
32 | |
33 | /* Watchpoint currently in use for each WRP. */ |
34 | static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]); |
35 | |
36 | /* Currently stepping a per-CPU kernel breakpoint. */ |
37 | static DEFINE_PER_CPU(int, stepping_kernel_bp); |
38 | |
39 | /* Number of BRP/WRP registers on this CPU. */ |
40 | static int core_num_brps; |
41 | static int core_num_wrps; |
42 | |
43 | int hw_breakpoint_slots(int type) |
44 | { |
45 | /* |
46 | * We can be called early, so don't rely on |
47 | * our static variables being initialised. |
48 | */ |
49 | switch (type) { |
50 | case TYPE_INST: |
51 | return get_num_brps(); |
52 | case TYPE_DATA: |
53 | return get_num_wrps(); |
54 | default: |
55 | pr_warn("unknown slot type: %d\n" , type); |
56 | return 0; |
57 | } |
58 | } |
59 | |
60 | #define READ_WB_REG_CASE(OFF, N, REG, VAL) \ |
61 | case (OFF + N): \ |
62 | AARCH64_DBG_READ(N, REG, VAL); \ |
63 | break |
64 | |
65 | #define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \ |
66 | case (OFF + N): \ |
67 | AARCH64_DBG_WRITE(N, REG, VAL); \ |
68 | break |
69 | |
70 | #define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \ |
71 | READ_WB_REG_CASE(OFF, 0, REG, VAL); \ |
72 | READ_WB_REG_CASE(OFF, 1, REG, VAL); \ |
73 | READ_WB_REG_CASE(OFF, 2, REG, VAL); \ |
74 | READ_WB_REG_CASE(OFF, 3, REG, VAL); \ |
75 | READ_WB_REG_CASE(OFF, 4, REG, VAL); \ |
76 | READ_WB_REG_CASE(OFF, 5, REG, VAL); \ |
77 | READ_WB_REG_CASE(OFF, 6, REG, VAL); \ |
78 | READ_WB_REG_CASE(OFF, 7, REG, VAL); \ |
79 | READ_WB_REG_CASE(OFF, 8, REG, VAL); \ |
80 | READ_WB_REG_CASE(OFF, 9, REG, VAL); \ |
81 | READ_WB_REG_CASE(OFF, 10, REG, VAL); \ |
82 | READ_WB_REG_CASE(OFF, 11, REG, VAL); \ |
83 | READ_WB_REG_CASE(OFF, 12, REG, VAL); \ |
84 | READ_WB_REG_CASE(OFF, 13, REG, VAL); \ |
85 | READ_WB_REG_CASE(OFF, 14, REG, VAL); \ |
86 | READ_WB_REG_CASE(OFF, 15, REG, VAL) |
87 | |
88 | #define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \ |
89 | WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \ |
90 | WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \ |
91 | WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \ |
92 | WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \ |
93 | WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \ |
94 | WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \ |
95 | WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \ |
96 | WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \ |
97 | WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \ |
98 | WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \ |
99 | WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \ |
100 | WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \ |
101 | WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \ |
102 | WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \ |
103 | WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ |
104 | WRITE_WB_REG_CASE(OFF, 15, REG, VAL) |
105 | |
106 | static u64 read_wb_reg(int reg, int n) |
107 | { |
108 | u64 val = 0; |
109 | |
110 | switch (reg + n) { |
111 | GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); |
112 | GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); |
113 | GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); |
114 | GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); |
115 | default: |
116 | pr_warn("attempt to read from unknown breakpoint register %d\n" , n); |
117 | } |
118 | |
119 | return val; |
120 | } |
121 | NOKPROBE_SYMBOL(read_wb_reg); |
122 | |
123 | static void write_wb_reg(int reg, int n, u64 val) |
124 | { |
125 | switch (reg + n) { |
126 | GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); |
127 | GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); |
128 | GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); |
129 | GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); |
130 | default: |
131 | pr_warn("attempt to write to unknown breakpoint register %d\n" , n); |
132 | } |
133 | isb(); |
134 | } |
135 | NOKPROBE_SYMBOL(write_wb_reg); |
136 | |
137 | /* |
138 | * Convert a breakpoint privilege level to the corresponding exception |
139 | * level. |
140 | */ |
141 | static enum dbg_active_el debug_exception_level(int privilege) |
142 | { |
143 | switch (privilege) { |
144 | case AARCH64_BREAKPOINT_EL0: |
145 | return DBG_ACTIVE_EL0; |
146 | case AARCH64_BREAKPOINT_EL1: |
147 | return DBG_ACTIVE_EL1; |
148 | default: |
149 | pr_warn("invalid breakpoint privilege level %d\n" , privilege); |
150 | return -EINVAL; |
151 | } |
152 | } |
153 | NOKPROBE_SYMBOL(debug_exception_level); |
154 | |
155 | enum hw_breakpoint_ops { |
156 | HW_BREAKPOINT_INSTALL, |
157 | HW_BREAKPOINT_UNINSTALL, |
158 | HW_BREAKPOINT_RESTORE |
159 | }; |
160 | |
161 | static int is_compat_bp(struct perf_event *bp) |
162 | { |
163 | struct task_struct *tsk = bp->hw.target; |
164 | |
165 | /* |
166 | * tsk can be NULL for per-cpu (non-ptrace) breakpoints. |
167 | * In this case, use the native interface, since we don't have |
168 | * the notion of a "compat CPU" and could end up relying on |
169 | * deprecated behaviour if we use unaligned watchpoints in |
170 | * AArch64 state. |
171 | */ |
172 | return tsk && is_compat_thread(task_thread_info(tsk)); |
173 | } |
174 | |
175 | /** |
176 | * hw_breakpoint_slot_setup - Find and setup a perf slot according to |
177 | * operations |
178 | * |
179 | * @slots: pointer to array of slots |
180 | * @max_slots: max number of slots |
181 | * @bp: perf_event to setup |
182 | * @ops: operation to be carried out on the slot |
183 | * |
184 | * Return: |
185 | * slot index on success |
186 | * -ENOSPC if no slot is available/matches |
187 | * -EINVAL on wrong operations parameter |
188 | */ |
189 | static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, |
190 | struct perf_event *bp, |
191 | enum hw_breakpoint_ops ops) |
192 | { |
193 | int i; |
194 | struct perf_event **slot; |
195 | |
196 | for (i = 0; i < max_slots; ++i) { |
197 | slot = &slots[i]; |
198 | switch (ops) { |
199 | case HW_BREAKPOINT_INSTALL: |
200 | if (!*slot) { |
201 | *slot = bp; |
202 | return i; |
203 | } |
204 | break; |
205 | case HW_BREAKPOINT_UNINSTALL: |
206 | if (*slot == bp) { |
207 | *slot = NULL; |
208 | return i; |
209 | } |
210 | break; |
211 | case HW_BREAKPOINT_RESTORE: |
212 | if (*slot == bp) |
213 | return i; |
214 | break; |
215 | default: |
216 | pr_warn_once("Unhandled hw breakpoint ops %d\n" , ops); |
217 | return -EINVAL; |
218 | } |
219 | } |
220 | return -ENOSPC; |
221 | } |
222 | |
223 | static int hw_breakpoint_control(struct perf_event *bp, |
224 | enum hw_breakpoint_ops ops) |
225 | { |
226 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
227 | struct perf_event **slots; |
228 | struct debug_info *debug_info = ¤t->thread.debug; |
229 | int i, max_slots, ctrl_reg, val_reg, reg_enable; |
230 | enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege); |
231 | u32 ctrl; |
232 | |
233 | if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { |
234 | /* Breakpoint */ |
235 | ctrl_reg = AARCH64_DBG_REG_BCR; |
236 | val_reg = AARCH64_DBG_REG_BVR; |
237 | slots = this_cpu_ptr(bp_on_reg); |
238 | max_slots = core_num_brps; |
239 | reg_enable = !debug_info->bps_disabled; |
240 | } else { |
241 | /* Watchpoint */ |
242 | ctrl_reg = AARCH64_DBG_REG_WCR; |
243 | val_reg = AARCH64_DBG_REG_WVR; |
244 | slots = this_cpu_ptr(wp_on_reg); |
245 | max_slots = core_num_wrps; |
246 | reg_enable = !debug_info->wps_disabled; |
247 | } |
248 | |
249 | i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); |
250 | |
251 | if (WARN_ONCE(i < 0, "Can't find any breakpoint slot" )) |
252 | return i; |
253 | |
254 | switch (ops) { |
255 | case HW_BREAKPOINT_INSTALL: |
256 | /* |
257 | * Ensure debug monitors are enabled at the correct exception |
258 | * level. |
259 | */ |
260 | enable_debug_monitors(dbg_el); |
261 | fallthrough; |
262 | case HW_BREAKPOINT_RESTORE: |
263 | /* Setup the address register. */ |
264 | write_wb_reg(reg: val_reg, n: i, val: info->address); |
265 | |
266 | /* Setup the control register. */ |
267 | ctrl = encode_ctrl_reg(info->ctrl); |
268 | write_wb_reg(reg: ctrl_reg, n: i, |
269 | val: reg_enable ? ctrl | 0x1 : ctrl & ~0x1); |
270 | break; |
271 | case HW_BREAKPOINT_UNINSTALL: |
272 | /* Reset the control register. */ |
273 | write_wb_reg(reg: ctrl_reg, n: i, val: 0); |
274 | |
275 | /* |
276 | * Release the debug monitors for the correct exception |
277 | * level. |
278 | */ |
279 | disable_debug_monitors(dbg_el); |
280 | break; |
281 | } |
282 | |
283 | return 0; |
284 | } |
285 | |
286 | /* |
287 | * Install a perf counter breakpoint. |
288 | */ |
289 | int arch_install_hw_breakpoint(struct perf_event *bp) |
290 | { |
291 | return hw_breakpoint_control(bp, ops: HW_BREAKPOINT_INSTALL); |
292 | } |
293 | |
294 | void arch_uninstall_hw_breakpoint(struct perf_event *bp) |
295 | { |
296 | hw_breakpoint_control(bp, ops: HW_BREAKPOINT_UNINSTALL); |
297 | } |
298 | |
299 | static int get_hbp_len(u8 hbp_len) |
300 | { |
301 | unsigned int len_in_bytes = 0; |
302 | |
303 | switch (hbp_len) { |
304 | case ARM_BREAKPOINT_LEN_1: |
305 | len_in_bytes = 1; |
306 | break; |
307 | case ARM_BREAKPOINT_LEN_2: |
308 | len_in_bytes = 2; |
309 | break; |
310 | case ARM_BREAKPOINT_LEN_3: |
311 | len_in_bytes = 3; |
312 | break; |
313 | case ARM_BREAKPOINT_LEN_4: |
314 | len_in_bytes = 4; |
315 | break; |
316 | case ARM_BREAKPOINT_LEN_5: |
317 | len_in_bytes = 5; |
318 | break; |
319 | case ARM_BREAKPOINT_LEN_6: |
320 | len_in_bytes = 6; |
321 | break; |
322 | case ARM_BREAKPOINT_LEN_7: |
323 | len_in_bytes = 7; |
324 | break; |
325 | case ARM_BREAKPOINT_LEN_8: |
326 | len_in_bytes = 8; |
327 | break; |
328 | } |
329 | |
330 | return len_in_bytes; |
331 | } |
332 | |
333 | /* |
334 | * Check whether bp virtual address is in kernel space. |
335 | */ |
336 | int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) |
337 | { |
338 | unsigned int len; |
339 | unsigned long va; |
340 | |
341 | va = hw->address; |
342 | len = get_hbp_len(hbp_len: hw->ctrl.len); |
343 | |
344 | return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); |
345 | } |
346 | |
347 | /* |
348 | * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. |
349 | * Hopefully this will disappear when ptrace can bypass the conversion |
350 | * to generic breakpoint descriptions. |
351 | */ |
352 | int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, |
353 | int *gen_len, int *gen_type, int *offset) |
354 | { |
355 | /* Type */ |
356 | switch (ctrl.type) { |
357 | case ARM_BREAKPOINT_EXECUTE: |
358 | *gen_type = HW_BREAKPOINT_X; |
359 | break; |
360 | case ARM_BREAKPOINT_LOAD: |
361 | *gen_type = HW_BREAKPOINT_R; |
362 | break; |
363 | case ARM_BREAKPOINT_STORE: |
364 | *gen_type = HW_BREAKPOINT_W; |
365 | break; |
366 | case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE: |
367 | *gen_type = HW_BREAKPOINT_RW; |
368 | break; |
369 | default: |
370 | return -EINVAL; |
371 | } |
372 | |
373 | if (!ctrl.len) |
374 | return -EINVAL; |
375 | *offset = __ffs(ctrl.len); |
376 | |
377 | /* Len */ |
378 | switch (ctrl.len >> *offset) { |
379 | case ARM_BREAKPOINT_LEN_1: |
380 | *gen_len = HW_BREAKPOINT_LEN_1; |
381 | break; |
382 | case ARM_BREAKPOINT_LEN_2: |
383 | *gen_len = HW_BREAKPOINT_LEN_2; |
384 | break; |
385 | case ARM_BREAKPOINT_LEN_3: |
386 | *gen_len = HW_BREAKPOINT_LEN_3; |
387 | break; |
388 | case ARM_BREAKPOINT_LEN_4: |
389 | *gen_len = HW_BREAKPOINT_LEN_4; |
390 | break; |
391 | case ARM_BREAKPOINT_LEN_5: |
392 | *gen_len = HW_BREAKPOINT_LEN_5; |
393 | break; |
394 | case ARM_BREAKPOINT_LEN_6: |
395 | *gen_len = HW_BREAKPOINT_LEN_6; |
396 | break; |
397 | case ARM_BREAKPOINT_LEN_7: |
398 | *gen_len = HW_BREAKPOINT_LEN_7; |
399 | break; |
400 | case ARM_BREAKPOINT_LEN_8: |
401 | *gen_len = HW_BREAKPOINT_LEN_8; |
402 | break; |
403 | default: |
404 | return -EINVAL; |
405 | } |
406 | |
407 | return 0; |
408 | } |
409 | |
410 | /* |
411 | * Construct an arch_hw_breakpoint from a perf_event. |
412 | */ |
413 | static int arch_build_bp_info(struct perf_event *bp, |
414 | const struct perf_event_attr *attr, |
415 | struct arch_hw_breakpoint *hw) |
416 | { |
417 | /* Type */ |
418 | switch (attr->bp_type) { |
419 | case HW_BREAKPOINT_X: |
420 | hw->ctrl.type = ARM_BREAKPOINT_EXECUTE; |
421 | break; |
422 | case HW_BREAKPOINT_R: |
423 | hw->ctrl.type = ARM_BREAKPOINT_LOAD; |
424 | break; |
425 | case HW_BREAKPOINT_W: |
426 | hw->ctrl.type = ARM_BREAKPOINT_STORE; |
427 | break; |
428 | case HW_BREAKPOINT_RW: |
429 | hw->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; |
430 | break; |
431 | default: |
432 | return -EINVAL; |
433 | } |
434 | |
435 | /* Len */ |
436 | switch (attr->bp_len) { |
437 | case HW_BREAKPOINT_LEN_1: |
438 | hw->ctrl.len = ARM_BREAKPOINT_LEN_1; |
439 | break; |
440 | case HW_BREAKPOINT_LEN_2: |
441 | hw->ctrl.len = ARM_BREAKPOINT_LEN_2; |
442 | break; |
443 | case HW_BREAKPOINT_LEN_3: |
444 | hw->ctrl.len = ARM_BREAKPOINT_LEN_3; |
445 | break; |
446 | case HW_BREAKPOINT_LEN_4: |
447 | hw->ctrl.len = ARM_BREAKPOINT_LEN_4; |
448 | break; |
449 | case HW_BREAKPOINT_LEN_5: |
450 | hw->ctrl.len = ARM_BREAKPOINT_LEN_5; |
451 | break; |
452 | case HW_BREAKPOINT_LEN_6: |
453 | hw->ctrl.len = ARM_BREAKPOINT_LEN_6; |
454 | break; |
455 | case HW_BREAKPOINT_LEN_7: |
456 | hw->ctrl.len = ARM_BREAKPOINT_LEN_7; |
457 | break; |
458 | case HW_BREAKPOINT_LEN_8: |
459 | hw->ctrl.len = ARM_BREAKPOINT_LEN_8; |
460 | break; |
461 | default: |
462 | return -EINVAL; |
463 | } |
464 | |
465 | /* |
466 | * On AArch64, we only permit breakpoints of length 4, whereas |
467 | * AArch32 also requires breakpoints of length 2 for Thumb. |
468 | * Watchpoints can be of length 1, 2, 4 or 8 bytes. |
469 | */ |
470 | if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE) { |
471 | if (is_compat_bp(bp)) { |
472 | if (hw->ctrl.len != ARM_BREAKPOINT_LEN_2 && |
473 | hw->ctrl.len != ARM_BREAKPOINT_LEN_4) |
474 | return -EINVAL; |
475 | } else if (hw->ctrl.len != ARM_BREAKPOINT_LEN_4) { |
476 | /* |
477 | * FIXME: Some tools (I'm looking at you perf) assume |
478 | * that breakpoints should be sizeof(long). This |
479 | * is nonsense. For now, we fix up the parameter |
480 | * but we should probably return -EINVAL instead. |
481 | */ |
482 | hw->ctrl.len = ARM_BREAKPOINT_LEN_4; |
483 | } |
484 | } |
485 | |
486 | /* Address */ |
487 | hw->address = attr->bp_addr; |
488 | |
489 | /* |
490 | * Privilege |
491 | * Note that we disallow combined EL0/EL1 breakpoints because |
492 | * that would complicate the stepping code. |
493 | */ |
494 | if (arch_check_bp_in_kernelspace(hw)) |
495 | hw->ctrl.privilege = AARCH64_BREAKPOINT_EL1; |
496 | else |
497 | hw->ctrl.privilege = AARCH64_BREAKPOINT_EL0; |
498 | |
499 | /* Enabled? */ |
500 | hw->ctrl.enabled = !attr->disabled; |
501 | |
502 | return 0; |
503 | } |
504 | |
505 | /* |
506 | * Validate the arch-specific HW Breakpoint register settings. |
507 | */ |
508 | int hw_breakpoint_arch_parse(struct perf_event *bp, |
509 | const struct perf_event_attr *attr, |
510 | struct arch_hw_breakpoint *hw) |
511 | { |
512 | int ret; |
513 | u64 alignment_mask, offset; |
514 | |
515 | /* Build the arch_hw_breakpoint. */ |
516 | ret = arch_build_bp_info(bp, attr, hw); |
517 | if (ret) |
518 | return ret; |
519 | |
520 | /* |
521 | * Check address alignment. |
522 | * We don't do any clever alignment correction for watchpoints |
523 | * because using 64-bit unaligned addresses is deprecated for |
524 | * AArch64. |
525 | * |
526 | * AArch32 tasks expect some simple alignment fixups, so emulate |
527 | * that here. |
528 | */ |
529 | if (is_compat_bp(bp)) { |
530 | if (hw->ctrl.len == ARM_BREAKPOINT_LEN_8) |
531 | alignment_mask = 0x7; |
532 | else |
533 | alignment_mask = 0x3; |
534 | offset = hw->address & alignment_mask; |
535 | switch (offset) { |
536 | case 0: |
537 | /* Aligned */ |
538 | break; |
539 | case 1: |
540 | case 2: |
541 | /* Allow halfword watchpoints and breakpoints. */ |
542 | if (hw->ctrl.len == ARM_BREAKPOINT_LEN_2) |
543 | break; |
544 | |
545 | fallthrough; |
546 | case 3: |
547 | /* Allow single byte watchpoint. */ |
548 | if (hw->ctrl.len == ARM_BREAKPOINT_LEN_1) |
549 | break; |
550 | |
551 | fallthrough; |
552 | default: |
553 | return -EINVAL; |
554 | } |
555 | } else { |
556 | if (hw->ctrl.type == ARM_BREAKPOINT_EXECUTE) |
557 | alignment_mask = 0x3; |
558 | else |
559 | alignment_mask = 0x7; |
560 | offset = hw->address & alignment_mask; |
561 | } |
562 | |
563 | hw->address &= ~alignment_mask; |
564 | hw->ctrl.len <<= offset; |
565 | |
566 | /* |
567 | * Disallow per-task kernel breakpoints since these would |
568 | * complicate the stepping code. |
569 | */ |
570 | if (hw->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target) |
571 | return -EINVAL; |
572 | |
573 | return 0; |
574 | } |
575 | |
576 | /* |
577 | * Enable/disable all of the breakpoints active at the specified |
578 | * exception level at the register level. |
579 | * This is used when single-stepping after a breakpoint exception. |
580 | */ |
581 | static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable) |
582 | { |
583 | int i, max_slots, privilege; |
584 | u32 ctrl; |
585 | struct perf_event **slots; |
586 | |
587 | switch (reg) { |
588 | case AARCH64_DBG_REG_BCR: |
589 | slots = this_cpu_ptr(bp_on_reg); |
590 | max_slots = core_num_brps; |
591 | break; |
592 | case AARCH64_DBG_REG_WCR: |
593 | slots = this_cpu_ptr(wp_on_reg); |
594 | max_slots = core_num_wrps; |
595 | break; |
596 | default: |
597 | return; |
598 | } |
599 | |
600 | for (i = 0; i < max_slots; ++i) { |
601 | if (!slots[i]) |
602 | continue; |
603 | |
604 | privilege = counter_arch_bp(bp: slots[i])->ctrl.privilege; |
605 | if (debug_exception_level(privilege) != el) |
606 | continue; |
607 | |
608 | ctrl = read_wb_reg(reg, n: i); |
609 | if (enable) |
610 | ctrl |= 0x1; |
611 | else |
612 | ctrl &= ~0x1; |
613 | write_wb_reg(reg, n: i, val: ctrl); |
614 | } |
615 | } |
616 | NOKPROBE_SYMBOL(toggle_bp_registers); |
617 | |
618 | /* |
619 | * Debug exception handlers. |
620 | */ |
621 | static int breakpoint_handler(unsigned long unused, unsigned long esr, |
622 | struct pt_regs *regs) |
623 | { |
624 | int i, step = 0, *kernel_step; |
625 | u32 ctrl_reg; |
626 | u64 addr, val; |
627 | struct perf_event *bp, **slots; |
628 | struct debug_info *debug_info; |
629 | struct arch_hw_breakpoint_ctrl ctrl; |
630 | |
631 | slots = this_cpu_ptr(bp_on_reg); |
632 | addr = instruction_pointer(regs); |
633 | debug_info = ¤t->thread.debug; |
634 | |
635 | for (i = 0; i < core_num_brps; ++i) { |
636 | rcu_read_lock(); |
637 | |
638 | bp = slots[i]; |
639 | |
640 | if (bp == NULL) |
641 | goto unlock; |
642 | |
643 | /* Check if the breakpoint value matches. */ |
644 | val = read_wb_reg(AARCH64_DBG_REG_BVR, i); |
645 | if (val != (addr & ~0x3)) |
646 | goto unlock; |
647 | |
648 | /* Possible match, check the byte address select to confirm. */ |
649 | ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i); |
650 | decode_ctrl_reg(ctrl_reg, &ctrl); |
651 | if (!((1 << (addr & 0x3)) & ctrl.len)) |
652 | goto unlock; |
653 | |
654 | counter_arch_bp(bp)->trigger = addr; |
655 | perf_bp_event(event: bp, data: regs); |
656 | |
657 | /* Do we need to handle the stepping? */ |
658 | if (uses_default_overflow_handler(event: bp)) |
659 | step = 1; |
660 | unlock: |
661 | rcu_read_unlock(); |
662 | } |
663 | |
664 | if (!step) |
665 | return 0; |
666 | |
667 | if (user_mode(regs)) { |
668 | debug_info->bps_disabled = 1; |
669 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0); |
670 | |
671 | /* If we're already stepping a watchpoint, just return. */ |
672 | if (debug_info->wps_disabled) |
673 | return 0; |
674 | |
675 | if (test_thread_flag(TIF_SINGLESTEP)) |
676 | debug_info->suspended_step = 1; |
677 | else |
678 | user_enable_single_step(current); |
679 | } else { |
680 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0); |
681 | kernel_step = this_cpu_ptr(&stepping_kernel_bp); |
682 | |
683 | if (*kernel_step != ARM_KERNEL_STEP_NONE) |
684 | return 0; |
685 | |
686 | if (kernel_active_single_step()) { |
687 | *kernel_step = ARM_KERNEL_STEP_SUSPEND; |
688 | } else { |
689 | *kernel_step = ARM_KERNEL_STEP_ACTIVE; |
690 | kernel_enable_single_step(regs); |
691 | } |
692 | } |
693 | |
694 | return 0; |
695 | } |
696 | NOKPROBE_SYMBOL(breakpoint_handler); |
697 | |
698 | /* |
699 | * Arm64 hardware does not always report a watchpoint hit address that matches |
700 | * one of the watchpoints set. It can also report an address "near" the |
701 | * watchpoint if a single instruction access both watched and unwatched |
702 | * addresses. There is no straight-forward way, short of disassembling the |
703 | * offending instruction, to map that address back to the watchpoint. This |
704 | * function computes the distance of the memory access from the watchpoint as a |
705 | * heuristic for the likelihood that a given access triggered the watchpoint. |
706 | * |
707 | * See Section D2.10.5 "Determining the memory location that caused a Watchpoint |
708 | * exception" of ARMv8 Architecture Reference Manual for details. |
709 | * |
710 | * The function returns the distance of the address from the bytes watched by |
711 | * the watchpoint. In case of an exact match, it returns 0. |
712 | */ |
713 | static u64 get_distance_from_watchpoint(unsigned long addr, u64 val, |
714 | struct arch_hw_breakpoint_ctrl *ctrl) |
715 | { |
716 | u64 wp_low, wp_high; |
717 | u32 lens, lene; |
718 | |
719 | addr = untagged_addr(addr); |
720 | |
721 | lens = __ffs(ctrl->len); |
722 | lene = __fls(word: ctrl->len); |
723 | |
724 | wp_low = val + lens; |
725 | wp_high = val + lene; |
726 | if (addr < wp_low) |
727 | return wp_low - addr; |
728 | else if (addr > wp_high) |
729 | return addr - wp_high; |
730 | else |
731 | return 0; |
732 | } |
733 | |
734 | static int watchpoint_report(struct perf_event *wp, unsigned long addr, |
735 | struct pt_regs *regs) |
736 | { |
737 | int step = uses_default_overflow_handler(event: wp); |
738 | struct arch_hw_breakpoint *info = counter_arch_bp(bp: wp); |
739 | |
740 | info->trigger = addr; |
741 | |
742 | /* |
743 | * If we triggered a user watchpoint from a uaccess routine, then |
744 | * handle the stepping ourselves since userspace really can't help |
745 | * us with this. |
746 | */ |
747 | if (!user_mode(regs) && info->ctrl.privilege == AARCH64_BREAKPOINT_EL0) |
748 | step = 1; |
749 | else |
750 | perf_bp_event(event: wp, data: regs); |
751 | |
752 | return step; |
753 | } |
754 | |
755 | static int watchpoint_handler(unsigned long addr, unsigned long esr, |
756 | struct pt_regs *regs) |
757 | { |
758 | int i, step = 0, *kernel_step, access, closest_match = 0; |
759 | u64 min_dist = -1, dist; |
760 | u32 ctrl_reg; |
761 | u64 val; |
762 | struct perf_event *wp, **slots; |
763 | struct debug_info *debug_info; |
764 | struct arch_hw_breakpoint_ctrl ctrl; |
765 | |
766 | slots = this_cpu_ptr(wp_on_reg); |
767 | debug_info = ¤t->thread.debug; |
768 | |
769 | /* |
770 | * Find all watchpoints that match the reported address. If no exact |
771 | * match is found. Attribute the hit to the closest watchpoint. |
772 | */ |
773 | rcu_read_lock(); |
774 | for (i = 0; i < core_num_wrps; ++i) { |
775 | wp = slots[i]; |
776 | if (wp == NULL) |
777 | continue; |
778 | |
779 | /* |
780 | * Check that the access type matches. |
781 | * 0 => load, otherwise => store |
782 | */ |
783 | access = (esr & ESR_ELx_WNR) ? HW_BREAKPOINT_W : |
784 | HW_BREAKPOINT_R; |
785 | if (!(access & hw_breakpoint_type(bp: wp))) |
786 | continue; |
787 | |
788 | /* Check if the watchpoint value and byte select match. */ |
789 | val = read_wb_reg(AARCH64_DBG_REG_WVR, i); |
790 | ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i); |
791 | decode_ctrl_reg(ctrl_reg, &ctrl); |
792 | dist = get_distance_from_watchpoint(addr, val, ctrl: &ctrl); |
793 | if (dist < min_dist) { |
794 | min_dist = dist; |
795 | closest_match = i; |
796 | } |
797 | /* Is this an exact match? */ |
798 | if (dist != 0) |
799 | continue; |
800 | |
801 | step = watchpoint_report(wp, addr, regs); |
802 | } |
803 | |
804 | /* No exact match found? */ |
805 | if (min_dist > 0 && min_dist != -1) |
806 | step = watchpoint_report(wp: slots[closest_match], addr, regs); |
807 | |
808 | rcu_read_unlock(); |
809 | |
810 | if (!step) |
811 | return 0; |
812 | |
813 | /* |
814 | * We always disable EL0 watchpoints because the kernel can |
815 | * cause these to fire via an unprivileged access. |
816 | */ |
817 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0); |
818 | |
819 | if (user_mode(regs)) { |
820 | debug_info->wps_disabled = 1; |
821 | |
822 | /* If we're already stepping a breakpoint, just return. */ |
823 | if (debug_info->bps_disabled) |
824 | return 0; |
825 | |
826 | if (test_thread_flag(TIF_SINGLESTEP)) |
827 | debug_info->suspended_step = 1; |
828 | else |
829 | user_enable_single_step(current); |
830 | } else { |
831 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0); |
832 | kernel_step = this_cpu_ptr(&stepping_kernel_bp); |
833 | |
834 | if (*kernel_step != ARM_KERNEL_STEP_NONE) |
835 | return 0; |
836 | |
837 | if (kernel_active_single_step()) { |
838 | *kernel_step = ARM_KERNEL_STEP_SUSPEND; |
839 | } else { |
840 | *kernel_step = ARM_KERNEL_STEP_ACTIVE; |
841 | kernel_enable_single_step(regs); |
842 | } |
843 | } |
844 | |
845 | return 0; |
846 | } |
847 | NOKPROBE_SYMBOL(watchpoint_handler); |
848 | |
849 | /* |
850 | * Handle single-step exception. |
851 | */ |
852 | int reinstall_suspended_bps(struct pt_regs *regs) |
853 | { |
854 | struct debug_info *debug_info = ¤t->thread.debug; |
855 | int handled_exception = 0, *kernel_step; |
856 | |
857 | kernel_step = this_cpu_ptr(&stepping_kernel_bp); |
858 | |
859 | /* |
860 | * Called from single-step exception handler. |
861 | * Return 0 if execution can resume, 1 if a SIGTRAP should be |
862 | * reported. |
863 | */ |
864 | if (user_mode(regs)) { |
865 | if (debug_info->bps_disabled) { |
866 | debug_info->bps_disabled = 0; |
867 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1); |
868 | handled_exception = 1; |
869 | } |
870 | |
871 | if (debug_info->wps_disabled) { |
872 | debug_info->wps_disabled = 0; |
873 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1); |
874 | handled_exception = 1; |
875 | } |
876 | |
877 | if (handled_exception) { |
878 | if (debug_info->suspended_step) { |
879 | debug_info->suspended_step = 0; |
880 | /* Allow exception handling to fall-through. */ |
881 | handled_exception = 0; |
882 | } else { |
883 | user_disable_single_step(current); |
884 | } |
885 | } |
886 | } else if (*kernel_step != ARM_KERNEL_STEP_NONE) { |
887 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1); |
888 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1); |
889 | |
890 | if (!debug_info->wps_disabled) |
891 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1); |
892 | |
893 | if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) { |
894 | kernel_disable_single_step(); |
895 | handled_exception = 1; |
896 | } else { |
897 | handled_exception = 0; |
898 | } |
899 | |
900 | *kernel_step = ARM_KERNEL_STEP_NONE; |
901 | } |
902 | |
903 | return !handled_exception; |
904 | } |
905 | NOKPROBE_SYMBOL(reinstall_suspended_bps); |
906 | |
907 | /* |
908 | * Context-switcher for restoring suspended breakpoints. |
909 | */ |
910 | void hw_breakpoint_thread_switch(struct task_struct *next) |
911 | { |
912 | /* |
913 | * current next |
914 | * disabled: 0 0 => The usual case, NOTIFY_DONE |
915 | * 0 1 => Disable the registers |
916 | * 1 0 => Enable the registers |
917 | * 1 1 => NOTIFY_DONE. per-task bps will |
918 | * get taken care of by perf. |
919 | */ |
920 | |
921 | struct debug_info *current_debug_info, *next_debug_info; |
922 | |
923 | current_debug_info = ¤t->thread.debug; |
924 | next_debug_info = &next->thread.debug; |
925 | |
926 | /* Update breakpoints. */ |
927 | if (current_debug_info->bps_disabled != next_debug_info->bps_disabled) |
928 | toggle_bp_registers(AARCH64_DBG_REG_BCR, |
929 | DBG_ACTIVE_EL0, |
930 | !next_debug_info->bps_disabled); |
931 | |
932 | /* Update watchpoints. */ |
933 | if (current_debug_info->wps_disabled != next_debug_info->wps_disabled) |
934 | toggle_bp_registers(AARCH64_DBG_REG_WCR, |
935 | DBG_ACTIVE_EL0, |
936 | !next_debug_info->wps_disabled); |
937 | } |
938 | |
939 | /* |
940 | * CPU initialisation. |
941 | */ |
942 | static int hw_breakpoint_reset(unsigned int cpu) |
943 | { |
944 | int i; |
945 | struct perf_event **slots; |
946 | /* |
947 | * When a CPU goes through cold-boot, it does not have any installed |
948 | * slot, so it is safe to share the same function for restoring and |
949 | * resetting breakpoints; when a CPU is hotplugged in, it goes |
950 | * through the slots, which are all empty, hence it just resets control |
951 | * and value for debug registers. |
952 | * When this function is triggered on warm-boot through a CPU PM |
953 | * notifier some slots might be initialized; if so they are |
954 | * reprogrammed according to the debug slots content. |
955 | */ |
956 | for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) { |
957 | if (slots[i]) { |
958 | hw_breakpoint_control(bp: slots[i], ops: HW_BREAKPOINT_RESTORE); |
959 | } else { |
960 | write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL); |
961 | write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL); |
962 | } |
963 | } |
964 | |
965 | for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) { |
966 | if (slots[i]) { |
967 | hw_breakpoint_control(bp: slots[i], ops: HW_BREAKPOINT_RESTORE); |
968 | } else { |
969 | write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL); |
970 | write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL); |
971 | } |
972 | } |
973 | |
974 | return 0; |
975 | } |
976 | |
977 | /* |
978 | * One-time initialisation. |
979 | */ |
980 | static int __init arch_hw_breakpoint_init(void) |
981 | { |
982 | int ret; |
983 | |
984 | core_num_brps = get_num_brps(); |
985 | core_num_wrps = get_num_wrps(); |
986 | |
987 | pr_info("found %d breakpoint and %d watchpoint registers.\n" , |
988 | core_num_brps, core_num_wrps); |
989 | |
990 | /* Register debug fault handlers. */ |
991 | hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP, |
992 | TRAP_HWBKPT, "hw-breakpoint handler" ); |
993 | hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP, |
994 | TRAP_HWBKPT, "hw-watchpoint handler" ); |
995 | |
996 | /* |
997 | * Reset the breakpoint resources. We assume that a halting |
998 | * debugger will leave the world in a nice state for us. |
999 | */ |
1000 | ret = cpuhp_setup_state(state: CPUHP_AP_PERF_ARM_HW_BREAKPOINT_STARTING, |
1001 | name: "perf/arm64/hw_breakpoint:starting" , |
1002 | startup: hw_breakpoint_reset, NULL); |
1003 | if (ret) |
1004 | pr_err("failed to register CPU hotplug notifier: %d\n" , ret); |
1005 | |
1006 | /* Register cpu_suspend hw breakpoint restore hook */ |
1007 | cpu_suspend_set_dbg_restorer(hw_breakpoint_reset); |
1008 | |
1009 | return ret; |
1010 | } |
1011 | arch_initcall(arch_hw_breakpoint_init); |
1012 | |
1013 | void hw_breakpoint_pmu_read(struct perf_event *bp) |
1014 | { |
1015 | } |
1016 | |
1017 | /* |
1018 | * Dummy function to register with die_notifier. |
1019 | */ |
1020 | int hw_breakpoint_exceptions_notify(struct notifier_block *unused, |
1021 | unsigned long val, void *data) |
1022 | { |
1023 | return NOTIFY_DONE; |
1024 | } |
1025 | |