1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2022-2023 Loongson Technology Corporation Limited |
4 | */ |
5 | #define pr_fmt(fmt) "hw-breakpoint: " fmt |
6 | |
7 | #include <linux/hw_breakpoint.h> |
8 | #include <linux/kprobes.h> |
9 | #include <linux/perf_event.h> |
10 | |
11 | #include <asm/hw_breakpoint.h> |
12 | |
13 | /* Breakpoint currently in use for each BRP. */ |
14 | static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[LOONGARCH_MAX_BRP]); |
15 | |
16 | /* Watchpoint currently in use for each WRP. */ |
17 | static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[LOONGARCH_MAX_WRP]); |
18 | |
19 | int hw_breakpoint_slots(int type) |
20 | { |
21 | /* |
22 | * We can be called early, so don't rely on |
23 | * our static variables being initialised. |
24 | */ |
25 | switch (type) { |
26 | case TYPE_INST: |
27 | return get_num_brps(); |
28 | case TYPE_DATA: |
29 | return get_num_wrps(); |
30 | default: |
31 | pr_warn("unknown slot type: %d\n" , type); |
32 | return 0; |
33 | } |
34 | } |
35 | |
36 | #define READ_WB_REG_CASE(OFF, N, REG, T, VAL) \ |
37 | case (OFF + N): \ |
38 | LOONGARCH_CSR_WATCH_READ(N, REG, T, VAL); \ |
39 | break |
40 | |
41 | #define WRITE_WB_REG_CASE(OFF, N, REG, T, VAL) \ |
42 | case (OFF + N): \ |
43 | LOONGARCH_CSR_WATCH_WRITE(N, REG, T, VAL); \ |
44 | break |
45 | |
46 | #define GEN_READ_WB_REG_CASES(OFF, REG, T, VAL) \ |
47 | READ_WB_REG_CASE(OFF, 0, REG, T, VAL); \ |
48 | READ_WB_REG_CASE(OFF, 1, REG, T, VAL); \ |
49 | READ_WB_REG_CASE(OFF, 2, REG, T, VAL); \ |
50 | READ_WB_REG_CASE(OFF, 3, REG, T, VAL); \ |
51 | READ_WB_REG_CASE(OFF, 4, REG, T, VAL); \ |
52 | READ_WB_REG_CASE(OFF, 5, REG, T, VAL); \ |
53 | READ_WB_REG_CASE(OFF, 6, REG, T, VAL); \ |
54 | READ_WB_REG_CASE(OFF, 7, REG, T, VAL); |
55 | |
56 | #define GEN_WRITE_WB_REG_CASES(OFF, REG, T, VAL) \ |
57 | WRITE_WB_REG_CASE(OFF, 0, REG, T, VAL); \ |
58 | WRITE_WB_REG_CASE(OFF, 1, REG, T, VAL); \ |
59 | WRITE_WB_REG_CASE(OFF, 2, REG, T, VAL); \ |
60 | WRITE_WB_REG_CASE(OFF, 3, REG, T, VAL); \ |
61 | WRITE_WB_REG_CASE(OFF, 4, REG, T, VAL); \ |
62 | WRITE_WB_REG_CASE(OFF, 5, REG, T, VAL); \ |
63 | WRITE_WB_REG_CASE(OFF, 6, REG, T, VAL); \ |
64 | WRITE_WB_REG_CASE(OFF, 7, REG, T, VAL); |
65 | |
66 | static u64 read_wb_reg(int reg, int n, int t) |
67 | { |
68 | u64 val = 0; |
69 | |
70 | switch (reg + n) { |
71 | GEN_READ_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); |
72 | GEN_READ_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); |
73 | GEN_READ_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); |
74 | GEN_READ_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); |
75 | default: |
76 | pr_warn("Attempt to read from unknown breakpoint register %d\n" , n); |
77 | } |
78 | |
79 | return val; |
80 | } |
81 | NOKPROBE_SYMBOL(read_wb_reg); |
82 | |
83 | static void write_wb_reg(int reg, int n, int t, u64 val) |
84 | { |
85 | switch (reg + n) { |
86 | GEN_WRITE_WB_REG_CASES(CSR_CFG_ADDR, ADDR, t, val); |
87 | GEN_WRITE_WB_REG_CASES(CSR_CFG_MASK, MASK, t, val); |
88 | GEN_WRITE_WB_REG_CASES(CSR_CFG_CTRL, CTRL, t, val); |
89 | GEN_WRITE_WB_REG_CASES(CSR_CFG_ASID, ASID, t, val); |
90 | default: |
91 | pr_warn("Attempt to write to unknown breakpoint register %d\n" , n); |
92 | } |
93 | } |
94 | NOKPROBE_SYMBOL(write_wb_reg); |
95 | |
96 | enum hw_breakpoint_ops { |
97 | HW_BREAKPOINT_INSTALL, |
98 | HW_BREAKPOINT_UNINSTALL, |
99 | }; |
100 | |
101 | /* |
102 | * hw_breakpoint_slot_setup - Find and setup a perf slot according to operations |
103 | * |
104 | * @slots: pointer to array of slots |
105 | * @max_slots: max number of slots |
106 | * @bp: perf_event to setup |
107 | * @ops: operation to be carried out on the slot |
108 | * |
109 | * Return: |
110 | * slot index on success |
111 | * -ENOSPC if no slot is available/matches |
112 | * -EINVAL on wrong operations parameter |
113 | */ |
114 | |
115 | static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, |
116 | struct perf_event *bp, enum hw_breakpoint_ops ops) |
117 | { |
118 | int i; |
119 | struct perf_event **slot; |
120 | |
121 | for (i = 0; i < max_slots; ++i) { |
122 | slot = &slots[i]; |
123 | switch (ops) { |
124 | case HW_BREAKPOINT_INSTALL: |
125 | if (!*slot) { |
126 | *slot = bp; |
127 | return i; |
128 | } |
129 | break; |
130 | case HW_BREAKPOINT_UNINSTALL: |
131 | if (*slot == bp) { |
132 | *slot = NULL; |
133 | return i; |
134 | } |
135 | break; |
136 | default: |
137 | pr_warn_once("Unhandled hw breakpoint ops %d\n" , ops); |
138 | return -EINVAL; |
139 | } |
140 | } |
141 | |
142 | return -ENOSPC; |
143 | } |
144 | |
145 | void ptrace_hw_copy_thread(struct task_struct *tsk) |
146 | { |
147 | memset(tsk->thread.hbp_break, 0, sizeof(tsk->thread.hbp_break)); |
148 | memset(tsk->thread.hbp_watch, 0, sizeof(tsk->thread.hbp_watch)); |
149 | } |
150 | |
151 | /* |
152 | * Unregister breakpoints from this task and reset the pointers in the thread_struct. |
153 | */ |
154 | void flush_ptrace_hw_breakpoint(struct task_struct *tsk) |
155 | { |
156 | int i; |
157 | struct thread_struct *t = &tsk->thread; |
158 | |
159 | for (i = 0; i < LOONGARCH_MAX_BRP; i++) { |
160 | if (t->hbp_break[i]) { |
161 | unregister_hw_breakpoint(t->hbp_break[i]); |
162 | t->hbp_break[i] = NULL; |
163 | } |
164 | } |
165 | |
166 | for (i = 0; i < LOONGARCH_MAX_WRP; i++) { |
167 | if (t->hbp_watch[i]) { |
168 | unregister_hw_breakpoint(t->hbp_watch[i]); |
169 | t->hbp_watch[i] = NULL; |
170 | } |
171 | } |
172 | } |
173 | |
174 | static int hw_breakpoint_control(struct perf_event *bp, |
175 | enum hw_breakpoint_ops ops) |
176 | { |
177 | u32 ctrl; |
178 | int i, max_slots, enable; |
179 | struct perf_event **slots; |
180 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
181 | |
182 | if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { |
183 | /* Breakpoint */ |
184 | slots = this_cpu_ptr(bp_on_reg); |
185 | max_slots = boot_cpu_data.watch_ireg_count; |
186 | } else { |
187 | /* Watchpoint */ |
188 | slots = this_cpu_ptr(wp_on_reg); |
189 | max_slots = boot_cpu_data.watch_dreg_count; |
190 | } |
191 | |
192 | i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); |
193 | |
194 | if (WARN_ONCE(i < 0, "Can't find any breakpoint slot" )) |
195 | return i; |
196 | |
197 | switch (ops) { |
198 | case HW_BREAKPOINT_INSTALL: |
199 | /* Set the FWPnCFG/MWPnCFG 1~4 register. */ |
200 | write_wb_reg(CSR_CFG_ADDR, i, 0, info->address); |
201 | write_wb_reg(CSR_CFG_ADDR, i, 1, info->address); |
202 | write_wb_reg(CSR_CFG_MASK, i, 0, info->mask); |
203 | write_wb_reg(CSR_CFG_MASK, i, 1, info->mask); |
204 | write_wb_reg(CSR_CFG_ASID, i, 0, 0); |
205 | write_wb_reg(CSR_CFG_ASID, i, 1, 0); |
206 | if (info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) { |
207 | write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); |
208 | } else { |
209 | ctrl = encode_ctrl_reg(info->ctrl); |
210 | write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl | CTRL_PLV_ENABLE); |
211 | } |
212 | enable = csr_read64(LOONGARCH_CSR_CRMD); |
213 | csr_write64(CSR_CRMD_WE | enable, LOONGARCH_CSR_CRMD); |
214 | break; |
215 | case HW_BREAKPOINT_UNINSTALL: |
216 | /* Reset the FWPnCFG/MWPnCFG 1~4 register. */ |
217 | write_wb_reg(CSR_CFG_ADDR, i, 0, 0); |
218 | write_wb_reg(CSR_CFG_ADDR, i, 1, 0); |
219 | write_wb_reg(CSR_CFG_MASK, i, 0, 0); |
220 | write_wb_reg(CSR_CFG_MASK, i, 1, 0); |
221 | write_wb_reg(CSR_CFG_CTRL, i, 0, 0); |
222 | write_wb_reg(CSR_CFG_CTRL, i, 1, 0); |
223 | write_wb_reg(CSR_CFG_ASID, i, 0, 0); |
224 | write_wb_reg(CSR_CFG_ASID, i, 1, 0); |
225 | break; |
226 | } |
227 | |
228 | return 0; |
229 | } |
230 | |
231 | /* |
232 | * Install a perf counter breakpoint. |
233 | */ |
234 | int arch_install_hw_breakpoint(struct perf_event *bp) |
235 | { |
236 | return hw_breakpoint_control(bp, ops: HW_BREAKPOINT_INSTALL); |
237 | } |
238 | |
239 | void arch_uninstall_hw_breakpoint(struct perf_event *bp) |
240 | { |
241 | hw_breakpoint_control(bp, ops: HW_BREAKPOINT_UNINSTALL); |
242 | } |
243 | |
244 | static int get_hbp_len(u8 hbp_len) |
245 | { |
246 | unsigned int len_in_bytes = 0; |
247 | |
248 | switch (hbp_len) { |
249 | case LOONGARCH_BREAKPOINT_LEN_1: |
250 | len_in_bytes = 1; |
251 | break; |
252 | case LOONGARCH_BREAKPOINT_LEN_2: |
253 | len_in_bytes = 2; |
254 | break; |
255 | case LOONGARCH_BREAKPOINT_LEN_4: |
256 | len_in_bytes = 4; |
257 | break; |
258 | case LOONGARCH_BREAKPOINT_LEN_8: |
259 | len_in_bytes = 8; |
260 | break; |
261 | } |
262 | |
263 | return len_in_bytes; |
264 | } |
265 | |
266 | /* |
267 | * Check whether bp virtual address is in kernel space. |
268 | */ |
269 | int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) |
270 | { |
271 | unsigned int len; |
272 | unsigned long va; |
273 | |
274 | va = hw->address; |
275 | len = get_hbp_len(hbp_len: hw->ctrl.len); |
276 | |
277 | return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); |
278 | } |
279 | |
280 | /* |
281 | * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. |
282 | * Hopefully this will disappear when ptrace can bypass the conversion |
283 | * to generic breakpoint descriptions. |
284 | */ |
285 | int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, |
286 | int *gen_len, int *gen_type, int *offset) |
287 | { |
288 | /* Type */ |
289 | switch (ctrl.type) { |
290 | case LOONGARCH_BREAKPOINT_EXECUTE: |
291 | *gen_type = HW_BREAKPOINT_X; |
292 | break; |
293 | case LOONGARCH_BREAKPOINT_LOAD: |
294 | *gen_type = HW_BREAKPOINT_R; |
295 | break; |
296 | case LOONGARCH_BREAKPOINT_STORE: |
297 | *gen_type = HW_BREAKPOINT_W; |
298 | break; |
299 | case LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE: |
300 | *gen_type = HW_BREAKPOINT_RW; |
301 | break; |
302 | default: |
303 | return -EINVAL; |
304 | } |
305 | |
306 | if (!ctrl.len) |
307 | return -EINVAL; |
308 | |
309 | *offset = __ffs(ctrl.len); |
310 | |
311 | /* Len */ |
312 | switch (ctrl.len) { |
313 | case LOONGARCH_BREAKPOINT_LEN_1: |
314 | *gen_len = HW_BREAKPOINT_LEN_1; |
315 | break; |
316 | case LOONGARCH_BREAKPOINT_LEN_2: |
317 | *gen_len = HW_BREAKPOINT_LEN_2; |
318 | break; |
319 | case LOONGARCH_BREAKPOINT_LEN_4: |
320 | *gen_len = HW_BREAKPOINT_LEN_4; |
321 | break; |
322 | case LOONGARCH_BREAKPOINT_LEN_8: |
323 | *gen_len = HW_BREAKPOINT_LEN_8; |
324 | break; |
325 | default: |
326 | return -EINVAL; |
327 | } |
328 | |
329 | return 0; |
330 | } |
331 | |
332 | /* |
333 | * Construct an arch_hw_breakpoint from a perf_event. |
334 | */ |
335 | static int arch_build_bp_info(struct perf_event *bp, |
336 | const struct perf_event_attr *attr, |
337 | struct arch_hw_breakpoint *hw) |
338 | { |
339 | /* Type */ |
340 | switch (attr->bp_type) { |
341 | case HW_BREAKPOINT_X: |
342 | hw->ctrl.type = LOONGARCH_BREAKPOINT_EXECUTE; |
343 | break; |
344 | case HW_BREAKPOINT_R: |
345 | hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD; |
346 | break; |
347 | case HW_BREAKPOINT_W: |
348 | hw->ctrl.type = LOONGARCH_BREAKPOINT_STORE; |
349 | break; |
350 | case HW_BREAKPOINT_RW: |
351 | hw->ctrl.type = LOONGARCH_BREAKPOINT_LOAD | LOONGARCH_BREAKPOINT_STORE; |
352 | break; |
353 | default: |
354 | return -EINVAL; |
355 | } |
356 | |
357 | /* Len */ |
358 | switch (attr->bp_len) { |
359 | case HW_BREAKPOINT_LEN_1: |
360 | hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_1; |
361 | break; |
362 | case HW_BREAKPOINT_LEN_2: |
363 | hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_2; |
364 | break; |
365 | case HW_BREAKPOINT_LEN_4: |
366 | hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_4; |
367 | break; |
368 | case HW_BREAKPOINT_LEN_8: |
369 | hw->ctrl.len = LOONGARCH_BREAKPOINT_LEN_8; |
370 | break; |
371 | default: |
372 | return -EINVAL; |
373 | } |
374 | |
375 | /* Address */ |
376 | hw->address = attr->bp_addr; |
377 | |
378 | return 0; |
379 | } |
380 | |
381 | /* |
382 | * Validate the arch-specific HW Breakpoint register settings. |
383 | */ |
384 | int hw_breakpoint_arch_parse(struct perf_event *bp, |
385 | const struct perf_event_attr *attr, |
386 | struct arch_hw_breakpoint *hw) |
387 | { |
388 | int ret; |
389 | u64 alignment_mask, offset; |
390 | |
391 | /* Build the arch_hw_breakpoint. */ |
392 | ret = arch_build_bp_info(bp, attr, hw); |
393 | if (ret) |
394 | return ret; |
395 | |
396 | if (hw->ctrl.type != LOONGARCH_BREAKPOINT_EXECUTE) |
397 | alignment_mask = 0x7; |
398 | else |
399 | alignment_mask = 0x3; |
400 | offset = hw->address & alignment_mask; |
401 | |
402 | hw->address &= ~alignment_mask; |
403 | hw->ctrl.len <<= offset; |
404 | |
405 | return 0; |
406 | } |
407 | |
408 | static void update_bp_registers(struct pt_regs *regs, int enable, int type) |
409 | { |
410 | u32 ctrl; |
411 | int i, max_slots; |
412 | struct perf_event **slots; |
413 | struct arch_hw_breakpoint *info; |
414 | |
415 | switch (type) { |
416 | case 0: |
417 | slots = this_cpu_ptr(bp_on_reg); |
418 | max_slots = boot_cpu_data.watch_ireg_count; |
419 | break; |
420 | case 1: |
421 | slots = this_cpu_ptr(wp_on_reg); |
422 | max_slots = boot_cpu_data.watch_dreg_count; |
423 | break; |
424 | default: |
425 | return; |
426 | } |
427 | |
428 | for (i = 0; i < max_slots; ++i) { |
429 | if (!slots[i]) |
430 | continue; |
431 | |
432 | info = counter_arch_bp(bp: slots[i]); |
433 | if (enable) { |
434 | if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { |
435 | write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); |
436 | write_wb_reg(CSR_CFG_CTRL, i, 0, CTRL_PLV_ENABLE); |
437 | } else { |
438 | ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); |
439 | if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) |
440 | ctrl |= 0x1 << MWPnCFG3_LoadEn; |
441 | if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) |
442 | ctrl |= 0x1 << MWPnCFG3_StoreEn; |
443 | write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); |
444 | } |
445 | regs->csr_prmd |= CSR_PRMD_PWE; |
446 | } else { |
447 | if ((info->ctrl.type == LOONGARCH_BREAKPOINT_EXECUTE) && (type == 0)) { |
448 | write_wb_reg(CSR_CFG_CTRL, i, 0, 0); |
449 | } else { |
450 | ctrl = read_wb_reg(CSR_CFG_CTRL, i, 1); |
451 | if (info->ctrl.type == LOONGARCH_BREAKPOINT_LOAD) |
452 | ctrl &= ~0x1 << MWPnCFG3_LoadEn; |
453 | if (info->ctrl.type == LOONGARCH_BREAKPOINT_STORE) |
454 | ctrl &= ~0x1 << MWPnCFG3_StoreEn; |
455 | write_wb_reg(CSR_CFG_CTRL, i, 1, ctrl); |
456 | } |
457 | regs->csr_prmd &= ~CSR_PRMD_PWE; |
458 | } |
459 | } |
460 | } |
461 | NOKPROBE_SYMBOL(update_bp_registers); |
462 | |
463 | /* |
464 | * Debug exception handlers. |
465 | */ |
466 | void breakpoint_handler(struct pt_regs *regs) |
467 | { |
468 | int i; |
469 | struct perf_event *bp, **slots; |
470 | |
471 | slots = this_cpu_ptr(bp_on_reg); |
472 | |
473 | for (i = 0; i < boot_cpu_data.watch_ireg_count; ++i) { |
474 | bp = slots[i]; |
475 | if (bp == NULL) |
476 | continue; |
477 | perf_bp_event(event: bp, data: regs); |
478 | } |
479 | update_bp_registers(regs, enable: 0, type: 0); |
480 | } |
481 | NOKPROBE_SYMBOL(breakpoint_handler); |
482 | |
483 | void watchpoint_handler(struct pt_regs *regs) |
484 | { |
485 | int i; |
486 | struct perf_event *wp, **slots; |
487 | |
488 | slots = this_cpu_ptr(wp_on_reg); |
489 | |
490 | for (i = 0; i < boot_cpu_data.watch_dreg_count; ++i) { |
491 | wp = slots[i]; |
492 | if (wp == NULL) |
493 | continue; |
494 | perf_bp_event(event: wp, data: regs); |
495 | } |
496 | update_bp_registers(regs, enable: 0, type: 1); |
497 | } |
498 | NOKPROBE_SYMBOL(watchpoint_handler); |
499 | |
500 | static int __init arch_hw_breakpoint_init(void) |
501 | { |
502 | int cpu; |
503 | |
504 | boot_cpu_data.watch_ireg_count = get_num_brps(); |
505 | boot_cpu_data.watch_dreg_count = get_num_wrps(); |
506 | |
507 | pr_info("Found %d breakpoint and %d watchpoint registers.\n" , |
508 | boot_cpu_data.watch_ireg_count, boot_cpu_data.watch_dreg_count); |
509 | |
510 | for (cpu = 1; cpu < NR_CPUS; cpu++) { |
511 | cpu_data[cpu].watch_ireg_count = boot_cpu_data.watch_ireg_count; |
512 | cpu_data[cpu].watch_dreg_count = boot_cpu_data.watch_dreg_count; |
513 | } |
514 | |
515 | return 0; |
516 | } |
517 | arch_initcall(arch_hw_breakpoint_init); |
518 | |
519 | void hw_breakpoint_thread_switch(struct task_struct *next) |
520 | { |
521 | u64 addr, mask; |
522 | struct pt_regs *regs = task_pt_regs(next); |
523 | |
524 | if (test_tsk_thread_flag(tsk: next, TIF_SINGLESTEP)) { |
525 | addr = read_wb_reg(CSR_CFG_ADDR, 0, 0); |
526 | mask = read_wb_reg(CSR_CFG_MASK, 0, 0); |
527 | if (!((regs->csr_era ^ addr) & ~mask)) |
528 | csr_write32(CSR_FWPC_SKIP, LOONGARCH_CSR_FWPS); |
529 | regs->csr_prmd |= CSR_PRMD_PWE; |
530 | } else { |
531 | /* Update breakpoints */ |
532 | update_bp_registers(regs, enable: 1, type: 0); |
533 | /* Update watchpoints */ |
534 | update_bp_registers(regs, enable: 1, type: 1); |
535 | } |
536 | } |
537 | |
538 | void hw_breakpoint_pmu_read(struct perf_event *bp) |
539 | { |
540 | } |
541 | |
542 | /* |
543 | * Dummy function to register with die_notifier. |
544 | */ |
545 | int hw_breakpoint_exceptions_notify(struct notifier_block *unused, |
546 | unsigned long val, void *data) |
547 | { |
548 | return NOTIFY_DONE; |
549 | } |
550 | |