1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | // |
3 | // Linux performance counter support for ARC CPUs. |
4 | // This code is inspired by the perf support of various other architectures. |
5 | // |
6 | // Copyright (C) 2013-2018 Synopsys, Inc. (www.synopsys.com) |
7 | |
8 | #include <linux/errno.h> |
9 | #include <linux/interrupt.h> |
10 | #include <linux/module.h> |
11 | #include <linux/of.h> |
12 | #include <linux/perf_event.h> |
13 | #include <linux/platform_device.h> |
14 | #include <asm/arcregs.h> |
15 | #include <asm/stacktrace.h> |
16 | |
17 | /* HW holds 8 symbols + one for null terminator */ |
18 | #define ARCPMU_EVENT_NAME_LEN 9 |
19 | |
20 | /* |
21 | * Some ARC pct quirks: |
22 | * |
23 | * PERF_COUNT_HW_STALLED_CYCLES_BACKEND |
24 | * PERF_COUNT_HW_STALLED_CYCLES_FRONTEND |
25 | * The ARC 700 can either measure stalls per pipeline stage, or all stalls |
26 | * combined; for now we assign all stalls to STALLED_CYCLES_BACKEND |
27 | * and all pipeline flushes (e.g. caused by mispredicts, etc.) to |
28 | * STALLED_CYCLES_FRONTEND. |
29 | * |
30 | * We could start multiple performance counters and combine everything |
31 | * afterwards, but that makes it complicated. |
32 | * |
33 | * Note that I$ cache misses aren't counted by either of the two! |
34 | */ |
35 | |
36 | /* |
37 | * ARC PCT has hardware conditions with fixed "names" but variable "indexes" |
38 | * (based on a specific RTL build) |
39 | * Below is the static map between perf generic/arc specific event_id and |
40 | * h/w condition names. |
41 | * At the time of probe, we loop thru each index and find it's name to |
42 | * complete the mapping of perf event_id to h/w index as latter is needed |
43 | * to program the counter really |
44 | */ |
45 | static const char * const arc_pmu_ev_hw_map[] = { |
46 | /* count cycles */ |
47 | [PERF_COUNT_HW_CPU_CYCLES] = "crun" , |
48 | [PERF_COUNT_HW_REF_CPU_CYCLES] = "crun" , |
49 | [PERF_COUNT_HW_BUS_CYCLES] = "crun" , |
50 | |
51 | [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = "bflush" , |
52 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = "bstall" , |
53 | |
54 | /* counts condition */ |
55 | [PERF_COUNT_HW_INSTRUCTIONS] = "iall" , |
56 | /* All jump instructions that are taken */ |
57 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = "ijmptak" , |
58 | #ifdef CONFIG_ISA_ARCV2 |
59 | [PERF_COUNT_HW_BRANCH_MISSES] = "bpmp" , |
60 | #else |
61 | [PERF_COUNT_ARC_BPOK] = "bpok" , /* NP-NT, PT-T, PNT-NT */ |
62 | [PERF_COUNT_HW_BRANCH_MISSES] = "bpfail" , /* NP-T, PT-NT, PNT-T */ |
63 | #endif |
64 | [PERF_COUNT_ARC_LDC] = "imemrdc" , /* Instr: mem read cached */ |
65 | [PERF_COUNT_ARC_STC] = "imemwrc" , /* Instr: mem write cached */ |
66 | |
67 | [PERF_COUNT_ARC_DCLM] = "dclm" , /* D-cache Load Miss */ |
68 | [PERF_COUNT_ARC_DCSM] = "dcsm" , /* D-cache Store Miss */ |
69 | [PERF_COUNT_ARC_ICM] = "icm" , /* I-cache Miss */ |
70 | [PERF_COUNT_ARC_EDTLB] = "edtlb" , /* D-TLB Miss */ |
71 | [PERF_COUNT_ARC_EITLB] = "eitlb" , /* I-TLB Miss */ |
72 | |
73 | [PERF_COUNT_HW_CACHE_REFERENCES] = "imemrdc" , /* Instr: mem read cached */ |
74 | [PERF_COUNT_HW_CACHE_MISSES] = "dclm" , /* D-cache Load Miss */ |
75 | }; |
76 | |
77 | #define C(_x) PERF_COUNT_HW_CACHE_##_x |
78 | #define CACHE_OP_UNSUPPORTED 0xffff |
79 | |
80 | static const unsigned int arc_pmu_cache_map[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { |
81 | [C(L1D)] = { |
82 | [C(OP_READ)] = { |
83 | [C(RESULT_ACCESS)] = PERF_COUNT_ARC_LDC, |
84 | [C(RESULT_MISS)] = PERF_COUNT_ARC_DCLM, |
85 | }, |
86 | [C(OP_WRITE)] = { |
87 | [C(RESULT_ACCESS)] = PERF_COUNT_ARC_STC, |
88 | [C(RESULT_MISS)] = PERF_COUNT_ARC_DCSM, |
89 | }, |
90 | [C(OP_PREFETCH)] = { |
91 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
92 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
93 | }, |
94 | }, |
95 | [C(L1I)] = { |
96 | [C(OP_READ)] = { |
97 | [C(RESULT_ACCESS)] = PERF_COUNT_HW_INSTRUCTIONS, |
98 | [C(RESULT_MISS)] = PERF_COUNT_ARC_ICM, |
99 | }, |
100 | [C(OP_WRITE)] = { |
101 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
102 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
103 | }, |
104 | [C(OP_PREFETCH)] = { |
105 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
106 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
107 | }, |
108 | }, |
109 | [C(LL)] = { |
110 | [C(OP_READ)] = { |
111 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
112 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
113 | }, |
114 | [C(OP_WRITE)] = { |
115 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
116 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
117 | }, |
118 | [C(OP_PREFETCH)] = { |
119 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
120 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
121 | }, |
122 | }, |
123 | [C(DTLB)] = { |
124 | [C(OP_READ)] = { |
125 | [C(RESULT_ACCESS)] = PERF_COUNT_ARC_LDC, |
126 | [C(RESULT_MISS)] = PERF_COUNT_ARC_EDTLB, |
127 | }, |
128 | /* DTLB LD/ST Miss not segregated by h/w*/ |
129 | [C(OP_WRITE)] = { |
130 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
131 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
132 | }, |
133 | [C(OP_PREFETCH)] = { |
134 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
135 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
136 | }, |
137 | }, |
138 | [C(ITLB)] = { |
139 | [C(OP_READ)] = { |
140 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
141 | [C(RESULT_MISS)] = PERF_COUNT_ARC_EITLB, |
142 | }, |
143 | [C(OP_WRITE)] = { |
144 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
145 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
146 | }, |
147 | [C(OP_PREFETCH)] = { |
148 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
149 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
150 | }, |
151 | }, |
152 | [C(BPU)] = { |
153 | [C(OP_READ)] = { |
154 | [C(RESULT_ACCESS)] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS, |
155 | [C(RESULT_MISS)] = PERF_COUNT_HW_BRANCH_MISSES, |
156 | }, |
157 | [C(OP_WRITE)] = { |
158 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
159 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
160 | }, |
161 | [C(OP_PREFETCH)] = { |
162 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
163 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
164 | }, |
165 | }, |
166 | [C(NODE)] = { |
167 | [C(OP_READ)] = { |
168 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
169 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
170 | }, |
171 | [C(OP_WRITE)] = { |
172 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
173 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
174 | }, |
175 | [C(OP_PREFETCH)] = { |
176 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, |
177 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, |
178 | }, |
179 | }, |
180 | }; |
181 | |
182 | enum arc_pmu_attr_groups { |
183 | ARCPMU_ATTR_GR_EVENTS, |
184 | ARCPMU_ATTR_GR_FORMATS, |
185 | ARCPMU_NR_ATTR_GR |
186 | }; |
187 | |
188 | struct arc_pmu_raw_event_entry { |
189 | char name[ARCPMU_EVENT_NAME_LEN]; |
190 | }; |
191 | |
192 | struct arc_pmu { |
193 | struct pmu pmu; |
194 | unsigned int irq; |
195 | int n_counters; |
196 | int n_events; |
197 | u64 max_period; |
198 | int ev_hw_idx[PERF_COUNT_ARC_HW_MAX]; |
199 | |
200 | struct arc_pmu_raw_event_entry *raw_entry; |
201 | struct attribute **attrs; |
202 | struct perf_pmu_events_attr *attr; |
203 | const struct attribute_group *attr_groups[ARCPMU_NR_ATTR_GR + 1]; |
204 | }; |
205 | |
206 | struct arc_pmu_cpu { |
207 | /* |
208 | * A 1 bit for an index indicates that the counter is being used for |
209 | * an event. A 0 means that the counter can be used. |
210 | */ |
211 | unsigned long used_mask[BITS_TO_LONGS(ARC_PERF_MAX_COUNTERS)]; |
212 | |
213 | /* |
214 | * The events that are active on the PMU for the given index. |
215 | */ |
216 | struct perf_event *act_counter[ARC_PERF_MAX_COUNTERS]; |
217 | }; |
218 | |
219 | struct arc_callchain_trace { |
220 | int depth; |
221 | void *perf_stuff; |
222 | }; |
223 | |
224 | static int callchain_trace(unsigned int addr, void *data) |
225 | { |
226 | struct arc_callchain_trace *ctrl = data; |
227 | struct perf_callchain_entry_ctx *entry = ctrl->perf_stuff; |
228 | |
229 | perf_callchain_store(ctx: entry, ip: addr); |
230 | |
231 | if (ctrl->depth++ < 3) |
232 | return 0; |
233 | |
234 | return -1; |
235 | } |
236 | |
237 | void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, |
238 | struct pt_regs *regs) |
239 | { |
240 | struct arc_callchain_trace ctrl = { |
241 | .depth = 0, |
242 | .perf_stuff = entry, |
243 | }; |
244 | |
245 | arc_unwind_core(NULL, regs, callchain_trace, &ctrl); |
246 | } |
247 | |
248 | void perf_callchain_user(struct perf_callchain_entry_ctx *entry, |
249 | struct pt_regs *regs) |
250 | { |
251 | /* |
252 | * User stack can't be unwound trivially with kernel dwarf unwinder |
253 | * So for now just record the user PC |
254 | */ |
255 | perf_callchain_store(ctx: entry, ip: instruction_pointer(regs)); |
256 | } |
257 | |
258 | static struct arc_pmu *arc_pmu; |
259 | static DEFINE_PER_CPU(struct arc_pmu_cpu, arc_pmu_cpu); |
260 | |
261 | /* read counter #idx; note that counter# != event# on ARC! */ |
262 | static u64 arc_pmu_read_counter(int idx) |
263 | { |
264 | u32 tmp; |
265 | u64 result; |
266 | |
267 | /* |
268 | * ARC supports making 'snapshots' of the counters, so we don't |
269 | * need to care about counters wrapping to 0 underneath our feet |
270 | */ |
271 | write_aux_reg(ARC_REG_PCT_INDEX, idx); |
272 | tmp = read_aux_reg(ARC_REG_PCT_CONTROL); |
273 | write_aux_reg(ARC_REG_PCT_CONTROL, tmp | ARC_REG_PCT_CONTROL_SN); |
274 | result = (u64) (read_aux_reg(ARC_REG_PCT_SNAPH)) << 32; |
275 | result |= read_aux_reg(ARC_REG_PCT_SNAPL); |
276 | |
277 | return result; |
278 | } |
279 | |
280 | static void arc_perf_event_update(struct perf_event *event, |
281 | struct hw_perf_event *hwc, int idx) |
282 | { |
283 | u64 prev_raw_count = local64_read(&hwc->prev_count); |
284 | u64 new_raw_count = arc_pmu_read_counter(idx); |
285 | s64 delta = new_raw_count - prev_raw_count; |
286 | |
287 | /* |
288 | * We aren't afraid of hwc->prev_count changing beneath our feet |
289 | * because there's no way for us to re-enter this function anytime. |
290 | */ |
291 | local64_set(&hwc->prev_count, new_raw_count); |
292 | local64_add(delta, &event->count); |
293 | local64_sub(delta, &hwc->period_left); |
294 | } |
295 | |
296 | static void arc_pmu_read(struct perf_event *event) |
297 | { |
298 | arc_perf_event_update(event, hwc: &event->hw, idx: event->hw.idx); |
299 | } |
300 | |
301 | static int arc_pmu_cache_event(u64 config) |
302 | { |
303 | unsigned int cache_type, cache_op, cache_result; |
304 | int ret; |
305 | |
306 | cache_type = (config >> 0) & 0xff; |
307 | cache_op = (config >> 8) & 0xff; |
308 | cache_result = (config >> 16) & 0xff; |
309 | if (cache_type >= PERF_COUNT_HW_CACHE_MAX) |
310 | return -EINVAL; |
311 | if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) |
312 | return -EINVAL; |
313 | if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) |
314 | return -EINVAL; |
315 | |
316 | ret = arc_pmu_cache_map[cache_type][cache_op][cache_result]; |
317 | |
318 | if (ret == CACHE_OP_UNSUPPORTED) |
319 | return -ENOENT; |
320 | |
321 | pr_debug("init cache event: type/op/result %d/%d/%d with h/w %d \'%s\'\n" , |
322 | cache_type, cache_op, cache_result, ret, |
323 | arc_pmu_ev_hw_map[ret]); |
324 | |
325 | return ret; |
326 | } |
327 | |
328 | /* initializes hw_perf_event structure if event is supported */ |
329 | static int arc_pmu_event_init(struct perf_event *event) |
330 | { |
331 | struct hw_perf_event *hwc = &event->hw; |
332 | int ret; |
333 | |
334 | if (!is_sampling_event(event)) { |
335 | hwc->sample_period = arc_pmu->max_period; |
336 | hwc->last_period = hwc->sample_period; |
337 | local64_set(&hwc->period_left, hwc->sample_period); |
338 | } |
339 | |
340 | hwc->config = 0; |
341 | |
342 | if (is_isa_arcv2()) { |
343 | /* "exclude user" means "count only kernel" */ |
344 | if (event->attr.exclude_user) |
345 | hwc->config |= ARC_REG_PCT_CONFIG_KERN; |
346 | |
347 | /* "exclude kernel" means "count only user" */ |
348 | if (event->attr.exclude_kernel) |
349 | hwc->config |= ARC_REG_PCT_CONFIG_USER; |
350 | } |
351 | |
352 | switch (event->attr.type) { |
353 | case PERF_TYPE_HARDWARE: |
354 | if (event->attr.config >= PERF_COUNT_HW_MAX) |
355 | return -ENOENT; |
356 | if (arc_pmu->ev_hw_idx[event->attr.config] < 0) |
357 | return -ENOENT; |
358 | hwc->config |= arc_pmu->ev_hw_idx[event->attr.config]; |
359 | pr_debug("init event %d with h/w %08x \'%s\'\n" , |
360 | (int)event->attr.config, (int)hwc->config, |
361 | arc_pmu_ev_hw_map[event->attr.config]); |
362 | return 0; |
363 | |
364 | case PERF_TYPE_HW_CACHE: |
365 | ret = arc_pmu_cache_event(config: event->attr.config); |
366 | if (ret < 0) |
367 | return ret; |
368 | hwc->config |= arc_pmu->ev_hw_idx[ret]; |
369 | pr_debug("init cache event with h/w %08x \'%s\'\n" , |
370 | (int)hwc->config, arc_pmu_ev_hw_map[ret]); |
371 | return 0; |
372 | |
373 | case PERF_TYPE_RAW: |
374 | if (event->attr.config >= arc_pmu->n_events) |
375 | return -ENOENT; |
376 | |
377 | hwc->config |= event->attr.config; |
378 | pr_debug("init raw event with idx %lld \'%s\'\n" , |
379 | event->attr.config, |
380 | arc_pmu->raw_entry[event->attr.config].name); |
381 | |
382 | return 0; |
383 | |
384 | default: |
385 | return -ENOENT; |
386 | } |
387 | } |
388 | |
389 | /* starts all counters */ |
390 | static void arc_pmu_enable(struct pmu *pmu) |
391 | { |
392 | u32 tmp; |
393 | tmp = read_aux_reg(ARC_REG_PCT_CONTROL); |
394 | write_aux_reg(ARC_REG_PCT_CONTROL, (tmp & 0xffff0000) | 0x1); |
395 | } |
396 | |
397 | /* stops all counters */ |
398 | static void arc_pmu_disable(struct pmu *pmu) |
399 | { |
400 | u32 tmp; |
401 | tmp = read_aux_reg(ARC_REG_PCT_CONTROL); |
402 | write_aux_reg(ARC_REG_PCT_CONTROL, (tmp & 0xffff0000) | 0x0); |
403 | } |
404 | |
405 | static int arc_pmu_event_set_period(struct perf_event *event) |
406 | { |
407 | struct hw_perf_event *hwc = &event->hw; |
408 | s64 left = local64_read(&hwc->period_left); |
409 | s64 period = hwc->sample_period; |
410 | int idx = hwc->idx; |
411 | int overflow = 0; |
412 | u64 value; |
413 | |
414 | if (unlikely(left <= -period)) { |
415 | /* left underflowed by more than period. */ |
416 | left = period; |
417 | local64_set(&hwc->period_left, left); |
418 | hwc->last_period = period; |
419 | overflow = 1; |
420 | } else if (unlikely(left <= 0)) { |
421 | /* left underflowed by less than period. */ |
422 | left += period; |
423 | local64_set(&hwc->period_left, left); |
424 | hwc->last_period = period; |
425 | overflow = 1; |
426 | } |
427 | |
428 | if (left > arc_pmu->max_period) |
429 | left = arc_pmu->max_period; |
430 | |
431 | value = arc_pmu->max_period - left; |
432 | local64_set(&hwc->prev_count, value); |
433 | |
434 | /* Select counter */ |
435 | write_aux_reg(ARC_REG_PCT_INDEX, idx); |
436 | |
437 | /* Write value */ |
438 | write_aux_reg(ARC_REG_PCT_COUNTL, lower_32_bits(value)); |
439 | write_aux_reg(ARC_REG_PCT_COUNTH, upper_32_bits(value)); |
440 | |
441 | perf_event_update_userpage(event); |
442 | |
443 | return overflow; |
444 | } |
445 | |
446 | /* |
447 | * Assigns hardware counter to hardware condition. |
448 | * Note that there is no separate start/stop mechanism; |
449 | * stopping is achieved by assigning the 'never' condition |
450 | */ |
451 | static void arc_pmu_start(struct perf_event *event, int flags) |
452 | { |
453 | struct hw_perf_event *hwc = &event->hw; |
454 | int idx = hwc->idx; |
455 | |
456 | if (WARN_ON_ONCE(idx == -1)) |
457 | return; |
458 | |
459 | if (flags & PERF_EF_RELOAD) |
460 | WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
461 | |
462 | hwc->state = 0; |
463 | |
464 | arc_pmu_event_set_period(event); |
465 | |
466 | /* Enable interrupt for this counter */ |
467 | if (is_sampling_event(event)) |
468 | write_aux_reg(ARC_REG_PCT_INT_CTRL, |
469 | read_aux_reg(ARC_REG_PCT_INT_CTRL) | BIT(idx)); |
470 | |
471 | /* enable ARC pmu here */ |
472 | write_aux_reg(ARC_REG_PCT_INDEX, idx); /* counter # */ |
473 | write_aux_reg(ARC_REG_PCT_CONFIG, hwc->config); /* condition */ |
474 | } |
475 | |
476 | static void arc_pmu_stop(struct perf_event *event, int flags) |
477 | { |
478 | struct hw_perf_event *hwc = &event->hw; |
479 | int idx = hwc->idx; |
480 | |
481 | /* Disable interrupt for this counter */ |
482 | if (is_sampling_event(event)) { |
483 | /* |
484 | * Reset interrupt flag by writing of 1. This is required |
485 | * to make sure pending interrupt was not left. |
486 | */ |
487 | write_aux_reg(ARC_REG_PCT_INT_ACT, BIT(idx)); |
488 | write_aux_reg(ARC_REG_PCT_INT_CTRL, |
489 | read_aux_reg(ARC_REG_PCT_INT_CTRL) & ~BIT(idx)); |
490 | } |
491 | |
492 | if (!(event->hw.state & PERF_HES_STOPPED)) { |
493 | /* stop hw counter here */ |
494 | write_aux_reg(ARC_REG_PCT_INDEX, idx); |
495 | |
496 | /* condition code #0 is always "never" */ |
497 | write_aux_reg(ARC_REG_PCT_CONFIG, 0); |
498 | |
499 | event->hw.state |= PERF_HES_STOPPED; |
500 | } |
501 | |
502 | if ((flags & PERF_EF_UPDATE) && |
503 | !(event->hw.state & PERF_HES_UPTODATE)) { |
504 | arc_perf_event_update(event, hwc: &event->hw, idx); |
505 | event->hw.state |= PERF_HES_UPTODATE; |
506 | } |
507 | } |
508 | |
509 | static void arc_pmu_del(struct perf_event *event, int flags) |
510 | { |
511 | struct arc_pmu_cpu *pmu_cpu = this_cpu_ptr(&arc_pmu_cpu); |
512 | |
513 | arc_pmu_stop(event, PERF_EF_UPDATE); |
514 | __clear_bit(event->hw.idx, pmu_cpu->used_mask); |
515 | |
516 | pmu_cpu->act_counter[event->hw.idx] = 0; |
517 | |
518 | perf_event_update_userpage(event); |
519 | } |
520 | |
521 | /* allocate hardware counter and optionally start counting */ |
522 | static int arc_pmu_add(struct perf_event *event, int flags) |
523 | { |
524 | struct arc_pmu_cpu *pmu_cpu = this_cpu_ptr(&arc_pmu_cpu); |
525 | struct hw_perf_event *hwc = &event->hw; |
526 | int idx; |
527 | |
528 | idx = ffz(pmu_cpu->used_mask[0]); |
529 | if (idx == arc_pmu->n_counters) |
530 | return -EAGAIN; |
531 | |
532 | __set_bit(idx, pmu_cpu->used_mask); |
533 | hwc->idx = idx; |
534 | |
535 | write_aux_reg(ARC_REG_PCT_INDEX, idx); |
536 | |
537 | pmu_cpu->act_counter[idx] = event; |
538 | |
539 | if (is_sampling_event(event)) { |
540 | /* Mimic full counter overflow as other arches do */ |
541 | write_aux_reg(ARC_REG_PCT_INT_CNTL, |
542 | lower_32_bits(arc_pmu->max_period)); |
543 | write_aux_reg(ARC_REG_PCT_INT_CNTH, |
544 | upper_32_bits(arc_pmu->max_period)); |
545 | } |
546 | |
547 | write_aux_reg(ARC_REG_PCT_CONFIG, 0); |
548 | write_aux_reg(ARC_REG_PCT_COUNTL, 0); |
549 | write_aux_reg(ARC_REG_PCT_COUNTH, 0); |
550 | local64_set(&hwc->prev_count, 0); |
551 | |
552 | hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; |
553 | if (flags & PERF_EF_START) |
554 | arc_pmu_start(event, PERF_EF_RELOAD); |
555 | |
556 | perf_event_update_userpage(event); |
557 | |
558 | return 0; |
559 | } |
560 | |
561 | #ifdef CONFIG_ISA_ARCV2 |
562 | static irqreturn_t arc_pmu_intr(int irq, void *dev) |
563 | { |
564 | struct perf_sample_data data; |
565 | struct arc_pmu_cpu *pmu_cpu = this_cpu_ptr(&arc_pmu_cpu); |
566 | struct pt_regs *regs; |
567 | unsigned int active_ints; |
568 | int idx; |
569 | |
570 | arc_pmu_disable(&arc_pmu->pmu); |
571 | |
572 | active_ints = read_aux_reg(ARC_REG_PCT_INT_ACT); |
573 | if (!active_ints) |
574 | goto done; |
575 | |
576 | regs = get_irq_regs(); |
577 | |
578 | do { |
579 | struct perf_event *event; |
580 | struct hw_perf_event *hwc; |
581 | |
582 | idx = __ffs(active_ints); |
583 | |
584 | /* Reset interrupt flag by writing of 1 */ |
585 | write_aux_reg(ARC_REG_PCT_INT_ACT, BIT(idx)); |
586 | |
587 | /* |
588 | * On reset of "interrupt active" bit corresponding |
589 | * "interrupt enable" bit gets automatically reset as well. |
590 | * Now we need to re-enable interrupt for the counter. |
591 | */ |
592 | write_aux_reg(ARC_REG_PCT_INT_CTRL, |
593 | read_aux_reg(ARC_REG_PCT_INT_CTRL) | BIT(idx)); |
594 | |
595 | event = pmu_cpu->act_counter[idx]; |
596 | hwc = &event->hw; |
597 | |
598 | WARN_ON_ONCE(hwc->idx != idx); |
599 | |
600 | arc_perf_event_update(event, &event->hw, event->hw.idx); |
601 | perf_sample_data_init(&data, 0, hwc->last_period); |
602 | if (arc_pmu_event_set_period(event)) { |
603 | if (perf_event_overflow(event, &data, regs)) |
604 | arc_pmu_stop(event, 0); |
605 | } |
606 | |
607 | active_ints &= ~BIT(idx); |
608 | } while (active_ints); |
609 | |
610 | done: |
611 | arc_pmu_enable(&arc_pmu->pmu); |
612 | |
613 | return IRQ_HANDLED; |
614 | } |
615 | #else |
616 | |
617 | static irqreturn_t arc_pmu_intr(int irq, void *dev) |
618 | { |
619 | return IRQ_NONE; |
620 | } |
621 | |
622 | #endif /* CONFIG_ISA_ARCV2 */ |
623 | |
624 | static void arc_cpu_pmu_irq_init(void *data) |
625 | { |
626 | int irq = *(int *)data; |
627 | |
628 | enable_percpu_irq(irq, IRQ_TYPE_NONE); |
629 | |
630 | /* Clear all pending interrupt flags */ |
631 | write_aux_reg(ARC_REG_PCT_INT_ACT, 0xffffffff); |
632 | } |
633 | |
634 | /* Event field occupies the bottom 15 bits of our config field */ |
635 | PMU_FORMAT_ATTR(event, "config:0-14" ); |
636 | static struct attribute *arc_pmu_format_attrs[] = { |
637 | &format_attr_event.attr, |
638 | NULL, |
639 | }; |
640 | |
641 | static struct attribute_group arc_pmu_format_attr_gr = { |
642 | .name = "format" , |
643 | .attrs = arc_pmu_format_attrs, |
644 | }; |
645 | |
646 | static ssize_t arc_pmu_events_sysfs_show(struct device *dev, |
647 | struct device_attribute *attr, |
648 | char *page) |
649 | { |
650 | struct perf_pmu_events_attr *pmu_attr; |
651 | |
652 | pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); |
653 | return sprintf(buf: page, fmt: "event=0x%04llx\n" , pmu_attr->id); |
654 | } |
655 | |
656 | /* |
657 | * We don't add attrs here as we don't have pre-defined list of perf events. |
658 | * We will generate and add attrs dynamically in probe() after we read HW |
659 | * configuration. |
660 | */ |
661 | static struct attribute_group arc_pmu_events_attr_gr = { |
662 | .name = "events" , |
663 | }; |
664 | |
665 | static void arc_pmu_add_raw_event_attr(int j, char *str) |
666 | { |
667 | memmove(arc_pmu->raw_entry[j].name, str, ARCPMU_EVENT_NAME_LEN - 1); |
668 | arc_pmu->attr[j].attr.attr.name = arc_pmu->raw_entry[j].name; |
669 | arc_pmu->attr[j].attr.attr.mode = VERIFY_OCTAL_PERMISSIONS(0444); |
670 | arc_pmu->attr[j].attr.show = arc_pmu_events_sysfs_show; |
671 | arc_pmu->attr[j].id = j; |
672 | arc_pmu->attrs[j] = &(arc_pmu->attr[j].attr.attr); |
673 | } |
674 | |
675 | static int arc_pmu_raw_alloc(struct device *dev) |
676 | { |
677 | arc_pmu->attr = devm_kmalloc_array(dev, n: arc_pmu->n_events + 1, |
678 | size: sizeof(*arc_pmu->attr), GFP_KERNEL | __GFP_ZERO); |
679 | if (!arc_pmu->attr) |
680 | return -ENOMEM; |
681 | |
682 | arc_pmu->attrs = devm_kmalloc_array(dev, n: arc_pmu->n_events + 1, |
683 | size: sizeof(*arc_pmu->attrs), GFP_KERNEL | __GFP_ZERO); |
684 | if (!arc_pmu->attrs) |
685 | return -ENOMEM; |
686 | |
687 | arc_pmu->raw_entry = devm_kmalloc_array(dev, n: arc_pmu->n_events, |
688 | size: sizeof(*arc_pmu->raw_entry), GFP_KERNEL | __GFP_ZERO); |
689 | if (!arc_pmu->raw_entry) |
690 | return -ENOMEM; |
691 | |
692 | return 0; |
693 | } |
694 | |
695 | static inline bool event_in_hw_event_map(int i, char *name) |
696 | { |
697 | if (!arc_pmu_ev_hw_map[i]) |
698 | return false; |
699 | |
700 | if (!strlen(arc_pmu_ev_hw_map[i])) |
701 | return false; |
702 | |
703 | if (strcmp(arc_pmu_ev_hw_map[i], name)) |
704 | return false; |
705 | |
706 | return true; |
707 | } |
708 | |
709 | static void arc_pmu_map_hw_event(int j, char *str) |
710 | { |
711 | int i; |
712 | |
713 | /* See if HW condition has been mapped to a perf event_id */ |
714 | for (i = 0; i < ARRAY_SIZE(arc_pmu_ev_hw_map); i++) { |
715 | if (event_in_hw_event_map(i, str)) { |
716 | pr_debug("mapping perf event %2d to h/w event \'%8s\' (idx %d)\n" , |
717 | i, str, j); |
718 | arc_pmu->ev_hw_idx[i] = j; |
719 | } |
720 | } |
721 | } |
722 | |
723 | static int arc_pmu_device_probe(struct platform_device *pdev) |
724 | { |
725 | struct arc_reg_pct_build pct_bcr; |
726 | struct arc_reg_cc_build cc_bcr; |
727 | int i, has_interrupts, irq = -1; |
728 | int counter_size; /* in bits */ |
729 | |
730 | union cc_name { |
731 | struct { |
732 | u32 word0, word1; |
733 | char sentinel; |
734 | } indiv; |
735 | char str[ARCPMU_EVENT_NAME_LEN]; |
736 | } cc_name; |
737 | |
738 | |
739 | READ_BCR(ARC_REG_PCT_BUILD, pct_bcr); |
740 | if (!pct_bcr.v) { |
741 | pr_err("This core does not have performance counters!\n" ); |
742 | return -ENODEV; |
743 | } |
744 | BUILD_BUG_ON(ARC_PERF_MAX_COUNTERS > 32); |
745 | if (WARN_ON(pct_bcr.c > ARC_PERF_MAX_COUNTERS)) |
746 | return -EINVAL; |
747 | |
748 | READ_BCR(ARC_REG_CC_BUILD, cc_bcr); |
749 | if (WARN(!cc_bcr.v, "Counters exist but No countable conditions?" )) |
750 | return -EINVAL; |
751 | |
752 | arc_pmu = devm_kzalloc(dev: &pdev->dev, size: sizeof(struct arc_pmu), GFP_KERNEL); |
753 | if (!arc_pmu) |
754 | return -ENOMEM; |
755 | |
756 | arc_pmu->n_events = cc_bcr.c; |
757 | |
758 | if (arc_pmu_raw_alloc(dev: &pdev->dev)) |
759 | return -ENOMEM; |
760 | |
761 | has_interrupts = is_isa_arcv2() ? pct_bcr.i : 0; |
762 | |
763 | arc_pmu->n_counters = pct_bcr.c; |
764 | counter_size = 32 + (pct_bcr.s << 4); |
765 | |
766 | arc_pmu->max_period = (1ULL << counter_size) / 2 - 1ULL; |
767 | |
768 | pr_info("ARC perf\t: %d counters (%d bits), %d conditions%s\n" , |
769 | arc_pmu->n_counters, counter_size, cc_bcr.c, |
770 | has_interrupts ? ", [overflow IRQ support]" : "" ); |
771 | |
772 | cc_name.str[ARCPMU_EVENT_NAME_LEN - 1] = 0; |
773 | for (i = 0; i < PERF_COUNT_ARC_HW_MAX; i++) |
774 | arc_pmu->ev_hw_idx[i] = -1; |
775 | |
776 | /* loop thru all available h/w condition indexes */ |
777 | for (i = 0; i < cc_bcr.c; i++) { |
778 | write_aux_reg(ARC_REG_CC_INDEX, i); |
779 | cc_name.indiv.word0 = le32_to_cpu(read_aux_reg(ARC_REG_CC_NAME0)); |
780 | cc_name.indiv.word1 = le32_to_cpu(read_aux_reg(ARC_REG_CC_NAME1)); |
781 | |
782 | arc_pmu_map_hw_event(j: i, str: cc_name.str); |
783 | arc_pmu_add_raw_event_attr(j: i, str: cc_name.str); |
784 | } |
785 | |
786 | arc_pmu_events_attr_gr.attrs = arc_pmu->attrs; |
787 | arc_pmu->attr_groups[ARCPMU_ATTR_GR_EVENTS] = &arc_pmu_events_attr_gr; |
788 | arc_pmu->attr_groups[ARCPMU_ATTR_GR_FORMATS] = &arc_pmu_format_attr_gr; |
789 | |
790 | arc_pmu->pmu = (struct pmu) { |
791 | .pmu_enable = arc_pmu_enable, |
792 | .pmu_disable = arc_pmu_disable, |
793 | .event_init = arc_pmu_event_init, |
794 | .add = arc_pmu_add, |
795 | .del = arc_pmu_del, |
796 | .start = arc_pmu_start, |
797 | .stop = arc_pmu_stop, |
798 | .read = arc_pmu_read, |
799 | .attr_groups = arc_pmu->attr_groups, |
800 | }; |
801 | |
802 | if (has_interrupts) { |
803 | irq = platform_get_irq(pdev, 0); |
804 | if (irq >= 0) { |
805 | int ret; |
806 | |
807 | arc_pmu->irq = irq; |
808 | |
809 | /* intc map function ensures irq_set_percpu_devid() called */ |
810 | ret = request_percpu_irq(irq, handler: arc_pmu_intr, devname: "ARC perf counters" , |
811 | this_cpu_ptr(&arc_pmu_cpu)); |
812 | |
813 | if (!ret) |
814 | on_each_cpu(func: arc_cpu_pmu_irq_init, info: &irq, wait: 1); |
815 | else |
816 | irq = -1; |
817 | } |
818 | |
819 | } |
820 | |
821 | if (irq == -1) |
822 | arc_pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; |
823 | |
824 | /* |
825 | * perf parser doesn't really like '-' symbol in events name, so let's |
826 | * use '_' in arc pct name as it goes to kernel PMU event prefix. |
827 | */ |
828 | return perf_pmu_register(pmu: &arc_pmu->pmu, name: "arc_pct" , type: PERF_TYPE_RAW); |
829 | } |
830 | |
831 | static const struct of_device_id arc_pmu_match[] = { |
832 | { .compatible = "snps,arc700-pct" }, |
833 | { .compatible = "snps,archs-pct" }, |
834 | {}, |
835 | }; |
836 | MODULE_DEVICE_TABLE(of, arc_pmu_match); |
837 | |
838 | static struct platform_driver arc_pmu_driver = { |
839 | .driver = { |
840 | .name = "arc-pct" , |
841 | .of_match_table = of_match_ptr(arc_pmu_match), |
842 | }, |
843 | .probe = arc_pmu_device_probe, |
844 | }; |
845 | |
846 | module_platform_driver(arc_pmu_driver); |
847 | |
848 | MODULE_LICENSE("GPL" ); |
849 | MODULE_AUTHOR("Mischa Jonker <mjonker@synopsys.com>" ); |
850 | MODULE_DESCRIPTION("ARC PMU driver" ); |
851 | |