1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * ARMv8 PMUv3 Performance Events handling code. |
4 | * |
5 | * Copyright (C) 2012 ARM Limited |
6 | * Author: Will Deacon <will.deacon@arm.com> |
7 | * |
8 | * This code is based heavily on the ARMv7 perf event code. |
9 | */ |
10 | |
11 | #include <asm/irq_regs.h> |
12 | #include <asm/perf_event.h> |
13 | #include <asm/virt.h> |
14 | |
15 | #include <clocksource/arm_arch_timer.h> |
16 | |
17 | #include <linux/acpi.h> |
18 | #include <linux/clocksource.h> |
19 | #include <linux/of.h> |
20 | #include <linux/perf/arm_pmu.h> |
21 | #include <linux/perf/arm_pmuv3.h> |
22 | #include <linux/platform_device.h> |
23 | #include <linux/sched_clock.h> |
24 | #include <linux/smp.h> |
25 | #include <linux/nmi.h> |
26 | |
27 | #include <asm/arm_pmuv3.h> |
28 | |
29 | /* ARMv8 Cortex-A53 specific event types. */ |
30 | #define ARMV8_A53_PERFCTR_PREF_LINEFILL 0xC2 |
31 | |
32 | /* ARMv8 Cavium ThunderX specific event types. */ |
33 | #define ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST 0xE9 |
34 | #define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS 0xEA |
35 | #define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS 0xEB |
36 | #define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS 0xEC |
37 | #define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS 0xED |
38 | |
39 | /* |
40 | * ARMv8 Architectural defined events, not all of these may |
41 | * be supported on any given implementation. Unsupported events will |
42 | * be disabled at run-time based on the PMCEID registers. |
43 | */ |
44 | static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = { |
45 | PERF_MAP_ALL_UNSUPPORTED, |
46 | [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CPU_CYCLES, |
47 | [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INST_RETIRED, |
48 | [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, |
49 | [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, |
50 | [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, |
51 | [PERF_COUNT_HW_BUS_CYCLES] = ARMV8_PMUV3_PERFCTR_BUS_CYCLES, |
52 | [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND, |
53 | [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = ARMV8_PMUV3_PERFCTR_STALL_BACKEND, |
54 | }; |
55 | |
56 | static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
57 | [PERF_COUNT_HW_CACHE_OP_MAX] |
58 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
59 | PERF_CACHE_MAP_ALL_UNSUPPORTED, |
60 | |
61 | [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE, |
62 | [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL, |
63 | |
64 | [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE, |
65 | [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL, |
66 | |
67 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL, |
68 | [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB, |
69 | |
70 | [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL, |
71 | [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB, |
72 | |
73 | [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD, |
74 | [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_LL_CACHE_RD, |
75 | |
76 | [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED, |
77 | [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED, |
78 | }; |
79 | |
80 | static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
81 | [PERF_COUNT_HW_CACHE_OP_MAX] |
82 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
83 | PERF_CACHE_MAP_ALL_UNSUPPORTED, |
84 | |
85 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL, |
86 | |
87 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, |
88 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, |
89 | }; |
90 | |
91 | static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
92 | [PERF_COUNT_HW_CACHE_OP_MAX] |
93 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
94 | PERF_CACHE_MAP_ALL_UNSUPPORTED, |
95 | |
96 | [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD, |
97 | [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD, |
98 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, |
99 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR, |
100 | |
101 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, |
102 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, |
103 | |
104 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, |
105 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, |
106 | }; |
107 | |
108 | static const unsigned armv8_a73_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
109 | [PERF_COUNT_HW_CACHE_OP_MAX] |
110 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
111 | PERF_CACHE_MAP_ALL_UNSUPPORTED, |
112 | |
113 | [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD, |
114 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, |
115 | }; |
116 | |
117 | static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
118 | [PERF_COUNT_HW_CACHE_OP_MAX] |
119 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
120 | PERF_CACHE_MAP_ALL_UNSUPPORTED, |
121 | |
122 | [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD, |
123 | [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD, |
124 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, |
125 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST, |
126 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS, |
127 | [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS, |
128 | |
129 | [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS, |
130 | [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS, |
131 | |
132 | [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD, |
133 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, |
134 | [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR, |
135 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, |
136 | }; |
137 | |
138 | static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] |
139 | [PERF_COUNT_HW_CACHE_OP_MAX] |
140 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { |
141 | PERF_CACHE_MAP_ALL_UNSUPPORTED, |
142 | |
143 | [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD, |
144 | [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD, |
145 | [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR, |
146 | [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR, |
147 | |
148 | [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD, |
149 | [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR, |
150 | [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD, |
151 | [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR, |
152 | |
153 | [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD, |
154 | [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR, |
155 | }; |
156 | |
157 | static ssize_t |
158 | armv8pmu_events_sysfs_show(struct device *dev, |
159 | struct device_attribute *attr, char *page) |
160 | { |
161 | struct perf_pmu_events_attr *pmu_attr; |
162 | |
163 | pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); |
164 | |
165 | return sprintf(buf: page, fmt: "event=0x%04llx\n" , pmu_attr->id); |
166 | } |
167 | |
168 | #define ARMV8_EVENT_ATTR(name, config) \ |
169 | PMU_EVENT_ATTR_ID(name, armv8pmu_events_sysfs_show, config) |
170 | |
171 | static struct attribute *armv8_pmuv3_event_attrs[] = { |
172 | ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR), |
173 | ARMV8_EVENT_ATTR(l1i_cache_refill, ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL), |
174 | ARMV8_EVENT_ATTR(l1i_tlb_refill, ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL), |
175 | ARMV8_EVENT_ATTR(l1d_cache_refill, ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL), |
176 | ARMV8_EVENT_ATTR(l1d_cache, ARMV8_PMUV3_PERFCTR_L1D_CACHE), |
177 | ARMV8_EVENT_ATTR(l1d_tlb_refill, ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL), |
178 | ARMV8_EVENT_ATTR(ld_retired, ARMV8_PMUV3_PERFCTR_LD_RETIRED), |
179 | ARMV8_EVENT_ATTR(st_retired, ARMV8_PMUV3_PERFCTR_ST_RETIRED), |
180 | ARMV8_EVENT_ATTR(inst_retired, ARMV8_PMUV3_PERFCTR_INST_RETIRED), |
181 | ARMV8_EVENT_ATTR(exc_taken, ARMV8_PMUV3_PERFCTR_EXC_TAKEN), |
182 | ARMV8_EVENT_ATTR(exc_return, ARMV8_PMUV3_PERFCTR_EXC_RETURN), |
183 | ARMV8_EVENT_ATTR(cid_write_retired, ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED), |
184 | ARMV8_EVENT_ATTR(pc_write_retired, ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED), |
185 | ARMV8_EVENT_ATTR(br_immed_retired, ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED), |
186 | ARMV8_EVENT_ATTR(br_return_retired, ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED), |
187 | ARMV8_EVENT_ATTR(unaligned_ldst_retired, ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED), |
188 | ARMV8_EVENT_ATTR(br_mis_pred, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED), |
189 | ARMV8_EVENT_ATTR(cpu_cycles, ARMV8_PMUV3_PERFCTR_CPU_CYCLES), |
190 | ARMV8_EVENT_ATTR(br_pred, ARMV8_PMUV3_PERFCTR_BR_PRED), |
191 | ARMV8_EVENT_ATTR(mem_access, ARMV8_PMUV3_PERFCTR_MEM_ACCESS), |
192 | ARMV8_EVENT_ATTR(l1i_cache, ARMV8_PMUV3_PERFCTR_L1I_CACHE), |
193 | ARMV8_EVENT_ATTR(l1d_cache_wb, ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB), |
194 | ARMV8_EVENT_ATTR(l2d_cache, ARMV8_PMUV3_PERFCTR_L2D_CACHE), |
195 | ARMV8_EVENT_ATTR(l2d_cache_refill, ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL), |
196 | ARMV8_EVENT_ATTR(l2d_cache_wb, ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB), |
197 | ARMV8_EVENT_ATTR(bus_access, ARMV8_PMUV3_PERFCTR_BUS_ACCESS), |
198 | ARMV8_EVENT_ATTR(memory_error, ARMV8_PMUV3_PERFCTR_MEMORY_ERROR), |
199 | ARMV8_EVENT_ATTR(inst_spec, ARMV8_PMUV3_PERFCTR_INST_SPEC), |
200 | ARMV8_EVENT_ATTR(ttbr_write_retired, ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED), |
201 | ARMV8_EVENT_ATTR(bus_cycles, ARMV8_PMUV3_PERFCTR_BUS_CYCLES), |
202 | /* Don't expose the chain event in /sys, since it's useless in isolation */ |
203 | ARMV8_EVENT_ATTR(l1d_cache_allocate, ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE), |
204 | ARMV8_EVENT_ATTR(l2d_cache_allocate, ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE), |
205 | ARMV8_EVENT_ATTR(br_retired, ARMV8_PMUV3_PERFCTR_BR_RETIRED), |
206 | ARMV8_EVENT_ATTR(br_mis_pred_retired, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED), |
207 | ARMV8_EVENT_ATTR(stall_frontend, ARMV8_PMUV3_PERFCTR_STALL_FRONTEND), |
208 | ARMV8_EVENT_ATTR(stall_backend, ARMV8_PMUV3_PERFCTR_STALL_BACKEND), |
209 | ARMV8_EVENT_ATTR(l1d_tlb, ARMV8_PMUV3_PERFCTR_L1D_TLB), |
210 | ARMV8_EVENT_ATTR(l1i_tlb, ARMV8_PMUV3_PERFCTR_L1I_TLB), |
211 | ARMV8_EVENT_ATTR(l2i_cache, ARMV8_PMUV3_PERFCTR_L2I_CACHE), |
212 | ARMV8_EVENT_ATTR(l2i_cache_refill, ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL), |
213 | ARMV8_EVENT_ATTR(l3d_cache_allocate, ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE), |
214 | ARMV8_EVENT_ATTR(l3d_cache_refill, ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL), |
215 | ARMV8_EVENT_ATTR(l3d_cache, ARMV8_PMUV3_PERFCTR_L3D_CACHE), |
216 | ARMV8_EVENT_ATTR(l3d_cache_wb, ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB), |
217 | ARMV8_EVENT_ATTR(l2d_tlb_refill, ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL), |
218 | ARMV8_EVENT_ATTR(l2i_tlb_refill, ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL), |
219 | ARMV8_EVENT_ATTR(l2d_tlb, ARMV8_PMUV3_PERFCTR_L2D_TLB), |
220 | ARMV8_EVENT_ATTR(l2i_tlb, ARMV8_PMUV3_PERFCTR_L2I_TLB), |
221 | ARMV8_EVENT_ATTR(remote_access, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS), |
222 | ARMV8_EVENT_ATTR(ll_cache, ARMV8_PMUV3_PERFCTR_LL_CACHE), |
223 | ARMV8_EVENT_ATTR(ll_cache_miss, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS), |
224 | ARMV8_EVENT_ATTR(dtlb_walk, ARMV8_PMUV3_PERFCTR_DTLB_WALK), |
225 | ARMV8_EVENT_ATTR(itlb_walk, ARMV8_PMUV3_PERFCTR_ITLB_WALK), |
226 | ARMV8_EVENT_ATTR(ll_cache_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_RD), |
227 | ARMV8_EVENT_ATTR(ll_cache_miss_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD), |
228 | ARMV8_EVENT_ATTR(remote_access_rd, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS_RD), |
229 | ARMV8_EVENT_ATTR(l1d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L1D_CACHE_LMISS_RD), |
230 | ARMV8_EVENT_ATTR(op_retired, ARMV8_PMUV3_PERFCTR_OP_RETIRED), |
231 | ARMV8_EVENT_ATTR(op_spec, ARMV8_PMUV3_PERFCTR_OP_SPEC), |
232 | ARMV8_EVENT_ATTR(stall, ARMV8_PMUV3_PERFCTR_STALL), |
233 | ARMV8_EVENT_ATTR(stall_slot_backend, ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND), |
234 | ARMV8_EVENT_ATTR(stall_slot_frontend, ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND), |
235 | ARMV8_EVENT_ATTR(stall_slot, ARMV8_PMUV3_PERFCTR_STALL_SLOT), |
236 | ARMV8_EVENT_ATTR(sample_pop, ARMV8_SPE_PERFCTR_SAMPLE_POP), |
237 | ARMV8_EVENT_ATTR(sample_feed, ARMV8_SPE_PERFCTR_SAMPLE_FEED), |
238 | ARMV8_EVENT_ATTR(sample_filtrate, ARMV8_SPE_PERFCTR_SAMPLE_FILTRATE), |
239 | ARMV8_EVENT_ATTR(sample_collision, ARMV8_SPE_PERFCTR_SAMPLE_COLLISION), |
240 | ARMV8_EVENT_ATTR(cnt_cycles, ARMV8_AMU_PERFCTR_CNT_CYCLES), |
241 | ARMV8_EVENT_ATTR(stall_backend_mem, ARMV8_AMU_PERFCTR_STALL_BACKEND_MEM), |
242 | ARMV8_EVENT_ATTR(l1i_cache_lmiss, ARMV8_PMUV3_PERFCTR_L1I_CACHE_LMISS), |
243 | ARMV8_EVENT_ATTR(l2d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L2D_CACHE_LMISS_RD), |
244 | ARMV8_EVENT_ATTR(l2i_cache_lmiss, ARMV8_PMUV3_PERFCTR_L2I_CACHE_LMISS), |
245 | ARMV8_EVENT_ATTR(l3d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L3D_CACHE_LMISS_RD), |
246 | ARMV8_EVENT_ATTR(trb_wrap, ARMV8_PMUV3_PERFCTR_TRB_WRAP), |
247 | ARMV8_EVENT_ATTR(trb_trig, ARMV8_PMUV3_PERFCTR_TRB_TRIG), |
248 | ARMV8_EVENT_ATTR(trcextout0, ARMV8_PMUV3_PERFCTR_TRCEXTOUT0), |
249 | ARMV8_EVENT_ATTR(trcextout1, ARMV8_PMUV3_PERFCTR_TRCEXTOUT1), |
250 | ARMV8_EVENT_ATTR(trcextout2, ARMV8_PMUV3_PERFCTR_TRCEXTOUT2), |
251 | ARMV8_EVENT_ATTR(trcextout3, ARMV8_PMUV3_PERFCTR_TRCEXTOUT3), |
252 | ARMV8_EVENT_ATTR(cti_trigout4, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT4), |
253 | ARMV8_EVENT_ATTR(cti_trigout5, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT5), |
254 | ARMV8_EVENT_ATTR(cti_trigout6, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT6), |
255 | ARMV8_EVENT_ATTR(cti_trigout7, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT7), |
256 | ARMV8_EVENT_ATTR(ldst_align_lat, ARMV8_PMUV3_PERFCTR_LDST_ALIGN_LAT), |
257 | ARMV8_EVENT_ATTR(ld_align_lat, ARMV8_PMUV3_PERFCTR_LD_ALIGN_LAT), |
258 | ARMV8_EVENT_ATTR(st_align_lat, ARMV8_PMUV3_PERFCTR_ST_ALIGN_LAT), |
259 | ARMV8_EVENT_ATTR(mem_access_checked, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED), |
260 | ARMV8_EVENT_ATTR(mem_access_checked_rd, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED_RD), |
261 | ARMV8_EVENT_ATTR(mem_access_checked_wr, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED_WR), |
262 | NULL, |
263 | }; |
264 | |
265 | static umode_t |
266 | armv8pmu_event_attr_is_visible(struct kobject *kobj, |
267 | struct attribute *attr, int unused) |
268 | { |
269 | struct device *dev = kobj_to_dev(kobj); |
270 | struct pmu *pmu = dev_get_drvdata(dev); |
271 | struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); |
272 | struct perf_pmu_events_attr *pmu_attr; |
273 | |
274 | pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr); |
275 | |
276 | if (pmu_attr->id < ARMV8_PMUV3_MAX_COMMON_EVENTS && |
277 | test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap)) |
278 | return attr->mode; |
279 | |
280 | if (pmu_attr->id >= ARMV8_PMUV3_EXT_COMMON_EVENT_BASE) { |
281 | u64 id = pmu_attr->id - ARMV8_PMUV3_EXT_COMMON_EVENT_BASE; |
282 | |
283 | if (id < ARMV8_PMUV3_MAX_COMMON_EVENTS && |
284 | test_bit(id, cpu_pmu->pmceid_ext_bitmap)) |
285 | return attr->mode; |
286 | } |
287 | |
288 | return 0; |
289 | } |
290 | |
291 | static const struct attribute_group armv8_pmuv3_events_attr_group = { |
292 | .name = "events" , |
293 | .attrs = armv8_pmuv3_event_attrs, |
294 | .is_visible = armv8pmu_event_attr_is_visible, |
295 | }; |
296 | |
297 | PMU_FORMAT_ATTR(event, "config:0-15" ); |
298 | PMU_FORMAT_ATTR(long, "config1:0" ); |
299 | PMU_FORMAT_ATTR(rdpmc, "config1:1" ); |
300 | |
301 | static int sysctl_perf_user_access __read_mostly; |
302 | |
303 | static inline bool armv8pmu_event_is_64bit(struct perf_event *event) |
304 | { |
305 | return event->attr.config1 & 0x1; |
306 | } |
307 | |
308 | static inline bool armv8pmu_event_want_user_access(struct perf_event *event) |
309 | { |
310 | return event->attr.config1 & 0x2; |
311 | } |
312 | |
313 | static struct attribute *armv8_pmuv3_format_attrs[] = { |
314 | &format_attr_event.attr, |
315 | &format_attr_long.attr, |
316 | &format_attr_rdpmc.attr, |
317 | NULL, |
318 | }; |
319 | |
320 | static const struct attribute_group armv8_pmuv3_format_attr_group = { |
321 | .name = "format" , |
322 | .attrs = armv8_pmuv3_format_attrs, |
323 | }; |
324 | |
325 | static ssize_t slots_show(struct device *dev, struct device_attribute *attr, |
326 | char *page) |
327 | { |
328 | struct pmu *pmu = dev_get_drvdata(dev); |
329 | struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); |
330 | u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK; |
331 | |
332 | return sysfs_emit(buf: page, fmt: "0x%08x\n" , slots); |
333 | } |
334 | |
335 | static DEVICE_ATTR_RO(slots); |
336 | |
337 | static ssize_t bus_slots_show(struct device *dev, struct device_attribute *attr, |
338 | char *page) |
339 | { |
340 | struct pmu *pmu = dev_get_drvdata(dev); |
341 | struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); |
342 | u32 bus_slots = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_SLOTS_SHIFT) |
343 | & ARMV8_PMU_BUS_SLOTS_MASK; |
344 | |
345 | return sysfs_emit(buf: page, fmt: "0x%08x\n" , bus_slots); |
346 | } |
347 | |
348 | static DEVICE_ATTR_RO(bus_slots); |
349 | |
350 | static ssize_t bus_width_show(struct device *dev, struct device_attribute *attr, |
351 | char *page) |
352 | { |
353 | struct pmu *pmu = dev_get_drvdata(dev); |
354 | struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); |
355 | u32 bus_width = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_WIDTH_SHIFT) |
356 | & ARMV8_PMU_BUS_WIDTH_MASK; |
357 | u32 val = 0; |
358 | |
359 | /* Encoded as Log2(number of bytes), plus one */ |
360 | if (bus_width > 2 && bus_width < 13) |
361 | val = 1 << (bus_width - 1); |
362 | |
363 | return sysfs_emit(buf: page, fmt: "0x%08x\n" , val); |
364 | } |
365 | |
366 | static DEVICE_ATTR_RO(bus_width); |
367 | |
368 | static struct attribute *armv8_pmuv3_caps_attrs[] = { |
369 | &dev_attr_slots.attr, |
370 | &dev_attr_bus_slots.attr, |
371 | &dev_attr_bus_width.attr, |
372 | NULL, |
373 | }; |
374 | |
375 | static const struct attribute_group armv8_pmuv3_caps_attr_group = { |
376 | .name = "caps" , |
377 | .attrs = armv8_pmuv3_caps_attrs, |
378 | }; |
379 | |
380 | /* |
381 | * Perf Events' indices |
382 | */ |
383 | #define ARMV8_IDX_CYCLE_COUNTER 0 |
384 | #define ARMV8_IDX_COUNTER0 1 |
385 | #define ARMV8_IDX_CYCLE_COUNTER_USER 32 |
386 | |
387 | /* |
388 | * We unconditionally enable ARMv8.5-PMU long event counter support |
389 | * (64-bit events) where supported. Indicate if this arm_pmu has long |
390 | * event counter support. |
391 | * |
392 | * On AArch32, long counters make no sense (you can't access the top |
393 | * bits), so we only enable this on AArch64. |
394 | */ |
395 | static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu) |
396 | { |
397 | return (IS_ENABLED(CONFIG_ARM64) && is_pmuv3p5(cpu_pmu->pmuver)); |
398 | } |
399 | |
400 | static inline bool armv8pmu_event_has_user_read(struct perf_event *event) |
401 | { |
402 | return event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT; |
403 | } |
404 | |
405 | /* |
406 | * We must chain two programmable counters for 64 bit events, |
407 | * except when we have allocated the 64bit cycle counter (for CPU |
408 | * cycles event) or when user space counter access is enabled. |
409 | */ |
410 | static inline bool armv8pmu_event_is_chained(struct perf_event *event) |
411 | { |
412 | int idx = event->hw.idx; |
413 | struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); |
414 | |
415 | return !armv8pmu_event_has_user_read(event) && |
416 | armv8pmu_event_is_64bit(event) && |
417 | !armv8pmu_has_long_event(cpu_pmu) && |
418 | (idx != ARMV8_IDX_CYCLE_COUNTER); |
419 | } |
420 | |
421 | /* |
422 | * ARMv8 low level PMU access |
423 | */ |
424 | |
425 | /* |
426 | * Perf Event to low level counters mapping |
427 | */ |
428 | #define ARMV8_IDX_TO_COUNTER(x) \ |
429 | (((x) - ARMV8_IDX_COUNTER0) & ARMV8_PMU_COUNTER_MASK) |
430 | |
431 | static inline u32 armv8pmu_pmcr_read(void) |
432 | { |
433 | return read_pmcr(); |
434 | } |
435 | |
436 | static inline void armv8pmu_pmcr_write(u32 val) |
437 | { |
438 | val &= ARMV8_PMU_PMCR_MASK; |
439 | isb(); |
440 | write_pmcr(val); |
441 | } |
442 | |
443 | static inline int armv8pmu_has_overflowed(u32 pmovsr) |
444 | { |
445 | return pmovsr & ARMV8_PMU_OVERFLOWED_MASK; |
446 | } |
447 | |
448 | static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx) |
449 | { |
450 | return pmnc & BIT(ARMV8_IDX_TO_COUNTER(idx)); |
451 | } |
452 | |
453 | static inline u64 armv8pmu_read_evcntr(int idx) |
454 | { |
455 | u32 counter = ARMV8_IDX_TO_COUNTER(idx); |
456 | |
457 | return read_pmevcntrn(counter); |
458 | } |
459 | |
460 | static inline u64 armv8pmu_read_hw_counter(struct perf_event *event) |
461 | { |
462 | int idx = event->hw.idx; |
463 | u64 val = armv8pmu_read_evcntr(idx); |
464 | |
465 | if (armv8pmu_event_is_chained(event)) |
466 | val = (val << 32) | armv8pmu_read_evcntr(idx: idx - 1); |
467 | return val; |
468 | } |
469 | |
470 | /* |
471 | * The cycle counter is always a 64-bit counter. When ARMV8_PMU_PMCR_LP |
472 | * is set the event counters also become 64-bit counters. Unless the |
473 | * user has requested a long counter (attr.config1) then we want to |
474 | * interrupt upon 32-bit overflow - we achieve this by applying a bias. |
475 | */ |
476 | static bool armv8pmu_event_needs_bias(struct perf_event *event) |
477 | { |
478 | struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); |
479 | struct hw_perf_event *hwc = &event->hw; |
480 | int idx = hwc->idx; |
481 | |
482 | if (armv8pmu_event_is_64bit(event)) |
483 | return false; |
484 | |
485 | if (armv8pmu_has_long_event(cpu_pmu) || |
486 | idx == ARMV8_IDX_CYCLE_COUNTER) |
487 | return true; |
488 | |
489 | return false; |
490 | } |
491 | |
492 | static u64 armv8pmu_bias_long_counter(struct perf_event *event, u64 value) |
493 | { |
494 | if (armv8pmu_event_needs_bias(event)) |
495 | value |= GENMASK_ULL(63, 32); |
496 | |
497 | return value; |
498 | } |
499 | |
500 | static u64 armv8pmu_unbias_long_counter(struct perf_event *event, u64 value) |
501 | { |
502 | if (armv8pmu_event_needs_bias(event)) |
503 | value &= ~GENMASK_ULL(63, 32); |
504 | |
505 | return value; |
506 | } |
507 | |
508 | static u64 armv8pmu_read_counter(struct perf_event *event) |
509 | { |
510 | struct hw_perf_event *hwc = &event->hw; |
511 | int idx = hwc->idx; |
512 | u64 value; |
513 | |
514 | if (idx == ARMV8_IDX_CYCLE_COUNTER) |
515 | value = read_pmccntr(); |
516 | else |
517 | value = armv8pmu_read_hw_counter(event); |
518 | |
519 | return armv8pmu_unbias_long_counter(event, value); |
520 | } |
521 | |
522 | static inline void armv8pmu_write_evcntr(int idx, u64 value) |
523 | { |
524 | u32 counter = ARMV8_IDX_TO_COUNTER(idx); |
525 | |
526 | write_pmevcntrn(counter, value); |
527 | } |
528 | |
529 | static inline void armv8pmu_write_hw_counter(struct perf_event *event, |
530 | u64 value) |
531 | { |
532 | int idx = event->hw.idx; |
533 | |
534 | if (armv8pmu_event_is_chained(event)) { |
535 | armv8pmu_write_evcntr(idx, upper_32_bits(value)); |
536 | armv8pmu_write_evcntr(idx: idx - 1, lower_32_bits(value)); |
537 | } else { |
538 | armv8pmu_write_evcntr(idx, value); |
539 | } |
540 | } |
541 | |
542 | static void armv8pmu_write_counter(struct perf_event *event, u64 value) |
543 | { |
544 | struct hw_perf_event *hwc = &event->hw; |
545 | int idx = hwc->idx; |
546 | |
547 | value = armv8pmu_bias_long_counter(event, value); |
548 | |
549 | if (idx == ARMV8_IDX_CYCLE_COUNTER) |
550 | write_pmccntr(value); |
551 | else |
552 | armv8pmu_write_hw_counter(event, value); |
553 | } |
554 | |
555 | static inline void armv8pmu_write_evtype(int idx, u32 val) |
556 | { |
557 | u32 counter = ARMV8_IDX_TO_COUNTER(idx); |
558 | |
559 | val &= ARMV8_PMU_EVTYPE_MASK; |
560 | write_pmevtypern(counter, val); |
561 | } |
562 | |
563 | static inline void armv8pmu_write_event_type(struct perf_event *event) |
564 | { |
565 | struct hw_perf_event *hwc = &event->hw; |
566 | int idx = hwc->idx; |
567 | |
568 | /* |
569 | * For chained events, the low counter is programmed to count |
570 | * the event of interest and the high counter is programmed |
571 | * with CHAIN event code with filters set to count at all ELs. |
572 | */ |
573 | if (armv8pmu_event_is_chained(event)) { |
574 | u32 chain_evt = ARMV8_PMUV3_PERFCTR_CHAIN | |
575 | ARMV8_PMU_INCLUDE_EL2; |
576 | |
577 | armv8pmu_write_evtype(idx: idx - 1, val: hwc->config_base); |
578 | armv8pmu_write_evtype(idx, val: chain_evt); |
579 | } else { |
580 | if (idx == ARMV8_IDX_CYCLE_COUNTER) |
581 | write_pmccfiltr(hwc->config_base); |
582 | else |
583 | armv8pmu_write_evtype(idx, val: hwc->config_base); |
584 | } |
585 | } |
586 | |
587 | static u32 armv8pmu_event_cnten_mask(struct perf_event *event) |
588 | { |
589 | int counter = ARMV8_IDX_TO_COUNTER(event->hw.idx); |
590 | u32 mask = BIT(counter); |
591 | |
592 | if (armv8pmu_event_is_chained(event)) |
593 | mask |= BIT(counter - 1); |
594 | return mask; |
595 | } |
596 | |
597 | static inline void armv8pmu_enable_counter(u32 mask) |
598 | { |
599 | /* |
600 | * Make sure event configuration register writes are visible before we |
601 | * enable the counter. |
602 | * */ |
603 | isb(); |
604 | write_pmcntenset(mask); |
605 | } |
606 | |
607 | static inline void armv8pmu_enable_event_counter(struct perf_event *event) |
608 | { |
609 | struct perf_event_attr *attr = &event->attr; |
610 | u32 mask = armv8pmu_event_cnten_mask(event); |
611 | |
612 | kvm_set_pmu_events(mask, attr); |
613 | |
614 | /* We rely on the hypervisor switch code to enable guest counters */ |
615 | if (!kvm_pmu_counter_deferred(attr)) |
616 | armv8pmu_enable_counter(mask); |
617 | } |
618 | |
619 | static inline void armv8pmu_disable_counter(u32 mask) |
620 | { |
621 | write_pmcntenclr(mask); |
622 | /* |
623 | * Make sure the effects of disabling the counter are visible before we |
624 | * start configuring the event. |
625 | */ |
626 | isb(); |
627 | } |
628 | |
629 | static inline void armv8pmu_disable_event_counter(struct perf_event *event) |
630 | { |
631 | struct perf_event_attr *attr = &event->attr; |
632 | u32 mask = armv8pmu_event_cnten_mask(event); |
633 | |
634 | kvm_clr_pmu_events(mask); |
635 | |
636 | /* We rely on the hypervisor switch code to disable guest counters */ |
637 | if (!kvm_pmu_counter_deferred(attr)) |
638 | armv8pmu_disable_counter(mask); |
639 | } |
640 | |
641 | static inline void armv8pmu_enable_intens(u32 mask) |
642 | { |
643 | write_pmintenset(mask); |
644 | } |
645 | |
646 | static inline void armv8pmu_enable_event_irq(struct perf_event *event) |
647 | { |
648 | u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx); |
649 | armv8pmu_enable_intens(BIT(counter)); |
650 | } |
651 | |
652 | static inline void armv8pmu_disable_intens(u32 mask) |
653 | { |
654 | write_pmintenclr(mask); |
655 | isb(); |
656 | /* Clear the overflow flag in case an interrupt is pending. */ |
657 | write_pmovsclr(mask); |
658 | isb(); |
659 | } |
660 | |
661 | static inline void armv8pmu_disable_event_irq(struct perf_event *event) |
662 | { |
663 | u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx); |
664 | armv8pmu_disable_intens(BIT(counter)); |
665 | } |
666 | |
667 | static inline u32 armv8pmu_getreset_flags(void) |
668 | { |
669 | u32 value; |
670 | |
671 | /* Read */ |
672 | value = read_pmovsclr(); |
673 | |
674 | /* Write to clear flags */ |
675 | value &= ARMV8_PMU_OVSR_MASK; |
676 | write_pmovsclr(value); |
677 | |
678 | return value; |
679 | } |
680 | |
681 | static void update_pmuserenr(u64 val) |
682 | { |
683 | lockdep_assert_irqs_disabled(); |
684 | |
685 | /* |
686 | * The current PMUSERENR_EL0 value might be the value for the guest. |
687 | * If that's the case, have KVM keep tracking of the register value |
688 | * for the host EL0 so that KVM can restore it before returning to |
689 | * the host EL0. Otherwise, update the register now. |
690 | */ |
691 | if (kvm_set_pmuserenr(val)) |
692 | return; |
693 | |
694 | write_pmuserenr(val); |
695 | } |
696 | |
697 | static void armv8pmu_disable_user_access(void) |
698 | { |
699 | update_pmuserenr(val: 0); |
700 | } |
701 | |
702 | static void armv8pmu_enable_user_access(struct arm_pmu *cpu_pmu) |
703 | { |
704 | int i; |
705 | struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); |
706 | |
707 | /* Clear any unused counters to avoid leaking their contents */ |
708 | for_each_clear_bit(i, cpuc->used_mask, cpu_pmu->num_events) { |
709 | if (i == ARMV8_IDX_CYCLE_COUNTER) |
710 | write_pmccntr(0); |
711 | else |
712 | armv8pmu_write_evcntr(idx: i, value: 0); |
713 | } |
714 | |
715 | update_pmuserenr(ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR); |
716 | } |
717 | |
718 | static void armv8pmu_enable_event(struct perf_event *event) |
719 | { |
720 | /* |
721 | * Enable counter and interrupt, and set the counter to count |
722 | * the event that we're interested in. |
723 | */ |
724 | armv8pmu_disable_event_counter(event); |
725 | armv8pmu_write_event_type(event); |
726 | armv8pmu_enable_event_irq(event); |
727 | armv8pmu_enable_event_counter(event); |
728 | } |
729 | |
730 | static void armv8pmu_disable_event(struct perf_event *event) |
731 | { |
732 | armv8pmu_disable_event_counter(event); |
733 | armv8pmu_disable_event_irq(event); |
734 | } |
735 | |
736 | static void armv8pmu_start(struct arm_pmu *cpu_pmu) |
737 | { |
738 | struct perf_event_context *ctx; |
739 | int nr_user = 0; |
740 | |
741 | ctx = perf_cpu_task_ctx(); |
742 | if (ctx) |
743 | nr_user = ctx->nr_user; |
744 | |
745 | if (sysctl_perf_user_access && nr_user) |
746 | armv8pmu_enable_user_access(cpu_pmu); |
747 | else |
748 | armv8pmu_disable_user_access(); |
749 | |
750 | /* Enable all counters */ |
751 | armv8pmu_pmcr_write(val: armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E); |
752 | |
753 | kvm_vcpu_pmu_resync_el0(); |
754 | } |
755 | |
756 | static void armv8pmu_stop(struct arm_pmu *cpu_pmu) |
757 | { |
758 | /* Disable all counters */ |
759 | armv8pmu_pmcr_write(val: armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E); |
760 | } |
761 | |
762 | static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu) |
763 | { |
764 | u32 pmovsr; |
765 | struct perf_sample_data data; |
766 | struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); |
767 | struct pt_regs *regs; |
768 | int idx; |
769 | |
770 | /* |
771 | * Get and reset the IRQ flags |
772 | */ |
773 | pmovsr = armv8pmu_getreset_flags(); |
774 | |
775 | /* |
776 | * Did an overflow occur? |
777 | */ |
778 | if (!armv8pmu_has_overflowed(pmovsr)) |
779 | return IRQ_NONE; |
780 | |
781 | /* |
782 | * Handle the counter(s) overflow(s) |
783 | */ |
784 | regs = get_irq_regs(); |
785 | |
786 | /* |
787 | * Stop the PMU while processing the counter overflows |
788 | * to prevent skews in group events. |
789 | */ |
790 | armv8pmu_stop(cpu_pmu); |
791 | for (idx = 0; idx < cpu_pmu->num_events; ++idx) { |
792 | struct perf_event *event = cpuc->events[idx]; |
793 | struct hw_perf_event *hwc; |
794 | |
795 | /* Ignore if we don't have an event. */ |
796 | if (!event) |
797 | continue; |
798 | |
799 | /* |
800 | * We have a single interrupt for all counters. Check that |
801 | * each counter has overflowed before we process it. |
802 | */ |
803 | if (!armv8pmu_counter_has_overflowed(pmnc: pmovsr, idx)) |
804 | continue; |
805 | |
806 | hwc = &event->hw; |
807 | armpmu_event_update(event); |
808 | perf_sample_data_init(data: &data, addr: 0, period: hwc->last_period); |
809 | if (!armpmu_event_set_period(event)) |
810 | continue; |
811 | |
812 | /* |
813 | * Perf event overflow will queue the processing of the event as |
814 | * an irq_work which will be taken care of in the handling of |
815 | * IPI_IRQ_WORK. |
816 | */ |
817 | if (perf_event_overflow(event, data: &data, regs)) |
818 | cpu_pmu->disable(event); |
819 | } |
820 | armv8pmu_start(cpu_pmu); |
821 | |
822 | return IRQ_HANDLED; |
823 | } |
824 | |
825 | static int armv8pmu_get_single_idx(struct pmu_hw_events *cpuc, |
826 | struct arm_pmu *cpu_pmu) |
827 | { |
828 | int idx; |
829 | |
830 | for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; idx++) { |
831 | if (!test_and_set_bit(nr: idx, addr: cpuc->used_mask)) |
832 | return idx; |
833 | } |
834 | return -EAGAIN; |
835 | } |
836 | |
837 | static int armv8pmu_get_chain_idx(struct pmu_hw_events *cpuc, |
838 | struct arm_pmu *cpu_pmu) |
839 | { |
840 | int idx; |
841 | |
842 | /* |
843 | * Chaining requires two consecutive event counters, where |
844 | * the lower idx must be even. |
845 | */ |
846 | for (idx = ARMV8_IDX_COUNTER0 + 1; idx < cpu_pmu->num_events; idx += 2) { |
847 | if (!test_and_set_bit(nr: idx, addr: cpuc->used_mask)) { |
848 | /* Check if the preceding even counter is available */ |
849 | if (!test_and_set_bit(nr: idx - 1, addr: cpuc->used_mask)) |
850 | return idx; |
851 | /* Release the Odd counter */ |
852 | clear_bit(nr: idx, addr: cpuc->used_mask); |
853 | } |
854 | } |
855 | return -EAGAIN; |
856 | } |
857 | |
858 | static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, |
859 | struct perf_event *event) |
860 | { |
861 | struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); |
862 | struct hw_perf_event *hwc = &event->hw; |
863 | unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT; |
864 | |
865 | /* Always prefer to place a cycle counter into the cycle counter. */ |
866 | if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) { |
867 | if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, addr: cpuc->used_mask)) |
868 | return ARMV8_IDX_CYCLE_COUNTER; |
869 | else if (armv8pmu_event_is_64bit(event) && |
870 | armv8pmu_event_want_user_access(event) && |
871 | !armv8pmu_has_long_event(cpu_pmu)) |
872 | return -EAGAIN; |
873 | } |
874 | |
875 | /* |
876 | * Otherwise use events counters |
877 | */ |
878 | if (armv8pmu_event_is_chained(event)) |
879 | return armv8pmu_get_chain_idx(cpuc, cpu_pmu); |
880 | else |
881 | return armv8pmu_get_single_idx(cpuc, cpu_pmu); |
882 | } |
883 | |
884 | static void armv8pmu_clear_event_idx(struct pmu_hw_events *cpuc, |
885 | struct perf_event *event) |
886 | { |
887 | int idx = event->hw.idx; |
888 | |
889 | clear_bit(nr: idx, addr: cpuc->used_mask); |
890 | if (armv8pmu_event_is_chained(event)) |
891 | clear_bit(nr: idx - 1, addr: cpuc->used_mask); |
892 | } |
893 | |
894 | static int armv8pmu_user_event_idx(struct perf_event *event) |
895 | { |
896 | if (!sysctl_perf_user_access || !armv8pmu_event_has_user_read(event)) |
897 | return 0; |
898 | |
899 | /* |
900 | * We remap the cycle counter index to 32 to |
901 | * match the offset applied to the rest of |
902 | * the counter indices. |
903 | */ |
904 | if (event->hw.idx == ARMV8_IDX_CYCLE_COUNTER) |
905 | return ARMV8_IDX_CYCLE_COUNTER_USER; |
906 | |
907 | return event->hw.idx; |
908 | } |
909 | |
910 | /* |
911 | * Add an event filter to a given event. |
912 | */ |
913 | static int armv8pmu_set_event_filter(struct hw_perf_event *event, |
914 | struct perf_event_attr *attr) |
915 | { |
916 | unsigned long config_base = 0; |
917 | |
918 | if (attr->exclude_idle) |
919 | return -EPERM; |
920 | |
921 | /* |
922 | * If we're running in hyp mode, then we *are* the hypervisor. |
923 | * Therefore we ignore exclude_hv in this configuration, since |
924 | * there's no hypervisor to sample anyway. This is consistent |
925 | * with other architectures (x86 and Power). |
926 | */ |
927 | if (is_kernel_in_hyp_mode()) { |
928 | if (!attr->exclude_kernel && !attr->exclude_host) |
929 | config_base |= ARMV8_PMU_INCLUDE_EL2; |
930 | if (attr->exclude_guest) |
931 | config_base |= ARMV8_PMU_EXCLUDE_EL1; |
932 | if (attr->exclude_host) |
933 | config_base |= ARMV8_PMU_EXCLUDE_EL0; |
934 | } else { |
935 | if (!attr->exclude_hv && !attr->exclude_host) |
936 | config_base |= ARMV8_PMU_INCLUDE_EL2; |
937 | } |
938 | |
939 | /* |
940 | * Filter out !VHE kernels and guest kernels |
941 | */ |
942 | if (attr->exclude_kernel) |
943 | config_base |= ARMV8_PMU_EXCLUDE_EL1; |
944 | |
945 | if (attr->exclude_user) |
946 | config_base |= ARMV8_PMU_EXCLUDE_EL0; |
947 | |
948 | /* |
949 | * Install the filter into config_base as this is used to |
950 | * construct the event type. |
951 | */ |
952 | event->config_base = config_base; |
953 | |
954 | return 0; |
955 | } |
956 | |
957 | static void armv8pmu_reset(void *info) |
958 | { |
959 | struct arm_pmu *cpu_pmu = (struct arm_pmu *)info; |
960 | u32 pmcr; |
961 | |
962 | /* The counter and interrupt enable registers are unknown at reset. */ |
963 | armv8pmu_disable_counter(U32_MAX); |
964 | armv8pmu_disable_intens(U32_MAX); |
965 | |
966 | /* Clear the counters we flip at guest entry/exit */ |
967 | kvm_clr_pmu_events(U32_MAX); |
968 | |
969 | /* |
970 | * Initialize & Reset PMNC. Request overflow interrupt for |
971 | * 64 bit cycle counter but cheat in armv8pmu_write_counter(). |
972 | */ |
973 | pmcr = ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C | ARMV8_PMU_PMCR_LC; |
974 | |
975 | /* Enable long event counter support where available */ |
976 | if (armv8pmu_has_long_event(cpu_pmu)) |
977 | pmcr |= ARMV8_PMU_PMCR_LP; |
978 | |
979 | armv8pmu_pmcr_write(val: pmcr); |
980 | } |
981 | |
982 | static int __armv8_pmuv3_map_event_id(struct arm_pmu *armpmu, |
983 | struct perf_event *event) |
984 | { |
985 | if (event->attr.type == PERF_TYPE_HARDWARE && |
986 | event->attr.config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) { |
987 | |
988 | if (test_bit(ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED, |
989 | armpmu->pmceid_bitmap)) |
990 | return ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED; |
991 | |
992 | if (test_bit(ARMV8_PMUV3_PERFCTR_BR_RETIRED, |
993 | armpmu->pmceid_bitmap)) |
994 | return ARMV8_PMUV3_PERFCTR_BR_RETIRED; |
995 | |
996 | return HW_OP_UNSUPPORTED; |
997 | } |
998 | |
999 | return armpmu_map_event(event, &armv8_pmuv3_perf_map, |
1000 | &armv8_pmuv3_perf_cache_map, |
1001 | ARMV8_PMU_EVTYPE_EVENT); |
1002 | } |
1003 | |
1004 | static int __armv8_pmuv3_map_event(struct perf_event *event, |
1005 | const unsigned (*) |
1006 | [PERF_COUNT_HW_MAX], |
1007 | const unsigned (*) |
1008 | [PERF_COUNT_HW_CACHE_MAX] |
1009 | [PERF_COUNT_HW_CACHE_OP_MAX] |
1010 | [PERF_COUNT_HW_CACHE_RESULT_MAX]) |
1011 | { |
1012 | int hw_event_id; |
1013 | struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
1014 | |
1015 | hw_event_id = __armv8_pmuv3_map_event_id(armpmu, event); |
1016 | |
1017 | /* |
1018 | * CHAIN events only work when paired with an adjacent counter, and it |
1019 | * never makes sense for a user to open one in isolation, as they'll be |
1020 | * rotated arbitrarily. |
1021 | */ |
1022 | if (hw_event_id == ARMV8_PMUV3_PERFCTR_CHAIN) |
1023 | return -EINVAL; |
1024 | |
1025 | if (armv8pmu_event_is_64bit(event)) |
1026 | event->hw.flags |= ARMPMU_EVT_64BIT; |
1027 | |
1028 | /* |
1029 | * User events must be allocated into a single counter, and so |
1030 | * must not be chained. |
1031 | * |
1032 | * Most 64-bit events require long counter support, but 64-bit |
1033 | * CPU_CYCLES events can be placed into the dedicated cycle |
1034 | * counter when this is free. |
1035 | */ |
1036 | if (armv8pmu_event_want_user_access(event)) { |
1037 | if (!(event->attach_state & PERF_ATTACH_TASK)) |
1038 | return -EINVAL; |
1039 | if (armv8pmu_event_is_64bit(event) && |
1040 | (hw_event_id != ARMV8_PMUV3_PERFCTR_CPU_CYCLES) && |
1041 | !armv8pmu_has_long_event(cpu_pmu: armpmu)) |
1042 | return -EOPNOTSUPP; |
1043 | |
1044 | event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; |
1045 | } |
1046 | |
1047 | /* Only expose micro/arch events supported by this PMU */ |
1048 | if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS) |
1049 | && test_bit(hw_event_id, armpmu->pmceid_bitmap)) { |
1050 | return hw_event_id; |
1051 | } |
1052 | |
1053 | return armpmu_map_event(event, extra_event_map, extra_cache_map, |
1054 | ARMV8_PMU_EVTYPE_EVENT); |
1055 | } |
1056 | |
1057 | static int armv8_pmuv3_map_event(struct perf_event *event) |
1058 | { |
1059 | return __armv8_pmuv3_map_event(event, NULL, NULL); |
1060 | } |
1061 | |
1062 | static int armv8_a53_map_event(struct perf_event *event) |
1063 | { |
1064 | return __armv8_pmuv3_map_event(event, NULL, extra_cache_map: &armv8_a53_perf_cache_map); |
1065 | } |
1066 | |
1067 | static int armv8_a57_map_event(struct perf_event *event) |
1068 | { |
1069 | return __armv8_pmuv3_map_event(event, NULL, extra_cache_map: &armv8_a57_perf_cache_map); |
1070 | } |
1071 | |
1072 | static int armv8_a73_map_event(struct perf_event *event) |
1073 | { |
1074 | return __armv8_pmuv3_map_event(event, NULL, extra_cache_map: &armv8_a73_perf_cache_map); |
1075 | } |
1076 | |
1077 | static int armv8_thunder_map_event(struct perf_event *event) |
1078 | { |
1079 | return __armv8_pmuv3_map_event(event, NULL, |
1080 | extra_cache_map: &armv8_thunder_perf_cache_map); |
1081 | } |
1082 | |
1083 | static int armv8_vulcan_map_event(struct perf_event *event) |
1084 | { |
1085 | return __armv8_pmuv3_map_event(event, NULL, |
1086 | extra_cache_map: &armv8_vulcan_perf_cache_map); |
1087 | } |
1088 | |
1089 | struct armv8pmu_probe_info { |
1090 | struct arm_pmu *pmu; |
1091 | bool present; |
1092 | }; |
1093 | |
1094 | static void __armv8pmu_probe_pmu(void *info) |
1095 | { |
1096 | struct armv8pmu_probe_info *probe = info; |
1097 | struct arm_pmu *cpu_pmu = probe->pmu; |
1098 | u64 pmceid_raw[2]; |
1099 | u32 pmceid[2]; |
1100 | int pmuver; |
1101 | |
1102 | pmuver = read_pmuver(); |
1103 | if (!pmuv3_implemented(pmuver)) |
1104 | return; |
1105 | |
1106 | cpu_pmu->pmuver = pmuver; |
1107 | probe->present = true; |
1108 | |
1109 | /* Read the nb of CNTx counters supported from PMNC */ |
1110 | cpu_pmu->num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT) |
1111 | & ARMV8_PMU_PMCR_N_MASK; |
1112 | |
1113 | /* Add the CPU cycles counter */ |
1114 | cpu_pmu->num_events += 1; |
1115 | |
1116 | pmceid[0] = pmceid_raw[0] = read_pmceid0(); |
1117 | pmceid[1] = pmceid_raw[1] = read_pmceid1(); |
1118 | |
1119 | bitmap_from_arr32(cpu_pmu->pmceid_bitmap, |
1120 | pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS); |
1121 | |
1122 | pmceid[0] = pmceid_raw[0] >> 32; |
1123 | pmceid[1] = pmceid_raw[1] >> 32; |
1124 | |
1125 | bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap, |
1126 | pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS); |
1127 | |
1128 | /* store PMMIR register for sysfs */ |
1129 | if (is_pmuv3p4(pmuver)) |
1130 | cpu_pmu->reg_pmmir = read_pmmir(); |
1131 | else |
1132 | cpu_pmu->reg_pmmir = 0; |
1133 | } |
1134 | |
1135 | static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu) |
1136 | { |
1137 | struct armv8pmu_probe_info probe = { |
1138 | .pmu = cpu_pmu, |
1139 | .present = false, |
1140 | }; |
1141 | int ret; |
1142 | |
1143 | ret = smp_call_function_any(mask: &cpu_pmu->supported_cpus, |
1144 | func: __armv8pmu_probe_pmu, |
1145 | info: &probe, wait: 1); |
1146 | if (ret) |
1147 | return ret; |
1148 | |
1149 | return probe.present ? 0 : -ENODEV; |
1150 | } |
1151 | |
1152 | static void armv8pmu_disable_user_access_ipi(void *unused) |
1153 | { |
1154 | armv8pmu_disable_user_access(); |
1155 | } |
1156 | |
1157 | static int armv8pmu_proc_user_access_handler(struct ctl_table *table, int write, |
1158 | void *buffer, size_t *lenp, loff_t *ppos) |
1159 | { |
1160 | int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); |
1161 | if (ret || !write || sysctl_perf_user_access) |
1162 | return ret; |
1163 | |
1164 | on_each_cpu(func: armv8pmu_disable_user_access_ipi, NULL, wait: 1); |
1165 | return 0; |
1166 | } |
1167 | |
1168 | static struct ctl_table armv8_pmu_sysctl_table[] = { |
1169 | { |
1170 | .procname = "perf_user_access" , |
1171 | .data = &sysctl_perf_user_access, |
1172 | .maxlen = sizeof(unsigned int), |
1173 | .mode = 0644, |
1174 | .proc_handler = armv8pmu_proc_user_access_handler, |
1175 | .extra1 = SYSCTL_ZERO, |
1176 | .extra2 = SYSCTL_ONE, |
1177 | }, |
1178 | }; |
1179 | |
1180 | static void armv8_pmu_register_sysctl_table(void) |
1181 | { |
1182 | static u32 tbl_registered = 0; |
1183 | |
1184 | if (!cmpxchg_relaxed(&tbl_registered, 0, 1)) |
1185 | register_sysctl("kernel" , armv8_pmu_sysctl_table); |
1186 | } |
1187 | |
1188 | static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name, |
1189 | int (*map_event)(struct perf_event *event)) |
1190 | { |
1191 | int ret = armv8pmu_probe_pmu(cpu_pmu); |
1192 | if (ret) |
1193 | return ret; |
1194 | |
1195 | cpu_pmu->handle_irq = armv8pmu_handle_irq; |
1196 | cpu_pmu->enable = armv8pmu_enable_event; |
1197 | cpu_pmu->disable = armv8pmu_disable_event; |
1198 | cpu_pmu->read_counter = armv8pmu_read_counter; |
1199 | cpu_pmu->write_counter = armv8pmu_write_counter; |
1200 | cpu_pmu->get_event_idx = armv8pmu_get_event_idx; |
1201 | cpu_pmu->clear_event_idx = armv8pmu_clear_event_idx; |
1202 | cpu_pmu->start = armv8pmu_start; |
1203 | cpu_pmu->stop = armv8pmu_stop; |
1204 | cpu_pmu->reset = armv8pmu_reset; |
1205 | cpu_pmu->set_event_filter = armv8pmu_set_event_filter; |
1206 | |
1207 | cpu_pmu->pmu.event_idx = armv8pmu_user_event_idx; |
1208 | |
1209 | cpu_pmu->name = name; |
1210 | cpu_pmu->map_event = map_event; |
1211 | cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &armv8_pmuv3_events_attr_group; |
1212 | cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &armv8_pmuv3_format_attr_group; |
1213 | cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = &armv8_pmuv3_caps_attr_group; |
1214 | armv8_pmu_register_sysctl_table(); |
1215 | return 0; |
1216 | } |
1217 | |
1218 | #define PMUV3_INIT_SIMPLE(name) \ |
1219 | static int name##_pmu_init(struct arm_pmu *cpu_pmu) \ |
1220 | { \ |
1221 | return armv8_pmu_init(cpu_pmu, #name, armv8_pmuv3_map_event); \ |
1222 | } |
1223 | |
1224 | PMUV3_INIT_SIMPLE(armv8_pmuv3) |
1225 | |
1226 | PMUV3_INIT_SIMPLE(armv8_cortex_a34) |
1227 | PMUV3_INIT_SIMPLE(armv8_cortex_a55) |
1228 | PMUV3_INIT_SIMPLE(armv8_cortex_a65) |
1229 | PMUV3_INIT_SIMPLE(armv8_cortex_a75) |
1230 | PMUV3_INIT_SIMPLE(armv8_cortex_a76) |
1231 | PMUV3_INIT_SIMPLE(armv8_cortex_a77) |
1232 | PMUV3_INIT_SIMPLE(armv8_cortex_a78) |
1233 | PMUV3_INIT_SIMPLE(armv9_cortex_a510) |
1234 | PMUV3_INIT_SIMPLE(armv9_cortex_a520) |
1235 | PMUV3_INIT_SIMPLE(armv9_cortex_a710) |
1236 | PMUV3_INIT_SIMPLE(armv9_cortex_a715) |
1237 | PMUV3_INIT_SIMPLE(armv9_cortex_a720) |
1238 | PMUV3_INIT_SIMPLE(armv8_cortex_x1) |
1239 | PMUV3_INIT_SIMPLE(armv9_cortex_x2) |
1240 | PMUV3_INIT_SIMPLE(armv9_cortex_x3) |
1241 | PMUV3_INIT_SIMPLE(armv9_cortex_x4) |
1242 | PMUV3_INIT_SIMPLE(armv8_neoverse_e1) |
1243 | PMUV3_INIT_SIMPLE(armv8_neoverse_n1) |
1244 | PMUV3_INIT_SIMPLE(armv9_neoverse_n2) |
1245 | PMUV3_INIT_SIMPLE(armv8_neoverse_v1) |
1246 | |
1247 | PMUV3_INIT_SIMPLE(armv8_nvidia_carmel) |
1248 | PMUV3_INIT_SIMPLE(armv8_nvidia_denver) |
1249 | |
1250 | static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu) |
1251 | { |
1252 | return armv8_pmu_init(cpu_pmu, name: "armv8_cortex_a35" , map_event: armv8_a53_map_event); |
1253 | } |
1254 | |
1255 | static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu) |
1256 | { |
1257 | return armv8_pmu_init(cpu_pmu, name: "armv8_cortex_a53" , map_event: armv8_a53_map_event); |
1258 | } |
1259 | |
1260 | static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu) |
1261 | { |
1262 | return armv8_pmu_init(cpu_pmu, name: "armv8_cortex_a57" , map_event: armv8_a57_map_event); |
1263 | } |
1264 | |
1265 | static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu) |
1266 | { |
1267 | return armv8_pmu_init(cpu_pmu, name: "armv8_cortex_a72" , map_event: armv8_a57_map_event); |
1268 | } |
1269 | |
1270 | static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu) |
1271 | { |
1272 | return armv8_pmu_init(cpu_pmu, name: "armv8_cortex_a73" , map_event: armv8_a73_map_event); |
1273 | } |
1274 | |
1275 | static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu) |
1276 | { |
1277 | return armv8_pmu_init(cpu_pmu, name: "armv8_cavium_thunder" , map_event: armv8_thunder_map_event); |
1278 | } |
1279 | |
1280 | static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu) |
1281 | { |
1282 | return armv8_pmu_init(cpu_pmu, name: "armv8_brcm_vulcan" , map_event: armv8_vulcan_map_event); |
1283 | } |
1284 | |
1285 | static const struct of_device_id armv8_pmu_of_device_ids[] = { |
1286 | {.compatible = "arm,armv8-pmuv3" , .data = armv8_pmuv3_pmu_init}, |
1287 | {.compatible = "arm,cortex-a34-pmu" , .data = armv8_cortex_a34_pmu_init}, |
1288 | {.compatible = "arm,cortex-a35-pmu" , .data = armv8_a35_pmu_init}, |
1289 | {.compatible = "arm,cortex-a53-pmu" , .data = armv8_a53_pmu_init}, |
1290 | {.compatible = "arm,cortex-a55-pmu" , .data = armv8_cortex_a55_pmu_init}, |
1291 | {.compatible = "arm,cortex-a57-pmu" , .data = armv8_a57_pmu_init}, |
1292 | {.compatible = "arm,cortex-a65-pmu" , .data = armv8_cortex_a65_pmu_init}, |
1293 | {.compatible = "arm,cortex-a72-pmu" , .data = armv8_a72_pmu_init}, |
1294 | {.compatible = "arm,cortex-a73-pmu" , .data = armv8_a73_pmu_init}, |
1295 | {.compatible = "arm,cortex-a75-pmu" , .data = armv8_cortex_a75_pmu_init}, |
1296 | {.compatible = "arm,cortex-a76-pmu" , .data = armv8_cortex_a76_pmu_init}, |
1297 | {.compatible = "arm,cortex-a77-pmu" , .data = armv8_cortex_a77_pmu_init}, |
1298 | {.compatible = "arm,cortex-a78-pmu" , .data = armv8_cortex_a78_pmu_init}, |
1299 | {.compatible = "arm,cortex-a510-pmu" , .data = armv9_cortex_a510_pmu_init}, |
1300 | {.compatible = "arm,cortex-a520-pmu" , .data = armv9_cortex_a520_pmu_init}, |
1301 | {.compatible = "arm,cortex-a710-pmu" , .data = armv9_cortex_a710_pmu_init}, |
1302 | {.compatible = "arm,cortex-a715-pmu" , .data = armv9_cortex_a715_pmu_init}, |
1303 | {.compatible = "arm,cortex-a720-pmu" , .data = armv9_cortex_a720_pmu_init}, |
1304 | {.compatible = "arm,cortex-x1-pmu" , .data = armv8_cortex_x1_pmu_init}, |
1305 | {.compatible = "arm,cortex-x2-pmu" , .data = armv9_cortex_x2_pmu_init}, |
1306 | {.compatible = "arm,cortex-x3-pmu" , .data = armv9_cortex_x3_pmu_init}, |
1307 | {.compatible = "arm,cortex-x4-pmu" , .data = armv9_cortex_x4_pmu_init}, |
1308 | {.compatible = "arm,neoverse-e1-pmu" , .data = armv8_neoverse_e1_pmu_init}, |
1309 | {.compatible = "arm,neoverse-n1-pmu" , .data = armv8_neoverse_n1_pmu_init}, |
1310 | {.compatible = "arm,neoverse-n2-pmu" , .data = armv9_neoverse_n2_pmu_init}, |
1311 | {.compatible = "arm,neoverse-v1-pmu" , .data = armv8_neoverse_v1_pmu_init}, |
1312 | {.compatible = "cavium,thunder-pmu" , .data = armv8_thunder_pmu_init}, |
1313 | {.compatible = "brcm,vulcan-pmu" , .data = armv8_vulcan_pmu_init}, |
1314 | {.compatible = "nvidia,carmel-pmu" , .data = armv8_nvidia_carmel_pmu_init}, |
1315 | {.compatible = "nvidia,denver-pmu" , .data = armv8_nvidia_denver_pmu_init}, |
1316 | {}, |
1317 | }; |
1318 | |
1319 | static int armv8_pmu_device_probe(struct platform_device *pdev) |
1320 | { |
1321 | return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids, NULL); |
1322 | } |
1323 | |
1324 | static struct platform_driver armv8_pmu_driver = { |
1325 | .driver = { |
1326 | .name = ARMV8_PMU_PDEV_NAME, |
1327 | .of_match_table = armv8_pmu_of_device_ids, |
1328 | .suppress_bind_attrs = true, |
1329 | }, |
1330 | .probe = armv8_pmu_device_probe, |
1331 | }; |
1332 | |
1333 | static int __init armv8_pmu_driver_init(void) |
1334 | { |
1335 | int ret; |
1336 | |
1337 | if (acpi_disabled) |
1338 | ret = platform_driver_register(&armv8_pmu_driver); |
1339 | else |
1340 | ret = arm_pmu_acpi_probe(armv8_pmuv3_pmu_init); |
1341 | |
1342 | if (!ret) |
1343 | lockup_detector_retry_init(); |
1344 | |
1345 | return ret; |
1346 | } |
1347 | device_initcall(armv8_pmu_driver_init) |
1348 | |
1349 | void arch_perf_update_userpage(struct perf_event *event, |
1350 | struct perf_event_mmap_page *userpg, u64 now) |
1351 | { |
1352 | struct clock_read_data *rd; |
1353 | unsigned int seq; |
1354 | u64 ns; |
1355 | |
1356 | userpg->cap_user_time = 0; |
1357 | userpg->cap_user_time_zero = 0; |
1358 | userpg->cap_user_time_short = 0; |
1359 | userpg->cap_user_rdpmc = armv8pmu_event_has_user_read(event); |
1360 | |
1361 | if (userpg->cap_user_rdpmc) { |
1362 | if (event->hw.flags & ARMPMU_EVT_64BIT) |
1363 | userpg->pmc_width = 64; |
1364 | else |
1365 | userpg->pmc_width = 32; |
1366 | } |
1367 | |
1368 | do { |
1369 | rd = sched_clock_read_begin(&seq); |
1370 | |
1371 | if (rd->read_sched_clock != arch_timer_read_counter) |
1372 | return; |
1373 | |
1374 | userpg->time_mult = rd->mult; |
1375 | userpg->time_shift = rd->shift; |
1376 | userpg->time_zero = rd->epoch_ns; |
1377 | userpg->time_cycles = rd->epoch_cyc; |
1378 | userpg->time_mask = rd->sched_clock_mask; |
1379 | |
1380 | /* |
1381 | * Subtract the cycle base, such that software that |
1382 | * doesn't know about cap_user_time_short still 'works' |
1383 | * assuming no wraps. |
1384 | */ |
1385 | ns = mul_u64_u32_shr(a: rd->epoch_cyc, mul: rd->mult, shift: rd->shift); |
1386 | userpg->time_zero -= ns; |
1387 | |
1388 | } while (sched_clock_read_retry(seq)); |
1389 | |
1390 | userpg->time_offset = userpg->time_zero - now; |
1391 | |
1392 | /* |
1393 | * time_shift is not expected to be greater than 31 due to |
1394 | * the original published conversion algorithm shifting a |
1395 | * 32-bit value (now specifies a 64-bit value) - refer |
1396 | * perf_event_mmap_page documentation in perf_event.h. |
1397 | */ |
1398 | if (userpg->time_shift == 32) { |
1399 | userpg->time_shift = 31; |
1400 | userpg->time_mult >>= 1; |
1401 | } |
1402 | |
1403 | /* |
1404 | * Internal timekeeping for enabled/running/stopped times |
1405 | * is always computed with the sched_clock. |
1406 | */ |
1407 | userpg->cap_user_time = 1; |
1408 | userpg->cap_user_time_zero = 1; |
1409 | userpg->cap_user_time_short = 1; |
1410 | } |
1411 | |