1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Alibaba DDR Sub-System Driveway PMU driver |
4 | * |
5 | * Copyright (C) 2022 Alibaba Inc |
6 | */ |
7 | |
8 | #define ALI_DRW_PMUNAME "ali_drw" |
9 | #define ALI_DRW_DRVNAME ALI_DRW_PMUNAME "_pmu" |
10 | #define pr_fmt(fmt) ALI_DRW_DRVNAME ": " fmt |
11 | |
12 | #include <linux/acpi.h> |
13 | #include <linux/bitfield.h> |
14 | #include <linux/bitmap.h> |
15 | #include <linux/bitops.h> |
16 | #include <linux/cpuhotplug.h> |
17 | #include <linux/cpumask.h> |
18 | #include <linux/device.h> |
19 | #include <linux/errno.h> |
20 | #include <linux/interrupt.h> |
21 | #include <linux/irq.h> |
22 | #include <linux/kernel.h> |
23 | #include <linux/list.h> |
24 | #include <linux/module.h> |
25 | #include <linux/mutex.h> |
26 | #include <linux/perf_event.h> |
27 | #include <linux/platform_device.h> |
28 | #include <linux/printk.h> |
29 | #include <linux/rculist.h> |
30 | #include <linux/refcount.h> |
31 | |
32 | |
33 | #define ALI_DRW_PMU_COMMON_MAX_COUNTERS 16 |
34 | #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE 19 |
35 | |
36 | #define ALI_DRW_PMU_PA_SHIFT 12 |
37 | #define ALI_DRW_PMU_CNT_INIT 0x00000000 |
38 | #define ALI_DRW_CNT_MAX_PERIOD 0xffffffff |
39 | #define ALI_DRW_PMU_CYCLE_EVT_ID 0x80 |
40 | |
41 | #define ALI_DRW_PMU_CNT_CTRL 0xC00 |
42 | #define ALI_DRW_PMU_CNT_RST BIT(2) |
43 | #define ALI_DRW_PMU_CNT_STOP BIT(1) |
44 | #define ALI_DRW_PMU_CNT_START BIT(0) |
45 | |
46 | #define ALI_DRW_PMU_CNT_STATE 0xC04 |
47 | #define ALI_DRW_PMU_TEST_CTRL 0xC08 |
48 | #define ALI_DRW_PMU_CNT_PRELOAD 0xC0C |
49 | |
50 | #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK GENMASK(23, 0) |
51 | #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK GENMASK(31, 0) |
52 | #define ALI_DRW_PMU_CYCLE_CNT_HIGH 0xC10 |
53 | #define ALI_DRW_PMU_CYCLE_CNT_LOW 0xC14 |
54 | |
55 | /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */ |
56 | #define ALI_DRW_PMU_EVENT_SEL0 0xC68 |
57 | /* counter 0-3 use sel0, counter 4-7 use sel1...*/ |
58 | #define ALI_DRW_PMU_EVENT_SELn(n) \ |
59 | (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4) |
60 | #define ALI_DRW_PMCOM_CNT_EN BIT(7) |
61 | #define ALI_DRW_PMCOM_CNT_EVENT_MASK GENMASK(5, 0) |
62 | #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \ |
63 | (8 * (n % 4)) |
64 | |
65 | /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */ |
66 | #define ALI_DRW_PMU_COMMON_COUNTER0 0xC78 |
67 | #define ALI_DRW_PMU_COMMON_COUNTERn(n) \ |
68 | (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n)) |
69 | |
70 | #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL 0xCB8 |
71 | #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL 0xCBC |
72 | #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS 0xCC0 |
73 | #define ALI_DRW_PMU_OV_INTR_CLR 0xCC4 |
74 | #define ALI_DRW_PMU_OV_INTR_STATUS 0xCC8 |
75 | #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK GENMASK(23, 8) |
76 | #define ALI_DRW_PMBW_CNT_OV_INTR_MASK GENMASK(7, 0) |
77 | #define ALI_DRW_PMU_OV_INTR_MASK GENMASK_ULL(63, 0) |
78 | |
79 | static int ali_drw_cpuhp_state_num; |
80 | |
81 | static LIST_HEAD(ali_drw_pmu_irqs); |
82 | static DEFINE_MUTEX(ali_drw_pmu_irqs_lock); |
83 | |
84 | struct ali_drw_pmu_irq { |
85 | struct hlist_node node; |
86 | struct list_head irqs_node; |
87 | struct list_head pmus_node; |
88 | int irq_num; |
89 | int cpu; |
90 | refcount_t refcount; |
91 | }; |
92 | |
93 | struct ali_drw_pmu { |
94 | void __iomem *cfg_base; |
95 | struct device *dev; |
96 | |
97 | struct list_head pmus_node; |
98 | struct ali_drw_pmu_irq *irq; |
99 | int irq_num; |
100 | int cpu; |
101 | DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS); |
102 | struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; |
103 | int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; |
104 | |
105 | struct pmu pmu; |
106 | }; |
107 | |
108 | #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu)) |
109 | |
110 | #define DRW_CONFIG_EVENTID GENMASK(7, 0) |
111 | #define GET_DRW_EVENTID(event) FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config) |
112 | |
113 | static ssize_t ali_drw_pmu_format_show(struct device *dev, |
114 | struct device_attribute *attr, char *buf) |
115 | { |
116 | struct dev_ext_attribute *eattr; |
117 | |
118 | eattr = container_of(attr, struct dev_ext_attribute, attr); |
119 | |
120 | return sprintf(buf, fmt: "%s\n" , (char *)eattr->var); |
121 | } |
122 | |
123 | /* |
124 | * PMU event attributes |
125 | */ |
126 | static ssize_t ali_drw_pmu_event_show(struct device *dev, |
127 | struct device_attribute *attr, char *page) |
128 | { |
129 | struct dev_ext_attribute *eattr; |
130 | |
131 | eattr = container_of(attr, struct dev_ext_attribute, attr); |
132 | |
133 | return sprintf(buf: page, fmt: "config=0x%lx\n" , (unsigned long)eattr->var); |
134 | } |
135 | |
136 | #define ALI_DRW_PMU_ATTR(_name, _func, _config) \ |
137 | (&((struct dev_ext_attribute[]) { \ |
138 | { __ATTR(_name, 0444, _func, NULL), (void *)_config } \ |
139 | })[0].attr.attr) |
140 | |
141 | #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config) \ |
142 | ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config) |
143 | #define ALI_DRW_PMU_EVENT_ATTR(_name, _config) \ |
144 | ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config) |
145 | |
146 | static struct attribute *ali_drw_pmu_events_attrs[] = { |
147 | ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr, 0x0), |
148 | ALI_DRW_PMU_EVENT_ATTR(hif_wr, 0x1), |
149 | ALI_DRW_PMU_EVENT_ATTR(hif_rd, 0x2), |
150 | ALI_DRW_PMU_EVENT_ATTR(hif_rmw, 0x3), |
151 | ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd, 0x4), |
152 | ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles, 0x7), |
153 | ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles, 0x8), |
154 | ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical, 0x9), |
155 | ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical, 0xA), |
156 | ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical, 0xB), |
157 | ALI_DRW_PMU_EVENT_ATTR(op_is_activate, 0xC), |
158 | ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr, 0xD), |
159 | ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate, 0xE), |
160 | ALI_DRW_PMU_EVENT_ATTR(op_is_rd, 0xF), |
161 | ALI_DRW_PMU_EVENT_ATTR(op_is_wr, 0x10), |
162 | ALI_DRW_PMU_EVENT_ATTR(op_is_mwr, 0x11), |
163 | ALI_DRW_PMU_EVENT_ATTR(op_is_precharge, 0x12), |
164 | ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr, 0x13), |
165 | ALI_DRW_PMU_EVENT_ATTR(precharge_for_other, 0x14), |
166 | ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions, 0x15), |
167 | ALI_DRW_PMU_EVENT_ATTR(write_combine, 0x16), |
168 | ALI_DRW_PMU_EVENT_ATTR(war_hazard, 0x17), |
169 | ALI_DRW_PMU_EVENT_ATTR(raw_hazard, 0x18), |
170 | ALI_DRW_PMU_EVENT_ATTR(waw_hazard, 0x19), |
171 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0, 0x1A), |
172 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1, 0x1B), |
173 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2, 0x1C), |
174 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3, 0x1D), |
175 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0, 0x1E), |
176 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1, 0x1F), |
177 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2, 0x20), |
178 | ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3, 0x21), |
179 | ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0, 0x26), |
180 | ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1, 0x27), |
181 | ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2, 0x28), |
182 | ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3, 0x29), |
183 | ALI_DRW_PMU_EVENT_ATTR(op_is_refresh, 0x2A), |
184 | ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref, 0x2B), |
185 | ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode, 0x2D), |
186 | ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl, 0x2E), |
187 | ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30), |
188 | ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31), |
189 | ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc, 0x34), |
190 | ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr, 0x35), |
191 | ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr, 0x36), |
192 | ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart, 0x37), |
193 | ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch, 0x38), |
194 | ALI_DRW_PMU_EVENT_ATTR(chi_txreq, 0x39), |
195 | ALI_DRW_PMU_EVENT_ATTR(chi_txdat, 0x3A), |
196 | ALI_DRW_PMU_EVENT_ATTR(chi_rxdat, 0x3B), |
197 | ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp, 0x3C), |
198 | ALI_DRW_PMU_EVENT_ATTR(tsz_vio, 0x3D), |
199 | ALI_DRW_PMU_EVENT_ATTR(cycle, 0x80), |
200 | NULL, |
201 | }; |
202 | |
203 | static struct attribute_group ali_drw_pmu_events_attr_group = { |
204 | .name = "events" , |
205 | .attrs = ali_drw_pmu_events_attrs, |
206 | }; |
207 | |
208 | static struct attribute *ali_drw_pmu_format_attr[] = { |
209 | ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7" ), |
210 | NULL, |
211 | }; |
212 | |
213 | static const struct attribute_group ali_drw_pmu_format_group = { |
214 | .name = "format" , |
215 | .attrs = ali_drw_pmu_format_attr, |
216 | }; |
217 | |
218 | static ssize_t ali_drw_pmu_cpumask_show(struct device *dev, |
219 | struct device_attribute *attr, |
220 | char *buf) |
221 | { |
222 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev)); |
223 | |
224 | return cpumap_print_to_pagebuf(list: true, buf, cpumask_of(drw_pmu->cpu)); |
225 | } |
226 | |
227 | static struct device_attribute ali_drw_pmu_cpumask_attr = |
228 | __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL); |
229 | |
230 | static struct attribute *ali_drw_pmu_cpumask_attrs[] = { |
231 | &ali_drw_pmu_cpumask_attr.attr, |
232 | NULL, |
233 | }; |
234 | |
235 | static const struct attribute_group ali_drw_pmu_cpumask_attr_group = { |
236 | .attrs = ali_drw_pmu_cpumask_attrs, |
237 | }; |
238 | |
239 | static ssize_t ali_drw_pmu_identifier_show(struct device *dev, |
240 | struct device_attribute *attr, |
241 | char *page) |
242 | { |
243 | return sysfs_emit(buf: page, fmt: "%s\n" , "ali_drw_pmu" ); |
244 | } |
245 | |
246 | static umode_t ali_drw_pmu_identifier_attr_visible(struct kobject *kobj, |
247 | struct attribute *attr, int n) |
248 | { |
249 | return attr->mode; |
250 | } |
251 | |
252 | static struct device_attribute ali_drw_pmu_identifier_attr = |
253 | __ATTR(identifier, 0444, ali_drw_pmu_identifier_show, NULL); |
254 | |
255 | static struct attribute *ali_drw_pmu_identifier_attrs[] = { |
256 | &ali_drw_pmu_identifier_attr.attr, |
257 | NULL |
258 | }; |
259 | |
260 | static const struct attribute_group ali_drw_pmu_identifier_attr_group = { |
261 | .attrs = ali_drw_pmu_identifier_attrs, |
262 | .is_visible = ali_drw_pmu_identifier_attr_visible |
263 | }; |
264 | |
265 | static const struct attribute_group *ali_drw_pmu_attr_groups[] = { |
266 | &ali_drw_pmu_events_attr_group, |
267 | &ali_drw_pmu_cpumask_attr_group, |
268 | &ali_drw_pmu_format_group, |
269 | &ali_drw_pmu_identifier_attr_group, |
270 | NULL, |
271 | }; |
272 | |
273 | /* find a counter for event, then in add func, hw.idx will equal to counter */ |
274 | static int ali_drw_get_counter_idx(struct perf_event *event) |
275 | { |
276 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
277 | int idx; |
278 | |
279 | for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) { |
280 | if (!test_and_set_bit(nr: idx, addr: drw_pmu->used_mask)) |
281 | return idx; |
282 | } |
283 | |
284 | /* The counters are all in use. */ |
285 | return -EBUSY; |
286 | } |
287 | |
288 | static u64 ali_drw_pmu_read_counter(struct perf_event *event) |
289 | { |
290 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
291 | u64 cycle_high, cycle_low; |
292 | |
293 | if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { |
294 | cycle_high = readl(addr: drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH); |
295 | cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK; |
296 | cycle_low = readl(addr: drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW); |
297 | cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK; |
298 | return (cycle_high << 32 | cycle_low); |
299 | } |
300 | |
301 | return readl(addr: drw_pmu->cfg_base + |
302 | ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx)); |
303 | } |
304 | |
305 | static void ali_drw_pmu_event_update(struct perf_event *event) |
306 | { |
307 | struct hw_perf_event *hwc = &event->hw; |
308 | u64 delta, prev, now; |
309 | |
310 | do { |
311 | prev = local64_read(&hwc->prev_count); |
312 | now = ali_drw_pmu_read_counter(event); |
313 | } while (local64_cmpxchg(l: &hwc->prev_count, old: prev, new: now) != prev); |
314 | |
315 | /* handle overflow. */ |
316 | delta = now - prev; |
317 | if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) |
318 | delta &= ALI_DRW_PMU_OV_INTR_MASK; |
319 | else |
320 | delta &= ALI_DRW_CNT_MAX_PERIOD; |
321 | local64_add(delta, &event->count); |
322 | } |
323 | |
324 | static void ali_drw_pmu_event_set_period(struct perf_event *event) |
325 | { |
326 | u64 pre_val; |
327 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
328 | |
329 | /* set a preload counter for test purpose */ |
330 | writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx, |
331 | addr: drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); |
332 | |
333 | /* set conunter initial value */ |
334 | pre_val = ALI_DRW_PMU_CNT_INIT; |
335 | writel(val: pre_val, addr: drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); |
336 | local64_set(&event->hw.prev_count, pre_val); |
337 | |
338 | /* set sel mode to zero to start test */ |
339 | writel(val: 0x0, addr: drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); |
340 | } |
341 | |
342 | static void ali_drw_pmu_enable_counter(struct perf_event *event) |
343 | { |
344 | u32 val, subval, reg, shift; |
345 | int counter = event->hw.idx; |
346 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
347 | |
348 | reg = ALI_DRW_PMU_EVENT_SELn(counter); |
349 | val = readl(addr: drw_pmu->cfg_base + reg); |
350 | subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) | |
351 | FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]); |
352 | |
353 | shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); |
354 | val &= ~(GENMASK(7, 0) << shift); |
355 | val |= subval << shift; |
356 | |
357 | writel(val, addr: drw_pmu->cfg_base + reg); |
358 | } |
359 | |
360 | static void ali_drw_pmu_disable_counter(struct perf_event *event) |
361 | { |
362 | u32 val, reg, subval, shift; |
363 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
364 | int counter = event->hw.idx; |
365 | |
366 | reg = ALI_DRW_PMU_EVENT_SELn(counter); |
367 | val = readl(addr: drw_pmu->cfg_base + reg); |
368 | subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) | |
369 | FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0); |
370 | |
371 | shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); |
372 | val &= ~(GENMASK(7, 0) << shift); |
373 | val |= subval << shift; |
374 | |
375 | writel(val, addr: drw_pmu->cfg_base + reg); |
376 | } |
377 | |
378 | static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data) |
379 | { |
380 | struct ali_drw_pmu_irq *irq = data; |
381 | struct ali_drw_pmu *drw_pmu; |
382 | irqreturn_t ret = IRQ_NONE; |
383 | |
384 | rcu_read_lock(); |
385 | list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) { |
386 | unsigned long status, clr_status; |
387 | struct perf_event *event; |
388 | unsigned int idx; |
389 | |
390 | for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { |
391 | event = drw_pmu->events[idx]; |
392 | if (!event) |
393 | continue; |
394 | ali_drw_pmu_disable_counter(event); |
395 | } |
396 | |
397 | /* common counter intr status */ |
398 | status = readl(addr: drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS); |
399 | status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status); |
400 | if (status) { |
401 | for_each_set_bit(idx, &status, |
402 | ALI_DRW_PMU_COMMON_MAX_COUNTERS) { |
403 | event = drw_pmu->events[idx]; |
404 | if (WARN_ON_ONCE(!event)) |
405 | continue; |
406 | ali_drw_pmu_event_update(event); |
407 | ali_drw_pmu_event_set_period(event); |
408 | } |
409 | |
410 | /* clear common counter intr status */ |
411 | clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1); |
412 | writel(val: clr_status, |
413 | addr: drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); |
414 | } |
415 | |
416 | for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { |
417 | event = drw_pmu->events[idx]; |
418 | if (!event) |
419 | continue; |
420 | if (!(event->hw.state & PERF_HES_STOPPED)) |
421 | ali_drw_pmu_enable_counter(event); |
422 | } |
423 | if (status) |
424 | ret = IRQ_HANDLED; |
425 | } |
426 | rcu_read_unlock(); |
427 | return ret; |
428 | } |
429 | |
430 | static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device |
431 | *pdev, int irq_num) |
432 | { |
433 | int ret; |
434 | struct ali_drw_pmu_irq *irq; |
435 | |
436 | list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) { |
437 | if (irq->irq_num == irq_num |
438 | && refcount_inc_not_zero(r: &irq->refcount)) |
439 | return irq; |
440 | } |
441 | |
442 | irq = kzalloc(size: sizeof(*irq), GFP_KERNEL); |
443 | if (!irq) |
444 | return ERR_PTR(error: -ENOMEM); |
445 | |
446 | INIT_LIST_HEAD(list: &irq->pmus_node); |
447 | |
448 | /* Pick one CPU to be the preferred one to use */ |
449 | irq->cpu = smp_processor_id(); |
450 | refcount_set(r: &irq->refcount, n: 1); |
451 | |
452 | /* |
453 | * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same |
454 | * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers |
455 | * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not |
456 | * share with other component. |
457 | */ |
458 | ret = devm_request_irq(dev: &pdev->dev, irq: irq_num, handler: ali_drw_pmu_isr, |
459 | IRQF_SHARED, devname: dev_name(dev: &pdev->dev), dev_id: irq); |
460 | if (ret < 0) { |
461 | dev_err(&pdev->dev, |
462 | "Fail to request IRQ:%d ret:%d\n" , irq_num, ret); |
463 | goto out_free; |
464 | } |
465 | |
466 | ret = irq_set_affinity_hint(irq: irq_num, cpumask_of(irq->cpu)); |
467 | if (ret) |
468 | goto out_free; |
469 | |
470 | ret = cpuhp_state_add_instance_nocalls(state: ali_drw_cpuhp_state_num, |
471 | node: &irq->node); |
472 | if (ret) |
473 | goto out_free; |
474 | |
475 | irq->irq_num = irq_num; |
476 | list_add(new: &irq->irqs_node, head: &ali_drw_pmu_irqs); |
477 | |
478 | return irq; |
479 | |
480 | out_free: |
481 | kfree(objp: irq); |
482 | return ERR_PTR(error: ret); |
483 | } |
484 | |
485 | static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu, |
486 | struct platform_device *pdev) |
487 | { |
488 | int irq_num; |
489 | struct ali_drw_pmu_irq *irq; |
490 | |
491 | /* Read and init IRQ */ |
492 | irq_num = platform_get_irq(pdev, 0); |
493 | if (irq_num < 0) |
494 | return irq_num; |
495 | |
496 | mutex_lock(&ali_drw_pmu_irqs_lock); |
497 | irq = __ali_drw_pmu_init_irq(pdev, irq_num); |
498 | mutex_unlock(lock: &ali_drw_pmu_irqs_lock); |
499 | |
500 | if (IS_ERR(ptr: irq)) |
501 | return PTR_ERR(ptr: irq); |
502 | |
503 | drw_pmu->irq = irq; |
504 | |
505 | mutex_lock(&ali_drw_pmu_irqs_lock); |
506 | list_add_rcu(new: &drw_pmu->pmus_node, head: &irq->pmus_node); |
507 | mutex_unlock(lock: &ali_drw_pmu_irqs_lock); |
508 | |
509 | return 0; |
510 | } |
511 | |
512 | static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu) |
513 | { |
514 | struct ali_drw_pmu_irq *irq = drw_pmu->irq; |
515 | |
516 | mutex_lock(&ali_drw_pmu_irqs_lock); |
517 | list_del_rcu(entry: &drw_pmu->pmus_node); |
518 | |
519 | if (!refcount_dec_and_test(r: &irq->refcount)) { |
520 | mutex_unlock(lock: &ali_drw_pmu_irqs_lock); |
521 | return; |
522 | } |
523 | |
524 | list_del(entry: &irq->irqs_node); |
525 | mutex_unlock(lock: &ali_drw_pmu_irqs_lock); |
526 | |
527 | WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL)); |
528 | cpuhp_state_remove_instance_nocalls(state: ali_drw_cpuhp_state_num, |
529 | node: &irq->node); |
530 | kfree(objp: irq); |
531 | } |
532 | |
533 | static int ali_drw_pmu_event_init(struct perf_event *event) |
534 | { |
535 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
536 | struct hw_perf_event *hwc = &event->hw; |
537 | struct perf_event *sibling; |
538 | struct device *dev = drw_pmu->pmu.dev; |
539 | |
540 | if (event->attr.type != event->pmu->type) |
541 | return -ENOENT; |
542 | |
543 | if (is_sampling_event(event)) { |
544 | dev_err(dev, "Sampling not supported!\n" ); |
545 | return -EOPNOTSUPP; |
546 | } |
547 | |
548 | if (event->attach_state & PERF_ATTACH_TASK) { |
549 | dev_err(dev, "Per-task counter cannot allocate!\n" ); |
550 | return -EOPNOTSUPP; |
551 | } |
552 | |
553 | event->cpu = drw_pmu->cpu; |
554 | if (event->cpu < 0) { |
555 | dev_err(dev, "Per-task mode not supported!\n" ); |
556 | return -EOPNOTSUPP; |
557 | } |
558 | |
559 | if (event->group_leader != event && |
560 | !is_software_event(event: event->group_leader)) { |
561 | dev_err(dev, "driveway only allow one event!\n" ); |
562 | return -EINVAL; |
563 | } |
564 | |
565 | for_each_sibling_event(sibling, event->group_leader) { |
566 | if (sibling != event && !is_software_event(event: sibling)) { |
567 | dev_err(dev, "driveway event not allowed!\n" ); |
568 | return -EINVAL; |
569 | } |
570 | } |
571 | |
572 | /* reset all the pmu counters */ |
573 | writel(ALI_DRW_PMU_CNT_RST, addr: drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); |
574 | |
575 | hwc->idx = -1; |
576 | |
577 | return 0; |
578 | } |
579 | |
580 | static void ali_drw_pmu_start(struct perf_event *event, int flags) |
581 | { |
582 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
583 | |
584 | event->hw.state = 0; |
585 | |
586 | if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { |
587 | writel(ALI_DRW_PMU_CNT_START, |
588 | addr: drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); |
589 | return; |
590 | } |
591 | |
592 | ali_drw_pmu_event_set_period(event); |
593 | if (flags & PERF_EF_RELOAD) { |
594 | unsigned long prev_raw_count = |
595 | local64_read(&event->hw.prev_count); |
596 | writel(val: prev_raw_count, |
597 | addr: drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); |
598 | } |
599 | |
600 | ali_drw_pmu_enable_counter(event); |
601 | |
602 | writel(ALI_DRW_PMU_CNT_START, addr: drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); |
603 | } |
604 | |
605 | static void ali_drw_pmu_stop(struct perf_event *event, int flags) |
606 | { |
607 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
608 | |
609 | if (event->hw.state & PERF_HES_STOPPED) |
610 | return; |
611 | |
612 | if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID) |
613 | ali_drw_pmu_disable_counter(event); |
614 | |
615 | writel(ALI_DRW_PMU_CNT_STOP, addr: drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); |
616 | |
617 | ali_drw_pmu_event_update(event); |
618 | event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
619 | } |
620 | |
621 | static int ali_drw_pmu_add(struct perf_event *event, int flags) |
622 | { |
623 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
624 | struct hw_perf_event *hwc = &event->hw; |
625 | int idx = -1; |
626 | int evtid; |
627 | |
628 | evtid = GET_DRW_EVENTID(event); |
629 | |
630 | if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) { |
631 | idx = ali_drw_get_counter_idx(event); |
632 | if (idx < 0) |
633 | return idx; |
634 | drw_pmu->events[idx] = event; |
635 | drw_pmu->evtids[idx] = evtid; |
636 | } |
637 | hwc->idx = idx; |
638 | |
639 | hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
640 | |
641 | if (flags & PERF_EF_START) |
642 | ali_drw_pmu_start(event, PERF_EF_RELOAD); |
643 | |
644 | /* Propagate our changes to the userspace mapping. */ |
645 | perf_event_update_userpage(event); |
646 | |
647 | return 0; |
648 | } |
649 | |
650 | static void ali_drw_pmu_del(struct perf_event *event, int flags) |
651 | { |
652 | struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); |
653 | struct hw_perf_event *hwc = &event->hw; |
654 | int idx = hwc->idx; |
655 | |
656 | ali_drw_pmu_stop(event, PERF_EF_UPDATE); |
657 | |
658 | if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) { |
659 | drw_pmu->events[idx] = NULL; |
660 | drw_pmu->evtids[idx] = 0; |
661 | clear_bit(nr: idx, addr: drw_pmu->used_mask); |
662 | } |
663 | |
664 | perf_event_update_userpage(event); |
665 | } |
666 | |
667 | static void ali_drw_pmu_read(struct perf_event *event) |
668 | { |
669 | ali_drw_pmu_event_update(event); |
670 | } |
671 | |
672 | static int ali_drw_pmu_probe(struct platform_device *pdev) |
673 | { |
674 | struct ali_drw_pmu *drw_pmu; |
675 | struct resource *res; |
676 | char *name; |
677 | int ret; |
678 | |
679 | drw_pmu = devm_kzalloc(dev: &pdev->dev, size: sizeof(*drw_pmu), GFP_KERNEL); |
680 | if (!drw_pmu) |
681 | return -ENOMEM; |
682 | |
683 | drw_pmu->dev = &pdev->dev; |
684 | platform_set_drvdata(pdev, data: drw_pmu); |
685 | |
686 | drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res); |
687 | if (IS_ERR(ptr: drw_pmu->cfg_base)) |
688 | return PTR_ERR(ptr: drw_pmu->cfg_base); |
689 | |
690 | name = devm_kasprintf(dev: drw_pmu->dev, GFP_KERNEL, fmt: "ali_drw_%llx" , |
691 | (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT)); |
692 | if (!name) |
693 | return -ENOMEM; |
694 | |
695 | writel(ALI_DRW_PMU_CNT_RST, addr: drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); |
696 | |
697 | /* enable the generation of interrupt by all common counters */ |
698 | writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, |
699 | addr: drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL); |
700 | |
701 | /* clearing interrupt status */ |
702 | writel(val: 0xffffff, addr: drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); |
703 | |
704 | drw_pmu->cpu = smp_processor_id(); |
705 | |
706 | ret = ali_drw_pmu_init_irq(drw_pmu, pdev); |
707 | if (ret) |
708 | return ret; |
709 | |
710 | drw_pmu->pmu = (struct pmu) { |
711 | .module = THIS_MODULE, |
712 | .task_ctx_nr = perf_invalid_context, |
713 | .event_init = ali_drw_pmu_event_init, |
714 | .add = ali_drw_pmu_add, |
715 | .del = ali_drw_pmu_del, |
716 | .start = ali_drw_pmu_start, |
717 | .stop = ali_drw_pmu_stop, |
718 | .read = ali_drw_pmu_read, |
719 | .attr_groups = ali_drw_pmu_attr_groups, |
720 | .capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
721 | }; |
722 | |
723 | ret = perf_pmu_register(pmu: &drw_pmu->pmu, name, type: -1); |
724 | if (ret) { |
725 | dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n" ); |
726 | ali_drw_pmu_uninit_irq(drw_pmu); |
727 | } |
728 | |
729 | return ret; |
730 | } |
731 | |
732 | static int ali_drw_pmu_remove(struct platform_device *pdev) |
733 | { |
734 | struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev); |
735 | |
736 | /* disable the generation of interrupt by all common counters */ |
737 | writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, |
738 | addr: drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL); |
739 | |
740 | ali_drw_pmu_uninit_irq(drw_pmu); |
741 | perf_pmu_unregister(pmu: &drw_pmu->pmu); |
742 | |
743 | return 0; |
744 | } |
745 | |
746 | static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) |
747 | { |
748 | struct ali_drw_pmu_irq *irq; |
749 | struct ali_drw_pmu *drw_pmu; |
750 | unsigned int target; |
751 | int ret; |
752 | cpumask_t node_online_cpus; |
753 | |
754 | irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node); |
755 | if (cpu != irq->cpu) |
756 | return 0; |
757 | |
758 | ret = cpumask_and(dstp: &node_online_cpus, |
759 | src1p: cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask); |
760 | if (ret) |
761 | target = cpumask_any_but(mask: &node_online_cpus, cpu); |
762 | else |
763 | target = cpumask_any_but(cpu_online_mask, cpu); |
764 | |
765 | if (target >= nr_cpu_ids) |
766 | return 0; |
767 | |
768 | /* We're only reading, but this isn't the place to be involving RCU */ |
769 | mutex_lock(&ali_drw_pmu_irqs_lock); |
770 | list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node) |
771 | perf_pmu_migrate_context(pmu: &drw_pmu->pmu, src_cpu: irq->cpu, dst_cpu: target); |
772 | mutex_unlock(lock: &ali_drw_pmu_irqs_lock); |
773 | |
774 | WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target))); |
775 | irq->cpu = target; |
776 | |
777 | return 0; |
778 | } |
779 | |
780 | /* |
781 | * Due to historical reasons, the HID used in the production environment is |
782 | * ARMHD700, so we leave ARMHD700 as Compatible ID. |
783 | */ |
784 | static const struct acpi_device_id ali_drw_acpi_match[] = { |
785 | {"BABA5000" , 0}, |
786 | {"ARMHD700" , 0}, |
787 | {} |
788 | }; |
789 | |
790 | MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match); |
791 | |
792 | static struct platform_driver ali_drw_pmu_driver = { |
793 | .driver = { |
794 | .name = "ali_drw_pmu" , |
795 | .acpi_match_table = ali_drw_acpi_match, |
796 | }, |
797 | .probe = ali_drw_pmu_probe, |
798 | .remove = ali_drw_pmu_remove, |
799 | }; |
800 | |
801 | static int __init ali_drw_pmu_init(void) |
802 | { |
803 | int ret; |
804 | |
805 | ret = cpuhp_setup_state_multi(state: CPUHP_AP_ONLINE_DYN, |
806 | name: "ali_drw_pmu:online" , |
807 | NULL, teardown: ali_drw_pmu_offline_cpu); |
808 | |
809 | if (ret < 0) { |
810 | pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n" , |
811 | ret); |
812 | return ret; |
813 | } |
814 | ali_drw_cpuhp_state_num = ret; |
815 | |
816 | ret = platform_driver_register(&ali_drw_pmu_driver); |
817 | if (ret) |
818 | cpuhp_remove_multi_state(state: ali_drw_cpuhp_state_num); |
819 | |
820 | return ret; |
821 | } |
822 | |
823 | static void __exit ali_drw_pmu_exit(void) |
824 | { |
825 | platform_driver_unregister(&ali_drw_pmu_driver); |
826 | cpuhp_remove_multi_state(state: ali_drw_cpuhp_state_num); |
827 | } |
828 | |
829 | module_init(ali_drw_pmu_init); |
830 | module_exit(ali_drw_pmu_exit); |
831 | |
832 | MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>" ); |
833 | MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>" ); |
834 | MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>" ); |
835 | MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver" ); |
836 | MODULE_LICENSE("GPL v2" ); |
837 | |