1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * This driver adds support for PCIe PMU RCiEP device. Related |
4 | * perf events are bandwidth, latency etc. |
5 | * |
6 | * Copyright (C) 2021 HiSilicon Limited |
7 | * Author: Qi Liu <liuqi115@huawei.com> |
8 | */ |
9 | #include <linux/bitfield.h> |
10 | #include <linux/bitmap.h> |
11 | #include <linux/bug.h> |
12 | #include <linux/device.h> |
13 | #include <linux/err.h> |
14 | #include <linux/interrupt.h> |
15 | #include <linux/irq.h> |
16 | #include <linux/kernel.h> |
17 | #include <linux/list.h> |
18 | #include <linux/module.h> |
19 | #include <linux/pci.h> |
20 | #include <linux/perf_event.h> |
21 | |
22 | #define DRV_NAME "hisi_pcie_pmu" |
23 | /* Define registers */ |
24 | #define HISI_PCIE_GLOBAL_CTRL 0x00 |
25 | #define HISI_PCIE_EVENT_CTRL 0x010 |
26 | #define HISI_PCIE_CNT 0x090 |
27 | #define HISI_PCIE_EXT_CNT 0x110 |
28 | #define HISI_PCIE_INT_STAT 0x150 |
29 | #define HISI_PCIE_INT_MASK 0x154 |
30 | #define HISI_PCIE_REG_BDF 0xfe0 |
31 | #define HISI_PCIE_REG_VERSION 0xfe4 |
32 | #define HISI_PCIE_REG_INFO 0xfe8 |
33 | |
34 | /* Define command in HISI_PCIE_GLOBAL_CTRL */ |
35 | #define HISI_PCIE_GLOBAL_EN 0x01 |
36 | #define HISI_PCIE_GLOBAL_NONE 0 |
37 | |
38 | /* Define command in HISI_PCIE_EVENT_CTRL */ |
39 | #define HISI_PCIE_EVENT_EN BIT_ULL(20) |
40 | #define HISI_PCIE_RESET_CNT BIT_ULL(22) |
41 | #define HISI_PCIE_INIT_SET BIT_ULL(34) |
42 | #define HISI_PCIE_THR_EN BIT_ULL(26) |
43 | #define HISI_PCIE_TARGET_EN BIT_ULL(32) |
44 | #define HISI_PCIE_TRIG_EN BIT_ULL(52) |
45 | |
46 | /* Define offsets in HISI_PCIE_EVENT_CTRL */ |
47 | #define HISI_PCIE_EVENT_M GENMASK_ULL(15, 0) |
48 | #define HISI_PCIE_THR_MODE_M GENMASK_ULL(27, 27) |
49 | #define HISI_PCIE_THR_M GENMASK_ULL(31, 28) |
50 | #define HISI_PCIE_LEN_M GENMASK_ULL(35, 34) |
51 | #define HISI_PCIE_TARGET_M GENMASK_ULL(52, 36) |
52 | #define HISI_PCIE_TRIG_MODE_M GENMASK_ULL(53, 53) |
53 | #define HISI_PCIE_TRIG_M GENMASK_ULL(59, 56) |
54 | |
55 | /* Default config of TLP length mode, will count both TLP headers and payloads */ |
56 | #define HISI_PCIE_LEN_M_DEFAULT 3ULL |
57 | |
58 | #define HISI_PCIE_MAX_COUNTERS 8 |
59 | #define HISI_PCIE_REG_STEP 8 |
60 | #define HISI_PCIE_THR_MAX_VAL 10 |
61 | #define HISI_PCIE_TRIG_MAX_VAL 10 |
62 | #define HISI_PCIE_MAX_PERIOD (GENMASK_ULL(63, 0)) |
63 | #define HISI_PCIE_INIT_VAL BIT_ULL(63) |
64 | |
65 | struct hisi_pcie_pmu { |
66 | struct perf_event *hw_events[HISI_PCIE_MAX_COUNTERS]; |
67 | struct hlist_node node; |
68 | struct pci_dev *pdev; |
69 | struct pmu pmu; |
70 | void __iomem *base; |
71 | int irq; |
72 | u32 identifier; |
73 | /* Minimum and maximum BDF of root ports monitored by PMU */ |
74 | u16 bdf_min; |
75 | u16 bdf_max; |
76 | int on_cpu; |
77 | }; |
78 | |
79 | struct hisi_pcie_reg_pair { |
80 | u16 lo; |
81 | u16 hi; |
82 | }; |
83 | |
84 | #define to_pcie_pmu(p) (container_of((p), struct hisi_pcie_pmu, pmu)) |
85 | #define GET_PCI_DEVFN(bdf) ((bdf) & 0xff) |
86 | |
87 | #define HISI_PCIE_PMU_FILTER_ATTR(_name, _config, _hi, _lo) \ |
88 | static u64 hisi_pcie_get_##_name(struct perf_event *event) \ |
89 | { \ |
90 | return FIELD_GET(GENMASK(_hi, _lo), event->attr._config); \ |
91 | } \ |
92 | |
93 | HISI_PCIE_PMU_FILTER_ATTR(event, config, 16, 0); |
94 | HISI_PCIE_PMU_FILTER_ATTR(thr_len, config1, 3, 0); |
95 | HISI_PCIE_PMU_FILTER_ATTR(thr_mode, config1, 4, 4); |
96 | HISI_PCIE_PMU_FILTER_ATTR(trig_len, config1, 8, 5); |
97 | HISI_PCIE_PMU_FILTER_ATTR(trig_mode, config1, 9, 9); |
98 | HISI_PCIE_PMU_FILTER_ATTR(len_mode, config1, 11, 10); |
99 | HISI_PCIE_PMU_FILTER_ATTR(port, config2, 15, 0); |
100 | HISI_PCIE_PMU_FILTER_ATTR(bdf, config2, 31, 16); |
101 | |
102 | static ssize_t hisi_pcie_format_sysfs_show(struct device *dev, struct device_attribute *attr, |
103 | char *buf) |
104 | { |
105 | struct dev_ext_attribute *eattr; |
106 | |
107 | eattr = container_of(attr, struct dev_ext_attribute, attr); |
108 | |
109 | return sysfs_emit(buf, fmt: "%s\n" , (char *)eattr->var); |
110 | } |
111 | |
112 | static ssize_t hisi_pcie_event_sysfs_show(struct device *dev, struct device_attribute *attr, |
113 | char *buf) |
114 | { |
115 | struct perf_pmu_events_attr *pmu_attr = |
116 | container_of(attr, struct perf_pmu_events_attr, attr); |
117 | |
118 | return sysfs_emit(buf, fmt: "config=0x%llx\n" , pmu_attr->id); |
119 | } |
120 | |
121 | #define HISI_PCIE_PMU_FORMAT_ATTR(_name, _format) \ |
122 | (&((struct dev_ext_attribute[]){ \ |
123 | { .attr = __ATTR(_name, 0444, hisi_pcie_format_sysfs_show, \ |
124 | NULL), \ |
125 | .var = (void *)_format } \ |
126 | })[0].attr.attr) |
127 | |
128 | #define HISI_PCIE_PMU_EVENT_ATTR(_name, _id) \ |
129 | PMU_EVENT_ATTR_ID(_name, hisi_pcie_event_sysfs_show, _id) |
130 | |
131 | static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) |
132 | { |
133 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); |
134 | |
135 | return cpumap_print_to_pagebuf(list: true, buf, cpumask_of(pcie_pmu->on_cpu)); |
136 | } |
137 | static DEVICE_ATTR_RO(cpumask); |
138 | |
139 | static ssize_t identifier_show(struct device *dev, struct device_attribute *attr, char *buf) |
140 | { |
141 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); |
142 | |
143 | return sysfs_emit(buf, fmt: "%#x\n" , pcie_pmu->identifier); |
144 | } |
145 | static DEVICE_ATTR_RO(identifier); |
146 | |
147 | static ssize_t bus_show(struct device *dev, struct device_attribute *attr, char *buf) |
148 | { |
149 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(dev_get_drvdata(dev)); |
150 | |
151 | return sysfs_emit(buf, fmt: "%#04x\n" , PCI_BUS_NUM(pcie_pmu->bdf_min)); |
152 | } |
153 | static DEVICE_ATTR_RO(bus); |
154 | |
155 | static struct hisi_pcie_reg_pair |
156 | hisi_pcie_parse_reg_value(struct hisi_pcie_pmu *pcie_pmu, u32 reg_off) |
157 | { |
158 | u32 val = readl_relaxed(pcie_pmu->base + reg_off); |
159 | struct hisi_pcie_reg_pair regs = { |
160 | .lo = val, |
161 | .hi = val >> 16, |
162 | }; |
163 | |
164 | return regs; |
165 | } |
166 | |
167 | /* |
168 | * Hardware counter and ext_counter work together for bandwidth, latency, bus |
169 | * utilization and buffer occupancy events. For example, RX memory write latency |
170 | * events(index = 0x0010), counter counts total delay cycles and ext_counter |
171 | * counts RX memory write PCIe packets number. |
172 | * |
173 | * As we don't want PMU driver to process these two data, "delay cycles" can |
174 | * be treated as an independent event(index = 0x0010), "RX memory write packets |
175 | * number" as another(index = 0x10010). BIT 16 is used to distinguish and 0-15 |
176 | * bits are "real" event index, which can be used to set HISI_PCIE_EVENT_CTRL. |
177 | */ |
178 | #define EXT_COUNTER_IS_USED(idx) ((idx) & BIT(16)) |
179 | |
180 | static u32 hisi_pcie_get_real_event(struct perf_event *event) |
181 | { |
182 | return hisi_pcie_get_event(event) & GENMASK(15, 0); |
183 | } |
184 | |
185 | static u32 hisi_pcie_pmu_get_offset(u32 offset, u32 idx) |
186 | { |
187 | return offset + HISI_PCIE_REG_STEP * idx; |
188 | } |
189 | |
190 | static u32 hisi_pcie_pmu_readl(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, |
191 | u32 idx) |
192 | { |
193 | u32 offset = hisi_pcie_pmu_get_offset(offset: reg_offset, idx); |
194 | |
195 | return readl_relaxed(pcie_pmu->base + offset); |
196 | } |
197 | |
198 | static void hisi_pcie_pmu_writel(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u32 val) |
199 | { |
200 | u32 offset = hisi_pcie_pmu_get_offset(offset: reg_offset, idx); |
201 | |
202 | writel_relaxed(val, pcie_pmu->base + offset); |
203 | } |
204 | |
205 | static u64 hisi_pcie_pmu_readq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx) |
206 | { |
207 | u32 offset = hisi_pcie_pmu_get_offset(offset: reg_offset, idx); |
208 | |
209 | return readq_relaxed(pcie_pmu->base + offset); |
210 | } |
211 | |
212 | static void hisi_pcie_pmu_writeq(struct hisi_pcie_pmu *pcie_pmu, u32 reg_offset, u32 idx, u64 val) |
213 | { |
214 | u32 offset = hisi_pcie_pmu_get_offset(offset: reg_offset, idx); |
215 | |
216 | writeq_relaxed(val, pcie_pmu->base + offset); |
217 | } |
218 | |
219 | static u64 hisi_pcie_pmu_get_event_ctrl_val(struct perf_event *event) |
220 | { |
221 | u64 port, trig_len, thr_len, len_mode; |
222 | u64 reg = HISI_PCIE_INIT_SET; |
223 | |
224 | /* Config HISI_PCIE_EVENT_CTRL according to event. */ |
225 | reg |= FIELD_PREP(HISI_PCIE_EVENT_M, hisi_pcie_get_real_event(event)); |
226 | |
227 | /* Config HISI_PCIE_EVENT_CTRL according to root port or EP device. */ |
228 | port = hisi_pcie_get_port(event); |
229 | if (port) |
230 | reg |= FIELD_PREP(HISI_PCIE_TARGET_M, port); |
231 | else |
232 | reg |= HISI_PCIE_TARGET_EN | |
233 | FIELD_PREP(HISI_PCIE_TARGET_M, hisi_pcie_get_bdf(event)); |
234 | |
235 | /* Config HISI_PCIE_EVENT_CTRL according to trigger condition. */ |
236 | trig_len = hisi_pcie_get_trig_len(event); |
237 | if (trig_len) { |
238 | reg |= FIELD_PREP(HISI_PCIE_TRIG_M, trig_len); |
239 | reg |= FIELD_PREP(HISI_PCIE_TRIG_MODE_M, hisi_pcie_get_trig_mode(event)); |
240 | reg |= HISI_PCIE_TRIG_EN; |
241 | } |
242 | |
243 | /* Config HISI_PCIE_EVENT_CTRL according to threshold condition. */ |
244 | thr_len = hisi_pcie_get_thr_len(event); |
245 | if (thr_len) { |
246 | reg |= FIELD_PREP(HISI_PCIE_THR_M, thr_len); |
247 | reg |= FIELD_PREP(HISI_PCIE_THR_MODE_M, hisi_pcie_get_thr_mode(event)); |
248 | reg |= HISI_PCIE_THR_EN; |
249 | } |
250 | |
251 | len_mode = hisi_pcie_get_len_mode(event); |
252 | if (len_mode) |
253 | reg |= FIELD_PREP(HISI_PCIE_LEN_M, len_mode); |
254 | else |
255 | reg |= FIELD_PREP(HISI_PCIE_LEN_M, HISI_PCIE_LEN_M_DEFAULT); |
256 | |
257 | return reg; |
258 | } |
259 | |
260 | static void hisi_pcie_pmu_config_event_ctrl(struct perf_event *event) |
261 | { |
262 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
263 | struct hw_perf_event *hwc = &event->hw; |
264 | u64 reg = hisi_pcie_pmu_get_event_ctrl_val(event); |
265 | |
266 | hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx: hwc->idx, val: reg); |
267 | } |
268 | |
269 | static void hisi_pcie_pmu_clear_event_ctrl(struct perf_event *event) |
270 | { |
271 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
272 | struct hw_perf_event *hwc = &event->hw; |
273 | |
274 | hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx: hwc->idx, HISI_PCIE_INIT_SET); |
275 | } |
276 | |
277 | static bool hisi_pcie_pmu_valid_requester_id(struct hisi_pcie_pmu *pcie_pmu, u32 bdf) |
278 | { |
279 | struct pci_dev *root_port, *pdev; |
280 | u16 rp_bdf; |
281 | |
282 | pdev = pci_get_domain_bus_and_slot(domain: pci_domain_nr(bus: pcie_pmu->pdev->bus), PCI_BUS_NUM(bdf), |
283 | GET_PCI_DEVFN(bdf)); |
284 | if (!pdev) |
285 | return false; |
286 | |
287 | root_port = pcie_find_root_port(dev: pdev); |
288 | if (!root_port) { |
289 | pci_dev_put(dev: pdev); |
290 | return false; |
291 | } |
292 | |
293 | pci_dev_put(dev: pdev); |
294 | rp_bdf = pci_dev_id(dev: root_port); |
295 | return rp_bdf >= pcie_pmu->bdf_min && rp_bdf <= pcie_pmu->bdf_max; |
296 | } |
297 | |
298 | static bool hisi_pcie_pmu_valid_filter(struct perf_event *event, |
299 | struct hisi_pcie_pmu *pcie_pmu) |
300 | { |
301 | u32 requester_id = hisi_pcie_get_bdf(event); |
302 | |
303 | if (hisi_pcie_get_thr_len(event) > HISI_PCIE_THR_MAX_VAL) |
304 | return false; |
305 | |
306 | if (hisi_pcie_get_trig_len(event) > HISI_PCIE_TRIG_MAX_VAL) |
307 | return false; |
308 | |
309 | /* Need to explicitly set filter of "port" or "bdf" */ |
310 | if (!hisi_pcie_get_port(event) && |
311 | !hisi_pcie_pmu_valid_requester_id(pcie_pmu, bdf: requester_id)) |
312 | return false; |
313 | |
314 | return true; |
315 | } |
316 | |
317 | /* |
318 | * Check Whether two events share the same config. The same config means not |
319 | * only the event code, but also the filter settings of the two events are |
320 | * the same. |
321 | */ |
322 | static bool hisi_pcie_pmu_cmp_event(struct perf_event *target, |
323 | struct perf_event *event) |
324 | { |
325 | return hisi_pcie_pmu_get_event_ctrl_val(event: target) == |
326 | hisi_pcie_pmu_get_event_ctrl_val(event); |
327 | } |
328 | |
329 | static bool hisi_pcie_pmu_validate_event_group(struct perf_event *event) |
330 | { |
331 | struct perf_event *sibling, *leader = event->group_leader; |
332 | struct perf_event *event_group[HISI_PCIE_MAX_COUNTERS]; |
333 | int counters = 1; |
334 | int num; |
335 | |
336 | event_group[0] = leader; |
337 | if (!is_software_event(event: leader)) { |
338 | if (leader->pmu != event->pmu) |
339 | return false; |
340 | |
341 | if (leader != event && !hisi_pcie_pmu_cmp_event(target: leader, event)) |
342 | event_group[counters++] = event; |
343 | } |
344 | |
345 | for_each_sibling_event(sibling, event->group_leader) { |
346 | if (is_software_event(event: sibling)) |
347 | continue; |
348 | |
349 | if (sibling->pmu != event->pmu) |
350 | return false; |
351 | |
352 | for (num = 0; num < counters; num++) { |
353 | if (hisi_pcie_pmu_cmp_event(target: event_group[num], event: sibling)) |
354 | break; |
355 | } |
356 | |
357 | if (num == counters) |
358 | event_group[counters++] = sibling; |
359 | } |
360 | |
361 | return counters <= HISI_PCIE_MAX_COUNTERS; |
362 | } |
363 | |
364 | static int hisi_pcie_pmu_event_init(struct perf_event *event) |
365 | { |
366 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
367 | struct hw_perf_event *hwc = &event->hw; |
368 | |
369 | /* Check the type first before going on, otherwise it's not our event */ |
370 | if (event->attr.type != event->pmu->type) |
371 | return -ENOENT; |
372 | |
373 | if (EXT_COUNTER_IS_USED(hisi_pcie_get_event(event))) |
374 | hwc->event_base = HISI_PCIE_EXT_CNT; |
375 | else |
376 | hwc->event_base = HISI_PCIE_CNT; |
377 | |
378 | /* Sampling is not supported. */ |
379 | if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) |
380 | return -EOPNOTSUPP; |
381 | |
382 | if (!hisi_pcie_pmu_valid_filter(event, pcie_pmu)) |
383 | return -EINVAL; |
384 | |
385 | if (!hisi_pcie_pmu_validate_event_group(event)) |
386 | return -EINVAL; |
387 | |
388 | event->cpu = pcie_pmu->on_cpu; |
389 | |
390 | return 0; |
391 | } |
392 | |
393 | static u64 hisi_pcie_pmu_read_counter(struct perf_event *event) |
394 | { |
395 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
396 | u32 idx = event->hw.idx; |
397 | |
398 | return hisi_pcie_pmu_readq(pcie_pmu, reg_offset: event->hw.event_base, idx); |
399 | } |
400 | |
401 | /* |
402 | * Check all work events, if a relevant event is found then we return it |
403 | * first, otherwise return the first idle counter (need to reset). |
404 | */ |
405 | static int hisi_pcie_pmu_get_event_idx(struct hisi_pcie_pmu *pcie_pmu, |
406 | struct perf_event *event) |
407 | { |
408 | int first_idle = -EAGAIN; |
409 | struct perf_event *sibling; |
410 | int idx; |
411 | |
412 | for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { |
413 | sibling = pcie_pmu->hw_events[idx]; |
414 | if (!sibling) { |
415 | if (first_idle == -EAGAIN) |
416 | first_idle = idx; |
417 | continue; |
418 | } |
419 | |
420 | /* Related events must be used in group */ |
421 | if (hisi_pcie_pmu_cmp_event(target: sibling, event) && |
422 | sibling->group_leader == event->group_leader) |
423 | return idx; |
424 | } |
425 | |
426 | return first_idle; |
427 | } |
428 | |
429 | static void hisi_pcie_pmu_event_update(struct perf_event *event) |
430 | { |
431 | struct hw_perf_event *hwc = &event->hw; |
432 | u64 new_cnt, prev_cnt, delta; |
433 | |
434 | do { |
435 | prev_cnt = local64_read(&hwc->prev_count); |
436 | new_cnt = hisi_pcie_pmu_read_counter(event); |
437 | } while (local64_cmpxchg(l: &hwc->prev_count, old: prev_cnt, |
438 | new: new_cnt) != prev_cnt); |
439 | |
440 | delta = (new_cnt - prev_cnt) & HISI_PCIE_MAX_PERIOD; |
441 | local64_add(delta, &event->count); |
442 | } |
443 | |
444 | static void hisi_pcie_pmu_read(struct perf_event *event) |
445 | { |
446 | hisi_pcie_pmu_event_update(event); |
447 | } |
448 | |
449 | static void hisi_pcie_pmu_set_period(struct perf_event *event) |
450 | { |
451 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
452 | struct hw_perf_event *hwc = &event->hw; |
453 | int idx = hwc->idx; |
454 | |
455 | local64_set(&hwc->prev_count, HISI_PCIE_INIT_VAL); |
456 | hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_CNT, idx, HISI_PCIE_INIT_VAL); |
457 | hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EXT_CNT, idx, HISI_PCIE_INIT_VAL); |
458 | } |
459 | |
460 | static void hisi_pcie_pmu_enable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) |
461 | { |
462 | u32 idx = hwc->idx; |
463 | u64 val; |
464 | |
465 | val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx); |
466 | val |= HISI_PCIE_EVENT_EN; |
467 | hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val); |
468 | } |
469 | |
470 | static void hisi_pcie_pmu_disable_counter(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) |
471 | { |
472 | u32 idx = hwc->idx; |
473 | u64 val; |
474 | |
475 | val = hisi_pcie_pmu_readq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx); |
476 | val &= ~HISI_PCIE_EVENT_EN; |
477 | hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, val); |
478 | } |
479 | |
480 | static void hisi_pcie_pmu_enable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) |
481 | { |
482 | u32 idx = hwc->idx; |
483 | |
484 | hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, val: 0); |
485 | } |
486 | |
487 | static void hisi_pcie_pmu_disable_int(struct hisi_pcie_pmu *pcie_pmu, struct hw_perf_event *hwc) |
488 | { |
489 | u32 idx = hwc->idx; |
490 | |
491 | hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_MASK, idx, val: 1); |
492 | } |
493 | |
494 | static void hisi_pcie_pmu_reset_counter(struct hisi_pcie_pmu *pcie_pmu, int idx) |
495 | { |
496 | hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_RESET_CNT); |
497 | hisi_pcie_pmu_writeq(pcie_pmu, HISI_PCIE_EVENT_CTRL, idx, HISI_PCIE_INIT_SET); |
498 | } |
499 | |
500 | static void hisi_pcie_pmu_start(struct perf_event *event, int flags) |
501 | { |
502 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
503 | struct hw_perf_event *hwc = &event->hw; |
504 | int idx = hwc->idx; |
505 | u64 prev_cnt; |
506 | |
507 | if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) |
508 | return; |
509 | |
510 | WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
511 | hwc->state = 0; |
512 | |
513 | hisi_pcie_pmu_config_event_ctrl(event); |
514 | hisi_pcie_pmu_enable_counter(pcie_pmu, hwc); |
515 | hisi_pcie_pmu_enable_int(pcie_pmu, hwc); |
516 | hisi_pcie_pmu_set_period(event); |
517 | |
518 | if (flags & PERF_EF_RELOAD) { |
519 | prev_cnt = local64_read(&hwc->prev_count); |
520 | hisi_pcie_pmu_writeq(pcie_pmu, reg_offset: hwc->event_base, idx, val: prev_cnt); |
521 | } |
522 | |
523 | perf_event_update_userpage(event); |
524 | } |
525 | |
526 | static void hisi_pcie_pmu_stop(struct perf_event *event, int flags) |
527 | { |
528 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
529 | struct hw_perf_event *hwc = &event->hw; |
530 | |
531 | hisi_pcie_pmu_event_update(event); |
532 | hisi_pcie_pmu_disable_int(pcie_pmu, hwc); |
533 | hisi_pcie_pmu_disable_counter(pcie_pmu, hwc); |
534 | hisi_pcie_pmu_clear_event_ctrl(event); |
535 | WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); |
536 | hwc->state |= PERF_HES_STOPPED; |
537 | |
538 | if (hwc->state & PERF_HES_UPTODATE) |
539 | return; |
540 | |
541 | hwc->state |= PERF_HES_UPTODATE; |
542 | } |
543 | |
544 | static int hisi_pcie_pmu_add(struct perf_event *event, int flags) |
545 | { |
546 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
547 | struct hw_perf_event *hwc = &event->hw; |
548 | int idx; |
549 | |
550 | hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
551 | |
552 | idx = hisi_pcie_pmu_get_event_idx(pcie_pmu, event); |
553 | if (idx < 0) |
554 | return idx; |
555 | |
556 | hwc->idx = idx; |
557 | |
558 | /* No enabled counter found with related event, reset it */ |
559 | if (!pcie_pmu->hw_events[idx]) { |
560 | hisi_pcie_pmu_reset_counter(pcie_pmu, idx); |
561 | pcie_pmu->hw_events[idx] = event; |
562 | } |
563 | |
564 | if (flags & PERF_EF_START) |
565 | hisi_pcie_pmu_start(event, PERF_EF_RELOAD); |
566 | |
567 | return 0; |
568 | } |
569 | |
570 | static void hisi_pcie_pmu_del(struct perf_event *event, int flags) |
571 | { |
572 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(event->pmu); |
573 | struct hw_perf_event *hwc = &event->hw; |
574 | |
575 | hisi_pcie_pmu_stop(event, PERF_EF_UPDATE); |
576 | pcie_pmu->hw_events[hwc->idx] = NULL; |
577 | perf_event_update_userpage(event); |
578 | } |
579 | |
580 | static void hisi_pcie_pmu_enable(struct pmu *pmu) |
581 | { |
582 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu); |
583 | int num; |
584 | |
585 | for (num = 0; num < HISI_PCIE_MAX_COUNTERS; num++) { |
586 | if (pcie_pmu->hw_events[num]) |
587 | break; |
588 | } |
589 | |
590 | if (num == HISI_PCIE_MAX_COUNTERS) |
591 | return; |
592 | |
593 | writel(HISI_PCIE_GLOBAL_EN, addr: pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL); |
594 | } |
595 | |
596 | static void hisi_pcie_pmu_disable(struct pmu *pmu) |
597 | { |
598 | struct hisi_pcie_pmu *pcie_pmu = to_pcie_pmu(pmu); |
599 | |
600 | writel(HISI_PCIE_GLOBAL_NONE, addr: pcie_pmu->base + HISI_PCIE_GLOBAL_CTRL); |
601 | } |
602 | |
603 | static irqreturn_t hisi_pcie_pmu_irq(int irq, void *data) |
604 | { |
605 | struct hisi_pcie_pmu *pcie_pmu = data; |
606 | irqreturn_t ret = IRQ_NONE; |
607 | struct perf_event *event; |
608 | u32 overflown; |
609 | int idx; |
610 | |
611 | for (idx = 0; idx < HISI_PCIE_MAX_COUNTERS; idx++) { |
612 | overflown = hisi_pcie_pmu_readl(pcie_pmu, HISI_PCIE_INT_STAT, idx); |
613 | if (!overflown) |
614 | continue; |
615 | |
616 | /* Clear status of interrupt. */ |
617 | hisi_pcie_pmu_writel(pcie_pmu, HISI_PCIE_INT_STAT, idx, val: 1); |
618 | event = pcie_pmu->hw_events[idx]; |
619 | if (!event) |
620 | continue; |
621 | |
622 | hisi_pcie_pmu_event_update(event); |
623 | hisi_pcie_pmu_set_period(event); |
624 | ret = IRQ_HANDLED; |
625 | } |
626 | |
627 | return ret; |
628 | } |
629 | |
630 | static int hisi_pcie_pmu_irq_register(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) |
631 | { |
632 | int irq, ret; |
633 | |
634 | ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: 1, PCI_IRQ_MSI); |
635 | if (ret < 0) { |
636 | pci_err(pdev, "Failed to enable MSI vectors: %d\n" , ret); |
637 | return ret; |
638 | } |
639 | |
640 | irq = pci_irq_vector(dev: pdev, nr: 0); |
641 | ret = request_irq(irq, handler: hisi_pcie_pmu_irq, IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME, |
642 | dev: pcie_pmu); |
643 | if (ret) { |
644 | pci_err(pdev, "Failed to register IRQ: %d\n" , ret); |
645 | pci_free_irq_vectors(dev: pdev); |
646 | return ret; |
647 | } |
648 | |
649 | pcie_pmu->irq = irq; |
650 | |
651 | return 0; |
652 | } |
653 | |
654 | static void hisi_pcie_pmu_irq_unregister(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) |
655 | { |
656 | free_irq(pcie_pmu->irq, pcie_pmu); |
657 | pci_free_irq_vectors(dev: pdev); |
658 | } |
659 | |
660 | static int hisi_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) |
661 | { |
662 | struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node); |
663 | |
664 | if (pcie_pmu->on_cpu == -1) { |
665 | pcie_pmu->on_cpu = cpumask_local_spread(i: 0, node: dev_to_node(dev: &pcie_pmu->pdev->dev)); |
666 | WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(pcie_pmu->on_cpu))); |
667 | } |
668 | |
669 | return 0; |
670 | } |
671 | |
672 | static int hisi_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) |
673 | { |
674 | struct hisi_pcie_pmu *pcie_pmu = hlist_entry_safe(node, struct hisi_pcie_pmu, node); |
675 | unsigned int target; |
676 | cpumask_t mask; |
677 | int numa_node; |
678 | |
679 | /* Nothing to do if this CPU doesn't own the PMU */ |
680 | if (pcie_pmu->on_cpu != cpu) |
681 | return 0; |
682 | |
683 | pcie_pmu->on_cpu = -1; |
684 | |
685 | /* Choose a local CPU from all online cpus. */ |
686 | numa_node = dev_to_node(dev: &pcie_pmu->pdev->dev); |
687 | if (cpumask_and(dstp: &mask, src1p: cpumask_of_node(node: numa_node), cpu_online_mask) && |
688 | cpumask_andnot(dstp: &mask, src1p: &mask, cpumask_of(cpu))) |
689 | target = cpumask_any(&mask); |
690 | else |
691 | target = cpumask_any_but(cpu_online_mask, cpu); |
692 | |
693 | if (target >= nr_cpu_ids) { |
694 | pci_err(pcie_pmu->pdev, "There is no CPU to set\n" ); |
695 | return 0; |
696 | } |
697 | |
698 | perf_pmu_migrate_context(pmu: &pcie_pmu->pmu, src_cpu: cpu, dst_cpu: target); |
699 | /* Use this CPU for event counting */ |
700 | pcie_pmu->on_cpu = target; |
701 | WARN_ON(irq_set_affinity(pcie_pmu->irq, cpumask_of(target))); |
702 | |
703 | return 0; |
704 | } |
705 | |
706 | static struct attribute *hisi_pcie_pmu_events_attr[] = { |
707 | HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_latency, 0x0010), |
708 | HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_cnt, 0x10010), |
709 | HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_latency, 0x0210), |
710 | HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_cnt, 0x10210), |
711 | HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_latency, 0x0011), |
712 | HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_cnt, 0x10011), |
713 | HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_flux, 0x0104), |
714 | HISI_PCIE_PMU_EVENT_ATTR(rx_mwr_time, 0x10104), |
715 | HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_flux, 0x0804), |
716 | HISI_PCIE_PMU_EVENT_ATTR(rx_mrd_time, 0x10804), |
717 | HISI_PCIE_PMU_EVENT_ATTR(rx_cpl_flux, 0x2004), |
718 | HISI_PCIE_PMU_EVENT_ATTR(rx_cpl_time, 0x12004), |
719 | HISI_PCIE_PMU_EVENT_ATTR(tx_mwr_flux, 0x0105), |
720 | HISI_PCIE_PMU_EVENT_ATTR(tx_mwr_time, 0x10105), |
721 | HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_flux, 0x0405), |
722 | HISI_PCIE_PMU_EVENT_ATTR(tx_mrd_time, 0x10405), |
723 | HISI_PCIE_PMU_EVENT_ATTR(tx_cpl_flux, 0x1005), |
724 | HISI_PCIE_PMU_EVENT_ATTR(tx_cpl_time, 0x11005), |
725 | NULL |
726 | }; |
727 | |
728 | static struct attribute_group hisi_pcie_pmu_events_group = { |
729 | .name = "events" , |
730 | .attrs = hisi_pcie_pmu_events_attr, |
731 | }; |
732 | |
733 | static struct attribute *hisi_pcie_pmu_format_attr[] = { |
734 | HISI_PCIE_PMU_FORMAT_ATTR(event, "config:0-16" ), |
735 | HISI_PCIE_PMU_FORMAT_ATTR(thr_len, "config1:0-3" ), |
736 | HISI_PCIE_PMU_FORMAT_ATTR(thr_mode, "config1:4" ), |
737 | HISI_PCIE_PMU_FORMAT_ATTR(trig_len, "config1:5-8" ), |
738 | HISI_PCIE_PMU_FORMAT_ATTR(trig_mode, "config1:9" ), |
739 | HISI_PCIE_PMU_FORMAT_ATTR(len_mode, "config1:10-11" ), |
740 | HISI_PCIE_PMU_FORMAT_ATTR(port, "config2:0-15" ), |
741 | HISI_PCIE_PMU_FORMAT_ATTR(bdf, "config2:16-31" ), |
742 | NULL |
743 | }; |
744 | |
745 | static const struct attribute_group hisi_pcie_pmu_format_group = { |
746 | .name = "format" , |
747 | .attrs = hisi_pcie_pmu_format_attr, |
748 | }; |
749 | |
750 | static struct attribute *hisi_pcie_pmu_bus_attrs[] = { |
751 | &dev_attr_bus.attr, |
752 | NULL |
753 | }; |
754 | |
755 | static const struct attribute_group hisi_pcie_pmu_bus_attr_group = { |
756 | .attrs = hisi_pcie_pmu_bus_attrs, |
757 | }; |
758 | |
759 | static struct attribute *hisi_pcie_pmu_cpumask_attrs[] = { |
760 | &dev_attr_cpumask.attr, |
761 | NULL |
762 | }; |
763 | |
764 | static const struct attribute_group hisi_pcie_pmu_cpumask_attr_group = { |
765 | .attrs = hisi_pcie_pmu_cpumask_attrs, |
766 | }; |
767 | |
768 | static struct attribute *hisi_pcie_pmu_identifier_attrs[] = { |
769 | &dev_attr_identifier.attr, |
770 | NULL |
771 | }; |
772 | |
773 | static const struct attribute_group hisi_pcie_pmu_identifier_attr_group = { |
774 | .attrs = hisi_pcie_pmu_identifier_attrs, |
775 | }; |
776 | |
777 | static const struct attribute_group *hisi_pcie_pmu_attr_groups[] = { |
778 | &hisi_pcie_pmu_events_group, |
779 | &hisi_pcie_pmu_format_group, |
780 | &hisi_pcie_pmu_bus_attr_group, |
781 | &hisi_pcie_pmu_cpumask_attr_group, |
782 | &hisi_pcie_pmu_identifier_attr_group, |
783 | NULL |
784 | }; |
785 | |
786 | static int hisi_pcie_alloc_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) |
787 | { |
788 | struct hisi_pcie_reg_pair regs; |
789 | u16 sicl_id, core_id; |
790 | char *name; |
791 | |
792 | regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_BDF); |
793 | pcie_pmu->bdf_min = regs.lo; |
794 | pcie_pmu->bdf_max = regs.hi; |
795 | |
796 | regs = hisi_pcie_parse_reg_value(pcie_pmu, HISI_PCIE_REG_INFO); |
797 | sicl_id = regs.hi; |
798 | core_id = regs.lo; |
799 | |
800 | name = devm_kasprintf(dev: &pdev->dev, GFP_KERNEL, fmt: "hisi_pcie%u_core%u" , sicl_id, core_id); |
801 | if (!name) |
802 | return -ENOMEM; |
803 | |
804 | pcie_pmu->pdev = pdev; |
805 | pcie_pmu->on_cpu = -1; |
806 | pcie_pmu->identifier = readl(addr: pcie_pmu->base + HISI_PCIE_REG_VERSION); |
807 | pcie_pmu->pmu = (struct pmu) { |
808 | .name = name, |
809 | .module = THIS_MODULE, |
810 | .event_init = hisi_pcie_pmu_event_init, |
811 | .pmu_enable = hisi_pcie_pmu_enable, |
812 | .pmu_disable = hisi_pcie_pmu_disable, |
813 | .add = hisi_pcie_pmu_add, |
814 | .del = hisi_pcie_pmu_del, |
815 | .start = hisi_pcie_pmu_start, |
816 | .stop = hisi_pcie_pmu_stop, |
817 | .read = hisi_pcie_pmu_read, |
818 | .task_ctx_nr = perf_invalid_context, |
819 | .attr_groups = hisi_pcie_pmu_attr_groups, |
820 | .capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
821 | }; |
822 | |
823 | return 0; |
824 | } |
825 | |
826 | static int hisi_pcie_init_pmu(struct pci_dev *pdev, struct hisi_pcie_pmu *pcie_pmu) |
827 | { |
828 | int ret; |
829 | |
830 | pcie_pmu->base = pci_ioremap_bar(pdev, bar: 2); |
831 | if (!pcie_pmu->base) { |
832 | pci_err(pdev, "Ioremap failed for pcie_pmu resource\n" ); |
833 | return -ENOMEM; |
834 | } |
835 | |
836 | ret = hisi_pcie_alloc_pmu(pdev, pcie_pmu); |
837 | if (ret) |
838 | goto err_iounmap; |
839 | |
840 | ret = hisi_pcie_pmu_irq_register(pdev, pcie_pmu); |
841 | if (ret) |
842 | goto err_iounmap; |
843 | |
844 | ret = cpuhp_state_add_instance(state: CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, node: &pcie_pmu->node); |
845 | if (ret) { |
846 | pci_err(pdev, "Failed to register hotplug: %d\n" , ret); |
847 | goto err_irq_unregister; |
848 | } |
849 | |
850 | ret = perf_pmu_register(pmu: &pcie_pmu->pmu, name: pcie_pmu->pmu.name, type: -1); |
851 | if (ret) { |
852 | pci_err(pdev, "Failed to register PCIe PMU: %d\n" , ret); |
853 | goto err_hotplug_unregister; |
854 | } |
855 | |
856 | return ret; |
857 | |
858 | err_hotplug_unregister: |
859 | cpuhp_state_remove_instance_nocalls( |
860 | state: CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, node: &pcie_pmu->node); |
861 | |
862 | err_irq_unregister: |
863 | hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu); |
864 | |
865 | err_iounmap: |
866 | iounmap(addr: pcie_pmu->base); |
867 | |
868 | return ret; |
869 | } |
870 | |
871 | static void hisi_pcie_uninit_pmu(struct pci_dev *pdev) |
872 | { |
873 | struct hisi_pcie_pmu *pcie_pmu = pci_get_drvdata(pdev); |
874 | |
875 | perf_pmu_unregister(pmu: &pcie_pmu->pmu); |
876 | cpuhp_state_remove_instance_nocalls( |
877 | state: CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, node: &pcie_pmu->node); |
878 | hisi_pcie_pmu_irq_unregister(pdev, pcie_pmu); |
879 | iounmap(addr: pcie_pmu->base); |
880 | } |
881 | |
882 | static int hisi_pcie_init_dev(struct pci_dev *pdev) |
883 | { |
884 | int ret; |
885 | |
886 | ret = pcim_enable_device(pdev); |
887 | if (ret) { |
888 | pci_err(pdev, "Failed to enable PCI device: %d\n" , ret); |
889 | return ret; |
890 | } |
891 | |
892 | ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME); |
893 | if (ret < 0) { |
894 | pci_err(pdev, "Failed to request PCI mem regions: %d\n" , ret); |
895 | return ret; |
896 | } |
897 | |
898 | pci_set_master(dev: pdev); |
899 | |
900 | return 0; |
901 | } |
902 | |
903 | static int hisi_pcie_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
904 | { |
905 | struct hisi_pcie_pmu *pcie_pmu; |
906 | int ret; |
907 | |
908 | pcie_pmu = devm_kzalloc(dev: &pdev->dev, size: sizeof(*pcie_pmu), GFP_KERNEL); |
909 | if (!pcie_pmu) |
910 | return -ENOMEM; |
911 | |
912 | ret = hisi_pcie_init_dev(pdev); |
913 | if (ret) |
914 | return ret; |
915 | |
916 | ret = hisi_pcie_init_pmu(pdev, pcie_pmu); |
917 | if (ret) |
918 | return ret; |
919 | |
920 | pci_set_drvdata(pdev, data: pcie_pmu); |
921 | |
922 | return ret; |
923 | } |
924 | |
925 | static void hisi_pcie_pmu_remove(struct pci_dev *pdev) |
926 | { |
927 | hisi_pcie_uninit_pmu(pdev); |
928 | pci_set_drvdata(pdev, NULL); |
929 | } |
930 | |
931 | static const struct pci_device_id hisi_pcie_pmu_ids[] = { |
932 | { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12d) }, |
933 | { 0, } |
934 | }; |
935 | MODULE_DEVICE_TABLE(pci, hisi_pcie_pmu_ids); |
936 | |
937 | static struct pci_driver hisi_pcie_pmu_driver = { |
938 | .name = DRV_NAME, |
939 | .id_table = hisi_pcie_pmu_ids, |
940 | .probe = hisi_pcie_pmu_probe, |
941 | .remove = hisi_pcie_pmu_remove, |
942 | }; |
943 | |
944 | static int __init hisi_pcie_module_init(void) |
945 | { |
946 | int ret; |
947 | |
948 | ret = cpuhp_setup_state_multi(state: CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, |
949 | name: "AP_PERF_ARM_HISI_PCIE_PMU_ONLINE" , |
950 | startup: hisi_pcie_pmu_online_cpu, |
951 | teardown: hisi_pcie_pmu_offline_cpu); |
952 | if (ret) { |
953 | pr_err("Failed to setup PCIe PMU hotplug: %d\n" , ret); |
954 | return ret; |
955 | } |
956 | |
957 | ret = pci_register_driver(&hisi_pcie_pmu_driver); |
958 | if (ret) |
959 | cpuhp_remove_multi_state(state: CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE); |
960 | |
961 | return ret; |
962 | } |
963 | module_init(hisi_pcie_module_init); |
964 | |
965 | static void __exit hisi_pcie_module_exit(void) |
966 | { |
967 | pci_unregister_driver(dev: &hisi_pcie_pmu_driver); |
968 | cpuhp_remove_multi_state(state: CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE); |
969 | } |
970 | module_exit(hisi_pcie_module_exit); |
971 | |
972 | MODULE_DESCRIPTION("HiSilicon PCIe PMU driver" ); |
973 | MODULE_LICENSE("GPL v2" ); |
974 | MODULE_AUTHOR("Qi Liu <liuqi115@huawei.com>" ); |
975 | |