1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2020 Intel Corporation. All rights rsvd. */
3
4#include <linux/sched/task.h>
5#include <linux/io-64-nonatomic-lo-hi.h>
6#include "idxd.h"
7#include "perfmon.h"
8
9/*
10 * These attributes specify the bits in the config word that the perf
11 * syscall uses to pass the event ids and categories to perfmon.
12 */
13DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3");
14DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31");
15
16/*
17 * These attributes specify the bits in the config1 word that the perf
18 * syscall uses to pass filter data to perfmon.
19 */
20DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31");
21DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39");
22DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43");
23DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51");
24DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59");
25
26#define PERFMON_FILTERS_START 2
27#define PERFMON_FILTERS_MAX 5
28
29static struct attribute *perfmon_format_attrs[] = {
30 &format_attr_idxd_event_category.attr,
31 &format_attr_idxd_event.attr,
32 &format_attr_idxd_filter_wq.attr,
33 &format_attr_idxd_filter_tc.attr,
34 &format_attr_idxd_filter_pgsz.attr,
35 &format_attr_idxd_filter_sz.attr,
36 &format_attr_idxd_filter_eng.attr,
37 NULL,
38};
39
40static struct attribute_group perfmon_format_attr_group = {
41 .name = "format",
42 .attrs = perfmon_format_attrs,
43};
44
45static const struct attribute_group *perfmon_attr_groups[] = {
46 &perfmon_format_attr_group,
47 NULL,
48};
49
50static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event)
51{
52 return &idxd_pmu->pmu == event->pmu;
53}
54
55static int perfmon_collect_events(struct idxd_pmu *idxd_pmu,
56 struct perf_event *leader,
57 bool do_grp)
58{
59 struct perf_event *event;
60 int n, max_count;
61
62 max_count = idxd_pmu->n_counters;
63 n = idxd_pmu->n_events;
64
65 if (n >= max_count)
66 return -EINVAL;
67
68 if (is_idxd_event(idxd_pmu, event: leader)) {
69 idxd_pmu->event_list[n] = leader;
70 idxd_pmu->event_list[n]->hw.idx = n;
71 n++;
72 }
73
74 if (!do_grp)
75 return n;
76
77 for_each_sibling_event(event, leader) {
78 if (!is_idxd_event(idxd_pmu, event) ||
79 event->state <= PERF_EVENT_STATE_OFF)
80 continue;
81
82 if (n >= max_count)
83 return -EINVAL;
84
85 idxd_pmu->event_list[n] = event;
86 idxd_pmu->event_list[n]->hw.idx = n;
87 n++;
88 }
89
90 return n;
91}
92
93static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu,
94 struct perf_event *event, int idx)
95{
96 struct idxd_device *idxd = idxd_pmu->idxd;
97 struct hw_perf_event *hwc = &event->hw;
98
99 hwc->idx = idx;
100 hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx));
101 hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx));
102}
103
104static int perfmon_assign_event(struct idxd_pmu *idxd_pmu,
105 struct perf_event *event)
106{
107 int i;
108
109 for (i = 0; i < IDXD_PMU_EVENT_MAX; i++)
110 if (!test_and_set_bit(nr: i, addr: idxd_pmu->used_mask))
111 return i;
112
113 return -EINVAL;
114}
115
116/*
117 * Check whether there are enough counters to satisfy that all the
118 * events in the group can actually be scheduled at the same time.
119 *
120 * To do this, create a fake idxd_pmu object so the event collection
121 * and assignment functions can be used without affecting the internal
122 * state of the real idxd_pmu object.
123 */
124static int perfmon_validate_group(struct idxd_pmu *pmu,
125 struct perf_event *event)
126{
127 struct perf_event *leader = event->group_leader;
128 struct idxd_pmu *fake_pmu;
129 int i, ret = 0, n, idx;
130
131 fake_pmu = kzalloc(sizeof(*fake_pmu), GFP_KERNEL);
132 if (!fake_pmu)
133 return -ENOMEM;
134
135 fake_pmu->pmu.name = pmu->pmu.name;
136 fake_pmu->n_counters = pmu->n_counters;
137
138 n = perfmon_collect_events(idxd_pmu: fake_pmu, leader, do_grp: true);
139 if (n < 0) {
140 ret = n;
141 goto out;
142 }
143
144 fake_pmu->n_events = n;
145 n = perfmon_collect_events(idxd_pmu: fake_pmu, leader: event, do_grp: false);
146 if (n < 0) {
147 ret = n;
148 goto out;
149 }
150
151 fake_pmu->n_events = n;
152
153 for (i = 0; i < n; i++) {
154 event = fake_pmu->event_list[i];
155
156 idx = perfmon_assign_event(idxd_pmu: fake_pmu, event);
157 if (idx < 0) {
158 ret = idx;
159 goto out;
160 }
161 }
162out:
163 kfree(objp: fake_pmu);
164
165 return ret;
166}
167
168static int perfmon_pmu_event_init(struct perf_event *event)
169{
170 struct idxd_device *idxd;
171 int ret = 0;
172
173 idxd = event_to_idxd(event);
174 event->hw.idx = -1;
175
176 if (event->attr.type != event->pmu->type)
177 return -ENOENT;
178
179 /* sampling not supported */
180 if (event->attr.sample_period)
181 return -EINVAL;
182
183 if (event->cpu < 0)
184 return -EINVAL;
185
186 if (event->pmu != &idxd->idxd_pmu->pmu)
187 return -EINVAL;
188
189 event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd));
190 event->hw.config = event->attr.config;
191
192 if (event->group_leader != event)
193 /* non-group events have themselves as leader */
194 ret = perfmon_validate_group(pmu: idxd->idxd_pmu, event);
195
196 return ret;
197}
198
199static inline u64 perfmon_pmu_read_counter(struct perf_event *event)
200{
201 struct hw_perf_event *hwc = &event->hw;
202 struct idxd_device *idxd;
203 int cntr = hwc->idx;
204
205 idxd = event_to_idxd(event);
206
207 return ioread64(CNTRDATA_REG(idxd, cntr));
208}
209
210static void perfmon_pmu_event_update(struct perf_event *event)
211{
212 struct idxd_device *idxd = event_to_idxd(event);
213 u64 prev_raw_count, new_raw_count, delta, p, n;
214 int shift = 64 - idxd->idxd_pmu->counter_width;
215 struct hw_perf_event *hwc = &event->hw;
216
217 prev_raw_count = local64_read(&hwc->prev_count);
218 do {
219 new_raw_count = perfmon_pmu_read_counter(event);
220 } while (!local64_try_cmpxchg(l: &hwc->prev_count,
221 old: &prev_raw_count, new: new_raw_count));
222 n = (new_raw_count << shift);
223 p = (prev_raw_count << shift);
224
225 delta = ((n - p) >> shift);
226
227 local64_add(delta, &event->count);
228}
229
230void perfmon_counter_overflow(struct idxd_device *idxd)
231{
232 int i, n_counters, max_loop = OVERFLOW_SIZE;
233 struct perf_event *event;
234 unsigned long ovfstatus;
235
236 n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);
237
238 ovfstatus = ioread32(OVFSTATUS_REG(idxd));
239
240 /*
241 * While updating overflowed counters, other counters behind
242 * them could overflow and be missed in a given pass.
243 * Normally this could happen at most n_counters times, but in
244 * theory a tiny counter width could result in continual
245 * overflows and endless looping. max_loop provides a
246 * failsafe in that highly unlikely case.
247 */
248 while (ovfstatus && max_loop--) {
249 /* Figure out which counter(s) overflowed */
250 for_each_set_bit(i, &ovfstatus, n_counters) {
251 unsigned long ovfstatus_clear = 0;
252
253 /* Update event->count for overflowed counter */
254 event = idxd->idxd_pmu->event_list[i];
255 perfmon_pmu_event_update(event);
256 /* Writing 1 to OVFSTATUS bit clears it */
257 set_bit(nr: i, addr: &ovfstatus_clear);
258 iowrite32(ovfstatus_clear, OVFSTATUS_REG(idxd));
259 }
260
261 ovfstatus = ioread32(OVFSTATUS_REG(idxd));
262 }
263
264 /*
265 * Should never happen. If so, it means a counter(s) looped
266 * around twice while this handler was running.
267 */
268 WARN_ON_ONCE(ovfstatus);
269}
270
271static inline void perfmon_reset_config(struct idxd_device *idxd)
272{
273 iowrite32(CONFIG_RESET, PERFRST_REG(idxd));
274 iowrite32(0, OVFSTATUS_REG(idxd));
275 iowrite32(0, PERFFRZ_REG(idxd));
276}
277
278static inline void perfmon_reset_counters(struct idxd_device *idxd)
279{
280 iowrite32(CNTR_RESET, PERFRST_REG(idxd));
281}
282
283static inline void perfmon_reset(struct idxd_device *idxd)
284{
285 perfmon_reset_config(idxd);
286 perfmon_reset_counters(idxd);
287}
288
289static void perfmon_pmu_event_start(struct perf_event *event, int mode)
290{
291 u32 flt_wq, flt_tc, flt_pg_sz, flt_xfer_sz, flt_eng = 0;
292 u64 cntr_cfg, cntrdata, event_enc, event_cat = 0;
293 struct hw_perf_event *hwc = &event->hw;
294 union filter_cfg flt_cfg;
295 union event_cfg event_cfg;
296 struct idxd_device *idxd;
297 int cntr;
298
299 idxd = event_to_idxd(event);
300
301 event->hw.idx = hwc->idx;
302 cntr = hwc->idx;
303
304 /* Obtain event category and event value from user space */
305 event_cfg.val = event->attr.config;
306 flt_cfg.val = event->attr.config1;
307 event_cat = event_cfg.event_cat;
308 event_enc = event_cfg.event_enc;
309
310 /* Obtain filter configuration from user space */
311 flt_wq = flt_cfg.wq;
312 flt_tc = flt_cfg.tc;
313 flt_pg_sz = flt_cfg.pg_sz;
314 flt_xfer_sz = flt_cfg.xfer_sz;
315 flt_eng = flt_cfg.eng;
316
317 if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))
318 iowrite32(flt_wq, FLTCFG_REG(idxd, cntr, FLT_WQ));
319 if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))
320 iowrite32(flt_tc, FLTCFG_REG(idxd, cntr, FLT_TC));
321 if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))
322 iowrite32(flt_pg_sz, FLTCFG_REG(idxd, cntr, FLT_PG_SZ));
323 if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))
324 iowrite32(flt_xfer_sz, FLTCFG_REG(idxd, cntr, FLT_XFER_SZ));
325 if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))
326 iowrite32(flt_eng, FLTCFG_REG(idxd, cntr, FLT_ENG));
327
328 /* Read the start value */
329 cntrdata = ioread64(CNTRDATA_REG(idxd, cntr));
330 local64_set(&event->hw.prev_count, cntrdata);
331
332 /* Set counter to event/category */
333 cntr_cfg = event_cat << CNTRCFG_CATEGORY_SHIFT;
334 cntr_cfg |= event_enc << CNTRCFG_EVENT_SHIFT;
335 /* Set interrupt on overflow and counter enable bits */
336 cntr_cfg |= (CNTRCFG_IRQ_OVERFLOW | CNTRCFG_ENABLE);
337
338 iowrite64(val: cntr_cfg, CNTRCFG_REG(idxd, cntr));
339}
340
341static void perfmon_pmu_event_stop(struct perf_event *event, int mode)
342{
343 struct hw_perf_event *hwc = &event->hw;
344 struct idxd_device *idxd;
345 int i, cntr = hwc->idx;
346 u64 cntr_cfg;
347
348 idxd = event_to_idxd(event);
349
350 /* remove this event from event list */
351 for (i = 0; i < idxd->idxd_pmu->n_events; i++) {
352 if (event != idxd->idxd_pmu->event_list[i])
353 continue;
354
355 for (++i; i < idxd->idxd_pmu->n_events; i++)
356 idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];
357 --idxd->idxd_pmu->n_events;
358 break;
359 }
360
361 cntr_cfg = ioread64(CNTRCFG_REG(idxd, cntr));
362 cntr_cfg &= ~CNTRCFG_ENABLE;
363 iowrite64(val: cntr_cfg, CNTRCFG_REG(idxd, cntr));
364
365 if (mode == PERF_EF_UPDATE)
366 perfmon_pmu_event_update(event);
367
368 event->hw.idx = -1;
369 clear_bit(nr: cntr, addr: idxd->idxd_pmu->used_mask);
370}
371
372static void perfmon_pmu_event_del(struct perf_event *event, int mode)
373{
374 perfmon_pmu_event_stop(event, PERF_EF_UPDATE);
375}
376
377static int perfmon_pmu_event_add(struct perf_event *event, int flags)
378{
379 struct idxd_device *idxd = event_to_idxd(event);
380 struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;
381 struct hw_perf_event *hwc = &event->hw;
382 int idx, n;
383
384 n = perfmon_collect_events(idxd_pmu, leader: event, do_grp: false);
385 if (n < 0)
386 return n;
387
388 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
389 if (!(flags & PERF_EF_START))
390 hwc->state |= PERF_HES_ARCH;
391
392 idx = perfmon_assign_event(idxd_pmu, event);
393 if (idx < 0)
394 return idx;
395
396 perfmon_assign_hw_event(idxd_pmu, event, idx);
397
398 if (flags & PERF_EF_START)
399 perfmon_pmu_event_start(event, mode: 0);
400
401 idxd_pmu->n_events = n;
402
403 return 0;
404}
405
406static void enable_perfmon_pmu(struct idxd_device *idxd)
407{
408 iowrite32(COUNTER_UNFREEZE, PERFFRZ_REG(idxd));
409}
410
411static void disable_perfmon_pmu(struct idxd_device *idxd)
412{
413 iowrite32(COUNTER_FREEZE, PERFFRZ_REG(idxd));
414}
415
416static void perfmon_pmu_enable(struct pmu *pmu)
417{
418 struct idxd_device *idxd = pmu_to_idxd(pmu);
419
420 enable_perfmon_pmu(idxd);
421}
422
423static void perfmon_pmu_disable(struct pmu *pmu)
424{
425 struct idxd_device *idxd = pmu_to_idxd(pmu);
426
427 disable_perfmon_pmu(idxd);
428}
429
430static void skip_filter(int i)
431{
432 int j;
433
434 for (j = i; j < PERFMON_FILTERS_MAX; j++)
435 perfmon_format_attrs[PERFMON_FILTERS_START + j] =
436 perfmon_format_attrs[PERFMON_FILTERS_START + j + 1];
437}
438
439static void idxd_pmu_init(struct idxd_pmu *idxd_pmu)
440{
441 int i;
442
443 for (i = 0 ; i < PERFMON_FILTERS_MAX; i++) {
444 if (!test_bit(i, &idxd_pmu->supported_filters))
445 skip_filter(i);
446 }
447
448 idxd_pmu->pmu.name = idxd_pmu->name;
449 idxd_pmu->pmu.attr_groups = perfmon_attr_groups;
450 idxd_pmu->pmu.task_ctx_nr = perf_invalid_context;
451 idxd_pmu->pmu.event_init = perfmon_pmu_event_init;
452 idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable;
453 idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable;
454 idxd_pmu->pmu.add = perfmon_pmu_event_add;
455 idxd_pmu->pmu.del = perfmon_pmu_event_del;
456 idxd_pmu->pmu.start = perfmon_pmu_event_start;
457 idxd_pmu->pmu.stop = perfmon_pmu_event_stop;
458 idxd_pmu->pmu.read = perfmon_pmu_event_update;
459 idxd_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE;
460 idxd_pmu->pmu.scope = PERF_PMU_SCOPE_SYS_WIDE;
461 idxd_pmu->pmu.module = THIS_MODULE;
462}
463
464void perfmon_pmu_remove(struct idxd_device *idxd)
465{
466 if (!idxd->idxd_pmu)
467 return;
468
469 perf_pmu_unregister(pmu: &idxd->idxd_pmu->pmu);
470 kfree(objp: idxd->idxd_pmu);
471 idxd->idxd_pmu = NULL;
472}
473
474int perfmon_pmu_init(struct idxd_device *idxd)
475{
476 union idxd_perfcap perfcap;
477 struct idxd_pmu *idxd_pmu;
478 int rc = -ENODEV;
479
480 /*
481 * If perfmon_offset or num_counters is 0, it means perfmon is
482 * not supported on this hardware.
483 */
484 if (idxd->perfmon_offset == 0)
485 return -ENODEV;
486
487 idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL);
488 if (!idxd_pmu)
489 return -ENOMEM;
490
491 idxd_pmu->idxd = idxd;
492 idxd->idxd_pmu = idxd_pmu;
493
494 if (idxd->data->type == IDXD_TYPE_DSA) {
495 rc = sprintf(buf: idxd_pmu->name, fmt: "dsa%d", idxd->id);
496 if (rc < 0)
497 goto free;
498 } else if (idxd->data->type == IDXD_TYPE_IAX) {
499 rc = sprintf(buf: idxd_pmu->name, fmt: "iax%d", idxd->id);
500 if (rc < 0)
501 goto free;
502 } else {
503 goto free;
504 }
505
506 perfmon_reset(idxd);
507
508 perfcap.bits = ioread64(PERFCAP_REG(idxd));
509
510 /*
511 * If total perf counter is 0, stop further registration.
512 * This is necessary in order to support driver running on
513 * guest which does not have pmon support.
514 */
515 if (perfcap.num_perf_counter == 0)
516 goto free;
517
518 /* A counter width of 0 means it can't count */
519 if (perfcap.counter_width == 0)
520 goto free;
521
522 /* Overflow interrupt and counter freeze support must be available */
523 if (!perfcap.overflow_interrupt || !perfcap.counter_freeze)
524 goto free;
525
526 /* Number of event categories cannot be 0 */
527 if (perfcap.num_event_category == 0)
528 goto free;
529
530 /*
531 * We don't support per-counter capabilities for now.
532 */
533 if (perfcap.cap_per_counter)
534 goto free;
535
536 idxd_pmu->n_event_categories = perfcap.num_event_category;
537 idxd_pmu->supported_event_categories = perfcap.global_event_category;
538 idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;
539
540 /* check filter capability. If 0, then filters are not supported */
541 idxd_pmu->supported_filters = perfcap.filter;
542 if (perfcap.filter)
543 idxd_pmu->n_filters = hweight8(perfcap.filter);
544
545 /* Store the total number of counters categories, and counter width */
546 idxd_pmu->n_counters = perfcap.num_perf_counter;
547 idxd_pmu->counter_width = perfcap.counter_width;
548
549 idxd_pmu_init(idxd_pmu);
550
551 rc = perf_pmu_register(pmu: &idxd_pmu->pmu, name: idxd_pmu->name, type: -1);
552 if (rc)
553 goto free;
554
555out:
556 return rc;
557free:
558 kfree(objp: idxd_pmu);
559 idxd->idxd_pmu = NULL;
560
561 goto out;
562}
563

Provided by KDAB

Privacy Policy
Improve your Profiling and Debugging skills
Find out more

source code of linux/drivers/dma/idxd/perfmon.c