1/*
2 * Support cstate residency counters
3 *
4 * Copyright (C) 2015, Intel Corp.
5 * Author: Kan Liang (kan.liang@intel.com)
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
16 *
17 */
18
19/*
20 * This file export cstate related free running (read-only) counters
21 * for perf. These counters may be use simultaneously by other tools,
22 * such as turbostat. However, it still make sense to implement them
23 * in perf. Because we can conveniently collect them together with
24 * other events, and allow to use them from tools without special MSR
25 * access code.
26 *
27 * The events only support system-wide mode counting. There is no
28 * sampling support because it is not supported by the hardware.
29 *
30 * According to counters' scope and category, two PMUs are registered
31 * with the perf_event core subsystem.
32 * - 'cstate_core': The counter is available for each physical core.
33 * The counters include CORE_C*_RESIDENCY.
34 * - 'cstate_pkg': The counter is available for each physical package.
35 * The counters include PKG_C*_RESIDENCY.
36 *
37 * All of these counters are specified in the Intel® 64 and IA-32
38 * Architectures Software Developer.s Manual Vol3b.
39 *
40 * Model specific counters:
41 * MSR_CORE_C1_RES: CORE C1 Residency Counter
42 * perf code: 0x00
43 * Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL
44 * MTL,SRF,GRR
45 * Scope: Core (each processor core has a MSR)
46 * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
47 * perf code: 0x01
48 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,GLM,
49 * CNL,KBL,CML,TNT
50 * Scope: Core
51 * MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter
52 * perf code: 0x02
53 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
54 * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
55 * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
56 * GRR
57 * Scope: Core
58 * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
59 * perf code: 0x03
60 * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML,
61 * ICL,TGL,RKL,ADL,RPL,MTL
62 * Scope: Core
63 * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter.
64 * perf code: 0x00
65 * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL,
66 * KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL,
67 * RPL,SPR,MTL
68 * Scope: Package (physical package)
69 * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter.
70 * perf code: 0x01
71 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL,
72 * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL,
73 * ADL,RPL,MTL
74 * Scope: Package (physical package)
75 * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter.
76 * perf code: 0x02
77 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
78 * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
79 * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF
80 * Scope: Package (physical package)
81 * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter.
82 * perf code: 0x03
83 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,CNL,
84 * KBL,CML,ICL,TGL,RKL,ADL,RPL,MTL
85 * Scope: Package (physical package)
86 * MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter.
87 * perf code: 0x04
88 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL,
89 * ADL,RPL,MTL
90 * Scope: Package (physical package)
91 * MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter.
92 * perf code: 0x05
93 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL,
94 * ADL,RPL,MTL
95 * Scope: Package (physical package)
96 * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
97 * perf code: 0x06
98 * Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL,
99 * TNT,RKL,ADL,RPL,MTL
100 * Scope: Package (physical package)
101 * MSR_MODULE_C6_RES_MS: Module C6 Residency Counter.
102 * perf code: 0x00
103 * Available model: SRF,GRR
104 * Scope: A cluster of cores shared L2 cache
105 *
106 */
107
108#include <linux/module.h>
109#include <linux/slab.h>
110#include <linux/perf_event.h>
111#include <linux/nospec.h>
112#include <asm/cpu_device_id.h>
113#include <asm/intel-family.h>
114#include "../perf_event.h"
115#include "../probe.h"
116
117MODULE_LICENSE("GPL");
118
119#define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format) \
120static ssize_t __cstate_##_var##_show(struct device *dev, \
121 struct device_attribute *attr, \
122 char *page) \
123{ \
124 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
125 return sprintf(page, _format "\n"); \
126} \
127static struct device_attribute format_attr_##_var = \
128 __ATTR(_name, 0444, __cstate_##_var##_show, NULL)
129
130static ssize_t cstate_get_attr_cpumask(struct device *dev,
131 struct device_attribute *attr,
132 char *buf);
133
134/* Model -> events mapping */
135struct cstate_model {
136 unsigned long core_events;
137 unsigned long pkg_events;
138 unsigned long module_events;
139 unsigned long quirks;
140};
141
142/* Quirk flags */
143#define SLM_PKG_C6_USE_C7_MSR (1UL << 0)
144#define KNL_CORE_C6_MSR (1UL << 1)
145
146struct perf_cstate_msr {
147 u64 msr;
148 struct perf_pmu_events_attr *attr;
149};
150
151
152/* cstate_core PMU */
153static struct pmu cstate_core_pmu;
154static bool has_cstate_core;
155
156enum perf_cstate_core_events {
157 PERF_CSTATE_CORE_C1_RES = 0,
158 PERF_CSTATE_CORE_C3_RES,
159 PERF_CSTATE_CORE_C6_RES,
160 PERF_CSTATE_CORE_C7_RES,
161
162 PERF_CSTATE_CORE_EVENT_MAX,
163};
164
165PMU_EVENT_ATTR_STRING(c1-residency, attr_cstate_core_c1, "event=0x00");
166PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_core_c3, "event=0x01");
167PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_core_c6, "event=0x02");
168PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_core_c7, "event=0x03");
169
170static unsigned long core_msr_mask;
171
172PMU_EVENT_GROUP(events, cstate_core_c1);
173PMU_EVENT_GROUP(events, cstate_core_c3);
174PMU_EVENT_GROUP(events, cstate_core_c6);
175PMU_EVENT_GROUP(events, cstate_core_c7);
176
177static bool test_msr(int idx, void *data)
178{
179 return test_bit(idx, (unsigned long *) data);
180}
181
182static struct perf_msr core_msr[] = {
183 [PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES, .grp: &group_cstate_core_c1, .test: test_msr },
184 [PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY, .grp: &group_cstate_core_c3, .test: test_msr },
185 [PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY, .grp: &group_cstate_core_c6, .test: test_msr },
186 [PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY, .grp: &group_cstate_core_c7, .test: test_msr },
187};
188
189static struct attribute *attrs_empty[] = {
190 NULL,
191};
192
193/*
194 * There are no default events, but we need to create
195 * "events" group (with empty attrs) before updating
196 * it with detected events.
197 */
198static struct attribute_group cstate_events_attr_group = {
199 .name = "events",
200 .attrs = attrs_empty,
201};
202
203DEFINE_CSTATE_FORMAT_ATTR(cstate_event, event, "config:0-63");
204static struct attribute *cstate_format_attrs[] = {
205 &format_attr_cstate_event.attr,
206 NULL,
207};
208
209static struct attribute_group cstate_format_attr_group = {
210 .name = "format",
211 .attrs = cstate_format_attrs,
212};
213
214static cpumask_t cstate_core_cpu_mask;
215static DEVICE_ATTR(cpumask, S_IRUGO, cstate_get_attr_cpumask, NULL);
216
217static struct attribute *cstate_cpumask_attrs[] = {
218 &dev_attr_cpumask.attr,
219 NULL,
220};
221
222static struct attribute_group cpumask_attr_group = {
223 .attrs = cstate_cpumask_attrs,
224};
225
226static const struct attribute_group *cstate_attr_groups[] = {
227 &cstate_events_attr_group,
228 &cstate_format_attr_group,
229 &cpumask_attr_group,
230 NULL,
231};
232
233/* cstate_pkg PMU */
234static struct pmu cstate_pkg_pmu;
235static bool has_cstate_pkg;
236
237enum perf_cstate_pkg_events {
238 PERF_CSTATE_PKG_C2_RES = 0,
239 PERF_CSTATE_PKG_C3_RES,
240 PERF_CSTATE_PKG_C6_RES,
241 PERF_CSTATE_PKG_C7_RES,
242 PERF_CSTATE_PKG_C8_RES,
243 PERF_CSTATE_PKG_C9_RES,
244 PERF_CSTATE_PKG_C10_RES,
245
246 PERF_CSTATE_PKG_EVENT_MAX,
247};
248
249PMU_EVENT_ATTR_STRING(c2-residency, attr_cstate_pkg_c2, "event=0x00");
250PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_pkg_c3, "event=0x01");
251PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_pkg_c6, "event=0x02");
252PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_pkg_c7, "event=0x03");
253PMU_EVENT_ATTR_STRING(c8-residency, attr_cstate_pkg_c8, "event=0x04");
254PMU_EVENT_ATTR_STRING(c9-residency, attr_cstate_pkg_c9, "event=0x05");
255PMU_EVENT_ATTR_STRING(c10-residency, attr_cstate_pkg_c10, "event=0x06");
256
257static unsigned long pkg_msr_mask;
258
259PMU_EVENT_GROUP(events, cstate_pkg_c2);
260PMU_EVENT_GROUP(events, cstate_pkg_c3);
261PMU_EVENT_GROUP(events, cstate_pkg_c6);
262PMU_EVENT_GROUP(events, cstate_pkg_c7);
263PMU_EVENT_GROUP(events, cstate_pkg_c8);
264PMU_EVENT_GROUP(events, cstate_pkg_c9);
265PMU_EVENT_GROUP(events, cstate_pkg_c10);
266
267static struct perf_msr pkg_msr[] = {
268 [PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY, .grp: &group_cstate_pkg_c2, .test: test_msr },
269 [PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY, .grp: &group_cstate_pkg_c3, .test: test_msr },
270 [PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY, .grp: &group_cstate_pkg_c6, .test: test_msr },
271 [PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY, .grp: &group_cstate_pkg_c7, .test: test_msr },
272 [PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY, .grp: &group_cstate_pkg_c8, .test: test_msr },
273 [PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY, .grp: &group_cstate_pkg_c9, .test: test_msr },
274 [PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY, .grp: &group_cstate_pkg_c10, .test: test_msr },
275};
276
277static cpumask_t cstate_pkg_cpu_mask;
278
279/* cstate_module PMU */
280static struct pmu cstate_module_pmu;
281static bool has_cstate_module;
282
283enum perf_cstate_module_events {
284 PERF_CSTATE_MODULE_C6_RES = 0,
285
286 PERF_CSTATE_MODULE_EVENT_MAX,
287};
288
289PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_module_c6, "event=0x00");
290
291static unsigned long module_msr_mask;
292
293PMU_EVENT_GROUP(events, cstate_module_c6);
294
295static struct perf_msr module_msr[] = {
296 [PERF_CSTATE_MODULE_C6_RES] = { MSR_MODULE_C6_RES_MS, .grp: &group_cstate_module_c6, .test: test_msr },
297};
298
299static cpumask_t cstate_module_cpu_mask;
300
301static ssize_t cstate_get_attr_cpumask(struct device *dev,
302 struct device_attribute *attr,
303 char *buf)
304{
305 struct pmu *pmu = dev_get_drvdata(dev);
306
307 if (pmu == &cstate_core_pmu)
308 return cpumap_print_to_pagebuf(list: true, buf, mask: &cstate_core_cpu_mask);
309 else if (pmu == &cstate_pkg_pmu)
310 return cpumap_print_to_pagebuf(list: true, buf, mask: &cstate_pkg_cpu_mask);
311 else if (pmu == &cstate_module_pmu)
312 return cpumap_print_to_pagebuf(list: true, buf, mask: &cstate_module_cpu_mask);
313 else
314 return 0;
315}
316
317static int cstate_pmu_event_init(struct perf_event *event)
318{
319 u64 cfg = event->attr.config;
320 int cpu;
321
322 if (event->attr.type != event->pmu->type)
323 return -ENOENT;
324
325 /* unsupported modes and filters */
326 if (event->attr.sample_period) /* no sampling */
327 return -EINVAL;
328
329 if (event->cpu < 0)
330 return -EINVAL;
331
332 if (event->pmu == &cstate_core_pmu) {
333 if (cfg >= PERF_CSTATE_CORE_EVENT_MAX)
334 return -EINVAL;
335 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_CORE_EVENT_MAX);
336 if (!(core_msr_mask & (1 << cfg)))
337 return -EINVAL;
338 event->hw.event_base = core_msr[cfg].msr;
339 cpu = cpumask_any_and(&cstate_core_cpu_mask,
340 topology_sibling_cpumask(event->cpu));
341 } else if (event->pmu == &cstate_pkg_pmu) {
342 if (cfg >= PERF_CSTATE_PKG_EVENT_MAX)
343 return -EINVAL;
344 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX);
345 if (!(pkg_msr_mask & (1 << cfg)))
346 return -EINVAL;
347
348 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
349
350 event->hw.event_base = pkg_msr[cfg].msr;
351 cpu = cpumask_any_and(&cstate_pkg_cpu_mask,
352 topology_die_cpumask(event->cpu));
353 } else if (event->pmu == &cstate_module_pmu) {
354 if (cfg >= PERF_CSTATE_MODULE_EVENT_MAX)
355 return -EINVAL;
356 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_MODULE_EVENT_MAX);
357 if (!(module_msr_mask & (1 << cfg)))
358 return -EINVAL;
359 event->hw.event_base = module_msr[cfg].msr;
360 cpu = cpumask_any_and(&cstate_module_cpu_mask,
361 topology_cluster_cpumask(event->cpu));
362 } else {
363 return -ENOENT;
364 }
365
366 if (cpu >= nr_cpu_ids)
367 return -ENODEV;
368
369 event->cpu = cpu;
370 event->hw.config = cfg;
371 event->hw.idx = -1;
372 return 0;
373}
374
375static inline u64 cstate_pmu_read_counter(struct perf_event *event)
376{
377 u64 val;
378
379 rdmsrl(event->hw.event_base, val);
380 return val;
381}
382
383static void cstate_pmu_event_update(struct perf_event *event)
384{
385 struct hw_perf_event *hwc = &event->hw;
386 u64 prev_raw_count, new_raw_count;
387
388 prev_raw_count = local64_read(&hwc->prev_count);
389 do {
390 new_raw_count = cstate_pmu_read_counter(event);
391 } while (!local64_try_cmpxchg(l: &hwc->prev_count,
392 old: &prev_raw_count, new: new_raw_count));
393
394 local64_add(new_raw_count - prev_raw_count, &event->count);
395}
396
397static void cstate_pmu_event_start(struct perf_event *event, int mode)
398{
399 local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event));
400}
401
402static void cstate_pmu_event_stop(struct perf_event *event, int mode)
403{
404 cstate_pmu_event_update(event);
405}
406
407static void cstate_pmu_event_del(struct perf_event *event, int mode)
408{
409 cstate_pmu_event_stop(event, PERF_EF_UPDATE);
410}
411
412static int cstate_pmu_event_add(struct perf_event *event, int mode)
413{
414 if (mode & PERF_EF_START)
415 cstate_pmu_event_start(event, mode);
416
417 return 0;
418}
419
420/*
421 * Check if exiting cpu is the designated reader. If so migrate the
422 * events when there is a valid target available
423 */
424static int cstate_cpu_exit(unsigned int cpu)
425{
426 unsigned int target;
427
428 if (has_cstate_core &&
429 cpumask_test_and_clear_cpu(cpu, cpumask: &cstate_core_cpu_mask)) {
430
431 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
432 /* Migrate events if there is a valid target */
433 if (target < nr_cpu_ids) {
434 cpumask_set_cpu(cpu: target, dstp: &cstate_core_cpu_mask);
435 perf_pmu_migrate_context(pmu: &cstate_core_pmu, src_cpu: cpu, dst_cpu: target);
436 }
437 }
438
439 if (has_cstate_pkg &&
440 cpumask_test_and_clear_cpu(cpu, cpumask: &cstate_pkg_cpu_mask)) {
441
442 target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
443 /* Migrate events if there is a valid target */
444 if (target < nr_cpu_ids) {
445 cpumask_set_cpu(cpu: target, dstp: &cstate_pkg_cpu_mask);
446 perf_pmu_migrate_context(pmu: &cstate_pkg_pmu, src_cpu: cpu, dst_cpu: target);
447 }
448 }
449
450 if (has_cstate_module &&
451 cpumask_test_and_clear_cpu(cpu, cpumask: &cstate_module_cpu_mask)) {
452
453 target = cpumask_any_but(topology_cluster_cpumask(cpu), cpu);
454 /* Migrate events if there is a valid target */
455 if (target < nr_cpu_ids) {
456 cpumask_set_cpu(cpu: target, dstp: &cstate_module_cpu_mask);
457 perf_pmu_migrate_context(pmu: &cstate_module_pmu, src_cpu: cpu, dst_cpu: target);
458 }
459 }
460 return 0;
461}
462
463static int cstate_cpu_init(unsigned int cpu)
464{
465 unsigned int target;
466
467 /*
468 * If this is the first online thread of that core, set it in
469 * the core cpu mask as the designated reader.
470 */
471 target = cpumask_any_and(&cstate_core_cpu_mask,
472 topology_sibling_cpumask(cpu));
473
474 if (has_cstate_core && target >= nr_cpu_ids)
475 cpumask_set_cpu(cpu, dstp: &cstate_core_cpu_mask);
476
477 /*
478 * If this is the first online thread of that package, set it
479 * in the package cpu mask as the designated reader.
480 */
481 target = cpumask_any_and(&cstate_pkg_cpu_mask,
482 topology_die_cpumask(cpu));
483 if (has_cstate_pkg && target >= nr_cpu_ids)
484 cpumask_set_cpu(cpu, dstp: &cstate_pkg_cpu_mask);
485
486 /*
487 * If this is the first online thread of that cluster, set it
488 * in the cluster cpu mask as the designated reader.
489 */
490 target = cpumask_any_and(&cstate_module_cpu_mask,
491 topology_cluster_cpumask(cpu));
492 if (has_cstate_module && target >= nr_cpu_ids)
493 cpumask_set_cpu(cpu, dstp: &cstate_module_cpu_mask);
494
495 return 0;
496}
497
498static const struct attribute_group *core_attr_update[] = {
499 &group_cstate_core_c1,
500 &group_cstate_core_c3,
501 &group_cstate_core_c6,
502 &group_cstate_core_c7,
503 NULL,
504};
505
506static const struct attribute_group *pkg_attr_update[] = {
507 &group_cstate_pkg_c2,
508 &group_cstate_pkg_c3,
509 &group_cstate_pkg_c6,
510 &group_cstate_pkg_c7,
511 &group_cstate_pkg_c8,
512 &group_cstate_pkg_c9,
513 &group_cstate_pkg_c10,
514 NULL,
515};
516
517static const struct attribute_group *module_attr_update[] = {
518 &group_cstate_module_c6,
519 NULL
520};
521
522static struct pmu cstate_core_pmu = {
523 .attr_groups = cstate_attr_groups,
524 .attr_update = core_attr_update,
525 .name = "cstate_core",
526 .task_ctx_nr = perf_invalid_context,
527 .event_init = cstate_pmu_event_init,
528 .add = cstate_pmu_event_add,
529 .del = cstate_pmu_event_del,
530 .start = cstate_pmu_event_start,
531 .stop = cstate_pmu_event_stop,
532 .read = cstate_pmu_event_update,
533 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
534 .module = THIS_MODULE,
535};
536
537static struct pmu cstate_pkg_pmu = {
538 .attr_groups = cstate_attr_groups,
539 .attr_update = pkg_attr_update,
540 .name = "cstate_pkg",
541 .task_ctx_nr = perf_invalid_context,
542 .event_init = cstate_pmu_event_init,
543 .add = cstate_pmu_event_add,
544 .del = cstate_pmu_event_del,
545 .start = cstate_pmu_event_start,
546 .stop = cstate_pmu_event_stop,
547 .read = cstate_pmu_event_update,
548 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
549 .module = THIS_MODULE,
550};
551
552static struct pmu cstate_module_pmu = {
553 .attr_groups = cstate_attr_groups,
554 .attr_update = module_attr_update,
555 .name = "cstate_module",
556 .task_ctx_nr = perf_invalid_context,
557 .event_init = cstate_pmu_event_init,
558 .add = cstate_pmu_event_add,
559 .del = cstate_pmu_event_del,
560 .start = cstate_pmu_event_start,
561 .stop = cstate_pmu_event_stop,
562 .read = cstate_pmu_event_update,
563 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
564 .module = THIS_MODULE,
565};
566
567static const struct cstate_model nhm_cstates __initconst = {
568 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
569 BIT(PERF_CSTATE_CORE_C6_RES),
570
571 .pkg_events = BIT(PERF_CSTATE_PKG_C3_RES) |
572 BIT(PERF_CSTATE_PKG_C6_RES) |
573 BIT(PERF_CSTATE_PKG_C7_RES),
574};
575
576static const struct cstate_model snb_cstates __initconst = {
577 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
578 BIT(PERF_CSTATE_CORE_C6_RES) |
579 BIT(PERF_CSTATE_CORE_C7_RES),
580
581 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
582 BIT(PERF_CSTATE_PKG_C3_RES) |
583 BIT(PERF_CSTATE_PKG_C6_RES) |
584 BIT(PERF_CSTATE_PKG_C7_RES),
585};
586
587static const struct cstate_model hswult_cstates __initconst = {
588 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
589 BIT(PERF_CSTATE_CORE_C6_RES) |
590 BIT(PERF_CSTATE_CORE_C7_RES),
591
592 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
593 BIT(PERF_CSTATE_PKG_C3_RES) |
594 BIT(PERF_CSTATE_PKG_C6_RES) |
595 BIT(PERF_CSTATE_PKG_C7_RES) |
596 BIT(PERF_CSTATE_PKG_C8_RES) |
597 BIT(PERF_CSTATE_PKG_C9_RES) |
598 BIT(PERF_CSTATE_PKG_C10_RES),
599};
600
601static const struct cstate_model cnl_cstates __initconst = {
602 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
603 BIT(PERF_CSTATE_CORE_C3_RES) |
604 BIT(PERF_CSTATE_CORE_C6_RES) |
605 BIT(PERF_CSTATE_CORE_C7_RES),
606
607 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
608 BIT(PERF_CSTATE_PKG_C3_RES) |
609 BIT(PERF_CSTATE_PKG_C6_RES) |
610 BIT(PERF_CSTATE_PKG_C7_RES) |
611 BIT(PERF_CSTATE_PKG_C8_RES) |
612 BIT(PERF_CSTATE_PKG_C9_RES) |
613 BIT(PERF_CSTATE_PKG_C10_RES),
614};
615
616static const struct cstate_model icl_cstates __initconst = {
617 .core_events = BIT(PERF_CSTATE_CORE_C6_RES) |
618 BIT(PERF_CSTATE_CORE_C7_RES),
619
620 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
621 BIT(PERF_CSTATE_PKG_C3_RES) |
622 BIT(PERF_CSTATE_PKG_C6_RES) |
623 BIT(PERF_CSTATE_PKG_C7_RES) |
624 BIT(PERF_CSTATE_PKG_C8_RES) |
625 BIT(PERF_CSTATE_PKG_C9_RES) |
626 BIT(PERF_CSTATE_PKG_C10_RES),
627};
628
629static const struct cstate_model icx_cstates __initconst = {
630 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
631 BIT(PERF_CSTATE_CORE_C6_RES),
632
633 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
634 BIT(PERF_CSTATE_PKG_C6_RES),
635};
636
637static const struct cstate_model adl_cstates __initconst = {
638 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
639 BIT(PERF_CSTATE_CORE_C6_RES) |
640 BIT(PERF_CSTATE_CORE_C7_RES),
641
642 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
643 BIT(PERF_CSTATE_PKG_C3_RES) |
644 BIT(PERF_CSTATE_PKG_C6_RES) |
645 BIT(PERF_CSTATE_PKG_C7_RES) |
646 BIT(PERF_CSTATE_PKG_C8_RES) |
647 BIT(PERF_CSTATE_PKG_C9_RES) |
648 BIT(PERF_CSTATE_PKG_C10_RES),
649};
650
651static const struct cstate_model slm_cstates __initconst = {
652 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
653 BIT(PERF_CSTATE_CORE_C6_RES),
654
655 .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES),
656 .quirks = SLM_PKG_C6_USE_C7_MSR,
657};
658
659
660static const struct cstate_model knl_cstates __initconst = {
661 .core_events = BIT(PERF_CSTATE_CORE_C6_RES),
662
663 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
664 BIT(PERF_CSTATE_PKG_C3_RES) |
665 BIT(PERF_CSTATE_PKG_C6_RES),
666 .quirks = KNL_CORE_C6_MSR,
667};
668
669
670static const struct cstate_model glm_cstates __initconst = {
671 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
672 BIT(PERF_CSTATE_CORE_C3_RES) |
673 BIT(PERF_CSTATE_CORE_C6_RES),
674
675 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
676 BIT(PERF_CSTATE_PKG_C3_RES) |
677 BIT(PERF_CSTATE_PKG_C6_RES) |
678 BIT(PERF_CSTATE_PKG_C10_RES),
679};
680
681static const struct cstate_model grr_cstates __initconst = {
682 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
683 BIT(PERF_CSTATE_CORE_C6_RES),
684
685 .module_events = BIT(PERF_CSTATE_MODULE_C6_RES),
686};
687
688static const struct cstate_model srf_cstates __initconst = {
689 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
690 BIT(PERF_CSTATE_CORE_C6_RES),
691
692 .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES),
693
694 .module_events = BIT(PERF_CSTATE_MODULE_C6_RES),
695};
696
697
698static const struct x86_cpu_id intel_cstates_match[] __initconst = {
699 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &nhm_cstates),
700 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &nhm_cstates),
701 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &nhm_cstates),
702
703 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &nhm_cstates),
704 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &nhm_cstates),
705 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &nhm_cstates),
706
707 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &snb_cstates),
708 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &snb_cstates),
709
710 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &snb_cstates),
711 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &snb_cstates),
712
713 X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &snb_cstates),
714 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &snb_cstates),
715 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &snb_cstates),
716
717 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &hswult_cstates),
718
719 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &slm_cstates),
720 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &slm_cstates),
721 X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &slm_cstates),
722
723 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &snb_cstates),
724 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &snb_cstates),
725 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &snb_cstates),
726 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &snb_cstates),
727
728 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &snb_cstates),
729 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &snb_cstates),
730 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &snb_cstates),
731
732 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &hswult_cstates),
733 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &hswult_cstates),
734 X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE_L, &hswult_cstates),
735 X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &hswult_cstates),
736
737 X86_MATCH_INTEL_FAM6_MODEL(CANNONLAKE_L, &cnl_cstates),
738
739 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &knl_cstates),
740 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &knl_cstates),
741
742 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &glm_cstates),
743 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &glm_cstates),
744 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &glm_cstates),
745 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &glm_cstates),
746 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT, &glm_cstates),
747 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_L, &glm_cstates),
748 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT, &adl_cstates),
749 X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT_X, &srf_cstates),
750 X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT, &grr_cstates),
751
752 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_cstates),
753 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_cstates),
754 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &icx_cstates),
755 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &icx_cstates),
756 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &icx_cstates),
757 X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &icx_cstates),
758 X86_MATCH_INTEL_FAM6_MODEL(GRANITERAPIDS_X, &icx_cstates),
759 X86_MATCH_INTEL_FAM6_MODEL(GRANITERAPIDS_D, &icx_cstates),
760
761 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &icl_cstates),
762 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &icl_cstates),
763 X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &icl_cstates),
764 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &adl_cstates),
765 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &adl_cstates),
766 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_cstates),
767 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &adl_cstates),
768 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &adl_cstates),
769 X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE, &adl_cstates),
770 X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &adl_cstates),
771 { },
772};
773MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);
774
775static int __init cstate_probe(const struct cstate_model *cm)
776{
777 /* SLM has different MSR for PKG C6 */
778 if (cm->quirks & SLM_PKG_C6_USE_C7_MSR)
779 pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY;
780
781 /* KNL has different MSR for CORE C6 */
782 if (cm->quirks & KNL_CORE_C6_MSR)
783 pkg_msr[PERF_CSTATE_CORE_C6_RES].msr = MSR_KNL_CORE_C6_RESIDENCY;
784
785
786 core_msr_mask = perf_msr_probe(msr: core_msr, cnt: PERF_CSTATE_CORE_EVENT_MAX,
787 no_zero: true, data: (void *) &cm->core_events);
788
789 pkg_msr_mask = perf_msr_probe(msr: pkg_msr, cnt: PERF_CSTATE_PKG_EVENT_MAX,
790 no_zero: true, data: (void *) &cm->pkg_events);
791
792 module_msr_mask = perf_msr_probe(msr: module_msr, cnt: PERF_CSTATE_MODULE_EVENT_MAX,
793 no_zero: true, data: (void *) &cm->module_events);
794
795 has_cstate_core = !!core_msr_mask;
796 has_cstate_pkg = !!pkg_msr_mask;
797 has_cstate_module = !!module_msr_mask;
798
799 return (has_cstate_core || has_cstate_pkg || has_cstate_module) ? 0 : -ENODEV;
800}
801
802static inline void cstate_cleanup(void)
803{
804 cpuhp_remove_state_nocalls(state: CPUHP_AP_PERF_X86_CSTATE_ONLINE);
805 cpuhp_remove_state_nocalls(state: CPUHP_AP_PERF_X86_CSTATE_STARTING);
806
807 if (has_cstate_core)
808 perf_pmu_unregister(pmu: &cstate_core_pmu);
809
810 if (has_cstate_pkg)
811 perf_pmu_unregister(pmu: &cstate_pkg_pmu);
812
813 if (has_cstate_module)
814 perf_pmu_unregister(pmu: &cstate_module_pmu);
815}
816
817static int __init cstate_init(void)
818{
819 int err;
820
821 cpuhp_setup_state(state: CPUHP_AP_PERF_X86_CSTATE_STARTING,
822 name: "perf/x86/cstate:starting", startup: cstate_cpu_init, NULL);
823 cpuhp_setup_state(state: CPUHP_AP_PERF_X86_CSTATE_ONLINE,
824 name: "perf/x86/cstate:online", NULL, teardown: cstate_cpu_exit);
825
826 if (has_cstate_core) {
827 err = perf_pmu_register(pmu: &cstate_core_pmu, name: cstate_core_pmu.name, type: -1);
828 if (err) {
829 has_cstate_core = false;
830 pr_info("Failed to register cstate core pmu\n");
831 cstate_cleanup();
832 return err;
833 }
834 }
835
836 if (has_cstate_pkg) {
837 if (topology_max_dies_per_package() > 1) {
838 err = perf_pmu_register(pmu: &cstate_pkg_pmu,
839 name: "cstate_die", type: -1);
840 } else {
841 err = perf_pmu_register(pmu: &cstate_pkg_pmu,
842 name: cstate_pkg_pmu.name, type: -1);
843 }
844 if (err) {
845 has_cstate_pkg = false;
846 pr_info("Failed to register cstate pkg pmu\n");
847 cstate_cleanup();
848 return err;
849 }
850 }
851
852 if (has_cstate_module) {
853 err = perf_pmu_register(pmu: &cstate_module_pmu, name: cstate_module_pmu.name, type: -1);
854 if (err) {
855 has_cstate_module = false;
856 pr_info("Failed to register cstate cluster pmu\n");
857 cstate_cleanup();
858 return err;
859 }
860 }
861 return 0;
862}
863
864static int __init cstate_pmu_init(void)
865{
866 const struct x86_cpu_id *id;
867 int err;
868
869 if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
870 return -ENODEV;
871
872 id = x86_match_cpu(match: intel_cstates_match);
873 if (!id)
874 return -ENODEV;
875
876 err = cstate_probe(cm: (const struct cstate_model *) id->driver_data);
877 if (err)
878 return err;
879
880 return cstate_init();
881}
882module_init(cstate_pmu_init);
883
884static void __exit cstate_pmu_exit(void)
885{
886 cstate_cleanup();
887}
888module_exit(cstate_pmu_exit);
889

source code of linux/arch/x86/events/intel/cstate.c