1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* turbostat -- show CPU frequency and C-state residency
4	* on modern Intel and AMD processors.
5	*
6	* Copyright (c) 2025 Intel Corporation.
7	* Len Brown <len.brown@intel.com>
8	*/
9
10	#define _GNU_SOURCE
11	#include MSRHEADER
12
13	// copied from arch/x86/include/asm/cpu_device_id.h
14	#define VFM_MODEL_BIT 0
15	#define VFM_FAMILY_BIT 8
16	#define VFM_VENDOR_BIT 16
17	#define VFM_RSVD_BIT 24
18
19	#define VFM_MODEL_MASK GENMASK(VFM_FAMILY_BIT - 1, VFM_MODEL_BIT)
20	#define VFM_FAMILY_MASK GENMASK(VFM_VENDOR_BIT - 1, VFM_FAMILY_BIT)
21	#define VFM_VENDOR_MASK GENMASK(VFM_RSVD_BIT - 1, VFM_VENDOR_BIT)
22
23	#define VFM_MODEL(vfm) (((vfm) & VFM_MODEL_MASK) >> VFM_MODEL_BIT)
24	#define VFM_FAMILY(vfm) (((vfm) & VFM_FAMILY_MASK) >> VFM_FAMILY_BIT)
25	#define VFM_VENDOR(vfm) (((vfm) & VFM_VENDOR_MASK) >> VFM_VENDOR_BIT)
26
27	#define VFM_MAKE(_vendor, _family, _model) ( \
28	((_model) << VFM_MODEL_BIT) | \
29	((_family) << VFM_FAMILY_BIT) | \
30	((_vendor) << VFM_VENDOR_BIT) \
31	)
32	// end copied section
33
34	#define CPUID_LEAF_MODEL_ID 0x1A
35	#define CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT 24
36
37	#define X86_VENDOR_INTEL 0
38
39	#include INTEL_FAMILY_HEADER
40	#include BUILD_BUG_HEADER
41	#include <stdarg.h>
42	#include <stdio.h>
43	#include <err.h>
44	#include <unistd.h>
45	#include <sys/types.h>
46	#include <sys/wait.h>
47	#include <sys/stat.h>
48	#include <sys/select.h>
49	#include <sys/resource.h>
50	#include <sys/mman.h>
51	#include <fcntl.h>
52	#include <signal.h>
53	#include <sys/time.h>
54	#include <stdlib.h>
55	#include <getopt.h>
56	#include <dirent.h>
57	#include <string.h>
58	#include <ctype.h>
59	#include <sched.h>
60	#include <time.h>
61	#include <cpuid.h>
62	#include <sys/capability.h>
63	#include <errno.h>
64	#include <math.h>
65	#include <linux/perf_event.h>
66	#include <asm/unistd.h>
67	#include <stdbool.h>
68	#include <assert.h>
69	#include <linux/kernel.h>
70	#include <limits.h>
71
72	#define UNUSED(x) (void)(x)
73
74	/*
75	* This list matches the column headers, except
76	* 1. built-in only, the sysfs counters are not here -- we learn of those at run-time
77	* 2. Core and CPU are moved to the end, we can't have strings that contain them
78	* matching on them for --show and --hide.
79	*/
80
81	/*
82	* buffer size used by sscanf() for added column names
83	* Usually truncated to 7 characters, but also handles 18 columns for raw 64-bit counters
84	*/
85	#define NAME_BYTES 20
86	#define PATH_BYTES 128
87	#define PERF_NAME_BYTES 128
88
89	#define MAX_NOFILE 0x8000
90
91	#define COUNTER_KIND_PERF_PREFIX "perf/"
92	#define COUNTER_KIND_PERF_PREFIX_LEN strlen(COUNTER_KIND_PERF_PREFIX)
93	#define PERF_DEV_NAME_BYTES 32
94	#define PERF_EVT_NAME_BYTES 32
95
96	#define INTEL_ECORE_TYPE 0x20
97	#define INTEL_PCORE_TYPE 0x40
98
99	#define ROUND_UP_TO_PAGE_SIZE(n) (((n) + 0x1000UL-1UL) & ~(0x1000UL-1UL))
100
101	enum counter_scope { SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE };
102	enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC, COUNTER_K2M };
103	enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE };
104	enum counter_source { COUNTER_SOURCE_NONE, COUNTER_SOURCE_PERF, COUNTER_SOURCE_MSR };
105
106	struct perf_counter_info {
107	struct perf_counter_info *next;
108
109	/* How to open the counter / What counter it is. */
110	char device[PERF_DEV_NAME_BYTES];
111	char event[PERF_EVT_NAME_BYTES];
112
113	/* How to show/format the counter. */
114	char name[PERF_NAME_BYTES];
115	unsigned int width;
116	enum counter_scope scope;
117	enum counter_type type;
118	enum counter_format format;
119	double scale;
120
121	/* For reading the counter. */
122	int *fd_perf_per_domain;
123	size_t num_domains;
124	};
125
126	struct sysfs_path {
127	char path[PATH_BYTES];
128	int id;
129	struct sysfs_path *next;
130	};
131
132	struct msr_counter {
133	unsigned int msr_num;
134	char name[NAME_BYTES];
135	struct sysfs_path *sp;
136	unsigned int width;
137	enum counter_type type;
138	enum counter_format format;
139	struct msr_counter *next;
140	unsigned int flags;
141	#define FLAGS_HIDE (1 << 0)
142	#define FLAGS_SHOW (1 << 1)
143	#define SYSFS_PERCPU (1 << 1)
144	};
145	static int use_android_msr_path;
146
147	struct msr_counter bic[] = {
148	{ 0x0, "usec", NULL, 0, 0, 0, NULL, 0 },
149	{ 0x0, "Time_Of_Day_Seconds", NULL, 0, 0, 0, NULL, 0 },
150	{ 0x0, "Package", NULL, 0, 0, 0, NULL, 0 },
151	{ 0x0, "Node", NULL, 0, 0, 0, NULL, 0 },
152	{ 0x0, "Avg_MHz", NULL, 0, 0, 0, NULL, 0 },
153	{ 0x0, "Busy%", NULL, 0, 0, 0, NULL, 0 },
154	{ 0x0, "Bzy_MHz", NULL, 0, 0, 0, NULL, 0 },
155	{ 0x0, "TSC_MHz", NULL, 0, 0, 0, NULL, 0 },
156	{ 0x0, "IRQ", NULL, 0, 0, 0, NULL, 0 },
157	{ 0x0, "SMI", NULL, 32, 0, FORMAT_DELTA, NULL, 0 },
158	{ 0x0, "cpuidle", NULL, 0, 0, 0, NULL, 0 },
159	{ 0x0, "CPU%c1", NULL, 0, 0, 0, NULL, 0 },
160	{ 0x0, "CPU%c3", NULL, 0, 0, 0, NULL, 0 },
161	{ 0x0, "CPU%c6", NULL, 0, 0, 0, NULL, 0 },
162	{ 0x0, "CPU%c7", NULL, 0, 0, 0, NULL, 0 },
163	{ 0x0, "ThreadC", NULL, 0, 0, 0, NULL, 0 },
164	{ 0x0, "CoreTmp", NULL, 0, 0, 0, NULL, 0 },
165	{ 0x0, "CoreCnt", NULL, 0, 0, 0, NULL, 0 },
166	{ 0x0, "PkgTmp", NULL, 0, 0, 0, NULL, 0 },
167	{ 0x0, "GFX%rc6", NULL, 0, 0, 0, NULL, 0 },
168	{ 0x0, "GFXMHz", NULL, 0, 0, 0, NULL, 0 },
169	{ 0x0, "Pkg%pc2", NULL, 0, 0, 0, NULL, 0 },
170	{ 0x0, "Pkg%pc3", NULL, 0, 0, 0, NULL, 0 },
171	{ 0x0, "Pkg%pc6", NULL, 0, 0, 0, NULL, 0 },
172	{ 0x0, "Pkg%pc7", NULL, 0, 0, 0, NULL, 0 },
173	{ 0x0, "Pkg%pc8", NULL, 0, 0, 0, NULL, 0 },
174	{ 0x0, "Pkg%pc9", NULL, 0, 0, 0, NULL, 0 },
175	{ 0x0, "Pk%pc10", NULL, 0, 0, 0, NULL, 0 },
176	{ 0x0, "CPU%LPI", NULL, 0, 0, 0, NULL, 0 },
177	{ 0x0, "SYS%LPI", NULL, 0, 0, 0, NULL, 0 },
178	{ 0x0, "PkgWatt", NULL, 0, 0, 0, NULL, 0 },
179	{ 0x0, "CorWatt", NULL, 0, 0, 0, NULL, 0 },
180	{ 0x0, "GFXWatt", NULL, 0, 0, 0, NULL, 0 },
181	{ 0x0, "PkgCnt", NULL, 0, 0, 0, NULL, 0 },
182	{ 0x0, "RAMWatt", NULL, 0, 0, 0, NULL, 0 },
183	{ 0x0, "PKG_%", NULL, 0, 0, 0, NULL, 0 },
184	{ 0x0, "RAM_%", NULL, 0, 0, 0, NULL, 0 },
185	{ 0x0, "Pkg_J", NULL, 0, 0, 0, NULL, 0 },
186	{ 0x0, "Cor_J", NULL, 0, 0, 0, NULL, 0 },
187	{ 0x0, "GFX_J", NULL, 0, 0, 0, NULL, 0 },
188	{ 0x0, "RAM_J", NULL, 0, 0, 0, NULL, 0 },
189	{ 0x0, "Mod%c6", NULL, 0, 0, 0, NULL, 0 },
190	{ 0x0, "Totl%C0", NULL, 0, 0, 0, NULL, 0 },
191	{ 0x0, "Any%C0", NULL, 0, 0, 0, NULL, 0 },
192	{ 0x0, "GFX%C0", NULL, 0, 0, 0, NULL, 0 },
193	{ 0x0, "CPUGFX%", NULL, 0, 0, 0, NULL, 0 },
194	{ 0x0, "Core", NULL, 0, 0, 0, NULL, 0 },
195	{ 0x0, "CPU", NULL, 0, 0, 0, NULL, 0 },
196	{ 0x0, "APIC", NULL, 0, 0, 0, NULL, 0 },
197	{ 0x0, "X2APIC", NULL, 0, 0, 0, NULL, 0 },
198	{ 0x0, "Die", NULL, 0, 0, 0, NULL, 0 },
199	{ 0x0, "L3", NULL, 0, 0, 0, NULL, 0 },
200	{ 0x0, "GFXAMHz", NULL, 0, 0, 0, NULL, 0 },
201	{ 0x0, "IPC", NULL, 0, 0, 0, NULL, 0 },
202	{ 0x0, "CoreThr", NULL, 0, 0, 0, NULL, 0 },
203	{ 0x0, "UncMHz", NULL, 0, 0, 0, NULL, 0 },
204	{ 0x0, "SAM%mc6", NULL, 0, 0, 0, NULL, 0 },
205	{ 0x0, "SAMMHz", NULL, 0, 0, 0, NULL, 0 },
206	{ 0x0, "SAMAMHz", NULL, 0, 0, 0, NULL, 0 },
207	{ 0x0, "Die%c6", NULL, 0, 0, 0, NULL, 0 },
208	{ 0x0, "SysWatt", NULL, 0, 0, 0, NULL, 0 },
209	{ 0x0, "Sys_J", NULL, 0, 0, 0, NULL, 0 },
210	{ 0x0, "NMI", NULL, 0, 0, 0, NULL, 0 },
211	{ 0x0, "CPU%c1e", NULL, 0, 0, 0, NULL, 0 },
212	{ 0x0, "pct_idle", NULL, 0, 0, 0, NULL, 0 },
213	{ 0x0, "LLCkRPS", NULL, 0, 0, 0, NULL, 0 },
214	{ 0x0, "LLC%hit", NULL, 0, 0, 0, NULL, 0 },
215	};
216
217	/* n.b. bic_names must match the order in bic[], above */
218	enum bic_names {
219	BIC_USEC,
220	BIC_TOD,
221	BIC_Package,
222	BIC_Node,
223	BIC_Avg_MHz,
224	BIC_Busy,
225	BIC_Bzy_MHz,
226	BIC_TSC_MHz,
227	BIC_IRQ,
228	BIC_SMI,
229	BIC_cpuidle,
230	BIC_CPU_c1,
231	BIC_CPU_c3,
232	BIC_CPU_c6,
233	BIC_CPU_c7,
234	BIC_ThreadC,
235	BIC_CoreTmp,
236	BIC_CoreCnt,
237	BIC_PkgTmp,
238	BIC_GFX_rc6,
239	BIC_GFXMHz,
240	BIC_Pkgpc2,
241	BIC_Pkgpc3,
242	BIC_Pkgpc6,
243	BIC_Pkgpc7,
244	BIC_Pkgpc8,
245	BIC_Pkgpc9,
246	BIC_Pkgpc10,
247	BIC_CPU_LPI,
248	BIC_SYS_LPI,
249	BIC_PkgWatt,
250	BIC_CorWatt,
251	BIC_GFXWatt,
252	BIC_PkgCnt,
253	BIC_RAMWatt,
254	BIC_PKG__,
255	BIC_RAM__,
256	BIC_Pkg_J,
257	BIC_Cor_J,
258	BIC_GFX_J,
259	BIC_RAM_J,
260	BIC_Mod_c6,
261	BIC_Totl_c0,
262	BIC_Any_c0,
263	BIC_GFX_c0,
264	BIC_CPUGFX,
265	BIC_Core,
266	BIC_CPU,
267	BIC_APIC,
268	BIC_X2APIC,
269	BIC_Die,
270	BIC_L3,
271	BIC_GFXACTMHz,
272	BIC_IPC,
273	BIC_CORE_THROT_CNT,
274	BIC_UNCORE_MHZ,
275	BIC_SAM_mc6,
276	BIC_SAMMHz,
277	BIC_SAMACTMHz,
278	BIC_Diec6,
279	BIC_SysWatt,
280	BIC_Sys_J,
281	BIC_NMI,
282	BIC_CPU_c1e,
283	BIC_pct_idle,
284	BIC_LLC_RPS,
285	BIC_LLC_HIT,
286	MAX_BIC
287	};
288
289	void print_bic_set(char *s, cpu_set_t *set)
290	{
291	int i;
292
293	assert(MAX_BIC < CPU_SETSIZE);
294
295	printf("%s:", s);
296
297	for (i = 0; i <= MAX_BIC; ++i) {
298
299	if (CPU_ISSET(i, set)) {
300	assert(i < MAX_BIC);
301	printf(" %s", bic[i].name);
302	}
303	}
304	putchar('\n');
305	}
306
307	static cpu_set_t bic_group_topology;
308	static cpu_set_t bic_group_thermal_pwr;
309	static cpu_set_t bic_group_frequency;
310	static cpu_set_t bic_group_hw_idle;
311	static cpu_set_t bic_group_sw_idle;
312	static cpu_set_t bic_group_idle;
313	static cpu_set_t bic_group_cache;
314	static cpu_set_t bic_group_other;
315	static cpu_set_t bic_group_disabled_by_default;
316	static cpu_set_t bic_enabled;
317	static cpu_set_t bic_present;
318
319	/* modify */
320	#define BIC_INIT(set) CPU_ZERO(set)
321
322	#define SET_BIC(COUNTER_NUMBER, set) CPU_SET(COUNTER_NUMBER, set)
323	#define CLR_BIC(COUNTER_NUMBER, set) CPU_CLR(COUNTER_NUMBER, set)
324
325	#define BIC_PRESENT(COUNTER_NUMBER) SET_BIC(COUNTER_NUMBER, &bic_present)
326	#define BIC_NOT_PRESENT(COUNTER_NUMBER) CPU_CLR(COUNTER_NUMBER, &bic_present)
327
328	/* test */
329	#define BIC_IS_ENABLED(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_enabled)
330	#define DO_BIC_READ(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_present)
331	#define DO_BIC(COUNTER_NUMBER) (CPU_ISSET(COUNTER_NUMBER, &bic_enabled) && CPU_ISSET(COUNTER_NUMBER, &bic_present))
332
333	static void bic_set_all(cpu_set_t *set)
334	{
335	int i;
336
337	assert(MAX_BIC < CPU_SETSIZE);
338
339	for (i = 0; i < MAX_BIC; ++i)
340	SET_BIC(i, set);
341	}
342
343	/*
344	* bic_clear_bits()
345	* clear all the bits from "clr" in "dst"
346	*/
347	static void bic_clear_bits(cpu_set_t *dst, cpu_set_t *clr)
348	{
349	int i;
350
351	assert(MAX_BIC < CPU_SETSIZE);
352
353	for (i = 0; i < MAX_BIC; ++i)
354	if (CPU_ISSET(i, clr))
355	CLR_BIC(i, dst);
356	}
357
358	static void bic_groups_init(void)
359	{
360	BIC_INIT(&bic_group_topology);
361	SET_BIC(BIC_Package, &bic_group_topology);
362	SET_BIC(BIC_Node, &bic_group_topology);
363	SET_BIC(BIC_CoreCnt, &bic_group_topology);
364	SET_BIC(BIC_PkgCnt, &bic_group_topology);
365	SET_BIC(BIC_Core, &bic_group_topology);
366	SET_BIC(BIC_CPU, &bic_group_topology);
367	SET_BIC(BIC_Die, &bic_group_topology);
368	SET_BIC(BIC_L3, &bic_group_topology);
369
370	BIC_INIT(&bic_group_thermal_pwr);
371	SET_BIC(BIC_CoreTmp, &bic_group_thermal_pwr);
372	SET_BIC(BIC_PkgTmp, &bic_group_thermal_pwr);
373	SET_BIC(BIC_PkgWatt, &bic_group_thermal_pwr);
374	SET_BIC(BIC_CorWatt, &bic_group_thermal_pwr);
375	SET_BIC(BIC_GFXWatt, &bic_group_thermal_pwr);
376	SET_BIC(BIC_RAMWatt, &bic_group_thermal_pwr);
377	SET_BIC(BIC_PKG__, &bic_group_thermal_pwr);
378	SET_BIC(BIC_RAM__, &bic_group_thermal_pwr);
379	SET_BIC(BIC_SysWatt, &bic_group_thermal_pwr);
380
381	BIC_INIT(&bic_group_frequency);
382	SET_BIC(BIC_Avg_MHz, &bic_group_frequency);
383	SET_BIC(BIC_Busy, &bic_group_frequency);
384	SET_BIC(BIC_Bzy_MHz, &bic_group_frequency);
385	SET_BIC(BIC_TSC_MHz, &bic_group_frequency);
386	SET_BIC(BIC_GFXMHz, &bic_group_frequency);
387	SET_BIC(BIC_GFXACTMHz, &bic_group_frequency);
388	SET_BIC(BIC_SAMMHz, &bic_group_frequency);
389	SET_BIC(BIC_SAMACTMHz, &bic_group_frequency);
390	SET_BIC(BIC_UNCORE_MHZ, &bic_group_frequency);
391
392	BIC_INIT(&bic_group_hw_idle);
393	SET_BIC(BIC_Busy, &bic_group_hw_idle);
394	SET_BIC(BIC_CPU_c1, &bic_group_hw_idle);
395	SET_BIC(BIC_CPU_c3, &bic_group_hw_idle);
396	SET_BIC(BIC_CPU_c6, &bic_group_hw_idle);
397	SET_BIC(BIC_CPU_c7, &bic_group_hw_idle);
398	SET_BIC(BIC_GFX_rc6, &bic_group_hw_idle);
399	SET_BIC(BIC_Pkgpc2, &bic_group_hw_idle);
400	SET_BIC(BIC_Pkgpc3, &bic_group_hw_idle);
401	SET_BIC(BIC_Pkgpc6, &bic_group_hw_idle);
402	SET_BIC(BIC_Pkgpc7, &bic_group_hw_idle);
403	SET_BIC(BIC_Pkgpc8, &bic_group_hw_idle);
404	SET_BIC(BIC_Pkgpc9, &bic_group_hw_idle);
405	SET_BIC(BIC_Pkgpc10, &bic_group_hw_idle);
406	SET_BIC(BIC_CPU_LPI, &bic_group_hw_idle);
407	SET_BIC(BIC_SYS_LPI, &bic_group_hw_idle);
408	SET_BIC(BIC_Mod_c6, &bic_group_hw_idle);
409	SET_BIC(BIC_Totl_c0, &bic_group_hw_idle);
410	SET_BIC(BIC_Any_c0, &bic_group_hw_idle);
411	SET_BIC(BIC_GFX_c0, &bic_group_hw_idle);
412	SET_BIC(BIC_CPUGFX, &bic_group_hw_idle);
413	SET_BIC(BIC_SAM_mc6, &bic_group_hw_idle);
414	SET_BIC(BIC_Diec6, &bic_group_hw_idle);
415
416	BIC_INIT(&bic_group_sw_idle);
417	SET_BIC(BIC_Busy, &bic_group_sw_idle);
418	SET_BIC(BIC_cpuidle, &bic_group_sw_idle);
419	SET_BIC(BIC_pct_idle, &bic_group_sw_idle);
420
421	BIC_INIT(&bic_group_idle);
422
423	CPU_OR(&bic_group_idle, &bic_group_idle, &bic_group_hw_idle);
424	SET_BIC(BIC_pct_idle, &bic_group_idle);
425
426	BIC_INIT(&bic_group_cache);
427	SET_BIC(BIC_LLC_RPS, &bic_group_cache);
428	SET_BIC(BIC_LLC_HIT, &bic_group_cache);
429
430	BIC_INIT(&bic_group_other);
431	SET_BIC(BIC_IRQ, &bic_group_other);
432	SET_BIC(BIC_NMI, &bic_group_other);
433	SET_BIC(BIC_SMI, &bic_group_other);
434	SET_BIC(BIC_ThreadC, &bic_group_other);
435	SET_BIC(BIC_CoreTmp, &bic_group_other);
436	SET_BIC(BIC_IPC, &bic_group_other);
437
438	BIC_INIT(&bic_group_disabled_by_default);
439	SET_BIC(BIC_USEC, &bic_group_disabled_by_default);
440	SET_BIC(BIC_TOD, &bic_group_disabled_by_default);
441	SET_BIC(BIC_cpuidle, &bic_group_disabled_by_default);
442	SET_BIC(BIC_APIC, &bic_group_disabled_by_default);
443	SET_BIC(BIC_X2APIC, &bic_group_disabled_by_default);
444
445	BIC_INIT(&bic_enabled);
446	bic_set_all(&bic_enabled);
447	bic_clear_bits(&bic_enabled, &bic_group_disabled_by_default);
448
449	BIC_INIT(&bic_present);
450	SET_BIC(BIC_USEC, &bic_present);
451	SET_BIC(BIC_TOD, &bic_present);
452	SET_BIC(BIC_cpuidle, &bic_present);
453	SET_BIC(BIC_APIC, &bic_present);
454	SET_BIC(BIC_X2APIC, &bic_present);
455	SET_BIC(BIC_pct_idle, &bic_present);
456	}
457
458	/*
459	* MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit:
460	* If you change the values, note they are used both in comparisons
461	* (>= PCL__7) and to index pkg_cstate_limit_strings[].
462	*/
463	#define PCLUKN 0 /* Unknown */
464	#define PCLRSV 1 /* Reserved */
465	#define PCL__0 2 /* PC0 */
466	#define PCL__1 3 /* PC1 */
467	#define PCL__2 4 /* PC2 */
468	#define PCL__3 5 /* PC3 */
469	#define PCL__4 6 /* PC4 */
470	#define PCL__6 7 /* PC6 */
471	#define PCL_6N 8 /* PC6 No Retention */
472	#define PCL_6R 9 /* PC6 Retention */
473	#define PCL__7 10 /* PC7 */
474	#define PCL_7S 11 /* PC7 Shrink */
475	#define PCL__8 12 /* PC8 */
476	#define PCL__9 13 /* PC9 */
477	#define PCL_10 14 /* PC10 */
478	#define PCLUNL 15 /* Unlimited */
479
480	char *proc_stat = "/proc/stat";
481	FILE *outf;
482	int *fd_percpu;
483	int *fd_instr_count_percpu;
484	int *fd_llc_percpu;
485	struct timeval interval_tv = { 5, 0 };
486	struct timespec interval_ts = { 5, 0 };
487
488	unsigned int num_iterations;
489	unsigned int header_iterations;
490	unsigned int debug;
491	unsigned int quiet;
492	unsigned int shown;
493	unsigned int sums_need_wide_columns;
494	unsigned int rapl_joules;
495	unsigned int valid_rapl_msrs;
496	unsigned int summary_only;
497	unsigned int list_header_only;
498	unsigned int dump_only;
499	unsigned int force_load;
500	unsigned int cpuid_has_aperf_mperf;
501	unsigned int has_aperf_access;
502	unsigned int has_epb;
503	unsigned int has_turbo;
504	unsigned int is_hybrid;
505	unsigned int units = 1000000; /* MHz etc */
506	unsigned int genuine_intel;
507	unsigned int authentic_amd;
508	unsigned int hygon_genuine;
509	unsigned int max_level, max_extended_level;
510	unsigned int has_invariant_tsc;
511	unsigned int aperf_mperf_multiplier = 1;
512	double bclk;
513	double base_hz;
514	unsigned int has_base_hz;
515	double tsc_tweak = 1.0;
516	unsigned int show_pkg_only;
517	unsigned int show_core_only;
518	char *output_buffer, *outp;
519	unsigned int do_dts;
520	unsigned int do_ptm;
521	unsigned int do_ipc;
522	unsigned long long cpuidle_cur_cpu_lpi_us;
523	unsigned long long cpuidle_cur_sys_lpi_us;
524	unsigned int tj_max;
525	unsigned int tj_max_override;
526	double rapl_power_units, rapl_time_units;
527	double rapl_dram_energy_units, rapl_energy_units, rapl_psys_energy_units;
528	double rapl_joule_counter_range;
529	unsigned int crystal_hz;
530	unsigned long long tsc_hz;
531	int base_cpu;
532	unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */
533	/* IA32_HWP_REQUEST, IA32_HWP_STATUS */
534	unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */
535	unsigned int has_hwp_activity_window; /* IA32_HWP_REQUEST[bits 41:32] */
536	unsigned int has_hwp_epp; /* IA32_HWP_REQUEST[bits 31:24] */
537	unsigned int has_hwp_pkg; /* IA32_HWP_REQUEST_PKG */
538	unsigned int first_counter_read = 1;
539
540	static struct timeval procsysfs_tv_begin;
541
542	int ignore_stdin;
543	bool no_msr;
544	bool no_perf;
545
546	enum gfx_sysfs_idx {
547	GFX_rc6,
548	GFX_MHz,
549	GFX_ACTMHz,
550	SAM_mc6,
551	SAM_MHz,
552	SAM_ACTMHz,
553	GFX_MAX
554	};
555
556	struct gfx_sysfs_info {
557	FILE *fp;
558	unsigned int val;
559	unsigned long long val_ull;
560	};
561
562	static struct gfx_sysfs_info gfx_info[GFX_MAX];
563
564	int get_msr(int cpu, off_t offset, unsigned long long *msr);
565	int add_counter(unsigned int msr_num, char *path, char *name,
566	unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int package_num);
567
568	/* Model specific support Start */
569
570	/* List of features that may diverge among different platforms */
571	struct platform_features {
572	bool has_msr_misc_feature_control; /* MSR_MISC_FEATURE_CONTROL */
573	bool has_msr_misc_pwr_mgmt; /* MSR_MISC_PWR_MGMT */
574	bool has_nhm_msrs; /* MSR_PLATFORM_INFO, MSR_IA32_TEMPERATURE_TARGET, MSR_SMI_COUNT, MSR_PKG_CST_CONFIG_CONTROL, MSR_IA32_POWER_CTL, TRL MSRs */
575	bool has_config_tdp; /* MSR_CONFIG_TDP_NOMINAL/LEVEL_1/LEVEL_2/CONTROL, MSR_TURBO_ACTIVATION_RATIO */
576	int bclk_freq; /* CPU base clock */
577	int crystal_freq; /* Crystal clock to use when not available from CPUID.15 */
578	int supported_cstates; /* Core cstates and Package cstates supported */
579	int cst_limit; /* MSR_PKG_CST_CONFIG_CONTROL */
580	bool has_cst_auto_convension; /* AUTOMATIC_CSTATE_CONVERSION bit in MSR_PKG_CST_CONFIG_CONTROL */
581	bool has_irtl_msrs; /* MSR_PKGC3/PKGC6/PKGC7/PKGC8/PKGC9/PKGC10_IRTL */
582	bool has_msr_core_c1_res; /* MSR_CORE_C1_RES */
583	bool has_msr_module_c6_res_ms; /* MSR_MODULE_C6_RES_MS */
584	bool has_msr_c6_demotion_policy_config; /* MSR_CC6_DEMOTION_POLICY_CONFIG/MSR_MC6_DEMOTION_POLICY_CONFIG */
585	bool has_msr_atom_pkg_c6_residency; /* MSR_ATOM_PKG_C6_RESIDENCY */
586	bool has_msr_knl_core_c6_residency; /* MSR_KNL_CORE_C6_RESIDENCY */
587	bool has_ext_cst_msrs; /* MSR_PKG_WEIGHTED_CORE_C0_RES/MSR_PKG_ANY_CORE_C0_RES/MSR_PKG_ANY_GFXE_C0_RES/MSR_PKG_BOTH_CORE_GFXE_C0_RES */
588	bool has_cst_prewake_bit; /* Cstate prewake bit in MSR_IA32_POWER_CTL */
589	int trl_msrs; /* MSR_TURBO_RATIO_LIMIT/LIMIT1/LIMIT2/SECONDARY, Atom TRL MSRs */
590	int plr_msrs; /* MSR_CORE/GFX/RING_PERF_LIMIT_REASONS */
591	int plat_rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */
592	bool has_per_core_rapl; /* Indicates cores energy collection is per-core, not per-package. AMD specific for now */
593	bool has_rapl_divisor; /* Divisor for Energy unit raw value from MSR_RAPL_POWER_UNIT */
594	bool has_fixed_rapl_unit; /* Fixed Energy Unit used for DRAM RAPL Domain */
595	bool has_fixed_rapl_psys_unit; /* Fixed Energy Unit used for PSYS RAPL Domain */
596	int rapl_quirk_tdp; /* Hardcoded TDP value when cannot be retrieved from hardware */
597	int tcc_offset_bits; /* TCC Offset bits in MSR_IA32_TEMPERATURE_TARGET */
598	bool enable_tsc_tweak; /* Use CPU Base freq instead of TSC freq for aperf/mperf counter */
599	bool need_perf_multiplier; /* mperf/aperf multiplier */
600	};
601
602	struct platform_data {
603	unsigned int vfm;
604	const struct platform_features *features;
605	};
606
607	/* For BCLK */
608	enum bclk_freq {
609	BCLK_100MHZ = 1,
610	BCLK_133MHZ,
611	BCLK_SLV,
612	};
613
614	#define SLM_BCLK_FREQS 5
615	double slm_freq_table[SLM_BCLK_FREQS] = { 83.3, 100.0, 133.3, 116.7, 80.0 };
616
617	double slm_bclk(void)
618	{
619	unsigned long long msr = 3;
620	unsigned int i;
621	double freq;
622
623	if (get_msr(cpu: base_cpu, MSR_FSB_FREQ, msr: &msr))
624	fprintf(outf, "SLM BCLK: unknown\n");
625
626	i = msr & 0xf;
627	if (i >= SLM_BCLK_FREQS) {
628	fprintf(outf, "SLM BCLK[%d] invalid\n", i);
629	i = 3;
630	}
631	freq = slm_freq_table[i];
632
633	if (!quiet)
634	fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq);
635
636	return freq;
637	}
638
639	/* For Package cstate limit */
640	enum package_cstate_limit {
641	CST_LIMIT_NHM = 1,
642	CST_LIMIT_SNB,
643	CST_LIMIT_HSW,
644	CST_LIMIT_SKX,
645	CST_LIMIT_ICX,
646	CST_LIMIT_SLV,
647	CST_LIMIT_AMT,
648	CST_LIMIT_KNL,
649	CST_LIMIT_GMT,
650	};
651
652	/* For Turbo Ratio Limit MSRs */
653	enum turbo_ratio_limit_msrs {
654	TRL_BASE = BIT(0),
655	TRL_LIMIT1 = BIT(1),
656	TRL_LIMIT2 = BIT(2),
657	TRL_ATOM = BIT(3),
658	TRL_KNL = BIT(4),
659	TRL_CORECOUNT = BIT(5),
660	};
661
662	/* For Perf Limit Reason MSRs */
663	enum perf_limit_reason_msrs {
664	PLR_CORE = BIT(0),
665	PLR_GFX = BIT(1),
666	PLR_RING = BIT(2),
667	};
668
669	/* For RAPL MSRs */
670	enum rapl_msrs {
671	RAPL_PKG_POWER_LIMIT = BIT(0), /* 0x610 MSR_PKG_POWER_LIMIT */
672	RAPL_PKG_ENERGY_STATUS = BIT(1), /* 0x611 MSR_PKG_ENERGY_STATUS */
673	RAPL_PKG_PERF_STATUS = BIT(2), /* 0x613 MSR_PKG_PERF_STATUS */
674	RAPL_PKG_POWER_INFO = BIT(3), /* 0x614 MSR_PKG_POWER_INFO */
675	RAPL_DRAM_POWER_LIMIT = BIT(4), /* 0x618 MSR_DRAM_POWER_LIMIT */
676	RAPL_DRAM_ENERGY_STATUS = BIT(5), /* 0x619 MSR_DRAM_ENERGY_STATUS */
677	RAPL_DRAM_PERF_STATUS = BIT(6), /* 0x61b MSR_DRAM_PERF_STATUS */
678	RAPL_DRAM_POWER_INFO = BIT(7), /* 0x61c MSR_DRAM_POWER_INFO */
679	RAPL_CORE_POWER_LIMIT = BIT(8), /* 0x638 MSR_PP0_POWER_LIMIT */
680	RAPL_CORE_ENERGY_STATUS = BIT(9), /* 0x639 MSR_PP0_ENERGY_STATUS */
681	RAPL_CORE_POLICY = BIT(10), /* 0x63a MSR_PP0_POLICY */
682	RAPL_GFX_POWER_LIMIT = BIT(11), /* 0x640 MSR_PP1_POWER_LIMIT */
683	RAPL_GFX_ENERGY_STATUS = BIT(12), /* 0x641 MSR_PP1_ENERGY_STATUS */
684	RAPL_GFX_POLICY = BIT(13), /* 0x642 MSR_PP1_POLICY */
685	RAPL_AMD_PWR_UNIT = BIT(14), /* 0xc0010299 MSR_AMD_RAPL_POWER_UNIT */
686	RAPL_AMD_CORE_ENERGY_STAT = BIT(15), /* 0xc001029a MSR_AMD_CORE_ENERGY_STATUS */
687	RAPL_AMD_PKG_ENERGY_STAT = BIT(16), /* 0xc001029b MSR_AMD_PKG_ENERGY_STATUS */
688	RAPL_PLATFORM_ENERGY_LIMIT = BIT(17), /* 0x64c MSR_PLATFORM_ENERGY_LIMIT */
689	RAPL_PLATFORM_ENERGY_STATUS = BIT(18), /* 0x64d MSR_PLATFORM_ENERGY_STATUS */
690	};
691
692	#define RAPL_PKG (RAPL_PKG_ENERGY_STATUS | RAPL_PKG_POWER_LIMIT)
693	#define RAPL_DRAM (RAPL_DRAM_ENERGY_STATUS | RAPL_DRAM_POWER_LIMIT)
694	#define RAPL_CORE (RAPL_CORE_ENERGY_STATUS | RAPL_CORE_POWER_LIMIT)
695	#define RAPL_GFX (RAPL_GFX_POWER_LIMIT | RAPL_GFX_ENERGY_STATUS)
696	#define RAPL_PSYS (RAPL_PLATFORM_ENERGY_STATUS | RAPL_PLATFORM_ENERGY_LIMIT)
697
698	#define RAPL_PKG_ALL (RAPL_PKG | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO)
699	#define RAPL_DRAM_ALL (RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_DRAM_POWER_INFO)
700	#define RAPL_CORE_ALL (RAPL_CORE | RAPL_CORE_POLICY)
701	#define RAPL_GFX_ALL (RAPL_GFX | RAPL_GFX_POLICY)
702
703	#define RAPL_AMD_F17H (RAPL_AMD_PWR_UNIT | RAPL_AMD_CORE_ENERGY_STAT | RAPL_AMD_PKG_ENERGY_STAT)
704
705	/* For Cstates */
706	enum cstates {
707	CC1 = BIT(0),
708	CC3 = BIT(1),
709	CC6 = BIT(2),
710	CC7 = BIT(3),
711	PC2 = BIT(4),
712	PC3 = BIT(5),
713	PC6 = BIT(6),
714	PC7 = BIT(7),
715	PC8 = BIT(8),
716	PC9 = BIT(9),
717	PC10 = BIT(10),
718	};
719
720	static const struct platform_features nhm_features = {
721	.has_msr_misc_pwr_mgmt = 1,
722	.has_nhm_msrs = 1,
723	.bclk_freq = BCLK_133MHZ,
724	.supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
725	.cst_limit = CST_LIMIT_NHM,
726	.trl_msrs = TRL_BASE,
727	};
728
729	static const struct platform_features nhx_features = {
730	.has_msr_misc_pwr_mgmt = 1,
731	.has_nhm_msrs = 1,
732	.bclk_freq = BCLK_133MHZ,
733	.supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
734	.cst_limit = CST_LIMIT_NHM,
735	};
736
737	static const struct platform_features snb_features = {
738	.has_msr_misc_feature_control = 1,
739	.has_msr_misc_pwr_mgmt = 1,
740	.has_nhm_msrs = 1,
741	.bclk_freq = BCLK_100MHZ,
742	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
743	.cst_limit = CST_LIMIT_SNB,
744	.has_irtl_msrs = 1,
745	.trl_msrs = TRL_BASE,
746	.plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
747	};
748
749	static const struct platform_features snx_features = {
750	.has_msr_misc_feature_control = 1,
751	.has_msr_misc_pwr_mgmt = 1,
752	.has_nhm_msrs = 1,
753	.bclk_freq = BCLK_100MHZ,
754	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
755	.cst_limit = CST_LIMIT_SNB,
756	.has_irtl_msrs = 1,
757	.trl_msrs = TRL_BASE,
758	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
759	};
760
761	static const struct platform_features ivb_features = {
762	.has_msr_misc_feature_control = 1,
763	.has_msr_misc_pwr_mgmt = 1,
764	.has_nhm_msrs = 1,
765	.has_config_tdp = 1,
766	.bclk_freq = BCLK_100MHZ,
767	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
768	.cst_limit = CST_LIMIT_SNB,
769	.has_irtl_msrs = 1,
770	.trl_msrs = TRL_BASE,
771	.plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
772	};
773
774	static const struct platform_features ivx_features = {
775	.has_msr_misc_feature_control = 1,
776	.has_msr_misc_pwr_mgmt = 1,
777	.has_nhm_msrs = 1,
778	.bclk_freq = BCLK_100MHZ,
779	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
780	.cst_limit = CST_LIMIT_SNB,
781	.has_irtl_msrs = 1,
782	.trl_msrs = TRL_BASE | TRL_LIMIT1,
783	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
784	};
785
786	static const struct platform_features hsw_features = {
787	.has_msr_misc_feature_control = 1,
788	.has_msr_misc_pwr_mgmt = 1,
789	.has_nhm_msrs = 1,
790	.has_config_tdp = 1,
791	.bclk_freq = BCLK_100MHZ,
792	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
793	.cst_limit = CST_LIMIT_HSW,
794	.has_irtl_msrs = 1,
795	.trl_msrs = TRL_BASE,
796	.plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
797	.plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
798	};
799
800	static const struct platform_features hsx_features = {
801	.has_msr_misc_feature_control = 1,
802	.has_msr_misc_pwr_mgmt = 1,
803	.has_nhm_msrs = 1,
804	.has_config_tdp = 1,
805	.bclk_freq = BCLK_100MHZ,
806	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
807	.cst_limit = CST_LIMIT_HSW,
808	.has_irtl_msrs = 1,
809	.trl_msrs = TRL_BASE | TRL_LIMIT1 | TRL_LIMIT2,
810	.plr_msrs = PLR_CORE | PLR_RING,
811	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
812	.has_fixed_rapl_unit = 1,
813	};
814
815	static const struct platform_features hswl_features = {
816	.has_msr_misc_feature_control = 1,
817	.has_msr_misc_pwr_mgmt = 1,
818	.has_nhm_msrs = 1,
819	.has_config_tdp = 1,
820	.bclk_freq = BCLK_100MHZ,
821	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
822	.cst_limit = CST_LIMIT_HSW,
823	.has_irtl_msrs = 1,
824	.trl_msrs = TRL_BASE,
825	.plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
826	.plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
827	};
828
829	static const struct platform_features hswg_features = {
830	.has_msr_misc_feature_control = 1,
831	.has_msr_misc_pwr_mgmt = 1,
832	.has_nhm_msrs = 1,
833	.has_config_tdp = 1,
834	.bclk_freq = BCLK_100MHZ,
835	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
836	.cst_limit = CST_LIMIT_HSW,
837	.has_irtl_msrs = 1,
838	.trl_msrs = TRL_BASE,
839	.plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
840	.plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
841	};
842
843	static const struct platform_features bdw_features = {
844	.has_msr_misc_feature_control = 1,
845	.has_msr_misc_pwr_mgmt = 1,
846	.has_nhm_msrs = 1,
847	.has_config_tdp = 1,
848	.bclk_freq = BCLK_100MHZ,
849	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
850	.cst_limit = CST_LIMIT_HSW,
851	.has_irtl_msrs = 1,
852	.trl_msrs = TRL_BASE,
853	.plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
854	};
855
856	static const struct platform_features bdwg_features = {
857	.has_msr_misc_feature_control = 1,
858	.has_msr_misc_pwr_mgmt = 1,
859	.has_nhm_msrs = 1,
860	.has_config_tdp = 1,
861	.bclk_freq = BCLK_100MHZ,
862	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
863	.cst_limit = CST_LIMIT_HSW,
864	.has_irtl_msrs = 1,
865	.trl_msrs = TRL_BASE,
866	.plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
867	};
868
869	static const struct platform_features bdx_features = {
870	.has_msr_misc_feature_control = 1,
871	.has_msr_misc_pwr_mgmt = 1,
872	.has_nhm_msrs = 1,
873	.has_config_tdp = 1,
874	.bclk_freq = BCLK_100MHZ,
875	.supported_cstates = CC1 | CC3 | CC6 | PC2 | PC3 | PC6,
876	.cst_limit = CST_LIMIT_HSW,
877	.has_irtl_msrs = 1,
878	.has_cst_auto_convension = 1,
879	.trl_msrs = TRL_BASE,
880	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
881	.has_fixed_rapl_unit = 1,
882	};
883
884	static const struct platform_features skl_features = {
885	.has_msr_misc_feature_control = 1,
886	.has_msr_misc_pwr_mgmt = 1,
887	.has_nhm_msrs = 1,
888	.has_config_tdp = 1,
889	.bclk_freq = BCLK_100MHZ,
890	.crystal_freq = 24000000,
891	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
892	.cst_limit = CST_LIMIT_HSW,
893	.has_irtl_msrs = 1,
894	.has_ext_cst_msrs = 1,
895	.trl_msrs = TRL_BASE,
896	.tcc_offset_bits = 6,
897	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS,
898	.enable_tsc_tweak = 1,
899	};
900
901	static const struct platform_features cnl_features = {
902	.has_msr_misc_feature_control = 1,
903	.has_msr_misc_pwr_mgmt = 1,
904	.has_nhm_msrs = 1,
905	.has_config_tdp = 1,
906	.bclk_freq = BCLK_100MHZ,
907	.supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
908	.cst_limit = CST_LIMIT_HSW,
909	.has_irtl_msrs = 1,
910	.has_msr_core_c1_res = 1,
911	.has_ext_cst_msrs = 1,
912	.trl_msrs = TRL_BASE,
913	.tcc_offset_bits = 6,
914	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS,
915	.enable_tsc_tweak = 1,
916	};
917
918	/* Copied from cnl_features, with PC7/PC9 removed */
919	static const struct platform_features adl_features = {
920	.has_msr_misc_feature_control = cnl_features.has_msr_misc_feature_control,
921	.has_msr_misc_pwr_mgmt = cnl_features.has_msr_misc_pwr_mgmt,
922	.has_nhm_msrs = cnl_features.has_nhm_msrs,
923	.has_config_tdp = cnl_features.has_config_tdp,
924	.bclk_freq = cnl_features.bclk_freq,
925	.supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC8 | PC10,
926	.cst_limit = cnl_features.cst_limit,
927	.has_irtl_msrs = cnl_features.has_irtl_msrs,
928	.has_msr_core_c1_res = cnl_features.has_msr_core_c1_res,
929	.has_ext_cst_msrs = cnl_features.has_ext_cst_msrs,
930	.trl_msrs = cnl_features.trl_msrs,
931	.tcc_offset_bits = cnl_features.tcc_offset_bits,
932	.plat_rapl_msrs = cnl_features.plat_rapl_msrs,
933	.enable_tsc_tweak = cnl_features.enable_tsc_tweak,
934	};
935
936	/* Copied from adl_features, with PC3/PC8 removed */
937	static const struct platform_features lnl_features = {
938	.has_msr_misc_feature_control = adl_features.has_msr_misc_feature_control,
939	.has_msr_misc_pwr_mgmt = adl_features.has_msr_misc_pwr_mgmt,
940	.has_nhm_msrs = adl_features.has_nhm_msrs,
941	.has_config_tdp = adl_features.has_config_tdp,
942	.bclk_freq = adl_features.bclk_freq,
943	.supported_cstates = CC1 | CC6 | CC7 | PC2 | PC6 | PC10,
944	.cst_limit = adl_features.cst_limit,
945	.has_irtl_msrs = adl_features.has_irtl_msrs,
946	.has_msr_core_c1_res = adl_features.has_msr_core_c1_res,
947	.has_ext_cst_msrs = adl_features.has_ext_cst_msrs,
948	.trl_msrs = adl_features.trl_msrs,
949	.tcc_offset_bits = adl_features.tcc_offset_bits,
950	.plat_rapl_msrs = adl_features.plat_rapl_msrs,
951	.enable_tsc_tweak = adl_features.enable_tsc_tweak,
952	};
953
954	static const struct platform_features skx_features = {
955	.has_msr_misc_feature_control = 1,
956	.has_msr_misc_pwr_mgmt = 1,
957	.has_nhm_msrs = 1,
958	.has_config_tdp = 1,
959	.bclk_freq = BCLK_100MHZ,
960	.supported_cstates = CC1 | CC6 | PC2 | PC6,
961	.cst_limit = CST_LIMIT_SKX,
962	.has_irtl_msrs = 1,
963	.has_cst_auto_convension = 1,
964	.trl_msrs = TRL_BASE | TRL_CORECOUNT,
965	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
966	.has_fixed_rapl_unit = 1,
967	};
968
969	static const struct platform_features icx_features = {
970	.has_msr_misc_feature_control = 1,
971	.has_msr_misc_pwr_mgmt = 1,
972	.has_nhm_msrs = 1,
973	.has_config_tdp = 1,
974	.bclk_freq = BCLK_100MHZ,
975	.supported_cstates = CC1 | CC6 | PC2 | PC6,
976	.cst_limit = CST_LIMIT_ICX,
977	.has_msr_core_c1_res = 1,
978	.has_irtl_msrs = 1,
979	.has_cst_prewake_bit = 1,
980	.trl_msrs = TRL_BASE | TRL_CORECOUNT,
981	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
982	.has_fixed_rapl_unit = 1,
983	};
984
985	static const struct platform_features spr_features = {
986	.has_msr_misc_feature_control = 1,
987	.has_msr_misc_pwr_mgmt = 1,
988	.has_nhm_msrs = 1,
989	.has_config_tdp = 1,
990	.bclk_freq = BCLK_100MHZ,
991	.supported_cstates = CC1 | CC6 | PC2 | PC6,
992	.cst_limit = CST_LIMIT_SKX,
993	.has_msr_core_c1_res = 1,
994	.has_irtl_msrs = 1,
995	.has_cst_prewake_bit = 1,
996	.has_fixed_rapl_psys_unit = 1,
997	.trl_msrs = TRL_BASE | TRL_CORECOUNT,
998	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
999	};
1000
1001	static const struct platform_features dmr_features = {
1002	.has_msr_misc_feature_control = spr_features.has_msr_misc_feature_control,
1003	.has_msr_misc_pwr_mgmt = spr_features.has_msr_misc_pwr_mgmt,
1004	.has_nhm_msrs = spr_features.has_nhm_msrs,
1005	.bclk_freq = spr_features.bclk_freq,
1006	.supported_cstates = spr_features.supported_cstates,
1007	.cst_limit = spr_features.cst_limit,
1008	.has_msr_core_c1_res = spr_features.has_msr_core_c1_res,
1009	.has_cst_prewake_bit = spr_features.has_cst_prewake_bit,
1010	.has_fixed_rapl_psys_unit = spr_features.has_fixed_rapl_psys_unit,
1011	.trl_msrs = spr_features.trl_msrs,
1012	.has_msr_module_c6_res_ms = 1, /* DMR has Dual-Core-Module and MC6 MSR */
1013	.plat_rapl_msrs = 0, /* DMR does not have RAPL MSRs */
1014	.plr_msrs = 0, /* DMR does not have PLR MSRs */
1015	.has_irtl_msrs = 0, /* DMR does not have IRTL MSRs */
1016	.has_config_tdp = 0, /* DMR does not have CTDP MSRs */
1017	};
1018
1019	static const struct platform_features srf_features = {
1020	.has_msr_misc_feature_control = 1,
1021	.has_msr_misc_pwr_mgmt = 1,
1022	.has_nhm_msrs = 1,
1023	.has_config_tdp = 1,
1024	.bclk_freq = BCLK_100MHZ,
1025	.supported_cstates = CC1 | CC6 | PC2 | PC6,
1026	.cst_limit = CST_LIMIT_SKX,
1027	.has_msr_core_c1_res = 1,
1028	.has_msr_module_c6_res_ms = 1,
1029	.has_irtl_msrs = 1,
1030	.has_cst_prewake_bit = 1,
1031	.trl_msrs = TRL_BASE | TRL_CORECOUNT,
1032	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1033	};
1034
1035	static const struct platform_features grr_features = {
1036	.has_msr_misc_feature_control = 1,
1037	.has_msr_misc_pwr_mgmt = 1,
1038	.has_nhm_msrs = 1,
1039	.has_config_tdp = 1,
1040	.bclk_freq = BCLK_100MHZ,
1041	.supported_cstates = CC1 | CC6,
1042	.cst_limit = CST_LIMIT_SKX,
1043	.has_msr_core_c1_res = 1,
1044	.has_msr_module_c6_res_ms = 1,
1045	.has_irtl_msrs = 1,
1046	.has_cst_prewake_bit = 1,
1047	.trl_msrs = TRL_BASE | TRL_CORECOUNT,
1048	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1049	};
1050
1051	static const struct platform_features slv_features = {
1052	.has_nhm_msrs = 1,
1053	.bclk_freq = BCLK_SLV,
1054	.supported_cstates = CC1 | CC6 | PC6,
1055	.cst_limit = CST_LIMIT_SLV,
1056	.has_msr_core_c1_res = 1,
1057	.has_msr_module_c6_res_ms = 1,
1058	.has_msr_c6_demotion_policy_config = 1,
1059	.has_msr_atom_pkg_c6_residency = 1,
1060	.trl_msrs = TRL_ATOM,
1061	.plat_rapl_msrs = RAPL_PKG | RAPL_CORE,
1062	.has_rapl_divisor = 1,
1063	.rapl_quirk_tdp = 30,
1064	};
1065
1066	static const struct platform_features slvd_features = {
1067	.has_msr_misc_pwr_mgmt = 1,
1068	.has_nhm_msrs = 1,
1069	.bclk_freq = BCLK_SLV,
1070	.supported_cstates = CC1 | CC6 | PC3 | PC6,
1071	.cst_limit = CST_LIMIT_SLV,
1072	.has_msr_atom_pkg_c6_residency = 1,
1073	.trl_msrs = TRL_BASE,
1074	.plat_rapl_msrs = RAPL_PKG | RAPL_CORE,
1075	.rapl_quirk_tdp = 30,
1076	};
1077
1078	static const struct platform_features amt_features = {
1079	.has_nhm_msrs = 1,
1080	.bclk_freq = BCLK_133MHZ,
1081	.supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
1082	.cst_limit = CST_LIMIT_AMT,
1083	.trl_msrs = TRL_BASE,
1084	};
1085
1086	static const struct platform_features gmt_features = {
1087	.has_msr_misc_pwr_mgmt = 1,
1088	.has_nhm_msrs = 1,
1089	.bclk_freq = BCLK_100MHZ,
1090	.crystal_freq = 19200000,
1091	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1092	.cst_limit = CST_LIMIT_GMT,
1093	.has_irtl_msrs = 1,
1094	.trl_msrs = TRL_BASE | TRL_CORECOUNT,
1095	.plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
1096	};
1097
1098	static const struct platform_features gmtd_features = {
1099	.has_msr_misc_pwr_mgmt = 1,
1100	.has_nhm_msrs = 1,
1101	.bclk_freq = BCLK_100MHZ,
1102	.crystal_freq = 25000000,
1103	.supported_cstates = CC1 | CC6 | PC2 | PC6,
1104	.cst_limit = CST_LIMIT_GMT,
1105	.has_irtl_msrs = 1,
1106	.has_msr_core_c1_res = 1,
1107	.trl_msrs = TRL_BASE | TRL_CORECOUNT,
1108	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS,
1109	};
1110
1111	static const struct platform_features gmtp_features = {
1112	.has_msr_misc_pwr_mgmt = 1,
1113	.has_nhm_msrs = 1,
1114	.bclk_freq = BCLK_100MHZ,
1115	.crystal_freq = 19200000,
1116	.supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1117	.cst_limit = CST_LIMIT_GMT,
1118	.has_irtl_msrs = 1,
1119	.trl_msrs = TRL_BASE,
1120	.plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
1121	};
1122
1123	static const struct platform_features tmt_features = {
1124	.has_msr_misc_pwr_mgmt = 1,
1125	.has_nhm_msrs = 1,
1126	.bclk_freq = BCLK_100MHZ,
1127	.supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1128	.cst_limit = CST_LIMIT_GMT,
1129	.has_irtl_msrs = 1,
1130	.trl_msrs = TRL_BASE,
1131	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX,
1132	.enable_tsc_tweak = 1,
1133	};
1134
1135	static const struct platform_features tmtd_features = {
1136	.has_msr_misc_pwr_mgmt = 1,
1137	.has_nhm_msrs = 1,
1138	.bclk_freq = BCLK_100MHZ,
1139	.supported_cstates = CC1 | CC6,
1140	.cst_limit = CST_LIMIT_GMT,
1141	.has_irtl_msrs = 1,
1142	.trl_msrs = TRL_BASE | TRL_CORECOUNT,
1143	.plat_rapl_msrs = RAPL_PKG_ALL,
1144	};
1145
1146	static const struct platform_features knl_features = {
1147	.has_msr_misc_pwr_mgmt = 1,
1148	.has_nhm_msrs = 1,
1149	.has_config_tdp = 1,
1150	.bclk_freq = BCLK_100MHZ,
1151	.supported_cstates = CC1 | CC6 | PC3 | PC6,
1152	.cst_limit = CST_LIMIT_KNL,
1153	.has_msr_knl_core_c6_residency = 1,
1154	.trl_msrs = TRL_KNL,
1155	.plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
1156	.has_fixed_rapl_unit = 1,
1157	.need_perf_multiplier = 1,
1158	};
1159
1160	static const struct platform_features default_features = {
1161	};
1162
1163	static const struct platform_features amd_features_with_rapl = {
1164	.plat_rapl_msrs = RAPL_AMD_F17H,
1165	.has_per_core_rapl = 1,
1166	.rapl_quirk_tdp = 280, /* This is the max stock TDP of HEDT/Server Fam17h+ chips */
1167	};
1168
1169	static const struct platform_data turbostat_pdata[] = {
1170	{ INTEL_NEHALEM, &nhm_features },
1171	{ INTEL_NEHALEM_G, &nhm_features },
1172	{ INTEL_NEHALEM_EP, &nhm_features },
1173	{ INTEL_NEHALEM_EX, &nhx_features },
1174	{ INTEL_WESTMERE, &nhm_features },
1175	{ INTEL_WESTMERE_EP, &nhm_features },
1176	{ INTEL_WESTMERE_EX, &nhx_features },
1177	{ INTEL_SANDYBRIDGE, &snb_features },
1178	{ INTEL_SANDYBRIDGE_X, &snx_features },
1179	{ INTEL_IVYBRIDGE, &ivb_features },
1180	{ INTEL_IVYBRIDGE_X, &ivx_features },
1181	{ INTEL_HASWELL, &hsw_features },
1182	{ INTEL_HASWELL_X, &hsx_features },
1183	{ INTEL_HASWELL_L, &hswl_features },
1184	{ INTEL_HASWELL_G, &hswg_features },
1185	{ INTEL_BROADWELL, &bdw_features },
1186	{ INTEL_BROADWELL_G, &bdwg_features },
1187	{ INTEL_BROADWELL_X, &bdx_features },
1188	{ INTEL_BROADWELL_D, &bdx_features },
1189	{ INTEL_SKYLAKE_L, &skl_features },
1190	{ INTEL_SKYLAKE, &skl_features },
1191	{ INTEL_SKYLAKE_X, &skx_features },
1192	{ INTEL_KABYLAKE_L, &skl_features },
1193	{ INTEL_KABYLAKE, &skl_features },
1194	{ INTEL_COMETLAKE, &skl_features },
1195	{ INTEL_COMETLAKE_L, &skl_features },
1196	{ INTEL_CANNONLAKE_L, &cnl_features },
1197	{ INTEL_ICELAKE_X, &icx_features },
1198	{ INTEL_ICELAKE_D, &icx_features },
1199	{ INTEL_ICELAKE_L, &cnl_features },
1200	{ INTEL_ICELAKE_NNPI, &cnl_features },
1201	{ INTEL_ROCKETLAKE, &cnl_features },
1202	{ INTEL_TIGERLAKE_L, &cnl_features },
1203	{ INTEL_TIGERLAKE, &cnl_features },
1204	{ INTEL_SAPPHIRERAPIDS_X, &spr_features },
1205	{ INTEL_EMERALDRAPIDS_X, &spr_features },
1206	{ INTEL_GRANITERAPIDS_X, &spr_features },
1207	{ INTEL_GRANITERAPIDS_D, &spr_features },
1208	{ INTEL_DIAMONDRAPIDS_X, &dmr_features },
1209	{ INTEL_LAKEFIELD, &cnl_features },
1210	{ INTEL_ALDERLAKE, &adl_features },
1211	{ INTEL_ALDERLAKE_L, &adl_features },
1212	{ INTEL_RAPTORLAKE, &adl_features },
1213	{ INTEL_RAPTORLAKE_P, &adl_features },
1214	{ INTEL_RAPTORLAKE_S, &adl_features },
1215	{ INTEL_BARTLETTLAKE, &adl_features },
1216	{ INTEL_METEORLAKE, &adl_features },
1217	{ INTEL_METEORLAKE_L, &adl_features },
1218	{ INTEL_ARROWLAKE_H, &adl_features },
1219	{ INTEL_ARROWLAKE_U, &adl_features },
1220	{ INTEL_ARROWLAKE, &adl_features },
1221	{ INTEL_LUNARLAKE_M, &lnl_features },
1222	{ INTEL_PANTHERLAKE_L, &lnl_features },
1223	{ INTEL_NOVALAKE, &lnl_features },
1224	{ INTEL_NOVALAKE_L, &lnl_features },
1225	{ INTEL_WILDCATLAKE_L, &lnl_features },
1226	{ INTEL_ATOM_SILVERMONT, &slv_features },
1227	{ INTEL_ATOM_SILVERMONT_D, &slvd_features },
1228	{ INTEL_ATOM_AIRMONT, &amt_features },
1229	{ INTEL_ATOM_GOLDMONT, &gmt_features },
1230	{ INTEL_ATOM_GOLDMONT_D, &gmtd_features },
1231	{ INTEL_ATOM_GOLDMONT_PLUS, &gmtp_features },
1232	{ INTEL_ATOM_TREMONT_D, &tmtd_features },
1233	{ INTEL_ATOM_TREMONT, &tmt_features },
1234	{ INTEL_ATOM_TREMONT_L, &tmt_features },
1235	{ INTEL_ATOM_GRACEMONT, &adl_features },
1236	{ INTEL_ATOM_CRESTMONT_X, &srf_features },
1237	{ INTEL_ATOM_CRESTMONT, &grr_features },
1238	{ INTEL_ATOM_DARKMONT_X, &srf_features },
1239	{ INTEL_XEON_PHI_KNL, &knl_features },
1240	{ INTEL_XEON_PHI_KNM, &knl_features },
1241	/*
1242	* Missing support for
1243	* INTEL_ICELAKE
1244	* INTEL_ATOM_SILVERMONT_MID
1245	* INTEL_ATOM_SILVERMONT_MID2
1246	* INTEL_ATOM_AIRMONT_NP
1247	*/
1248	{ 0, NULL },
1249	};
1250
1251	static const struct platform_features *platform;
1252
1253	void probe_platform_features(unsigned int family, unsigned int model)
1254	{
1255	int i;
1256
1257	if (authentic_amd || hygon_genuine) {
1258	/* fallback to default features on unsupported models */
1259	force_load++;
1260	if (max_extended_level >= 0x80000007) {
1261	unsigned int eax, ebx, ecx, edx;
1262
1263	__cpuid(0x80000007, eax, ebx, ecx, edx);
1264	/* RAPL (Fam 17h+) */
1265	if ((edx & (1 << 14)) && family >= 0x17)
1266	platform = &amd_features_with_rapl;
1267	}
1268	goto end;
1269	}
1270
1271	if (!genuine_intel)
1272	goto end;
1273
1274	for (i = 0; turbostat_pdata[i].features; i++) {
1275	if (VFM_FAMILY(turbostat_pdata[i].vfm) == family && VFM_MODEL(turbostat_pdata[i].vfm) == model) {
1276	platform = turbostat_pdata[i].features;
1277	return;
1278	}
1279	}
1280
1281	end:
1282	if (force_load && !platform) {
1283	fprintf(outf, "Forced to run on unsupported platform!\n");
1284	platform = &default_features;
1285	}
1286
1287	if (platform)
1288	return;
1289
1290	fprintf(stderr, "Unsupported platform detected.\n\tSee RUN THE LATEST VERSION on turbostat(8)\n");
1291	exit(1);
1292	}
1293
1294	/* Model specific support End */
1295
1296	#define TJMAX_DEFAULT 100
1297
1298	/* MSRs that are not yet in the kernel-provided header. */
1299	#define MSR_RAPL_PWR_UNIT 0xc0010299
1300	#define MSR_CORE_ENERGY_STAT 0xc001029a
1301	#define MSR_PKG_ENERGY_STAT 0xc001029b
1302
1303	#define MAX(a, b) ((a) > (b) ? (a) : (b))
1304
1305	int backwards_count;
1306	char *progname;
1307
1308	#define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */
1309	cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset;
1310	size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size;
1311	#define MAX_ADDED_THREAD_COUNTERS 24
1312	#define MAX_ADDED_CORE_COUNTERS 8
1313	#define MAX_ADDED_PACKAGE_COUNTERS 16
1314	#define PMT_MAX_ADDED_THREAD_COUNTERS 24
1315	#define PMT_MAX_ADDED_CORE_COUNTERS 8
1316	#define PMT_MAX_ADDED_PACKAGE_COUNTERS 16
1317	#define BITMASK_SIZE 32
1318
1319	#define ZERO_ARRAY(arr) (memset(arr, 0, sizeof(arr)) + __must_be_array(arr))
1320
1321	/* Indexes used to map data read from perf and MSRs into global variables */
1322	enum rapl_rci_index {
1323	RAPL_RCI_INDEX_ENERGY_PKG = 0,
1324	RAPL_RCI_INDEX_ENERGY_CORES = 1,
1325	RAPL_RCI_INDEX_DRAM = 2,
1326	RAPL_RCI_INDEX_GFX = 3,
1327	RAPL_RCI_INDEX_PKG_PERF_STATUS = 4,
1328	RAPL_RCI_INDEX_DRAM_PERF_STATUS = 5,
1329	RAPL_RCI_INDEX_CORE_ENERGY = 6,
1330	RAPL_RCI_INDEX_ENERGY_PLATFORM = 7,
1331	NUM_RAPL_COUNTERS,
1332	};
1333
1334	enum rapl_unit {
1335	RAPL_UNIT_INVALID,
1336	RAPL_UNIT_JOULES,
1337	RAPL_UNIT_WATTS,
1338	};
1339
1340	struct rapl_counter_info_t {
1341	unsigned long long data[NUM_RAPL_COUNTERS];
1342	enum counter_source source[NUM_RAPL_COUNTERS];
1343	unsigned long long flags[NUM_RAPL_COUNTERS];
1344	double scale[NUM_RAPL_COUNTERS];
1345	enum rapl_unit unit[NUM_RAPL_COUNTERS];
1346	unsigned long long msr[NUM_RAPL_COUNTERS];
1347	unsigned long long msr_mask[NUM_RAPL_COUNTERS];
1348	int msr_shift[NUM_RAPL_COUNTERS];
1349
1350	int fd_perf;
1351	};
1352
1353	/* struct rapl_counter_info_t for each RAPL domain */
1354	struct rapl_counter_info_t *rapl_counter_info_perdomain;
1355	unsigned int rapl_counter_info_perdomain_size;
1356
1357	#define RAPL_COUNTER_FLAG_PLATFORM_COUNTER (1u << 0)
1358	#define RAPL_COUNTER_FLAG_USE_MSR_SUM (1u << 1)
1359
1360	struct rapl_counter_arch_info {
1361	int feature_mask; /* Mask for testing if the counter is supported on host */
1362	const char *perf_subsys;
1363	const char *perf_name;
1364	unsigned long long msr;
1365	unsigned long long msr_mask;
1366	int msr_shift; /* Positive mean shift right, negative mean shift left */
1367	double *platform_rapl_msr_scale; /* Scale applied to values read by MSR (platform dependent, filled at runtime) */
1368	unsigned int rci_index; /* Maps data from perf counters to global variables */
1369	unsigned int bic_number;
1370	double compat_scale; /* Some counters require constant scaling to be in the same range as other, similar ones */
1371	unsigned long long flags;
1372	};
1373
1374	static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = {
1375	{
1376	.feature_mask = RAPL_PKG,
1377	.perf_subsys = "power",
1378	.perf_name = "energy-pkg",
1379	.msr = MSR_PKG_ENERGY_STATUS,
1380	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1381	.msr_shift = 0,
1382	.platform_rapl_msr_scale = &rapl_energy_units,
1383	.rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1384	.bic_number = BIC_PkgWatt,
1385	.compat_scale = 1.0,
1386	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1387	},
1388	{
1389	.feature_mask = RAPL_PKG,
1390	.perf_subsys = "power",
1391	.perf_name = "energy-pkg",
1392	.msr = MSR_PKG_ENERGY_STATUS,
1393	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1394	.msr_shift = 0,
1395	.platform_rapl_msr_scale = &rapl_energy_units,
1396	.rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1397	.bic_number = BIC_Pkg_J,
1398	.compat_scale = 1.0,
1399	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1400	},
1401	{
1402	.feature_mask = RAPL_AMD_F17H,
1403	.perf_subsys = "power",
1404	.perf_name = "energy-pkg",
1405	.msr = MSR_PKG_ENERGY_STAT,
1406	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1407	.msr_shift = 0,
1408	.platform_rapl_msr_scale = &rapl_energy_units,
1409	.rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1410	.bic_number = BIC_PkgWatt,
1411	.compat_scale = 1.0,
1412	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1413	},
1414	{
1415	.feature_mask = RAPL_AMD_F17H,
1416	.perf_subsys = "power",
1417	.perf_name = "energy-pkg",
1418	.msr = MSR_PKG_ENERGY_STAT,
1419	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1420	.msr_shift = 0,
1421	.platform_rapl_msr_scale = &rapl_energy_units,
1422	.rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1423	.bic_number = BIC_Pkg_J,
1424	.compat_scale = 1.0,
1425	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1426	},
1427	{
1428	.feature_mask = RAPL_CORE_ENERGY_STATUS,
1429	.perf_subsys = "power",
1430	.perf_name = "energy-cores",
1431	.msr = MSR_PP0_ENERGY_STATUS,
1432	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1433	.msr_shift = 0,
1434	.platform_rapl_msr_scale = &rapl_energy_units,
1435	.rci_index = RAPL_RCI_INDEX_ENERGY_CORES,
1436	.bic_number = BIC_CorWatt,
1437	.compat_scale = 1.0,
1438	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1439	},
1440	{
1441	.feature_mask = RAPL_CORE_ENERGY_STATUS,
1442	.perf_subsys = "power",
1443	.perf_name = "energy-cores",
1444	.msr = MSR_PP0_ENERGY_STATUS,
1445	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1446	.msr_shift = 0,
1447	.platform_rapl_msr_scale = &rapl_energy_units,
1448	.rci_index = RAPL_RCI_INDEX_ENERGY_CORES,
1449	.bic_number = BIC_Cor_J,
1450	.compat_scale = 1.0,
1451	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1452	},
1453	{
1454	.feature_mask = RAPL_DRAM,
1455	.perf_subsys = "power",
1456	.perf_name = "energy-ram",
1457	.msr = MSR_DRAM_ENERGY_STATUS,
1458	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1459	.msr_shift = 0,
1460	.platform_rapl_msr_scale = &rapl_dram_energy_units,
1461	.rci_index = RAPL_RCI_INDEX_DRAM,
1462	.bic_number = BIC_RAMWatt,
1463	.compat_scale = 1.0,
1464	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1465	},
1466	{
1467	.feature_mask = RAPL_DRAM,
1468	.perf_subsys = "power",
1469	.perf_name = "energy-ram",
1470	.msr = MSR_DRAM_ENERGY_STATUS,
1471	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1472	.msr_shift = 0,
1473	.platform_rapl_msr_scale = &rapl_dram_energy_units,
1474	.rci_index = RAPL_RCI_INDEX_DRAM,
1475	.bic_number = BIC_RAM_J,
1476	.compat_scale = 1.0,
1477	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1478	},
1479	{
1480	.feature_mask = RAPL_GFX,
1481	.perf_subsys = "power",
1482	.perf_name = "energy-gpu",
1483	.msr = MSR_PP1_ENERGY_STATUS,
1484	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1485	.msr_shift = 0,
1486	.platform_rapl_msr_scale = &rapl_energy_units,
1487	.rci_index = RAPL_RCI_INDEX_GFX,
1488	.bic_number = BIC_GFXWatt,
1489	.compat_scale = 1.0,
1490	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1491	},
1492	{
1493	.feature_mask = RAPL_GFX,
1494	.perf_subsys = "power",
1495	.perf_name = "energy-gpu",
1496	.msr = MSR_PP1_ENERGY_STATUS,
1497	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1498	.msr_shift = 0,
1499	.platform_rapl_msr_scale = &rapl_energy_units,
1500	.rci_index = RAPL_RCI_INDEX_GFX,
1501	.bic_number = BIC_GFX_J,
1502	.compat_scale = 1.0,
1503	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1504	},
1505	{
1506	.feature_mask = RAPL_PKG_PERF_STATUS,
1507	.perf_subsys = NULL,
1508	.perf_name = NULL,
1509	.msr = MSR_PKG_PERF_STATUS,
1510	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1511	.msr_shift = 0,
1512	.platform_rapl_msr_scale = &rapl_time_units,
1513	.rci_index = RAPL_RCI_INDEX_PKG_PERF_STATUS,
1514	.bic_number = BIC_PKG__,
1515	.compat_scale = 100.0,
1516	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1517	},
1518	{
1519	.feature_mask = RAPL_DRAM_PERF_STATUS,
1520	.perf_subsys = NULL,
1521	.perf_name = NULL,
1522	.msr = MSR_DRAM_PERF_STATUS,
1523	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1524	.msr_shift = 0,
1525	.platform_rapl_msr_scale = &rapl_time_units,
1526	.rci_index = RAPL_RCI_INDEX_DRAM_PERF_STATUS,
1527	.bic_number = BIC_RAM__,
1528	.compat_scale = 100.0,
1529	.flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1530	},
1531	{
1532	.feature_mask = RAPL_AMD_F17H,
1533	.perf_subsys = NULL,
1534	.perf_name = NULL,
1535	.msr = MSR_CORE_ENERGY_STAT,
1536	.msr_mask = 0xFFFFFFFF,
1537	.msr_shift = 0,
1538	.platform_rapl_msr_scale = &rapl_energy_units,
1539	.rci_index = RAPL_RCI_INDEX_CORE_ENERGY,
1540	.bic_number = BIC_CorWatt,
1541	.compat_scale = 1.0,
1542	.flags = 0,
1543	},
1544	{
1545	.feature_mask = RAPL_AMD_F17H,
1546	.perf_subsys = NULL,
1547	.perf_name = NULL,
1548	.msr = MSR_CORE_ENERGY_STAT,
1549	.msr_mask = 0xFFFFFFFF,
1550	.msr_shift = 0,
1551	.platform_rapl_msr_scale = &rapl_energy_units,
1552	.rci_index = RAPL_RCI_INDEX_CORE_ENERGY,
1553	.bic_number = BIC_Cor_J,
1554	.compat_scale = 1.0,
1555	.flags = 0,
1556	},
1557	{
1558	.feature_mask = RAPL_PSYS,
1559	.perf_subsys = "power",
1560	.perf_name = "energy-psys",
1561	.msr = MSR_PLATFORM_ENERGY_STATUS,
1562	.msr_mask = 0x00000000FFFFFFFF,
1563	.msr_shift = 0,
1564	.platform_rapl_msr_scale = &rapl_psys_energy_units,
1565	.rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM,
1566	.bic_number = BIC_SysWatt,
1567	.compat_scale = 1.0,
1568	.flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM,
1569	},
1570	{
1571	.feature_mask = RAPL_PSYS,
1572	.perf_subsys = "power",
1573	.perf_name = "energy-psys",
1574	.msr = MSR_PLATFORM_ENERGY_STATUS,
1575	.msr_mask = 0x00000000FFFFFFFF,
1576	.msr_shift = 0,
1577	.platform_rapl_msr_scale = &rapl_psys_energy_units,
1578	.rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM,
1579	.bic_number = BIC_Sys_J,
1580	.compat_scale = 1.0,
1581	.flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM,
1582	},
1583	};
1584
1585	struct rapl_counter {
1586	unsigned long long raw_value;
1587	enum rapl_unit unit;
1588	double scale;
1589	};
1590
1591	/* Indexes used to map data read from perf and MSRs into global variables */
1592	enum ccstate_rci_index {
1593	CCSTATE_RCI_INDEX_C1_RESIDENCY = 0,
1594	CCSTATE_RCI_INDEX_C3_RESIDENCY = 1,
1595	CCSTATE_RCI_INDEX_C6_RESIDENCY = 2,
1596	CCSTATE_RCI_INDEX_C7_RESIDENCY = 3,
1597	PCSTATE_RCI_INDEX_C2_RESIDENCY = 4,
1598	PCSTATE_RCI_INDEX_C3_RESIDENCY = 5,
1599	PCSTATE_RCI_INDEX_C6_RESIDENCY = 6,
1600	PCSTATE_RCI_INDEX_C7_RESIDENCY = 7,
1601	PCSTATE_RCI_INDEX_C8_RESIDENCY = 8,
1602	PCSTATE_RCI_INDEX_C9_RESIDENCY = 9,
1603	PCSTATE_RCI_INDEX_C10_RESIDENCY = 10,
1604	NUM_CSTATE_COUNTERS,
1605	};
1606
1607	struct cstate_counter_info_t {
1608	unsigned long long data[NUM_CSTATE_COUNTERS];
1609	enum counter_source source[NUM_CSTATE_COUNTERS];
1610	unsigned long long msr[NUM_CSTATE_COUNTERS];
1611	int fd_perf_core;
1612	int fd_perf_pkg;
1613	};
1614
1615	struct cstate_counter_info_t *ccstate_counter_info;
1616	unsigned int ccstate_counter_info_size;
1617
1618	#define CSTATE_COUNTER_FLAG_COLLECT_PER_CORE (1u << 0)
1619	#define CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD ((1u << 1) | CSTATE_COUNTER_FLAG_COLLECT_PER_CORE)
1620	#define CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY (1u << 2)
1621
1622	struct cstate_counter_arch_info {
1623	int feature_mask; /* Mask for testing if the counter is supported on host */
1624	const char *perf_subsys;
1625	const char *perf_name;
1626	unsigned long long msr;
1627	unsigned int rci_index; /* Maps data from perf counters to global variables */
1628	unsigned int bic_number;
1629	unsigned long long flags;
1630	int pkg_cstate_limit;
1631	};
1632
1633	static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = {
1634	{
1635	.feature_mask = CC1,
1636	.perf_subsys = "cstate_core",
1637	.perf_name = "c1-residency",
1638	.msr = MSR_CORE_C1_RES,
1639	.rci_index = CCSTATE_RCI_INDEX_C1_RESIDENCY,
1640	.bic_number = BIC_CPU_c1,
1641	.flags = CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD,
1642	.pkg_cstate_limit = 0,
1643	},
1644	{
1645	.feature_mask = CC3,
1646	.perf_subsys = "cstate_core",
1647	.perf_name = "c3-residency",
1648	.msr = MSR_CORE_C3_RESIDENCY,
1649	.rci_index = CCSTATE_RCI_INDEX_C3_RESIDENCY,
1650	.bic_number = BIC_CPU_c3,
1651	.flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1652	.pkg_cstate_limit = 0,
1653	},
1654	{
1655	.feature_mask = CC6,
1656	.perf_subsys = "cstate_core",
1657	.perf_name = "c6-residency",
1658	.msr = MSR_CORE_C6_RESIDENCY,
1659	.rci_index = CCSTATE_RCI_INDEX_C6_RESIDENCY,
1660	.bic_number = BIC_CPU_c6,
1661	.flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1662	.pkg_cstate_limit = 0,
1663	},
1664	{
1665	.feature_mask = CC7,
1666	.perf_subsys = "cstate_core",
1667	.perf_name = "c7-residency",
1668	.msr = MSR_CORE_C7_RESIDENCY,
1669	.rci_index = CCSTATE_RCI_INDEX_C7_RESIDENCY,
1670	.bic_number = BIC_CPU_c7,
1671	.flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1672	.pkg_cstate_limit = 0,
1673	},
1674	{
1675	.feature_mask = PC2,
1676	.perf_subsys = "cstate_pkg",
1677	.perf_name = "c2-residency",
1678	.msr = MSR_PKG_C2_RESIDENCY,
1679	.rci_index = PCSTATE_RCI_INDEX_C2_RESIDENCY,
1680	.bic_number = BIC_Pkgpc2,
1681	.flags = 0,
1682	.pkg_cstate_limit = PCL__2,
1683	},
1684	{
1685	.feature_mask = PC3,
1686	.perf_subsys = "cstate_pkg",
1687	.perf_name = "c3-residency",
1688	.msr = MSR_PKG_C3_RESIDENCY,
1689	.rci_index = PCSTATE_RCI_INDEX_C3_RESIDENCY,
1690	.bic_number = BIC_Pkgpc3,
1691	.flags = 0,
1692	.pkg_cstate_limit = PCL__3,
1693	},
1694	{
1695	.feature_mask = PC6,
1696	.perf_subsys = "cstate_pkg",
1697	.perf_name = "c6-residency",
1698	.msr = MSR_PKG_C6_RESIDENCY,
1699	.rci_index = PCSTATE_RCI_INDEX_C6_RESIDENCY,
1700	.bic_number = BIC_Pkgpc6,
1701	.flags = 0,
1702	.pkg_cstate_limit = PCL__6,
1703	},
1704	{
1705	.feature_mask = PC7,
1706	.perf_subsys = "cstate_pkg",
1707	.perf_name = "c7-residency",
1708	.msr = MSR_PKG_C7_RESIDENCY,
1709	.rci_index = PCSTATE_RCI_INDEX_C7_RESIDENCY,
1710	.bic_number = BIC_Pkgpc7,
1711	.flags = 0,
1712	.pkg_cstate_limit = PCL__7,
1713	},
1714	{
1715	.feature_mask = PC8,
1716	.perf_subsys = "cstate_pkg",
1717	.perf_name = "c8-residency",
1718	.msr = MSR_PKG_C8_RESIDENCY,
1719	.rci_index = PCSTATE_RCI_INDEX_C8_RESIDENCY,
1720	.bic_number = BIC_Pkgpc8,
1721	.flags = 0,
1722	.pkg_cstate_limit = PCL__8,
1723	},
1724	{
1725	.feature_mask = PC9,
1726	.perf_subsys = "cstate_pkg",
1727	.perf_name = "c9-residency",
1728	.msr = MSR_PKG_C9_RESIDENCY,
1729	.rci_index = PCSTATE_RCI_INDEX_C9_RESIDENCY,
1730	.bic_number = BIC_Pkgpc9,
1731	.flags = 0,
1732	.pkg_cstate_limit = PCL__9,
1733	},
1734	{
1735	.feature_mask = PC10,
1736	.perf_subsys = "cstate_pkg",
1737	.perf_name = "c10-residency",
1738	.msr = MSR_PKG_C10_RESIDENCY,
1739	.rci_index = PCSTATE_RCI_INDEX_C10_RESIDENCY,
1740	.bic_number = BIC_Pkgpc10,
1741	.flags = 0,
1742	.pkg_cstate_limit = PCL_10,
1743	},
1744	};
1745
1746	/* Indexes used to map data read from perf and MSRs into global variables */
1747	enum msr_rci_index {
1748	MSR_RCI_INDEX_APERF = 0,
1749	MSR_RCI_INDEX_MPERF = 1,
1750	MSR_RCI_INDEX_SMI = 2,
1751	NUM_MSR_COUNTERS,
1752	};
1753
1754	struct msr_counter_info_t {
1755	unsigned long long data[NUM_MSR_COUNTERS];
1756	enum counter_source source[NUM_MSR_COUNTERS];
1757	unsigned long long msr[NUM_MSR_COUNTERS];
1758	unsigned long long msr_mask[NUM_MSR_COUNTERS];
1759	int fd_perf;
1760	};
1761
1762	struct msr_counter_info_t *msr_counter_info;
1763	unsigned int msr_counter_info_size;
1764
1765	struct msr_counter_arch_info {
1766	const char *perf_subsys;
1767	const char *perf_name;
1768	unsigned long long msr;
1769	unsigned long long msr_mask;
1770	unsigned int rci_index; /* Maps data from perf counters to global variables */
1771	bool needed;
1772	bool present;
1773	};
1774
1775	enum msr_arch_info_index {
1776	MSR_ARCH_INFO_APERF_INDEX = 0,
1777	MSR_ARCH_INFO_MPERF_INDEX = 1,
1778	MSR_ARCH_INFO_SMI_INDEX = 2,
1779	};
1780
1781	static struct msr_counter_arch_info msr_counter_arch_infos[] = {
1782	[MSR_ARCH_INFO_APERF_INDEX] = {
1783	.perf_subsys = "msr",
1784	.perf_name = "aperf",
1785	.msr = MSR_IA32_APERF,
1786	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1787	.rci_index = MSR_RCI_INDEX_APERF,
1788	},
1789
1790	[MSR_ARCH_INFO_MPERF_INDEX] = {
1791	.perf_subsys = "msr",
1792	.perf_name = "mperf",
1793	.msr = MSR_IA32_MPERF,
1794	.msr_mask = 0xFFFFFFFFFFFFFFFF,
1795	.rci_index = MSR_RCI_INDEX_MPERF,
1796	},
1797
1798	[MSR_ARCH_INFO_SMI_INDEX] = {
1799	.perf_subsys = "msr",
1800	.perf_name = "smi",
1801	.msr = MSR_SMI_COUNT,
1802	.msr_mask = 0xFFFFFFFF,
1803	.rci_index = MSR_RCI_INDEX_SMI,
1804	},
1805	};
1806
1807	/* Can be redefined when compiling, useful for testing. */
1808	#ifndef SYSFS_TELEM_PATH
1809	#define SYSFS_TELEM_PATH "/sys/class/intel_pmt"
1810	#endif
1811
1812	#define PMT_COUNTER_MTL_DC6_OFFSET 120
1813	#define PMT_COUNTER_MTL_DC6_LSB 0
1814	#define PMT_COUNTER_MTL_DC6_MSB 63
1815	#define PMT_MTL_DC6_GUID 0x1a067102
1816	#define PMT_MTL_DC6_SEQ 0
1817
1818	#define PMT_COUNTER_CWF_MC1E_OFFSET_BASE 20936
1819	#define PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT 24
1820	#define PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE 12
1821	#define PMT_COUNTER_CWF_CPUS_PER_MODULE 4
1822	#define PMT_COUNTER_CWF_MC1E_LSB 0
1823	#define PMT_COUNTER_CWF_MC1E_MSB 63
1824	#define PMT_CWF_MC1E_GUID 0x14421519
1825
1826	unsigned long long tcore_clock_freq_hz = 800000000;
1827
1828	#define PMT_COUNTER_NAME_SIZE_BYTES 16
1829	#define PMT_COUNTER_TYPE_NAME_SIZE_BYTES 32
1830
1831	struct pmt_mmio {
1832	struct pmt_mmio *next;
1833
1834	unsigned int guid;
1835	unsigned int size;
1836
1837	/* Base pointer to the mmaped memory. */
1838	void *mmio_base;
1839
1840	/*
1841	* Offset to be applied to the mmio_base
1842	* to get the beginning of the PMT counters for given GUID.
1843	*/
1844	unsigned long pmt_offset;
1845	} *pmt_mmios;
1846
1847	enum pmt_datatype {
1848	PMT_TYPE_RAW,
1849	PMT_TYPE_XTAL_TIME,
1850	PMT_TYPE_TCORE_CLOCK,
1851	};
1852
1853	struct pmt_domain_info {
1854	/*
1855	* Pointer to the MMIO obtained by applying a counter offset
1856	* to the mmio_base of the mmaped region for the given GUID.
1857	*
1858	* This is where to read the raw value of the counter from.
1859	*/
1860	unsigned long *pcounter;
1861	};
1862
1863	struct pmt_counter {
1864	struct pmt_counter *next;
1865
1866	/* PMT metadata */
1867	char name[PMT_COUNTER_NAME_SIZE_BYTES];
1868	enum pmt_datatype type;
1869	enum counter_scope scope;
1870	unsigned int lsb;
1871	unsigned int msb;
1872
1873	/* BIC-like metadata */
1874	enum counter_format format;
1875
1876	unsigned int num_domains;
1877	struct pmt_domain_info *domains;
1878	};
1879
1880	/*
1881	* PMT telemetry directory iterator.
1882	* Used to iterate telemetry files in sysfs in correct order.
1883	*/
1884	struct pmt_diriter_t {
1885	DIR *dir;
1886	struct dirent **namelist;
1887	unsigned int num_names;
1888	unsigned int current_name_idx;
1889	};
1890
1891	int pmt_telemdir_filter(const struct dirent *e)
1892	{
1893	unsigned int dummy;
1894
1895	return sscanf(e->d_name, "telem%u", &dummy);
1896	}
1897
1898	int pmt_telemdir_sort(const struct dirent **a, const struct dirent **b)
1899	{
1900	unsigned int aidx = 0, bidx = 0;
1901
1902	sscanf((*a)->d_name, "telem%u", &aidx);
1903	sscanf((*b)->d_name, "telem%u", &bidx);
1904
1905	return (aidx > bidx) ? 1 : (aidx < bidx) ? -1 : 0;
1906	}
1907
1908	const struct dirent *pmt_diriter_next(struct pmt_diriter_t *iter)
1909	{
1910	const struct dirent *ret = NULL;
1911
1912	if (!iter->dir)
1913	return NULL;
1914
1915	if (iter->current_name_idx >= iter->num_names)
1916	return NULL;
1917
1918	ret = iter->namelist[iter->current_name_idx];
1919	++iter->current_name_idx;
1920
1921	return ret;
1922	}
1923
1924	const struct dirent *pmt_diriter_begin(struct pmt_diriter_t *iter, const char *pmt_root_path)
1925	{
1926	int num_names = iter->num_names;
1927
1928	if (!iter->dir) {
1929	iter->dir = opendir(pmt_root_path);
1930	if (iter->dir == NULL)
1931	return NULL;
1932
1933	num_names = scandir(pmt_root_path, &iter->namelist, pmt_telemdir_filter, pmt_telemdir_sort);
1934	if (num_names == -1)
1935	return NULL;
1936	}
1937
1938	iter->current_name_idx = 0;
1939	iter->num_names = num_names;
1940
1941	return pmt_diriter_next(iter);
1942	}
1943
1944	void pmt_diriter_init(struct pmt_diriter_t *iter)
1945	{
1946	memset(iter, 0, sizeof(*iter));
1947	}
1948
1949	void pmt_diriter_remove(struct pmt_diriter_t *iter)
1950	{
1951	if (iter->namelist) {
1952	for (unsigned int i = 0; i < iter->num_names; i++) {
1953	free(iter->namelist[i]);
1954	iter->namelist[i] = NULL;
1955	}
1956	}
1957
1958	free(iter->namelist);
1959	iter->namelist = NULL;
1960	iter->num_names = 0;
1961	iter->current_name_idx = 0;
1962
1963	closedir(iter->dir);
1964	iter->dir = NULL;
1965	}
1966
1967	unsigned int pmt_counter_get_width(const struct pmt_counter *p)
1968	{
1969	return (p->msb - p->lsb) + 1;
1970	}
1971
1972	void pmt_counter_resize_(struct pmt_counter *pcounter, unsigned int new_size)
1973	{
1974	struct pmt_domain_info *new_mem;
1975
1976	new_mem = (struct pmt_domain_info *)reallocarray(pcounter->domains, new_size, sizeof(*pcounter->domains));
1977	if (!new_mem) {
1978	fprintf(stderr, "%s: failed to allocate memory for PMT counters\n", __func__);
1979	exit(1);
1980	}
1981
1982	/* Zero initialize just allocated memory. */
1983	const size_t num_new_domains = new_size - pcounter->num_domains;
1984
1985	memset(&new_mem[pcounter->num_domains], 0, num_new_domains * sizeof(*pcounter->domains));
1986
1987	pcounter->num_domains = new_size;
1988	pcounter->domains = new_mem;
1989	}
1990
1991	void pmt_counter_resize(struct pmt_counter *pcounter, unsigned int new_size)
1992	{
1993	/*
1994	* Allocate more memory ahead of time.
1995	*
1996	* Always allocate space for at least 8 elements
1997	* and double the size when growing.
1998	*/
1999	if (new_size < 8)
2000	new_size = 8;
2001	new_size = MAX(new_size, pcounter->num_domains * 2);
2002
2003	pmt_counter_resize_(pcounter, new_size);
2004	}
2005
2006	struct llc_stats {
2007	unsigned long long references;
2008	unsigned long long misses;
2009	};
2010	struct thread_data {
2011	struct timeval tv_begin;
2012	struct timeval tv_end;
2013	struct timeval tv_delta;
2014	unsigned long long tsc;
2015	unsigned long long aperf;
2016	unsigned long long mperf;
2017	unsigned long long c1;
2018	unsigned long long instr_count;
2019	unsigned long long irq_count;
2020	unsigned long long nmi_count;
2021	unsigned int smi_count;
2022	struct llc_stats llc;
2023	unsigned int cpu_id;
2024	unsigned int apic_id;
2025	unsigned int x2apic_id;
2026	unsigned int flags;
2027	bool is_atom;
2028	unsigned long long counter[MAX_ADDED_THREAD_COUNTERS];
2029	unsigned long long perf_counter[MAX_ADDED_THREAD_COUNTERS];
2030	unsigned long long pmt_counter[PMT_MAX_ADDED_THREAD_COUNTERS];
2031	} *thread_even, *thread_odd;
2032
2033	struct core_data {
2034	int base_cpu;
2035	unsigned long long c3;
2036	unsigned long long c6;
2037	unsigned long long c7;
2038	unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */
2039	unsigned int core_temp_c;
2040	struct rapl_counter core_energy; /* MSR_CORE_ENERGY_STAT */
2041	unsigned int core_id;
2042	unsigned long long core_throt_cnt;
2043	unsigned long long counter[MAX_ADDED_CORE_COUNTERS];
2044	unsigned long long perf_counter[MAX_ADDED_CORE_COUNTERS];
2045	unsigned long long pmt_counter[PMT_MAX_ADDED_CORE_COUNTERS];
2046	} *core_even, *core_odd;
2047
2048	struct pkg_data {
2049	int base_cpu;
2050	unsigned long long pc2;
2051	unsigned long long pc3;
2052	unsigned long long pc6;
2053	unsigned long long pc7;
2054	unsigned long long pc8;
2055	unsigned long long pc9;
2056	unsigned long long pc10;
2057	long long cpu_lpi;
2058	long long sys_lpi;
2059	unsigned long long pkg_wtd_core_c0;
2060	unsigned long long pkg_any_core_c0;
2061	unsigned long long pkg_any_gfxe_c0;
2062	unsigned long long pkg_both_core_gfxe_c0;
2063	long long gfx_rc6_ms;
2064	unsigned int gfx_mhz;
2065	unsigned int gfx_act_mhz;
2066	long long sam_mc6_ms;
2067	unsigned int sam_mhz;
2068	unsigned int sam_act_mhz;
2069	unsigned int package_id;
2070	struct rapl_counter energy_pkg; /* MSR_PKG_ENERGY_STATUS */
2071	struct rapl_counter energy_dram; /* MSR_DRAM_ENERGY_STATUS */
2072	struct rapl_counter energy_cores; /* MSR_PP0_ENERGY_STATUS */
2073	struct rapl_counter energy_gfx; /* MSR_PP1_ENERGY_STATUS */
2074	struct rapl_counter rapl_pkg_perf_status; /* MSR_PKG_PERF_STATUS */
2075	struct rapl_counter rapl_dram_perf_status; /* MSR_DRAM_PERF_STATUS */
2076	unsigned int pkg_temp_c;
2077	unsigned int uncore_mhz;
2078	unsigned long long die_c6;
2079	unsigned long long counter[MAX_ADDED_PACKAGE_COUNTERS];
2080	unsigned long long perf_counter[MAX_ADDED_PACKAGE_COUNTERS];
2081	unsigned long long pmt_counter[PMT_MAX_ADDED_PACKAGE_COUNTERS];
2082	} *package_even, *package_odd;
2083
2084	#define ODD_COUNTERS thread_odd, core_odd, package_odd
2085	#define EVEN_COUNTERS thread_even, core_even, package_even
2086
2087	#define GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no) \
2088	((thread_base) + \
2089	((pkg_no) * \
2090	topo.nodes_per_pkg * topo.cores_per_node * topo.threads_per_core) + \
2091	((node_no) * topo.cores_per_node * topo.threads_per_core) + \
2092	((core_no) * topo.threads_per_core) + \
2093	(thread_no))
2094
2095	#define GET_CORE(core_base, core_no, node_no, pkg_no) \
2096	((core_base) + \
2097	((pkg_no) * topo.nodes_per_pkg * topo.cores_per_node) + \
2098	((node_no) * topo.cores_per_node) + \
2099	(core_no))
2100
2101	/*
2102	* The accumulated sum of MSR is defined as a monotonic
2103	* increasing MSR, it will be accumulated periodically,
2104	* despite its register's bit width.
2105	*/
2106	enum {
2107	IDX_PKG_ENERGY,
2108	IDX_DRAM_ENERGY,
2109	IDX_PP0_ENERGY,
2110	IDX_PP1_ENERGY,
2111	IDX_PKG_PERF,
2112	IDX_DRAM_PERF,
2113	IDX_PSYS_ENERGY,
2114	IDX_COUNT,
2115	};
2116
2117	int get_msr_sum(int cpu, off_t offset, unsigned long long *msr);
2118
2119	struct msr_sum_array {
2120	/* get_msr_sum() = sum + (get_msr() - last) */
2121	struct {
2122	/*The accumulated MSR value is updated by the timer */
2123	unsigned long long sum;
2124	/*The MSR footprint recorded in last timer */
2125	unsigned long long last;
2126	} entries[IDX_COUNT];
2127	};
2128
2129	/* The percpu MSR sum array.*/
2130	struct msr_sum_array *per_cpu_msr_sum;
2131
2132	off_t idx_to_offset(int idx)
2133	{
2134	off_t offset;
2135
2136	switch (idx) {
2137	case IDX_PKG_ENERGY:
2138	if (valid_rapl_msrs & RAPL_AMD_F17H)
2139	offset = MSR_PKG_ENERGY_STAT;
2140	else
2141	offset = MSR_PKG_ENERGY_STATUS;
2142	break;
2143	case IDX_DRAM_ENERGY:
2144	offset = MSR_DRAM_ENERGY_STATUS;
2145	break;
2146	case IDX_PP0_ENERGY:
2147	offset = MSR_PP0_ENERGY_STATUS;
2148	break;
2149	case IDX_PP1_ENERGY:
2150	offset = MSR_PP1_ENERGY_STATUS;
2151	break;
2152	case IDX_PKG_PERF:
2153	offset = MSR_PKG_PERF_STATUS;
2154	break;
2155	case IDX_DRAM_PERF:
2156	offset = MSR_DRAM_PERF_STATUS;
2157	break;
2158	case IDX_PSYS_ENERGY:
2159	offset = MSR_PLATFORM_ENERGY_STATUS;
2160	break;
2161	default:
2162	offset = -1;
2163	}
2164	return offset;
2165	}
2166
2167	int offset_to_idx(off_t offset)
2168	{
2169	int idx;
2170
2171	switch (offset) {
2172	case MSR_PKG_ENERGY_STATUS:
2173	case MSR_PKG_ENERGY_STAT:
2174	idx = IDX_PKG_ENERGY;
2175	break;
2176	case MSR_DRAM_ENERGY_STATUS:
2177	idx = IDX_DRAM_ENERGY;
2178	break;
2179	case MSR_PP0_ENERGY_STATUS:
2180	idx = IDX_PP0_ENERGY;
2181	break;
2182	case MSR_PP1_ENERGY_STATUS:
2183	idx = IDX_PP1_ENERGY;
2184	break;
2185	case MSR_PKG_PERF_STATUS:
2186	idx = IDX_PKG_PERF;
2187	break;
2188	case MSR_DRAM_PERF_STATUS:
2189	idx = IDX_DRAM_PERF;
2190	break;
2191	case MSR_PLATFORM_ENERGY_STATUS:
2192	idx = IDX_PSYS_ENERGY;
2193	break;
2194	default:
2195	idx = -1;
2196	}
2197	return idx;
2198	}
2199
2200	int idx_valid(int idx)
2201	{
2202	switch (idx) {
2203	case IDX_PKG_ENERGY:
2204	return valid_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H);
2205	case IDX_DRAM_ENERGY:
2206	return valid_rapl_msrs & RAPL_DRAM;
2207	case IDX_PP0_ENERGY:
2208	return valid_rapl_msrs & RAPL_CORE_ENERGY_STATUS;
2209	case IDX_PP1_ENERGY:
2210	return valid_rapl_msrs & RAPL_GFX;
2211	case IDX_PKG_PERF:
2212	return valid_rapl_msrs & RAPL_PKG_PERF_STATUS;
2213	case IDX_DRAM_PERF:
2214	return valid_rapl_msrs & RAPL_DRAM_PERF_STATUS;
2215	case IDX_PSYS_ENERGY:
2216	return valid_rapl_msrs & RAPL_PSYS;
2217	default:
2218	return 0;
2219	}
2220	}
2221
2222	struct sys_counters {
2223	/* MSR added counters */
2224	unsigned int added_thread_counters;
2225	unsigned int added_core_counters;
2226	unsigned int added_package_counters;
2227	struct msr_counter *tp;
2228	struct msr_counter *cp;
2229	struct msr_counter *pp;
2230
2231	/* perf added counters */
2232	unsigned int added_thread_perf_counters;
2233	unsigned int added_core_perf_counters;
2234	unsigned int added_package_perf_counters;
2235	struct perf_counter_info *perf_tp;
2236	struct perf_counter_info *perf_cp;
2237	struct perf_counter_info *perf_pp;
2238
2239	struct pmt_counter *pmt_tp;
2240	struct pmt_counter *pmt_cp;
2241	struct pmt_counter *pmt_pp;
2242	} sys;
2243
2244	static size_t free_msr_counters_(struct msr_counter **pp)
2245	{
2246	struct msr_counter *p = NULL;
2247	size_t num_freed = 0;
2248
2249	while (*pp) {
2250	p = *pp;
2251
2252	if (p->msr_num != 0) {
2253	*pp = p->next;
2254
2255	free(p);
2256	++num_freed;
2257
2258	continue;
2259	}
2260
2261	pp = &p->next;
2262	}
2263
2264	return num_freed;
2265	}
2266
2267	/*
2268	* Free all added counters accessed via msr.
2269	*/
2270	static void free_sys_msr_counters(void)
2271	{
2272	/* Thread counters */
2273	sys.added_thread_counters -= free_msr_counters_(pp: &sys.tp);
2274
2275	/* Core counters */
2276	sys.added_core_counters -= free_msr_counters_(pp: &sys.cp);
2277
2278	/* Package counters */
2279	sys.added_package_counters -= free_msr_counters_(pp: &sys.pp);
2280	}
2281
2282	struct system_summary {
2283	struct thread_data threads;
2284	struct core_data cores;
2285	struct pkg_data packages;
2286	} average;
2287
2288	struct platform_counters {
2289	struct rapl_counter energy_psys; /* MSR_PLATFORM_ENERGY_STATUS */
2290	} platform_counters_odd, platform_counters_even;
2291
2292	struct cpu_topology {
2293	int physical_package_id;
2294	int die_id;
2295	int l3_id;
2296	int logical_cpu_id;
2297	int physical_node_id;
2298	int logical_node_id; /* 0-based count within the package */
2299	int physical_core_id;
2300	int thread_id;
2301	int type;
2302	cpu_set_t *put_ids; /* Processing Unit/Thread IDs */
2303	} *cpus;
2304
2305	struct topo_params {
2306	int num_packages;
2307	int num_die;
2308	int num_cpus;
2309	int num_cores;
2310	int allowed_packages;
2311	int allowed_cpus;
2312	int allowed_cores;
2313	int max_cpu_num;
2314	int max_core_id;
2315	int max_package_id;
2316	int max_die_id;
2317	int max_l3_id;
2318	int max_node_num;
2319	int nodes_per_pkg;
2320	int cores_per_node;
2321	int threads_per_core;
2322	} topo;
2323
2324	struct timeval tv_even, tv_odd, tv_delta;
2325
2326	int *irq_column_2_cpu; /* /proc/interrupts column numbers */
2327	int *irqs_per_cpu; /* indexed by cpu_num */
2328	int *nmi_per_cpu; /* indexed by cpu_num */
2329
2330	void setup_all_buffers(bool startup);
2331
2332	char *sys_lpi_file;
2333	char *sys_lpi_file_sysfs = "/sys/devices/system/cpu/cpuidle/low_power_idle_system_residency_us";
2334	char *sys_lpi_file_debugfs = "/sys/kernel/debug/pmc_core/slp_s0_residency_usec";
2335
2336	int cpu_is_not_present(int cpu)
2337	{
2338	return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set);
2339	}
2340
2341	int cpu_is_not_allowed(int cpu)
2342	{
2343	return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set);
2344	}
2345
2346	/*
2347	* run func(thread, core, package) in topology order
2348	* skip non-present cpus
2349	*/
2350
2351	#define PER_THREAD_PARAMS struct thread_data *t, struct core_data *c, struct pkg_data *p
2352
2353	int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *),
2354	struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base)
2355	{
2356	int retval, pkg_no, core_no, thread_no, node_no;
2357
2358	retval = 0;
2359
2360	for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
2361	for (node_no = 0; node_no < topo.nodes_per_pkg; node_no++) {
2362	for (core_no = 0; core_no < topo.cores_per_node; ++core_no) {
2363	for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) {
2364	struct thread_data *t;
2365	struct core_data *c;
2366
2367	t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no);
2368
2369	if (cpu_is_not_allowed(cpu: t->cpu_id))
2370	continue;
2371
2372	c = GET_CORE(core_base, core_no, node_no, pkg_no);
2373
2374	retval |= func(t, c, &pkg_base[pkg_no]);
2375	}
2376	}
2377	}
2378	}
2379	return retval;
2380	}
2381
2382	int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c)
2383	{
2384	return ((int)t->cpu_id == c->base_cpu || c->base_cpu < 0);
2385	}
2386
2387	int is_cpu_first_core_in_package(struct thread_data *t, struct pkg_data *p)
2388	{
2389	return ((int)t->cpu_id == p->base_cpu || p->base_cpu < 0);
2390	}
2391
2392	int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p)
2393	{
2394	return is_cpu_first_thread_in_core(t, c) && is_cpu_first_core_in_package(t, p);
2395	}
2396
2397	int cpu_migrate(int cpu)
2398	{
2399	CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
2400	CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set);
2401	if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1)
2402	return -1;
2403	else
2404	return 0;
2405	}
2406
2407	int get_msr_fd(int cpu)
2408	{
2409	char pathname[32];
2410	int fd;
2411
2412	fd = fd_percpu[cpu];
2413
2414	if (fd)
2415	return fd;
2416	sprintf(buf: pathname, fmt: use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu);
2417	fd = open(pathname, O_RDONLY);
2418	if (fd < 0)
2419	err(-1, "%s open failed, try chown or chmod +r %s, "
2420	"or run with --no-msr, or run as root", pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr");
2421	fd_percpu[cpu] = fd;
2422
2423	return fd;
2424	}
2425
2426	static void bic_disable_msr_access(void)
2427	{
2428	CLR_BIC(BIC_Mod_c6, &bic_enabled);
2429	CLR_BIC(BIC_CoreTmp, &bic_enabled);
2430	CLR_BIC(BIC_Totl_c0, &bic_enabled);
2431	CLR_BIC(BIC_Any_c0, &bic_enabled);
2432	CLR_BIC(BIC_GFX_c0, &bic_enabled);
2433	CLR_BIC(BIC_CPUGFX, &bic_enabled);
2434	CLR_BIC(BIC_PkgTmp, &bic_enabled);
2435
2436	free_sys_msr_counters();
2437	}
2438
2439	static void bic_disable_perf_access(void)
2440	{
2441	CLR_BIC(BIC_IPC, &bic_enabled);
2442	CLR_BIC(BIC_LLC_RPS, &bic_enabled);
2443	CLR_BIC(BIC_LLC_HIT, &bic_enabled);
2444	}
2445
2446	static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags)
2447	{
2448	assert(!no_perf);
2449
2450	return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags);
2451	}
2452
2453	static long open_perf_counter(int cpu, unsigned int type, unsigned int config, int group_fd, __u64 read_format)
2454	{
2455	struct perf_event_attr attr;
2456	const pid_t pid = -1;
2457	const unsigned long flags = 0;
2458
2459	assert(!no_perf);
2460
2461	memset(&attr, 0, sizeof(struct perf_event_attr));
2462
2463	attr.type = type;
2464	attr.size = sizeof(struct perf_event_attr);
2465	attr.config = config;
2466	attr.disabled = 0;
2467	attr.sample_type = PERF_SAMPLE_IDENTIFIER;
2468	attr.read_format = read_format;
2469
2470	const int fd = perf_event_open(hw_event: &attr, pid, cpu, group_fd, flags);
2471
2472	return fd;
2473	}
2474
2475	int get_instr_count_fd(int cpu)
2476	{
2477	if (fd_instr_count_percpu[cpu])
2478	return fd_instr_count_percpu[cpu];
2479
2480	fd_instr_count_percpu[cpu] = open_perf_counter(cpu, type: PERF_TYPE_HARDWARE, config: PERF_COUNT_HW_INSTRUCTIONS, group_fd: -1, read_format: 0);
2481
2482	return fd_instr_count_percpu[cpu];
2483	}
2484
2485	int get_msr(int cpu, off_t offset, unsigned long long *msr)
2486	{
2487	ssize_t retval;
2488
2489	assert(!no_msr);
2490
2491	retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset);
2492
2493	if (retval != sizeof *msr)
2494	err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset);
2495
2496	return 0;
2497	}
2498
2499	int add_msr_counter(int cpu, off_t offset)
2500	{
2501	ssize_t retval;
2502	unsigned long long value;
2503
2504	if (no_msr)
2505	return -1;
2506
2507	if (!offset)
2508	return -1;
2509
2510	retval = pread(get_msr_fd(cpu), &value, sizeof(value), offset);
2511
2512	/* if the read failed, the probe fails */
2513	if (retval != sizeof(value))
2514	return -1;
2515
2516	if (value == 0)
2517	return 0;
2518
2519	return 1;
2520	}
2521
2522	int add_rapl_msr_counter(int cpu, const struct rapl_counter_arch_info *cai)
2523	{
2524	int ret;
2525
2526	if (!(valid_rapl_msrs & cai->feature_mask))
2527	return -1;
2528
2529	ret = add_msr_counter(cpu, offset: cai->msr);
2530	if (ret < 0)
2531	return -1;
2532
2533	switch (cai->rci_index) {
2534	case RAPL_RCI_INDEX_ENERGY_PKG:
2535	case RAPL_RCI_INDEX_ENERGY_CORES:
2536	case RAPL_RCI_INDEX_DRAM:
2537	case RAPL_RCI_INDEX_GFX:
2538	case RAPL_RCI_INDEX_ENERGY_PLATFORM:
2539	if (ret == 0)
2540	return 1;
2541	}
2542
2543	/* PKG,DRAM_PERF_STATUS MSRs, can return any value */
2544	return 1;
2545	}
2546
2547	/* Convert CPU ID to domain ID for given added perf counter. */
2548	unsigned int cpu_to_domain(const struct perf_counter_info *pc, int cpu)
2549	{
2550	switch (pc->scope) {
2551	case SCOPE_CPU:
2552	return cpu;
2553
2554	case SCOPE_CORE:
2555	return cpus[cpu].physical_core_id;
2556
2557	case SCOPE_PACKAGE:
2558	return cpus[cpu].physical_package_id;
2559	}
2560
2561	__builtin_unreachable();
2562	}
2563
2564	#define MAX_DEFERRED 16
2565	char *deferred_add_names[MAX_DEFERRED];
2566	char *deferred_skip_names[MAX_DEFERRED];
2567	int deferred_add_index;
2568	int deferred_skip_index;
2569	unsigned int deferred_add_consumed;
2570	unsigned int deferred_skip_consumed;
2571
2572	/*
2573	* HIDE_LIST - hide this list of counters, show the rest [default]
2574	* SHOW_LIST - show this list of counters, hide the rest
2575	*/
2576	enum show_hide_mode { SHOW_LIST, HIDE_LIST } global_show_hide_mode = HIDE_LIST;
2577
2578	void help(void)
2579	{
2580	fprintf(outf,
2581	"Usage: turbostat [OPTIONS][(--interval seconds) | COMMAND ...]\n"
2582	"\n"
2583	"Turbostat forks the specified COMMAND and prints statistics\n"
2584	"when COMMAND completes.\n"
2585	"If no COMMAND is specified, turbostat wakes every 5-seconds\n"
2586	"to print statistics, until interrupted.\n"
2587	" -a, --add counter\n"
2588	" add a counter\n"
2589	" eg. --add msr0x10,u64,cpu,delta,MY_TSC\n"
2590	" eg. --add perf/cstate_pkg/c2-residency,package,delta,percent,perfPC2\n"
2591	" eg. --add pmt,name=XTAL,type=raw,domain=package0,offset=0,lsb=0,msb=63,guid=0x1a067102\n"
2592	" -c, --cpu cpu-set\n"
2593	" limit output to summary plus cpu-set:\n"
2594	" {core | package | j,k,l..m,n-p }\n"
2595	" -d, --debug\n"
2596	" displays usec, Time_Of_Day_Seconds and more debugging\n"
2597	" debug messages are printed to stderr\n"
2598	" -D, --Dump\n"
2599	" displays the raw counter values\n"
2600	" -e, --enable [all | column]\n"
2601	" shows all or the specified disabled column\n"
2602	" -f, --force\n"
2603	" force load turbostat with minimum default features on unsupported platforms.\n"
2604	" -H, --hide [column | column,column,...]\n"
2605	" hide the specified column(s)\n"
2606	" -i, --interval sec.subsec\n"
2607	" override default 5-second measurement interval\n"
2608	" -J, --Joules\n"
2609	" displays energy in Joules instead of Watts\n"
2610	" -l, --list\n"
2611	" list column headers only\n"
2612	" -M, --no-msr\n"
2613	" disable all uses of the MSR driver\n"
2614	" -P, --no-perf\n"
2615	" disable all uses of the perf API\n"
2616	" -n, --num_iterations num\n"
2617	" number of the measurement iterations\n"
2618	" -N, --header_iterations num\n"
2619	" print header every num iterations\n"
2620	" -o, --out file\n"
2621	" create or truncate \"file\" for all output\n"
2622	" -q, --quiet\n"
2623	" skip decoding system configuration header\n"
2624	" -s, --show [column | column,column,...]\n"
2625	" show only the specified column(s)\n"
2626	" -S, --Summary\n"
2627	" limits output to 1-line system summary per interval\n"
2628	" -T, --TCC temperature\n"
2629	" sets the Thermal Control Circuit temperature in\n"
2630	" degrees Celsius\n"
2631	" -h, --help\n"
2632	" print this help message\n"
2633	" -v, --version\n\t\tprint version information\n\nFor more help, run \"man turbostat\"\n");
2634	}
2635
2636	/*
2637	* bic_lookup
2638	* for all the strings in comma separate name_list,
2639	* set the approprate bit in return value.
2640	*/
2641	void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode)
2642	{
2643	unsigned int i;
2644
2645	while (name_list) {
2646	char *comma;
2647
2648	comma = strchr(name_list, ',');
2649
2650	if (comma)
2651	*comma = '\0';
2652
2653	for (i = 0; i < MAX_BIC; ++i) {
2654	if (!strcmp(name_list, bic[i].name)) {
2655	SET_BIC(i, ret_set);
2656	break;
2657	}
2658	if (!strcmp(name_list, "all")) {
2659	bic_set_all(ret_set);
2660	break;
2661	} else if (!strcmp(name_list, "topology")) {
2662	CPU_OR(ret_set, ret_set, &bic_group_topology);
2663	break;
2664	} else if (!strcmp(name_list, "power")) {
2665	CPU_OR(ret_set, ret_set, &bic_group_thermal_pwr);
2666	break;
2667	} else if (!strcmp(name_list, "idle")) {
2668	CPU_OR(ret_set, ret_set, &bic_group_idle);
2669	break;
2670	} else if (!strcmp(name_list, "cache")) {
2671	CPU_OR(ret_set, ret_set, &bic_group_cache);
2672	break;
2673	} else if (!strcmp(name_list, "llc")) {
2674	CPU_OR(ret_set, ret_set, &bic_group_cache);
2675	break;
2676	} else if (!strcmp(name_list, "swidle")) {
2677	CPU_OR(ret_set, ret_set, &bic_group_sw_idle);
2678	break;
2679	} else if (!strcmp(name_list, "sysfs")) { /* legacy compatibility */
2680	CPU_OR(ret_set, ret_set, &bic_group_sw_idle);
2681	break;
2682	} else if (!strcmp(name_list, "hwidle")) {
2683	CPU_OR(ret_set, ret_set, &bic_group_hw_idle);
2684	break;
2685	} else if (!strcmp(name_list, "frequency")) {
2686	CPU_OR(ret_set, ret_set, &bic_group_frequency);
2687	break;
2688	} else if (!strcmp(name_list, "other")) {
2689	CPU_OR(ret_set, ret_set, &bic_group_other);
2690	break;
2691	}
2692	}
2693	if (i == MAX_BIC) {
2694	if (mode == SHOW_LIST) {
2695	deferred_add_names[deferred_add_index++] = name_list;
2696	if (deferred_add_index >= MAX_DEFERRED) {
2697	fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", MAX_DEFERRED, name_list);
2698	help();
2699	exit(1);
2700	}
2701	} else {
2702	deferred_skip_names[deferred_skip_index++] = name_list;
2703	if (debug)
2704	fprintf(stderr, "deferred \"%s\"\n", name_list);
2705	if (deferred_skip_index >= MAX_DEFERRED) {
2706	fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", MAX_DEFERRED, name_list);
2707	help();
2708	exit(1);
2709	}
2710	}
2711	}
2712
2713	name_list = comma;
2714	if (name_list)
2715	name_list++;
2716
2717	}
2718	}
2719
2720	/*
2721	* print_name()
2722	* Print column header name for raw 64-bit counter in 16 columns (at least 8-char plus a tab)
2723	* Otherwise, allow the name + tab to fit within 8-coumn tab-stop.
2724	* In both cases, left justififed, just like other turbostat columns,
2725	* to allow the column values to consume the tab.
2726	*
2727	* Yes, 32-bit counters can overflow 8-columns, and
2728	* 64-bit counters can overflow 16-columns, but that is uncommon.
2729	*/
2730	static inline int print_name(int width, int *printed, char *delim, char *name, enum counter_type type, enum counter_format format)
2731	{
2732	UNUSED(type);
2733
2734	if (format == FORMAT_RAW && width >= 64)
2735	return (sprintf(buf: outp, fmt: "%s%-8s", (*printed++ ? delim : ""), name));
2736	else
2737	return (sprintf(buf: outp, fmt: "%s%s", (*printed++ ? delim : ""), name));
2738	}
2739
2740	static inline int print_hex_value(int width, int *printed, char *delim, unsigned long long value)
2741	{
2742	if (width <= 32)
2743	return (sprintf(buf: outp, fmt: "%s%08x", (*printed++ ? delim : ""), (unsigned int)value));
2744	else
2745	return (sprintf(buf: outp, fmt: "%s%016llx", (*printed++ ? delim : ""), value));
2746	}
2747
2748	static inline int print_decimal_value(int width, int *printed, char *delim, unsigned long long value)
2749	{
2750	if (width <= 32)
2751	return (sprintf(buf: outp, fmt: "%s%d", (*printed++ ? delim : ""), (unsigned int)value));
2752	else
2753	return (sprintf(buf: outp, fmt: "%s%-8lld", (*printed++ ? delim : ""), value));
2754	}
2755
2756	static inline int print_float_value(int *printed, char *delim, double value)
2757	{
2758	return (sprintf(buf: outp, fmt: "%s%0.2f", (*printed++ ? delim : ""), value));
2759	}
2760
2761	void print_header(char *delim)
2762	{
2763	struct msr_counter *mp;
2764	struct perf_counter_info *pp;
2765	struct pmt_counter *ppmt;
2766	int printed = 0;
2767
2768	if (DO_BIC(BIC_USEC))
2769	outp += sprintf(buf: outp, fmt: "%susec", (printed++ ? delim : ""));
2770	if (DO_BIC(BIC_TOD))
2771	outp += sprintf(buf: outp, fmt: "%sTime_Of_Day_Seconds", (printed++ ? delim : ""));
2772	if (DO_BIC(BIC_Package))
2773	outp += sprintf(buf: outp, fmt: "%sPackage", (printed++ ? delim : ""));
2774	if (DO_BIC(BIC_Die))
2775	outp += sprintf(buf: outp, fmt: "%sDie", (printed++ ? delim : ""));
2776	if (DO_BIC(BIC_L3))
2777	outp += sprintf(buf: outp, fmt: "%sL3", (printed++ ? delim : ""));
2778	if (DO_BIC(BIC_Node))
2779	outp += sprintf(buf: outp, fmt: "%sNode", (printed++ ? delim : ""));
2780	if (DO_BIC(BIC_Core))
2781	outp += sprintf(buf: outp, fmt: "%sCore", (printed++ ? delim : ""));
2782	if (DO_BIC(BIC_CPU))
2783	outp += sprintf(buf: outp, fmt: "%sCPU", (printed++ ? delim : ""));
2784	if (DO_BIC(BIC_APIC))
2785	outp += sprintf(buf: outp, fmt: "%sAPIC", (printed++ ? delim : ""));
2786	if (DO_BIC(BIC_X2APIC))
2787	outp += sprintf(buf: outp, fmt: "%sX2APIC", (printed++ ? delim : ""));
2788	if (DO_BIC(BIC_Avg_MHz))
2789	outp += sprintf(buf: outp, fmt: "%sAvg_MHz", (printed++ ? delim : ""));
2790	if (DO_BIC(BIC_Busy))
2791	outp += sprintf(buf: outp, fmt: "%sBusy%%", (printed++ ? delim : ""));
2792	if (DO_BIC(BIC_Bzy_MHz))
2793	outp += sprintf(buf: outp, fmt: "%sBzy_MHz", (printed++ ? delim : ""));
2794	if (DO_BIC(BIC_TSC_MHz))
2795	outp += sprintf(buf: outp, fmt: "%sTSC_MHz", (printed++ ? delim : ""));
2796
2797	if (DO_BIC(BIC_IPC))
2798	outp += sprintf(buf: outp, fmt: "%sIPC", (printed++ ? delim : ""));
2799
2800	if (DO_BIC(BIC_IRQ)) {
2801	if (sums_need_wide_columns)
2802	outp += sprintf(buf: outp, fmt: "%s IRQ", (printed++ ? delim : ""));
2803	else
2804	outp += sprintf(buf: outp, fmt: "%sIRQ", (printed++ ? delim : ""));
2805	}
2806	if (DO_BIC(BIC_NMI)) {
2807	if (sums_need_wide_columns)
2808	outp += sprintf(buf: outp, fmt: "%s NMI", (printed++ ? delim : ""));
2809	else
2810	outp += sprintf(buf: outp, fmt: "%sNMI", (printed++ ? delim : ""));
2811	}
2812
2813	if (DO_BIC(BIC_SMI))
2814	outp += sprintf(buf: outp, fmt: "%sSMI", (printed++ ? delim : ""));
2815
2816	if (DO_BIC(BIC_LLC_RPS))
2817	outp += sprintf(buf: outp, fmt: "%sLLCkRPS", (printed++ ? delim : ""));
2818
2819	if (DO_BIC(BIC_LLC_HIT))
2820	outp += sprintf(buf: outp, fmt: "%sLLC%%hit", (printed++ ? delim : ""));
2821
2822	for (mp = sys.tp; mp; mp = mp->next)
2823	outp += print_name(width: mp->width, printed: &printed, delim, name: mp->name, type: mp->type, format: mp->format);
2824
2825	for (pp = sys.perf_tp; pp; pp = pp->next)
2826	outp += print_name(width: pp->width, printed: &printed, delim, name: pp->name, type: pp->type, format: pp->format);
2827
2828	ppmt = sys.pmt_tp;
2829	while (ppmt) {
2830	switch (ppmt->type) {
2831	case PMT_TYPE_RAW:
2832	outp += print_name(width: pmt_counter_get_width(p: ppmt), printed: &printed, delim, name: ppmt->name, type: COUNTER_ITEMS, format: ppmt->format);
2833	break;
2834
2835	case PMT_TYPE_XTAL_TIME:
2836	case PMT_TYPE_TCORE_CLOCK:
2837	outp += print_name(width: 32, printed: &printed, delim, name: ppmt->name, type: COUNTER_ITEMS, format: ppmt->format);
2838	break;
2839	}
2840
2841	ppmt = ppmt->next;
2842	}
2843
2844	if (DO_BIC(BIC_CPU_c1))
2845	outp += sprintf(buf: outp, fmt: "%sCPU%%c1", (printed++ ? delim : ""));
2846	if (DO_BIC(BIC_CPU_c3))
2847	outp += sprintf(buf: outp, fmt: "%sCPU%%c3", (printed++ ? delim : ""));
2848	if (DO_BIC(BIC_CPU_c6))
2849	outp += sprintf(buf: outp, fmt: "%sCPU%%c6", (printed++ ? delim : ""));
2850	if (DO_BIC(BIC_CPU_c7))
2851	outp += sprintf(buf: outp, fmt: "%sCPU%%c7", (printed++ ? delim : ""));
2852
2853	if (DO_BIC(BIC_Mod_c6))
2854	outp += sprintf(buf: outp, fmt: "%sMod%%c6", (printed++ ? delim : ""));
2855
2856	if (DO_BIC(BIC_CoreTmp))
2857	outp += sprintf(buf: outp, fmt: "%sCoreTmp", (printed++ ? delim : ""));
2858
2859	if (DO_BIC(BIC_CORE_THROT_CNT))
2860	outp += sprintf(buf: outp, fmt: "%sCoreThr", (printed++ ? delim : ""));
2861
2862	if (valid_rapl_msrs && !rapl_joules) {
2863	if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
2864	outp += sprintf(buf: outp, fmt: "%sCorWatt", (printed++ ? delim : ""));
2865	} else if (valid_rapl_msrs && rapl_joules) {
2866	if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
2867	outp += sprintf(buf: outp, fmt: "%sCor_J", (printed++ ? delim : ""));
2868	}
2869
2870	for (mp = sys.cp; mp; mp = mp->next)
2871	outp += print_name(width: mp->width, printed: &printed, delim, name: mp->name, type: mp->type, format: mp->format);
2872
2873	for (pp = sys.perf_cp; pp; pp = pp->next)
2874	outp += print_name(width: pp->width, printed: &printed, delim, name: pp->name, type: pp->type, format: pp->format);
2875
2876	ppmt = sys.pmt_cp;
2877	while (ppmt) {
2878	switch (ppmt->type) {
2879	case PMT_TYPE_RAW:
2880	outp += print_name(width: pmt_counter_get_width(p: ppmt), printed: &printed, delim, name: ppmt->name, type: COUNTER_ITEMS, format: ppmt->format);
2881
2882	break;
2883
2884	case PMT_TYPE_XTAL_TIME:
2885	case PMT_TYPE_TCORE_CLOCK:
2886	outp += print_name(width: 32, printed: &printed, delim, name: ppmt->name, type: COUNTER_ITEMS, format: ppmt->format);
2887	break;
2888	}
2889
2890	ppmt = ppmt->next;
2891	}
2892	if (DO_BIC(BIC_PkgTmp))
2893	outp += sprintf(buf: outp, fmt: "%sPkgTmp", (printed++ ? delim : ""));
2894
2895	if (DO_BIC(BIC_GFX_rc6))
2896	outp += sprintf(buf: outp, fmt: "%sGFX%%rc6", (printed++ ? delim : ""));
2897
2898	if (DO_BIC(BIC_GFXMHz))
2899	outp += sprintf(buf: outp, fmt: "%sGFXMHz", (printed++ ? delim : ""));
2900
2901	if (DO_BIC(BIC_GFXACTMHz))
2902	outp += sprintf(buf: outp, fmt: "%sGFXAMHz", (printed++ ? delim : ""));
2903
2904	if (DO_BIC(BIC_SAM_mc6))
2905	outp += sprintf(buf: outp, fmt: "%sSAM%%mc6", (printed++ ? delim : ""));
2906
2907	if (DO_BIC(BIC_SAMMHz))
2908	outp += sprintf(buf: outp, fmt: "%sSAMMHz", (printed++ ? delim : ""));
2909
2910	if (DO_BIC(BIC_SAMACTMHz))
2911	outp += sprintf(buf: outp, fmt: "%sSAMAMHz", (printed++ ? delim : ""));
2912
2913	if (DO_BIC(BIC_Totl_c0))
2914	outp += sprintf(buf: outp, fmt: "%sTotl%%C0", (printed++ ? delim : ""));
2915	if (DO_BIC(BIC_Any_c0))
2916	outp += sprintf(buf: outp, fmt: "%sAny%%C0", (printed++ ? delim : ""));
2917	if (DO_BIC(BIC_GFX_c0))
2918	outp += sprintf(buf: outp, fmt: "%sGFX%%C0", (printed++ ? delim : ""));
2919	if (DO_BIC(BIC_CPUGFX))
2920	outp += sprintf(buf: outp, fmt: "%sCPUGFX%%", (printed++ ? delim : ""));
2921
2922	if (DO_BIC(BIC_Pkgpc2))
2923	outp += sprintf(buf: outp, fmt: "%sPkg%%pc2", (printed++ ? delim : ""));
2924	if (DO_BIC(BIC_Pkgpc3))
2925	outp += sprintf(buf: outp, fmt: "%sPkg%%pc3", (printed++ ? delim : ""));
2926	if (DO_BIC(BIC_Pkgpc6))
2927	outp += sprintf(buf: outp, fmt: "%sPkg%%pc6", (printed++ ? delim : ""));
2928	if (DO_BIC(BIC_Pkgpc7))
2929	outp += sprintf(buf: outp, fmt: "%sPkg%%pc7", (printed++ ? delim : ""));
2930	if (DO_BIC(BIC_Pkgpc8))
2931	outp += sprintf(buf: outp, fmt: "%sPkg%%pc8", (printed++ ? delim : ""));
2932	if (DO_BIC(BIC_Pkgpc9))
2933	outp += sprintf(buf: outp, fmt: "%sPkg%%pc9", (printed++ ? delim : ""));
2934	if (DO_BIC(BIC_Pkgpc10))
2935	outp += sprintf(buf: outp, fmt: "%sPk%%pc10", (printed++ ? delim : ""));
2936	if (DO_BIC(BIC_Diec6))
2937	outp += sprintf(buf: outp, fmt: "%sDie%%c6", (printed++ ? delim : ""));
2938	if (DO_BIC(BIC_CPU_LPI))
2939	outp += sprintf(buf: outp, fmt: "%sCPU%%LPI", (printed++ ? delim : ""));
2940	if (DO_BIC(BIC_SYS_LPI))
2941	outp += sprintf(buf: outp, fmt: "%sSYS%%LPI", (printed++ ? delim : ""));
2942
2943	if (!rapl_joules) {
2944	if (DO_BIC(BIC_PkgWatt))
2945	outp += sprintf(buf: outp, fmt: "%sPkgWatt", (printed++ ? delim : ""));
2946	if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
2947	outp += sprintf(buf: outp, fmt: "%sCorWatt", (printed++ ? delim : ""));
2948	if (DO_BIC(BIC_GFXWatt))
2949	outp += sprintf(buf: outp, fmt: "%sGFXWatt", (printed++ ? delim : ""));
2950	if (DO_BIC(BIC_RAMWatt))
2951	outp += sprintf(buf: outp, fmt: "%sRAMWatt", (printed++ ? delim : ""));
2952	if (DO_BIC(BIC_PKG__))
2953	outp += sprintf(buf: outp, fmt: "%sPKG_%%", (printed++ ? delim : ""));
2954	if (DO_BIC(BIC_RAM__))
2955	outp += sprintf(buf: outp, fmt: "%sRAM_%%", (printed++ ? delim : ""));
2956	} else {
2957	if (DO_BIC(BIC_Pkg_J))
2958	outp += sprintf(buf: outp, fmt: "%sPkg_J", (printed++ ? delim : ""));
2959	if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
2960	outp += sprintf(buf: outp, fmt: "%sCor_J", (printed++ ? delim : ""));
2961	if (DO_BIC(BIC_GFX_J))
2962	outp += sprintf(buf: outp, fmt: "%sGFX_J", (printed++ ? delim : ""));
2963	if (DO_BIC(BIC_RAM_J))
2964	outp += sprintf(buf: outp, fmt: "%sRAM_J", (printed++ ? delim : ""));
2965	if (DO_BIC(BIC_PKG__))
2966	outp += sprintf(buf: outp, fmt: "%sPKG_%%", (printed++ ? delim : ""));
2967	if (DO_BIC(BIC_RAM__))
2968	outp += sprintf(buf: outp, fmt: "%sRAM_%%", (printed++ ? delim : ""));
2969	}
2970	if (DO_BIC(BIC_UNCORE_MHZ))
2971	outp += sprintf(buf: outp, fmt: "%sUncMHz", (printed++ ? delim : ""));
2972
2973	for (mp = sys.pp; mp; mp = mp->next)
2974	outp += print_name(width: mp->width, printed: &printed, delim, name: mp->name, type: mp->type, format: mp->format);
2975
2976	for (pp = sys.perf_pp; pp; pp = pp->next)
2977	outp += print_name(width: pp->width, printed: &printed, delim, name: pp->name, type: pp->type, format: pp->format);
2978
2979	ppmt = sys.pmt_pp;
2980	while (ppmt) {
2981	switch (ppmt->type) {
2982	case PMT_TYPE_RAW:
2983	outp += print_name(width: pmt_counter_get_width(p: ppmt), printed: &printed, delim, name: ppmt->name, type: COUNTER_ITEMS, format: ppmt->format);
2984	break;
2985
2986	case PMT_TYPE_XTAL_TIME:
2987	case PMT_TYPE_TCORE_CLOCK:
2988	outp += print_name(width: 32, printed: &printed, delim, name: ppmt->name, type: COUNTER_ITEMS, format: ppmt->format);
2989	break;
2990	}
2991
2992	ppmt = ppmt->next;
2993	}
2994
2995	if (DO_BIC(BIC_SysWatt))
2996	outp += sprintf(buf: outp, fmt: "%sSysWatt", (printed++ ? delim : ""));
2997	if (DO_BIC(BIC_Sys_J))
2998	outp += sprintf(buf: outp, fmt: "%sSys_J", (printed++ ? delim : ""));
2999
3000	outp += sprintf(buf: outp, fmt: "\n");
3001	}
3002
3003	/*
3004	* pct()
3005	*
3006	* If absolute value is < 1.1, return percentage
3007	* otherwise, return nan
3008	*
3009	* return value is appropriate for printing percentages with %f
3010	* while flagging some obvious erroneous values.
3011	*/
3012	double pct(double d)
3013	{
3014
3015	double abs = fabs(d);
3016
3017	if (abs < 1.10)
3018	return (100.0 * d);
3019	return nan("");
3020	}
3021
3022	int dump_counters(PER_THREAD_PARAMS)
3023	{
3024	int i;
3025	struct msr_counter *mp;
3026	struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even;
3027
3028	outp += sprintf(buf: outp, fmt: "t %p, c %p, p %p\n", t, c, p);
3029
3030	if (t) {
3031	outp += sprintf(buf: outp, fmt: "CPU: %d flags 0x%x\n", t->cpu_id, t->flags);
3032	outp += sprintf(buf: outp, fmt: "TSC: %016llX\n", t->tsc);
3033	outp += sprintf(buf: outp, fmt: "aperf: %016llX\n", t->aperf);
3034	outp += sprintf(buf: outp, fmt: "mperf: %016llX\n", t->mperf);
3035	outp += sprintf(buf: outp, fmt: "c1: %016llX\n", t->c1);
3036
3037	if (DO_BIC(BIC_IPC))
3038	outp += sprintf(buf: outp, fmt: "IPC: %lld\n", t->instr_count);
3039
3040	if (DO_BIC(BIC_IRQ))
3041	outp += sprintf(buf: outp, fmt: "IRQ: %lld\n", t->irq_count);
3042	if (DO_BIC(BIC_NMI))
3043	outp += sprintf(buf: outp, fmt: "IRQ: %lld\n", t->nmi_count);
3044	if (DO_BIC(BIC_SMI))
3045	outp += sprintf(buf: outp, fmt: "SMI: %d\n", t->smi_count);
3046
3047	outp += sprintf(buf: outp, fmt: "LLC refs: %lld", t->llc.references);
3048	outp += sprintf(buf: outp, fmt: "LLC miss: %lld", t->llc.misses);
3049	outp += sprintf(buf: outp, fmt: "LLC Hit%%: %.2f", pct(d: (t->llc.references - t->llc.misses) / t->llc.references));
3050
3051	for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3052	outp += sprintf(buf: outp, fmt: "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path);
3053	}
3054	}
3055
3056	if (c && is_cpu_first_thread_in_core(t, c)) {
3057	outp += sprintf(buf: outp, fmt: "core: %d\n", c->core_id);
3058	outp += sprintf(buf: outp, fmt: "c3: %016llX\n", c->c3);
3059	outp += sprintf(buf: outp, fmt: "c6: %016llX\n", c->c6);
3060	outp += sprintf(buf: outp, fmt: "c7: %016llX\n", c->c7);
3061	outp += sprintf(buf: outp, fmt: "DTS: %dC\n", c->core_temp_c);
3062	outp += sprintf(buf: outp, fmt: "cpu_throt_count: %016llX\n", c->core_throt_cnt);
3063
3064	const unsigned long long energy_value = c->core_energy.raw_value * c->core_energy.scale;
3065	const double energy_scale = c->core_energy.scale;
3066
3067	if (c->core_energy.unit == RAPL_UNIT_JOULES)
3068	outp += sprintf(buf: outp, fmt: "Joules: %0llX (scale: %lf)\n", energy_value, energy_scale);
3069
3070	for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3071	outp += sprintf(buf: outp, fmt: "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, c->counter[i], mp->sp->path);
3072	}
3073	outp += sprintf(buf: outp, fmt: "mc6_us: %016llX\n", c->mc6_us);
3074	}
3075
3076	if (p && is_cpu_first_core_in_package(t, p)) {
3077	outp += sprintf(buf: outp, fmt: "package: %d\n", p->package_id);
3078
3079	outp += sprintf(buf: outp, fmt: "Weighted cores: %016llX\n", p->pkg_wtd_core_c0);
3080	outp += sprintf(buf: outp, fmt: "Any cores: %016llX\n", p->pkg_any_core_c0);
3081	outp += sprintf(buf: outp, fmt: "Any GFX: %016llX\n", p->pkg_any_gfxe_c0);
3082	outp += sprintf(buf: outp, fmt: "CPU + GFX: %016llX\n", p->pkg_both_core_gfxe_c0);
3083
3084	outp += sprintf(buf: outp, fmt: "pc2: %016llX\n", p->pc2);
3085	if (DO_BIC(BIC_Pkgpc3))
3086	outp += sprintf(buf: outp, fmt: "pc3: %016llX\n", p->pc3);
3087	if (DO_BIC(BIC_Pkgpc6))
3088	outp += sprintf(buf: outp, fmt: "pc6: %016llX\n", p->pc6);
3089	if (DO_BIC(BIC_Pkgpc7))
3090	outp += sprintf(buf: outp, fmt: "pc7: %016llX\n", p->pc7);
3091	outp += sprintf(buf: outp, fmt: "pc8: %016llX\n", p->pc8);
3092	outp += sprintf(buf: outp, fmt: "pc9: %016llX\n", p->pc9);
3093	outp += sprintf(buf: outp, fmt: "pc10: %016llX\n", p->pc10);
3094	outp += sprintf(buf: outp, fmt: "cpu_lpi: %016llX\n", p->cpu_lpi);
3095	outp += sprintf(buf: outp, fmt: "sys_lpi: %016llX\n", p->sys_lpi);
3096	outp += sprintf(buf: outp, fmt: "Joules PKG: %0llX\n", p->energy_pkg.raw_value);
3097	outp += sprintf(buf: outp, fmt: "Joules COR: %0llX\n", p->energy_cores.raw_value);
3098	outp += sprintf(buf: outp, fmt: "Joules GFX: %0llX\n", p->energy_gfx.raw_value);
3099	outp += sprintf(buf: outp, fmt: "Joules RAM: %0llX\n", p->energy_dram.raw_value);
3100	outp += sprintf(buf: outp, fmt: "Joules PSYS: %0llX\n", pplat_cnt->energy_psys.raw_value);
3101	outp += sprintf(buf: outp, fmt: "Throttle PKG: %0llX\n", p->rapl_pkg_perf_status.raw_value);
3102	outp += sprintf(buf: outp, fmt: "Throttle RAM: %0llX\n", p->rapl_dram_perf_status.raw_value);
3103	outp += sprintf(buf: outp, fmt: "PTM: %dC\n", p->pkg_temp_c);
3104
3105	for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3106	outp += sprintf(buf: outp, fmt: "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, p->counter[i], mp->sp->path);
3107	}
3108	}
3109
3110	outp += sprintf(buf: outp, fmt: "\n");
3111
3112	return 0;
3113	}
3114
3115	double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desired_unit, double interval)
3116	{
3117	assert(desired_unit != RAPL_UNIT_INVALID);
3118
3119	/*
3120	* For now we don't expect anything other than joules,
3121	* so just simplify the logic.
3122	*/
3123	assert(c->unit == RAPL_UNIT_JOULES);
3124
3125	const double scaled = c->raw_value * c->scale;
3126
3127	if (desired_unit == RAPL_UNIT_WATTS)
3128	return scaled / interval;
3129	return scaled;
3130	}
3131
3132	void get_perf_llc_stats(int cpu, struct llc_stats *llc)
3133	{
3134	struct read_format {
3135	unsigned long long num_read;
3136	struct llc_stats llc;
3137	} r;
3138	const ssize_t expected_read_size = sizeof(r);
3139	ssize_t actual_read_size;
3140
3141	actual_read_size = read(fd_llc_percpu[cpu], &r, expected_read_size);
3142
3143	if (actual_read_size == -1)
3144	err(-1, "%s(cpu%d,) %d,,%ld\n", __func__, cpu, fd_llc_percpu[cpu], expected_read_size);
3145
3146	llc->references = r.llc.references;
3147	llc->misses = r.llc.misses;
3148	if (actual_read_size != expected_read_size)
3149	warn("%s: failed to read perf_data (req %zu act %zu)", __func__, expected_read_size, actual_read_size);
3150	}
3151
3152	/*
3153	* column formatting convention & formats
3154	*/
3155	int format_counters(PER_THREAD_PARAMS)
3156	{
3157	static int count;
3158
3159	struct platform_counters *pplat_cnt = NULL;
3160	double interval_float, tsc;
3161	char *fmt8 = "%s%.2f";
3162
3163	int i;
3164	struct msr_counter *mp;
3165	struct perf_counter_info *pp;
3166	struct pmt_counter *ppmt;
3167	char *delim = "\t";
3168	int printed = 0;
3169
3170	if (t == &average.threads) {
3171	pplat_cnt = count & 1 ? &platform_counters_odd : &platform_counters_even;
3172	++count;
3173	}
3174
3175	/* if showing only 1st thread in core and this isn't one, bail out */
3176	if (show_core_only && !is_cpu_first_thread_in_core(t, c))
3177	return 0;
3178
3179	/* if showing only 1st thread in pkg and this isn't one, bail out */
3180	if (show_pkg_only && !is_cpu_first_core_in_package(t, p))
3181	return 0;
3182
3183	/*if not summary line and --cpu is used */
3184	if ((t != &average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset)))
3185	return 0;
3186
3187	if (DO_BIC(BIC_USEC)) {
3188	/* on each row, print how many usec each timestamp took to gather */
3189	struct timeval tv;
3190
3191	timersub(&t->tv_end, &t->tv_begin, &tv);
3192	outp += sprintf(buf: outp, fmt: "%5ld\t", tv.tv_sec * 1000000 + tv.tv_usec);
3193	}
3194
3195	/* Time_Of_Day_Seconds: on each row, print sec.usec last timestamp taken */
3196	if (DO_BIC(BIC_TOD))
3197	outp += sprintf(buf: outp, fmt: "%10ld.%06ld\t", t->tv_end.tv_sec, t->tv_end.tv_usec);
3198
3199	interval_float = t->tv_delta.tv_sec + t->tv_delta.tv_usec / 1000000.0;
3200
3201	tsc = t->tsc * tsc_tweak;
3202
3203	/* topo columns, print blanks on 1st (average) line */
3204	if (t == &average.threads) {
3205	if (DO_BIC(BIC_Package))
3206	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3207	if (DO_BIC(BIC_Die))
3208	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3209	if (DO_BIC(BIC_L3))
3210	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3211	if (DO_BIC(BIC_Node))
3212	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3213	if (DO_BIC(BIC_Core))
3214	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3215	if (DO_BIC(BIC_CPU))
3216	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3217	if (DO_BIC(BIC_APIC))
3218	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3219	if (DO_BIC(BIC_X2APIC))
3220	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3221	} else {
3222	if (DO_BIC(BIC_Package)) {
3223	if (p)
3224	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), p->package_id);
3225	else
3226	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3227	}
3228	if (DO_BIC(BIC_Die)) {
3229	if (c)
3230	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].die_id);
3231	else
3232	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3233	}
3234	if (DO_BIC(BIC_L3)) {
3235	if (c)
3236	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].l3_id);
3237	else
3238	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3239	}
3240	if (DO_BIC(BIC_Node)) {
3241	if (t)
3242	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id);
3243	else
3244	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3245	}
3246	if (DO_BIC(BIC_Core)) {
3247	if (c)
3248	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), c->core_id);
3249	else
3250	outp += sprintf(buf: outp, fmt: "%s-", (printed++ ? delim : ""));
3251	}
3252	if (DO_BIC(BIC_CPU))
3253	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), t->cpu_id);
3254	if (DO_BIC(BIC_APIC))
3255	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), t->apic_id);
3256	if (DO_BIC(BIC_X2APIC))
3257	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), t->x2apic_id);
3258	}
3259
3260	if (DO_BIC(BIC_Avg_MHz))
3261	outp += sprintf(buf: outp, fmt: "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float);
3262
3263	if (DO_BIC(BIC_Busy))
3264	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: t->mperf / tsc));
3265
3266	if (DO_BIC(BIC_Bzy_MHz)) {
3267	if (has_base_hz)
3268	outp += sprintf(buf: outp, fmt: "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf);
3269	else
3270	outp += sprintf(buf: outp, fmt: "%s%.0f", (printed++ ? delim : ""), tsc / units * t->aperf / t->mperf / interval_float);
3271	}
3272
3273	if (DO_BIC(BIC_TSC_MHz))
3274	outp += sprintf(buf: outp, fmt: "%s%.0f", (printed++ ? delim : ""), 1.0 * t->tsc / units / interval_float);
3275
3276	if (DO_BIC(BIC_IPC))
3277	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), 1.0 * t->instr_count / t->aperf);
3278
3279	/* IRQ */
3280	if (DO_BIC(BIC_IRQ)) {
3281	if (sums_need_wide_columns)
3282	outp += sprintf(buf: outp, fmt: "%s%8lld", (printed++ ? delim : ""), t->irq_count);
3283	else
3284	outp += sprintf(buf: outp, fmt: "%s%lld", (printed++ ? delim : ""), t->irq_count);
3285	}
3286
3287	/* NMI */
3288	if (DO_BIC(BIC_NMI)) {
3289	if (sums_need_wide_columns)
3290	outp += sprintf(buf: outp, fmt: "%s%8lld", (printed++ ? delim : ""), t->nmi_count);
3291	else
3292	outp += sprintf(buf: outp, fmt: "%s%lld", (printed++ ? delim : ""), t->nmi_count);
3293	}
3294
3295	/* SMI */
3296	if (DO_BIC(BIC_SMI))
3297	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), t->smi_count);
3298
3299	/* LLC Stats */
3300	if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT)) {
3301	if (DO_BIC(BIC_LLC_RPS))
3302	outp += sprintf(buf: outp, fmt: "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000);
3303
3304	if (DO_BIC(BIC_LLC_HIT))
3305	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), pct(d: (t->llc.references - t->llc.misses) / t->llc.references));
3306	}
3307
3308	/* Added Thread Counters */
3309	for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3310	if (mp->format == FORMAT_RAW)
3311	outp += print_hex_value(width: mp->width, printed: &printed, delim, value: t->counter[i]);
3312	else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3313	outp += print_decimal_value(width: mp->width, printed: &printed, delim, value: t->counter[i]);
3314	else if (mp->format == FORMAT_PERCENT) {
3315	if (mp->type == COUNTER_USEC)
3316	outp += print_float_value(printed: &printed, delim, value: t->counter[i] / interval_float / 10000);
3317	else
3318	outp += print_float_value(printed: &printed, delim, value: pct(d: t->counter[i] / tsc));
3319	}
3320	}
3321
3322	/* Added perf Thread Counters */
3323	for (i = 0, pp = sys.perf_tp; pp; ++i, pp = pp->next) {
3324	if (pp->format == FORMAT_RAW)
3325	outp += print_hex_value(width: pp->width, printed: &printed, delim, value: t->perf_counter[i]);
3326	else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3327	outp += print_decimal_value(width: pp->width, printed: &printed, delim, value: t->perf_counter[i]);
3328	else if (pp->format == FORMAT_PERCENT) {
3329	if (pp->type == COUNTER_USEC)
3330	outp += print_float_value(printed: &printed, delim, value: t->perf_counter[i] / interval_float / 10000);
3331	else
3332	outp += print_float_value(printed: &printed, delim, value: pct(d: t->perf_counter[i] / tsc));
3333	}
3334	}
3335
3336	/* Added PMT Thread Counters */
3337	for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
3338	const unsigned long value_raw = t->pmt_counter[i];
3339	double value_converted;
3340	switch (ppmt->type) {
3341	case PMT_TYPE_RAW:
3342	outp += print_hex_value(width: pmt_counter_get_width(p: ppmt), printed: &printed, delim, value: t->pmt_counter[i]);
3343	break;
3344
3345	case PMT_TYPE_XTAL_TIME:
3346	value_converted = pct(d: value_raw / crystal_hz / interval_float);
3347	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), value_converted);
3348	break;
3349
3350	case PMT_TYPE_TCORE_CLOCK:
3351	value_converted = pct(d: value_raw / tcore_clock_freq_hz / interval_float);
3352	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), value_converted);
3353	}
3354	}
3355
3356	/* C1 */
3357	if (DO_BIC(BIC_CPU_c1))
3358	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: t->c1 / tsc));
3359
3360	/* print per-core data only for 1st thread in core */
3361	if (!is_cpu_first_thread_in_core(t, c))
3362	goto done;
3363
3364	if (DO_BIC(BIC_CPU_c3))
3365	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: c->c3 / tsc));
3366	if (DO_BIC(BIC_CPU_c6))
3367	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: c->c6 / tsc));
3368	if (DO_BIC(BIC_CPU_c7))
3369	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: c->c7 / tsc));
3370
3371	/* Mod%c6 */
3372	if (DO_BIC(BIC_Mod_c6))
3373	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: c->mc6_us / tsc));
3374
3375	if (DO_BIC(BIC_CoreTmp))
3376	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), c->core_temp_c);
3377
3378	/* Core throttle count */
3379	if (DO_BIC(BIC_CORE_THROT_CNT))
3380	outp += sprintf(buf: outp, fmt: "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt);
3381
3382	/* Added Core Counters */
3383	for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3384	if (mp->format == FORMAT_RAW)
3385	outp += print_hex_value(width: mp->width, printed: &printed, delim, value: c->counter[i]);
3386	else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3387	outp += print_decimal_value(width: mp->width, printed: &printed, delim, value: c->counter[i]);
3388	else if (mp->format == FORMAT_PERCENT)
3389	outp += print_float_value(printed: &printed, delim, value: pct(d: c->counter[i] / tsc));
3390	}
3391
3392	/* Added perf Core counters */
3393	for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3394	if (pp->format == FORMAT_RAW)
3395	outp += print_hex_value(width: pp->width, printed: &printed, delim, value: c->perf_counter[i]);
3396	else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3397	outp += print_decimal_value(width: pp->width, printed: &printed, delim, value: c->perf_counter[i]);
3398	else if (pp->format == FORMAT_PERCENT)
3399	outp += print_float_value(printed: &printed, delim, value: pct(d: c->perf_counter[i] / tsc));
3400	}
3401
3402	/* Added PMT Core counters */
3403	for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3404	const unsigned long value_raw = c->pmt_counter[i];
3405	double value_converted;
3406	switch (ppmt->type) {
3407	case PMT_TYPE_RAW:
3408	outp += print_hex_value(width: pmt_counter_get_width(p: ppmt), printed: &printed, delim, value: c->pmt_counter[i]);
3409	break;
3410
3411	case PMT_TYPE_XTAL_TIME:
3412	value_converted = pct(d: value_raw / crystal_hz / interval_float);
3413	outp += print_float_value(printed: &printed, delim, value: value_converted);
3414	break;
3415
3416	case PMT_TYPE_TCORE_CLOCK:
3417	value_converted = pct(d: value_raw / tcore_clock_freq_hz / interval_float);
3418	outp += print_float_value(printed: &printed, delim, value: value_converted);
3419	}
3420	}
3421
3422	if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
3423	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &c->core_energy, desired_unit: RAPL_UNIT_WATTS, interval: interval_float));
3424	if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
3425	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &c->core_energy, desired_unit: RAPL_UNIT_JOULES, interval: interval_float));
3426
3427	/* print per-package data only for 1st core in package */
3428	if (!is_cpu_first_core_in_package(t, p))
3429	goto done;
3430
3431	/* PkgTmp */
3432	if (DO_BIC(BIC_PkgTmp))
3433	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), p->pkg_temp_c);
3434
3435	/* GFXrc6 */
3436	if (DO_BIC(BIC_GFX_rc6)) {
3437	if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */
3438	outp += sprintf(buf: outp, fmt: "%s**.**", (printed++ ? delim : ""));
3439	} else {
3440	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), p->gfx_rc6_ms / 10.0 / interval_float);
3441	}
3442	}
3443
3444	/* GFXMHz */
3445	if (DO_BIC(BIC_GFXMHz))
3446	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), p->gfx_mhz);
3447
3448	/* GFXACTMHz */
3449	if (DO_BIC(BIC_GFXACTMHz))
3450	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), p->gfx_act_mhz);
3451
3452	/* SAMmc6 */
3453	if (DO_BIC(BIC_SAM_mc6)) {
3454	if (p->sam_mc6_ms == -1) { /* detect GFX counter reset */
3455	outp += sprintf(buf: outp, fmt: "%s**.**", (printed++ ? delim : ""));
3456	} else {
3457	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), p->sam_mc6_ms / 10.0 / interval_float);
3458	}
3459	}
3460
3461	/* SAMMHz */
3462	if (DO_BIC(BIC_SAMMHz))
3463	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), p->sam_mhz);
3464
3465	/* SAMACTMHz */
3466	if (DO_BIC(BIC_SAMACTMHz))
3467	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), p->sam_act_mhz);
3468
3469	/* Totl%C0, Any%C0 GFX%C0 CPUGFX% */
3470	if (DO_BIC(BIC_Totl_c0))
3471	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */
3472	if (DO_BIC(BIC_Any_c0))
3473	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pkg_any_core_c0 / tsc));
3474	if (DO_BIC(BIC_GFX_c0))
3475	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pkg_any_gfxe_c0 / tsc));
3476	if (DO_BIC(BIC_CPUGFX))
3477	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pkg_both_core_gfxe_c0 / tsc));
3478
3479	if (DO_BIC(BIC_Pkgpc2))
3480	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pc2 / tsc));
3481	if (DO_BIC(BIC_Pkgpc3))
3482	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pc3 / tsc));
3483	if (DO_BIC(BIC_Pkgpc6))
3484	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pc6 / tsc));
3485	if (DO_BIC(BIC_Pkgpc7))
3486	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pc7 / tsc));
3487	if (DO_BIC(BIC_Pkgpc8))
3488	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pc8 / tsc));
3489	if (DO_BIC(BIC_Pkgpc9))
3490	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pc9 / tsc));
3491	if (DO_BIC(BIC_Pkgpc10))
3492	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->pc10 / tsc));
3493
3494	if (DO_BIC(BIC_Diec6))
3495	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->die_c6 / crystal_hz / interval_float));
3496
3497	if (DO_BIC(BIC_CPU_LPI)) {
3498	if (p->cpu_lpi >= 0)
3499	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->cpu_lpi / 1000000.0 / interval_float));
3500	else
3501	outp += sprintf(buf: outp, fmt: "%s(neg)", (printed++ ? delim : ""));
3502	}
3503	if (DO_BIC(BIC_SYS_LPI)) {
3504	if (p->sys_lpi >= 0)
3505	outp += sprintf(buf: outp, fmt: "%s%.2f", (printed++ ? delim : ""), pct(d: p->sys_lpi / 1000000.0 / interval_float));
3506	else
3507	outp += sprintf(buf: outp, fmt: "%s(neg)", (printed++ ? delim : ""));
3508	}
3509
3510	if (DO_BIC(BIC_PkgWatt))
3511	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->energy_pkg, desired_unit: RAPL_UNIT_WATTS, interval: interval_float));
3512	if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
3513	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->energy_cores, desired_unit: RAPL_UNIT_WATTS, interval: interval_float));
3514	if (DO_BIC(BIC_GFXWatt))
3515	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->energy_gfx, desired_unit: RAPL_UNIT_WATTS, interval: interval_float));
3516	if (DO_BIC(BIC_RAMWatt))
3517	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->energy_dram, desired_unit: RAPL_UNIT_WATTS, interval: interval_float));
3518	if (DO_BIC(BIC_Pkg_J))
3519	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->energy_pkg, desired_unit: RAPL_UNIT_JOULES, interval: interval_float));
3520	if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
3521	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->energy_cores, desired_unit: RAPL_UNIT_JOULES, interval: interval_float));
3522	if (DO_BIC(BIC_GFX_J))
3523	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->energy_gfx, desired_unit: RAPL_UNIT_JOULES, interval: interval_float));
3524	if (DO_BIC(BIC_RAM_J))
3525	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->energy_dram, desired_unit: RAPL_UNIT_JOULES, interval: interval_float));
3526	if (DO_BIC(BIC_PKG__))
3527	outp +=
3528	sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->rapl_pkg_perf_status, desired_unit: RAPL_UNIT_WATTS, interval: interval_float));
3529	if (DO_BIC(BIC_RAM__))
3530	outp +=
3531	sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""), rapl_counter_get_value(c: &p->rapl_dram_perf_status, desired_unit: RAPL_UNIT_WATTS, interval: interval_float));
3532	/* UncMHz */
3533	if (DO_BIC(BIC_UNCORE_MHZ))
3534	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), p->uncore_mhz);
3535
3536	/* Added Package Counters */
3537	for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3538	if (mp->format == FORMAT_RAW)
3539	outp += print_hex_value(width: mp->width, printed: &printed, delim, value: p->counter[i]);
3540	else if (mp->type == COUNTER_K2M)
3541	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), (unsigned int)p->counter[i] / 1000);
3542	else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3543	outp += print_decimal_value(width: mp->width, printed: &printed, delim, value: p->counter[i]);
3544	else if (mp->format == FORMAT_PERCENT)
3545	outp += print_float_value(printed: &printed, delim, value: pct(d: p->counter[i] / tsc));
3546	}
3547
3548	/* Added perf Package Counters */
3549	for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3550	if (pp->format == FORMAT_RAW)
3551	outp += print_hex_value(width: pp->width, printed: &printed, delim, value: p->perf_counter[i]);
3552	else if (pp->type == COUNTER_K2M)
3553	outp += sprintf(buf: outp, fmt: "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000);
3554	else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3555	outp += print_decimal_value(width: pp->width, printed: &printed, delim, value: p->perf_counter[i]);
3556	else if (pp->format == FORMAT_PERCENT)
3557	outp += print_float_value(printed: &printed, delim, value: pct(d: p->perf_counter[i] / tsc));
3558	}
3559
3560	/* Added PMT Package Counters */
3561	for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3562	const unsigned long value_raw = p->pmt_counter[i];
3563	double value_converted;
3564	switch (ppmt->type) {
3565	case PMT_TYPE_RAW:
3566	outp += print_hex_value(width: pmt_counter_get_width(p: ppmt), printed: &printed, delim, value: p->pmt_counter[i]);
3567	break;
3568
3569	case PMT_TYPE_XTAL_TIME:
3570	value_converted = pct(d: value_raw / crystal_hz / interval_float);
3571	outp += print_float_value(printed: &printed, delim, value: value_converted);
3572	break;
3573
3574	case PMT_TYPE_TCORE_CLOCK:
3575	value_converted = pct(d: value_raw / tcore_clock_freq_hz / interval_float);
3576	outp += print_float_value(printed: &printed, delim, value: value_converted);
3577	}
3578	}
3579
3580	if (DO_BIC(BIC_SysWatt) && (t == &average.threads))
3581	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""),
3582	rapl_counter_get_value(c: &pplat_cnt->energy_psys, desired_unit: RAPL_UNIT_WATTS, interval: interval_float));
3583	if (DO_BIC(BIC_Sys_J) && (t == &average.threads))
3584	outp += sprintf(buf: outp, fmt: fmt8, (printed++ ? delim : ""),
3585	rapl_counter_get_value(c: &pplat_cnt->energy_psys, desired_unit: RAPL_UNIT_JOULES, interval: interval_float));
3586
3587	done:
3588	if (*(outp - 1) != '\n')
3589	outp += sprintf(buf: outp, fmt: "\n");
3590
3591	return 0;
3592	}
3593
3594	void flush_output_stdout(void)
3595	{
3596	FILE *filep;
3597
3598	if (outf == stderr)
3599	filep = stdout;
3600	else
3601	filep = outf;
3602
3603	fputs(output_buffer, filep);
3604	fflush(filep);
3605
3606	outp = output_buffer;
3607	}
3608
3609	void flush_output_stderr(void)
3610	{
3611	fputs(output_buffer, outf);
3612	fflush(outf);
3613	outp = output_buffer;
3614	}
3615
3616	void format_all_counters(PER_THREAD_PARAMS)
3617	{
3618	static int count;
3619
3620	if ((!count || (header_iterations && !(count % header_iterations))) || !summary_only)
3621	print_header(delim: "\t");
3622
3623	format_counters(t: &average.threads, c: &average.cores, p: &average.packages);
3624
3625	count++;
3626
3627	if (summary_only)
3628	return;
3629
3630	for_all_cpus(func: format_counters, thread_base: t, core_base: c, pkg_base: p);
3631	}
3632
3633	#define DELTA_WRAP32(new, old) \
3634	old = ((((unsigned long long)new << 32) - ((unsigned long long)old << 32)) >> 32);
3635
3636	int delta_package(struct pkg_data *new, struct pkg_data *old)
3637	{
3638	int i;
3639	struct msr_counter *mp;
3640	struct perf_counter_info *pp;
3641	struct pmt_counter *ppmt;
3642
3643	if (DO_BIC(BIC_Totl_c0))
3644	old->pkg_wtd_core_c0 = new->pkg_wtd_core_c0 - old->pkg_wtd_core_c0;
3645	if (DO_BIC(BIC_Any_c0))
3646	old->pkg_any_core_c0 = new->pkg_any_core_c0 - old->pkg_any_core_c0;
3647	if (DO_BIC(BIC_GFX_c0))
3648	old->pkg_any_gfxe_c0 = new->pkg_any_gfxe_c0 - old->pkg_any_gfxe_c0;
3649	if (DO_BIC(BIC_CPUGFX))
3650	old->pkg_both_core_gfxe_c0 = new->pkg_both_core_gfxe_c0 - old->pkg_both_core_gfxe_c0;
3651
3652	old->pc2 = new->pc2 - old->pc2;
3653	if (DO_BIC(BIC_Pkgpc3))
3654	old->pc3 = new->pc3 - old->pc3;
3655	if (DO_BIC(BIC_Pkgpc6))
3656	old->pc6 = new->pc6 - old->pc6;
3657	if (DO_BIC(BIC_Pkgpc7))
3658	old->pc7 = new->pc7 - old->pc7;
3659	old->pc8 = new->pc8 - old->pc8;
3660	old->pc9 = new->pc9 - old->pc9;
3661	old->pc10 = new->pc10 - old->pc10;
3662	old->die_c6 = new->die_c6 - old->die_c6;
3663	old->cpu_lpi = new->cpu_lpi - old->cpu_lpi;
3664	old->sys_lpi = new->sys_lpi - old->sys_lpi;
3665	old->pkg_temp_c = new->pkg_temp_c;
3666
3667	/* flag an error when rc6 counter resets/wraps */
3668	if (old->gfx_rc6_ms > new->gfx_rc6_ms)
3669	old->gfx_rc6_ms = -1;
3670	else
3671	old->gfx_rc6_ms = new->gfx_rc6_ms - old->gfx_rc6_ms;
3672
3673	old->uncore_mhz = new->uncore_mhz;
3674	old->gfx_mhz = new->gfx_mhz;
3675	old->gfx_act_mhz = new->gfx_act_mhz;
3676
3677	/* flag an error when mc6 counter resets/wraps */
3678	if (old->sam_mc6_ms > new->sam_mc6_ms)
3679	old->sam_mc6_ms = -1;
3680	else
3681	old->sam_mc6_ms = new->sam_mc6_ms - old->sam_mc6_ms;
3682
3683	old->sam_mhz = new->sam_mhz;
3684	old->sam_act_mhz = new->sam_act_mhz;
3685
3686	old->energy_pkg.raw_value = new->energy_pkg.raw_value - old->energy_pkg.raw_value;
3687	old->energy_cores.raw_value = new->energy_cores.raw_value - old->energy_cores.raw_value;
3688	old->energy_gfx.raw_value = new->energy_gfx.raw_value - old->energy_gfx.raw_value;
3689	old->energy_dram.raw_value = new->energy_dram.raw_value - old->energy_dram.raw_value;
3690	old->rapl_pkg_perf_status.raw_value = new->rapl_pkg_perf_status.raw_value - old->rapl_pkg_perf_status.raw_value;
3691	old->rapl_dram_perf_status.raw_value = new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value;
3692
3693	for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3694	if (mp->format == FORMAT_RAW)
3695	old->counter[i] = new->counter[i];
3696	else if (mp->format == FORMAT_AVERAGE)
3697	old->counter[i] = new->counter[i];
3698	else
3699	old->counter[i] = new->counter[i] - old->counter[i];
3700	}
3701
3702	for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3703	if (pp->format == FORMAT_RAW)
3704	old->perf_counter[i] = new->perf_counter[i];
3705	else if (pp->format == FORMAT_AVERAGE)
3706	old->perf_counter[i] = new->perf_counter[i];
3707	else
3708	old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3709	}
3710
3711	for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3712	if (ppmt->format == FORMAT_RAW)
3713	old->pmt_counter[i] = new->pmt_counter[i];
3714	else
3715	old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3716	}
3717
3718	return 0;
3719	}
3720
3721	void delta_core(struct core_data *new, struct core_data *old)
3722	{
3723	int i;
3724	struct msr_counter *mp;
3725	struct perf_counter_info *pp;
3726	struct pmt_counter *ppmt;
3727
3728	old->c3 = new->c3 - old->c3;
3729	old->c6 = new->c6 - old->c6;
3730	old->c7 = new->c7 - old->c7;
3731	old->core_temp_c = new->core_temp_c;
3732	old->core_throt_cnt = new->core_throt_cnt - old->core_throt_cnt;
3733	old->mc6_us = new->mc6_us - old->mc6_us;
3734
3735	DELTA_WRAP32(new->core_energy.raw_value, old->core_energy.raw_value);
3736
3737	for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3738	if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE)
3739	old->counter[i] = new->counter[i];
3740	else
3741	old->counter[i] = new->counter[i] - old->counter[i];
3742	}
3743
3744	for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3745	if (pp->format == FORMAT_RAW)
3746	old->perf_counter[i] = new->perf_counter[i];
3747	else
3748	old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3749	}
3750
3751	for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3752	if (ppmt->format == FORMAT_RAW)
3753	old->pmt_counter[i] = new->pmt_counter[i];
3754	else
3755	old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3756	}
3757	}
3758
3759	int soft_c1_residency_display(int bic)
3760	{
3761	if (!DO_BIC(BIC_CPU_c1) || platform->has_msr_core_c1_res)
3762	return 0;
3763
3764	return DO_BIC_READ(bic);
3765	}
3766
3767	/*
3768	* old = new - old
3769	*/
3770	int delta_thread(struct thread_data *new, struct thread_data *old, struct core_data *core_delta)
3771	{
3772	int i;
3773	struct msr_counter *mp;
3774	struct perf_counter_info *pp;
3775	struct pmt_counter *ppmt;
3776
3777	/* we run cpuid just the 1st time, copy the results */
3778	if (DO_BIC(BIC_APIC))
3779	new->apic_id = old->apic_id;
3780	if (DO_BIC(BIC_X2APIC))
3781	new->x2apic_id = old->x2apic_id;
3782
3783	/*
3784	* the timestamps from start of measurement interval are in "old"
3785	* the timestamp from end of measurement interval are in "new"
3786	* over-write old w/ new so we can print end of interval values
3787	*/
3788
3789	timersub(&new->tv_begin, &old->tv_begin, &old->tv_delta);
3790	old->tv_begin = new->tv_begin;
3791	old->tv_end = new->tv_end;
3792
3793	old->tsc = new->tsc - old->tsc;
3794
3795	/* check for TSC < 1 Mcycles over interval */
3796	if (old->tsc < (1000 * 1000))
3797	errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n"
3798	"You can disable all c-states by booting with \"idle=poll\"\n" "or just the deep ones with \"processor.max_cstate=1\"");
3799
3800	old->c1 = new->c1 - old->c1;
3801
3802	if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz) || DO_BIC(BIC_IPC)
3803	|| soft_c1_residency_display(bic: BIC_Avg_MHz)) {
3804	if ((new->aperf > old->aperf) && (new->mperf > old->mperf)) {
3805	old->aperf = new->aperf - old->aperf;
3806	old->mperf = new->mperf - old->mperf;
3807	} else {
3808	return -1;
3809	}
3810	}
3811
3812	if (platform->has_msr_core_c1_res) {
3813	/*
3814	* Some models have a dedicated C1 residency MSR,
3815	* which should be more accurate than the derivation below.
3816	*/
3817	} else {
3818	/*
3819	* As counter collection is not atomic,
3820	* it is possible for mperf's non-halted cycles + idle states
3821	* to exceed TSC's all cycles: show c1 = 0% in that case.
3822	*/
3823	if ((old->mperf + core_delta->c3 + core_delta->c6 + core_delta->c7) > (old->tsc * tsc_tweak))
3824	old->c1 = 0;
3825	else {
3826	/* normal case, derive c1 */
3827	old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - core_delta->c6 - core_delta->c7;
3828	}
3829	}
3830
3831	if (old->mperf == 0) {
3832	if (debug > 1)
3833	fprintf(outf, "cpu%d MPERF 0!\n", old->cpu_id);
3834	old->mperf = 1; /* divide by 0 protection */
3835	}
3836
3837	if (DO_BIC(BIC_IPC))
3838	old->instr_count = new->instr_count - old->instr_count;
3839
3840	if (DO_BIC(BIC_IRQ))
3841	old->irq_count = new->irq_count - old->irq_count;
3842
3843	if (DO_BIC(BIC_NMI))
3844	old->nmi_count = new->nmi_count - old->nmi_count;
3845
3846	if (DO_BIC(BIC_SMI))
3847	old->smi_count = new->smi_count - old->smi_count;
3848
3849	if (DO_BIC(BIC_LLC_RPS))
3850	old->llc.references = new->llc.references - old->llc.references;
3851
3852	if (DO_BIC(BIC_LLC_HIT))
3853	old->llc.misses = new->llc.misses - old->llc.misses;
3854
3855	for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3856	if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE)
3857	old->counter[i] = new->counter[i];
3858	else
3859	old->counter[i] = new->counter[i] - old->counter[i];
3860	}
3861
3862	for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
3863	if (pp->format == FORMAT_RAW)
3864	old->perf_counter[i] = new->perf_counter[i];
3865	else
3866	old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3867	}
3868
3869	for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
3870	if (ppmt->format == FORMAT_RAW)
3871	old->pmt_counter[i] = new->pmt_counter[i];
3872	else
3873	old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3874	}
3875
3876	return 0;
3877	}
3878
3879	int delta_cpu(struct thread_data *t, struct core_data *c, struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2)
3880	{
3881	int retval = 0;
3882
3883	/* calculate core delta only for 1st thread in core */
3884	if (is_cpu_first_thread_in_core(t, c))
3885	delta_core(new: c, old: c2);
3886
3887	/* always calculate thread delta */
3888	retval = delta_thread(new: t, old: t2, core_delta: c2); /* c2 is core delta */
3889
3890	/* calculate package delta only for 1st core in package */
3891	if (is_cpu_first_core_in_package(t, p))
3892	retval |= delta_package(new: p, old: p2);
3893
3894	return retval;
3895	}
3896
3897	void delta_platform(struct platform_counters *new, struct platform_counters *old)
3898	{
3899	old->energy_psys.raw_value = new->energy_psys.raw_value - old->energy_psys.raw_value;
3900	}
3901
3902	void rapl_counter_clear(struct rapl_counter *c)
3903	{
3904	c->raw_value = 0;
3905	c->scale = 0.0;
3906	c->unit = RAPL_UNIT_INVALID;
3907	}
3908
3909	void clear_counters(PER_THREAD_PARAMS)
3910	{
3911	int i;
3912	struct msr_counter *mp;
3913
3914	t->tv_begin.tv_sec = 0;
3915	t->tv_begin.tv_usec = 0;
3916	t->tv_end.tv_sec = 0;
3917	t->tv_end.tv_usec = 0;
3918	t->tv_delta.tv_sec = 0;
3919	t->tv_delta.tv_usec = 0;
3920
3921	t->tsc = 0;
3922	t->aperf = 0;
3923	t->mperf = 0;
3924	t->c1 = 0;
3925
3926	t->instr_count = 0;
3927
3928	t->irq_count = 0;
3929	t->nmi_count = 0;
3930	t->smi_count = 0;
3931
3932	t->llc.references = 0;
3933	t->llc.misses = 0;
3934
3935	c->c3 = 0;
3936	c->c6 = 0;
3937	c->c7 = 0;
3938	c->mc6_us = 0;
3939	c->core_temp_c = 0;
3940	rapl_counter_clear(c: &c->core_energy);
3941	c->core_throt_cnt = 0;
3942
3943	t->llc.references = 0;
3944	t->llc.misses = 0;
3945
3946	p->pkg_wtd_core_c0 = 0;
3947	p->pkg_any_core_c0 = 0;
3948	p->pkg_any_gfxe_c0 = 0;
3949	p->pkg_both_core_gfxe_c0 = 0;
3950
3951	p->pc2 = 0;
3952	if (DO_BIC(BIC_Pkgpc3))
3953	p->pc3 = 0;
3954	if (DO_BIC(BIC_Pkgpc6))
3955	p->pc6 = 0;
3956	if (DO_BIC(BIC_Pkgpc7))
3957	p->pc7 = 0;
3958	p->pc8 = 0;
3959	p->pc9 = 0;
3960	p->pc10 = 0;
3961	p->die_c6 = 0;
3962	p->cpu_lpi = 0;
3963	p->sys_lpi = 0;
3964
3965	rapl_counter_clear(c: &p->energy_pkg);
3966	rapl_counter_clear(c: &p->energy_dram);
3967	rapl_counter_clear(c: &p->energy_cores);
3968	rapl_counter_clear(c: &p->energy_gfx);
3969	rapl_counter_clear(c: &p->rapl_pkg_perf_status);
3970	rapl_counter_clear(c: &p->rapl_dram_perf_status);
3971	p->pkg_temp_c = 0;
3972
3973	p->gfx_rc6_ms = 0;
3974	p->uncore_mhz = 0;
3975	p->gfx_mhz = 0;
3976	p->gfx_act_mhz = 0;
3977	p->sam_mc6_ms = 0;
3978	p->sam_mhz = 0;
3979	p->sam_act_mhz = 0;
3980	for (i = 0, mp = sys.tp; mp; i++, mp = mp->next)
3981	t->counter[i] = 0;
3982
3983	for (i = 0, mp = sys.cp; mp; i++, mp = mp->next)
3984	c->counter[i] = 0;
3985
3986	for (i = 0, mp = sys.pp; mp; i++, mp = mp->next)
3987	p->counter[i] = 0;
3988
3989	memset(&t->perf_counter[0], 0, sizeof(t->perf_counter));
3990	memset(&c->perf_counter[0], 0, sizeof(c->perf_counter));
3991	memset(&p->perf_counter[0], 0, sizeof(p->perf_counter));
3992
3993	memset(&t->pmt_counter[0], 0, ARRAY_SIZE(t->pmt_counter));
3994	memset(&c->pmt_counter[0], 0, ARRAY_SIZE(c->pmt_counter));
3995	memset(&p->pmt_counter[0], 0, ARRAY_SIZE(p->pmt_counter));
3996	}
3997
3998	void rapl_counter_accumulate(struct rapl_counter *dst, const struct rapl_counter *src)
3999	{
4000	/* Copy unit and scale from src if dst is not initialized */
4001	if (dst->unit == RAPL_UNIT_INVALID) {
4002	dst->unit = src->unit;
4003	dst->scale = src->scale;
4004	}
4005
4006	assert(dst->unit == src->unit);
4007	assert(dst->scale == src->scale);
4008
4009	dst->raw_value += src->raw_value;
4010	}
4011
4012	int sum_counters(PER_THREAD_PARAMS)
4013	{
4014	int i;
4015	struct msr_counter *mp;
4016	struct perf_counter_info *pp;
4017	struct pmt_counter *ppmt;
4018
4019	/* copy un-changing apic_id's */
4020	if (DO_BIC(BIC_APIC))
4021	average.threads.apic_id = t->apic_id;
4022	if (DO_BIC(BIC_X2APIC))
4023	average.threads.x2apic_id = t->x2apic_id;
4024
4025	/* remember first tv_begin */
4026	if (average.threads.tv_begin.tv_sec == 0)
4027	average.threads.tv_begin = procsysfs_tv_begin;
4028
4029	/* remember last tv_end */
4030	average.threads.tv_end = t->tv_end;
4031
4032	average.threads.tsc += t->tsc;
4033	average.threads.aperf += t->aperf;
4034	average.threads.mperf += t->mperf;
4035	average.threads.c1 += t->c1;
4036
4037	average.threads.instr_count += t->instr_count;
4038
4039	average.threads.irq_count += t->irq_count;
4040	average.threads.nmi_count += t->nmi_count;
4041	average.threads.smi_count += t->smi_count;
4042
4043	average.threads.llc.references += t->llc.references;
4044	average.threads.llc.misses += t->llc.misses;
4045
4046	for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4047	if (mp->format == FORMAT_RAW)
4048	continue;
4049	average.threads.counter[i] += t->counter[i];
4050	}
4051
4052	for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4053	if (pp->format == FORMAT_RAW)
4054	continue;
4055	average.threads.perf_counter[i] += t->perf_counter[i];
4056	}
4057
4058	for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4059	average.threads.pmt_counter[i] += t->pmt_counter[i];
4060	}
4061
4062	/* sum per-core values only for 1st thread in core */
4063	if (!is_cpu_first_thread_in_core(t, c))
4064	return 0;
4065
4066	average.cores.c3 += c->c3;
4067	average.cores.c6 += c->c6;
4068	average.cores.c7 += c->c7;
4069	average.cores.mc6_us += c->mc6_us;
4070
4071	average.cores.core_temp_c = MAX(average.cores.core_temp_c, c->core_temp_c);
4072	average.cores.core_throt_cnt = MAX(average.cores.core_throt_cnt, c->core_throt_cnt);
4073
4074	rapl_counter_accumulate(dst: &average.cores.core_energy, src: &c->core_energy);
4075
4076	for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
4077	if (mp->format == FORMAT_RAW)
4078	continue;
4079	average.cores.counter[i] += c->counter[i];
4080	}
4081
4082	for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
4083	if (pp->format == FORMAT_RAW)
4084	continue;
4085	average.cores.perf_counter[i] += c->perf_counter[i];
4086	}
4087
4088	for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
4089	average.cores.pmt_counter[i] += c->pmt_counter[i];
4090	}
4091
4092	/* sum per-pkg values only for 1st core in pkg */
4093	if (!is_cpu_first_core_in_package(t, p))
4094	return 0;
4095
4096	if (DO_BIC(BIC_Totl_c0))
4097	average.packages.pkg_wtd_core_c0 += p->pkg_wtd_core_c0;
4098	if (DO_BIC(BIC_Any_c0))
4099	average.packages.pkg_any_core_c0 += p->pkg_any_core_c0;
4100	if (DO_BIC(BIC_GFX_c0))
4101	average.packages.pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0;
4102	if (DO_BIC(BIC_CPUGFX))
4103	average.packages.pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0;
4104
4105	average.packages.pc2 += p->pc2;
4106	if (DO_BIC(BIC_Pkgpc3))
4107	average.packages.pc3 += p->pc3;
4108	if (DO_BIC(BIC_Pkgpc6))
4109	average.packages.pc6 += p->pc6;
4110	if (DO_BIC(BIC_Pkgpc7))
4111	average.packages.pc7 += p->pc7;
4112	average.packages.pc8 += p->pc8;
4113	average.packages.pc9 += p->pc9;
4114	average.packages.pc10 += p->pc10;
4115	average.packages.die_c6 += p->die_c6;
4116
4117	average.packages.cpu_lpi = p->cpu_lpi;
4118	average.packages.sys_lpi = p->sys_lpi;
4119
4120	rapl_counter_accumulate(dst: &average.packages.energy_pkg, src: &p->energy_pkg);
4121	rapl_counter_accumulate(dst: &average.packages.energy_dram, src: &p->energy_dram);
4122	rapl_counter_accumulate(dst: &average.packages.energy_cores, src: &p->energy_cores);
4123	rapl_counter_accumulate(dst: &average.packages.energy_gfx, src: &p->energy_gfx);
4124
4125	average.packages.gfx_rc6_ms = p->gfx_rc6_ms;
4126	average.packages.uncore_mhz = p->uncore_mhz;
4127	average.packages.gfx_mhz = p->gfx_mhz;
4128	average.packages.gfx_act_mhz = p->gfx_act_mhz;
4129	average.packages.sam_mc6_ms = p->sam_mc6_ms;
4130	average.packages.sam_mhz = p->sam_mhz;
4131	average.packages.sam_act_mhz = p->sam_act_mhz;
4132
4133	average.packages.pkg_temp_c = MAX(average.packages.pkg_temp_c, p->pkg_temp_c);
4134
4135	rapl_counter_accumulate(dst: &average.packages.rapl_pkg_perf_status, src: &p->rapl_pkg_perf_status);
4136	rapl_counter_accumulate(dst: &average.packages.rapl_dram_perf_status, src: &p->rapl_dram_perf_status);
4137
4138	for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
4139	if ((mp->format == FORMAT_RAW) && (topo.num_packages == 0))
4140	average.packages.counter[i] = p->counter[i];
4141	else
4142	average.packages.counter[i] += p->counter[i];
4143	}
4144
4145	for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
4146	if ((pp->format == FORMAT_RAW) && (topo.num_packages == 0))
4147	average.packages.perf_counter[i] = p->perf_counter[i];
4148	else
4149	average.packages.perf_counter[i] += p->perf_counter[i];
4150	}
4151
4152	for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
4153	average.packages.pmt_counter[i] += p->pmt_counter[i];
4154	}
4155
4156	return 0;
4157	}
4158
4159	/*
4160	* sum the counters for all cpus in the system
4161	* compute the weighted average
4162	*/
4163	void compute_average(PER_THREAD_PARAMS)
4164	{
4165	int i;
4166	struct msr_counter *mp;
4167	struct perf_counter_info *pp;
4168	struct pmt_counter *ppmt;
4169
4170	clear_counters(t: &average.threads, c: &average.cores, p: &average.packages);
4171
4172	for_all_cpus(func: sum_counters, thread_base: t, core_base: c, pkg_base: p);
4173
4174	/* Use the global time delta for the average. */
4175	average.threads.tv_delta = tv_delta;
4176
4177	average.threads.tsc /= topo.allowed_cpus;
4178	average.threads.aperf /= topo.allowed_cpus;
4179	average.threads.mperf /= topo.allowed_cpus;
4180	average.threads.instr_count /= topo.allowed_cpus;
4181	average.threads.c1 /= topo.allowed_cpus;
4182
4183	if (average.threads.irq_count > 9999999)
4184	sums_need_wide_columns = 1;
4185	if (average.threads.nmi_count > 9999999)
4186	sums_need_wide_columns = 1;
4187
4188	average.cores.c3 /= topo.allowed_cores;
4189	average.cores.c6 /= topo.allowed_cores;
4190	average.cores.c7 /= topo.allowed_cores;
4191	average.cores.mc6_us /= topo.allowed_cores;
4192
4193	if (DO_BIC(BIC_Totl_c0))
4194	average.packages.pkg_wtd_core_c0 /= topo.allowed_packages;
4195	if (DO_BIC(BIC_Any_c0))
4196	average.packages.pkg_any_core_c0 /= topo.allowed_packages;
4197	if (DO_BIC(BIC_GFX_c0))
4198	average.packages.pkg_any_gfxe_c0 /= topo.allowed_packages;
4199	if (DO_BIC(BIC_CPUGFX))
4200	average.packages.pkg_both_core_gfxe_c0 /= topo.allowed_packages;
4201
4202	average.packages.pc2 /= topo.allowed_packages;
4203	if (DO_BIC(BIC_Pkgpc3))
4204	average.packages.pc3 /= topo.allowed_packages;
4205	if (DO_BIC(BIC_Pkgpc6))
4206	average.packages.pc6 /= topo.allowed_packages;
4207	if (DO_BIC(BIC_Pkgpc7))
4208	average.packages.pc7 /= topo.allowed_packages;
4209
4210	average.packages.pc8 /= topo.allowed_packages;
4211	average.packages.pc9 /= topo.allowed_packages;
4212	average.packages.pc10 /= topo.allowed_packages;
4213	average.packages.die_c6 /= topo.allowed_packages;
4214
4215	for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4216	if (mp->format == FORMAT_RAW)
4217	continue;
4218	if (mp->type == COUNTER_ITEMS) {
4219	if (average.threads.counter[i] > 9999999)
4220	sums_need_wide_columns = 1;
4221	continue;
4222	}
4223	average.threads.counter[i] /= topo.allowed_cpus;
4224	}
4225	for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
4226	if (mp->format == FORMAT_RAW)
4227	continue;
4228	if (mp->type == COUNTER_ITEMS) {
4229	if (average.cores.counter[i] > 9999999)
4230	sums_need_wide_columns = 1;
4231	}
4232	average.cores.counter[i] /= topo.allowed_cores;
4233	}
4234	for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
4235	if (mp->format == FORMAT_RAW)
4236	continue;
4237	if (mp->type == COUNTER_ITEMS) {
4238	if (average.packages.counter[i] > 9999999)
4239	sums_need_wide_columns = 1;
4240	}
4241	average.packages.counter[i] /= topo.allowed_packages;
4242	}
4243
4244	for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4245	if (pp->format == FORMAT_RAW)
4246	continue;
4247	if (pp->type == COUNTER_ITEMS) {
4248	if (average.threads.perf_counter[i] > 9999999)
4249	sums_need_wide_columns = 1;
4250	continue;
4251	}
4252	average.threads.perf_counter[i] /= topo.allowed_cpus;
4253	}
4254	for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
4255	if (pp->format == FORMAT_RAW)
4256	continue;
4257	if (pp->type == COUNTER_ITEMS) {
4258	if (average.cores.perf_counter[i] > 9999999)
4259	sums_need_wide_columns = 1;
4260	}
4261	average.cores.perf_counter[i] /= topo.allowed_cores;
4262	}
4263	for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
4264	if (pp->format == FORMAT_RAW)
4265	continue;
4266	if (pp->type == COUNTER_ITEMS) {
4267	if (average.packages.perf_counter[i] > 9999999)
4268	sums_need_wide_columns = 1;
4269	}
4270	average.packages.perf_counter[i] /= topo.allowed_packages;
4271	}
4272
4273	for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4274	average.threads.pmt_counter[i] /= topo.allowed_cpus;
4275	}
4276	for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
4277	average.cores.pmt_counter[i] /= topo.allowed_cores;
4278	}
4279	for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
4280	average.packages.pmt_counter[i] /= topo.allowed_packages;
4281	}
4282	}
4283
4284	static unsigned long long rdtsc(void)
4285	{
4286	unsigned int low, high;
4287
4288	asm volatile ("rdtsc":"=a" (low), "=d"(high));
4289
4290	return low | ((unsigned long long)high) << 32;
4291	}
4292
4293	/*
4294	* Open a file, and exit on failure
4295	*/
4296	FILE *fopen_or_die(const char *path, const char *mode)
4297	{
4298	FILE *filep = fopen(path, mode);
4299
4300	if (!filep)
4301	err(1, "%s: open failed", path);
4302	return filep;
4303	}
4304
4305	/*
4306	* snapshot_sysfs_counter()
4307	*
4308	* return snapshot of given counter
4309	*/
4310	unsigned long long snapshot_sysfs_counter(char *path)
4311	{
4312	FILE *fp;
4313	int retval;
4314	unsigned long long counter;
4315
4316	fp = fopen_or_die(path, "r");
4317
4318	retval = fscanf(fp, "%lld", &counter);
4319	if (retval != 1)
4320	err(1, "snapshot_sysfs_counter(%s)", path);
4321
4322	fclose(fp);
4323
4324	return counter;
4325	}
4326
4327	int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp, char *counter_path)
4328	{
4329	if (mp->msr_num != 0) {
4330	assert(!no_msr);
4331	if (get_msr(cpu, offset: mp->msr_num, msr: counterp))
4332	return -1;
4333	} else {
4334	char path[128 + PATH_BYTES];
4335
4336	if (mp->flags & SYSFS_PERCPU) {
4337	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/%s", cpu, mp->sp->path);
4338
4339	*counterp = snapshot_sysfs_counter(path);
4340	} else {
4341	*counterp = snapshot_sysfs_counter(path: counter_path);
4342	}
4343	}
4344
4345	return 0;
4346	}
4347
4348	unsigned long long get_legacy_uncore_mhz(int package)
4349	{
4350	char path[128];
4351	int die;
4352	static int warn_once;
4353
4354	/*
4355	* for this package, use the first die_id that exists
4356	*/
4357	for (die = 0; die <= topo.max_die_id; ++die) {
4358
4359	sprintf(buf: path, fmt: "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", package, die);
4360
4361	if (access(path, R_OK) == 0)
4362	return (snapshot_sysfs_counter(path) / 1000);
4363	}
4364	if (!warn_once) {
4365	warnx("BUG: %s: No %s", __func__, path);
4366	warn_once = 1;
4367	}
4368
4369	return 0;
4370	}
4371
4372	int get_epb(int cpu)
4373	{
4374	char path[128 + PATH_BYTES];
4375	unsigned long long msr;
4376	int ret, epb = -1;
4377	FILE *fp;
4378
4379	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/power/energy_perf_bias", cpu);
4380
4381	fp = fopen(path, "r");
4382	if (!fp)
4383	goto msr_fallback;
4384
4385	ret = fscanf(fp, "%d", &epb);
4386	if (ret != 1)
4387	err(1, "%s(%s)", __func__, path);
4388
4389	fclose(fp);
4390
4391	return epb;
4392
4393	msr_fallback:
4394	if (no_msr)
4395	return -1;
4396
4397	get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, msr: &msr);
4398
4399	return msr & 0xf;
4400	}
4401
4402	void get_apic_id(struct thread_data *t)
4403	{
4404	unsigned int eax, ebx, ecx, edx;
4405
4406	if (DO_BIC(BIC_APIC)) {
4407	eax = ebx = ecx = edx = 0;
4408	__cpuid(1, eax, ebx, ecx, edx);
4409
4410	t->apic_id = (ebx >> 24) & 0xff;
4411	}
4412
4413	if (!DO_BIC(BIC_X2APIC))
4414	return;
4415
4416	if (authentic_amd || hygon_genuine) {
4417	unsigned int topology_extensions;
4418
4419	if (max_extended_level < 0x8000001e)
4420	return;
4421
4422	eax = ebx = ecx = edx = 0;
4423	__cpuid(0x80000001, eax, ebx, ecx, edx);
4424	topology_extensions = ecx & (1 << 22);
4425
4426	if (topology_extensions == 0)
4427	return;
4428
4429	eax = ebx = ecx = edx = 0;
4430	__cpuid(0x8000001e, eax, ebx, ecx, edx);
4431
4432	t->x2apic_id = eax;
4433	return;
4434	}
4435
4436	if (!genuine_intel)
4437	return;
4438
4439	if (max_level < 0xb)
4440	return;
4441
4442	ecx = 0;
4443	__cpuid(0xb, eax, ebx, ecx, edx);
4444	t->x2apic_id = edx;
4445
4446	if (debug && (t->apic_id != (t->x2apic_id & 0xff)))
4447	fprintf(outf, "cpu%d: BIOS BUG: apic 0x%x x2apic 0x%x\n", t->cpu_id, t->apic_id, t->x2apic_id);
4448	}
4449
4450	int get_core_throt_cnt(int cpu, unsigned long long *cnt)
4451	{
4452	char path[128 + PATH_BYTES];
4453	unsigned long long tmp;
4454	FILE *fp;
4455	int ret;
4456
4457	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/thermal_throttle/core_throttle_count", cpu);
4458	fp = fopen(path, "r");
4459	if (!fp)
4460	return -1;
4461	ret = fscanf(fp, "%lld", &tmp);
4462	fclose(fp);
4463	if (ret != 1)
4464	return -1;
4465	*cnt = tmp;
4466
4467	return 0;
4468	}
4469
4470	static int read_perf_counter_info(const char *const path, const char *const parse_format, void *value_ptr)
4471	{
4472	int fdmt;
4473	int bytes_read;
4474	char buf[64];
4475	int ret = -1;
4476
4477	fdmt = open(path, O_RDONLY, 0);
4478	if (fdmt == -1) {
4479	if (debug)
4480	fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4481	ret = -1;
4482	goto cleanup_and_exit;
4483	}
4484
4485	bytes_read = read(fdmt, buf, sizeof(buf) - 1);
4486	if (bytes_read <= 0 || bytes_read >= (int)sizeof(buf)) {
4487	if (debug)
4488	fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4489	ret = -1;
4490	goto cleanup_and_exit;
4491	}
4492
4493	buf[bytes_read] = '\0';
4494
4495	if (sscanf(buf, parse_format, value_ptr) != 1) {
4496	if (debug)
4497	fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4498	ret = -1;
4499	goto cleanup_and_exit;
4500	}
4501
4502	ret = 0;
4503
4504	cleanup_and_exit:
4505	close(fdmt);
4506	return ret;
4507	}
4508
4509	static unsigned int read_perf_counter_info_n(const char *const path, const char *const parse_format)
4510	{
4511	unsigned int v;
4512	int status;
4513
4514	status = read_perf_counter_info(path, parse_format, value_ptr: &v);
4515	if (status)
4516	v = -1;
4517
4518	return v;
4519	}
4520
4521	static unsigned int read_perf_type(const char *subsys)
4522	{
4523	const char *const path_format = "/sys/bus/event_source/devices/%s/type";
4524	const char *const format = "%u";
4525	char path[128];
4526
4527	snprintf(buf: path, size: sizeof(path), fmt: path_format, subsys);
4528
4529	return read_perf_counter_info_n(path, parse_format: format);
4530	}
4531
4532	static unsigned int read_perf_config(const char *subsys, const char *event_name)
4533	{
4534	const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s";
4535	FILE *fconfig = NULL;
4536	char path[128];
4537	char config_str[64];
4538	unsigned int config;
4539	unsigned int umask;
4540	bool has_config = false;
4541	bool has_umask = false;
4542	unsigned int ret = -1;
4543
4544	snprintf(buf: path, size: sizeof(path), fmt: path_format, subsys, event_name);
4545
4546	fconfig = fopen(path, "r");
4547	if (!fconfig)
4548	return -1;
4549
4550	if (fgets(config_str, ARRAY_SIZE(config_str), fconfig) != config_str)
4551	goto cleanup_and_exit;
4552
4553	for (char *pconfig_str = &config_str[0]; pconfig_str;) {
4554	if (sscanf(pconfig_str, "event=%x", &config) == 1) {
4555	has_config = true;
4556	goto next;
4557	}
4558
4559	if (sscanf(pconfig_str, "umask=%x", &umask) == 1) {
4560	has_umask = true;
4561	goto next;
4562	}
4563
4564	next:
4565	pconfig_str = strchr(pconfig_str, ',');
4566	if (pconfig_str) {
4567	*pconfig_str = '\0';
4568	++pconfig_str;
4569	}
4570	}
4571
4572	if (!has_umask)
4573	umask = 0;
4574
4575	if (has_config)
4576	ret = (umask << 8) | config;
4577
4578	cleanup_and_exit:
4579	fclose(fconfig);
4580	return ret;
4581	}
4582
4583	static unsigned int read_perf_rapl_unit(const char *subsys, const char *event_name)
4584	{
4585	const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.unit";
4586	const char *const format = "%s";
4587	char path[128];
4588	char unit_buffer[16];
4589
4590	snprintf(buf: path, size: sizeof(path), fmt: path_format, subsys, event_name);
4591
4592	read_perf_counter_info(path, parse_format: format, value_ptr: &unit_buffer);
4593	if (strcmp("Joules", unit_buffer) == 0)
4594	return RAPL_UNIT_JOULES;
4595
4596	return RAPL_UNIT_INVALID;
4597	}
4598
4599	static double read_perf_scale(const char *subsys, const char *event_name)
4600	{
4601	const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.scale";
4602	const char *const format = "%lf";
4603	char path[128];
4604	double scale;
4605
4606	snprintf(buf: path, size: sizeof(path), fmt: path_format, subsys, event_name);
4607
4608	if (read_perf_counter_info(path, parse_format: format, value_ptr: &scale))
4609	return 0.0;
4610
4611	return scale;
4612	}
4613
4614	size_t rapl_counter_info_count_perf(const struct rapl_counter_info_t *rci)
4615	{
4616	size_t ret = 0;
4617
4618	for (int i = 0; i < NUM_RAPL_COUNTERS; ++i)
4619	if (rci->source[i] == COUNTER_SOURCE_PERF)
4620	++ret;
4621
4622	return ret;
4623	}
4624
4625	static size_t cstate_counter_info_count_perf(const struct cstate_counter_info_t *cci)
4626	{
4627	size_t ret = 0;
4628
4629	for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i)
4630	if (cci->source[i] == COUNTER_SOURCE_PERF)
4631	++ret;
4632
4633	return ret;
4634	}
4635
4636	void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci, unsigned int idx)
4637	{
4638	if (rci->source[idx] == COUNTER_SOURCE_NONE)
4639	return;
4640
4641	rc->raw_value = rci->data[idx];
4642	rc->unit = rci->unit[idx];
4643	rc->scale = rci->scale[idx];
4644	}
4645
4646	int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct pkg_data *p)
4647	{
4648	struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even;
4649	unsigned long long perf_data[NUM_RAPL_COUNTERS + 1];
4650	struct rapl_counter_info_t *rci;
4651
4652	if (debug >= 2)
4653	fprintf(stderr, "%s: cpu%d domain%d\n", __func__, cpu, domain);
4654
4655	assert(rapl_counter_info_perdomain);
4656	assert(domain < rapl_counter_info_perdomain_size);
4657
4658	rci = &rapl_counter_info_perdomain[domain];
4659
4660	/*
4661	* If we have any perf counters to read, read them all now, in bulk
4662	*/
4663	if (rci->fd_perf != -1) {
4664	size_t num_perf_counters = rapl_counter_info_count_perf(rci);
4665	const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
4666	const ssize_t actual_read_size = read(rci->fd_perf, &perf_data[0], sizeof(perf_data));
4667
4668	if (actual_read_size != expected_read_size)
4669	err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size);
4670	}
4671
4672	for (unsigned int i = 0, pi = 1; i < NUM_RAPL_COUNTERS; ++i) {
4673	switch (rci->source[i]) {
4674	case COUNTER_SOURCE_NONE:
4675	rci->data[i] = 0;
4676	break;
4677
4678	case COUNTER_SOURCE_PERF:
4679	assert(pi < ARRAY_SIZE(perf_data));
4680	assert(rci->fd_perf != -1);
4681
4682	if (debug >= 2)
4683	fprintf(stderr, "Reading rapl counter via perf at %u (%llu %e %lf)\n",
4684	i, perf_data[pi], rci->scale[i], perf_data[pi] * rci->scale[i]);
4685
4686	rci->data[i] = perf_data[pi];
4687
4688	++pi;
4689	break;
4690
4691	case COUNTER_SOURCE_MSR:
4692	if (debug >= 2)
4693	fprintf(stderr, "Reading rapl counter via msr at %u\n", i);
4694
4695	assert(!no_msr);
4696	if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM) {
4697	if (get_msr_sum(cpu, offset: rci->msr[i], msr: &rci->data[i]))
4698	return -13 - i;
4699	} else {
4700	if (get_msr(cpu, offset: rci->msr[i], msr: &rci->data[i]))
4701	return -13 - i;
4702	}
4703
4704	rci->data[i] &= rci->msr_mask[i];
4705	if (rci->msr_shift[i] >= 0)
4706	rci->data[i] >>= abs(rci->msr_shift[i]);
4707	else
4708	rci->data[i] <<= abs(rci->msr_shift[i]);
4709
4710	break;
4711	}
4712	}
4713
4714	BUILD_BUG_ON(NUM_RAPL_COUNTERS != 8);
4715	write_rapl_counter(rc: &p->energy_pkg, rci, idx: RAPL_RCI_INDEX_ENERGY_PKG);
4716	write_rapl_counter(rc: &p->energy_cores, rci, idx: RAPL_RCI_INDEX_ENERGY_CORES);
4717	write_rapl_counter(rc: &p->energy_dram, rci, idx: RAPL_RCI_INDEX_DRAM);
4718	write_rapl_counter(rc: &p->energy_gfx, rci, idx: RAPL_RCI_INDEX_GFX);
4719	write_rapl_counter(rc: &p->rapl_pkg_perf_status, rci, idx: RAPL_RCI_INDEX_PKG_PERF_STATUS);
4720	write_rapl_counter(rc: &p->rapl_dram_perf_status, rci, idx: RAPL_RCI_INDEX_DRAM_PERF_STATUS);
4721	write_rapl_counter(rc: &c->core_energy, rci, idx: RAPL_RCI_INDEX_CORE_ENERGY);
4722	write_rapl_counter(rc: &pplat_cnt->energy_psys, rci, idx: RAPL_RCI_INDEX_ENERGY_PLATFORM);
4723
4724	return 0;
4725	}
4726
4727	char *find_sysfs_path_by_id(struct sysfs_path *sp, int id)
4728	{
4729	while (sp) {
4730	if (sp->id == id)
4731	return (sp->path);
4732	sp = sp->next;
4733	}
4734	if (debug)
4735	warnx("%s: id%d not found", __func__, id);
4736	return NULL;
4737	}
4738
4739	int get_cstate_counters(unsigned int cpu, PER_THREAD_PARAMS)
4740	{
4741	/*
4742	* Overcommit memory a little bit here,
4743	* but skip calculating exact sizes for the buffers.
4744	*/
4745	unsigned long long perf_data[NUM_CSTATE_COUNTERS];
4746	unsigned long long perf_data_core[NUM_CSTATE_COUNTERS + 1];
4747	unsigned long long perf_data_pkg[NUM_CSTATE_COUNTERS + 1];
4748
4749	struct cstate_counter_info_t *cci;
4750
4751	if (debug >= 2)
4752	fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
4753
4754	assert(ccstate_counter_info);
4755	assert(cpu <= ccstate_counter_info_size);
4756
4757	ZERO_ARRAY(perf_data);
4758	ZERO_ARRAY(perf_data_core);
4759	ZERO_ARRAY(perf_data_pkg);
4760
4761	cci = &ccstate_counter_info[cpu];
4762
4763	/*
4764	* If we have any perf counters to read, read them all now, in bulk
4765	*/
4766	const size_t num_perf_counters = cstate_counter_info_count_perf(cci);
4767	ssize_t expected_read_size = num_perf_counters * sizeof(unsigned long long);
4768	ssize_t actual_read_size_core = 0, actual_read_size_pkg = 0;
4769
4770	if (cci->fd_perf_core != -1) {
4771	/* Each descriptor read begins with number of counters read. */
4772	expected_read_size += sizeof(unsigned long long);
4773
4774	actual_read_size_core = read(cci->fd_perf_core, &perf_data_core[0], sizeof(perf_data_core));
4775
4776	if (actual_read_size_core <= 0)
4777	err(-1, "%s: read perf %s: %ld", __func__, "core", actual_read_size_core);
4778	}
4779
4780	if (cci->fd_perf_pkg != -1) {
4781	/* Each descriptor read begins with number of counters read. */
4782	expected_read_size += sizeof(unsigned long long);
4783
4784	actual_read_size_pkg = read(cci->fd_perf_pkg, &perf_data_pkg[0], sizeof(perf_data_pkg));
4785
4786	if (actual_read_size_pkg <= 0)
4787	err(-1, "%s: read perf %s: %ld", __func__, "pkg", actual_read_size_pkg);
4788	}
4789
4790	const ssize_t actual_read_size_total = actual_read_size_core + actual_read_size_pkg;
4791
4792	if (actual_read_size_total != expected_read_size)
4793	err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size_total);
4794
4795	/*
4796	* Copy ccstate and pcstate data into unified buffer.
4797	*
4798	* Skip first element from core and pkg buffers.
4799	* Kernel puts there how many counters were read.
4800	*/
4801	const size_t num_core_counters = perf_data_core[0];
4802	const size_t num_pkg_counters = perf_data_pkg[0];
4803
4804	assert(num_perf_counters == num_core_counters + num_pkg_counters);
4805
4806	/* Copy ccstate perf data */
4807	memcpy(&perf_data[0], &perf_data_core[1], num_core_counters * sizeof(unsigned long long));
4808
4809	/* Copy pcstate perf data */
4810	memcpy(&perf_data[num_core_counters], &perf_data_pkg[1], num_pkg_counters * sizeof(unsigned long long));
4811
4812	for (unsigned int i = 0, pi = 0; i < NUM_CSTATE_COUNTERS; ++i) {
4813	switch (cci->source[i]) {
4814	case COUNTER_SOURCE_NONE:
4815	break;
4816
4817	case COUNTER_SOURCE_PERF:
4818	assert(pi < ARRAY_SIZE(perf_data));
4819	assert(cci->fd_perf_core != -1 || cci->fd_perf_pkg != -1);
4820
4821	if (debug >= 2)
4822	fprintf(stderr, "cstate via %s %u: %llu\n", "perf", i, perf_data[pi]);
4823
4824	cci->data[i] = perf_data[pi];
4825
4826	++pi;
4827	break;
4828
4829	case COUNTER_SOURCE_MSR:
4830	assert(!no_msr);
4831	if (get_msr(cpu, offset: cci->msr[i], msr: &cci->data[i]))
4832	return -13 - i;
4833
4834	if (debug >= 2)
4835	fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", "msr", cci->msr[i], i, cci->data[i]);
4836
4837	break;
4838	}
4839	}
4840
4841	/*
4842	* Helper to write the data only if the source of
4843	* the counter for the current cpu is not none.
4844	*
4845	* Otherwise we would overwrite core data with 0 (default value),
4846	* when invoked for the thread sibling.
4847	*/
4848	#define PERF_COUNTER_WRITE_DATA(out_counter, index) do { \
4849	if (cci->source[index] != COUNTER_SOURCE_NONE) \
4850	out_counter = cci->data[index]; \
4851	} while (0)
4852
4853	BUILD_BUG_ON(NUM_CSTATE_COUNTERS != 11);
4854
4855	PERF_COUNTER_WRITE_DATA(t->c1, CCSTATE_RCI_INDEX_C1_RESIDENCY);
4856	PERF_COUNTER_WRITE_DATA(c->c3, CCSTATE_RCI_INDEX_C3_RESIDENCY);
4857	PERF_COUNTER_WRITE_DATA(c->c6, CCSTATE_RCI_INDEX_C6_RESIDENCY);
4858	PERF_COUNTER_WRITE_DATA(c->c7, CCSTATE_RCI_INDEX_C7_RESIDENCY);
4859
4860	PERF_COUNTER_WRITE_DATA(p->pc2, PCSTATE_RCI_INDEX_C2_RESIDENCY);
4861	PERF_COUNTER_WRITE_DATA(p->pc3, PCSTATE_RCI_INDEX_C3_RESIDENCY);
4862	PERF_COUNTER_WRITE_DATA(p->pc6, PCSTATE_RCI_INDEX_C6_RESIDENCY);
4863	PERF_COUNTER_WRITE_DATA(p->pc7, PCSTATE_RCI_INDEX_C7_RESIDENCY);
4864	PERF_COUNTER_WRITE_DATA(p->pc8, PCSTATE_RCI_INDEX_C8_RESIDENCY);
4865	PERF_COUNTER_WRITE_DATA(p->pc9, PCSTATE_RCI_INDEX_C9_RESIDENCY);
4866	PERF_COUNTER_WRITE_DATA(p->pc10, PCSTATE_RCI_INDEX_C10_RESIDENCY);
4867
4868	#undef PERF_COUNTER_WRITE_DATA
4869
4870	return 0;
4871	}
4872
4873	size_t msr_counter_info_count_perf(const struct msr_counter_info_t *mci)
4874	{
4875	size_t ret = 0;
4876
4877	for (int i = 0; i < NUM_MSR_COUNTERS; ++i)
4878	if (mci->source[i] == COUNTER_SOURCE_PERF)
4879	++ret;
4880
4881	return ret;
4882	}
4883
4884	int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t)
4885	{
4886	unsigned long long perf_data[NUM_MSR_COUNTERS + 1];
4887
4888	struct msr_counter_info_t *mci;
4889
4890	if (debug >= 2)
4891	fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
4892
4893	assert(msr_counter_info);
4894	assert(cpu <= msr_counter_info_size);
4895
4896	mci = &msr_counter_info[cpu];
4897
4898	ZERO_ARRAY(perf_data);
4899	ZERO_ARRAY(mci->data);
4900
4901	if (mci->fd_perf != -1) {
4902	const size_t num_perf_counters = msr_counter_info_count_perf(mci);
4903	const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
4904	const ssize_t actual_read_size = read(mci->fd_perf, &perf_data[0], sizeof(perf_data));
4905
4906	if (actual_read_size != expected_read_size)
4907	err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size);
4908	}
4909
4910	for (unsigned int i = 0, pi = 1; i < NUM_MSR_COUNTERS; ++i) {
4911	switch (mci->source[i]) {
4912	case COUNTER_SOURCE_NONE:
4913	break;
4914
4915	case COUNTER_SOURCE_PERF:
4916	assert(pi < ARRAY_SIZE(perf_data));
4917	assert(mci->fd_perf != -1);
4918
4919	if (debug >= 2)
4920	fprintf(stderr, "Reading msr counter via perf at %u: %llu\n", i, perf_data[pi]);
4921
4922	mci->data[i] = perf_data[pi];
4923
4924	++pi;
4925	break;
4926
4927	case COUNTER_SOURCE_MSR:
4928	assert(!no_msr);
4929
4930	if (get_msr(cpu, offset: mci->msr[i], msr: &mci->data[i]))
4931	return -2 - i;
4932
4933	mci->data[i] &= mci->msr_mask[i];
4934
4935	if (debug >= 2)
4936	fprintf(stderr, "Reading msr counter via msr at %u: %llu\n", i, mci->data[i]);
4937
4938	break;
4939	}
4940	}
4941
4942	BUILD_BUG_ON(NUM_MSR_COUNTERS != 3);
4943	t->aperf = mci->data[MSR_RCI_INDEX_APERF];
4944	t->mperf = mci->data[MSR_RCI_INDEX_MPERF];
4945	t->smi_count = mci->data[MSR_RCI_INDEX_SMI];
4946
4947	return 0;
4948	}
4949
4950	int perf_counter_info_read_values(struct perf_counter_info *pp, int cpu, unsigned long long *out, size_t out_size)
4951	{
4952	unsigned int domain;
4953	unsigned long long value;
4954	int fd_counter;
4955
4956	for (size_t i = 0; pp; ++i, pp = pp->next) {
4957	domain = cpu_to_domain(pc: pp, cpu);
4958	assert(domain < pp->num_domains);
4959
4960	fd_counter = pp->fd_perf_per_domain[domain];
4961
4962	if (fd_counter == -1)
4963	continue;
4964
4965	if (read(fd_counter, &value, sizeof(value)) != sizeof(value))
4966	return 1;
4967
4968	assert(i < out_size);
4969	out[i] = value * pp->scale;
4970	}
4971
4972	return 0;
4973	}
4974
4975	unsigned long pmt_gen_value_mask(unsigned int lsb, unsigned int msb)
4976	{
4977	unsigned long mask;
4978
4979	if (msb == 63)
4980	mask = 0xffffffffffffffff;
4981	else
4982	mask = ((1 << (msb + 1)) - 1);
4983
4984	mask -= (1 << lsb) - 1;
4985
4986	return mask;
4987	}
4988
4989	unsigned long pmt_read_counter(struct pmt_counter *ppmt, unsigned int domain_id)
4990	{
4991	if (domain_id >= ppmt->num_domains)
4992	return 0;
4993
4994	const unsigned long *pmmio = ppmt->domains[domain_id].pcounter;
4995	const unsigned long value = pmmio ? *pmmio : 0;
4996	const unsigned long value_mask = pmt_gen_value_mask(lsb: ppmt->lsb, msb: ppmt->msb);
4997	const unsigned long value_shift = ppmt->lsb;
4998
4999	return (value & value_mask) >> value_shift;
5000	}
5001
5002	/* Rapl domain enumeration helpers */
5003	static inline int get_rapl_num_domains(void)
5004	{
5005	int num_packages = topo.max_package_id + 1;
5006	int num_cores_per_package;
5007	int num_cores;
5008
5009	if (!platform->has_per_core_rapl)
5010	return num_packages;
5011
5012	num_cores_per_package = topo.max_core_id + 1;
5013	num_cores = num_cores_per_package * num_packages;
5014
5015	return num_cores;
5016	}
5017
5018	static inline int get_rapl_domain_id(int cpu)
5019	{
5020	int nr_cores_per_package = topo.max_core_id + 1;
5021	int rapl_core_id;
5022
5023	if (!platform->has_per_core_rapl)
5024	return cpus[cpu].physical_package_id;
5025
5026	/* Compute the system-wide unique core-id for @cpu */
5027	rapl_core_id = cpus[cpu].physical_core_id;
5028	rapl_core_id += cpus[cpu].physical_package_id * nr_cores_per_package;
5029
5030	return rapl_core_id;
5031	}
5032
5033	/*
5034	* get_counters(...)
5035	* migrate to cpu
5036	* acquire and record local counters for that cpu
5037	*/
5038	int get_counters(PER_THREAD_PARAMS)
5039	{
5040	int cpu = t->cpu_id;
5041	unsigned long long msr;
5042	struct msr_counter *mp;
5043	struct pmt_counter *pp;
5044	int i;
5045	int status;
5046
5047	if (cpu_migrate(cpu)) {
5048	fprintf(outf, "%s: Could not migrate to CPU %d\n", __func__, cpu);
5049	return -1;
5050	}
5051
5052	gettimeofday(&t->tv_begin, (struct timezone *)NULL);
5053
5054	if (first_counter_read)
5055	get_apic_id(t);
5056
5057	t->tsc = rdtsc(); /* we are running on local CPU of interest */
5058
5059	get_smi_aperf_mperf(cpu, t);
5060
5061	if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT))
5062	get_perf_llc_stats(cpu, llc: &t->llc);
5063
5064	if (DO_BIC(BIC_IPC))
5065	if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long))
5066	return -4;
5067
5068	if (DO_BIC(BIC_IRQ))
5069	t->irq_count = irqs_per_cpu[cpu];
5070	if (DO_BIC(BIC_NMI))
5071	t->nmi_count = nmi_per_cpu[cpu];
5072
5073	get_cstate_counters(cpu, t, c, p);
5074
5075	for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
5076	if (get_mp(cpu, mp, counterp: &t->counter[i], counter_path: mp->sp->path))
5077	return -10;
5078	}
5079
5080	if (perf_counter_info_read_values(pp: sys.perf_tp, cpu, out: t->perf_counter, MAX_ADDED_THREAD_COUNTERS))
5081	return -10;
5082
5083	for (i = 0, pp = sys.pmt_tp; pp; i++, pp = pp->next)
5084	t->pmt_counter[i] = pmt_read_counter(ppmt: pp, domain_id: t->cpu_id);
5085
5086	/* collect core counters only for 1st thread in core */
5087	if (!is_cpu_first_thread_in_core(t, c))
5088	goto done;
5089
5090	if (platform->has_per_core_rapl) {
5091	status = get_rapl_counters(cpu, domain: get_rapl_domain_id(cpu), c, p);
5092	if (status != 0)
5093	return status;
5094	}
5095
5096	if (DO_BIC(BIC_CPU_c7) && t->is_atom) {
5097	/*
5098	* For Atom CPUs that has core cstate deeper than c6,
5099	* MSR_CORE_C6_RESIDENCY returns residency of cc6 and deeper.
5100	* Minus CC7 (and deeper cstates) residency to get
5101	* accturate cc6 residency.
5102	*/
5103	c->c6 -= c->c7;
5104	}
5105
5106	if (DO_BIC(BIC_Mod_c6))
5107	if (get_msr(cpu, MSR_MODULE_C6_RES_MS, msr: &c->mc6_us))
5108	return -8;
5109
5110	if (DO_BIC(BIC_CoreTmp)) {
5111	if (get_msr(cpu, MSR_IA32_THERM_STATUS, msr: &msr))
5112	return -9;
5113	c->core_temp_c = tj_max - ((msr >> 16) & 0x7F);
5114	}
5115
5116	if (DO_BIC(BIC_CORE_THROT_CNT))
5117	get_core_throt_cnt(cpu, cnt: &c->core_throt_cnt);
5118
5119	for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
5120	if (get_mp(cpu, mp, counterp: &c->counter[i], counter_path: mp->sp->path))
5121	return -10;
5122	}
5123
5124	if (perf_counter_info_read_values(pp: sys.perf_cp, cpu, out: c->perf_counter, MAX_ADDED_CORE_COUNTERS))
5125	return -10;
5126
5127	for (i = 0, pp = sys.pmt_cp; pp; i++, pp = pp->next)
5128	c->pmt_counter[i] = pmt_read_counter(ppmt: pp, domain_id: c->core_id);
5129
5130	/* collect package counters only for 1st core in package */
5131	if (!is_cpu_first_core_in_package(t, p))
5132	goto done;
5133
5134	if (DO_BIC(BIC_Totl_c0)) {
5135	if (get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, msr: &p->pkg_wtd_core_c0))
5136	return -10;
5137	}
5138	if (DO_BIC(BIC_Any_c0)) {
5139	if (get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, msr: &p->pkg_any_core_c0))
5140	return -11;
5141	}
5142	if (DO_BIC(BIC_GFX_c0)) {
5143	if (get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, msr: &p->pkg_any_gfxe_c0))
5144	return -12;
5145	}
5146	if (DO_BIC(BIC_CPUGFX)) {
5147	if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, msr: &p->pkg_both_core_gfxe_c0))
5148	return -13;
5149	}
5150
5151	if (DO_BIC(BIC_CPU_LPI))
5152	p->cpu_lpi = cpuidle_cur_cpu_lpi_us;
5153	if (DO_BIC(BIC_SYS_LPI))
5154	p->sys_lpi = cpuidle_cur_sys_lpi_us;
5155
5156	if (!platform->has_per_core_rapl) {
5157	status = get_rapl_counters(cpu, domain: get_rapl_domain_id(cpu), c, p);
5158	if (status != 0)
5159	return status;
5160	}
5161
5162	if (DO_BIC(BIC_PkgTmp)) {
5163	if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, msr: &msr))
5164	return -17;
5165	p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F);
5166	}
5167
5168	if (DO_BIC(BIC_UNCORE_MHZ))
5169	p->uncore_mhz = get_legacy_uncore_mhz(package: p->package_id);
5170
5171	if (DO_BIC(BIC_GFX_rc6))
5172	p->gfx_rc6_ms = gfx_info[GFX_rc6].val_ull;
5173
5174	if (DO_BIC(BIC_GFXMHz))
5175	p->gfx_mhz = gfx_info[GFX_MHz].val;
5176
5177	if (DO_BIC(BIC_GFXACTMHz))
5178	p->gfx_act_mhz = gfx_info[GFX_ACTMHz].val;
5179
5180	if (DO_BIC(BIC_SAM_mc6))
5181	p->sam_mc6_ms = gfx_info[SAM_mc6].val_ull;
5182
5183	if (DO_BIC(BIC_SAMMHz))
5184	p->sam_mhz = gfx_info[SAM_MHz].val;
5185
5186	if (DO_BIC(BIC_SAMACTMHz))
5187	p->sam_act_mhz = gfx_info[SAM_ACTMHz].val;
5188
5189	for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
5190	char *path = NULL;
5191
5192	if (mp->msr_num == 0) {
5193	path = find_sysfs_path_by_id(sp: mp->sp, id: p->package_id);
5194	if (path == NULL) {
5195	warnx("%s: package_id %d not found", __func__, p->package_id);
5196	return -10;
5197	}
5198	}
5199	if (get_mp(cpu, mp, counterp: &p->counter[i], counter_path: path))
5200	return -10;
5201	}
5202
5203	if (perf_counter_info_read_values(pp: sys.perf_pp, cpu, out: p->perf_counter, MAX_ADDED_PACKAGE_COUNTERS))
5204	return -10;
5205
5206	for (i = 0, pp = sys.pmt_pp; pp; i++, pp = pp->next)
5207	p->pmt_counter[i] = pmt_read_counter(ppmt: pp, domain_id: p->package_id);
5208
5209	done:
5210	gettimeofday(&t->tv_end, (struct timezone *)NULL);
5211
5212	return 0;
5213	}
5214
5215	int pkg_cstate_limit = PCLUKN;
5216	char *pkg_cstate_limit_strings[] = { "unknown", "reserved", "pc0", "pc1", "pc2",
5217	"pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited"
5218	};
5219
5220	int nhm_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5221	PCLRSV, PCLRSV
5222	};
5223
5224	int snb_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5225	PCLRSV, PCLRSV
5226	};
5227
5228	int hsw_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5229	PCLRSV, PCLRSV
5230	};
5231
5232	int slv_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5233	PCL__6, PCL__7
5234	};
5235
5236	int amt_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5237	PCLRSV, PCLRSV
5238	};
5239
5240	int phi_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5241	PCLRSV, PCLRSV
5242	};
5243
5244	int glm_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5245	PCLRSV, PCLRSV
5246	};
5247
5248	int skx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5249	PCLRSV, PCLRSV
5250	};
5251
5252	int icx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5253	PCLRSV, PCLRSV
5254	};
5255
5256	void probe_cst_limit(void)
5257	{
5258	unsigned long long msr;
5259	int *pkg_cstate_limits;
5260
5261	if (!platform->has_nhm_msrs || no_msr)
5262	return;
5263
5264	switch (platform->cst_limit) {
5265	case CST_LIMIT_NHM:
5266	pkg_cstate_limits = nhm_pkg_cstate_limits;
5267	break;
5268	case CST_LIMIT_SNB:
5269	pkg_cstate_limits = snb_pkg_cstate_limits;
5270	break;
5271	case CST_LIMIT_HSW:
5272	pkg_cstate_limits = hsw_pkg_cstate_limits;
5273	break;
5274	case CST_LIMIT_SKX:
5275	pkg_cstate_limits = skx_pkg_cstate_limits;
5276	break;
5277	case CST_LIMIT_ICX:
5278	pkg_cstate_limits = icx_pkg_cstate_limits;
5279	break;
5280	case CST_LIMIT_SLV:
5281	pkg_cstate_limits = slv_pkg_cstate_limits;
5282	break;
5283	case CST_LIMIT_AMT:
5284	pkg_cstate_limits = amt_pkg_cstate_limits;
5285	break;
5286	case CST_LIMIT_KNL:
5287	pkg_cstate_limits = phi_pkg_cstate_limits;
5288	break;
5289	case CST_LIMIT_GMT:
5290	pkg_cstate_limits = glm_pkg_cstate_limits;
5291	break;
5292	default:
5293	return;
5294	}
5295
5296	get_msr(cpu: base_cpu, MSR_PKG_CST_CONFIG_CONTROL, msr: &msr);
5297	pkg_cstate_limit = pkg_cstate_limits[msr & 0xF];
5298	}
5299
5300	static void dump_platform_info(void)
5301	{
5302	unsigned long long msr;
5303	unsigned int ratio;
5304
5305	if (!platform->has_nhm_msrs || no_msr)
5306	return;
5307
5308	get_msr(cpu: base_cpu, MSR_PLATFORM_INFO, msr: &msr);
5309
5310	fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", base_cpu, msr);
5311
5312	ratio = (msr >> 40) & 0xFF;
5313	fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n", ratio, bclk, ratio * bclk);
5314
5315	ratio = (msr >> 8) & 0xFF;
5316	fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
5317	}
5318
5319	static void dump_power_ctl(void)
5320	{
5321	unsigned long long msr;
5322
5323	if (!platform->has_nhm_msrs || no_msr)
5324	return;
5325
5326	get_msr(cpu: base_cpu, MSR_IA32_POWER_CTL, msr: &msr);
5327	fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", base_cpu, msr, msr & 0x2 ? "EN" : "DIS");
5328
5329	/* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */
5330	if (platform->has_cst_prewake_bit)
5331	fprintf(outf, "C-state Pre-wake: %sabled\n", msr & 0x40000000 ? "DIS" : "EN");
5332
5333	return;
5334	}
5335
5336	static void dump_turbo_ratio_limit2(void)
5337	{
5338	unsigned long long msr;
5339	unsigned int ratio;
5340
5341	get_msr(cpu: base_cpu, MSR_TURBO_RATIO_LIMIT2, msr: &msr);
5342
5343	fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", base_cpu, msr);
5344
5345	ratio = (msr >> 8) & 0xFF;
5346	if (ratio)
5347	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 18 active cores\n", ratio, bclk, ratio * bclk);
5348
5349	ratio = (msr >> 0) & 0xFF;
5350	if (ratio)
5351	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 17 active cores\n", ratio, bclk, ratio * bclk);
5352	return;
5353	}
5354
5355	static void dump_turbo_ratio_limit1(void)
5356	{
5357	unsigned long long msr;
5358	unsigned int ratio;
5359
5360	get_msr(cpu: base_cpu, MSR_TURBO_RATIO_LIMIT1, msr: &msr);
5361
5362	fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, msr);
5363
5364	ratio = (msr >> 56) & 0xFF;
5365	if (ratio)
5366	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 16 active cores\n", ratio, bclk, ratio * bclk);
5367
5368	ratio = (msr >> 48) & 0xFF;
5369	if (ratio)
5370	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 15 active cores\n", ratio, bclk, ratio * bclk);
5371
5372	ratio = (msr >> 40) & 0xFF;
5373	if (ratio)
5374	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 14 active cores\n", ratio, bclk, ratio * bclk);
5375
5376	ratio = (msr >> 32) & 0xFF;
5377	if (ratio)
5378	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 13 active cores\n", ratio, bclk, ratio * bclk);
5379
5380	ratio = (msr >> 24) & 0xFF;
5381	if (ratio)
5382	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 12 active cores\n", ratio, bclk, ratio * bclk);
5383
5384	ratio = (msr >> 16) & 0xFF;
5385	if (ratio)
5386	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 11 active cores\n", ratio, bclk, ratio * bclk);
5387
5388	ratio = (msr >> 8) & 0xFF;
5389	if (ratio)
5390	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 10 active cores\n", ratio, bclk, ratio * bclk);
5391
5392	ratio = (msr >> 0) & 0xFF;
5393	if (ratio)
5394	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 9 active cores\n", ratio, bclk, ratio * bclk);
5395	return;
5396	}
5397
5398	static void dump_turbo_ratio_limits(int trl_msr_offset)
5399	{
5400	unsigned long long msr, core_counts;
5401	int shift;
5402
5403	get_msr(cpu: base_cpu, offset: trl_msr_offset, msr: &msr);
5404	fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n",
5405	base_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr);
5406
5407	if (platform->trl_msrs & TRL_CORECOUNT) {
5408	get_msr(cpu: base_cpu, MSR_TURBO_RATIO_LIMIT1, msr: &core_counts);
5409	fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, core_counts);
5410	} else {
5411	core_counts = 0x0807060504030201;
5412	}
5413
5414	for (shift = 56; shift >= 0; shift -= 8) {
5415	unsigned int ratio, group_size;
5416
5417	ratio = (msr >> shift) & 0xFF;
5418	group_size = (core_counts >> shift) & 0xFF;
5419	if (ratio)
5420	fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio, bclk, ratio * bclk, group_size);
5421	}
5422
5423	return;
5424	}
5425
5426	static void dump_atom_turbo_ratio_limits(void)
5427	{
5428	unsigned long long msr;
5429	unsigned int ratio;
5430
5431	get_msr(cpu: base_cpu, MSR_ATOM_CORE_RATIOS, msr: &msr);
5432	fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF);
5433
5434	ratio = (msr >> 0) & 0x3F;
5435	if (ratio)
5436	fprintf(outf, "%d * %.1f = %.1f MHz minimum operating frequency\n", ratio, bclk, ratio * bclk);
5437
5438	ratio = (msr >> 8) & 0x3F;
5439	if (ratio)
5440	fprintf(outf, "%d * %.1f = %.1f MHz low frequency mode (LFM)\n", ratio, bclk, ratio * bclk);
5441
5442	ratio = (msr >> 16) & 0x3F;
5443	if (ratio)
5444	fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
5445
5446	get_msr(cpu: base_cpu, MSR_ATOM_CORE_TURBO_RATIOS, msr: &msr);
5447	fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF);
5448
5449	ratio = (msr >> 24) & 0x3F;
5450	if (ratio)
5451	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 4 active cores\n", ratio, bclk, ratio * bclk);
5452
5453	ratio = (msr >> 16) & 0x3F;
5454	if (ratio)
5455	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 3 active cores\n", ratio, bclk, ratio * bclk);
5456
5457	ratio = (msr >> 8) & 0x3F;
5458	if (ratio)
5459	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 2 active cores\n", ratio, bclk, ratio * bclk);
5460
5461	ratio = (msr >> 0) & 0x3F;
5462	if (ratio)
5463	fprintf(outf, "%d * %.1f = %.1f MHz max turbo 1 active core\n", ratio, bclk, ratio * bclk);
5464	}
5465
5466	static void dump_knl_turbo_ratio_limits(void)
5467	{
5468	const unsigned int buckets_no = 7;
5469
5470	unsigned long long msr;
5471	int delta_cores, delta_ratio;
5472	int i, b_nr;
5473	unsigned int cores[buckets_no];
5474	unsigned int ratio[buckets_no];
5475
5476	get_msr(cpu: base_cpu, MSR_TURBO_RATIO_LIMIT, msr: &msr);
5477
5478	fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", base_cpu, msr);
5479
5480	/*
5481	* Turbo encoding in KNL is as follows:
5482	* [0] -- Reserved
5483	* [7:1] -- Base value of number of active cores of bucket 1.
5484	* [15:8] -- Base value of freq ratio of bucket 1.
5485	* [20:16] -- +ve delta of number of active cores of bucket 2.
5486	* i.e. active cores of bucket 2 =
5487	* active cores of bucket 1 + delta
5488	* [23:21] -- Negative delta of freq ratio of bucket 2.
5489	* i.e. freq ratio of bucket 2 =
5490	* freq ratio of bucket 1 - delta
5491	* [28:24]-- +ve delta of number of active cores of bucket 3.
5492	* [31:29]-- -ve delta of freq ratio of bucket 3.
5493	* [36:32]-- +ve delta of number of active cores of bucket 4.
5494	* [39:37]-- -ve delta of freq ratio of bucket 4.
5495	* [44:40]-- +ve delta of number of active cores of bucket 5.
5496	* [47:45]-- -ve delta of freq ratio of bucket 5.
5497	* [52:48]-- +ve delta of number of active cores of bucket 6.
5498	* [55:53]-- -ve delta of freq ratio of bucket 6.
5499	* [60:56]-- +ve delta of number of active cores of bucket 7.
5500	* [63:61]-- -ve delta of freq ratio of bucket 7.
5501	*/
5502
5503	b_nr = 0;
5504	cores[b_nr] = (msr & 0xFF) >> 1;
5505	ratio[b_nr] = (msr >> 8) & 0xFF;
5506
5507	for (i = 16; i < 64; i += 8) {
5508	delta_cores = (msr >> i) & 0x1F;
5509	delta_ratio = (msr >> (i + 5)) & 0x7;
5510
5511	cores[b_nr + 1] = cores[b_nr] + delta_cores;
5512	ratio[b_nr + 1] = ratio[b_nr] - delta_ratio;
5513	b_nr++;
5514	}
5515
5516	for (i = buckets_no - 1; i >= 0; i--)
5517	if (i > 0 ? ratio[i] != ratio[i - 1] : 1)
5518	fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio[i], bclk, ratio[i] * bclk, cores[i]);
5519	}
5520
5521	static void dump_cst_cfg(void)
5522	{
5523	unsigned long long msr;
5524
5525	if (!platform->has_nhm_msrs || no_msr)
5526	return;
5527
5528	get_msr(cpu: base_cpu, MSR_PKG_CST_CONFIG_CONTROL, msr: &msr);
5529
5530	fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", base_cpu, msr);
5531
5532	fprintf(outf, " (%s%s%s%s%slocked, pkg-cstate-limit=%d (%s)",
5533	(msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "",
5534	(msr & SNB_C1_AUTO_UNDEMOTE) ? "UNdemote-C1, " : "",
5535	(msr & NHM_C3_AUTO_DEMOTE) ? "demote-C3, " : "",
5536	(msr & NHM_C1_AUTO_DEMOTE) ? "demote-C1, " : "",
5537	(msr & (1 << 15)) ? "" : "UN", (unsigned int)msr & 0xF, pkg_cstate_limit_strings[pkg_cstate_limit]);
5538
5539	#define AUTOMATIC_CSTATE_CONVERSION (1UL << 16)
5540	if (platform->has_cst_auto_convension) {
5541	fprintf(outf, ", automatic c-state conversion=%s", (msr & AUTOMATIC_CSTATE_CONVERSION) ? "on" : "off");
5542	}
5543
5544	fprintf(outf, ")\n");
5545
5546	return;
5547	}
5548
5549	static void dump_config_tdp(void)
5550	{
5551	unsigned long long msr;
5552
5553	get_msr(cpu: base_cpu, MSR_CONFIG_TDP_NOMINAL, msr: &msr);
5554	fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", base_cpu, msr);
5555	fprintf(outf, " (base_ratio=%d)\n", (unsigned int)msr & 0xFF);
5556
5557	get_msr(cpu: base_cpu, MSR_CONFIG_TDP_LEVEL_1, msr: &msr);
5558	fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", base_cpu, msr);
5559	if (msr) {
5560	fprintf(outf, "PKG_MIN_PWR_LVL1=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5561	fprintf(outf, "PKG_MAX_PWR_LVL1=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5562	fprintf(outf, "LVL1_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5563	fprintf(outf, "PKG_TDP_LVL1=%d", (unsigned int)(msr) & 0x7FFF);
5564	}
5565	fprintf(outf, ")\n");
5566
5567	get_msr(cpu: base_cpu, MSR_CONFIG_TDP_LEVEL_2, msr: &msr);
5568	fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", base_cpu, msr);
5569	if (msr) {
5570	fprintf(outf, "PKG_MIN_PWR_LVL2=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5571	fprintf(outf, "PKG_MAX_PWR_LVL2=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5572	fprintf(outf, "LVL2_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5573	fprintf(outf, "PKG_TDP_LVL2=%d", (unsigned int)(msr) & 0x7FFF);
5574	}
5575	fprintf(outf, ")\n");
5576
5577	get_msr(cpu: base_cpu, MSR_CONFIG_TDP_CONTROL, msr: &msr);
5578	fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", base_cpu, msr);
5579	if ((msr) & 0x3)
5580	fprintf(outf, "TDP_LEVEL=%d ", (unsigned int)(msr) & 0x3);
5581	fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5582	fprintf(outf, ")\n");
5583
5584	get_msr(cpu: base_cpu, MSR_TURBO_ACTIVATION_RATIO, msr: &msr);
5585	fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", base_cpu, msr);
5586	fprintf(outf, "MAX_NON_TURBO_RATIO=%d", (unsigned int)(msr) & 0xFF);
5587	fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5588	fprintf(outf, ")\n");
5589	}
5590
5591	unsigned int irtl_time_units[] = { 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };
5592
5593	void print_irtl(void)
5594	{
5595	unsigned long long msr;
5596
5597	if (!platform->has_irtl_msrs || no_msr)
5598	return;
5599
5600	if (platform->supported_cstates & PC3) {
5601	get_msr(cpu: base_cpu, MSR_PKGC3_IRTL, msr: &msr);
5602	fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", base_cpu, msr);
5603	fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5604	}
5605
5606	if (platform->supported_cstates & PC6) {
5607	get_msr(cpu: base_cpu, MSR_PKGC6_IRTL, msr: &msr);
5608	fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", base_cpu, msr);
5609	fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5610	}
5611
5612	if (platform->supported_cstates & PC7) {
5613	get_msr(cpu: base_cpu, MSR_PKGC7_IRTL, msr: &msr);
5614	fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", base_cpu, msr);
5615	fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5616	}
5617
5618	if (platform->supported_cstates & PC8) {
5619	get_msr(cpu: base_cpu, MSR_PKGC8_IRTL, msr: &msr);
5620	fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", base_cpu, msr);
5621	fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5622	}
5623
5624	if (platform->supported_cstates & PC9) {
5625	get_msr(cpu: base_cpu, MSR_PKGC9_IRTL, msr: &msr);
5626	fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", base_cpu, msr);
5627	fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5628	}
5629
5630	if (platform->supported_cstates & PC10) {
5631	get_msr(cpu: base_cpu, MSR_PKGC10_IRTL, msr: &msr);
5632	fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", base_cpu, msr);
5633	fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5634	}
5635	}
5636
5637	void free_fd_percpu(void)
5638	{
5639	int i;
5640
5641	if (!fd_percpu)
5642	return;
5643
5644	for (i = 0; i < topo.max_cpu_num + 1; ++i) {
5645	if (fd_percpu[i] != 0)
5646	close(fd_percpu[i]);
5647	}
5648
5649	free(fd_percpu);
5650	fd_percpu = NULL;
5651	}
5652
5653	void free_fd_instr_count_percpu(void)
5654	{
5655	if (!fd_instr_count_percpu)
5656	return;
5657
5658	for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5659	if (fd_instr_count_percpu[i] != 0)
5660	close(fd_instr_count_percpu[i]);
5661	}
5662
5663	free(fd_instr_count_percpu);
5664	fd_instr_count_percpu = NULL;
5665	}
5666
5667	void free_fd_llc_percpu(void)
5668	{
5669	if (!fd_llc_percpu)
5670	return;
5671
5672	for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5673	if (fd_llc_percpu[i] != 0)
5674	close(fd_llc_percpu[i]);
5675	}
5676
5677	free(fd_llc_percpu);
5678	fd_llc_percpu = NULL;
5679	}
5680
5681	void free_fd_cstate(void)
5682	{
5683	if (!ccstate_counter_info)
5684	return;
5685
5686	const int counter_info_num = ccstate_counter_info_size;
5687
5688	for (int counter_id = 0; counter_id < counter_info_num; ++counter_id) {
5689	if (ccstate_counter_info[counter_id].fd_perf_core != -1)
5690	close(ccstate_counter_info[counter_id].fd_perf_core);
5691
5692	if (ccstate_counter_info[counter_id].fd_perf_pkg != -1)
5693	close(ccstate_counter_info[counter_id].fd_perf_pkg);
5694	}
5695
5696	free(ccstate_counter_info);
5697	ccstate_counter_info = NULL;
5698	ccstate_counter_info_size = 0;
5699	}
5700
5701	void free_fd_msr(void)
5702	{
5703	if (!msr_counter_info)
5704	return;
5705
5706	for (int cpu = 0; cpu < topo.max_cpu_num; ++cpu) {
5707	if (msr_counter_info[cpu].fd_perf != -1)
5708	close(msr_counter_info[cpu].fd_perf);
5709	}
5710
5711	free(msr_counter_info);
5712	msr_counter_info = NULL;
5713	msr_counter_info_size = 0;
5714	}
5715
5716	void free_fd_rapl_percpu(void)
5717	{
5718	if (!rapl_counter_info_perdomain)
5719	return;
5720
5721	const int num_domains = rapl_counter_info_perdomain_size;
5722
5723	for (int domain_id = 0; domain_id < num_domains; ++domain_id) {
5724	if (rapl_counter_info_perdomain[domain_id].fd_perf != -1)
5725	close(rapl_counter_info_perdomain[domain_id].fd_perf);
5726	}
5727
5728	free(rapl_counter_info_perdomain);
5729	rapl_counter_info_perdomain = NULL;
5730	rapl_counter_info_perdomain_size = 0;
5731	}
5732
5733	void free_fd_added_perf_counters_(struct perf_counter_info *pp)
5734	{
5735	if (!pp)
5736	return;
5737
5738	if (!pp->fd_perf_per_domain)
5739	return;
5740
5741	while (pp) {
5742	for (size_t domain = 0; domain < pp->num_domains; ++domain) {
5743	if (pp->fd_perf_per_domain[domain] != -1) {
5744	close(pp->fd_perf_per_domain[domain]);
5745	pp->fd_perf_per_domain[domain] = -1;
5746	}
5747	}
5748
5749	free(pp->fd_perf_per_domain);
5750	pp->fd_perf_per_domain = NULL;
5751
5752	pp = pp->next;
5753	}
5754	}
5755
5756	void free_fd_added_perf_counters(void)
5757	{
5758	free_fd_added_perf_counters_(pp: sys.perf_tp);
5759	free_fd_added_perf_counters_(pp: sys.perf_cp);
5760	free_fd_added_perf_counters_(pp: sys.perf_pp);
5761	}
5762
5763	void free_all_buffers(void)
5764	{
5765	int i;
5766
5767	CPU_FREE(cpu_present_set);
5768	cpu_present_set = NULL;
5769	cpu_present_setsize = 0;
5770
5771	CPU_FREE(cpu_effective_set);
5772	cpu_effective_set = NULL;
5773	cpu_effective_setsize = 0;
5774
5775	CPU_FREE(cpu_allowed_set);
5776	cpu_allowed_set = NULL;
5777	cpu_allowed_setsize = 0;
5778
5779	CPU_FREE(cpu_affinity_set);
5780	cpu_affinity_set = NULL;
5781	cpu_affinity_setsize = 0;
5782
5783	free(thread_even);
5784	free(core_even);
5785	free(package_even);
5786
5787	thread_even = NULL;
5788	core_even = NULL;
5789	package_even = NULL;
5790
5791	free(thread_odd);
5792	free(core_odd);
5793	free(package_odd);
5794
5795	thread_odd = NULL;
5796	core_odd = NULL;
5797	package_odd = NULL;
5798
5799	free(output_buffer);
5800	output_buffer = NULL;
5801	outp = NULL;
5802
5803	free_fd_percpu();
5804	free_fd_instr_count_percpu();
5805	free_fd_llc_percpu();
5806	free_fd_msr();
5807	free_fd_rapl_percpu();
5808	free_fd_cstate();
5809	free_fd_added_perf_counters();
5810
5811	free(irq_column_2_cpu);
5812	free(irqs_per_cpu);
5813	free(nmi_per_cpu);
5814
5815	for (i = 0; i <= topo.max_cpu_num; ++i) {
5816	if (cpus[i].put_ids)
5817	CPU_FREE(cpus[i].put_ids);
5818	}
5819	free(cpus);
5820	}
5821
5822	/*
5823	* Parse a file containing a single int.
5824	* Return 0 if file can not be opened
5825	* Exit if file can be opened, but can not be parsed
5826	*/
5827	int parse_int_file(const char *fmt, ...)
5828	{
5829	va_list args;
5830	char path[PATH_MAX];
5831	FILE *filep;
5832	int value;
5833
5834	va_start(args, fmt);
5835	vsnprintf(buf: path, size: sizeof(path), fmt, args);
5836	va_end(args);
5837	filep = fopen(path, "r");
5838	if (!filep)
5839	return 0;
5840	if (fscanf(filep, "%d", &value) != 1)
5841	err(1, "%s: failed to parse number from file", path);
5842	fclose(filep);
5843	return value;
5844	}
5845
5846	/*
5847	* cpu_is_first_core_in_package(cpu)
5848	* return 1 if given CPU is 1st core in package
5849	*/
5850	int cpu_is_first_core_in_package(int cpu)
5851	{
5852	return cpu == parse_int_file(fmt: "/sys/devices/system/cpu/cpu%d/topology/core_siblings_list", cpu);
5853	}
5854
5855	int get_physical_package_id(int cpu)
5856	{
5857	return parse_int_file(fmt: "/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu);
5858	}
5859
5860	int get_die_id(int cpu)
5861	{
5862	return parse_int_file(fmt: "/sys/devices/system/cpu/cpu%d/topology/die_id", cpu);
5863	}
5864
5865	int get_l3_id(int cpu)
5866	{
5867	return parse_int_file(fmt: "/sys/devices/system/cpu/cpu%d/cache/index3/id", cpu);
5868	}
5869
5870	int get_core_id(int cpu)
5871	{
5872	return parse_int_file(fmt: "/sys/devices/system/cpu/cpu%d/topology/core_id", cpu);
5873	}
5874
5875	void set_node_data(void)
5876	{
5877	int pkg, node, lnode, cpu, cpux;
5878	int cpu_count;
5879
5880	/* initialize logical_node_id */
5881	for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu)
5882	cpus[cpu].logical_node_id = -1;
5883
5884	cpu_count = 0;
5885	for (pkg = 0; pkg < topo.num_packages; pkg++) {
5886	lnode = 0;
5887	for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
5888	if (cpus[cpu].physical_package_id != pkg)
5889	continue;
5890	/* find a cpu with an unset logical_node_id */
5891	if (cpus[cpu].logical_node_id != -1)
5892	continue;
5893	cpus[cpu].logical_node_id = lnode;
5894	node = cpus[cpu].physical_node_id;
5895	cpu_count++;
5896	/*
5897	* find all matching cpus on this pkg and set
5898	* the logical_node_id
5899	*/
5900	for (cpux = cpu; cpux <= topo.max_cpu_num; cpux++) {
5901	if ((cpus[cpux].physical_package_id == pkg) && (cpus[cpux].physical_node_id == node)) {
5902	cpus[cpux].logical_node_id = lnode;
5903	cpu_count++;
5904	}
5905	}
5906	lnode++;
5907	if (lnode > topo.nodes_per_pkg)
5908	topo.nodes_per_pkg = lnode;
5909	}
5910	if (cpu_count >= topo.max_cpu_num)
5911	break;
5912	}
5913	}
5914
5915	int get_physical_node_id(struct cpu_topology *thiscpu)
5916	{
5917	char path[80];
5918	FILE *filep;
5919	int i;
5920	int cpu = thiscpu->logical_cpu_id;
5921
5922	for (i = 0; i <= topo.max_cpu_num; i++) {
5923	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/node%i/cpulist", cpu, i);
5924	filep = fopen(path, "r");
5925	if (!filep)
5926	continue;
5927	fclose(filep);
5928	return i;
5929	}
5930	return -1;
5931	}
5932
5933	static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size)
5934	{
5935	unsigned int start, end;
5936	char *next = cpu_str;
5937
5938	while (next && *next) {
5939
5940	if (*next == '-') /* no negative cpu numbers */
5941	return 1;
5942
5943	if (*next == '\0' || *next == '\n')
5944	break;
5945
5946	start = strtoul(next, &next, 10);
5947
5948	if (start >= CPU_SUBSET_MAXCPUS)
5949	return 1;
5950	CPU_SET_S(start, cpu_set_size, cpu_set);
5951
5952	if (*next == '\0' || *next == '\n')
5953	break;
5954
5955	if (*next == ',') {
5956	next += 1;
5957	continue;
5958	}
5959
5960	if (*next == '-') {
5961	next += 1; /* start range */
5962	} else if (*next == '.') {
5963	next += 1;
5964	if (*next == '.')
5965	next += 1; /* start range */
5966	else
5967	return 1;
5968	}
5969
5970	end = strtoul(next, &next, 10);
5971	if (end <= start)
5972	return 1;
5973
5974	while (++start <= end) {
5975	if (start >= CPU_SUBSET_MAXCPUS)
5976	return 1;
5977	CPU_SET_S(start, cpu_set_size, cpu_set);
5978	}
5979
5980	if (*next == ',')
5981	next += 1;
5982	else if (*next != '\0' && *next != '\n')
5983	return 1;
5984	}
5985
5986	return 0;
5987	}
5988
5989	int get_thread_siblings(struct cpu_topology *thiscpu)
5990	{
5991	char path[80], character;
5992	FILE *filep;
5993	unsigned long map;
5994	int so, shift, sib_core;
5995	int cpu = thiscpu->logical_cpu_id;
5996	int offset = topo.max_cpu_num + 1;
5997	size_t size;
5998	int thread_id = 0;
5999
6000	thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1));
6001	if (thiscpu->thread_id < 0)
6002	thiscpu->thread_id = thread_id++;
6003	if (!thiscpu->put_ids)
6004	return -1;
6005
6006	size = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
6007	CPU_ZERO_S(size, thiscpu->put_ids);
6008
6009	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", cpu);
6010	filep = fopen(path, "r");
6011
6012	if (!filep) {
6013	warnx("%s: open failed", path);
6014	return -1;
6015	}
6016	do {
6017	offset -= BITMASK_SIZE;
6018	if (fscanf(filep, "%lx%c", &map, &character) != 2)
6019	err(1, "%s: failed to parse file", path);
6020	for (shift = 0; shift < BITMASK_SIZE; shift++) {
6021	if ((map >> shift) & 0x1) {
6022	so = shift + offset;
6023	sib_core = get_core_id(cpu: so);
6024	if (sib_core == thiscpu->physical_core_id) {
6025	CPU_SET_S(so, size, thiscpu->put_ids);
6026	if ((so != cpu) && (cpus[so].thread_id < 0))
6027	cpus[so].thread_id = thread_id++;
6028	}
6029	}
6030	}
6031	} while (character == ',');
6032	fclose(filep);
6033
6034	return CPU_COUNT_S(size, thiscpu->put_ids);
6035	}
6036
6037	/*
6038	* run func(thread, core, package) in topology order
6039	* skip non-present cpus
6040	*/
6041
6042	int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *,
6043	struct pkg_data *, struct thread_data *, struct core_data *,
6044	struct pkg_data *), struct thread_data *thread_base,
6045	struct core_data *core_base, struct pkg_data *pkg_base,
6046	struct thread_data *thread_base2, struct core_data *core_base2, struct pkg_data *pkg_base2)
6047	{
6048	int retval, pkg_no, node_no, core_no, thread_no;
6049
6050	retval = 0;
6051
6052	for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
6053	for (node_no = 0; node_no < topo.nodes_per_pkg; ++node_no) {
6054	for (core_no = 0; core_no < topo.cores_per_node; ++core_no) {
6055	for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) {
6056	struct thread_data *t, *t2;
6057	struct core_data *c, *c2;
6058
6059	t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no);
6060
6061	if (cpu_is_not_allowed(cpu: t->cpu_id))
6062	continue;
6063
6064	t2 = GET_THREAD(thread_base2, thread_no, core_no, node_no, pkg_no);
6065
6066	c = GET_CORE(core_base, core_no, node_no, pkg_no);
6067	c2 = GET_CORE(core_base2, core_no, node_no, pkg_no);
6068
6069	retval |= func(t, c, &pkg_base[pkg_no], t2, c2, &pkg_base2[pkg_no]);
6070	}
6071	}
6072	}
6073	}
6074	return retval;
6075	}
6076
6077	/*
6078	* run func(cpu) on every cpu in /proc/stat
6079	* return max_cpu number
6080	*/
6081	int for_all_proc_cpus(int (func) (int))
6082	{
6083	FILE *fp;
6084	int cpu_num;
6085	int retval;
6086
6087	fp = fopen_or_die(proc_stat, "r");
6088
6089	retval = fscanf(fp, "cpu %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n");
6090	if (retval != 0)
6091	err(1, "%s: failed to parse format", proc_stat);
6092
6093	while (1) {
6094	retval = fscanf(fp, "cpu%u %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n", &cpu_num);
6095	if (retval != 1)
6096	break;
6097
6098	retval = func(cpu_num);
6099	if (retval) {
6100	fclose(fp);
6101	return (retval);
6102	}
6103	}
6104	fclose(fp);
6105	return 0;
6106	}
6107
6108	#define PATH_EFFECTIVE_CPUS "/sys/fs/cgroup/cpuset.cpus.effective"
6109
6110	static char cpu_effective_str[1024];
6111
6112	static int update_effective_str(bool startup)
6113	{
6114	FILE *fp;
6115	char *pos;
6116	char buf[1024];
6117	int ret;
6118
6119	if (cpu_effective_str[0] == '\0' && !startup)
6120	return 0;
6121
6122	fp = fopen(PATH_EFFECTIVE_CPUS, "r");
6123	if (!fp)
6124	return 0;
6125
6126	pos = fgets(buf, 1024, fp);
6127	if (!pos)
6128	err(1, "%s: file read failed\n", PATH_EFFECTIVE_CPUS);
6129
6130	fclose(fp);
6131
6132	ret = strncmp(cpu_effective_str, buf, 1024);
6133	if (!ret)
6134	return 0;
6135
6136	strncpy(p: cpu_effective_str, q: buf, size: 1024);
6137	return 1;
6138	}
6139
6140	static void update_effective_set(bool startup)
6141	{
6142	update_effective_str(startup);
6143
6144	if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize))
6145	err(1, "%s: cpu str malformat %s\n", PATH_EFFECTIVE_CPUS, cpu_effective_str);
6146	}
6147
6148	void linux_perf_init(void);
6149	void msr_perf_init(void);
6150	void rapl_perf_init(void);
6151	void cstate_perf_init(void);
6152	void perf_llc_init(void);
6153	void added_perf_counters_init(void);
6154	void pmt_init(void);
6155
6156	void re_initialize(void)
6157	{
6158	free_all_buffers();
6159	setup_all_buffers(false);
6160	linux_perf_init();
6161	msr_perf_init();
6162	rapl_perf_init();
6163	cstate_perf_init();
6164	perf_llc_init();
6165	added_perf_counters_init();
6166	pmt_init();
6167	fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus);
6168	}
6169
6170	void set_max_cpu_num(void)
6171	{
6172	FILE *filep;
6173	int base_cpu;
6174	unsigned long dummy;
6175	char pathname[64];
6176
6177	base_cpu = sched_getcpu();
6178	if (base_cpu < 0)
6179	err(1, "cannot find calling cpu ID");
6180	sprintf(buf: pathname, fmt: "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", base_cpu);
6181
6182	filep = fopen_or_die(pathname, "r");
6183	topo.max_cpu_num = 0;
6184	while (fscanf(filep, "%lx,", &dummy) == 1)
6185	topo.max_cpu_num += BITMASK_SIZE;
6186	fclose(filep);
6187	topo.max_cpu_num--; /* 0 based */
6188	}
6189
6190	/*
6191	* count_cpus()
6192	* remember the last one seen, it will be the max
6193	*/
6194	int count_cpus(int cpu)
6195	{
6196	UNUSED(cpu);
6197
6198	topo.num_cpus++;
6199	return 0;
6200	}
6201
6202	int mark_cpu_present(int cpu)
6203	{
6204	CPU_SET_S(cpu, cpu_present_setsize, cpu_present_set);
6205	return 0;
6206	}
6207
6208	int init_thread_id(int cpu)
6209	{
6210	cpus[cpu].thread_id = -1;
6211	return 0;
6212	}
6213
6214	int set_my_cpu_type(void)
6215	{
6216	unsigned int eax, ebx, ecx, edx;
6217	unsigned int max_level;
6218
6219	__cpuid(0, max_level, ebx, ecx, edx);
6220
6221	if (max_level < CPUID_LEAF_MODEL_ID)
6222	return 0;
6223
6224	__cpuid(CPUID_LEAF_MODEL_ID, eax, ebx, ecx, edx);
6225
6226	return (eax >> CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT);
6227	}
6228
6229	int set_cpu_hybrid_type(int cpu)
6230	{
6231	if (cpu_migrate(cpu))
6232	return -1;
6233
6234	int type = set_my_cpu_type();
6235
6236	cpus[cpu].type = type;
6237	return 0;
6238	}
6239
6240	/*
6241	* snapshot_proc_interrupts()
6242	*
6243	* read and record summary of /proc/interrupts
6244	*
6245	* return 1 if config change requires a restart, else return 0
6246	*/
6247	int snapshot_proc_interrupts(void)
6248	{
6249	static FILE *fp;
6250	int column, retval;
6251
6252	if (fp == NULL)
6253	fp = fopen_or_die("/proc/interrupts", "r");
6254	else
6255	rewind(fp);
6256
6257	/* read 1st line of /proc/interrupts to get cpu* name for each column */
6258	for (column = 0; column < topo.num_cpus; ++column) {
6259	int cpu_number;
6260
6261	retval = fscanf(fp, " CPU%d", &cpu_number);
6262	if (retval != 1)
6263	break;
6264
6265	if (cpu_number > topo.max_cpu_num) {
6266	warn("/proc/interrupts: cpu%d: > %d", cpu_number, topo.max_cpu_num);
6267	return 1;
6268	}
6269
6270	irq_column_2_cpu[column] = cpu_number;
6271	irqs_per_cpu[cpu_number] = 0;
6272	nmi_per_cpu[cpu_number] = 0;
6273	}
6274
6275	/* read /proc/interrupt count lines and sum up irqs per cpu */
6276	while (1) {
6277	int column;
6278	char buf[64];
6279	int this_row_is_nmi = 0;
6280
6281	retval = fscanf(fp, " %s:", buf); /* irq# "N:" */
6282	if (retval != 1)
6283	break;
6284
6285	if (strncmp(buf, "NMI", strlen("NMI")) == 0)
6286	this_row_is_nmi = 1;
6287
6288	/* read the count per cpu */
6289	for (column = 0; column < topo.num_cpus; ++column) {
6290
6291	int cpu_number, irq_count;
6292
6293	retval = fscanf(fp, " %d", &irq_count);
6294
6295	if (retval != 1)
6296	break;
6297
6298	cpu_number = irq_column_2_cpu[column];
6299	irqs_per_cpu[cpu_number] += irq_count;
6300	if (this_row_is_nmi)
6301	nmi_per_cpu[cpu_number] += irq_count;
6302	}
6303	while (getc(fp) != '\n') ; /* flush interrupt description */
6304
6305	}
6306	return 0;
6307	}
6308
6309	/*
6310	* snapshot_graphics()
6311	*
6312	* record snapshot of specified graphics sysfs knob
6313	*
6314	* return 1 if config change requires a restart, else return 0
6315	*/
6316	int snapshot_graphics(int idx)
6317	{
6318	int retval;
6319
6320	rewind(gfx_info[idx].fp);
6321	fflush(gfx_info[idx].fp);
6322
6323	switch (idx) {
6324	case GFX_rc6:
6325	case SAM_mc6:
6326	retval = fscanf(gfx_info[idx].fp, "%lld", &gfx_info[idx].val_ull);
6327	if (retval != 1)
6328	err(1, "rc6");
6329	return 0;
6330	case GFX_MHz:
6331	case GFX_ACTMHz:
6332	case SAM_MHz:
6333	case SAM_ACTMHz:
6334	retval = fscanf(gfx_info[idx].fp, "%d", &gfx_info[idx].val);
6335	if (retval != 1)
6336	err(1, "MHz");
6337	return 0;
6338	default:
6339	return -EINVAL;
6340	}
6341	}
6342
6343	/*
6344	* snapshot_cpu_lpi()
6345	*
6346	* record snapshot of
6347	* /sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us
6348	*/
6349	int snapshot_cpu_lpi_us(void)
6350	{
6351	FILE *fp;
6352	int retval;
6353
6354	fp = fopen_or_die("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", "r");
6355
6356	retval = fscanf(fp, "%lld", &cpuidle_cur_cpu_lpi_us);
6357	if (retval != 1) {
6358	fprintf(stderr, "Disabling Low Power Idle CPU output\n");
6359	BIC_NOT_PRESENT(BIC_CPU_LPI);
6360	fclose(fp);
6361	return -1;
6362	}
6363
6364	fclose(fp);
6365
6366	return 0;
6367	}
6368
6369	/*
6370	* snapshot_sys_lpi()
6371	*
6372	* record snapshot of sys_lpi_file
6373	*/
6374	int snapshot_sys_lpi_us(void)
6375	{
6376	FILE *fp;
6377	int retval;
6378
6379	fp = fopen_or_die(sys_lpi_file, "r");
6380
6381	retval = fscanf(fp, "%lld", &cpuidle_cur_sys_lpi_us);
6382	if (retval != 1) {
6383	fprintf(stderr, "Disabling Low Power Idle System output\n");
6384	BIC_NOT_PRESENT(BIC_SYS_LPI);
6385	fclose(fp);
6386	return -1;
6387	}
6388	fclose(fp);
6389
6390	return 0;
6391	}
6392
6393	/*
6394	* snapshot /proc and /sys files
6395	*
6396	* return 1 if configuration restart needed, else return 0
6397	*/
6398	int snapshot_proc_sysfs_files(void)
6399	{
6400	gettimeofday(&procsysfs_tv_begin, (struct timezone *)NULL);
6401
6402	if (DO_BIC(BIC_IRQ) || DO_BIC(BIC_NMI))
6403	if (snapshot_proc_interrupts())
6404	return 1;
6405
6406	if (DO_BIC(BIC_GFX_rc6))
6407	snapshot_graphics(idx: GFX_rc6);
6408
6409	if (DO_BIC(BIC_GFXMHz))
6410	snapshot_graphics(idx: GFX_MHz);
6411
6412	if (DO_BIC(BIC_GFXACTMHz))
6413	snapshot_graphics(idx: GFX_ACTMHz);
6414
6415	if (DO_BIC(BIC_SAM_mc6))
6416	snapshot_graphics(idx: SAM_mc6);
6417
6418	if (DO_BIC(BIC_SAMMHz))
6419	snapshot_graphics(idx: SAM_MHz);
6420
6421	if (DO_BIC(BIC_SAMACTMHz))
6422	snapshot_graphics(idx: SAM_ACTMHz);
6423
6424	if (DO_BIC(BIC_CPU_LPI))
6425	snapshot_cpu_lpi_us();
6426
6427	if (DO_BIC(BIC_SYS_LPI))
6428	snapshot_sys_lpi_us();
6429
6430	return 0;
6431	}
6432
6433	int exit_requested;
6434
6435	static void signal_handler(int signal)
6436	{
6437	switch (signal) {
6438	case SIGINT:
6439	exit_requested = 1;
6440	if (debug)
6441	fprintf(stderr, " SIGINT\n");
6442	break;
6443	case SIGUSR1:
6444	if (debug > 1)
6445	fprintf(stderr, "SIGUSR1\n");
6446	break;
6447	}
6448	}
6449
6450	void setup_signal_handler(void)
6451	{
6452	struct sigaction sa;
6453
6454	memset(&sa, 0, sizeof(sa));
6455
6456	sa.sa_handler = &signal_handler;
6457
6458	if (sigaction(SIGINT, &sa, NULL) < 0)
6459	err(1, "sigaction SIGINT");
6460	if (sigaction(SIGUSR1, &sa, NULL) < 0)
6461	err(1, "sigaction SIGUSR1");
6462	}
6463
6464	void do_sleep(void)
6465	{
6466	struct timeval tout;
6467	struct timespec rest;
6468	fd_set readfds;
6469	int retval;
6470
6471	FD_ZERO(&readfds);
6472	FD_SET(0, &readfds);
6473
6474	if (ignore_stdin) {
6475	nanosleep(&interval_ts, NULL);
6476	return;
6477	}
6478
6479	tout = interval_tv;
6480	retval = select(1, &readfds, NULL, NULL, &tout);
6481
6482	if (retval == 1) {
6483	switch (getc(stdin)) {
6484	case 'q':
6485	exit_requested = 1;
6486	break;
6487	case EOF:
6488	/*
6489	* 'stdin' is a pipe closed on the other end. There
6490	* won't be any further input.
6491	*/
6492	ignore_stdin = 1;
6493	/* Sleep the rest of the time */
6494	rest.tv_sec = (tout.tv_sec + tout.tv_usec / 1000000);
6495	rest.tv_nsec = (tout.tv_usec % 1000000) * 1000;
6496	nanosleep(&rest, NULL);
6497	}
6498	}
6499	}
6500
6501	int get_msr_sum(int cpu, off_t offset, unsigned long long *msr)
6502	{
6503	int ret, idx;
6504	unsigned long long msr_cur, msr_last;
6505
6506	assert(!no_msr);
6507
6508	if (!per_cpu_msr_sum)
6509	return 1;
6510
6511	idx = offset_to_idx(offset);
6512	if (idx < 0)
6513	return idx;
6514	/* get_msr_sum() = sum + (get_msr() - last) */
6515	ret = get_msr(cpu, offset, msr: &msr_cur);
6516	if (ret)
6517	return ret;
6518	msr_last = per_cpu_msr_sum[cpu].entries[idx].last;
6519	DELTA_WRAP32(msr_cur, msr_last);
6520	*msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum;
6521
6522	return 0;
6523	}
6524
6525	timer_t timerid;
6526
6527	/* Timer callback, update the sum of MSRs periodically. */
6528	static int update_msr_sum(PER_THREAD_PARAMS)
6529	{
6530	int i, ret;
6531	int cpu = t->cpu_id;
6532
6533	UNUSED(c);
6534	UNUSED(p);
6535
6536	assert(!no_msr);
6537
6538	for (i = IDX_PKG_ENERGY; i < IDX_COUNT; i++) {
6539	unsigned long long msr_cur, msr_last;
6540	off_t offset;
6541
6542	if (!idx_valid(idx: i))
6543	continue;
6544	offset = idx_to_offset(idx: i);
6545	if (offset < 0)
6546	continue;
6547	ret = get_msr(cpu, offset, msr: &msr_cur);
6548	if (ret) {
6549	fprintf(outf, "Can not update msr(0x%llx)\n", (unsigned long long)offset);
6550	continue;
6551	}
6552
6553	msr_last = per_cpu_msr_sum[cpu].entries[i].last;
6554	per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff;
6555
6556	DELTA_WRAP32(msr_cur, msr_last);
6557	per_cpu_msr_sum[cpu].entries[i].sum += msr_last;
6558	}
6559	return 0;
6560	}
6561
6562	static void msr_record_handler(union sigval v)
6563	{
6564	UNUSED(v);
6565
6566	for_all_cpus(func: update_msr_sum, EVEN_COUNTERS);
6567	}
6568
6569	void msr_sum_record(void)
6570	{
6571	struct itimerspec its;
6572	struct sigevent sev;
6573
6574	per_cpu_msr_sum = calloc(topo.max_cpu_num + 1, sizeof(struct msr_sum_array));
6575	if (!per_cpu_msr_sum) {
6576	fprintf(outf, "Can not allocate memory for long time MSR.\n");
6577	return;
6578	}
6579	/*
6580	* Signal handler might be restricted, so use thread notifier instead.
6581	*/
6582	memset(&sev, 0, sizeof(struct sigevent));
6583	sev.sigev_notify = SIGEV_THREAD;
6584	sev.sigev_notify_function = msr_record_handler;
6585
6586	sev.sigev_value.sival_ptr = &timerid;
6587	if (timer_create(CLOCK_REALTIME, &sev, &timerid) == -1) {
6588	fprintf(outf, "Can not create timer.\n");
6589	goto release_msr;
6590	}
6591
6592	its.it_value.tv_sec = 0;
6593	its.it_value.tv_nsec = 1;
6594	/*
6595	* A wraparound time has been calculated early.
6596	* Some sources state that the peak power for a
6597	* microprocessor is usually 1.5 times the TDP rating,
6598	* use 2 * TDP for safety.
6599	*/
6600	its.it_interval.tv_sec = rapl_joule_counter_range / 2;
6601	its.it_interval.tv_nsec = 0;
6602
6603	if (timer_settime(timerid, 0, &its, NULL) == -1) {
6604	fprintf(outf, "Can not set timer.\n");
6605	goto release_timer;
6606	}
6607	return;
6608
6609	release_timer:
6610	timer_delete(timer: timerid);
6611	release_msr:
6612	free(per_cpu_msr_sum);
6613	per_cpu_msr_sum = NULL;
6614	}
6615
6616	/*
6617	* set_my_sched_priority(pri)
6618	* return previous priority on success
6619	* return value < -20 on failure
6620	*/
6621	int set_my_sched_priority(int priority)
6622	{
6623	int retval;
6624	int original_priority;
6625
6626	errno = 0;
6627	original_priority = getpriority(PRIO_PROCESS, 0);
6628	if (errno && (original_priority == -1))
6629	return -21;
6630
6631	retval = setpriority(PRIO_PROCESS, 0, priority);
6632	if (retval)
6633	return -21;
6634
6635	errno = 0;
6636	retval = getpriority(PRIO_PROCESS, 0);
6637	if (retval != priority)
6638	return -21;
6639
6640	return original_priority;
6641	}
6642
6643	void turbostat_loop()
6644	{
6645	int retval;
6646	int restarted = 0;
6647	unsigned int done_iters = 0;
6648
6649	setup_signal_handler();
6650
6651	/*
6652	* elevate own priority for interval mode
6653	*
6654	* ignore on error - we probably don't have permission to set it, but
6655	* it's not a big deal
6656	*/
6657	set_my_sched_priority(-20);
6658
6659	restart:
6660	restarted++;
6661
6662	snapshot_proc_sysfs_files();
6663	retval = for_all_cpus(func: get_counters, EVEN_COUNTERS);
6664	first_counter_read = 0;
6665	if (retval < -1) {
6666	exit(retval);
6667	} else if (retval == -1) {
6668	if (restarted > 10) {
6669	exit(retval);
6670	}
6671	re_initialize();
6672	goto restart;
6673	}
6674	restarted = 0;
6675	done_iters = 0;
6676	gettimeofday(&tv_even, (struct timezone *)NULL);
6677
6678	while (1) {
6679	if (for_all_proc_cpus(func: cpu_is_not_present)) {
6680	re_initialize();
6681	goto restart;
6682	}
6683	if (update_effective_str(startup: false)) {
6684	re_initialize();
6685	goto restart;
6686	}
6687	do_sleep();
6688	if (snapshot_proc_sysfs_files())
6689	goto restart;
6690	retval = for_all_cpus(func: get_counters, ODD_COUNTERS);
6691	if (retval < -1) {
6692	exit(retval);
6693	} else if (retval == -1) {
6694	re_initialize();
6695	goto restart;
6696	}
6697	gettimeofday(&tv_odd, (struct timezone *)NULL);
6698	timersub(&tv_odd, &tv_even, &tv_delta);
6699	if (for_all_cpus_2(func: delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) {
6700	re_initialize();
6701	goto restart;
6702	}
6703	delta_platform(new: &platform_counters_odd, old: &platform_counters_even);
6704	compute_average(EVEN_COUNTERS);
6705	format_all_counters(EVEN_COUNTERS);
6706	flush_output_stdout();
6707	if (exit_requested)
6708	break;
6709	if (num_iterations && ++done_iters >= num_iterations)
6710	break;
6711	do_sleep();
6712	if (snapshot_proc_sysfs_files())
6713	goto restart;
6714	retval = for_all_cpus(func: get_counters, EVEN_COUNTERS);
6715	if (retval < -1) {
6716	exit(retval);
6717	} else if (retval == -1) {
6718	re_initialize();
6719	goto restart;
6720	}
6721	gettimeofday(&tv_even, (struct timezone *)NULL);
6722	timersub(&tv_even, &tv_odd, &tv_delta);
6723	if (for_all_cpus_2(func: delta_cpu, EVEN_COUNTERS, ODD_COUNTERS)) {
6724	re_initialize();
6725	goto restart;
6726	}
6727	delta_platform(new: &platform_counters_even, old: &platform_counters_odd);
6728	compute_average(ODD_COUNTERS);
6729	format_all_counters(ODD_COUNTERS);
6730	flush_output_stdout();
6731	if (exit_requested)
6732	break;
6733	if (num_iterations && ++done_iters >= num_iterations)
6734	break;
6735	}
6736	}
6737
6738	int probe_dev_msr(void)
6739	{
6740	struct stat sb;
6741	char pathname[32];
6742
6743	sprintf(buf: pathname, fmt: "/dev/msr%d", base_cpu);
6744	return !stat(pathname, &sb);
6745	}
6746
6747	int probe_dev_cpu_msr(void)
6748	{
6749	struct stat sb;
6750	char pathname[32];
6751
6752	sprintf(buf: pathname, fmt: "/dev/cpu/%d/msr", base_cpu);
6753	return !stat(pathname, &sb);
6754	}
6755
6756	int probe_msr_driver(void)
6757	{
6758	if (probe_dev_msr()) {
6759	use_android_msr_path = 1;
6760	return 1;
6761	}
6762	return probe_dev_cpu_msr();
6763	}
6764
6765	void check_msr_driver(void)
6766	{
6767	if (probe_msr_driver())
6768	return;
6769
6770	if (system("/sbin/modprobe msr > /dev/null 2>&1"))
6771	no_msr = 1;
6772
6773	if (!probe_msr_driver())
6774	no_msr = 1;
6775	}
6776
6777	/*
6778	* check for CAP_SYS_RAWIO
6779	* return 0 on success
6780	* return 1 on fail
6781	*/
6782	int check_for_cap_sys_rawio(void)
6783	{
6784	cap_t caps;
6785	cap_flag_value_t cap_flag_value;
6786	int ret = 0;
6787
6788	caps = cap_get_proc();
6789	if (caps == NULL) {
6790	/*
6791	* CONFIG_MULTIUSER=n kernels have no cap_get_proc()
6792	* Allow them to continue and attempt to access MSRs
6793	*/
6794	if (errno == ENOSYS)
6795	return 0;
6796
6797	return 1;
6798	}
6799
6800	if (cap_get_flag(caps, CAP_SYS_RAWIO, CAP_EFFECTIVE, &cap_flag_value)) {
6801	ret = 1;
6802	goto free_and_exit;
6803	}
6804
6805	if (cap_flag_value != CAP_SET) {
6806	ret = 1;
6807	goto free_and_exit;
6808	}
6809
6810	free_and_exit:
6811	if (cap_free(caps) == -1)
6812	err(-6, "cap_free\n");
6813
6814	return ret;
6815	}
6816
6817	void check_msr_permission(void)
6818	{
6819	int failed = 0;
6820	char pathname[32];
6821
6822	if (no_msr)
6823	return;
6824
6825	/* check for CAP_SYS_RAWIO */
6826	failed += check_for_cap_sys_rawio();
6827
6828	/* test file permissions */
6829	sprintf(buf: pathname, fmt: use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", base_cpu);
6830	if (euidaccess(pathname, R_OK)) {
6831	failed++;
6832	}
6833
6834	if (failed) {
6835	warnx("Failed to access %s. Some of the counters may not be available\n"
6836	"\tRun as root to enable them or use %s to disable the access explicitly", pathname, "--no-msr");
6837	no_msr = 1;
6838	}
6839	}
6840
6841	void probe_bclk(void)
6842	{
6843	unsigned long long msr;
6844	unsigned int base_ratio;
6845
6846	if (!platform->has_nhm_msrs || no_msr)
6847	return;
6848
6849	if (platform->bclk_freq == BCLK_100MHZ)
6850	bclk = 100.00;
6851	else if (platform->bclk_freq == BCLK_133MHZ)
6852	bclk = 133.33;
6853	else if (platform->bclk_freq == BCLK_SLV)
6854	bclk = slm_bclk();
6855	else
6856	return;
6857
6858	get_msr(cpu: base_cpu, MSR_PLATFORM_INFO, msr: &msr);
6859	base_ratio = (msr >> 8) & 0xFF;
6860
6861	base_hz = base_ratio * bclk * 1000000;
6862	has_base_hz = 1;
6863
6864	if (platform->enable_tsc_tweak)
6865	tsc_tweak = base_hz / tsc_hz;
6866	}
6867
6868	static void remove_underbar(char *s)
6869	{
6870	char *to = s;
6871
6872	while (*s) {
6873	if (*s != '_')
6874	*to++ = *s;
6875	s++;
6876	}
6877
6878	*to = 0;
6879	}
6880
6881	static void dump_turbo_ratio_info(void)
6882	{
6883	if (!has_turbo)
6884	return;
6885
6886	if (!platform->has_nhm_msrs || no_msr)
6887	return;
6888
6889	if (platform->trl_msrs & TRL_LIMIT2)
6890	dump_turbo_ratio_limit2();
6891
6892	if (platform->trl_msrs & TRL_LIMIT1)
6893	dump_turbo_ratio_limit1();
6894
6895	if (platform->trl_msrs & TRL_BASE) {
6896	dump_turbo_ratio_limits(MSR_TURBO_RATIO_LIMIT);
6897
6898	if (is_hybrid)
6899	dump_turbo_ratio_limits(MSR_SECONDARY_TURBO_RATIO_LIMIT);
6900	}
6901
6902	if (platform->trl_msrs & TRL_ATOM)
6903	dump_atom_turbo_ratio_limits();
6904
6905	if (platform->trl_msrs & TRL_KNL)
6906	dump_knl_turbo_ratio_limits();
6907
6908	if (platform->has_config_tdp)
6909	dump_config_tdp();
6910	}
6911
6912	static int read_sysfs_int(char *path)
6913	{
6914	FILE *input;
6915	int retval = -1;
6916
6917	input = fopen(path, "r");
6918	if (input == NULL) {
6919	if (debug)
6920	fprintf(outf, "NSFOD %s\n", path);
6921	return (-1);
6922	}
6923	if (fscanf(input, "%d", &retval) != 1)
6924	err(1, "%s: failed to read int from file", path);
6925	fclose(input);
6926
6927	return (retval);
6928	}
6929
6930	static void dump_sysfs_file(char *path)
6931	{
6932	FILE *input;
6933	char cpuidle_buf[64];
6934
6935	input = fopen(path, "r");
6936	if (input == NULL) {
6937	if (debug)
6938	fprintf(outf, "NSFOD %s\n", path);
6939	return;
6940	}
6941	if (!fgets(cpuidle_buf, sizeof(cpuidle_buf), input))
6942	err(1, "%s: failed to read file", path);
6943	fclose(input);
6944
6945	fprintf(outf, "%s: %s", strrchr(path, '/') + 1, cpuidle_buf);
6946	}
6947
6948	static void probe_intel_uncore_frequency_legacy(void)
6949	{
6950	int i, j;
6951	char path[256];
6952
6953	for (i = 0; i < topo.num_packages; ++i) {
6954	for (j = 0; j <= topo.max_die_id; ++j) {
6955	int k, l;
6956	char path_base[128];
6957
6958	sprintf(buf: path_base, fmt: "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, j);
6959
6960	sprintf(buf: path, fmt: "%s/current_freq_khz", path_base);
6961	if (access(path, R_OK))
6962	continue;
6963
6964	BIC_PRESENT(BIC_UNCORE_MHZ);
6965
6966	if (quiet)
6967	return;
6968
6969	sprintf(buf: path, fmt: "%s/min_freq_khz", path_base);
6970	k = read_sysfs_int(path);
6971	sprintf(buf: path, fmt: "%s/max_freq_khz", path_base);
6972	l = read_sysfs_int(path);
6973	fprintf(outf, "Uncore Frequency package%d die%d: %d - %d MHz ", i, j, k / 1000, l / 1000);
6974
6975	sprintf(buf: path, fmt: "%s/initial_min_freq_khz", path_base);
6976	k = read_sysfs_int(path);
6977	sprintf(buf: path, fmt: "%s/initial_max_freq_khz", path_base);
6978	l = read_sysfs_int(path);
6979	fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
6980
6981	sprintf(buf: path, fmt: "%s/current_freq_khz", path_base);
6982	k = read_sysfs_int(path);
6983	fprintf(outf, " %d MHz\n", k / 1000);
6984	}
6985	}
6986	}
6987
6988	static void probe_intel_uncore_frequency_cluster(void)
6989	{
6990	int i, uncore_max_id;
6991	char path[256];
6992	char path_base[128];
6993
6994	if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00/current_freq_khz", R_OK))
6995	return;
6996
6997	for (uncore_max_id = 0;; ++uncore_max_id) {
6998
6999	sprintf(buf: path_base, fmt: "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", uncore_max_id);
7000
7001	/* uncore## start at 00 and skips no numbers, so stop upon first missing */
7002	if (access(path_base, R_OK)) {
7003	uncore_max_id -= 1;
7004	break;
7005	}
7006	}
7007	for (i = uncore_max_id; i >= 0; --i) {
7008	int k, l;
7009	int package_id, domain_id, cluster_id;
7010	char name_buf[16];
7011
7012	sprintf(buf: path_base, fmt: "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i);
7013
7014	if (access(path_base, R_OK))
7015	err(1, "%s: %s\n", __func__, path_base);
7016
7017	sprintf(buf: path, fmt: "%s/package_id", path_base);
7018	package_id = read_sysfs_int(path);
7019
7020	sprintf(buf: path, fmt: "%s/domain_id", path_base);
7021	domain_id = read_sysfs_int(path);
7022
7023	sprintf(buf: path, fmt: "%s/fabric_cluster_id", path_base);
7024	cluster_id = read_sysfs_int(path);
7025
7026	sprintf(buf: path, fmt: "%s/current_freq_khz", path_base);
7027	sprintf(buf: name_buf, fmt: "UMHz%d.%d", domain_id, cluster_id);
7028
7029	/*
7030	* Once add_couter() is called, that counter is always read
7031	* and reported -- So it is effectively (enabled & present).
7032	* Only call add_counter() here if legacy BIC_UNCORE_MHZ (UncMHz)
7033	* is (enabled). Since we are in this routine, we
7034	* know we will not probe and set (present) the legacy counter.
7035	*
7036	* This allows "--show/--hide UncMHz" to be effective for
7037	* the clustered MHz counters, as a group.
7038	*/
7039	if BIC_IS_ENABLED
7040	(BIC_UNCORE_MHZ)
7041	add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, package_id);
7042
7043	if (quiet)
7044	continue;
7045
7046	sprintf(buf: path, fmt: "%s/min_freq_khz", path_base);
7047	k = read_sysfs_int(path);
7048	sprintf(buf: path, fmt: "%s/max_freq_khz", path_base);
7049	l = read_sysfs_int(path);
7050	fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", package_id, domain_id, cluster_id, k / 1000, l / 1000);
7051
7052	sprintf(buf: path, fmt: "%s/initial_min_freq_khz", path_base);
7053	k = read_sysfs_int(path);
7054	sprintf(buf: path, fmt: "%s/initial_max_freq_khz", path_base);
7055	l = read_sysfs_int(path);
7056	fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
7057
7058	sprintf(buf: path, fmt: "%s/current_freq_khz", path_base);
7059	k = read_sysfs_int(path);
7060	fprintf(outf, " %d MHz\n", k / 1000);
7061	}
7062	}
7063
7064	static void probe_intel_uncore_frequency(void)
7065	{
7066	if (!genuine_intel)
7067	return;
7068
7069	if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00", R_OK) == 0)
7070	probe_intel_uncore_frequency_cluster();
7071	else
7072	probe_intel_uncore_frequency_legacy();
7073	}
7074
7075	static void set_graphics_fp(char *path, int idx)
7076	{
7077	if (!access(path, R_OK))
7078	gfx_info[idx].fp = fopen_or_die(path, "r");
7079	}
7080
7081	/* Enlarge this if there are /sys/class/drm/card2 ... */
7082	#define GFX_MAX_CARDS 2
7083
7084	static void probe_graphics(void)
7085	{
7086	char path[PATH_MAX];
7087	int i;
7088
7089	/* Xe graphics sysfs knobs */
7090	if (!access("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", R_OK)) {
7091	FILE *fp;
7092	char buf[8];
7093	bool gt0_is_gt;
7094
7095	fp = fopen("/sys/class/drm/card0/device/tile0/gt0/gtidle/name", "r");
7096	if (!fp)
7097	goto next;
7098
7099	if (!fread(buf, sizeof(char), 7, fp)) {
7100	fclose(fp);
7101	goto next;
7102	}
7103	fclose(fp);
7104
7105	if (!strncmp(buf, "gt0-rc", strlen("gt0-rc")))
7106	gt0_is_gt = true;
7107	else if (!strncmp(buf, "gt0-mc", strlen("gt0-mc")))
7108	gt0_is_gt = false;
7109	else
7110	goto next;
7111
7112	set_graphics_fp(path: "/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", idx: gt0_is_gt ? GFX_rc6 : SAM_mc6);
7113
7114	set_graphics_fp(path: "/sys/class/drm/card0/device/tile0/gt0/freq0/cur_freq", idx: gt0_is_gt ? GFX_MHz : SAM_MHz);
7115
7116	set_graphics_fp(path: "/sys/class/drm/card0/device/tile0/gt0/freq0/act_freq", idx: gt0_is_gt ? GFX_ACTMHz : SAM_ACTMHz);
7117
7118	set_graphics_fp(path: "/sys/class/drm/card0/device/tile0/gt1/gtidle/idle_residency_ms", idx: gt0_is_gt ? SAM_mc6 : GFX_rc6);
7119
7120	set_graphics_fp(path: "/sys/class/drm/card0/device/tile0/gt1/freq0/cur_freq", idx: gt0_is_gt ? SAM_MHz : GFX_MHz);
7121
7122	set_graphics_fp(path: "/sys/class/drm/card0/device/tile0/gt1/freq0/act_freq", idx: gt0_is_gt ? SAM_ACTMHz : GFX_ACTMHz);
7123
7124	goto end;
7125	}
7126
7127	next:
7128	/* New i915 graphics sysfs knobs */
7129	for (i = 0; i < GFX_MAX_CARDS; i++) {
7130	snprintf(buf: path, PATH_MAX, fmt: "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i);
7131	if (!access(path, R_OK))
7132	break;
7133	}
7134
7135	if (i == GFX_MAX_CARDS)
7136	goto legacy_i915;
7137
7138	snprintf(buf: path, PATH_MAX, fmt: "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i);
7139	set_graphics_fp(path, idx: GFX_rc6);
7140
7141	snprintf(buf: path, PATH_MAX, fmt: "/sys/class/drm/card%d/gt/gt0/rps_cur_freq_mhz", i);
7142	set_graphics_fp(path, idx: GFX_MHz);
7143
7144	snprintf(buf: path, PATH_MAX, fmt: "/sys/class/drm/card%d/gt/gt0/rps_act_freq_mhz", i);
7145	set_graphics_fp(path, idx: GFX_ACTMHz);
7146
7147	snprintf(buf: path, PATH_MAX, fmt: "/sys/class/drm/card%d/gt/gt1/rc6_residency_ms", i);
7148	set_graphics_fp(path, idx: SAM_mc6);
7149
7150	snprintf(buf: path, PATH_MAX, fmt: "/sys/class/drm/card%d/gt/gt1/rps_cur_freq_mhz", i);
7151	set_graphics_fp(path, idx: SAM_MHz);
7152
7153	snprintf(buf: path, PATH_MAX, fmt: "/sys/class/drm/card%d/gt/gt1/rps_act_freq_mhz", i);
7154	set_graphics_fp(path, idx: SAM_ACTMHz);
7155
7156	goto end;
7157
7158	legacy_i915:
7159	/* Fall back to traditional i915 graphics sysfs knobs */
7160	set_graphics_fp(path: "/sys/class/drm/card0/power/rc6_residency_ms", idx: GFX_rc6);
7161
7162	set_graphics_fp(path: "/sys/class/drm/card0/gt_cur_freq_mhz", idx: GFX_MHz);
7163	if (!gfx_info[GFX_MHz].fp)
7164	set_graphics_fp(path: "/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", idx: GFX_MHz);
7165
7166	set_graphics_fp(path: "/sys/class/drm/card0/gt_act_freq_mhz", idx: GFX_ACTMHz);
7167	if (!gfx_info[GFX_ACTMHz].fp)
7168	set_graphics_fp(path: "/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz", idx: GFX_ACTMHz);
7169
7170	end:
7171	if (gfx_info[GFX_rc6].fp)
7172	BIC_PRESENT(BIC_GFX_rc6);
7173	if (gfx_info[GFX_MHz].fp)
7174	BIC_PRESENT(BIC_GFXMHz);
7175	if (gfx_info[GFX_ACTMHz].fp)
7176	BIC_PRESENT(BIC_GFXACTMHz);
7177	if (gfx_info[SAM_mc6].fp)
7178	BIC_PRESENT(BIC_SAM_mc6);
7179	if (gfx_info[SAM_MHz].fp)
7180	BIC_PRESENT(BIC_SAMMHz);
7181	if (gfx_info[SAM_ACTMHz].fp)
7182	BIC_PRESENT(BIC_SAMACTMHz);
7183	}
7184
7185	static void dump_sysfs_cstate_config(void)
7186	{
7187	char path[64];
7188	char name_buf[16];
7189	char desc[64];
7190	FILE *input;
7191	int state;
7192	char *sp;
7193
7194	if (access("/sys/devices/system/cpu/cpuidle", R_OK)) {
7195	fprintf(outf, "cpuidle not loaded\n");
7196	return;
7197	}
7198
7199	dump_sysfs_file(path: "/sys/devices/system/cpu/cpuidle/current_driver");
7200	dump_sysfs_file(path: "/sys/devices/system/cpu/cpuidle/current_governor");
7201	dump_sysfs_file(path: "/sys/devices/system/cpu/cpuidle/current_governor_ro");
7202
7203	for (state = 0; state < 10; ++state) {
7204
7205	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state);
7206	input = fopen(path, "r");
7207	if (input == NULL)
7208	continue;
7209	if (!fgets(name_buf, sizeof(name_buf), input))
7210	err(1, "%s: failed to read file", path);
7211
7212	/* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
7213	sp = strchr(name_buf, '-');
7214	if (!sp)
7215	sp = strchrnul(name_buf, '\n');
7216	*sp = '\0';
7217	fclose(input);
7218
7219	remove_underbar(s: name_buf);
7220
7221	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", base_cpu, state);
7222	input = fopen(path, "r");
7223	if (input == NULL)
7224	continue;
7225	if (!fgets(desc, sizeof(desc), input))
7226	err(1, "%s: failed to read file", path);
7227
7228	fprintf(outf, "cpu%d: %s: %s", base_cpu, name_buf, desc);
7229	fclose(input);
7230	}
7231	}
7232
7233	static void dump_sysfs_pstate_config(void)
7234	{
7235	char path[64];
7236	char driver_buf[64];
7237	char governor_buf[64];
7238	FILE *input;
7239	int turbo;
7240
7241	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", base_cpu);
7242	input = fopen(path, "r");
7243	if (input == NULL) {
7244	fprintf(outf, "NSFOD %s\n", path);
7245	return;
7246	}
7247	if (!fgets(driver_buf, sizeof(driver_buf), input))
7248	err(1, "%s: failed to read file", path);
7249	fclose(input);
7250
7251	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", base_cpu);
7252	input = fopen(path, "r");
7253	if (input == NULL) {
7254	fprintf(outf, "NSFOD %s\n", path);
7255	return;
7256	}
7257	if (!fgets(governor_buf, sizeof(governor_buf), input))
7258	err(1, "%s: failed to read file", path);
7259	fclose(input);
7260
7261	fprintf(outf, "cpu%d: cpufreq driver: %s", base_cpu, driver_buf);
7262	fprintf(outf, "cpu%d: cpufreq governor: %s", base_cpu, governor_buf);
7263
7264	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpufreq/boost");
7265	input = fopen(path, "r");
7266	if (input != NULL) {
7267	if (fscanf(input, "%d", &turbo) != 1)
7268	err(1, "%s: failed to parse number from file", path);
7269	fprintf(outf, "cpufreq boost: %d\n", turbo);
7270	fclose(input);
7271	}
7272
7273	sprintf(buf: path, fmt: "/sys/devices/system/cpu/intel_pstate/no_turbo");
7274	input = fopen(path, "r");
7275	if (input != NULL) {
7276	if (fscanf(input, "%d", &turbo) != 1)
7277	err(1, "%s: failed to parse number from file", path);
7278	fprintf(outf, "cpufreq intel_pstate no_turbo: %d\n", turbo);
7279	fclose(input);
7280	}
7281	}
7282
7283	/*
7284	* print_epb()
7285	* Decode the ENERGY_PERF_BIAS MSR
7286	*/
7287	int print_epb(PER_THREAD_PARAMS)
7288	{
7289	char *epb_string;
7290	int cpu, epb;
7291
7292	UNUSED(c);
7293	UNUSED(p);
7294
7295	if (!has_epb)
7296	return 0;
7297
7298	cpu = t->cpu_id;
7299
7300	/* EPB is per-package */
7301	if (!is_cpu_first_thread_in_package(t, c, p))
7302	return 0;
7303
7304	if (cpu_migrate(cpu)) {
7305	fprintf(outf, "print_epb: Could not migrate to CPU %d\n", cpu);
7306	return -1;
7307	}
7308
7309	epb = get_epb(cpu);
7310	if (epb < 0)
7311	return 0;
7312
7313	switch (epb) {
7314	case ENERGY_PERF_BIAS_PERFORMANCE:
7315	epb_string = "performance";
7316	break;
7317	case ENERGY_PERF_BIAS_NORMAL:
7318	epb_string = "balanced";
7319	break;
7320	case ENERGY_PERF_BIAS_POWERSAVE:
7321	epb_string = "powersave";
7322	break;
7323	default:
7324	epb_string = "custom";
7325	break;
7326	}
7327	fprintf(outf, "cpu%d: EPB: %d (%s)\n", cpu, epb, epb_string);
7328
7329	return 0;
7330	}
7331
7332	/*
7333	* print_hwp()
7334	* Decode the MSR_HWP_CAPABILITIES
7335	*/
7336	int print_hwp(PER_THREAD_PARAMS)
7337	{
7338	unsigned long long msr;
7339	int cpu;
7340
7341	UNUSED(c);
7342	UNUSED(p);
7343
7344	if (no_msr)
7345	return 0;
7346
7347	if (!has_hwp)
7348	return 0;
7349
7350	cpu = t->cpu_id;
7351
7352	/* MSR_HWP_CAPABILITIES is per-package */
7353	if (!is_cpu_first_thread_in_package(t, c, p))
7354	return 0;
7355
7356	if (cpu_migrate(cpu)) {
7357	fprintf(outf, "print_hwp: Could not migrate to CPU %d\n", cpu);
7358	return -1;
7359	}
7360
7361	if (get_msr(cpu, MSR_PM_ENABLE, msr: &msr))
7362	return 0;
7363
7364	fprintf(outf, "cpu%d: MSR_PM_ENABLE: 0x%08llx (%sHWP)\n", cpu, msr, (msr & (1 << 0)) ? "" : "No-");
7365
7366	/* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */
7367	if ((msr & (1 << 0)) == 0)
7368	return 0;
7369
7370	if (get_msr(cpu, MSR_HWP_CAPABILITIES, msr: &msr))
7371	return 0;
7372
7373	fprintf(outf, "cpu%d: MSR_HWP_CAPABILITIES: 0x%08llx "
7374	"(high %d guar %d eff %d low %d)\n",
7375	cpu, msr,
7376	(unsigned int)HWP_HIGHEST_PERF(msr),
7377	(unsigned int)HWP_GUARANTEED_PERF(msr), (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr));
7378
7379	if (get_msr(cpu, MSR_HWP_REQUEST, msr: &msr))
7380	return 0;
7381
7382	fprintf(outf, "cpu%d: MSR_HWP_REQUEST: 0x%08llx "
7383	"(min %d max %d des %d epp 0x%x window 0x%x pkg 0x%x)\n",
7384	cpu, msr,
7385	(unsigned int)(((msr) >> 0) & 0xff),
7386	(unsigned int)(((msr) >> 8) & 0xff),
7387	(unsigned int)(((msr) >> 16) & 0xff),
7388	(unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1));
7389
7390	if (has_hwp_pkg) {
7391	if (get_msr(cpu, MSR_HWP_REQUEST_PKG, msr: &msr))
7392	return 0;
7393
7394	fprintf(outf, "cpu%d: MSR_HWP_REQUEST_PKG: 0x%08llx "
7395	"(min %d max %d des %d epp 0x%x window 0x%x)\n",
7396	cpu, msr,
7397	(unsigned int)(((msr) >> 0) & 0xff),
7398	(unsigned int)(((msr) >> 8) & 0xff),
7399	(unsigned int)(((msr) >> 16) & 0xff), (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3));
7400	}
7401	if (has_hwp_notify) {
7402	if (get_msr(cpu, MSR_HWP_INTERRUPT, msr: &msr))
7403	return 0;
7404
7405	fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx "
7406	"(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis");
7407	}
7408	if (get_msr(cpu, MSR_HWP_STATUS, msr: &msr))
7409	return 0;
7410
7411	fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx "
7412	"(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-");
7413
7414	return 0;
7415	}
7416
7417	/*
7418	* print_perf_limit()
7419	*/
7420	int print_perf_limit(PER_THREAD_PARAMS)
7421	{
7422	unsigned long long msr;
7423	int cpu;
7424
7425	UNUSED(c);
7426	UNUSED(p);
7427
7428	if (no_msr)
7429	return 0;
7430
7431	cpu = t->cpu_id;
7432
7433	/* per-package */
7434	if (!is_cpu_first_thread_in_package(t, c, p))
7435	return 0;
7436
7437	if (cpu_migrate(cpu)) {
7438	fprintf(outf, "print_perf_limit: Could not migrate to CPU %d\n", cpu);
7439	return -1;
7440	}
7441
7442	if (platform->plr_msrs & PLR_CORE) {
7443	get_msr(cpu, MSR_CORE_PERF_LIMIT_REASONS, msr: &msr);
7444	fprintf(outf, "cpu%d: MSR_CORE_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7445	fprintf(outf, " (Active: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)",
7446	(msr & 1 << 15) ? "bit15, " : "",
7447	(msr & 1 << 14) ? "bit14, " : "",
7448	(msr & 1 << 13) ? "Transitions, " : "",
7449	(msr & 1 << 12) ? "MultiCoreTurbo, " : "",
7450	(msr & 1 << 11) ? "PkgPwrL2, " : "",
7451	(msr & 1 << 10) ? "PkgPwrL1, " : "",
7452	(msr & 1 << 9) ? "CorePwr, " : "",
7453	(msr & 1 << 8) ? "Amps, " : "",
7454	(msr & 1 << 6) ? "VR-Therm, " : "",
7455	(msr & 1 << 5) ? "Auto-HWP, " : "",
7456	(msr & 1 << 4) ? "Graphics, " : "",
7457	(msr & 1 << 2) ? "bit2, " : "", (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : "");
7458	fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n",
7459	(msr & 1 << 31) ? "bit31, " : "",
7460	(msr & 1 << 30) ? "bit30, " : "",
7461	(msr & 1 << 29) ? "Transitions, " : "",
7462	(msr & 1 << 28) ? "MultiCoreTurbo, " : "",
7463	(msr & 1 << 27) ? "PkgPwrL2, " : "",
7464	(msr & 1 << 26) ? "PkgPwrL1, " : "",
7465	(msr & 1 << 25) ? "CorePwr, " : "",
7466	(msr & 1 << 24) ? "Amps, " : "",
7467	(msr & 1 << 22) ? "VR-Therm, " : "",
7468	(msr & 1 << 21) ? "Auto-HWP, " : "",
7469	(msr & 1 << 20) ? "Graphics, " : "",
7470	(msr & 1 << 18) ? "bit18, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : "");
7471
7472	}
7473	if (platform->plr_msrs & PLR_GFX) {
7474	get_msr(cpu, MSR_GFX_PERF_LIMIT_REASONS, msr: &msr);
7475	fprintf(outf, "cpu%d: MSR_GFX_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7476	fprintf(outf, " (Active: %s%s%s%s%s%s%s%s)",
7477	(msr & 1 << 0) ? "PROCHOT, " : "",
7478	(msr & 1 << 1) ? "ThermStatus, " : "",
7479	(msr & 1 << 4) ? "Graphics, " : "",
7480	(msr & 1 << 6) ? "VR-Therm, " : "",
7481	(msr & 1 << 8) ? "Amps, " : "",
7482	(msr & 1 << 9) ? "GFXPwr, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
7483	fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s)\n",
7484	(msr & 1 << 16) ? "PROCHOT, " : "",
7485	(msr & 1 << 17) ? "ThermStatus, " : "",
7486	(msr & 1 << 20) ? "Graphics, " : "",
7487	(msr & 1 << 22) ? "VR-Therm, " : "",
7488	(msr & 1 << 24) ? "Amps, " : "",
7489	(msr & 1 << 25) ? "GFXPwr, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
7490	}
7491	if (platform->plr_msrs & PLR_RING) {
7492	get_msr(cpu, MSR_RING_PERF_LIMIT_REASONS, msr: &msr);
7493	fprintf(outf, "cpu%d: MSR_RING_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7494	fprintf(outf, " (Active: %s%s%s%s%s%s)",
7495	(msr & 1 << 0) ? "PROCHOT, " : "",
7496	(msr & 1 << 1) ? "ThermStatus, " : "",
7497	(msr & 1 << 6) ? "VR-Therm, " : "",
7498	(msr & 1 << 8) ? "Amps, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
7499	fprintf(outf, " (Logged: %s%s%s%s%s%s)\n",
7500	(msr & 1 << 16) ? "PROCHOT, " : "",
7501	(msr & 1 << 17) ? "ThermStatus, " : "",
7502	(msr & 1 << 22) ? "VR-Therm, " : "",
7503	(msr & 1 << 24) ? "Amps, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
7504	}
7505	return 0;
7506	}
7507
7508	#define RAPL_POWER_GRANULARITY 0x7FFF /* 15 bit power granularity */
7509	#define RAPL_TIME_GRANULARITY 0x3F /* 6 bit time granularity */
7510
7511	double get_quirk_tdp(void)
7512	{
7513	if (platform->rapl_quirk_tdp)
7514	return platform->rapl_quirk_tdp;
7515
7516	return 135.0;
7517	}
7518
7519	double get_tdp_intel(void)
7520	{
7521	unsigned long long msr;
7522
7523	if (valid_rapl_msrs & RAPL_PKG_POWER_INFO)
7524	if (!get_msr(cpu: base_cpu, MSR_PKG_POWER_INFO, msr: &msr))
7525	return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units;
7526	return get_quirk_tdp();
7527	}
7528
7529	double get_tdp_amd(void)
7530	{
7531	return get_quirk_tdp();
7532	}
7533
7534	void rapl_probe_intel(void)
7535	{
7536	unsigned long long msr;
7537	unsigned int time_unit;
7538	double tdp;
7539
7540	if (rapl_joules) {
7541	CLR_BIC(BIC_SysWatt, &bic_enabled);
7542	CLR_BIC(BIC_PkgWatt, &bic_enabled);
7543	CLR_BIC(BIC_CorWatt, &bic_enabled);
7544	CLR_BIC(BIC_RAMWatt, &bic_enabled);
7545	CLR_BIC(BIC_GFXWatt, &bic_enabled);
7546	} else {
7547	CLR_BIC(BIC_Sys_J, &bic_enabled);
7548	CLR_BIC(BIC_Pkg_J, &bic_enabled);
7549	CLR_BIC(BIC_Cor_J, &bic_enabled);
7550	CLR_BIC(BIC_RAM_J, &bic_enabled);
7551	CLR_BIC(BIC_GFX_J, &bic_enabled);
7552	}
7553
7554	if (!valid_rapl_msrs || no_msr)
7555	return;
7556
7557	if (!(valid_rapl_msrs & RAPL_PKG_PERF_STATUS))
7558	CLR_BIC(BIC_PKG__, &bic_enabled);
7559	if (!(valid_rapl_msrs & RAPL_DRAM_PERF_STATUS))
7560	CLR_BIC(BIC_RAM__, &bic_enabled);
7561
7562	/* units on package 0, verify later other packages match */
7563	if (get_msr(cpu: base_cpu, MSR_RAPL_POWER_UNIT, msr: &msr))
7564	return;
7565
7566	rapl_power_units = 1.0 / (1 << (msr & 0xF));
7567	if (platform->has_rapl_divisor)
7568	rapl_energy_units = 1.0 * (1 << (msr >> 8 & 0x1F)) / 1000000;
7569	else
7570	rapl_energy_units = 1.0 / (1 << (msr >> 8 & 0x1F));
7571
7572	if (platform->has_fixed_rapl_unit)
7573	rapl_dram_energy_units = (15.3 / 1000000);
7574	else
7575	rapl_dram_energy_units = rapl_energy_units;
7576
7577	if (platform->has_fixed_rapl_psys_unit)
7578	rapl_psys_energy_units = 1.0;
7579	else
7580	rapl_psys_energy_units = rapl_energy_units;
7581
7582	time_unit = msr >> 16 & 0xF;
7583	if (time_unit == 0)
7584	time_unit = 0xA;
7585
7586	rapl_time_units = 1.0 / (1 << (time_unit));
7587
7588	tdp = get_tdp_intel();
7589
7590	rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7591	if (!quiet)
7592	fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7593	}
7594
7595	void rapl_probe_amd(void)
7596	{
7597	unsigned long long msr;
7598	double tdp;
7599
7600	if (rapl_joules) {
7601	CLR_BIC(BIC_SysWatt, &bic_enabled);
7602	CLR_BIC(BIC_CorWatt, &bic_enabled);
7603	} else {
7604	CLR_BIC(BIC_Pkg_J, &bic_enabled);
7605	CLR_BIC(BIC_Cor_J, &bic_enabled);
7606	}
7607
7608	if (!valid_rapl_msrs || no_msr)
7609	return;
7610
7611	if (get_msr(cpu: base_cpu, MSR_RAPL_PWR_UNIT, msr: &msr))
7612	return;
7613
7614	rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf));
7615	rapl_energy_units = ldexp(1.0, -(msr >> 8 & 0x1f));
7616	rapl_power_units = ldexp(1.0, -(msr & 0xf));
7617
7618	tdp = get_tdp_amd();
7619
7620	rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7621	if (!quiet)
7622	fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7623	}
7624
7625	void print_power_limit_msr(int cpu, unsigned long long msr, char *label)
7626	{
7627	fprintf(outf, "cpu%d: %s: %sabled (%0.3f Watts, %f sec, clamp %sabled)\n",
7628	cpu, label,
7629	((msr >> 15) & 1) ? "EN" : "DIS",
7630	((msr >> 0) & 0x7FFF) * rapl_power_units,
7631	(1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, (((msr >> 16) & 1) ? "EN" : "DIS"));
7632
7633	return;
7634	}
7635
7636	static int fread_int(char *path, int *val)
7637	{
7638	FILE *filep;
7639	int ret;
7640
7641	filep = fopen(path, "r");
7642	if (!filep)
7643	return -1;
7644
7645	ret = fscanf(filep, "%d", val);
7646	fclose(filep);
7647	return ret;
7648	}
7649
7650	static int fread_ull(char *path, unsigned long long *val)
7651	{
7652	FILE *filep;
7653	int ret;
7654
7655	filep = fopen(path, "r");
7656	if (!filep)
7657	return -1;
7658
7659	ret = fscanf(filep, "%llu", val);
7660	fclose(filep);
7661	return ret;
7662	}
7663
7664	static int fread_str(char *path, char *buf, int size)
7665	{
7666	FILE *filep;
7667	int ret;
7668	char *cp;
7669
7670	filep = fopen(path, "r");
7671	if (!filep)
7672	return -1;
7673
7674	ret = fread(buf, 1, size, filep);
7675	fclose(filep);
7676
7677	/* replace '\n' with '\0' */
7678	cp = strchr(buf, '\n');
7679	if (cp != NULL)
7680	*cp = '\0';
7681
7682	return ret;
7683	}
7684
7685	#define PATH_RAPL_SYSFS "/sys/class/powercap"
7686
7687	static int dump_one_domain(char *domain_path)
7688	{
7689	char path[PATH_MAX];
7690	char str[PATH_MAX];
7691	unsigned long long val;
7692	int constraint;
7693	int enable;
7694	int ret;
7695
7696	snprintf(buf: path, PATH_MAX, fmt: "%s/name", domain_path);
7697	ret = fread_str(path, buf: str, PATH_MAX);
7698	if (ret <= 0)
7699	return -1;
7700
7701	fprintf(outf, "%s: %s", domain_path + strlen(PATH_RAPL_SYSFS) + 1, str);
7702
7703	snprintf(buf: path, PATH_MAX, fmt: "%s/enabled", domain_path);
7704	ret = fread_int(path, val: &enable);
7705	if (ret <= 0)
7706	return -1;
7707
7708	if (!enable) {
7709	fputs(" disabled\n", outf);
7710	return 0;
7711	}
7712
7713	for (constraint = 0;; constraint++) {
7714	snprintf(buf: path, PATH_MAX, fmt: "%s/constraint_%d_time_window_us", domain_path, constraint);
7715	ret = fread_ull(path, val: &val);
7716	if (ret <= 0)
7717	break;
7718
7719	if (val > 1000000)
7720	fprintf(outf, " %0.1fs", (double)val / 1000000);
7721	else if (val > 1000)
7722	fprintf(outf, " %0.1fms", (double)val / 1000);
7723	else
7724	fprintf(outf, " %0.1fus", (double)val);
7725
7726	snprintf(buf: path, PATH_MAX, fmt: "%s/constraint_%d_power_limit_uw", domain_path, constraint);
7727	ret = fread_ull(path, val: &val);
7728	if (ret > 0 && val)
7729	fprintf(outf, ":%lluW", val / 1000000);
7730
7731	snprintf(buf: path, PATH_MAX, fmt: "%s/constraint_%d_max_power_uw", domain_path, constraint);
7732	ret = fread_ull(path, val: &val);
7733	if (ret > 0 && val)
7734	fprintf(outf, ",max:%lluW", val / 1000000);
7735	}
7736	fputc('\n', outf);
7737
7738	return 0;
7739	}
7740
7741	static int print_rapl_sysfs(void)
7742	{
7743	DIR *dir, *cdir;
7744	struct dirent *entry, *centry;
7745	char path[PATH_MAX];
7746	char str[PATH_MAX];
7747
7748	if ((dir = opendir(PATH_RAPL_SYSFS)) == NULL) {
7749	warn("open %s failed", PATH_RAPL_SYSFS);
7750	return 1;
7751	}
7752
7753	while ((entry = readdir(dir)) != NULL) {
7754	if (strlen(entry->d_name) > 100)
7755	continue;
7756
7757	if (strncmp(entry->d_name, "intel-rapl", strlen("intel-rapl")))
7758	continue;
7759
7760	snprintf(buf: path, PATH_MAX, fmt: "%s/%s/name", PATH_RAPL_SYSFS, entry->d_name);
7761
7762	/* Parse top level domains first, including package and psys */
7763	fread_str(path, buf: str, PATH_MAX);
7764	if (strncmp(str, "package", strlen("package")) && strncmp(str, "psys", strlen("psys")))
7765	continue;
7766
7767	snprintf(buf: path, PATH_MAX, fmt: "%s/%s", PATH_RAPL_SYSFS, entry->d_name);
7768	if ((cdir = opendir(path)) == NULL) {
7769	perror("opendir() error");
7770	return 1;
7771	}
7772
7773	dump_one_domain(domain_path: path);
7774
7775	while ((centry = readdir(cdir)) != NULL) {
7776	if (strncmp(centry->d_name, "intel-rapl", strlen("intel-rapl")))
7777	continue;
7778	snprintf(buf: path, PATH_MAX, fmt: "%s/%s/%s", PATH_RAPL_SYSFS, entry->d_name, centry->d_name);
7779	dump_one_domain(domain_path: path);
7780	}
7781	closedir(cdir);
7782	}
7783
7784	closedir(dir);
7785	return 0;
7786	}
7787
7788	int print_rapl(PER_THREAD_PARAMS)
7789	{
7790	unsigned long long msr;
7791	const char *msr_name;
7792	int cpu;
7793
7794	UNUSED(c);
7795	UNUSED(p);
7796
7797	if (!valid_rapl_msrs)
7798	return 0;
7799
7800	/* RAPL counters are per package, so print only for 1st thread/package */
7801	if (!is_cpu_first_thread_in_package(t, c, p))
7802	return 0;
7803
7804	cpu = t->cpu_id;
7805	if (cpu_migrate(cpu)) {
7806	fprintf(outf, "print_rapl: Could not migrate to CPU %d\n", cpu);
7807	return -1;
7808	}
7809
7810	if (valid_rapl_msrs & RAPL_AMD_F17H) {
7811	msr_name = "MSR_RAPL_PWR_UNIT";
7812	if (get_msr(cpu, MSR_RAPL_PWR_UNIT, msr: &msr))
7813	return -1;
7814	} else {
7815	msr_name = "MSR_RAPL_POWER_UNIT";
7816	if (get_msr(cpu, MSR_RAPL_POWER_UNIT, msr: &msr))
7817	return -1;
7818	}
7819
7820	fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr,
7821	rapl_power_units, rapl_energy_units, rapl_time_units);
7822
7823	if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) {
7824
7825	if (get_msr(cpu, MSR_PKG_POWER_INFO, msr: &msr))
7826	return -5;
7827
7828	fprintf(outf, "cpu%d: MSR_PKG_POWER_INFO: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
7829	cpu, msr,
7830	((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7831	((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7832	((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
7833
7834	}
7835	if (valid_rapl_msrs & RAPL_PKG) {
7836
7837	if (get_msr(cpu, MSR_PKG_POWER_LIMIT, msr: &msr))
7838	return -9;
7839
7840	fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 63) & 1 ? "" : "UN");
7841
7842	print_power_limit_msr(cpu, msr, label: "PKG Limit #1");
7843	fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%0.3f Watts, %f* sec, clamp %sabled)\n",
7844	cpu,
7845	((msr >> 47) & 1) ? "EN" : "DIS",
7846	((msr >> 32) & 0x7FFF) * rapl_power_units,
7847	(1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, ((msr >> 48) & 1) ? "EN" : "DIS");
7848
7849	if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, msr: &msr))
7850	return -9;
7851
7852	fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr);
7853	fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n",
7854	cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN");
7855	}
7856
7857	if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) {
7858	if (get_msr(cpu, MSR_DRAM_POWER_INFO, msr: &msr))
7859	return -6;
7860
7861	fprintf(outf, "cpu%d: MSR_DRAM_POWER_INFO,: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
7862	cpu, msr,
7863	((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7864	((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7865	((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
7866	}
7867	if (valid_rapl_msrs & RAPL_DRAM) {
7868	if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, msr: &msr))
7869	return -9;
7870	fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
7871
7872	print_power_limit_msr(cpu, msr, label: "DRAM Limit");
7873	}
7874	if (valid_rapl_msrs & RAPL_CORE_POLICY) {
7875	if (get_msr(cpu, MSR_PP0_POLICY, msr: &msr))
7876	return -7;
7877
7878	fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF);
7879	}
7880	if (valid_rapl_msrs & RAPL_CORE_POWER_LIMIT) {
7881	if (get_msr(cpu, MSR_PP0_POWER_LIMIT, msr: &msr))
7882	return -9;
7883	fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
7884	print_power_limit_msr(cpu, msr, label: "Cores Limit");
7885	}
7886	if (valid_rapl_msrs & RAPL_GFX) {
7887	if (get_msr(cpu, MSR_PP1_POLICY, msr: &msr))
7888	return -8;
7889
7890	fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF);
7891
7892	if (get_msr(cpu, MSR_PP1_POWER_LIMIT, msr: &msr))
7893	return -9;
7894	fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
7895	print_power_limit_msr(cpu, msr, label: "GFX Limit");
7896	}
7897	return 0;
7898	}
7899
7900	/*
7901	* probe_rapl_msrs
7902	*
7903	* initialize global valid_rapl_msrs to platform->plat_rapl_msrs
7904	* only if PKG_ENERGY counter is enumerated and reads non-zero
7905	*/
7906	void probe_rapl_msrs(void)
7907	{
7908	int ret;
7909	off_t offset;
7910	unsigned long long msr_value;
7911
7912	if (no_msr)
7913	return;
7914
7915	if ((platform->plat_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H)) == 0)
7916	return;
7917
7918	offset = idx_to_offset(idx: IDX_PKG_ENERGY);
7919	if (offset < 0)
7920	return;
7921
7922	ret = get_msr(cpu: base_cpu, offset, msr: &msr_value);
7923	if (ret) {
7924	if (debug)
7925	fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset);
7926	return;
7927	}
7928	if (msr_value == 0) {
7929	if (debug)
7930	fprintf(outf, "RAPL_PKG_ENERGY MSR(0x%llx) == ZERO: disabling all RAPL MSRs\n", (unsigned long long)offset);
7931	return;
7932	}
7933
7934	valid_rapl_msrs = platform->plat_rapl_msrs; /* success */
7935	}
7936
7937	/*
7938	* probe_rapl()
7939	*
7940	* sets rapl_power_units, rapl_energy_units, rapl_time_units
7941	*/
7942	void probe_rapl(void)
7943	{
7944	probe_rapl_msrs();
7945
7946	if (genuine_intel)
7947	rapl_probe_intel();
7948	if (authentic_amd || hygon_genuine)
7949	rapl_probe_amd();
7950
7951	if (quiet)
7952	return;
7953
7954	print_rapl_sysfs();
7955
7956	if (!valid_rapl_msrs || no_msr)
7957	return;
7958
7959	for_all_cpus(func: print_rapl, ODD_COUNTERS);
7960	}
7961
7962	/*
7963	* MSR_IA32_TEMPERATURE_TARGET indicates the temperature where
7964	* the Thermal Control Circuit (TCC) activates.
7965	* This is usually equal to tjMax.
7966	*
7967	* Older processors do not have this MSR, so there we guess,
7968	* but also allow cmdline over-ride with -T.
7969	*
7970	* Several MSR temperature values are in units of degrees-C
7971	* below this value, including the Digital Thermal Sensor (DTS),
7972	* Package Thermal Management Sensor (PTM), and thermal event thresholds.
7973	*/
7974	int set_temperature_target(PER_THREAD_PARAMS)
7975	{
7976	unsigned long long msr;
7977	unsigned int tcc_default, tcc_offset;
7978	int cpu;
7979
7980	UNUSED(c);
7981	UNUSED(p);
7982
7983	/* tj_max is used only for dts or ptm */
7984	if (!(do_dts || do_ptm))
7985	return 0;
7986
7987	/* this is a per-package concept */
7988	if (!is_cpu_first_thread_in_package(t, c, p))
7989	return 0;
7990
7991	cpu = t->cpu_id;
7992	if (cpu_migrate(cpu)) {
7993	fprintf(outf, "Could not migrate to CPU %d\n", cpu);
7994	return -1;
7995	}
7996
7997	if (tj_max_override != 0) {
7998	tj_max = tj_max_override;
7999	fprintf(outf, "cpu%d: Using cmdline TCC Target (%d C)\n", cpu, tj_max);
8000	return 0;
8001	}
8002
8003	/* Temperature Target MSR is Nehalem and newer only */
8004	if (!platform->has_nhm_msrs || no_msr)
8005	goto guess;
8006
8007	if (get_msr(cpu: base_cpu, MSR_IA32_TEMPERATURE_TARGET, msr: &msr))
8008	goto guess;
8009
8010	tcc_default = (msr >> 16) & 0xFF;
8011
8012	if (!quiet) {
8013	int bits = platform->tcc_offset_bits;
8014	unsigned long long enabled = 0;
8015
8016	if (bits && !get_msr(cpu: base_cpu, MSR_PLATFORM_INFO, msr: &enabled))
8017	enabled = (enabled >> 30) & 1;
8018
8019	if (bits && enabled) {
8020	tcc_offset = (msr >> 24) & GENMASK(bits - 1, 0);
8021	fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C) (%d default - %d offset)\n",
8022	cpu, msr, tcc_default - tcc_offset, tcc_default, tcc_offset);
8023	} else {
8024	fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C)\n", cpu, msr, tcc_default);
8025	}
8026	}
8027
8028	if (!tcc_default)
8029	goto guess;
8030
8031	tj_max = tcc_default;
8032
8033	return 0;
8034
8035	guess:
8036	tj_max = TJMAX_DEFAULT;
8037	fprintf(outf, "cpu%d: Guessing tjMax %d C, Please use -T to specify\n", cpu, tj_max);
8038
8039	return 0;
8040	}
8041
8042	int print_thermal(PER_THREAD_PARAMS)
8043	{
8044	unsigned long long msr;
8045	unsigned int dts, dts2;
8046	int cpu;
8047
8048	UNUSED(c);
8049	UNUSED(p);
8050
8051	if (no_msr)
8052	return 0;
8053
8054	if (!(do_dts || do_ptm))
8055	return 0;
8056
8057	cpu = t->cpu_id;
8058
8059	/* DTS is per-core, no need to print for each thread */
8060	if (!is_cpu_first_thread_in_core(t, c))
8061	return 0;
8062
8063	if (cpu_migrate(cpu)) {
8064	fprintf(outf, "print_thermal: Could not migrate to CPU %d\n", cpu);
8065	return -1;
8066	}
8067
8068	if (do_ptm && is_cpu_first_core_in_package(t, p)) {
8069	if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, msr: &msr))
8070	return 0;
8071
8072	dts = (msr >> 16) & 0x7F;
8073	fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_STATUS: 0x%08llx (%d C)\n", cpu, msr, tj_max - dts);
8074
8075	if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, msr: &msr))
8076	return 0;
8077
8078	dts = (msr >> 16) & 0x7F;
8079	dts2 = (msr >> 8) & 0x7F;
8080	fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
8081	}
8082
8083	if (do_dts && debug) {
8084	unsigned int resolution;
8085
8086	if (get_msr(cpu, MSR_IA32_THERM_STATUS, msr: &msr))
8087	return 0;
8088
8089	dts = (msr >> 16) & 0x7F;
8090	resolution = (msr >> 27) & 0xF;
8091	fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", cpu, msr, tj_max - dts, resolution);
8092
8093	if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, msr: &msr))
8094	return 0;
8095
8096	dts = (msr >> 16) & 0x7F;
8097	dts2 = (msr >> 8) & 0x7F;
8098	fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
8099	}
8100
8101	return 0;
8102	}
8103
8104	void probe_thermal(void)
8105	{
8106	if (!access("/sys/devices/system/cpu/cpu0/thermal_throttle/core_throttle_count", R_OK))
8107	BIC_PRESENT(BIC_CORE_THROT_CNT);
8108	else
8109	BIC_NOT_PRESENT(BIC_CORE_THROT_CNT);
8110
8111	for_all_cpus(func: set_temperature_target, ODD_COUNTERS);
8112
8113	if (quiet)
8114	return;
8115
8116	for_all_cpus(func: print_thermal, ODD_COUNTERS);
8117	}
8118
8119	int get_cpu_type(PER_THREAD_PARAMS)
8120	{
8121	unsigned int eax, ebx, ecx, edx;
8122
8123	UNUSED(c);
8124	UNUSED(p);
8125
8126	if (!genuine_intel)
8127	return 0;
8128
8129	if (cpu_migrate(cpu: t->cpu_id)) {
8130	fprintf(outf, "Could not migrate to CPU %d\n", t->cpu_id);
8131	return -1;
8132	}
8133
8134	if (max_level < 0x1a)
8135	return 0;
8136
8137	__cpuid(0x1a, eax, ebx, ecx, edx);
8138	eax = (eax >> 24) & 0xFF;
8139	if (eax == 0x20)
8140	t->is_atom = true;
8141	return 0;
8142	}
8143
8144	void decode_feature_control_msr(void)
8145	{
8146	unsigned long long msr;
8147
8148	if (no_msr)
8149	return;
8150
8151	if (!get_msr(cpu: base_cpu, MSR_IA32_FEAT_CTL, msr: &msr))
8152	fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n",
8153	base_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : "");
8154	}
8155
8156	void decode_misc_enable_msr(void)
8157	{
8158	unsigned long long msr;
8159
8160	if (no_msr)
8161	return;
8162
8163	if (!genuine_intel)
8164	return;
8165
8166	if (!get_msr(cpu: base_cpu, MSR_IA32_MISC_ENABLE, msr: &msr))
8167	fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n",
8168	base_cpu, msr,
8169	msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-",
8170	msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-",
8171	msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-",
8172	msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : "");
8173	}
8174
8175	void decode_misc_feature_control(void)
8176	{
8177	unsigned long long msr;
8178
8179	if (no_msr)
8180	return;
8181
8182	if (!platform->has_msr_misc_feature_control)
8183	return;
8184
8185	if (!get_msr(cpu: base_cpu, MSR_MISC_FEATURE_CONTROL, msr: &msr))
8186	fprintf(outf,
8187	"cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n",
8188	base_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "",
8189	msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : "");
8190	}
8191
8192	/*
8193	* Decode MSR_MISC_PWR_MGMT
8194	*
8195	* Decode the bits according to the Nehalem documentation
8196	* bit[0] seems to continue to have same meaning going forward
8197	* bit[1] less so...
8198	*/
8199	void decode_misc_pwr_mgmt_msr(void)
8200	{
8201	unsigned long long msr;
8202
8203	if (no_msr)
8204	return;
8205
8206	if (!platform->has_msr_misc_pwr_mgmt)
8207	return;
8208
8209	if (!get_msr(cpu: base_cpu, MSR_MISC_PWR_MGMT, msr: &msr))
8210	fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n",
8211	base_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS");
8212	}
8213
8214	/*
8215	* Decode MSR_CC6_DEMOTION_POLICY_CONFIG, MSR_MC6_DEMOTION_POLICY_CONFIG
8216	*
8217	* This MSRs are present on Silvermont processors,
8218	* Intel Atom processor E3000 series (Baytrail), and friends.
8219	*/
8220	void decode_c6_demotion_policy_msr(void)
8221	{
8222	unsigned long long msr;
8223
8224	if (no_msr)
8225	return;
8226
8227	if (!platform->has_msr_c6_demotion_policy_config)
8228	return;
8229
8230	if (!get_msr(cpu: base_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, msr: &msr))
8231	fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n",
8232	base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
8233
8234	if (!get_msr(cpu: base_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, msr: &msr))
8235	fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n",
8236	base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
8237	}
8238
8239	void print_dev_latency(void)
8240	{
8241	char *path = "/dev/cpu_dma_latency";
8242	int fd;
8243	int value;
8244	int retval;
8245
8246	fd = open(path, O_RDONLY);
8247	if (fd < 0) {
8248	if (debug)
8249	warnx("Read %s failed", path);
8250	return;
8251	}
8252
8253	retval = read(fd, (void *)&value, sizeof(int));
8254	if (retval != sizeof(int)) {
8255	warn("read failed %s", path);
8256	close(fd);
8257	return;
8258	}
8259	fprintf(outf, "/dev/cpu_dma_latency: %d usec (%s)\n", value, value == 2000000000 ? "default" : "constrained");
8260
8261	close(fd);
8262	}
8263
8264	static int has_perf_instr_count_access(void)
8265	{
8266	int fd;
8267
8268	if (no_perf)
8269	return 0;
8270
8271	fd = open_perf_counter(cpu: base_cpu, type: PERF_TYPE_HARDWARE, config: PERF_COUNT_HW_INSTRUCTIONS, group_fd: -1, read_format: 0);
8272	if (fd != -1)
8273	close(fd);
8274
8275	if (fd == -1)
8276	warnx("Failed to access %s. Some of the counters may not be available\n"
8277	"\tRun as root to enable them or use %s to disable the access explicitly", "perf instructions retired counter",
8278	"'--hide IPC' or '--no-perf'");
8279
8280	return (fd != -1);
8281	}
8282
8283	int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, double *scale_, enum rapl_unit *unit_)
8284	{
8285	int ret = -1;
8286
8287	if (no_perf)
8288	return -1;
8289
8290	if (!cai->perf_name)
8291	return -1;
8292
8293	const double scale = read_perf_scale(subsys: cai->perf_subsys, event_name: cai->perf_name);
8294
8295	if (scale == 0.0)
8296	goto end;
8297
8298	const enum rapl_unit unit = read_perf_rapl_unit(subsys: cai->perf_subsys, event_name: cai->perf_name);
8299
8300	if (unit == RAPL_UNIT_INVALID)
8301	goto end;
8302
8303	const unsigned int rapl_type = read_perf_type(subsys: cai->perf_subsys);
8304	const unsigned int rapl_energy_pkg_config = read_perf_config(subsys: cai->perf_subsys, event_name: cai->perf_name);
8305
8306	ret = open_perf_counter(cpu, type: rapl_type, config: rapl_energy_pkg_config, group_fd: rci->fd_perf, read_format: PERF_FORMAT_GROUP);
8307	if (ret == -1)
8308	goto end;
8309
8310	/* If it's the first counter opened, make it a group descriptor */
8311	if (rci->fd_perf == -1)
8312	rci->fd_perf = ret;
8313
8314	*scale_ = scale;
8315	*unit_ = unit;
8316
8317	end:
8318	if (debug >= 2)
8319	fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
8320
8321	return ret;
8322	}
8323
8324	/*
8325	* Linux-perf manages the HW instructions-retired counter
8326	* by enabling when requested, and hiding rollover
8327	*/
8328	void linux_perf_init(void)
8329	{
8330	if (access("/proc/sys/kernel/perf_event_paranoid", F_OK))
8331	return;
8332
8333	if (BIC_IS_ENABLED(BIC_IPC) && cpuid_has_aperf_mperf) {
8334	fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8335	if (fd_instr_count_percpu == NULL)
8336	err(-1, "calloc fd_instr_count_percpu");
8337	}
8338	if (BIC_IS_ENABLED(BIC_LLC_RPS)) {
8339	fd_llc_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8340	if (fd_llc_percpu == NULL)
8341	err(-1, "calloc fd_llc_percpu");
8342	}
8343	}
8344
8345	void rapl_perf_init(void)
8346	{
8347	const unsigned int num_domains = get_rapl_num_domains();
8348	bool *domain_visited = calloc(num_domains, sizeof(bool));
8349
8350	rapl_counter_info_perdomain = calloc(num_domains, sizeof(*rapl_counter_info_perdomain));
8351	if (rapl_counter_info_perdomain == NULL)
8352	err(-1, "calloc rapl_counter_info_percpu");
8353	rapl_counter_info_perdomain_size = num_domains;
8354
8355	/*
8356	* Initialize rapl_counter_info_percpu
8357	*/
8358	for (unsigned int domain_id = 0; domain_id < num_domains; ++domain_id) {
8359	struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[domain_id];
8360
8361	rci->fd_perf = -1;
8362	for (size_t i = 0; i < NUM_RAPL_COUNTERS; ++i) {
8363	rci->data[i] = 0;
8364	rci->source[i] = COUNTER_SOURCE_NONE;
8365	}
8366	}
8367
8368	/*
8369	* Open/probe the counters
8370	* If can't get it via perf, fallback to MSR
8371	*/
8372	for (size_t i = 0; i < ARRAY_SIZE(rapl_counter_arch_infos); ++i) {
8373
8374	const struct rapl_counter_arch_info *const cai = &rapl_counter_arch_infos[i];
8375	bool has_counter = 0;
8376	double scale;
8377	enum rapl_unit unit;
8378	unsigned int next_domain;
8379
8380	if (!BIC_IS_ENABLED(cai->bic_number))
8381	continue;
8382
8383	memset(domain_visited, 0, num_domains * sizeof(*domain_visited));
8384
8385	for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
8386
8387	if (cpu_is_not_allowed(cpu))
8388	continue;
8389
8390	/* Skip already seen and handled RAPL domains */
8391	next_domain = get_rapl_domain_id(cpu);
8392
8393	assert(next_domain < num_domains);
8394
8395	if (domain_visited[next_domain])
8396	continue;
8397
8398	domain_visited[next_domain] = 1;
8399
8400	if ((cai->flags & RAPL_COUNTER_FLAG_PLATFORM_COUNTER) && (cpu != base_cpu))
8401	continue;
8402
8403	struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[next_domain];
8404
8405	/*
8406	* rapl_counter_arch_infos[] can have multiple entries describing the same
8407	* counter, due to the difference from different platforms/Vendors.
8408	* E.g. rapl_counter_arch_infos[0] and rapl_counter_arch_infos[1] share the
8409	* same perf_subsys and perf_name, but with different MSR address.
8410	* rapl_counter_arch_infos[0] is for Intel and rapl_counter_arch_infos[1]
8411	* is for AMD.
8412	* In this case, it is possible that multiple rapl_counter_arch_infos[]
8413	* entries are probed just because their perf/msr is duplicate and valid.
8414	*
8415	* Thus need a check to avoid re-probe the same counters.
8416	*/
8417	if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE)
8418	break;
8419
8420	/* Use perf API for this counter */
8421	if (add_rapl_perf_counter(cpu, rci, cai, scale_: &scale, unit_: &unit) != -1) {
8422	rci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8423	rci->scale[cai->rci_index] = scale * cai->compat_scale;
8424	rci->unit[cai->rci_index] = unit;
8425	rci->flags[cai->rci_index] = cai->flags;
8426
8427	/* Use MSR for this counter */
8428	} else if (add_rapl_msr_counter(cpu, cai) >= 0) {
8429	rci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8430	rci->msr[cai->rci_index] = cai->msr;
8431	rci->msr_mask[cai->rci_index] = cai->msr_mask;
8432	rci->msr_shift[cai->rci_index] = cai->msr_shift;
8433	rci->unit[cai->rci_index] = RAPL_UNIT_JOULES;
8434	rci->scale[cai->rci_index] = *cai->platform_rapl_msr_scale * cai->compat_scale;
8435	rci->flags[cai->rci_index] = cai->flags;
8436	}
8437
8438	if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE)
8439	has_counter = 1;
8440	}
8441
8442	/* If any CPU has access to the counter, make it present */
8443	if (has_counter)
8444	BIC_PRESENT(cai->bic_number);
8445	}
8446
8447	free(domain_visited);
8448	}
8449
8450	/* Assumes msr_counter_info is populated */
8451	static int has_amperf_access(void)
8452	{
8453	return cpuid_has_aperf_mperf && msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present &&
8454	msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present;
8455	}
8456
8457	int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *group_name)
8458	{
8459	if (strcmp(group_name, "cstate_core") == 0)
8460	return &cci->fd_perf_core;
8461
8462	if (strcmp(group_name, "cstate_pkg") == 0)
8463	return &cci->fd_perf_pkg;
8464
8465	return NULL;
8466	}
8467
8468	int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai)
8469	{
8470	int ret = -1;
8471
8472	if (no_perf)
8473	return -1;
8474
8475	if (!cai->perf_name)
8476	return -1;
8477
8478	int *pfd_group = get_cstate_perf_group_fd(cci, group_name: cai->perf_subsys);
8479
8480	if (pfd_group == NULL)
8481	goto end;
8482
8483	const unsigned int type = read_perf_type(subsys: cai->perf_subsys);
8484	const unsigned int config = read_perf_config(subsys: cai->perf_subsys, event_name: cai->perf_name);
8485
8486	ret = open_perf_counter(cpu, type, config, group_fd: *pfd_group, read_format: PERF_FORMAT_GROUP);
8487
8488	if (ret == -1)
8489	goto end;
8490
8491	/* If it's the first counter opened, make it a group descriptor */
8492	if (*pfd_group == -1)
8493	*pfd_group = ret;
8494
8495	end:
8496	if (debug >= 2)
8497	fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
8498
8499	return ret;
8500	}
8501
8502	int add_msr_perf_counter(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai)
8503	{
8504	int ret = -1;
8505
8506	if (no_perf)
8507	return -1;
8508
8509	if (!cai->perf_name)
8510	return -1;
8511
8512	const unsigned int type = read_perf_type(subsys: cai->perf_subsys);
8513	const unsigned int config = read_perf_config(subsys: cai->perf_subsys, event_name: cai->perf_name);
8514
8515	ret = open_perf_counter(cpu, type, config, group_fd: cci->fd_perf, read_format: PERF_FORMAT_GROUP);
8516
8517	if (ret == -1)
8518	goto end;
8519
8520	/* If it's the first counter opened, make it a group descriptor */
8521	if (cci->fd_perf == -1)
8522	cci->fd_perf = ret;
8523
8524	end:
8525	if (debug)
8526	fprintf(stderr, "%s: %s/%s: %d (cpu: %d)\n", __func__, cai->perf_subsys, cai->perf_name, ret, cpu);
8527
8528	return ret;
8529	}
8530
8531	void msr_perf_init_(void)
8532	{
8533	const int mci_num = topo.max_cpu_num + 1;
8534
8535	msr_counter_info = calloc(mci_num, sizeof(*msr_counter_info));
8536	if (!msr_counter_info)
8537	err(1, "calloc msr_counter_info");
8538	msr_counter_info_size = mci_num;
8539
8540	for (int cpu = 0; cpu < mci_num; ++cpu)
8541	msr_counter_info[cpu].fd_perf = -1;
8542
8543	for (int cidx = 0; cidx < NUM_MSR_COUNTERS; ++cidx) {
8544
8545	struct msr_counter_arch_info *cai = &msr_counter_arch_infos[cidx];
8546
8547	cai->present = false;
8548
8549	for (int cpu = 0; cpu < mci_num; ++cpu) {
8550
8551	struct msr_counter_info_t *const cci = &msr_counter_info[cpu];
8552
8553	if (cpu_is_not_allowed(cpu))
8554	continue;
8555
8556	if (cai->needed) {
8557	/* Use perf API for this counter */
8558	if (add_msr_perf_counter(cpu, cci, cai) != -1) {
8559	cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8560	cai->present = true;
8561
8562	/* User MSR for this counter */
8563	} else if (add_msr_counter(cpu, offset: cai->msr) >= 0) {
8564	cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8565	cci->msr[cai->rci_index] = cai->msr;
8566	cci->msr_mask[cai->rci_index] = cai->msr_mask;
8567	cai->present = true;
8568	}
8569	}
8570	}
8571	}
8572	}
8573
8574	/* Initialize data for reading perf counters from the MSR group. */
8575	void msr_perf_init(void)
8576	{
8577	bool need_amperf = false, need_smi = false;
8578	const bool need_soft_c1 = (!platform->has_msr_core_c1_res) && (platform->supported_cstates & CC1);
8579
8580	need_amperf = BIC_IS_ENABLED(BIC_Avg_MHz) || BIC_IS_ENABLED(BIC_Busy) || BIC_IS_ENABLED(BIC_Bzy_MHz)
8581	|| BIC_IS_ENABLED(BIC_IPC) || need_soft_c1;
8582
8583	if (BIC_IS_ENABLED(BIC_SMI))
8584	need_smi = true;
8585
8586	/* Enable needed counters */
8587	msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].needed = need_amperf;
8588	msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].needed = need_amperf;
8589	msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].needed = need_smi;
8590
8591	msr_perf_init_();
8592
8593	const bool has_amperf = has_amperf_access();
8594	const bool has_smi = msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].present;
8595
8596	has_aperf_access = has_amperf;
8597
8598	if (has_amperf) {
8599	BIC_PRESENT(BIC_Avg_MHz);
8600	BIC_PRESENT(BIC_Busy);
8601	BIC_PRESENT(BIC_Bzy_MHz);
8602	BIC_PRESENT(BIC_SMI);
8603	}
8604
8605	if (has_smi)
8606	BIC_PRESENT(BIC_SMI);
8607	}
8608
8609	void cstate_perf_init_(bool soft_c1)
8610	{
8611	bool has_counter;
8612	bool *cores_visited = NULL, *pkg_visited = NULL;
8613	const int cores_visited_elems = topo.max_core_id + 1;
8614	const int pkg_visited_elems = topo.max_package_id + 1;
8615	const int cci_num = topo.max_cpu_num + 1;
8616
8617	ccstate_counter_info = calloc(cci_num, sizeof(*ccstate_counter_info));
8618	if (!ccstate_counter_info)
8619	err(1, "calloc ccstate_counter_arch_info");
8620	ccstate_counter_info_size = cci_num;
8621
8622	cores_visited = calloc(cores_visited_elems, sizeof(*cores_visited));
8623	if (!cores_visited)
8624	err(1, "calloc cores_visited");
8625
8626	pkg_visited = calloc(pkg_visited_elems, sizeof(*pkg_visited));
8627	if (!pkg_visited)
8628	err(1, "calloc pkg_visited");
8629
8630	/* Initialize cstate_counter_info_percpu */
8631	for (int cpu = 0; cpu < cci_num; ++cpu) {
8632	ccstate_counter_info[cpu].fd_perf_core = -1;
8633	ccstate_counter_info[cpu].fd_perf_pkg = -1;
8634	}
8635
8636	for (int cidx = 0; cidx < NUM_CSTATE_COUNTERS; ++cidx) {
8637	has_counter = false;
8638	memset(cores_visited, 0, cores_visited_elems * sizeof(*cores_visited));
8639	memset(pkg_visited, 0, pkg_visited_elems * sizeof(*pkg_visited));
8640
8641	const struct cstate_counter_arch_info *cai = &ccstate_counter_arch_infos[cidx];
8642
8643	for (int cpu = 0; cpu < cci_num; ++cpu) {
8644
8645	struct cstate_counter_info_t *const cci = &ccstate_counter_info[cpu];
8646
8647	if (cpu_is_not_allowed(cpu))
8648	continue;
8649
8650	const int core_id = cpus[cpu].physical_core_id;
8651	const int pkg_id = cpus[cpu].physical_package_id;
8652
8653	assert(core_id < cores_visited_elems);
8654	assert(pkg_id < pkg_visited_elems);
8655
8656	const bool per_thread = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD;
8657	const bool per_core = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_CORE;
8658
8659	if (!per_thread && cores_visited[core_id])
8660	continue;
8661
8662	if (!per_core && pkg_visited[pkg_id])
8663	continue;
8664
8665	const bool counter_needed = BIC_IS_ENABLED(cai->bic_number) ||
8666	(soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY));
8667	const bool counter_supported = (platform->supported_cstates & cai->feature_mask);
8668
8669	if (counter_needed && counter_supported) {
8670	/* Use perf API for this counter */
8671	if (add_cstate_perf_counter(cpu, cci, cai) != -1) {
8672
8673	cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8674
8675	/* User MSR for this counter */
8676	} else if (pkg_cstate_limit >= cai->pkg_cstate_limit && add_msr_counter(cpu, offset: cai->msr) >= 0) {
8677	cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8678	cci->msr[cai->rci_index] = cai->msr;
8679	}
8680	}
8681
8682	if (cci->source[cai->rci_index] != COUNTER_SOURCE_NONE) {
8683	has_counter = true;
8684	cores_visited[core_id] = true;
8685	pkg_visited[pkg_id] = true;
8686	}
8687	}
8688
8689	/* If any CPU has access to the counter, make it present */
8690	if (has_counter)
8691	BIC_PRESENT(cai->bic_number);
8692	}
8693
8694	free(cores_visited);
8695	free(pkg_visited);
8696	}
8697
8698	void cstate_perf_init(void)
8699	{
8700	/*
8701	* If we don't have a C1 residency MSR, we calculate it "in software",
8702	* but we need APERF, MPERF too.
8703	*/
8704	const bool soft_c1 = !platform->has_msr_core_c1_res && has_amperf_access()
8705	&& platform->supported_cstates & CC1;
8706
8707	if (soft_c1)
8708	BIC_PRESENT(BIC_CPU_c1);
8709
8710	cstate_perf_init_(soft_c1);
8711	}
8712
8713	void probe_cstates(void)
8714	{
8715	probe_cst_limit();
8716
8717	if (platform->has_msr_module_c6_res_ms)
8718	BIC_PRESENT(BIC_Mod_c6);
8719
8720	if (platform->has_ext_cst_msrs && !no_msr) {
8721	BIC_PRESENT(BIC_Totl_c0);
8722	BIC_PRESENT(BIC_Any_c0);
8723	BIC_PRESENT(BIC_GFX_c0);
8724	BIC_PRESENT(BIC_CPUGFX);
8725	}
8726
8727	if (quiet)
8728	return;
8729
8730	dump_power_ctl();
8731	dump_cst_cfg();
8732	decode_c6_demotion_policy_msr();
8733	print_dev_latency();
8734	dump_sysfs_cstate_config();
8735	print_irtl();
8736	}
8737
8738	void probe_lpi(void)
8739	{
8740	if (!access("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", R_OK))
8741	BIC_PRESENT(BIC_CPU_LPI);
8742	else
8743	BIC_NOT_PRESENT(BIC_CPU_LPI);
8744
8745	if (!access(sys_lpi_file_sysfs, R_OK)) {
8746	sys_lpi_file = sys_lpi_file_sysfs;
8747	BIC_PRESENT(BIC_SYS_LPI);
8748	} else if (!access(sys_lpi_file_debugfs, R_OK)) {
8749	sys_lpi_file = sys_lpi_file_debugfs;
8750	BIC_PRESENT(BIC_SYS_LPI);
8751	} else {
8752	sys_lpi_file_sysfs = NULL;
8753	BIC_NOT_PRESENT(BIC_SYS_LPI);
8754	}
8755
8756	}
8757
8758	void probe_pstates(void)
8759	{
8760	probe_bclk();
8761
8762	if (quiet)
8763	return;
8764
8765	dump_platform_info();
8766	dump_turbo_ratio_info();
8767	dump_sysfs_pstate_config();
8768	decode_misc_pwr_mgmt_msr();
8769
8770	for_all_cpus(func: print_hwp, ODD_COUNTERS);
8771	for_all_cpus(func: print_epb, ODD_COUNTERS);
8772	for_all_cpus(func: print_perf_limit, ODD_COUNTERS);
8773	}
8774
8775	void process_cpuid()
8776	{
8777	unsigned int eax, ebx, ecx, edx;
8778	unsigned int fms, family, model, stepping, ecx_flags, edx_flags;
8779	unsigned long long ucode_patch = 0;
8780	bool ucode_patch_valid = false;
8781
8782	eax = ebx = ecx = edx = 0;
8783
8784	__cpuid(0, max_level, ebx, ecx, edx);
8785
8786	if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
8787	genuine_intel = 1;
8788	else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
8789	authentic_amd = 1;
8790	else if (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e)
8791	hygon_genuine = 1;
8792
8793	if (!quiet)
8794	fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", (char *)&ebx, (char *)&edx, (char *)&ecx, max_level);
8795
8796	__cpuid(1, fms, ebx, ecx, edx);
8797	family = (fms >> 8) & 0xf;
8798	model = (fms >> 4) & 0xf;
8799	stepping = fms & 0xf;
8800	if (family == 0xf)
8801	family += (fms >> 20) & 0xff;
8802	if (family >= 6)
8803	model += ((fms >> 16) & 0xf) << 4;
8804	ecx_flags = ecx;
8805	edx_flags = edx;
8806
8807	if (!no_msr) {
8808	if (get_msr(cpu: sched_getcpu(), MSR_IA32_UCODE_REV, msr: &ucode_patch))
8809	warnx("get_msr(UCODE)");
8810	else
8811	ucode_patch_valid = true;
8812	}
8813
8814	/*
8815	* check max extended function levels of CPUID.
8816	* This is needed to check for invariant TSC.
8817	* This check is valid for both Intel and AMD.
8818	*/
8819	ebx = ecx = edx = 0;
8820	__cpuid(0x80000000, max_extended_level, ebx, ecx, edx);
8821
8822	if (!quiet) {
8823	fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", family, model, stepping, family, model, stepping);
8824	if (ucode_patch_valid)
8825	fprintf(outf, " microcode 0x%x", (unsigned int)((ucode_patch >> 32) & 0xFFFFFFFF));
8826	fputc('\n', outf);
8827
8828	fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level);
8829	fprintf(outf, "CPUID(1): %s %s %s %s %s %s %s %s %s %s\n",
8830	ecx_flags & (1 << 0) ? "SSE3" : "-",
8831	ecx_flags & (1 << 3) ? "MONITOR" : "-",
8832	ecx_flags & (1 << 6) ? "SMX" : "-",
8833	ecx_flags & (1 << 7) ? "EIST" : "-",
8834	ecx_flags & (1 << 8) ? "TM2" : "-",
8835	edx_flags & (1 << 4) ? "TSC" : "-",
8836	edx_flags & (1 << 5) ? "MSR" : "-",
8837	edx_flags & (1 << 22) ? "ACPI-TM" : "-", edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-");
8838	}
8839
8840	probe_platform_features(family, model);
8841
8842	if (!(edx_flags & (1 << 5)))
8843	errx(1, "CPUID: no MSR");
8844
8845	if (max_extended_level >= 0x80000007) {
8846
8847	/*
8848	* Non-Stop TSC is advertised by CPUID.EAX=0x80000007: EDX.bit8
8849	* this check is valid for both Intel and AMD
8850	*/
8851	__cpuid(0x80000007, eax, ebx, ecx, edx);
8852	has_invariant_tsc = edx & (1 << 8);
8853	}
8854
8855	/*
8856	* APERF/MPERF is advertised by CPUID.EAX=0x6: ECX.bit0
8857	* this check is valid for both Intel and AMD
8858	*/
8859
8860	__cpuid(0x6, eax, ebx, ecx, edx);
8861	cpuid_has_aperf_mperf = ecx & (1 << 0);
8862	do_dts = eax & (1 << 0);
8863	if (do_dts)
8864	BIC_PRESENT(BIC_CoreTmp);
8865	has_turbo = eax & (1 << 1);
8866	do_ptm = eax & (1 << 6);
8867	if (do_ptm)
8868	BIC_PRESENT(BIC_PkgTmp);
8869	has_hwp = eax & (1 << 7);
8870	has_hwp_notify = eax & (1 << 8);
8871	has_hwp_activity_window = eax & (1 << 9);
8872	has_hwp_epp = eax & (1 << 10);
8873	has_hwp_pkg = eax & (1 << 11);
8874	has_epb = ecx & (1 << 3);
8875
8876	if (!quiet)
8877	fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, "
8878	"%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n",
8879	cpuid_has_aperf_mperf ? "" : "No-",
8880	has_turbo ? "" : "No-",
8881	do_dts ? "" : "No-",
8882	do_ptm ? "" : "No-",
8883	has_hwp ? "" : "No-",
8884	has_hwp_notify ? "" : "No-",
8885	has_hwp_activity_window ? "" : "No-", has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-");
8886
8887	if (!quiet)
8888	decode_misc_enable_msr();
8889
8890	if (max_level >= 0x7 && !quiet) {
8891	int has_sgx;
8892
8893	ecx = 0;
8894
8895	__cpuid_count(0x7, 0, eax, ebx, ecx, edx);
8896
8897	has_sgx = ebx & (1 << 2);
8898
8899	is_hybrid = edx & (1 << 15);
8900
8901	fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-");
8902
8903	if (has_sgx)
8904	decode_feature_control_msr();
8905	}
8906
8907	if (max_level >= 0x15) {
8908	unsigned int eax_crystal;
8909	unsigned int ebx_tsc;
8910
8911	/*
8912	* CPUID 15H TSC/Crystal ratio, possibly Crystal Hz
8913	*/
8914	eax_crystal = ebx_tsc = crystal_hz = edx = 0;
8915	__cpuid(0x15, eax_crystal, ebx_tsc, crystal_hz, edx);
8916
8917	if (ebx_tsc != 0) {
8918	if (!quiet && (ebx != 0))
8919	fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", eax_crystal, ebx_tsc, crystal_hz);
8920
8921	if (crystal_hz == 0)
8922	crystal_hz = platform->crystal_freq;
8923
8924	if (crystal_hz) {
8925	tsc_hz = (unsigned long long)crystal_hz *ebx_tsc / eax_crystal;
8926	if (!quiet)
8927	fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n",
8928	tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal);
8929	}
8930	}
8931	}
8932	if (max_level >= 0x16) {
8933	unsigned int base_mhz, max_mhz, bus_mhz, edx;
8934
8935	/*
8936	* CPUID 16H Base MHz, Max MHz, Bus MHz
8937	*/
8938	base_mhz = max_mhz = bus_mhz = edx = 0;
8939
8940	__cpuid(0x16, base_mhz, max_mhz, bus_mhz, edx);
8941
8942	bclk = bus_mhz;
8943
8944	base_hz = base_mhz * 1000000;
8945	has_base_hz = 1;
8946
8947	if (platform->enable_tsc_tweak)
8948	tsc_tweak = base_hz / tsc_hz;
8949
8950	if (!quiet)
8951	fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", base_mhz, max_mhz, bus_mhz);
8952	}
8953
8954	if (cpuid_has_aperf_mperf)
8955	aperf_mperf_multiplier = platform->need_perf_multiplier ? 1024 : 1;
8956
8957	BIC_PRESENT(BIC_IRQ);
8958	BIC_PRESENT(BIC_NMI);
8959	BIC_PRESENT(BIC_TSC_MHz);
8960	}
8961
8962	static void counter_info_init(void)
8963	{
8964	for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i) {
8965	struct cstate_counter_arch_info *const cai = &ccstate_counter_arch_infos[i];
8966
8967	if (platform->has_msr_knl_core_c6_residency && cai->msr == MSR_CORE_C6_RESIDENCY)
8968	cai->msr = MSR_KNL_CORE_C6_RESIDENCY;
8969
8970	if (!platform->has_msr_core_c1_res && cai->msr == MSR_CORE_C1_RES)
8971	cai->msr = 0;
8972
8973	if (platform->has_msr_atom_pkg_c6_residency && cai->msr == MSR_PKG_C6_RESIDENCY)
8974	cai->msr = MSR_ATOM_PKG_C6_RESIDENCY;
8975	}
8976
8977	for (int i = 0; i < NUM_MSR_COUNTERS; ++i) {
8978	msr_counter_arch_infos[i].present = false;
8979	msr_counter_arch_infos[i].needed = false;
8980	}
8981	}
8982
8983	void probe_pm_features(void)
8984	{
8985	probe_pstates();
8986
8987	probe_cstates();
8988
8989	probe_lpi();
8990
8991	probe_intel_uncore_frequency();
8992
8993	probe_graphics();
8994
8995	probe_rapl();
8996
8997	probe_thermal();
8998
8999	if (platform->has_nhm_msrs && !no_msr)
9000	BIC_PRESENT(BIC_SMI);
9001
9002	if (!quiet)
9003	decode_misc_feature_control();
9004	}
9005
9006	/* perf_llc_probe
9007	*
9008	* return 1 on success, else 0
9009	*/
9010	int has_perf_llc_access(void)
9011	{
9012	int fd;
9013
9014	if (no_perf)
9015	return 0;
9016
9017	fd = open_perf_counter(cpu: base_cpu, type: PERF_TYPE_HARDWARE, config: PERF_COUNT_HW_CACHE_REFERENCES, group_fd: -1, read_format: PERF_FORMAT_GROUP);
9018	if (fd != -1)
9019	close(fd);
9020
9021	if (fd == -1)
9022	warnx("Failed to access %s. Some of the counters may not be available\n"
9023	"\tRun as root to enable them or use %s to disable the access explicitly", "perf LLC counters", "'--hide LLC' or '--no-perf'");
9024
9025	return (fd != -1);
9026	}
9027
9028	void perf_llc_init(void)
9029	{
9030	int cpu;
9031	int retval;
9032
9033	if (no_perf)
9034	return;
9035	if (!(BIC_IS_ENABLED(BIC_LLC_RPS) && BIC_IS_ENABLED(BIC_LLC_HIT)))
9036	return;
9037
9038	for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
9039
9040	if (cpu_is_not_allowed(cpu))
9041	continue;
9042
9043	assert(fd_llc_percpu != 0);
9044	fd_llc_percpu[cpu] = open_perf_counter(cpu, type: PERF_TYPE_HARDWARE, config: PERF_COUNT_HW_CACHE_REFERENCES, group_fd: -1, read_format: PERF_FORMAT_GROUP);
9045	if (fd_llc_percpu[cpu] == -1) {
9046	warnx("%s: perf REFS: failed to open counter on cpu%d", __func__, cpu);
9047	free_fd_llc_percpu();
9048	return;
9049	}
9050	assert(fd_llc_percpu != 0);
9051	retval = open_perf_counter(cpu, type: PERF_TYPE_HARDWARE, config: PERF_COUNT_HW_CACHE_MISSES, group_fd: fd_llc_percpu[cpu], read_format: PERF_FORMAT_GROUP);
9052	if (retval == -1) {
9053	warnx("%s: perf MISS: failed to open counter on cpu%d", __func__, cpu);
9054	free_fd_llc_percpu();
9055	return;
9056	}
9057	}
9058	BIC_PRESENT(BIC_LLC_RPS);
9059	BIC_PRESENT(BIC_LLC_HIT);
9060	}
9061
9062	/*
9063	* in /dev/cpu/ return success for names that are numbers
9064	* ie. filter out ".", "..", "microcode".
9065	*/
9066	int dir_filter(const struct dirent *dirp)
9067	{
9068	if (isdigit(c: dirp->d_name[0]))
9069	return 1;
9070	else
9071	return 0;
9072	}
9073
9074	char *possible_file = "/sys/devices/system/cpu/possible";
9075	char possible_buf[1024];
9076
9077	int initialize_cpu_possible_set(void)
9078	{
9079	FILE *fp;
9080
9081	fp = fopen(possible_file, "r");
9082	if (!fp) {
9083	warn("open %s", possible_file);
9084	return -1;
9085	}
9086	if (fread(possible_buf, sizeof(char), 1024, fp) == 0) {
9087	warn("read %s", possible_file);
9088	goto err;
9089	}
9090	if (parse_cpu_str(possible_buf, cpu_possible_set, cpu_possible_setsize)) {
9091	warnx("%s: cpu str malformat %s\n", possible_file, cpu_effective_str);
9092	goto err;
9093	}
9094	return 0;
9095
9096	err:
9097	fclose(fp);
9098	return -1;
9099	}
9100
9101	void topology_probe(bool startup)
9102	{
9103	int i;
9104	int max_core_id = 0;
9105	int max_package_id = 0;
9106	int max_siblings = 0;
9107
9108	/* Initialize num_cpus, max_cpu_num */
9109	set_max_cpu_num();
9110	topo.num_cpus = 0;
9111	for_all_proc_cpus(func: count_cpus);
9112	if (!summary_only)
9113	BIC_PRESENT(BIC_CPU);
9114
9115	if (debug > 1)
9116	fprintf(outf, "num_cpus %d max_cpu_num %d\n", topo.num_cpus, topo.max_cpu_num);
9117
9118	cpus = calloc(1, (topo.max_cpu_num + 1) * sizeof(struct cpu_topology));
9119	if (cpus == NULL)
9120	err(1, "calloc cpus");
9121
9122	/*
9123	* Allocate and initialize cpu_present_set
9124	*/
9125	cpu_present_set = CPU_ALLOC((topo.max_cpu_num + 1));
9126	if (cpu_present_set == NULL)
9127	err(3, "CPU_ALLOC");
9128	cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9129	CPU_ZERO_S(cpu_present_setsize, cpu_present_set);
9130	for_all_proc_cpus(func: mark_cpu_present);
9131
9132	/*
9133	* Allocate and initialize cpu_possible_set
9134	*/
9135	cpu_possible_set = CPU_ALLOC((topo.max_cpu_num + 1));
9136	if (cpu_possible_set == NULL)
9137	err(3, "CPU_ALLOC");
9138	cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9139	CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set);
9140	initialize_cpu_possible_set();
9141
9142	/*
9143	* Allocate and initialize cpu_effective_set
9144	*/
9145	cpu_effective_set = CPU_ALLOC((topo.max_cpu_num + 1));
9146	if (cpu_effective_set == NULL)
9147	err(3, "CPU_ALLOC");
9148	cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9149	CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set);
9150	update_effective_set(startup);
9151
9152	/*
9153	* Allocate and initialize cpu_allowed_set
9154	*/
9155	cpu_allowed_set = CPU_ALLOC((topo.max_cpu_num + 1));
9156	if (cpu_allowed_set == NULL)
9157	err(3, "CPU_ALLOC");
9158	cpu_allowed_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9159	CPU_ZERO_S(cpu_allowed_setsize, cpu_allowed_set);
9160
9161	/*
9162	* Validate and update cpu_allowed_set.
9163	*
9164	* Make sure all cpus in cpu_subset are also in cpu_present_set during startup.
9165	* Give a warning when cpus in cpu_subset become unavailable at runtime.
9166	* Give a warning when cpus are not effective because of cgroup setting.
9167	*
9168	* cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpu_subset.
9169	*/
9170	for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) {
9171	if (cpu_subset && !CPU_ISSET_S(i, cpu_subset_size, cpu_subset))
9172	continue;
9173
9174	if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set)) {
9175	if (cpu_subset) {
9176	/* cpus in cpu_subset must be in cpu_present_set during startup */
9177	if (startup)
9178	err(1, "cpu%d not present", i);
9179	else
9180	fprintf(stderr, "cpu%d not present\n", i);
9181	}
9182	continue;
9183	}
9184
9185	if (CPU_COUNT_S(cpu_effective_setsize, cpu_effective_set)) {
9186	if (!CPU_ISSET_S(i, cpu_effective_setsize, cpu_effective_set)) {
9187	fprintf(stderr, "cpu%d not effective\n", i);
9188	continue;
9189	}
9190	}
9191
9192	CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set);
9193	}
9194
9195	if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set))
9196	err(-ENODEV, "No valid cpus found");
9197	sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set);
9198
9199	/*
9200	* Allocate and initialize cpu_affinity_set
9201	*/
9202	cpu_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1));
9203	if (cpu_affinity_set == NULL)
9204	err(3, "CPU_ALLOC");
9205	cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9206	CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
9207
9208	for_all_proc_cpus(func: init_thread_id);
9209
9210	for_all_proc_cpus(func: set_cpu_hybrid_type);
9211
9212	/*
9213	* For online cpus
9214	* find max_core_id, max_package_id
9215	*/
9216	for (i = 0; i <= topo.max_cpu_num; ++i) {
9217	int siblings;
9218
9219	if (cpu_is_not_present(cpu: i)) {
9220	if (debug > 1)
9221	fprintf(outf, "cpu%d NOT PRESENT\n", i);
9222	continue;
9223	}
9224
9225	cpus[i].logical_cpu_id = i;
9226
9227	/* get package information */
9228	cpus[i].physical_package_id = get_physical_package_id(cpu: i);
9229	if (cpus[i].physical_package_id > max_package_id)
9230	max_package_id = cpus[i].physical_package_id;
9231
9232	/* get die information */
9233	cpus[i].die_id = get_die_id(cpu: i);
9234	if (cpus[i].die_id > topo.max_die_id)
9235	topo.max_die_id = cpus[i].die_id;
9236
9237	/* get l3 information */
9238	cpus[i].l3_id = get_l3_id(cpu: i);
9239	if (cpus[i].l3_id > topo.max_l3_id)
9240	topo.max_l3_id = cpus[i].l3_id;
9241
9242	/* get numa node information */
9243	cpus[i].physical_node_id = get_physical_node_id(thiscpu: &cpus[i]);
9244	if (cpus[i].physical_node_id > topo.max_node_num)
9245	topo.max_node_num = cpus[i].physical_node_id;
9246
9247	/* get core information */
9248	cpus[i].physical_core_id = get_core_id(cpu: i);
9249	if (cpus[i].physical_core_id > max_core_id)
9250	max_core_id = cpus[i].physical_core_id;
9251
9252	/* get thread information */
9253	siblings = get_thread_siblings(thiscpu: &cpus[i]);
9254	if (siblings > max_siblings)
9255	max_siblings = siblings;
9256	if (cpus[i].thread_id == 0)
9257	topo.num_cores++;
9258	}
9259	topo.max_core_id = max_core_id;
9260	topo.max_package_id = max_package_id;
9261
9262	topo.cores_per_node = max_core_id + 1;
9263	if (debug > 1)
9264	fprintf(outf, "max_core_id %d, sizing for %d cores per package\n", max_core_id, topo.cores_per_node);
9265	if (!summary_only)
9266	BIC_PRESENT(BIC_Core);
9267
9268	topo.num_die = topo.max_die_id + 1;
9269	if (debug > 1)
9270	fprintf(outf, "max_die_id %d, sizing for %d die\n", topo.max_die_id, topo.num_die);
9271	if (!summary_only && topo.num_die > 1)
9272	BIC_PRESENT(BIC_Die);
9273
9274	if (!summary_only && topo.max_l3_id > 0)
9275	BIC_PRESENT(BIC_L3);
9276
9277	topo.num_packages = max_package_id + 1;
9278	if (debug > 1)
9279	fprintf(outf, "max_package_id %d, sizing for %d packages\n", max_package_id, topo.num_packages);
9280	if (!summary_only && topo.num_packages > 1)
9281	BIC_PRESENT(BIC_Package);
9282
9283	set_node_data();
9284	if (debug > 1)
9285	fprintf(outf, "nodes_per_pkg %d\n", topo.nodes_per_pkg);
9286	if (!summary_only && topo.nodes_per_pkg > 1)
9287	BIC_PRESENT(BIC_Node);
9288
9289	topo.threads_per_core = max_siblings;
9290	if (debug > 1)
9291	fprintf(outf, "max_siblings %d\n", max_siblings);
9292
9293	if (debug < 1)
9294	return;
9295
9296	for (i = 0; i <= topo.max_cpu_num; ++i) {
9297	if (cpu_is_not_present(cpu: i))
9298	continue;
9299	fprintf(outf,
9300	"cpu %d pkg %d die %d l3 %d node %d lnode %d core %d thread %d\n",
9301	i, cpus[i].physical_package_id, cpus[i].die_id, cpus[i].l3_id,
9302	cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].physical_core_id, cpus[i].thread_id);
9303	}
9304
9305	}
9306
9307	void allocate_counters(struct thread_data **t, struct core_data **c, struct pkg_data **p)
9308	{
9309	int i;
9310	int num_cores = topo.cores_per_node * topo.nodes_per_pkg * topo.num_packages;
9311	int num_threads = topo.threads_per_core * num_cores;
9312
9313	*t = calloc(num_threads, sizeof(struct thread_data));
9314	if (*t == NULL)
9315	goto error;
9316
9317	for (i = 0; i < num_threads; i++)
9318	(*t)[i].cpu_id = -1;
9319
9320	*c = calloc(num_cores, sizeof(struct core_data));
9321	if (*c == NULL)
9322	goto error;
9323
9324	for (i = 0; i < num_cores; i++) {
9325	(*c)[i].core_id = -1;
9326	(*c)[i].base_cpu = -1;
9327	}
9328
9329	*p = calloc(topo.num_packages, sizeof(struct pkg_data));
9330	if (*p == NULL)
9331	goto error;
9332
9333	for (i = 0; i < topo.num_packages; i++) {
9334	(*p)[i].package_id = i;
9335	(*p)[i].base_cpu = -1;
9336	}
9337
9338	return;
9339	error:
9340	err(1, "calloc counters");
9341	}
9342
9343	/*
9344	* init_counter()
9345	*
9346	* set FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE
9347	*/
9348	void init_counter(struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base, int cpu_id)
9349	{
9350	int pkg_id = cpus[cpu_id].physical_package_id;
9351	int node_id = cpus[cpu_id].logical_node_id;
9352	int core_id = cpus[cpu_id].physical_core_id;
9353	int thread_id = cpus[cpu_id].thread_id;
9354	struct thread_data *t;
9355	struct core_data *c;
9356
9357	/* Workaround for systems where physical_node_id==-1
9358	* and logical_node_id==(-1 - topo.num_cpus)
9359	*/
9360	if (node_id < 0)
9361	node_id = 0;
9362
9363	t = GET_THREAD(thread_base, thread_id, core_id, node_id, pkg_id);
9364	c = GET_CORE(core_base, core_id, node_id, pkg_id);
9365
9366	t->cpu_id = cpu_id;
9367	if (!cpu_is_not_allowed(cpu: cpu_id)) {
9368
9369	if (c->base_cpu < 0)
9370	c->base_cpu = t->cpu_id;
9371	if (pkg_base[pkg_id].base_cpu < 0)
9372	pkg_base[pkg_id].base_cpu = t->cpu_id;
9373	}
9374
9375	c->core_id = core_id;
9376	pkg_base[pkg_id].package_id = pkg_id;
9377	}
9378
9379	int initialize_counters(int cpu_id)
9380	{
9381	init_counter(EVEN_COUNTERS, cpu_id);
9382	init_counter(ODD_COUNTERS, cpu_id);
9383	return 0;
9384	}
9385
9386	void allocate_output_buffer()
9387	{
9388	output_buffer = calloc(1, (1 + topo.num_cpus) * 2048);
9389	outp = output_buffer;
9390	if (outp == NULL)
9391	err(-1, "calloc output buffer");
9392	}
9393
9394	void allocate_fd_percpu(void)
9395	{
9396	fd_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9397	if (fd_percpu == NULL)
9398	err(-1, "calloc fd_percpu");
9399	}
9400
9401	void allocate_irq_buffers(void)
9402	{
9403	irq_column_2_cpu = calloc(topo.num_cpus, sizeof(int));
9404	if (irq_column_2_cpu == NULL)
9405	err(-1, "calloc %d", topo.num_cpus);
9406
9407	irqs_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9408	if (irqs_per_cpu == NULL)
9409	err(-1, "calloc %d IRQ", topo.max_cpu_num + 1);
9410
9411	nmi_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9412	if (nmi_per_cpu == NULL)
9413	err(-1, "calloc %d NMI", topo.max_cpu_num + 1);
9414	}
9415
9416	int update_topo(PER_THREAD_PARAMS)
9417	{
9418	topo.allowed_cpus++;
9419	if ((int)t->cpu_id == c->base_cpu)
9420	topo.allowed_cores++;
9421	if ((int)t->cpu_id == p->base_cpu)
9422	topo.allowed_packages++;
9423
9424	return 0;
9425	}
9426
9427	void topology_update(void)
9428	{
9429	topo.allowed_cpus = 0;
9430	topo.allowed_cores = 0;
9431	topo.allowed_packages = 0;
9432	for_all_cpus(func: update_topo, ODD_COUNTERS);
9433	}
9434
9435	void setup_all_buffers(bool startup)
9436	{
9437	topology_probe(startup);
9438	allocate_irq_buffers();
9439	allocate_fd_percpu();
9440	allocate_counters(t: &thread_even, c: &core_even, p: &package_even);
9441	allocate_counters(t: &thread_odd, c: &core_odd, p: &package_odd);
9442	allocate_output_buffer();
9443	for_all_proc_cpus(func: initialize_counters);
9444	topology_update();
9445	}
9446
9447	void set_base_cpu(void)
9448	{
9449	int i;
9450
9451	for (i = 0; i < topo.max_cpu_num + 1; ++i) {
9452	if (cpu_is_not_allowed(cpu: i))
9453	continue;
9454	base_cpu = i;
9455	if (debug > 1)
9456	fprintf(outf, "base_cpu = %d\n", base_cpu);
9457	return;
9458	}
9459	err(-ENODEV, "No valid cpus found");
9460	}
9461
9462	bool has_added_counters(void)
9463	{
9464	/*
9465	* It only makes sense to call this after the command line is parsed,
9466	* otherwise sys structure is not populated.
9467	*/
9468
9469	return sys.added_core_counters | sys.added_thread_counters | sys.added_package_counters;
9470	}
9471
9472	void check_msr_access(void)
9473	{
9474	check_msr_driver();
9475	check_msr_permission();
9476
9477	if (no_msr)
9478	bic_disable_msr_access();
9479	}
9480
9481	void check_perf_access(void)
9482	{
9483	if (BIC_IS_ENABLED(BIC_IPC))
9484	if (!has_perf_instr_count_access())
9485	no_perf = 1;
9486
9487	if (BIC_IS_ENABLED(BIC_LLC_RPS) || BIC_IS_ENABLED(BIC_LLC_HIT))
9488	if (!has_perf_llc_access())
9489	no_perf = 1;
9490
9491	if (no_perf)
9492	bic_disable_perf_access();
9493	}
9494
9495	bool perf_has_hybrid_devices(void)
9496	{
9497	/*
9498	* 0: unknown
9499	* 1: has separate perf device for p and e core
9500	* -1: doesn't have separate perf device for p and e core
9501	*/
9502	static int cached;
9503
9504	if (cached > 0)
9505	return true;
9506
9507	if (cached < 0)
9508	return false;
9509
9510	if (access("/sys/bus/event_source/devices/cpu_core", F_OK)) {
9511	cached = -1;
9512	return false;
9513	}
9514
9515	if (access("/sys/bus/event_source/devices/cpu_atom", F_OK)) {
9516	cached = -1;
9517	return false;
9518	}
9519
9520	cached = 1;
9521	return true;
9522	}
9523
9524	int added_perf_counters_init_(struct perf_counter_info *pinfo)
9525	{
9526	size_t num_domains = 0;
9527	unsigned int next_domain;
9528	bool *domain_visited;
9529	unsigned int perf_type, perf_config;
9530	double perf_scale;
9531	int fd_perf;
9532
9533	if (!pinfo)
9534	return 0;
9535
9536	const size_t max_num_domains = MAX(topo.max_cpu_num + 1, MAX(topo.max_core_id + 1, topo.max_package_id + 1));
9537
9538	domain_visited = calloc(max_num_domains, sizeof(*domain_visited));
9539
9540	while (pinfo) {
9541	switch (pinfo->scope) {
9542	case SCOPE_CPU:
9543	num_domains = topo.max_cpu_num + 1;
9544	break;
9545
9546	case SCOPE_CORE:
9547	num_domains = topo.max_core_id + 1;
9548	break;
9549
9550	case SCOPE_PACKAGE:
9551	num_domains = topo.max_package_id + 1;
9552	break;
9553	}
9554
9555	/* Allocate buffer for file descriptor for each domain. */
9556	pinfo->fd_perf_per_domain = calloc(num_domains, sizeof(*pinfo->fd_perf_per_domain));
9557	if (!pinfo->fd_perf_per_domain)
9558	errx(1, "%s: alloc %s", __func__, "fd_perf_per_domain");
9559
9560	for (size_t i = 0; i < num_domains; ++i)
9561	pinfo->fd_perf_per_domain[i] = -1;
9562
9563	pinfo->num_domains = num_domains;
9564	pinfo->scale = 1.0;
9565
9566	memset(domain_visited, 0, max_num_domains * sizeof(*domain_visited));
9567
9568	for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
9569
9570	next_domain = cpu_to_domain(pc: pinfo, cpu);
9571
9572	assert(next_domain < num_domains);
9573
9574	if (cpu_is_not_allowed(cpu))
9575	continue;
9576
9577	if (domain_visited[next_domain])
9578	continue;
9579
9580	/*
9581	* Intel hybrid platforms expose different perf devices for P and E cores.
9582	* Instead of one, "/sys/bus/event_source/devices/cpu" device, there are
9583	* "/sys/bus/event_source/devices/{cpu_core,cpu_atom}".
9584	*
9585	* This makes it more complicated to the user, because most of the counters
9586	* are available on both and have to be handled manually, otherwise.
9587	*
9588	* Code below, allow user to use the old "cpu" name, which is translated accordingly.
9589	*/
9590	const char *perf_device = pinfo->device;
9591
9592	if (strcmp(perf_device, "cpu") == 0 && perf_has_hybrid_devices()) {
9593	switch (cpus[cpu].type) {
9594	case INTEL_PCORE_TYPE:
9595	perf_device = "cpu_core";
9596	break;
9597
9598	case INTEL_ECORE_TYPE:
9599	perf_device = "cpu_atom";
9600	break;
9601
9602	default: /* Don't change, we will probably fail and report a problem soon. */
9603	break;
9604	}
9605	}
9606
9607	perf_type = read_perf_type(subsys: perf_device);
9608	if (perf_type == (unsigned int)-1) {
9609	warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "type");
9610	continue;
9611	}
9612
9613	perf_config = read_perf_config(subsys: perf_device, event_name: pinfo->event);
9614	if (perf_config == (unsigned int)-1) {
9615	warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "config");
9616	continue;
9617	}
9618
9619	/* Scale is not required, some counters just don't have it. */
9620	perf_scale = read_perf_scale(subsys: perf_device, event_name: pinfo->event);
9621	if (perf_scale == 0.0)
9622	perf_scale = 1.0;
9623
9624	fd_perf = open_perf_counter(cpu, type: perf_type, config: perf_config, group_fd: -1, read_format: 0);
9625	if (fd_perf == -1) {
9626	warnx("%s: perf/%s/%s: failed to open counter on cpu%d", __func__, perf_device, pinfo->event, cpu);
9627	continue;
9628	}
9629
9630	domain_visited[next_domain] = 1;
9631	pinfo->fd_perf_per_domain[next_domain] = fd_perf;
9632	pinfo->scale = perf_scale;
9633
9634	if (debug)
9635	fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n", perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]);
9636	}
9637
9638	pinfo = pinfo->next;
9639	}
9640
9641	free(domain_visited);
9642
9643	return 0;
9644	}
9645
9646	void added_perf_counters_init(void)
9647	{
9648	if (added_perf_counters_init_(pinfo: sys.perf_tp))
9649	errx(1, "%s: %s", __func__, "thread");
9650
9651	if (added_perf_counters_init_(pinfo: sys.perf_cp))
9652	errx(1, "%s: %s", __func__, "core");
9653
9654	if (added_perf_counters_init_(pinfo: sys.perf_pp))
9655	errx(1, "%s: %s", __func__, "package");
9656	}
9657
9658	int parse_telem_info_file(int fd_dir, const char *info_filename, const char *format, unsigned long *output)
9659	{
9660	int fd_telem_info;
9661	FILE *file_telem_info;
9662	unsigned long value;
9663
9664	fd_telem_info = openat(fd_dir, info_filename, O_RDONLY);
9665	if (fd_telem_info == -1)
9666	return -1;
9667
9668	file_telem_info = fdopen(fd_telem_info, "r");
9669	if (file_telem_info == NULL) {
9670	close(fd_telem_info);
9671	return -1;
9672	}
9673
9674	if (fscanf(file_telem_info, format, &value) != 1) {
9675	fclose(file_telem_info);
9676	return -1;
9677	}
9678
9679	fclose(file_telem_info);
9680
9681	*output = value;
9682
9683	return 0;
9684	}
9685
9686	struct pmt_mmio *pmt_mmio_open(unsigned int target_guid)
9687	{
9688	struct pmt_diriter_t pmt_iter;
9689	const struct dirent *entry;
9690	struct stat st;
9691	int fd_telem_dir, fd_pmt;
9692	unsigned long guid, size, offset;
9693	size_t mmap_size;
9694	void *mmio;
9695	struct pmt_mmio *head = NULL, *last = NULL;
9696	struct pmt_mmio *new_pmt = NULL;
9697
9698	if (stat(SYSFS_TELEM_PATH, &st) == -1)
9699	return NULL;
9700
9701	pmt_diriter_init(iter: &pmt_iter);
9702	entry = pmt_diriter_begin(iter: &pmt_iter, SYSFS_TELEM_PATH);
9703	if (!entry) {
9704	pmt_diriter_remove(iter: &pmt_iter);
9705	return NULL;
9706	}
9707
9708	for (; entry != NULL; entry = pmt_diriter_next(iter: &pmt_iter)) {
9709	if (fstatat(dirfd(pmt_iter.dir), entry->d_name, &st, 0) == -1)
9710	break;
9711
9712	if (!S_ISDIR(st.st_mode))
9713	continue;
9714
9715	fd_telem_dir = openat(dirfd(pmt_iter.dir), entry->d_name, O_RDONLY);
9716	if (fd_telem_dir == -1)
9717	break;
9718
9719	if (parse_telem_info_file(fd_dir: fd_telem_dir, info_filename: "guid", format: "%lx", output: &guid)) {
9720	close(fd_telem_dir);
9721	break;
9722	}
9723
9724	if (parse_telem_info_file(fd_dir: fd_telem_dir, info_filename: "size", format: "%lu", output: &size)) {
9725	close(fd_telem_dir);
9726	break;
9727	}
9728
9729	if (guid != target_guid) {
9730	close(fd_telem_dir);
9731	continue;
9732	}
9733
9734	if (parse_telem_info_file(fd_dir: fd_telem_dir, info_filename: "offset", format: "%lu", output: &offset)) {
9735	close(fd_telem_dir);
9736	break;
9737	}
9738
9739	assert(offset == 0);
9740
9741	fd_pmt = openat(fd_telem_dir, "telem", O_RDONLY);
9742	if (fd_pmt == -1)
9743	goto loop_cleanup_and_break;
9744
9745	mmap_size = ROUND_UP_TO_PAGE_SIZE(size);
9746	mmio = mmap(0, mmap_size, PROT_READ, MAP_SHARED, fd_pmt, 0);
9747	if (mmio != MAP_FAILED) {
9748	if (debug)
9749	fprintf(stderr, "%s: 0x%lx mmaped at: %p\n", __func__, guid, mmio);
9750
9751	new_pmt = calloc(1, sizeof(*new_pmt));
9752
9753	if (!new_pmt) {
9754	fprintf(stderr, "%s: Failed to allocate pmt_mmio\n", __func__);
9755	exit(1);
9756	}
9757
9758	/*
9759	* Create linked list of mmaped regions,
9760	* but preserve the ordering from sysfs.
9761	* Ordering is important for the user to
9762	* use the seq=%u parameter when adding a counter.
9763	*/
9764	new_pmt->guid = guid;
9765	new_pmt->mmio_base = mmio;
9766	new_pmt->pmt_offset = offset;
9767	new_pmt->size = size;
9768	new_pmt->next = pmt_mmios;
9769
9770	if (last)
9771	last->next = new_pmt;
9772	else
9773	head = new_pmt;
9774
9775	last = new_pmt;
9776	}
9777
9778	loop_cleanup_and_break:
9779	close(fd_pmt);
9780	close(fd_telem_dir);
9781	}
9782
9783	pmt_diriter_remove(iter: &pmt_iter);
9784
9785	/*
9786	* If we found something, stick just
9787	* created linked list to the front.
9788	*/
9789	if (head)
9790	pmt_mmios = head;
9791
9792	return head;
9793	}
9794
9795	struct pmt_mmio *pmt_mmio_find(unsigned int guid)
9796	{
9797	struct pmt_mmio *pmmio = pmt_mmios;
9798
9799	while (pmmio) {
9800	if (pmmio->guid == guid)
9801	return pmmio;
9802
9803	pmmio = pmmio->next;
9804	}
9805
9806	return NULL;
9807	}
9808
9809	void *pmt_get_counter_pointer(struct pmt_mmio *pmmio, unsigned long counter_offset)
9810	{
9811	char *ret;
9812
9813	/* Get base of mmaped PMT file. */
9814	ret = (char *)pmmio->mmio_base;
9815
9816	/*
9817	* Apply PMT MMIO offset to obtain beginning of the mmaped telemetry data.
9818	* It's not guaranteed that the mmaped memory begins with the telemetry data
9819	* - we might have to apply the offset first.
9820	*/
9821	ret += pmmio->pmt_offset;
9822
9823	/* Apply the counter offset to get the address to the mmaped counter. */
9824	ret += counter_offset;
9825
9826	return ret;
9827	}
9828
9829	struct pmt_mmio *pmt_add_guid(unsigned int guid, unsigned int seq)
9830	{
9831	struct pmt_mmio *ret;
9832
9833	ret = pmt_mmio_find(guid);
9834	if (!ret)
9835	ret = pmt_mmio_open(target_guid: guid);
9836
9837	while (ret && seq) {
9838	ret = ret->next;
9839	--seq;
9840	}
9841
9842	return ret;
9843	}
9844
9845	enum pmt_open_mode {
9846	PMT_OPEN_TRY, /* Open failure is not an error. */
9847	PMT_OPEN_REQUIRED, /* Open failure is a fatal error. */
9848	};
9849
9850	struct pmt_counter *pmt_find_counter(struct pmt_counter *pcounter, const char *name)
9851	{
9852	while (pcounter) {
9853	if (strcmp(pcounter->name, name) == 0)
9854	break;
9855
9856	pcounter = pcounter->next;
9857	}
9858
9859	return pcounter;
9860	}
9861
9862	struct pmt_counter **pmt_get_scope_root(enum counter_scope scope)
9863	{
9864	switch (scope) {
9865	case SCOPE_CPU:
9866	return &sys.pmt_tp;
9867	case SCOPE_CORE:
9868	return &sys.pmt_cp;
9869	case SCOPE_PACKAGE:
9870	return &sys.pmt_pp;
9871	}
9872
9873	__builtin_unreachable();
9874	}
9875
9876	void pmt_counter_add_domain(struct pmt_counter *pcounter, unsigned long *pmmio, unsigned int domain_id)
9877	{
9878	/* Make sure the new domain fits. */
9879	if (domain_id >= pcounter->num_domains)
9880	pmt_counter_resize(pcounter, new_size: domain_id + 1);
9881
9882	assert(pcounter->domains);
9883	assert(domain_id < pcounter->num_domains);
9884
9885	pcounter->domains[domain_id].pcounter = pmmio;
9886	}
9887
9888	int pmt_add_counter(unsigned int guid, unsigned int seq, const char *name, enum pmt_datatype type,
9889	unsigned int lsb, unsigned int msb, unsigned int offset, enum counter_scope scope,
9890	enum counter_format format, unsigned int domain_id, enum pmt_open_mode mode)
9891	{
9892	struct pmt_mmio *mmio;
9893	struct pmt_counter *pcounter;
9894	struct pmt_counter **const pmt_root = pmt_get_scope_root(scope);
9895	bool new_counter = false;
9896	int conflict = 0;
9897
9898	if (lsb > msb) {
9899	fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "lsb <= msb", name);
9900	exit(1);
9901	}
9902
9903	if (msb >= 64) {
9904	fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "msb < 64", name);
9905	exit(1);
9906	}
9907
9908	mmio = pmt_add_guid(guid, seq);
9909	if (!mmio) {
9910	if (mode != PMT_OPEN_TRY) {
9911	fprintf(stderr, "%s: failed to map PMT MMIO for guid %x, seq %u\n", __func__, guid, seq);
9912	exit(1);
9913	}
9914
9915	return 1;
9916	}
9917
9918	if (offset >= mmio->size) {
9919	if (mode != PMT_OPEN_TRY) {
9920	fprintf(stderr, "%s: offset %u outside of PMT MMIO size %u\n", __func__, offset, mmio->size);
9921	exit(1);
9922	}
9923
9924	return 1;
9925	}
9926
9927	pcounter = pmt_find_counter(pcounter: *pmt_root, name);
9928	if (!pcounter) {
9929	pcounter = calloc(1, sizeof(*pcounter));
9930	new_counter = true;
9931	}
9932
9933	if (new_counter) {
9934	strncpy(p: pcounter->name, q: name, ARRAY_SIZE(pcounter->name) - 1);
9935	pcounter->type = type;
9936	pcounter->scope = scope;
9937	pcounter->lsb = lsb;
9938	pcounter->msb = msb;
9939	pcounter->format = format;
9940	} else {
9941	conflict += pcounter->type != type;
9942	conflict += pcounter->scope != scope;
9943	conflict += pcounter->lsb != lsb;
9944	conflict += pcounter->msb != msb;
9945	conflict += pcounter->format != format;
9946	}
9947
9948	if (conflict) {
9949	fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n", __func__, name);
9950	exit(1);
9951	}
9952
9953	pmt_counter_add_domain(pcounter, pmmio: pmt_get_counter_pointer(pmmio: mmio, counter_offset: offset), domain_id);
9954
9955	if (new_counter) {
9956	pcounter->next = *pmt_root;
9957	*pmt_root = pcounter;
9958	}
9959
9960	return 0;
9961	}
9962
9963	void pmt_init(void)
9964	{
9965	int cpu_num;
9966	unsigned long seq, offset, mod_num;
9967
9968	if (BIC_IS_ENABLED(BIC_Diec6)) {
9969	pmt_add_counter(PMT_MTL_DC6_GUID, PMT_MTL_DC6_SEQ, name: "Die%c6", type: PMT_TYPE_XTAL_TIME,
9970	PMT_COUNTER_MTL_DC6_LSB, PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET,
9971	scope: SCOPE_PACKAGE, format: FORMAT_DELTA, domain_id: 0, mode: PMT_OPEN_TRY);
9972	}
9973
9974	if (BIC_IS_ENABLED(BIC_CPU_c1e)) {
9975	seq = 0;
9976	offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE;
9977	mod_num = 0; /* Relative module number for current PMT file. */
9978
9979	/* Open the counter for each CPU. */
9980	for (cpu_num = 0; cpu_num < topo.max_cpu_num;) {
9981
9982	if (cpu_is_not_allowed(cpu: cpu_num))
9983	goto next_loop_iter;
9984
9985	/*
9986	* Set the scope to CPU, even though CWF report the counter per module.
9987	* CPUs inside the same module will read from the same location, instead of reporting zeros.
9988	*
9989	* CWF with newer firmware might require a PMT_TYPE_XTAL_TIME intead of PMT_TYPE_TCORE_CLOCK.
9990	*/
9991	pmt_add_counter(PMT_CWF_MC1E_GUID, seq, name: "CPU%c1e", type: PMT_TYPE_TCORE_CLOCK,
9992	PMT_COUNTER_CWF_MC1E_LSB, PMT_COUNTER_CWF_MC1E_MSB, offset, scope: SCOPE_CPU, format: FORMAT_DELTA, domain_id: cpu_num, mode: PMT_OPEN_TRY);
9993
9994	/*
9995	* Rather complex logic for each time we go to the next loop iteration,
9996	* so keep it as a label.
9997	*/
9998	next_loop_iter:
9999	/*
10000	* Advance the cpu number and check if we should also advance offset to
10001	* the next counter inside the PMT file.
10002	*
10003	* On Clearwater Forest platform, the counter is reported per module,
10004	* so open the same counter for all of the CPUs inside the module.
10005	* That way, reported table show the correct value for all of the CPUs inside the module,
10006	* instead of zeros.
10007	*/
10008	++cpu_num;
10009	if (cpu_num % PMT_COUNTER_CWF_CPUS_PER_MODULE == 0) {
10010	offset += PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT;
10011	++mod_num;
10012	}
10013
10014	/*
10015	* There are PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE in each PMT file.
10016	*
10017	* If that number is reached, seq must be incremented to advance to the next file in a sequence.
10018	* Offset inside that file and a module counter has to be reset.
10019	*/
10020	if (mod_num == PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE) {
10021	++seq;
10022	offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE;
10023	mod_num = 0;
10024	}
10025	}
10026	}
10027	}
10028
10029	void turbostat_init()
10030	{
10031	setup_all_buffers(true);
10032	set_base_cpu();
10033	check_msr_access();
10034	check_perf_access();
10035	process_cpuid();
10036	counter_info_init();
10037	probe_pm_features();
10038	msr_perf_init();
10039	linux_perf_init();
10040	rapl_perf_init();
10041	cstate_perf_init();
10042	perf_llc_init();
10043	added_perf_counters_init();
10044	pmt_init();
10045
10046	for_all_cpus(func: get_cpu_type, ODD_COUNTERS);
10047	for_all_cpus(func: get_cpu_type, EVEN_COUNTERS);
10048
10049	if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(cpu: base_cpu) != -1)
10050	BIC_PRESENT(BIC_IPC);
10051
10052	/*
10053	* If TSC tweak is needed, but couldn't get it,
10054	* disable more BICs, since it can't be reported accurately.
10055	*/
10056	if (platform->enable_tsc_tweak && !has_base_hz) {
10057	CLR_BIC(BIC_Busy, &bic_enabled);
10058	CLR_BIC(BIC_Bzy_MHz, &bic_enabled);
10059	}
10060	}
10061
10062	void affinitize_child(void)
10063	{
10064	/* Prefer cpu_possible_set, if available */
10065	if (sched_setaffinity(0, cpu_possible_setsize, cpu_possible_set)) {
10066	warn("sched_setaffinity cpu_possible_set");
10067
10068	/* Otherwise, allow child to run on same cpu set as turbostat */
10069	if (sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set))
10070	warn("sched_setaffinity cpu_allowed_set");
10071	}
10072	}
10073
10074	int fork_it(char **argv)
10075	{
10076	pid_t child_pid;
10077	int status;
10078
10079	snapshot_proc_sysfs_files();
10080	status = for_all_cpus(func: get_counters, EVEN_COUNTERS);
10081	first_counter_read = 0;
10082	if (status)
10083	exit(status);
10084	gettimeofday(&tv_even, (struct timezone *)NULL);
10085
10086	child_pid = fork();
10087	if (!child_pid) {
10088	/* child */
10089	affinitize_child();
10090	execvp(argv[0], argv);
10091	err(errno, "exec %s", argv[0]);
10092	} else {
10093
10094	/* parent */
10095	if (child_pid == -1)
10096	err(1, "fork");
10097
10098	signal(SIGINT, SIG_IGN);
10099	signal(SIGQUIT, SIG_IGN);
10100	if (waitpid(child_pid, &status, 0) == -1)
10101	err(status, "waitpid");
10102
10103	if (WIFEXITED(status))
10104	status = WEXITSTATUS(status);
10105	}
10106	/*
10107	* n.b. fork_it() does not check for errors from for_all_cpus()
10108	* because re-starting is problematic when forking
10109	*/
10110	snapshot_proc_sysfs_files();
10111	for_all_cpus(func: get_counters, ODD_COUNTERS);
10112	gettimeofday(&tv_odd, (struct timezone *)NULL);
10113	timersub(&tv_odd, &tv_even, &tv_delta);
10114	if (for_all_cpus_2(func: delta_cpu, ODD_COUNTERS, EVEN_COUNTERS))
10115	fprintf(outf, "%s: Counter reset detected\n", progname);
10116	delta_platform(new: &platform_counters_odd, old: &platform_counters_even);
10117
10118	compute_average(EVEN_COUNTERS);
10119	format_all_counters(EVEN_COUNTERS);
10120
10121	fprintf(outf, "%.6f sec\n", tv_delta.tv_sec + tv_delta.tv_usec / 1000000.0);
10122
10123	flush_output_stderr();
10124
10125	return status;
10126	}
10127
10128	int get_and_dump_counters(void)
10129	{
10130	int status;
10131
10132	snapshot_proc_sysfs_files();
10133	status = for_all_cpus(func: get_counters, ODD_COUNTERS);
10134	if (status)
10135	return status;
10136
10137	status = for_all_cpus(func: dump_counters, ODD_COUNTERS);
10138	if (status)
10139	return status;
10140
10141	flush_output_stdout();
10142
10143	return status;
10144	}
10145
10146	void print_version()
10147	{
10148	fprintf(outf, "turbostat version 2025.12.02 - Len Brown <lenb@kernel.org>\n");
10149	}
10150
10151	#define COMMAND_LINE_SIZE 2048
10152
10153	void print_bootcmd(void)
10154	{
10155	char bootcmd[COMMAND_LINE_SIZE];
10156	FILE *fp;
10157	int ret;
10158
10159	memset(bootcmd, 0, COMMAND_LINE_SIZE);
10160	fp = fopen("/proc/cmdline", "r");
10161	if (!fp)
10162	return;
10163
10164	ret = fread(bootcmd, sizeof(char), COMMAND_LINE_SIZE - 1, fp);
10165	if (ret) {
10166	bootcmd[ret] = '\0';
10167	/* the last character is already '\n' */
10168	fprintf(outf, "Kernel command line: %s", bootcmd);
10169	}
10170
10171	fclose(fp);
10172	}
10173
10174	struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name)
10175	{
10176	struct msr_counter *mp;
10177
10178	for (mp = head; mp; mp = mp->next) {
10179	if (debug)
10180	fprintf(stderr, "%s: %s %s\n", __func__, name, mp->name);
10181	if (!strcmp(name, mp->name))
10182	return mp;
10183	}
10184	return NULL;
10185	}
10186
10187	int add_counter(unsigned int msr_num, char *path, char *name,
10188	unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int id)
10189	{
10190	struct msr_counter *msrp;
10191
10192	if (no_msr && msr_num)
10193	errx(1, "Requested MSR counter 0x%x, but in --no-msr mode", msr_num);
10194
10195	if (debug)
10196	fprintf(stderr, "%s(msr%d, %s, %s, width%d, scope%d, type%d, format%d, flags%x, id%d)\n",
10197	__func__, msr_num, path, name, width, scope, type, format, flags, id);
10198
10199	switch (scope) {
10200
10201	case SCOPE_CPU:
10202	msrp = find_msrp_by_name(head: sys.tp, name);
10203	if (msrp) {
10204	if (debug)
10205	fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10206	break;
10207	}
10208	if (sys.added_thread_counters++ >= MAX_ADDED_THREAD_COUNTERS) {
10209	warnx("ignoring thread counter %s", name);
10210	return -1;
10211	}
10212	break;
10213	case SCOPE_CORE:
10214	msrp = find_msrp_by_name(head: sys.cp, name);
10215	if (msrp) {
10216	if (debug)
10217	fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10218	break;
10219	}
10220	if (sys.added_core_counters++ >= MAX_ADDED_CORE_COUNTERS) {
10221	warnx("ignoring core counter %s", name);
10222	return -1;
10223	}
10224	break;
10225	case SCOPE_PACKAGE:
10226	msrp = find_msrp_by_name(head: sys.pp, name);
10227	if (msrp) {
10228	if (debug)
10229	fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10230	break;
10231	}
10232	if (sys.added_package_counters++ >= MAX_ADDED_PACKAGE_COUNTERS) {
10233	warnx("ignoring package counter %s", name);
10234	return -1;
10235	}
10236	break;
10237	default:
10238	warnx("ignoring counter %s with unknown scope", name);
10239	return -1;
10240	}
10241
10242	if (msrp == NULL) {
10243	msrp = calloc(1, sizeof(struct msr_counter));
10244	if (msrp == NULL)
10245	err(-1, "calloc msr_counter");
10246
10247	msrp->msr_num = msr_num;
10248	strncpy(p: msrp->name, q: name, NAME_BYTES - 1);
10249	msrp->width = width;
10250	msrp->type = type;
10251	msrp->format = format;
10252	msrp->flags = flags;
10253
10254	switch (scope) {
10255	case SCOPE_CPU:
10256	msrp->next = sys.tp;
10257	sys.tp = msrp;
10258	break;
10259	case SCOPE_CORE:
10260	msrp->next = sys.cp;
10261	sys.cp = msrp;
10262	break;
10263	case SCOPE_PACKAGE:
10264	msrp->next = sys.pp;
10265	sys.pp = msrp;
10266	break;
10267	}
10268	}
10269
10270	if (path) {
10271	struct sysfs_path *sp;
10272
10273	sp = calloc(1, sizeof(struct sysfs_path));
10274	if (sp == NULL) {
10275	perror("calloc");
10276	exit(1);
10277	}
10278	strncpy(p: sp->path, q: path, PATH_BYTES - 1);
10279	sp->id = id;
10280	sp->next = msrp->sp;
10281	msrp->sp = sp;
10282	}
10283
10284	return 0;
10285	}
10286
10287	/*
10288	* Initialize the fields used for identifying and opening the counter.
10289	*
10290	* Defer the initialization of any runtime buffers for actually reading
10291	* the counters for when we initialize all perf counters, so we can later
10292	* easily call re_initialize().
10293	*/
10294	struct perf_counter_info *make_perf_counter_info(const char *perf_device,
10295	const char *perf_event,
10296	const char *name,
10297	unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format)
10298	{
10299	struct perf_counter_info *pinfo;
10300
10301	pinfo = calloc(1, sizeof(*pinfo));
10302	if (!pinfo)
10303	errx(1, "%s: Failed to allocate %s/%s\n", __func__, perf_device, perf_event);
10304
10305	strncpy(p: pinfo->device, q: perf_device, ARRAY_SIZE(pinfo->device) - 1);
10306	strncpy(p: pinfo->event, q: perf_event, ARRAY_SIZE(pinfo->event) - 1);
10307
10308	strncpy(p: pinfo->name, q: name, ARRAY_SIZE(pinfo->name) - 1);
10309	pinfo->width = width;
10310	pinfo->scope = scope;
10311	pinfo->type = type;
10312	pinfo->format = format;
10313
10314	return pinfo;
10315	}
10316
10317	int add_perf_counter(const char *perf_device, const char *perf_event, const char *name_buffer, unsigned int width,
10318	enum counter_scope scope, enum counter_type type, enum counter_format format)
10319	{
10320	struct perf_counter_info *pinfo;
10321
10322	switch (scope) {
10323	case SCOPE_CPU:
10324	if (sys.added_thread_perf_counters >= MAX_ADDED_THREAD_COUNTERS) {
10325	warnx("ignoring thread counter perf/%s/%s", perf_device, perf_event);
10326	return -1;
10327	}
10328	break;
10329
10330	case SCOPE_CORE:
10331	if (sys.added_core_perf_counters >= MAX_ADDED_CORE_COUNTERS) {
10332	warnx("ignoring core counter perf/%s/%s", perf_device, perf_event);
10333	return -1;
10334	}
10335	break;
10336
10337	case SCOPE_PACKAGE:
10338	if (sys.added_package_perf_counters >= MAX_ADDED_PACKAGE_COUNTERS) {
10339	warnx("ignoring package counter perf/%s/%s", perf_device, perf_event);
10340	return -1;
10341	}
10342	break;
10343	}
10344
10345	pinfo = make_perf_counter_info(perf_device, perf_event, name: name_buffer, width, scope, type, format);
10346
10347	if (!pinfo)
10348	return -1;
10349
10350	switch (scope) {
10351	case SCOPE_CPU:
10352	pinfo->next = sys.perf_tp;
10353	sys.perf_tp = pinfo;
10354	++sys.added_thread_perf_counters;
10355	break;
10356
10357	case SCOPE_CORE:
10358	pinfo->next = sys.perf_cp;
10359	sys.perf_cp = pinfo;
10360	++sys.added_core_perf_counters;
10361	break;
10362
10363	case SCOPE_PACKAGE:
10364	pinfo->next = sys.perf_pp;
10365	sys.perf_pp = pinfo;
10366	++sys.added_package_perf_counters;
10367	break;
10368	}
10369
10370	// FIXME: we might not have debug here yet
10371	if (debug)
10372	fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n", __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope);
10373
10374	return 0;
10375	}
10376
10377	void parse_add_command_msr(char *add_command)
10378	{
10379	int msr_num = 0;
10380	char *path = NULL;
10381	char perf_device[PERF_DEV_NAME_BYTES] = "";
10382	char perf_event[PERF_EVT_NAME_BYTES] = "";
10383	char name_buffer[PERF_NAME_BYTES] = "";
10384	int width = 64;
10385	int fail = 0;
10386	enum counter_scope scope = SCOPE_CPU;
10387	enum counter_type type = COUNTER_CYCLES;
10388	enum counter_format format = FORMAT_DELTA;
10389
10390	while (add_command) {
10391
10392	if (sscanf(add_command, "msr0x%x", &msr_num) == 1)
10393	goto next;
10394
10395	if (sscanf(add_command, "msr%d", &msr_num) == 1)
10396	goto next;
10397
10398	BUILD_BUG_ON(ARRAY_SIZE(perf_device) <= 31);
10399	BUILD_BUG_ON(ARRAY_SIZE(perf_event) <= 31);
10400	if (sscanf(add_command, "perf/%31[^/]/%31[^,]", &perf_device[0], &perf_event[0]) == 2)
10401	goto next;
10402
10403	if (*add_command == '/') {
10404	path = add_command;
10405	goto next;
10406	}
10407
10408	if (sscanf(add_command, "u%d", &width) == 1) {
10409	if ((width == 32) || (width == 64))
10410	goto next;
10411	width = 64;
10412	}
10413	if (!strncmp(add_command, "cpu", strlen("cpu"))) {
10414	scope = SCOPE_CPU;
10415	goto next;
10416	}
10417	if (!strncmp(add_command, "core", strlen("core"))) {
10418	scope = SCOPE_CORE;
10419	goto next;
10420	}
10421	if (!strncmp(add_command, "package", strlen("package"))) {
10422	scope = SCOPE_PACKAGE;
10423	goto next;
10424	}
10425	if (!strncmp(add_command, "cycles", strlen("cycles"))) {
10426	type = COUNTER_CYCLES;
10427	goto next;
10428	}
10429	if (!strncmp(add_command, "seconds", strlen("seconds"))) {
10430	type = COUNTER_SECONDS;
10431	goto next;
10432	}
10433	if (!strncmp(add_command, "usec", strlen("usec"))) {
10434	type = COUNTER_USEC;
10435	goto next;
10436	}
10437	if (!strncmp(add_command, "raw", strlen("raw"))) {
10438	format = FORMAT_RAW;
10439	goto next;
10440	}
10441	if (!strncmp(add_command, "average", strlen("average"))) {
10442	format = FORMAT_AVERAGE;
10443	goto next;
10444	}
10445	if (!strncmp(add_command, "delta", strlen("delta"))) {
10446	format = FORMAT_DELTA;
10447	goto next;
10448	}
10449	if (!strncmp(add_command, "percent", strlen("percent"))) {
10450	format = FORMAT_PERCENT;
10451	goto next;
10452	}
10453
10454	BUILD_BUG_ON(ARRAY_SIZE(name_buffer) <= 18);
10455	if (sscanf(add_command, "%18s,%*s", name_buffer) == 1) {
10456	char *eos;
10457
10458	eos = strchr(name_buffer, ',');
10459	if (eos)
10460	*eos = '\0';
10461	goto next;
10462	}
10463
10464	next:
10465	add_command = strchr(add_command, ',');
10466	if (add_command) {
10467	*add_command = '\0';
10468	add_command++;
10469	}
10470
10471	}
10472	if ((msr_num == 0) && (path == NULL) && (perf_device[0] == '\0' || perf_event[0] == '\0')) {
10473	fprintf(stderr, "--add: (msrDDD | msr0xXXX | /path_to_counter | perf/device/event) required\n");
10474	fail++;
10475	}
10476
10477	/* Test for non-empty perf_device and perf_event */
10478	const bool is_perf_counter = perf_device[0] && perf_event[0];
10479
10480	/* generate default column header */
10481	if (*name_buffer == '\0') {
10482	if (is_perf_counter) {
10483	snprintf(buf: name_buffer, ARRAY_SIZE(name_buffer), fmt: "perf/%s", perf_event);
10484	} else {
10485	if (width == 32)
10486	sprintf(buf: name_buffer, fmt: "M0x%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
10487	else
10488	sprintf(buf: name_buffer, fmt: "M0X%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
10489	}
10490	}
10491
10492	if (is_perf_counter) {
10493	if (add_perf_counter(perf_device, perf_event, name_buffer, width, scope, type, format))
10494	fail++;
10495	} else {
10496	if (add_counter(msr_num, path, name: name_buffer, width, scope, type, format, flags: 0, id: 0))
10497	fail++;
10498	}
10499
10500	if (fail) {
10501	help();
10502	exit(1);
10503	}
10504	}
10505
10506	bool starts_with(const char *str, const char *prefix)
10507	{
10508	return strncmp(prefix, str, strlen(prefix)) == 0;
10509	}
10510
10511	int pmt_parse_from_path(const char *target_path, unsigned int *out_guid, unsigned int *out_seq)
10512	{
10513	struct pmt_diriter_t pmt_iter;
10514	const struct dirent *dirname;
10515	struct stat stat, target_stat;
10516	int fd_telem_dir = -1;
10517	int fd_target_dir;
10518	unsigned int seq = 0;
10519	unsigned long guid, target_guid;
10520	int ret = -1;
10521
10522	fd_target_dir = open(target_path, O_RDONLY | O_DIRECTORY);
10523	if (fd_target_dir == -1) {
10524	return -1;
10525	}
10526
10527	if (fstat(fd_target_dir, &target_stat) == -1) {
10528	fprintf(stderr, "%s: Failed to stat the target: %s", __func__, strerror(errno));
10529	exit(1);
10530	}
10531
10532	if (parse_telem_info_file(fd_dir: fd_target_dir, info_filename: "guid", format: "%lx", output: &target_guid)) {
10533	fprintf(stderr, "%s: Failed to parse the target guid file: %s", __func__, strerror(errno));
10534	exit(1);
10535	}
10536
10537	close(fd_target_dir);
10538
10539	pmt_diriter_init(iter: &pmt_iter);
10540
10541	for (dirname = pmt_diriter_begin(iter: &pmt_iter, SYSFS_TELEM_PATH); dirname != NULL; dirname = pmt_diriter_next(iter: &pmt_iter)) {
10542
10543	fd_telem_dir = openat(dirfd(pmt_iter.dir), dirname->d_name, O_RDONLY | O_DIRECTORY);
10544	if (fd_telem_dir == -1)
10545	continue;
10546
10547	if (parse_telem_info_file(fd_dir: fd_telem_dir, info_filename: "guid", format: "%lx", output: &guid)) {
10548	fprintf(stderr, "%s: Failed to parse the guid file: %s", __func__, strerror(errno));
10549	continue;
10550	}
10551
10552	if (fstat(fd_telem_dir, &stat) == -1) {
10553	fprintf(stderr, "%s: Failed to stat %s directory: %s", __func__, dirname->d_name, strerror(errno));
10554	continue;
10555	}
10556
10557	/*
10558	* If reached the same directory as target, exit the loop.
10559	* Seq has the correct value now.
10560	*/
10561	if (stat.st_dev == target_stat.st_dev && stat.st_ino == target_stat.st_ino) {
10562	ret = 0;
10563	break;
10564	}
10565
10566	/*
10567	* If reached directory with the same guid,
10568	* but it's not the target directory yet,
10569	* increment seq and continue the search.
10570	*/
10571	if (guid == target_guid)
10572	++seq;
10573
10574	close(fd_telem_dir);
10575	fd_telem_dir = -1;
10576	}
10577
10578	pmt_diriter_remove(iter: &pmt_iter);
10579
10580	if (fd_telem_dir != -1)
10581	close(fd_telem_dir);
10582
10583	if (!ret) {
10584	*out_guid = target_guid;
10585	*out_seq = seq;
10586	}
10587
10588	return ret;
10589	}
10590
10591	void parse_add_command_pmt(char *add_command)
10592	{
10593	char *name = NULL;
10594	char *type_name = NULL;
10595	char *format_name = NULL;
10596	char *direct_path = NULL;
10597	static const char direct_path_prefix[] = "path=";
10598	unsigned int offset;
10599	unsigned int lsb;
10600	unsigned int msb;
10601	unsigned int guid;
10602	unsigned int seq = 0; /* By default, pick first file in a sequence with a given GUID. */
10603	unsigned int domain_id;
10604	enum counter_scope scope = 0;
10605	enum pmt_datatype type = PMT_TYPE_RAW;
10606	enum counter_format format = FORMAT_RAW;
10607	bool has_offset = false;
10608	bool has_lsb = false;
10609	bool has_msb = false;
10610	bool has_format = true; /* Format has a default value. */
10611	bool has_guid = false;
10612	bool has_scope = false;
10613	bool has_type = true; /* Type has a default value. */
10614
10615	/* Consume the "pmt," prefix. */
10616	add_command = strchr(add_command, ',');
10617	if (!add_command) {
10618	help();
10619	exit(1);
10620	}
10621	++add_command;
10622
10623	while (add_command) {
10624	if (starts_with(str: add_command, prefix: "name=")) {
10625	name = add_command + strlen("name=");
10626	goto next;
10627	}
10628
10629	if (starts_with(str: add_command, prefix: "type=")) {
10630	type_name = add_command + strlen("type=");
10631	goto next;
10632	}
10633
10634	if (starts_with(str: add_command, prefix: "domain=")) {
10635	const size_t prefix_len = strlen("domain=");
10636
10637	if (sscanf(add_command + prefix_len, "cpu%u", &domain_id) == 1) {
10638	scope = SCOPE_CPU;
10639	has_scope = true;
10640	} else if (sscanf(add_command + prefix_len, "core%u", &domain_id) == 1) {
10641	scope = SCOPE_CORE;
10642	has_scope = true;
10643	} else if (sscanf(add_command + prefix_len, "package%u", &domain_id) == 1) {
10644	scope = SCOPE_PACKAGE;
10645	has_scope = true;
10646	}
10647
10648	if (!has_scope) {
10649	printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n", __func__);
10650	exit(1);
10651	}
10652
10653	goto next;
10654	}
10655
10656	if (starts_with(str: add_command, prefix: "format=")) {
10657	format_name = add_command + strlen("format=");
10658	goto next;
10659	}
10660
10661	if (sscanf(add_command, "offset=%u", &offset) == 1) {
10662	has_offset = true;
10663	goto next;
10664	}
10665
10666	if (sscanf(add_command, "lsb=%u", &lsb) == 1) {
10667	has_lsb = true;
10668	goto next;
10669	}
10670
10671	if (sscanf(add_command, "msb=%u", &msb) == 1) {
10672	has_msb = true;
10673	goto next;
10674	}
10675
10676	if (sscanf(add_command, "guid=%x", &guid) == 1) {
10677	has_guid = true;
10678	goto next;
10679	}
10680
10681	if (sscanf(add_command, "seq=%x", &seq) == 1)
10682	goto next;
10683
10684	if (strncmp(add_command, direct_path_prefix, strlen(direct_path_prefix)) == 0) {
10685	direct_path = add_command + strlen(direct_path_prefix);
10686	goto next;
10687	}
10688	next:
10689	add_command = strchr(add_command, ',');
10690	if (add_command) {
10691	*add_command = '\0';
10692	add_command++;
10693	}
10694	}
10695
10696	if (!name) {
10697	printf("%s: missing %s\n", __func__, "name");
10698	exit(1);
10699	}
10700
10701	if (strlen(name) >= PMT_COUNTER_NAME_SIZE_BYTES) {
10702	printf("%s: name has to be at most %d characters long\n", __func__, PMT_COUNTER_NAME_SIZE_BYTES);
10703	exit(1);
10704	}
10705
10706	if (format_name) {
10707	has_format = false;
10708
10709	if (strcmp("raw", format_name) == 0) {
10710	format = FORMAT_RAW;
10711	has_format = true;
10712	}
10713
10714	if (strcmp("average", format_name) == 0) {
10715	format = FORMAT_AVERAGE;
10716	has_format = true;
10717	}
10718
10719	if (strcmp("delta", format_name) == 0) {
10720	format = FORMAT_DELTA;
10721	has_format = true;
10722	}
10723
10724	if (!has_format) {
10725	fprintf(stderr, "%s: Invalid format %s. Expected raw, average or delta\n", __func__, format_name);
10726	exit(1);
10727	}
10728	}
10729
10730	if (type_name) {
10731	has_type = false;
10732
10733	if (strcmp("raw", type_name) == 0) {
10734	type = PMT_TYPE_RAW;
10735	has_type = true;
10736	}
10737
10738	if (strcmp("txtal_time", type_name) == 0) {
10739	type = PMT_TYPE_XTAL_TIME;
10740	has_type = true;
10741	}
10742
10743	if (strcmp("tcore_clock", type_name) == 0) {
10744	type = PMT_TYPE_TCORE_CLOCK;
10745	has_type = true;
10746	}
10747
10748	if (!has_type) {
10749	printf("%s: invalid %s: %s\n", __func__, "type", type_name);
10750	exit(1);
10751	}
10752	}
10753
10754	if (!has_offset) {
10755	printf("%s : missing %s\n", __func__, "offset");
10756	exit(1);
10757	}
10758
10759	if (!has_lsb) {
10760	printf("%s: missing %s\n", __func__, "lsb");
10761	exit(1);
10762	}
10763
10764	if (!has_msb) {
10765	printf("%s: missing %s\n", __func__, "msb");
10766	exit(1);
10767	}
10768
10769	if (direct_path && has_guid) {
10770	printf("%s: path and guid+seq parameters are mutually exclusive\n"
10771	"notice: passed guid=0x%x and path=%s\n", __func__, guid, direct_path);
10772	exit(1);
10773	}
10774
10775	if (direct_path) {
10776	if (pmt_parse_from_path(target_path: direct_path, out_guid: &guid, out_seq: &seq)) {
10777	printf("%s: failed to parse PMT file from %s\n", __func__, direct_path);
10778	exit(1);
10779	}
10780
10781	/* GUID was just infered from the direct path. */
10782	has_guid = true;
10783	}
10784
10785	if (!has_guid) {
10786	printf("%s: missing %s\n", __func__, "guid or path");
10787	exit(1);
10788	}
10789
10790	if (!has_scope) {
10791	printf("%s: missing %s\n", __func__, "scope");
10792	exit(1);
10793	}
10794
10795	if (lsb > msb) {
10796	printf("%s: lsb > msb doesn't make sense\n", __func__);
10797	exit(1);
10798	}
10799
10800	pmt_add_counter(guid, seq, name, type, lsb, msb, offset, scope, format, domain_id, mode: PMT_OPEN_REQUIRED);
10801	}
10802
10803	void parse_add_command(char *add_command)
10804	{
10805	if (strncmp(add_command, "pmt", strlen("pmt")) == 0)
10806	return parse_add_command_pmt(add_command);
10807	return parse_add_command_msr(add_command);
10808	}
10809
10810	int is_deferred_add(char *name)
10811	{
10812	int i;
10813
10814	for (i = 0; i < deferred_add_index; ++i)
10815	if (!strcmp(name, deferred_add_names[i])) {
10816	deferred_add_consumed |= (1 << i);
10817	return 1;
10818	}
10819	return 0;
10820	}
10821
10822	int is_deferred_skip(char *name)
10823	{
10824	int i;
10825
10826	for (i = 0; i < deferred_skip_index; ++i)
10827	if (!strcmp(name, deferred_skip_names[i])) {
10828	deferred_skip_consumed |= (1 << i);
10829	return 1;
10830	}
10831	return 0;
10832	}
10833
10834	void verify_deferred_consumed(void)
10835	{
10836	int i;
10837	int fail = 0;
10838
10839	for (i = 0; i < deferred_add_index; ++i) {
10840	if (!(deferred_add_consumed & (1 << i))) {
10841	warnx("Counter '%s' can not be added.", deferred_add_names[i]);
10842	fail++;
10843	}
10844	}
10845	for (i = 0; i < deferred_skip_index; ++i) {
10846	if (!(deferred_skip_consumed & (1 << i))) {
10847	warnx("Counter '%s' can not be skipped.", deferred_skip_names[i]);
10848	fail++;
10849	}
10850	}
10851	if (fail)
10852	exit(-EINVAL);
10853	}
10854
10855	void probe_cpuidle_residency(void)
10856	{
10857	char path[64];
10858	char name_buf[16];
10859	FILE *input;
10860	int state;
10861	int min_state = 1024, max_state = 0;
10862	char *sp;
10863
10864	for (state = 10; state >= 0; --state) {
10865
10866	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state);
10867	input = fopen(path, "r");
10868	if (input == NULL)
10869	continue;
10870	if (!fgets(name_buf, sizeof(name_buf), input))
10871	err(1, "%s: failed to read file", path);
10872
10873	/* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
10874	sp = strchr(name_buf, '-');
10875	if (!sp)
10876	sp = strchrnul(name_buf, '\n');
10877	*sp = '%';
10878	*(sp + 1) = '\0';
10879
10880	remove_underbar(s: name_buf);
10881
10882	fclose(input);
10883
10884	sprintf(buf: path, fmt: "cpuidle/state%d/time", state);
10885
10886	if (!DO_BIC(BIC_pct_idle) && !is_deferred_add(name: name_buf))
10887	continue;
10888
10889	if (is_deferred_skip(name: name_buf))
10890	continue;
10891
10892	add_counter(msr_num: 0, path, name: name_buf, width: 32, scope: SCOPE_CPU, type: COUNTER_USEC, format: FORMAT_PERCENT, SYSFS_PERCPU, id: 0);
10893
10894	if (state > max_state)
10895	max_state = state;
10896	if (state < min_state)
10897	min_state = state;
10898	}
10899	}
10900
10901	void probe_cpuidle_counts(void)
10902	{
10903	char path[64];
10904	char name_buf[16];
10905	FILE *input;
10906	int state;
10907	int min_state = 1024, max_state = 0;
10908	char *sp;
10909
10910	if (!DO_BIC(BIC_cpuidle))
10911	return;
10912
10913	for (state = 10; state >= 0; --state) {
10914
10915	sprintf(buf: path, fmt: "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state);
10916	input = fopen(path, "r");
10917	if (input == NULL)
10918	continue;
10919	if (!fgets(name_buf, sizeof(name_buf), input))
10920	err(1, "%s: failed to read file", path);
10921	fclose(input);
10922
10923	remove_underbar(s: name_buf);
10924
10925	if (!DO_BIC(BIC_cpuidle) && !is_deferred_add(name: name_buf))
10926	continue;
10927
10928	if (is_deferred_skip(name: name_buf))
10929	continue;
10930
10931	/* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
10932	sp = strchr(name_buf, '-');
10933	if (!sp)
10934	sp = strchrnul(name_buf, '\n');
10935
10936	/*
10937	* The 'below' sysfs file always contains 0 for the deepest state (largest index),
10938	* do not add it.
10939	*/
10940	if (state != max_state) {
10941	/*
10942	* Add 'C1+' for C1, and so on. The 'below' sysfs file always contains 0 for
10943	* the last state, so do not add it.
10944	*/
10945
10946	*sp = '+';
10947	*(sp + 1) = '\0';
10948	sprintf(buf: path, fmt: "cpuidle/state%d/below", state);
10949	add_counter(msr_num: 0, path, name: name_buf, width: 64, scope: SCOPE_CPU, type: COUNTER_ITEMS, format: FORMAT_DELTA, SYSFS_PERCPU, id: 0);
10950	}
10951
10952	*sp = '\0';
10953	sprintf(buf: path, fmt: "cpuidle/state%d/usage", state);
10954	add_counter(msr_num: 0, path, name: name_buf, width: 64, scope: SCOPE_CPU, type: COUNTER_ITEMS, format: FORMAT_DELTA, SYSFS_PERCPU, id: 0);
10955
10956	/*
10957	* The 'above' sysfs file always contains 0 for the shallowest state (smallest
10958	* index), do not add it.
10959	*/
10960	if (state != min_state) {
10961	*sp = '-';
10962	*(sp + 1) = '\0';
10963	sprintf(buf: path, fmt: "cpuidle/state%d/above", state);
10964	add_counter(msr_num: 0, path, name: name_buf, width: 64, scope: SCOPE_CPU, type: COUNTER_ITEMS, format: FORMAT_DELTA, SYSFS_PERCPU, id: 0);
10965	}
10966	}
10967	}
10968
10969	/*
10970	* parse cpuset with following syntax
10971	* 1,2,4..6,8-10 and set bits in cpu_subset
10972	*/
10973	void parse_cpu_command(char *optarg)
10974	{
10975	if (!strcmp(optarg, "core")) {
10976	if (cpu_subset)
10977	goto error;
10978	show_core_only++;
10979	return;
10980	}
10981	if (!strcmp(optarg, "package")) {
10982	if (cpu_subset)
10983	goto error;
10984	show_pkg_only++;
10985	return;
10986	}
10987	if (show_core_only || show_pkg_only)
10988	goto error;
10989
10990	cpu_subset = CPU_ALLOC(CPU_SUBSET_MAXCPUS);
10991	if (cpu_subset == NULL)
10992	err(3, "CPU_ALLOC");
10993	cpu_subset_size = CPU_ALLOC_SIZE(CPU_SUBSET_MAXCPUS);
10994
10995	CPU_ZERO_S(cpu_subset_size, cpu_subset);
10996
10997	if (parse_cpu_str(optarg, cpu_subset, cpu_subset_size))
10998	goto error;
10999
11000	return;
11001
11002	error:
11003	fprintf(stderr, "\"--cpu %s\" malformed\n", optarg);
11004	help();
11005	exit(-1);
11006	}
11007
11008	void cmdline(int argc, char **argv)
11009	{
11010	int opt;
11011	int option_index = 0;
11012	static struct option long_options[] = {
11013	{ "add", required_argument, 0, 'a' },
11014	{ "cpu", required_argument, 0, 'c' },
11015	{ "Dump", no_argument, 0, 'D' },
11016	{ "debug", no_argument, 0, 'd' }, /* internal, not documented */
11017	{ "enable", required_argument, 0, 'e' },
11018	{ "force", no_argument, 0, 'f' },
11019	{ "interval", required_argument, 0, 'i' },
11020	{ "IPC", no_argument, 0, 'I' },
11021	{ "num_iterations", required_argument, 0, 'n' },
11022	{ "header_iterations", required_argument, 0, 'N' },
11023	{ "help", no_argument, 0, 'h' },
11024	{ "hide", required_argument, 0, 'H' }, // meh, -h taken by --help
11025	{ "Joules", no_argument, 0, 'J' },
11026	{ "list", no_argument, 0, 'l' },
11027	{ "out", required_argument, 0, 'o' },
11028	{ "quiet", no_argument, 0, 'q' },
11029	{ "no-msr", no_argument, 0, 'M' },
11030	{ "no-perf", no_argument, 0, 'P' },
11031	{ "show", required_argument, 0, 's' },
11032	{ "Summary", no_argument, 0, 'S' },
11033	{ "TCC", required_argument, 0, 'T' },
11034	{ "version", no_argument, 0, 'v' },
11035	{ 0, 0, 0, 0 }
11036	};
11037
11038	progname = argv[0];
11039
11040	/*
11041	* Parse some options early, because they may make other options invalid,
11042	* like adding the MSR counter with --add and at the same time using --no-msr.
11043	*/
11044	while ((opt = getopt_long_only(argc, argv, "+MPn:", long_options, &option_index)) != -1) {
11045	switch (opt) {
11046	case 'M':
11047	no_msr = 1;
11048	break;
11049	case 'P':
11050	no_perf = 1;
11051	break;
11052	default:
11053	break;
11054	}
11055	}
11056	optind = 0;
11057
11058	while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:o:qMST:v", long_options, &option_index)) != -1) {
11059	switch (opt) {
11060	case 'a':
11061	parse_add_command(optarg);
11062	break;
11063	case 'c':
11064	parse_cpu_command(optarg);
11065	break;
11066	case 'D':
11067	dump_only++;
11068	/*
11069	* Force the no_perf early to prevent using it as a source.
11070	* User asks for raw values, but perf returns them relative
11071	* to the opening of the file descriptor.
11072	*/
11073	no_perf = 1;
11074	break;
11075	case 'e':
11076	/* --enable specified counter, without clearning existing list */
11077	bic_lookup(&bic_enabled, optarg, SHOW_LIST);
11078	break;
11079	case 'f':
11080	force_load++;
11081	break;
11082	case 'd':
11083	debug++;
11084	bic_set_all(&bic_enabled);
11085	break;
11086	case 'H':
11087	/*
11088	* --hide: do not show those specified
11089	* multiple invocations simply clear more bits in enabled mask
11090	*/
11091	{
11092	cpu_set_t bic_group_hide;
11093
11094	BIC_INIT(&bic_group_hide);
11095
11096	bic_lookup(&bic_group_hide, optarg, HIDE_LIST);
11097	bic_clear_bits(&bic_enabled, &bic_group_hide);
11098	}
11099	break;
11100	case 'h':
11101	default:
11102	help();
11103	exit(1);
11104	case 'i':
11105	{
11106	double interval = strtod(optarg, NULL);
11107
11108	if (interval < 0.001) {
11109	fprintf(outf, "interval %f seconds is too small\n", interval);
11110	exit(2);
11111	}
11112
11113	interval_tv.tv_sec = interval_ts.tv_sec = interval;
11114	interval_tv.tv_usec = (interval - interval_tv.tv_sec) * 1000000;
11115	interval_ts.tv_nsec = (interval - interval_ts.tv_sec) * 1000000000;
11116	}
11117	break;
11118	case 'J':
11119	rapl_joules++;
11120	break;
11121	case 'l':
11122	bic_set_all(&bic_enabled);
11123	list_header_only++;
11124	quiet++;
11125	break;
11126	case 'o':
11127	outf = fopen_or_die(optarg, "w");
11128	break;
11129	case 'q':
11130	quiet = 1;
11131	break;
11132	case 'M':
11133	case 'P':
11134	/* Parsed earlier */
11135	break;
11136	case 'n':
11137	num_iterations = strtod(optarg, NULL);
11138
11139	if (num_iterations <= 0) {
11140	fprintf(outf, "iterations %d should be positive number\n", num_iterations);
11141	exit(2);
11142	}
11143	break;
11144	case 'N':
11145	header_iterations = strtod(optarg, NULL);
11146
11147	if (header_iterations <= 0) {
11148	fprintf(outf, "iterations %d should be positive number\n", header_iterations);
11149	exit(2);
11150	}
11151	break;
11152	case 's':
11153	/*
11154	* --show: show only those specified
11155	* The 1st invocation will clear and replace the enabled mask
11156	* subsequent invocations can add to it.
11157	*/
11158	if (shown == 0)
11159	BIC_INIT(&bic_enabled);
11160	bic_lookup(&bic_enabled, optarg, SHOW_LIST);
11161	shown = 1;
11162	break;
11163	case 'S':
11164	summary_only++;
11165	break;
11166	case 'T':
11167	tj_max_override = atoi(optarg);
11168	break;
11169	case 'v':
11170	print_version();
11171	exit(0);
11172	break;
11173	}
11174	}
11175	}
11176
11177	void set_rlimit(void)
11178	{
11179	struct rlimit limit;
11180
11181	if (getrlimit(RLIMIT_NOFILE, &limit) < 0)
11182	err(1, "Failed to get rlimit");
11183
11184	if (limit.rlim_max < MAX_NOFILE)
11185	limit.rlim_max = MAX_NOFILE;
11186	if (limit.rlim_cur < MAX_NOFILE)
11187	limit.rlim_cur = MAX_NOFILE;
11188
11189	if (setrlimit(RLIMIT_NOFILE, &limit) < 0)
11190	err(1, "Failed to set rlimit");
11191	}
11192
11193	int main(int argc, char **argv)
11194	{
11195	int fd, ret;
11196
11197	bic_groups_init();
11198
11199	fd = open("/sys/fs/cgroup/cgroup.procs", O_WRONLY);
11200	if (fd < 0)
11201	goto skip_cgroup_setting;
11202
11203	ret = write(fd, "0\n", 2);
11204	if (ret == -1)
11205	perror("Can't update cgroup\n");
11206
11207	close(fd);
11208
11209	skip_cgroup_setting:
11210	outf = stderr;
11211	cmdline(argc, argv);
11212
11213	if (!quiet) {
11214	print_version();
11215	print_bootcmd();
11216	}
11217
11218	probe_cpuidle_residency();
11219	probe_cpuidle_counts();
11220
11221	verify_deferred_consumed();
11222
11223	if (!getuid())
11224	set_rlimit();
11225
11226	turbostat_init();
11227
11228	if (!no_msr)
11229	msr_sum_record();
11230
11231	/* dump counters and exit */
11232	if (dump_only)
11233	return get_and_dump_counters();
11234
11235	/* list header and exit */
11236	if (list_header_only) {
11237	print_header(delim: ",");
11238	flush_output_stdout();
11239	return 0;
11240	}
11241
11242	/*
11243	* if any params left, it must be a command to fork
11244	*/
11245	if (argc - optind)
11246	return fork_it(argv + optind);
11247	else
11248	turbostat_loop();
11249
11250	return 0;
11251	}
11252