1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* This is rewrite of original c2c tool introduced in here:
4	* http://lwn.net/Articles/588866/
5	*
6	* The original tool was changed to fit in current perf state.
7	*
8	* Original authors:
9	* Don Zickus <dzickus@redhat.com>
10	* Dick Fowles <fowles@inreach.com>
11	* Joe Mario <jmario@redhat.com>
12	*/
13	#include <errno.h>
14	#include <inttypes.h>
15	#include <linux/compiler.h>
16	#include <linux/err.h>
17	#include <linux/kernel.h>
18	#include <linux/stringify.h>
19	#include <linux/zalloc.h>
20	#include <asm/bug.h>
21	#include <sys/param.h>
22	#include "debug.h"
23	#include "builtin.h"
24	#include <perf/cpumap.h>
25	#include <subcmd/pager.h>
26	#include <subcmd/parse-options.h>
27	#include "map_symbol.h"
28	#include "mem-events.h"
29	#include "session.h"
30	#include "hist.h"
31	#include "sort.h"
32	#include "tool.h"
33	#include "cacheline.h"
34	#include "data.h"
35	#include "event.h"
36	#include "evlist.h"
37	#include "evsel.h"
38	#include "ui/browsers/hists.h"
39	#include "thread.h"
40	#include "mem2node.h"
41	#include "mem-info.h"
42	#include "symbol.h"
43	#include "ui/ui.h"
44	#include "ui/progress.h"
45	#include "pmus.h"
46	#include "string2.h"
47	#include "util/util.h"
48
49	struct c2c_hists {
50	struct hists hists;
51	struct perf_hpp_list list;
52	struct c2c_stats stats;
53	};
54
55	struct compute_stats {
56	struct stats lcl_hitm;
57	struct stats rmt_hitm;
58	struct stats lcl_peer;
59	struct stats rmt_peer;
60	struct stats load;
61	};
62
63	struct c2c_hist_entry {
64	struct c2c_hists *hists;
65	struct c2c_stats stats;
66	unsigned long *cpuset;
67	unsigned long *nodeset;
68	struct c2c_stats *node_stats;
69	unsigned int cacheline_idx;
70
71	struct compute_stats cstats;
72
73	unsigned long paddr;
74	unsigned long paddr_cnt;
75	bool paddr_zero;
76	char *nodestr;
77
78	/*
79	* must be at the end,
80	* because of its callchain dynamic entry
81	*/
82	struct hist_entry he;
83	};
84
85	static char const *coalesce_default = "iaddr";
86
87	struct perf_c2c {
88	struct perf_tool tool;
89	struct c2c_hists hists;
90	struct mem2node mem2node;
91
92	unsigned long **nodes;
93	int nodes_cnt;
94	int cpus_cnt;
95	int *cpu2node;
96	int node_info;
97
98	bool show_src;
99	bool show_all;
100	bool use_stdio;
101	bool stats_only;
102	bool symbol_full;
103	bool stitch_lbr;
104
105	/* Shared cache line stats */
106	struct c2c_stats shared_clines_stats;
107	int shared_clines;
108
109	int display;
110
111	const char *coalesce;
112	char *cl_sort;
113	char *cl_resort;
114	char *cl_output;
115	};
116
117	enum {
118	DISPLAY_LCL_HITM,
119	DISPLAY_RMT_HITM,
120	DISPLAY_TOT_HITM,
121	DISPLAY_SNP_PEER,
122	DISPLAY_MAX,
123	};
124
125	static const char *display_str[DISPLAY_MAX] = {
126	[DISPLAY_LCL_HITM] = "Local HITMs",
127	[DISPLAY_RMT_HITM] = "Remote HITMs",
128	[DISPLAY_TOT_HITM] = "Total HITMs",
129	[DISPLAY_SNP_PEER] = "Peer Snoop",
130	};
131
132	static const struct option c2c_options[] = {
133	OPT_INCR('v', "verbose", &verbose, "be more verbose (show counter open errors, etc)"),
134	OPT_END()
135	};
136
137	static struct perf_c2c c2c;
138
139	static void *c2c_he_zalloc(size_t size)
140	{
141	struct c2c_hist_entry *c2c_he;
142
143	c2c_he = zalloc(size + sizeof(*c2c_he));
144	if (!c2c_he)
145	return NULL;
146
147	c2c_he->cpuset = bitmap_zalloc(c2c.cpus_cnt);
148	if (!c2c_he->cpuset)
149	goto out_free;
150
151	c2c_he->nodeset = bitmap_zalloc(c2c.nodes_cnt);
152	if (!c2c_he->nodeset)
153	goto out_free;
154
155	c2c_he->node_stats = zalloc(c2c.nodes_cnt * sizeof(*c2c_he->node_stats));
156	if (!c2c_he->node_stats)
157	goto out_free;
158
159	init_stats(&c2c_he->cstats.lcl_hitm);
160	init_stats(&c2c_he->cstats.rmt_hitm);
161	init_stats(&c2c_he->cstats.lcl_peer);
162	init_stats(&c2c_he->cstats.rmt_peer);
163	init_stats(&c2c_he->cstats.load);
164
165	return &c2c_he->he;
166
167	out_free:
168	zfree(&c2c_he->nodeset);
169	zfree(&c2c_he->cpuset);
170	free(c2c_he);
171	return NULL;
172	}
173
174	static void c2c_he_free(void *he)
175	{
176	struct c2c_hist_entry *c2c_he;
177
178	c2c_he = container_of(he, struct c2c_hist_entry, he);
179	if (c2c_he->hists) {
180	hists__delete_entries(&c2c_he->hists->hists);
181	zfree(&c2c_he->hists);
182	}
183
184	zfree(&c2c_he->cpuset);
185	zfree(&c2c_he->nodeset);
186	zfree(&c2c_he->nodestr);
187	zfree(&c2c_he->node_stats);
188	free(c2c_he);
189	}
190
191	static struct hist_entry_ops c2c_entry_ops = {
192	.new = c2c_he_zalloc,
193	.free = c2c_he_free,
194	};
195
196	static int c2c_hists__init(struct c2c_hists *hists,
197	const char *sort,
198	int nr_header_lines);
199
200	static struct c2c_hists*
201	he__get_c2c_hists(struct hist_entry *he,
202	const char *sort,
203	int nr_header_lines)
204	{
205	struct c2c_hist_entry *c2c_he;
206	struct c2c_hists *hists;
207	int ret;
208
209	c2c_he = container_of(he, struct c2c_hist_entry, he);
210	if (c2c_he->hists)
211	return c2c_he->hists;
212
213	hists = c2c_he->hists = zalloc(sizeof(*hists));
214	if (!hists)
215	return NULL;
216
217	ret = c2c_hists__init(hists, sort, nr_header_lines);
218	if (ret) {
219	free(hists);
220	return NULL;
221	}
222
223	return hists;
224	}
225
226	static void c2c_he__set_cpu(struct c2c_hist_entry *c2c_he,
227	struct perf_sample *sample)
228	{
229	if (WARN_ONCE(sample->cpu == (unsigned int) -1,
230	"WARNING: no sample cpu value"))
231	return;
232
233	__set_bit(sample->cpu, c2c_he->cpuset);
234	}
235
236	static void c2c_he__set_node(struct c2c_hist_entry *c2c_he,
237	struct perf_sample *sample)
238	{
239	int node;
240
241	if (!sample->phys_addr) {
242	c2c_he->paddr_zero = true;
243	return;
244	}
245
246	node = mem2node__node(&c2c.mem2node, sample->phys_addr);
247	if (WARN_ONCE(node < 0, "WARNING: failed to find node\n"))
248	return;
249
250	__set_bit(node, c2c_he->nodeset);
251
252	if (c2c_he->paddr != sample->phys_addr) {
253	c2c_he->paddr_cnt++;
254	c2c_he->paddr = sample->phys_addr;
255	}
256	}
257
258	static void compute_stats(struct c2c_hist_entry *c2c_he,
259	struct c2c_stats *stats,
260	u64 weight)
261	{
262	struct compute_stats *cstats = &c2c_he->cstats;
263
264	if (stats->rmt_hitm)
265	update_stats(&cstats->rmt_hitm, weight);
266	else if (stats->lcl_hitm)
267	update_stats(&cstats->lcl_hitm, weight);
268	else if (stats->rmt_peer)
269	update_stats(&cstats->rmt_peer, weight);
270	else if (stats->lcl_peer)
271	update_stats(&cstats->lcl_peer, weight);
272	else if (stats->load)
273	update_stats(&cstats->load, weight);
274	}
275
276	static int process_sample_event(const struct perf_tool *tool __maybe_unused,
277	union perf_event *event,
278	struct perf_sample *sample,
279	struct evsel *evsel,
280	struct machine *machine)
281	{
282	struct c2c_hists *c2c_hists = &c2c.hists;
283	struct c2c_hist_entry *c2c_he;
284	struct c2c_stats stats = { .nr_entries = 0, };
285	struct hist_entry *he;
286	struct addr_location al;
287	struct mem_info *mi, *mi_dup;
288	struct callchain_cursor *cursor;
289	int ret;
290
291	addr_location__init(&al);
292	if (machine__resolve(machine, &al, sample) < 0) {
293	pr_debug("problem processing %d event, skipping it.\n",
294	event->header.type);
295	ret = -1;
296	goto out;
297	}
298
299	if (c2c.stitch_lbr)
300	thread__set_lbr_stitch_enable(al.thread, true);
301
302	cursor = get_tls_callchain_cursor();
303	ret = sample__resolve_callchain(sample, cursor, NULL,
304	evsel, &al, sysctl_perf_event_max_stack);
305	if (ret)
306	goto out;
307
308	mi = sample__resolve_mem(sample, &al);
309	if (mi == NULL) {
310	ret = -ENOMEM;
311	goto out;
312	}
313
314	/*
315	* The mi object is released in hists__add_entry_ops,
316	* if it gets sorted out into existing data, so we need
317	* to take the copy now.
318	*/
319	mi_dup = mem_info__get(mi);
320
321	c2c_decode_stats(&stats, mi);
322
323	he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops,
324	&al, NULL, NULL, mi, NULL,
325	sample, true);
326	if (he == NULL)
327	goto free_mi;
328
329	c2c_he = container_of(he, struct c2c_hist_entry, he);
330	c2c_add_stats(&c2c_he->stats, &stats);
331	c2c_add_stats(&c2c_hists->stats, &stats);
332
333	c2c_he__set_cpu(c2c_he, sample: sample);
334	c2c_he__set_node(c2c_he, sample: sample);
335
336	hists__inc_nr_samples(&c2c_hists->hists, he->filtered);
337	ret = hist_entry__append_callchain(he, sample);
338
339	if (!ret) {
340	/*
341	* There's already been warning about missing
342	* sample's cpu value. Let's account all to
343	* node 0 in this case, without any further
344	* warning.
345	*
346	* Doing node stats only for single callchain data.
347	*/
348	int cpu = sample->cpu == (unsigned int) -1 ? 0 : sample->cpu;
349	int node = c2c.cpu2node[cpu];
350
351	mi = mi_dup;
352
353	c2c_hists = he__get_c2c_hists(he, sort: c2c.cl_sort, nr_header_lines: 2);
354	if (!c2c_hists)
355	goto free_mi;
356
357	he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops,
358	&al, NULL, NULL, mi, NULL,
359	sample, true);
360	if (he == NULL)
361	goto free_mi;
362
363	c2c_he = container_of(he, struct c2c_hist_entry, he);
364	c2c_add_stats(&c2c_he->stats, &stats);
365	c2c_add_stats(&c2c_hists->stats, &stats);
366	c2c_add_stats(&c2c_he->node_stats[node], &stats);
367
368	compute_stats(c2c_he, &stats, sample->weight);
369
370	c2c_he__set_cpu(c2c_he, sample);
371	c2c_he__set_node(c2c_he, sample);
372
373	hists__inc_nr_samples(&c2c_hists->hists, he->filtered);
374	ret = hist_entry__append_callchain(he, sample);
375	}
376
377	out:
378	addr_location__exit(&al);
379	return ret;
380
381	free_mi:
382	mem_info__put(mi_dup);
383	mem_info__put(mi);
384	ret = -ENOMEM;
385	goto out;
386	}
387
388	static const char * const c2c_usage[] = {
389	"perf c2c {record|report}",
390	NULL
391	};
392
393	static const char * const __usage_report[] = {
394	"perf c2c report",
395	NULL
396	};
397
398	static const char * const *report_c2c_usage = __usage_report;
399
400	#define C2C_HEADER_MAX 2
401
402	struct c2c_header {
403	struct {
404	const char *text;
405	int span;
406	} line[C2C_HEADER_MAX];
407	};
408
409	struct c2c_dimension {
410	struct c2c_header header;
411	const char *name;
412	int width;
413	struct sort_entry *se;
414
415	int64_t (*cmp)(struct perf_hpp_fmt *fmt,
416	struct hist_entry *, struct hist_entry *);
417	int (*entry)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
418	struct hist_entry *he);
419	int (*color)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
420	struct hist_entry *he);
421	};
422
423	struct c2c_fmt {
424	struct perf_hpp_fmt fmt;
425	struct c2c_dimension *dim;
426	};
427
428	#define SYMBOL_WIDTH 30
429
430	static struct c2c_dimension dim_symbol;
431	static struct c2c_dimension dim_srcline;
432
433	static int symbol_width(struct hists *hists, struct sort_entry *se)
434	{
435	int width = hists__col_len(hists, se->se_width_idx);
436
437	if (!c2c.symbol_full)
438	width = MIN(width, SYMBOL_WIDTH);
439
440	return width;
441	}
442
443	static int c2c_width(struct perf_hpp_fmt *fmt,
444	struct perf_hpp *hpp __maybe_unused,
445	struct hists *hists)
446	{
447	struct c2c_fmt *c2c_fmt;
448	struct c2c_dimension *dim;
449
450	c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
451	dim = c2c_fmt->dim;
452
453	if (dim == &dim_symbol || dim == &dim_srcline)
454	return symbol_width(hists, dim->se);
455
456	return dim->se ? hists__col_len(hists, dim->se->se_width_idx) :
457	c2c_fmt->dim->width;
458	}
459
460	static int c2c_header(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
461	struct hists *hists, int line, int *span)
462	{
463	struct perf_hpp_list *hpp_list = hists->hpp_list;
464	struct c2c_fmt *c2c_fmt;
465	struct c2c_dimension *dim;
466	const char *text = NULL;
467	int width = c2c_width(fmt, hpp, hists);
468
469	c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
470	dim = c2c_fmt->dim;
471
472	if (dim->se) {
473	text = dim->header.line[line].text;
474	/* Use the last line from sort_entry if not defined. */
475	if (!text && (line == hpp_list->nr_header_lines - 1))
476	text = dim->se->se_header;
477	} else {
478	text = dim->header.line[line].text;
479
480	if (*span) {
481	(*span)--;
482	return 0;
483	} else {
484	*span = dim->header.line[line].span;
485	}
486	}
487
488	if (text == NULL)
489	text = "";
490
491	return scnprintf(hpp->buf, hpp->size, "%*s", width, text);
492	}
493
494	#define HEX_STR(__s, __v) \
495	({ \
496	scnprintf(__s, sizeof(__s), "0x%" PRIx64, __v); \
497	__s; \
498	})
499
500	static int64_t
501	dcacheline_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
502	struct hist_entry *left, struct hist_entry *right)
503	{
504	return sort__dcacheline_cmp(left, right);
505	}
506
507	static int dcacheline_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
508	struct hist_entry *he)
509	{
510	uint64_t addr = 0;
511	int width = c2c_width(fmt, hpp, he->hists);
512	char buf[20];
513
514	if (he->mem_info)
515	addr = cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl);
516
517	return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr));
518	}
519
520	static int
521	dcacheline_node_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
522	struct hist_entry *he)
523	{
524	struct c2c_hist_entry *c2c_he;
525	int width = c2c_width(fmt, hpp, he->hists);
526
527	c2c_he = container_of(he, struct c2c_hist_entry, he);
528	if (WARN_ON_ONCE(!c2c_he->nodestr))
529	return 0;
530
531	return scnprintf(hpp->buf, hpp->size, "%*s", width, c2c_he->nodestr);
532	}
533
534	static int
535	dcacheline_node_count(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
536	struct hist_entry *he)
537	{
538	struct c2c_hist_entry *c2c_he;
539	int width = c2c_width(fmt, hpp, he->hists);
540
541	c2c_he = container_of(he, struct c2c_hist_entry, he);
542	return scnprintf(hpp->buf, hpp->size, "%*lu", width, c2c_he->paddr_cnt);
543	}
544
545	static int offset_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
546	struct hist_entry *he)
547	{
548	uint64_t addr = 0;
549	int width = c2c_width(fmt, hpp, he->hists);
550	char buf[20];
551
552	if (he->mem_info)
553	addr = cl_offset(mem_info__daddr(he->mem_info)->al_addr, chk_double_cl);
554
555	return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr));
556	}
557
558	static int64_t
559	offset_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
560	struct hist_entry *left, struct hist_entry *right)
561	{
562	uint64_t l = 0, r = 0;
563
564	if (left->mem_info)
565	l = cl_offset(mem_info__daddr(left->mem_info)->addr, chk_double_cl);
566
567	if (right->mem_info)
568	r = cl_offset(mem_info__daddr(right->mem_info)->addr, chk_double_cl);
569
570	return (int64_t)(r - l);
571	}
572
573	static int
574	iaddr_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
575	struct hist_entry *he)
576	{
577	uint64_t addr = 0;
578	int width = c2c_width(fmt, hpp, he->hists);
579	char buf[20];
580
581	if (he->mem_info)
582	addr = mem_info__iaddr(he->mem_info)->addr;
583
584	return scnprintf(hpp->buf, hpp->size, "%*s", width, HEX_STR(buf, addr));
585	}
586
587	static int64_t
588	iaddr_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
589	struct hist_entry *left, struct hist_entry *right)
590	{
591	return sort__iaddr_cmp(left, right);
592	}
593
594	static int
595	tot_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
596	struct hist_entry *he)
597	{
598	struct c2c_hist_entry *c2c_he;
599	int width = c2c_width(fmt, hpp, he->hists);
600	unsigned int tot_hitm;
601
602	c2c_he = container_of(he, struct c2c_hist_entry, he);
603	tot_hitm = c2c_he->stats.lcl_hitm + c2c_he->stats.rmt_hitm;
604
605	return scnprintf(hpp->buf, hpp->size, "%*u", width, tot_hitm);
606	}
607
608	static int64_t
609	tot_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
610	struct hist_entry *left, struct hist_entry *right)
611	{
612	struct c2c_hist_entry *c2c_left;
613	struct c2c_hist_entry *c2c_right;
614	uint64_t tot_hitm_left;
615	uint64_t tot_hitm_right;
616
617	c2c_left = container_of(left, struct c2c_hist_entry, he);
618	c2c_right = container_of(right, struct c2c_hist_entry, he);
619
620	tot_hitm_left = c2c_left->stats.lcl_hitm + c2c_left->stats.rmt_hitm;
621	tot_hitm_right = c2c_right->stats.lcl_hitm + c2c_right->stats.rmt_hitm;
622
623	return tot_hitm_left - tot_hitm_right;
624	}
625
626	#define STAT_FN_ENTRY(__f) \
627	static int \
628	__f ## _entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, \
629	struct hist_entry *he) \
630	{ \
631	struct c2c_hist_entry *c2c_he; \
632	int width = c2c_width(fmt, hpp, he->hists); \
633	\
634	c2c_he = container_of(he, struct c2c_hist_entry, he); \
635	return scnprintf(hpp->buf, hpp->size, "%*u", width, \
636	c2c_he->stats.__f); \
637	}
638
639	#define STAT_FN_CMP(__f) \
640	static int64_t \
641	__f ## _cmp(struct perf_hpp_fmt *fmt __maybe_unused, \
642	struct hist_entry *left, struct hist_entry *right) \
643	{ \
644	struct c2c_hist_entry *c2c_left, *c2c_right; \
645	\
646	c2c_left = container_of(left, struct c2c_hist_entry, he); \
647	c2c_right = container_of(right, struct c2c_hist_entry, he); \
648	return (uint64_t) c2c_left->stats.__f - \
649	(uint64_t) c2c_right->stats.__f; \
650	}
651
652	#define STAT_FN(__f) \
653	STAT_FN_ENTRY(__f) \
654	STAT_FN_CMP(__f)
655
656	STAT_FN(rmt_hitm)
657	STAT_FN(lcl_hitm)
658	STAT_FN(rmt_peer)
659	STAT_FN(lcl_peer)
660	STAT_FN(tot_peer)
661	STAT_FN(store)
662	STAT_FN(st_l1hit)
663	STAT_FN(st_l1miss)
664	STAT_FN(st_na)
665	STAT_FN(ld_fbhit)
666	STAT_FN(ld_l1hit)
667	STAT_FN(ld_l2hit)
668	STAT_FN(ld_llchit)
669	STAT_FN(rmt_hit)
670
671	static uint64_t get_load_llc_misses(struct c2c_stats *stats)
672	{
673	return stats->lcl_dram +
674	stats->rmt_dram +
675	stats->rmt_hitm +
676	stats->rmt_hit;
677	}
678
679	static uint64_t get_load_cache_hits(struct c2c_stats *stats)
680	{
681	return stats->ld_fbhit +
682	stats->ld_l1hit +
683	stats->ld_l2hit +
684	stats->ld_llchit +
685	stats->lcl_hitm;
686	}
687
688	static uint64_t get_stores(struct c2c_stats *stats)
689	{
690	return stats->st_l1hit +
691	stats->st_l1miss +
692	stats->st_na;
693	}
694
695	static uint64_t total_records(struct c2c_stats *stats)
696	{
697	return get_load_llc_misses(stats) +
698	get_load_cache_hits(stats) +
699	get_stores(stats);
700	}
701
702	static int
703	tot_recs_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
704	struct hist_entry *he)
705	{
706	struct c2c_hist_entry *c2c_he;
707	int width = c2c_width(fmt, hpp, he->hists);
708	uint64_t tot_recs;
709
710	c2c_he = container_of(he, struct c2c_hist_entry, he);
711	tot_recs = total_records(stats: &c2c_he->stats);
712
713	return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, tot_recs);
714	}
715
716	static int64_t
717	tot_recs_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
718	struct hist_entry *left, struct hist_entry *right)
719	{
720	struct c2c_hist_entry *c2c_left;
721	struct c2c_hist_entry *c2c_right;
722	uint64_t tot_recs_left;
723	uint64_t tot_recs_right;
724
725	c2c_left = container_of(left, struct c2c_hist_entry, he);
726	c2c_right = container_of(right, struct c2c_hist_entry, he);
727
728	tot_recs_left = total_records(stats: &c2c_left->stats);
729	tot_recs_right = total_records(stats: &c2c_right->stats);
730
731	return tot_recs_left - tot_recs_right;
732	}
733
734	static uint64_t total_loads(struct c2c_stats *stats)
735	{
736	return get_load_llc_misses(stats) +
737	get_load_cache_hits(stats);
738	}
739
740	static int
741	tot_loads_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
742	struct hist_entry *he)
743	{
744	struct c2c_hist_entry *c2c_he;
745	int width = c2c_width(fmt, hpp, he->hists);
746	uint64_t tot_recs;
747
748	c2c_he = container_of(he, struct c2c_hist_entry, he);
749	tot_recs = total_loads(stats: &c2c_he->stats);
750
751	return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, tot_recs);
752	}
753
754	static int64_t
755	tot_loads_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
756	struct hist_entry *left, struct hist_entry *right)
757	{
758	struct c2c_hist_entry *c2c_left;
759	struct c2c_hist_entry *c2c_right;
760	uint64_t tot_recs_left;
761	uint64_t tot_recs_right;
762
763	c2c_left = container_of(left, struct c2c_hist_entry, he);
764	c2c_right = container_of(right, struct c2c_hist_entry, he);
765
766	tot_recs_left = total_loads(stats: &c2c_left->stats);
767	tot_recs_right = total_loads(stats: &c2c_right->stats);
768
769	return tot_recs_left - tot_recs_right;
770	}
771
772	typedef double (get_percent_cb)(struct c2c_hist_entry *);
773
774	static int
775	percent_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
776	struct hist_entry *he, get_percent_cb get_percent)
777	{
778	struct c2c_hist_entry *c2c_he;
779	int width = c2c_width(fmt, hpp, he->hists);
780	double per;
781
782	c2c_he = container_of(he, struct c2c_hist_entry, he);
783	per = get_percent(c2c_he);
784
785	#ifdef HAVE_SLANG_SUPPORT
786	if (use_browser)
787	return __hpp__slsmg_color_printf(hpp, "%*.2f%%", width - 1, per);
788	#endif
789	return hpp_color_scnprintf(hpp, "%*.2f%%", width - 1, per);
790	}
791
792	static double percent_costly_snoop(struct c2c_hist_entry *c2c_he)
793	{
794	struct c2c_hists *hists;
795	struct c2c_stats *stats;
796	struct c2c_stats *total;
797	int tot = 0, st = 0;
798	double p;
799
800	hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);
801	stats = &c2c_he->stats;
802	total = &hists->stats;
803
804	switch (c2c.display) {
805	case DISPLAY_RMT_HITM:
806	st = stats->rmt_hitm;
807	tot = total->rmt_hitm;
808	break;
809	case DISPLAY_LCL_HITM:
810	st = stats->lcl_hitm;
811	tot = total->lcl_hitm;
812	break;
813	case DISPLAY_TOT_HITM:
814	st = stats->tot_hitm;
815	tot = total->tot_hitm;
816	break;
817	case DISPLAY_SNP_PEER:
818	st = stats->tot_peer;
819	tot = total->tot_peer;
820	break;
821	default:
822	break;
823	}
824
825	p = tot ? (double) st / tot : 0;
826
827	return 100 * p;
828	}
829
830	#define PERC_STR(__s, __v) \
831	({ \
832	scnprintf(__s, sizeof(__s), "%.2F%%", __v); \
833	__s; \
834	})
835
836	static int
837	percent_costly_snoop_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
838	struct hist_entry *he)
839	{
840	struct c2c_hist_entry *c2c_he;
841	int width = c2c_width(fmt, hpp, he->hists);
842	char buf[10];
843	double per;
844
845	c2c_he = container_of(he, struct c2c_hist_entry, he);
846	per = percent_costly_snoop(c2c_he);
847	return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
848	}
849
850	static int
851	percent_costly_snoop_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
852	struct hist_entry *he)
853	{
854	return percent_color(fmt, hpp, he, get_percent: percent_costly_snoop);
855	}
856
857	static int64_t
858	percent_costly_snoop_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
859	struct hist_entry *left, struct hist_entry *right)
860	{
861	struct c2c_hist_entry *c2c_left;
862	struct c2c_hist_entry *c2c_right;
863	double per_left;
864	double per_right;
865
866	c2c_left = container_of(left, struct c2c_hist_entry, he);
867	c2c_right = container_of(right, struct c2c_hist_entry, he);
868
869	per_left = percent_costly_snoop(c2c_he: c2c_left);
870	per_right = percent_costly_snoop(c2c_he: c2c_right);
871
872	return per_left - per_right;
873	}
874
875	static struct c2c_stats *he_stats(struct hist_entry *he)
876	{
877	struct c2c_hist_entry *c2c_he;
878
879	c2c_he = container_of(he, struct c2c_hist_entry, he);
880	return &c2c_he->stats;
881	}
882
883	static struct c2c_stats *total_stats(struct hist_entry *he)
884	{
885	struct c2c_hists *hists;
886
887	hists = container_of(he->hists, struct c2c_hists, hists);
888	return &hists->stats;
889	}
890
891	static double percent(u32 st, u32 tot)
892	{
893	return tot ? 100. * (double) st / (double) tot : 0;
894	}
895
896	#define PERCENT(__h, __f) percent(he_stats(__h)->__f, total_stats(__h)->__f)
897
898	#define PERCENT_FN(__f) \
899	static double percent_ ## __f(struct c2c_hist_entry *c2c_he) \
900	{ \
901	struct c2c_hists *hists; \
902	\
903	hists = container_of(c2c_he->he.hists, struct c2c_hists, hists); \
904	return percent(c2c_he->stats.__f, hists->stats.__f); \
905	}
906
907	PERCENT_FN(rmt_hitm)
908	PERCENT_FN(lcl_hitm)
909	PERCENT_FN(rmt_peer)
910	PERCENT_FN(lcl_peer)
911	PERCENT_FN(st_l1hit)
912	PERCENT_FN(st_l1miss)
913	PERCENT_FN(st_na)
914
915	static int
916	percent_rmt_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
917	struct hist_entry *he)
918	{
919	int width = c2c_width(fmt, hpp, he->hists);
920	double per = PERCENT(he, rmt_hitm);
921	char buf[10];
922
923	return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
924	}
925
926	static int
927	percent_rmt_hitm_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
928	struct hist_entry *he)
929	{
930	return percent_color(fmt, hpp, he, get_percent: percent_rmt_hitm);
931	}
932
933	static int64_t
934	percent_rmt_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
935	struct hist_entry *left, struct hist_entry *right)
936	{
937	double per_left;
938	double per_right;
939
940	per_left = PERCENT(left, rmt_hitm);
941	per_right = PERCENT(right, rmt_hitm);
942
943	return per_left - per_right;
944	}
945
946	static int
947	percent_lcl_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
948	struct hist_entry *he)
949	{
950	int width = c2c_width(fmt, hpp, he->hists);
951	double per = PERCENT(he, lcl_hitm);
952	char buf[10];
953
954	return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
955	}
956
957	static int
958	percent_lcl_hitm_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
959	struct hist_entry *he)
960	{
961	return percent_color(fmt, hpp, he, get_percent: percent_lcl_hitm);
962	}
963
964	static int64_t
965	percent_lcl_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
966	struct hist_entry *left, struct hist_entry *right)
967	{
968	double per_left;
969	double per_right;
970
971	per_left = PERCENT(left, lcl_hitm);
972	per_right = PERCENT(right, lcl_hitm);
973
974	return per_left - per_right;
975	}
976
977	static int
978	percent_lcl_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
979	struct hist_entry *he)
980	{
981	int width = c2c_width(fmt, hpp, he->hists);
982	double per = PERCENT(he, lcl_peer);
983	char buf[10];
984
985	return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
986	}
987
988	static int
989	percent_lcl_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
990	struct hist_entry *he)
991	{
992	return percent_color(fmt, hpp, he, get_percent: percent_lcl_peer);
993	}
994
995	static int64_t
996	percent_lcl_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
997	struct hist_entry *left, struct hist_entry *right)
998	{
999	double per_left;
1000	double per_right;
1001
1002	per_left = PERCENT(left, lcl_peer);
1003	per_right = PERCENT(right, lcl_peer);
1004
1005	return per_left - per_right;
1006	}
1007
1008	static int
1009	percent_rmt_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1010	struct hist_entry *he)
1011	{
1012	int width = c2c_width(fmt, hpp, he->hists);
1013	double per = PERCENT(he, rmt_peer);
1014	char buf[10];
1015
1016	return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1017	}
1018
1019	static int
1020	percent_rmt_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1021	struct hist_entry *he)
1022	{
1023	return percent_color(fmt, hpp, he, get_percent: percent_rmt_peer);
1024	}
1025
1026	static int64_t
1027	percent_rmt_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1028	struct hist_entry *left, struct hist_entry *right)
1029	{
1030	double per_left;
1031	double per_right;
1032
1033	per_left = PERCENT(left, rmt_peer);
1034	per_right = PERCENT(right, rmt_peer);
1035
1036	return per_left - per_right;
1037	}
1038
1039	static int
1040	percent_stores_l1hit_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1041	struct hist_entry *he)
1042	{
1043	int width = c2c_width(fmt, hpp, he->hists);
1044	double per = PERCENT(he, st_l1hit);
1045	char buf[10];
1046
1047	return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1048	}
1049
1050	static int
1051	percent_stores_l1hit_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1052	struct hist_entry *he)
1053	{
1054	return percent_color(fmt, hpp, he, get_percent: percent_st_l1hit);
1055	}
1056
1057	static int64_t
1058	percent_stores_l1hit_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1059	struct hist_entry *left, struct hist_entry *right)
1060	{
1061	double per_left;
1062	double per_right;
1063
1064	per_left = PERCENT(left, st_l1hit);
1065	per_right = PERCENT(right, st_l1hit);
1066
1067	return per_left - per_right;
1068	}
1069
1070	static int
1071	percent_stores_l1miss_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1072	struct hist_entry *he)
1073	{
1074	int width = c2c_width(fmt, hpp, he->hists);
1075	double per = PERCENT(he, st_l1miss);
1076	char buf[10];
1077
1078	return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1079	}
1080
1081	static int
1082	percent_stores_l1miss_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1083	struct hist_entry *he)
1084	{
1085	return percent_color(fmt, hpp, he, get_percent: percent_st_l1miss);
1086	}
1087
1088	static int64_t
1089	percent_stores_l1miss_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1090	struct hist_entry *left, struct hist_entry *right)
1091	{
1092	double per_left;
1093	double per_right;
1094
1095	per_left = PERCENT(left, st_l1miss);
1096	per_right = PERCENT(right, st_l1miss);
1097
1098	return per_left - per_right;
1099	}
1100
1101	static int
1102	percent_stores_na_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1103	struct hist_entry *he)
1104	{
1105	int width = c2c_width(fmt, hpp, he->hists);
1106	double per = PERCENT(he, st_na);
1107	char buf[10];
1108
1109	return scnprintf(hpp->buf, hpp->size, "%*s", width, PERC_STR(buf, per));
1110	}
1111
1112	static int
1113	percent_stores_na_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1114	struct hist_entry *he)
1115	{
1116	return percent_color(fmt, hpp, he, get_percent: percent_st_na);
1117	}
1118
1119	static int64_t
1120	percent_stores_na_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1121	struct hist_entry *left, struct hist_entry *right)
1122	{
1123	double per_left;
1124	double per_right;
1125
1126	per_left = PERCENT(left, st_na);
1127	per_right = PERCENT(right, st_na);
1128
1129	return per_left - per_right;
1130	}
1131
1132	STAT_FN(lcl_dram)
1133	STAT_FN(rmt_dram)
1134
1135	static int
1136	pid_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1137	struct hist_entry *he)
1138	{
1139	int width = c2c_width(fmt, hpp, he->hists);
1140
1141	return scnprintf(hpp->buf, hpp->size, "%*d", width, thread__pid(he->thread));
1142	}
1143
1144	static int64_t
1145	pid_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1146	struct hist_entry *left, struct hist_entry *right)
1147	{
1148	return thread__pid(left->thread) - thread__pid(right->thread);
1149	}
1150
1151	static int64_t
1152	empty_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1153	struct hist_entry *left __maybe_unused,
1154	struct hist_entry *right __maybe_unused)
1155	{
1156	return 0;
1157	}
1158
1159	static int display_metrics(struct perf_hpp *hpp, u32 val, u32 sum)
1160	{
1161	int ret;
1162
1163	if (sum != 0)
1164	ret = scnprintf(hpp->buf, hpp->size, "%5.1f%% ",
1165	percent(val, sum));
1166	else
1167	ret = scnprintf(hpp->buf, hpp->size, "%6s ", "n/a");
1168
1169	return ret;
1170	}
1171
1172	static int
1173	node_entry(struct perf_hpp_fmt *fmt __maybe_unused, struct perf_hpp *hpp,
1174	struct hist_entry *he)
1175	{
1176	struct c2c_hist_entry *c2c_he;
1177	bool first = true;
1178	int node;
1179	int ret = 0;
1180
1181	c2c_he = container_of(he, struct c2c_hist_entry, he);
1182
1183	for (node = 0; node < c2c.nodes_cnt; node++) {
1184	DECLARE_BITMAP(set, c2c.cpus_cnt);
1185
1186	bitmap_zero(set, c2c.cpus_cnt);
1187	bitmap_and(set, c2c_he->cpuset, c2c.nodes[node], c2c.cpus_cnt);
1188
1189	if (bitmap_empty(set, c2c.cpus_cnt)) {
1190	if (c2c.node_info == 1) {
1191	ret = scnprintf(hpp->buf, hpp->size, "%21s", " ");
1192	advance_hpp(hpp, ret);
1193	}
1194	continue;
1195	}
1196
1197	if (!first) {
1198	ret = scnprintf(hpp->buf, hpp->size, " ");
1199	advance_hpp(hpp, ret);
1200	}
1201
1202	switch (c2c.node_info) {
1203	case 0:
1204	ret = scnprintf(hpp->buf, hpp->size, "%2d", node);
1205	advance_hpp(hpp, ret);
1206	break;
1207	case 1:
1208	{
1209	int num = bitmap_weight(set, c2c.cpus_cnt);
1210	struct c2c_stats *stats = &c2c_he->node_stats[node];
1211
1212	ret = scnprintf(hpp->buf, hpp->size, "%2d{%2d ", node, num);
1213	advance_hpp(hpp, ret);
1214
1215	switch (c2c.display) {
1216	case DISPLAY_RMT_HITM:
1217	ret = display_metrics(hpp, stats->rmt_hitm,
1218	c2c_he->stats.rmt_hitm);
1219	break;
1220	case DISPLAY_LCL_HITM:
1221	ret = display_metrics(hpp, stats->lcl_hitm,
1222	c2c_he->stats.lcl_hitm);
1223	break;
1224	case DISPLAY_TOT_HITM:
1225	ret = display_metrics(hpp, stats->tot_hitm,
1226	c2c_he->stats.tot_hitm);
1227	break;
1228	case DISPLAY_SNP_PEER:
1229	ret = display_metrics(hpp, stats->tot_peer,
1230	c2c_he->stats.tot_peer);
1231	break;
1232	default:
1233	break;
1234	}
1235
1236	advance_hpp(hpp, ret);
1237
1238	if (c2c_he->stats.store > 0) {
1239	ret = scnprintf(hpp->buf, hpp->size, "%5.1f%%}",
1240	percent(stats->store, c2c_he->stats.store));
1241	} else {
1242	ret = scnprintf(hpp->buf, hpp->size, "%6s}", "n/a");
1243	}
1244
1245	advance_hpp(hpp, ret);
1246	break;
1247	}
1248	case 2:
1249	ret = scnprintf(hpp->buf, hpp->size, "%2d{", node);
1250	advance_hpp(hpp, ret);
1251
1252	ret = bitmap_scnprintf(set, c2c.cpus_cnt, hpp->buf, hpp->size);
1253	advance_hpp(hpp, ret);
1254
1255	ret = scnprintf(hpp->buf, hpp->size, "}");
1256	advance_hpp(hpp, ret);
1257	break;
1258	default:
1259	break;
1260	}
1261
1262	first = false;
1263	}
1264
1265	return 0;
1266	}
1267
1268	static int
1269	mean_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1270	struct hist_entry *he, double mean)
1271	{
1272	int width = c2c_width(fmt, hpp, he->hists);
1273	char buf[10];
1274
1275	scnprintf(buf, 10, "%6.0f", mean);
1276	return scnprintf(hpp->buf, hpp->size, "%*s", width, buf);
1277	}
1278
1279	#define MEAN_ENTRY(__func, __val) \
1280	static int \
1281	__func(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hist_entry *he) \
1282	{ \
1283	struct c2c_hist_entry *c2c_he; \
1284	c2c_he = container_of(he, struct c2c_hist_entry, he); \
1285	return mean_entry(fmt, hpp, he, avg_stats(&c2c_he->cstats.__val)); \
1286	}
1287
1288	MEAN_ENTRY(mean_rmt_entry, rmt_hitm);
1289	MEAN_ENTRY(mean_lcl_entry, lcl_hitm);
1290	MEAN_ENTRY(mean_load_entry, load);
1291	MEAN_ENTRY(mean_rmt_peer_entry, rmt_peer);
1292	MEAN_ENTRY(mean_lcl_peer_entry, lcl_peer);
1293
1294	static int
1295	cpucnt_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1296	struct hist_entry *he)
1297	{
1298	struct c2c_hist_entry *c2c_he;
1299	int width = c2c_width(fmt, hpp, he->hists);
1300	char buf[10];
1301
1302	c2c_he = container_of(he, struct c2c_hist_entry, he);
1303
1304	scnprintf(buf, 10, "%d", bitmap_weight(c2c_he->cpuset, c2c.cpus_cnt));
1305	return scnprintf(hpp->buf, hpp->size, "%*s", width, buf);
1306	}
1307
1308	static int
1309	cl_idx_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1310	struct hist_entry *he)
1311	{
1312	struct c2c_hist_entry *c2c_he;
1313	int width = c2c_width(fmt, hpp, he->hists);
1314	char buf[10];
1315
1316	c2c_he = container_of(he, struct c2c_hist_entry, he);
1317
1318	scnprintf(buf, 10, "%u", c2c_he->cacheline_idx);
1319	return scnprintf(hpp->buf, hpp->size, "%*s", width, buf);
1320	}
1321
1322	static int
1323	cl_idx_empty_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1324	struct hist_entry *he)
1325	{
1326	int width = c2c_width(fmt, hpp, he->hists);
1327
1328	return scnprintf(hpp->buf, hpp->size, "%*s", width, "");
1329	}
1330
1331	#define HEADER_LOW(__h) \
1332	{ \
1333	.line[1] = { \
1334	.text = __h, \
1335	}, \
1336	}
1337
1338	#define HEADER_BOTH(__h0, __h1) \
1339	{ \
1340	.line[0] = { \
1341	.text = __h0, \
1342	}, \
1343	.line[1] = { \
1344	.text = __h1, \
1345	}, \
1346	}
1347
1348	#define HEADER_SPAN(__h0, __h1, __s) \
1349	{ \
1350	.line[0] = { \
1351	.text = __h0, \
1352	.span = __s, \
1353	}, \
1354	.line[1] = { \
1355	.text = __h1, \
1356	}, \
1357	}
1358
1359	#define HEADER_SPAN_LOW(__h) \
1360	{ \
1361	.line[1] = { \
1362	.text = __h, \
1363	}, \
1364	}
1365
1366	static struct c2c_dimension dim_dcacheline = {
1367	.header = HEADER_SPAN("--- Cacheline ----", "Address", 2),
1368	.name = "dcacheline",
1369	.cmp = dcacheline_cmp,
1370	.entry = dcacheline_entry,
1371	.width = 18,
1372	};
1373
1374	static struct c2c_dimension dim_dcacheline_node = {
1375	.header = HEADER_LOW("Node"),
1376	.name = "dcacheline_node",
1377	.cmp = empty_cmp,
1378	.entry = dcacheline_node_entry,
1379	.width = 4,
1380	};
1381
1382	static struct c2c_dimension dim_dcacheline_count = {
1383	.header = HEADER_LOW("PA cnt"),
1384	.name = "dcacheline_count",
1385	.cmp = empty_cmp,
1386	.entry = dcacheline_node_count,
1387	.width = 6,
1388	};
1389
1390	static struct c2c_header header_offset_tui = HEADER_SPAN("-----", "Off", 2);
1391
1392	static struct c2c_dimension dim_offset = {
1393	.header = HEADER_SPAN("--- Data address -", "Offset", 2),
1394	.name = "offset",
1395	.cmp = offset_cmp,
1396	.entry = offset_entry,
1397	.width = 18,
1398	};
1399
1400	static struct c2c_dimension dim_offset_node = {
1401	.header = HEADER_LOW("Node"),
1402	.name = "offset_node",
1403	.cmp = empty_cmp,
1404	.entry = dcacheline_node_entry,
1405	.width = 4,
1406	};
1407
1408	static struct c2c_dimension dim_iaddr = {
1409	.header = HEADER_LOW("Code address"),
1410	.name = "iaddr",
1411	.cmp = iaddr_cmp,
1412	.entry = iaddr_entry,
1413	.width = 18,
1414	};
1415
1416	static struct c2c_dimension dim_tot_hitm = {
1417	.header = HEADER_SPAN("------- Load Hitm -------", "Total", 2),
1418	.name = "tot_hitm",
1419	.cmp = tot_hitm_cmp,
1420	.entry = tot_hitm_entry,
1421	.width = 7,
1422	};
1423
1424	static struct c2c_dimension dim_lcl_hitm = {
1425	.header = HEADER_SPAN_LOW("LclHitm"),
1426	.name = "lcl_hitm",
1427	.cmp = lcl_hitm_cmp,
1428	.entry = lcl_hitm_entry,
1429	.width = 7,
1430	};
1431
1432	static struct c2c_dimension dim_rmt_hitm = {
1433	.header = HEADER_SPAN_LOW("RmtHitm"),
1434	.name = "rmt_hitm",
1435	.cmp = rmt_hitm_cmp,
1436	.entry = rmt_hitm_entry,
1437	.width = 7,
1438	};
1439
1440	static struct c2c_dimension dim_tot_peer = {
1441	.header = HEADER_SPAN("------- Load Peer -------", "Total", 2),
1442	.name = "tot_peer",
1443	.cmp = tot_peer_cmp,
1444	.entry = tot_peer_entry,
1445	.width = 7,
1446	};
1447
1448	static struct c2c_dimension dim_lcl_peer = {
1449	.header = HEADER_SPAN_LOW("Local"),
1450	.name = "lcl_peer",
1451	.cmp = lcl_peer_cmp,
1452	.entry = lcl_peer_entry,
1453	.width = 7,
1454	};
1455
1456	static struct c2c_dimension dim_rmt_peer = {
1457	.header = HEADER_SPAN_LOW("Remote"),
1458	.name = "rmt_peer",
1459	.cmp = rmt_peer_cmp,
1460	.entry = rmt_peer_entry,
1461	.width = 7,
1462	};
1463
1464	static struct c2c_dimension dim_cl_rmt_hitm = {
1465	.header = HEADER_SPAN("----- HITM -----", "Rmt", 1),
1466	.name = "cl_rmt_hitm",
1467	.cmp = rmt_hitm_cmp,
1468	.entry = rmt_hitm_entry,
1469	.width = 7,
1470	};
1471
1472	static struct c2c_dimension dim_cl_lcl_hitm = {
1473	.header = HEADER_SPAN_LOW("Lcl"),
1474	.name = "cl_lcl_hitm",
1475	.cmp = lcl_hitm_cmp,
1476	.entry = lcl_hitm_entry,
1477	.width = 7,
1478	};
1479
1480	static struct c2c_dimension dim_cl_rmt_peer = {
1481	.header = HEADER_SPAN("----- Peer -----", "Rmt", 1),
1482	.name = "cl_rmt_peer",
1483	.cmp = rmt_peer_cmp,
1484	.entry = rmt_peer_entry,
1485	.width = 7,
1486	};
1487
1488	static struct c2c_dimension dim_cl_lcl_peer = {
1489	.header = HEADER_SPAN_LOW("Lcl"),
1490	.name = "cl_lcl_peer",
1491	.cmp = lcl_peer_cmp,
1492	.entry = lcl_peer_entry,
1493	.width = 7,
1494	};
1495
1496	static struct c2c_dimension dim_tot_stores = {
1497	.header = HEADER_BOTH("Total", "Stores"),
1498	.name = "tot_stores",
1499	.cmp = store_cmp,
1500	.entry = store_entry,
1501	.width = 7,
1502	};
1503
1504	static struct c2c_dimension dim_stores_l1hit = {
1505	.header = HEADER_SPAN("--------- Stores --------", "L1Hit", 2),
1506	.name = "stores_l1hit",
1507	.cmp = st_l1hit_cmp,
1508	.entry = st_l1hit_entry,
1509	.width = 7,
1510	};
1511
1512	static struct c2c_dimension dim_stores_l1miss = {
1513	.header = HEADER_SPAN_LOW("L1Miss"),
1514	.name = "stores_l1miss",
1515	.cmp = st_l1miss_cmp,
1516	.entry = st_l1miss_entry,
1517	.width = 7,
1518	};
1519
1520	static struct c2c_dimension dim_stores_na = {
1521	.header = HEADER_SPAN_LOW("N/A"),
1522	.name = "stores_na",
1523	.cmp = st_na_cmp,
1524	.entry = st_na_entry,
1525	.width = 7,
1526	};
1527
1528	static struct c2c_dimension dim_cl_stores_l1hit = {
1529	.header = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2),
1530	.name = "cl_stores_l1hit",
1531	.cmp = st_l1hit_cmp,
1532	.entry = st_l1hit_entry,
1533	.width = 7,
1534	};
1535
1536	static struct c2c_dimension dim_cl_stores_l1miss = {
1537	.header = HEADER_SPAN_LOW("L1 Miss"),
1538	.name = "cl_stores_l1miss",
1539	.cmp = st_l1miss_cmp,
1540	.entry = st_l1miss_entry,
1541	.width = 7,
1542	};
1543
1544	static struct c2c_dimension dim_cl_stores_na = {
1545	.header = HEADER_SPAN_LOW("N/A"),
1546	.name = "cl_stores_na",
1547	.cmp = st_na_cmp,
1548	.entry = st_na_entry,
1549	.width = 7,
1550	};
1551
1552	static struct c2c_dimension dim_ld_fbhit = {
1553	.header = HEADER_SPAN("----- Core Load Hit -----", "FB", 2),
1554	.name = "ld_fbhit",
1555	.cmp = ld_fbhit_cmp,
1556	.entry = ld_fbhit_entry,
1557	.width = 7,
1558	};
1559
1560	static struct c2c_dimension dim_ld_l1hit = {
1561	.header = HEADER_SPAN_LOW("L1"),
1562	.name = "ld_l1hit",
1563	.cmp = ld_l1hit_cmp,
1564	.entry = ld_l1hit_entry,
1565	.width = 7,
1566	};
1567
1568	static struct c2c_dimension dim_ld_l2hit = {
1569	.header = HEADER_SPAN_LOW("L2"),
1570	.name = "ld_l2hit",
1571	.cmp = ld_l2hit_cmp,
1572	.entry = ld_l2hit_entry,
1573	.width = 7,
1574	};
1575
1576	static struct c2c_dimension dim_ld_llchit = {
1577	.header = HEADER_SPAN("- LLC Load Hit --", "LclHit", 1),
1578	.name = "ld_lclhit",
1579	.cmp = ld_llchit_cmp,
1580	.entry = ld_llchit_entry,
1581	.width = 8,
1582	};
1583
1584	static struct c2c_dimension dim_ld_rmthit = {
1585	.header = HEADER_SPAN("- RMT Load Hit --", "RmtHit", 1),
1586	.name = "ld_rmthit",
1587	.cmp = rmt_hit_cmp,
1588	.entry = rmt_hit_entry,
1589	.width = 8,
1590	};
1591
1592	static struct c2c_dimension dim_tot_recs = {
1593	.header = HEADER_BOTH("Total", "records"),
1594	.name = "tot_recs",
1595	.cmp = tot_recs_cmp,
1596	.entry = tot_recs_entry,
1597	.width = 7,
1598	};
1599
1600	static struct c2c_dimension dim_tot_loads = {
1601	.header = HEADER_BOTH("Total", "Loads"),
1602	.name = "tot_loads",
1603	.cmp = tot_loads_cmp,
1604	.entry = tot_loads_entry,
1605	.width = 7,
1606	};
1607
1608	static struct c2c_header percent_costly_snoop_header[] = {
1609	[DISPLAY_LCL_HITM] = HEADER_BOTH("Lcl", "Hitm"),
1610	[DISPLAY_RMT_HITM] = HEADER_BOTH("Rmt", "Hitm"),
1611	[DISPLAY_TOT_HITM] = HEADER_BOTH("Tot", "Hitm"),
1612	[DISPLAY_SNP_PEER] = HEADER_BOTH("Peer", "Snoop"),
1613	};
1614
1615	static struct c2c_dimension dim_percent_costly_snoop = {
1616	.name = "percent_costly_snoop",
1617	.cmp = percent_costly_snoop_cmp,
1618	.entry = percent_costly_snoop_entry,
1619	.color = percent_costly_snoop_color,
1620	.width = 7,
1621	};
1622
1623	static struct c2c_dimension dim_percent_rmt_hitm = {
1624	.header = HEADER_SPAN("----- HITM -----", "RmtHitm", 1),
1625	.name = "percent_rmt_hitm",
1626	.cmp = percent_rmt_hitm_cmp,
1627	.entry = percent_rmt_hitm_entry,
1628	.color = percent_rmt_hitm_color,
1629	.width = 7,
1630	};
1631
1632	static struct c2c_dimension dim_percent_lcl_hitm = {
1633	.header = HEADER_SPAN_LOW("LclHitm"),
1634	.name = "percent_lcl_hitm",
1635	.cmp = percent_lcl_hitm_cmp,
1636	.entry = percent_lcl_hitm_entry,
1637	.color = percent_lcl_hitm_color,
1638	.width = 7,
1639	};
1640
1641	static struct c2c_dimension dim_percent_rmt_peer = {
1642	.header = HEADER_SPAN("-- Peer Snoop --", "Rmt", 1),
1643	.name = "percent_rmt_peer",
1644	.cmp = percent_rmt_peer_cmp,
1645	.entry = percent_rmt_peer_entry,
1646	.color = percent_rmt_peer_color,
1647	.width = 7,
1648	};
1649
1650	static struct c2c_dimension dim_percent_lcl_peer = {
1651	.header = HEADER_SPAN_LOW("Lcl"),
1652	.name = "percent_lcl_peer",
1653	.cmp = percent_lcl_peer_cmp,
1654	.entry = percent_lcl_peer_entry,
1655	.color = percent_lcl_peer_color,
1656	.width = 7,
1657	};
1658
1659	static struct c2c_dimension dim_percent_stores_l1hit = {
1660	.header = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2),
1661	.name = "percent_stores_l1hit",
1662	.cmp = percent_stores_l1hit_cmp,
1663	.entry = percent_stores_l1hit_entry,
1664	.color = percent_stores_l1hit_color,
1665	.width = 7,
1666	};
1667
1668	static struct c2c_dimension dim_percent_stores_l1miss = {
1669	.header = HEADER_SPAN_LOW("L1 Miss"),
1670	.name = "percent_stores_l1miss",
1671	.cmp = percent_stores_l1miss_cmp,
1672	.entry = percent_stores_l1miss_entry,
1673	.color = percent_stores_l1miss_color,
1674	.width = 7,
1675	};
1676
1677	static struct c2c_dimension dim_percent_stores_na = {
1678	.header = HEADER_SPAN_LOW("N/A"),
1679	.name = "percent_stores_na",
1680	.cmp = percent_stores_na_cmp,
1681	.entry = percent_stores_na_entry,
1682	.color = percent_stores_na_color,
1683	.width = 7,
1684	};
1685
1686	static struct c2c_dimension dim_dram_lcl = {
1687	.header = HEADER_SPAN("--- Load Dram ----", "Lcl", 1),
1688	.name = "dram_lcl",
1689	.cmp = lcl_dram_cmp,
1690	.entry = lcl_dram_entry,
1691	.width = 8,
1692	};
1693
1694	static struct c2c_dimension dim_dram_rmt = {
1695	.header = HEADER_SPAN_LOW("Rmt"),
1696	.name = "dram_rmt",
1697	.cmp = rmt_dram_cmp,
1698	.entry = rmt_dram_entry,
1699	.width = 8,
1700	};
1701
1702	static struct c2c_dimension dim_pid = {
1703	.header = HEADER_LOW("Pid"),
1704	.name = "pid",
1705	.cmp = pid_cmp,
1706	.entry = pid_entry,
1707	.width = 7,
1708	};
1709
1710	static struct c2c_dimension dim_tid = {
1711	.header = HEADER_LOW("Tid"),
1712	.name = "tid",
1713	.se = &sort_thread,
1714	};
1715
1716	static struct c2c_dimension dim_symbol = {
1717	.name = "symbol",
1718	.se = &sort_sym,
1719	};
1720
1721	static struct c2c_dimension dim_dso = {
1722	.header = HEADER_BOTH("Shared", "Object"),
1723	.name = "dso",
1724	.se = &sort_dso,
1725	};
1726
1727	static struct c2c_dimension dim_node = {
1728	.name = "node",
1729	.cmp = empty_cmp,
1730	.entry = node_entry,
1731	.width = 4,
1732	};
1733
1734	static struct c2c_dimension dim_mean_rmt = {
1735	.header = HEADER_SPAN("---------- cycles ----------", "rmt hitm", 2),
1736	.name = "mean_rmt",
1737	.cmp = empty_cmp,
1738	.entry = mean_rmt_entry,
1739	.width = 8,
1740	};
1741
1742	static struct c2c_dimension dim_mean_lcl = {
1743	.header = HEADER_SPAN_LOW("lcl hitm"),
1744	.name = "mean_lcl",
1745	.cmp = empty_cmp,
1746	.entry = mean_lcl_entry,
1747	.width = 8,
1748	};
1749
1750	static struct c2c_dimension dim_mean_load = {
1751	.header = HEADER_SPAN_LOW("load"),
1752	.name = "mean_load",
1753	.cmp = empty_cmp,
1754	.entry = mean_load_entry,
1755	.width = 8,
1756	};
1757
1758	static struct c2c_dimension dim_mean_rmt_peer = {
1759	.header = HEADER_SPAN("---------- cycles ----------", "rmt peer", 2),
1760	.name = "mean_rmt_peer",
1761	.cmp = empty_cmp,
1762	.entry = mean_rmt_peer_entry,
1763	.width = 8,
1764	};
1765
1766	static struct c2c_dimension dim_mean_lcl_peer = {
1767	.header = HEADER_SPAN_LOW("lcl peer"),
1768	.name = "mean_lcl_peer",
1769	.cmp = empty_cmp,
1770	.entry = mean_lcl_peer_entry,
1771	.width = 8,
1772	};
1773
1774	static struct c2c_dimension dim_cpucnt = {
1775	.header = HEADER_BOTH("cpu", "cnt"),
1776	.name = "cpucnt",
1777	.cmp = empty_cmp,
1778	.entry = cpucnt_entry,
1779	.width = 8,
1780	};
1781
1782	static struct c2c_dimension dim_srcline = {
1783	.name = "cl_srcline",
1784	.se = &sort_srcline,
1785	};
1786
1787	static struct c2c_dimension dim_dcacheline_idx = {
1788	.header = HEADER_LOW("Index"),
1789	.name = "cl_idx",
1790	.cmp = empty_cmp,
1791	.entry = cl_idx_entry,
1792	.width = 5,
1793	};
1794
1795	static struct c2c_dimension dim_dcacheline_num = {
1796	.header = HEADER_LOW("Num"),
1797	.name = "cl_num",
1798	.cmp = empty_cmp,
1799	.entry = cl_idx_entry,
1800	.width = 5,
1801	};
1802
1803	static struct c2c_dimension dim_dcacheline_num_empty = {
1804	.header = HEADER_LOW("Num"),
1805	.name = "cl_num_empty",
1806	.cmp = empty_cmp,
1807	.entry = cl_idx_empty_entry,
1808	.width = 5,
1809	};
1810
1811	static struct c2c_dimension *dimensions[] = {
1812	&dim_dcacheline,
1813	&dim_dcacheline_node,
1814	&dim_dcacheline_count,
1815	&dim_offset,
1816	&dim_offset_node,
1817	&dim_iaddr,
1818	&dim_tot_hitm,
1819	&dim_lcl_hitm,
1820	&dim_rmt_hitm,
1821	&dim_tot_peer,
1822	&dim_lcl_peer,
1823	&dim_rmt_peer,
1824	&dim_cl_lcl_hitm,
1825	&dim_cl_rmt_hitm,
1826	&dim_cl_lcl_peer,
1827	&dim_cl_rmt_peer,
1828	&dim_tot_stores,
1829	&dim_stores_l1hit,
1830	&dim_stores_l1miss,
1831	&dim_stores_na,
1832	&dim_cl_stores_l1hit,
1833	&dim_cl_stores_l1miss,
1834	&dim_cl_stores_na,
1835	&dim_ld_fbhit,
1836	&dim_ld_l1hit,
1837	&dim_ld_l2hit,
1838	&dim_ld_llchit,
1839	&dim_ld_rmthit,
1840	&dim_tot_recs,
1841	&dim_tot_loads,
1842	&dim_percent_costly_snoop,
1843	&dim_percent_rmt_hitm,
1844	&dim_percent_lcl_hitm,
1845	&dim_percent_rmt_peer,
1846	&dim_percent_lcl_peer,
1847	&dim_percent_stores_l1hit,
1848	&dim_percent_stores_l1miss,
1849	&dim_percent_stores_na,
1850	&dim_dram_lcl,
1851	&dim_dram_rmt,
1852	&dim_pid,
1853	&dim_tid,
1854	&dim_symbol,
1855	&dim_dso,
1856	&dim_node,
1857	&dim_mean_rmt,
1858	&dim_mean_lcl,
1859	&dim_mean_rmt_peer,
1860	&dim_mean_lcl_peer,
1861	&dim_mean_load,
1862	&dim_cpucnt,
1863	&dim_srcline,
1864	&dim_dcacheline_idx,
1865	&dim_dcacheline_num,
1866	&dim_dcacheline_num_empty,
1867	NULL,
1868	};
1869
1870	static void fmt_free(struct perf_hpp_fmt *fmt)
1871	{
1872	struct c2c_fmt *c2c_fmt;
1873
1874	c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1875	free(c2c_fmt);
1876	}
1877
1878	static bool fmt_equal(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b)
1879	{
1880	struct c2c_fmt *c2c_a = container_of(a, struct c2c_fmt, fmt);
1881	struct c2c_fmt *c2c_b = container_of(b, struct c2c_fmt, fmt);
1882
1883	return c2c_a->dim == c2c_b->dim;
1884	}
1885
1886	static struct c2c_dimension *get_dimension(const char *name)
1887	{
1888	unsigned int i;
1889
1890	for (i = 0; dimensions[i]; i++) {
1891	struct c2c_dimension *dim = dimensions[i];
1892
1893	if (!strcmp(dim->name, name))
1894	return dim;
1895	}
1896
1897	return NULL;
1898	}
1899
1900	static int c2c_se_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1901	struct hist_entry *he)
1902	{
1903	struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1904	struct c2c_dimension *dim = c2c_fmt->dim;
1905	size_t len = fmt->user_len;
1906
1907	if (!len) {
1908	len = hists__col_len(he->hists, dim->se->se_width_idx);
1909
1910	if (dim == &dim_symbol || dim == &dim_srcline)
1911	len = symbol_width(hists: he->hists, se: dim->se);
1912	}
1913
1914	return dim->se->se_snprintf(he, hpp->buf, hpp->size, len);
1915	}
1916
1917	static int64_t c2c_se_cmp(struct perf_hpp_fmt *fmt,
1918	struct hist_entry *a, struct hist_entry *b)
1919	{
1920	struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1921	struct c2c_dimension *dim = c2c_fmt->dim;
1922
1923	return dim->se->se_cmp(a, b);
1924	}
1925
1926	static int64_t c2c_se_collapse(struct perf_hpp_fmt *fmt,
1927	struct hist_entry *a, struct hist_entry *b)
1928	{
1929	struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1930	struct c2c_dimension *dim = c2c_fmt->dim;
1931	int64_t (*collapse_fn)(struct hist_entry *, struct hist_entry *);
1932
1933	collapse_fn = dim->se->se_collapse ?: dim->se->se_cmp;
1934	return collapse_fn(a, b);
1935	}
1936
1937	static struct c2c_fmt *get_format(const char *name)
1938	{
1939	struct c2c_dimension *dim = get_dimension(name);
1940	struct c2c_fmt *c2c_fmt;
1941	struct perf_hpp_fmt *fmt;
1942
1943	if (!dim)
1944	return NULL;
1945
1946	c2c_fmt = zalloc(sizeof(*c2c_fmt));
1947	if (!c2c_fmt)
1948	return NULL;
1949
1950	c2c_fmt->dim = dim;
1951
1952	fmt = &c2c_fmt->fmt;
1953	INIT_LIST_HEAD(&fmt->list);
1954	INIT_LIST_HEAD(&fmt->sort_list);
1955
1956	fmt->cmp = dim->se ? c2c_se_cmp : dim->cmp;
1957	fmt->sort = dim->se ? c2c_se_cmp : dim->cmp;
1958	fmt->color = dim->se ? NULL : dim->color;
1959	fmt->entry = dim->se ? c2c_se_entry : dim->entry;
1960	fmt->header = c2c_header;
1961	fmt->width = c2c_width;
1962	fmt->collapse = dim->se ? c2c_se_collapse : dim->cmp;
1963	fmt->equal = fmt_equal;
1964	fmt->free = fmt_free;
1965
1966	return c2c_fmt;
1967	}
1968
1969	static int c2c_hists__init_output(struct perf_hpp_list *hpp_list, char *name)
1970	{
1971	struct c2c_fmt *c2c_fmt = get_format(name);
1972	int level = 0;
1973
1974	if (!c2c_fmt) {
1975	reset_dimensions();
1976	return output_field_add(hpp_list, name, &level);
1977	}
1978
1979	perf_hpp_list__column_register(hpp_list, &c2c_fmt->fmt);
1980	return 0;
1981	}
1982
1983	static int c2c_hists__init_sort(struct perf_hpp_list *hpp_list, char *name)
1984	{
1985	struct c2c_fmt *c2c_fmt = get_format(name);
1986	struct c2c_dimension *dim;
1987
1988	if (!c2c_fmt) {
1989	reset_dimensions();
1990	return sort_dimension__add(hpp_list, name, NULL, 0);
1991	}
1992
1993	dim = c2c_fmt->dim;
1994	if (dim == &dim_dso)
1995	hpp_list->dso = 1;
1996
1997	perf_hpp_list__register_sort_field(hpp_list, &c2c_fmt->fmt);
1998	return 0;
1999	}
2000
2001	#define PARSE_LIST(_list, _fn) \
2002	do { \
2003	char *tmp, *tok; \
2004	ret = 0; \
2005	\
2006	if (!_list) \
2007	break; \
2008	\
2009	for (tok = strtok_r((char *)_list, ", ", &tmp); \
2010	tok; tok = strtok_r(NULL, ", ", &tmp)) { \
2011	ret = _fn(hpp_list, tok); \
2012	if (ret == -EINVAL) { \
2013	pr_err("Invalid --fields key: `%s'", tok); \
2014	break; \
2015	} else if (ret == -ESRCH) { \
2016	pr_err("Unknown --fields key: `%s'", tok); \
2017	break; \
2018	} \
2019	} \
2020	} while (0)
2021
2022	static int hpp_list__parse(struct perf_hpp_list *hpp_list,
2023	const char *output_,
2024	const char *sort_)
2025	{
2026	char *output = output_ ? strdup(output_) : NULL;
2027	char *sort = sort_ ? strdup(sort_) : NULL;
2028	int ret;
2029
2030	PARSE_LIST(output, c2c_hists__init_output);
2031	PARSE_LIST(sort, c2c_hists__init_sort);
2032
2033	/* copy sort keys to output fields */
2034	perf_hpp__setup_output_field(hpp_list);
2035
2036	/*
2037	* We don't need other sorting keys other than those
2038	* we already specified. It also really slows down
2039	* the processing a lot with big number of output
2040	* fields, so switching this off for c2c.
2041	*/
2042
2043	#if 0
2044	/* and then copy output fields to sort keys */
2045	perf_hpp__append_sort_keys(&hists->list);
2046	#endif
2047
2048	free(output);
2049	free(sort);
2050	return ret;
2051	}
2052
2053	static int c2c_hists__init(struct c2c_hists *hists,
2054	const char *sort,
2055	int nr_header_lines)
2056	{
2057	__hists__init(&hists->hists, &hists->list);
2058
2059	/*
2060	* Initialize only with sort fields, we need to resort
2061	* later anyway, and that's where we add output fields
2062	* as well.
2063	*/
2064	perf_hpp_list__init(&hists->list);
2065
2066	/* Overload number of header lines.*/
2067	hists->list.nr_header_lines = nr_header_lines;
2068
2069	return hpp_list__parse(hpp_list: &hists->list, NULL, sort_: sort);
2070	}
2071
2072	static int c2c_hists__reinit(struct c2c_hists *c2c_hists,
2073	const char *output,
2074	const char *sort)
2075	{
2076	perf_hpp__reset_output_field(&c2c_hists->list);
2077	return hpp_list__parse(hpp_list: &c2c_hists->list, output_: output, sort_: sort);
2078	}
2079
2080	#define DISPLAY_LINE_LIMIT 0.001
2081
2082	static u8 filter_display(u32 val, u32 sum)
2083	{
2084	if (sum == 0 || ((double)val / sum) < DISPLAY_LINE_LIMIT)
2085	return HIST_FILTER__C2C;
2086
2087	return 0;
2088	}
2089
2090	static bool he__display(struct hist_entry *he, struct c2c_stats *stats)
2091	{
2092	struct c2c_hist_entry *c2c_he;
2093
2094	if (c2c.show_all)
2095	return true;
2096
2097	c2c_he = container_of(he, struct c2c_hist_entry, he);
2098
2099	switch (c2c.display) {
2100	case DISPLAY_LCL_HITM:
2101	he->filtered = filter_display(c2c_he->stats.lcl_hitm,
2102	stats->lcl_hitm);
2103	break;
2104	case DISPLAY_RMT_HITM:
2105	he->filtered = filter_display(c2c_he->stats.rmt_hitm,
2106	stats->rmt_hitm);
2107	break;
2108	case DISPLAY_TOT_HITM:
2109	he->filtered = filter_display(c2c_he->stats.tot_hitm,
2110	stats->tot_hitm);
2111	break;
2112	case DISPLAY_SNP_PEER:
2113	he->filtered = filter_display(c2c_he->stats.tot_peer,
2114	stats->tot_peer);
2115	break;
2116	default:
2117	break;
2118	}
2119
2120	return he->filtered == 0;
2121	}
2122
2123	static inline bool is_valid_hist_entry(struct hist_entry *he)
2124	{
2125	struct c2c_hist_entry *c2c_he;
2126	bool has_record = false;
2127
2128	c2c_he = container_of(he, struct c2c_hist_entry, he);
2129
2130	/* It's a valid entry if contains stores */
2131	if (c2c_he->stats.store)
2132	return true;
2133
2134	switch (c2c.display) {
2135	case DISPLAY_LCL_HITM:
2136	has_record = !!c2c_he->stats.lcl_hitm;
2137	break;
2138	case DISPLAY_RMT_HITM:
2139	has_record = !!c2c_he->stats.rmt_hitm;
2140	break;
2141	case DISPLAY_TOT_HITM:
2142	has_record = !!c2c_he->stats.tot_hitm;
2143	break;
2144	case DISPLAY_SNP_PEER:
2145	has_record = !!c2c_he->stats.tot_peer;
2146	default:
2147	break;
2148	}
2149
2150	return has_record;
2151	}
2152
2153	static void set_node_width(struct c2c_hist_entry *c2c_he, int len)
2154	{
2155	struct c2c_dimension *dim;
2156
2157	dim = &c2c.hists == c2c_he->hists ?
2158	&dim_dcacheline_node : &dim_offset_node;
2159
2160	if (len > dim->width)
2161	dim->width = len;
2162	}
2163
2164	static int set_nodestr(struct c2c_hist_entry *c2c_he)
2165	{
2166	char buf[30];
2167	int len;
2168
2169	if (c2c_he->nodestr)
2170	return 0;
2171
2172	if (!bitmap_empty(c2c_he->nodeset, c2c.nodes_cnt)) {
2173	len = bitmap_scnprintf(c2c_he->nodeset, c2c.nodes_cnt,
2174	buf, sizeof(buf));
2175	} else {
2176	len = scnprintf(buf, sizeof(buf), "N/A");
2177	}
2178
2179	set_node_width(c2c_he, len);
2180	c2c_he->nodestr = strdup(buf);
2181	return c2c_he->nodestr ? 0 : -ENOMEM;
2182	}
2183
2184	static void calc_width(struct c2c_hist_entry *c2c_he)
2185	{
2186	struct c2c_hists *c2c_hists;
2187
2188	c2c_hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);
2189	hists__calc_col_len(&c2c_hists->hists, &c2c_he->he);
2190	set_nodestr(c2c_he);
2191	}
2192
2193	static int filter_cb(struct hist_entry *he, void *arg __maybe_unused)
2194	{
2195	struct c2c_hist_entry *c2c_he;
2196
2197	c2c_he = container_of(he, struct c2c_hist_entry, he);
2198
2199	if (c2c.show_src && !he->srcline)
2200	he->srcline = hist_entry__srcline(he);
2201
2202	calc_width(c2c_he);
2203
2204	if (!is_valid_hist_entry(he))
2205	he->filtered = HIST_FILTER__C2C;
2206
2207	return 0;
2208	}
2209
2210	static int resort_cl_cb(struct hist_entry *he, void *arg __maybe_unused)
2211	{
2212	struct c2c_hist_entry *c2c_he;
2213	struct c2c_hists *c2c_hists;
2214	bool display = he__display(he, stats: &c2c.shared_clines_stats);
2215
2216	c2c_he = container_of(he, struct c2c_hist_entry, he);
2217	c2c_hists = c2c_he->hists;
2218
2219	if (display && c2c_hists) {
2220	static unsigned int idx;
2221
2222	c2c_he->cacheline_idx = idx++;
2223	calc_width(c2c_he);
2224
2225	c2c_hists__reinit(c2c_hists, output: c2c.cl_output, sort: c2c.cl_resort);
2226
2227	hists__collapse_resort(&c2c_hists->hists, NULL);
2228	hists__output_resort_cb(&c2c_hists->hists, NULL, filter_cb);
2229	}
2230
2231	return 0;
2232	}
2233
2234	static struct c2c_header header_node_0 = HEADER_LOW("Node");
2235	static struct c2c_header header_node_1_hitms_stores =
2236	HEADER_LOW("Node{cpus %hitms %stores}");
2237	static struct c2c_header header_node_1_peers_stores =
2238	HEADER_LOW("Node{cpus %peers %stores}");
2239	static struct c2c_header header_node_2 = HEADER_LOW("Node{cpu list}");
2240
2241	static void setup_nodes_header(void)
2242	{
2243	switch (c2c.node_info) {
2244	case 0:
2245	dim_node.header = header_node_0;
2246	break;
2247	case 1:
2248	if (c2c.display == DISPLAY_SNP_PEER)
2249	dim_node.header = header_node_1_peers_stores;
2250	else
2251	dim_node.header = header_node_1_hitms_stores;
2252	break;
2253	case 2:
2254	dim_node.header = header_node_2;
2255	break;
2256	default:
2257	break;
2258	}
2259
2260	return;
2261	}
2262
2263	static int setup_nodes(struct perf_session *session)
2264	{
2265	struct numa_node *n;
2266	unsigned long **nodes;
2267	int node, idx;
2268	struct perf_cpu cpu;
2269	int *cpu2node;
2270
2271	if (c2c.node_info > 2)
2272	c2c.node_info = 2;
2273
2274	c2c.nodes_cnt = session->header.env.nr_numa_nodes;
2275	c2c.cpus_cnt = session->header.env.nr_cpus_avail;
2276
2277	n = session->header.env.numa_nodes;
2278	if (!n)
2279	return -EINVAL;
2280
2281	nodes = zalloc(sizeof(unsigned long *) * c2c.nodes_cnt);
2282	if (!nodes)
2283	return -ENOMEM;
2284
2285	c2c.nodes = nodes;
2286
2287	cpu2node = zalloc(sizeof(int) * c2c.cpus_cnt);
2288	if (!cpu2node)
2289	return -ENOMEM;
2290
2291	for (idx = 0; idx < c2c.cpus_cnt; idx++)
2292	cpu2node[idx] = -1;
2293
2294	c2c.cpu2node = cpu2node;
2295
2296	for (node = 0; node < c2c.nodes_cnt; node++) {
2297	struct perf_cpu_map *map = n[node].map;
2298	unsigned long *set;
2299
2300	set = bitmap_zalloc(c2c.cpus_cnt);
2301	if (!set)
2302	return -ENOMEM;
2303
2304	nodes[node] = set;
2305
2306	perf_cpu_map__for_each_cpu_skip_any(cpu, idx, map) {
2307	__set_bit(cpu.cpu, set);
2308
2309	if (WARN_ONCE(cpu2node[cpu.cpu] != -1, "node/cpu topology bug"))
2310	return -EINVAL;
2311
2312	cpu2node[cpu.cpu] = node;
2313	}
2314	}
2315
2316	setup_nodes_header();
2317	return 0;
2318	}
2319
2320	#define HAS_HITMS(__h) ((__h)->stats.lcl_hitm || (__h)->stats.rmt_hitm)
2321	#define HAS_PEER(__h) ((__h)->stats.lcl_peer || (__h)->stats.rmt_peer)
2322
2323	static int resort_shared_cl_cb(struct hist_entry *he, void *arg __maybe_unused)
2324	{
2325	struct c2c_hist_entry *c2c_he;
2326	c2c_he = container_of(he, struct c2c_hist_entry, he);
2327
2328	if (HAS_HITMS(c2c_he) || HAS_PEER(c2c_he)) {
2329	c2c.shared_clines++;
2330	c2c_add_stats(&c2c.shared_clines_stats, &c2c_he->stats);
2331	}
2332
2333	return 0;
2334	}
2335
2336	static int hists__iterate_cb(struct hists *hists, hists__resort_cb_t cb)
2337	{
2338	struct rb_node *next = rb_first_cached(&hists->entries);
2339	int ret = 0;
2340
2341	while (next) {
2342	struct hist_entry *he;
2343
2344	he = rb_entry(next, struct hist_entry, rb_node);
2345	ret = cb(he, NULL);
2346	if (ret)
2347	break;
2348	next = rb_next(&he->rb_node);
2349	}
2350
2351	return ret;
2352	}
2353
2354	static void print_c2c__display_stats(FILE *out)
2355	{
2356	int llc_misses;
2357	struct c2c_stats *stats = &c2c.hists.stats;
2358
2359	llc_misses = get_load_llc_misses(stats);
2360
2361	fprintf(stream: out, format: "=================================================\n");
2362	fprintf(stream: out, format: " Trace Event Information \n");
2363	fprintf(stream: out, format: "=================================================\n");
2364	fprintf(stream: out, format: " Total records : %10d\n", stats->nr_entries);
2365	fprintf(stream: out, format: " Locked Load/Store Operations : %10d\n", stats->locks);
2366	fprintf(stream: out, format: " Load Operations : %10d\n", stats->load);
2367	fprintf(stream: out, format: " Loads - uncacheable : %10d\n", stats->ld_uncache);
2368	fprintf(stream: out, format: " Loads - IO : %10d\n", stats->ld_io);
2369	fprintf(stream: out, format: " Loads - Miss : %10d\n", stats->ld_miss);
2370	fprintf(stream: out, format: " Loads - no mapping : %10d\n", stats->ld_noadrs);
2371	fprintf(stream: out, format: " Load Fill Buffer Hit : %10d\n", stats->ld_fbhit);
2372	fprintf(stream: out, format: " Load L1D hit : %10d\n", stats->ld_l1hit);
2373	fprintf(stream: out, format: " Load L2D hit : %10d\n", stats->ld_l2hit);
2374	fprintf(stream: out, format: " Load LLC hit : %10d\n", stats->ld_llchit + stats->lcl_hitm);
2375	fprintf(stream: out, format: " Load Local HITM : %10d\n", stats->lcl_hitm);
2376	fprintf(stream: out, format: " Load Remote HITM : %10d\n", stats->rmt_hitm);
2377	fprintf(stream: out, format: " Load Remote HIT : %10d\n", stats->rmt_hit);
2378	fprintf(stream: out, format: " Load Local DRAM : %10d\n", stats->lcl_dram);
2379	fprintf(stream: out, format: " Load Remote DRAM : %10d\n", stats->rmt_dram);
2380	fprintf(stream: out, format: " Load MESI State Exclusive : %10d\n", stats->ld_excl);
2381	fprintf(stream: out, format: " Load MESI State Shared : %10d\n", stats->ld_shared);
2382	fprintf(stream: out, format: " Load LLC Misses : %10d\n", llc_misses);
2383	fprintf(stream: out, format: " Load access blocked by data : %10d\n", stats->blk_data);
2384	fprintf(stream: out, format: " Load access blocked by address : %10d\n", stats->blk_addr);
2385	fprintf(stream: out, format: " Load HIT Local Peer : %10d\n", stats->lcl_peer);
2386	fprintf(stream: out, format: " Load HIT Remote Peer : %10d\n", stats->rmt_peer);
2387	fprintf(stream: out, format: " LLC Misses to Local DRAM : %10.1f%%\n", ((double)stats->lcl_dram/(double)llc_misses) * 100.);
2388	fprintf(stream: out, format: " LLC Misses to Remote DRAM : %10.1f%%\n", ((double)stats->rmt_dram/(double)llc_misses) * 100.);
2389	fprintf(stream: out, format: " LLC Misses to Remote cache (HIT) : %10.1f%%\n", ((double)stats->rmt_hit /(double)llc_misses) * 100.);
2390	fprintf(stream: out, format: " LLC Misses to Remote cache (HITM) : %10.1f%%\n", ((double)stats->rmt_hitm/(double)llc_misses) * 100.);
2391	fprintf(stream: out, format: " Store Operations : %10d\n", stats->store);
2392	fprintf(stream: out, format: " Store - uncacheable : %10d\n", stats->st_uncache);
2393	fprintf(stream: out, format: " Store - no mapping : %10d\n", stats->st_noadrs);
2394	fprintf(stream: out, format: " Store L1D Hit : %10d\n", stats->st_l1hit);
2395	fprintf(stream: out, format: " Store L1D Miss : %10d\n", stats->st_l1miss);
2396	fprintf(stream: out, format: " Store No available memory level : %10d\n", stats->st_na);
2397	fprintf(stream: out, format: " No Page Map Rejects : %10d\n", stats->nomap);
2398	fprintf(stream: out, format: " Unable to parse data source : %10d\n", stats->noparse);
2399	}
2400
2401	static void print_shared_cacheline_info(FILE *out)
2402	{
2403	struct c2c_stats *stats = &c2c.shared_clines_stats;
2404	int hitm_cnt = stats->lcl_hitm + stats->rmt_hitm;
2405
2406	fprintf(stream: out, format: "=================================================\n");
2407	fprintf(stream: out, format: " Global Shared Cache Line Event Information \n");
2408	fprintf(stream: out, format: "=================================================\n");
2409	fprintf(stream: out, format: " Total Shared Cache Lines : %10d\n", c2c.shared_clines);
2410	fprintf(stream: out, format: " Load HITs on shared lines : %10d\n", stats->load);
2411	fprintf(stream: out, format: " Fill Buffer Hits on shared lines : %10d\n", stats->ld_fbhit);
2412	fprintf(stream: out, format: " L1D hits on shared lines : %10d\n", stats->ld_l1hit);
2413	fprintf(stream: out, format: " L2D hits on shared lines : %10d\n", stats->ld_l2hit);
2414	fprintf(stream: out, format: " LLC hits on shared lines : %10d\n", stats->ld_llchit + stats->lcl_hitm);
2415	fprintf(stream: out, format: " Load hits on peer cache or nodes : %10d\n", stats->lcl_peer + stats->rmt_peer);
2416	fprintf(stream: out, format: " Locked Access on shared lines : %10d\n", stats->locks);
2417	fprintf(stream: out, format: " Blocked Access on shared lines : %10d\n", stats->blk_data + stats->blk_addr);
2418	fprintf(stream: out, format: " Store HITs on shared lines : %10d\n", stats->store);
2419	fprintf(stream: out, format: " Store L1D hits on shared lines : %10d\n", stats->st_l1hit);
2420	fprintf(stream: out, format: " Store No available memory level : %10d\n", stats->st_na);
2421	fprintf(stream: out, format: " Total Merged records : %10d\n", hitm_cnt + stats->store);
2422	}
2423
2424	static void print_cacheline(struct c2c_hists *c2c_hists,
2425	struct hist_entry *he_cl,
2426	struct perf_hpp_list *hpp_list,
2427	FILE *out)
2428	{
2429	char bf[1000];
2430	struct perf_hpp hpp = {
2431	.buf = bf,
2432	.size = 1000,
2433	};
2434	static bool once;
2435
2436	if (!once) {
2437	hists__fprintf_headers(&c2c_hists->hists, out);
2438	once = true;
2439	} else {
2440	fprintf(stream: out, format: "\n");
2441	}
2442
2443	fprintf(stream: out, format: " ----------------------------------------------------------------------\n");
2444	__hist_entry__snprintf(he_cl, &hpp, hpp_list);
2445	fprintf(stream: out, format: "%s\n", bf);
2446	fprintf(stream: out, format: " ----------------------------------------------------------------------\n");
2447
2448	hists__fprintf(&c2c_hists->hists, false, 0, 0, 0, out, false);
2449	}
2450
2451	static void print_pareto(FILE *out)
2452	{
2453	struct perf_hpp_list hpp_list;
2454	struct rb_node *nd;
2455	int ret;
2456	const char *cl_output;
2457
2458	if (c2c.display != DISPLAY_SNP_PEER)
2459	cl_output = "cl_num,"
2460	"cl_rmt_hitm,"
2461	"cl_lcl_hitm,"
2462	"cl_stores_l1hit,"
2463	"cl_stores_l1miss,"
2464	"cl_stores_na,"
2465	"dcacheline";
2466	else
2467	cl_output = "cl_num,"
2468	"cl_rmt_peer,"
2469	"cl_lcl_peer,"
2470	"cl_stores_l1hit,"
2471	"cl_stores_l1miss,"
2472	"cl_stores_na,"
2473	"dcacheline";
2474
2475	perf_hpp_list__init(&hpp_list);
2476	ret = hpp_list__parse(hpp_list: &hpp_list, output_: cl_output, NULL);
2477
2478	if (WARN_ONCE(ret, "failed to setup sort entries\n"))
2479	return;
2480
2481	nd = rb_first_cached(&c2c.hists.hists.entries);
2482
2483	for (; nd; nd = rb_next(nd)) {
2484	struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
2485	struct c2c_hist_entry *c2c_he;
2486
2487	if (he->filtered)
2488	continue;
2489
2490	c2c_he = container_of(he, struct c2c_hist_entry, he);
2491	print_cacheline(c2c_hists: c2c_he->hists, he_cl: he, hpp_list: &hpp_list, out);
2492	}
2493	}
2494
2495	static void print_c2c_info(FILE *out, struct perf_session *session)
2496	{
2497	struct evlist *evlist = session->evlist;
2498	struct evsel *evsel;
2499	bool first = true;
2500
2501	fprintf(stream: out, format: "=================================================\n");
2502	fprintf(stream: out, format: " c2c details \n");
2503	fprintf(stream: out, format: "=================================================\n");
2504
2505	evlist__for_each_entry(evlist, evsel) {
2506	fprintf(stream: out, format: "%-36s: %s\n", first ? " Events" : "", evsel__name(evsel));
2507	first = false;
2508	}
2509	fprintf(stream: out, format: " Cachelines sort on : %s\n",
2510	display_str[c2c.display]);
2511	fprintf(stream: out, format: " Cacheline data grouping : %s\n", c2c.cl_sort);
2512	}
2513
2514	static void perf_c2c__hists_fprintf(FILE *out, struct perf_session *session)
2515	{
2516	setup_pager();
2517
2518	print_c2c__display_stats(out);
2519	fprintf(stream: out, format: "\n");
2520	print_shared_cacheline_info(out);
2521	fprintf(stream: out, format: "\n");
2522	print_c2c_info(out, session);
2523
2524	if (c2c.stats_only)
2525	return;
2526
2527	fprintf(stream: out, format: "\n");
2528	fprintf(stream: out, format: "=================================================\n");
2529	fprintf(stream: out, format: " Shared Data Cache Line Table \n");
2530	fprintf(stream: out, format: "=================================================\n");
2531	fprintf(stream: out, format: "#\n");
2532
2533	hists__fprintf(&c2c.hists.hists, true, 0, 0, 0, stdout, true);
2534
2535	fprintf(stream: out, format: "\n");
2536	fprintf(stream: out, format: "=================================================\n");
2537	fprintf(stream: out, format: " Shared Cache Line Distribution Pareto \n");
2538	fprintf(stream: out, format: "=================================================\n");
2539	fprintf(stream: out, format: "#\n");
2540
2541	print_pareto(out);
2542	}
2543
2544	#ifdef HAVE_SLANG_SUPPORT
2545	static void c2c_browser__update_nr_entries(struct hist_browser *hb)
2546	{
2547	u64 nr_entries = 0;
2548	struct rb_node *nd = rb_first_cached(&hb->hists->entries);
2549
2550	while (nd) {
2551	struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
2552
2553	if (!he->filtered)
2554	nr_entries++;
2555
2556	nd = rb_next(nd);
2557	}
2558
2559	hb->nr_non_filtered_entries = nr_entries;
2560	}
2561
2562	struct c2c_cacheline_browser {
2563	struct hist_browser hb;
2564	struct hist_entry *he;
2565	};
2566
2567	static int
2568	perf_c2c_cacheline_browser__title(struct hist_browser *browser,
2569	char *bf, size_t size)
2570	{
2571	struct c2c_cacheline_browser *cl_browser;
2572	struct hist_entry *he;
2573	uint64_t addr = 0;
2574
2575	cl_browser = container_of(browser, struct c2c_cacheline_browser, hb);
2576	he = cl_browser->he;
2577
2578	if (he->mem_info)
2579	addr = cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl);
2580
2581	scnprintf(bf, size, "Cacheline 0x%lx", addr);
2582	return 0;
2583	}
2584
2585	static struct c2c_cacheline_browser*
2586	c2c_cacheline_browser__new(struct hists *hists, struct hist_entry *he)
2587	{
2588	struct c2c_cacheline_browser *browser;
2589
2590	browser = zalloc(sizeof(*browser));
2591	if (browser) {
2592	hist_browser__init(&browser->hb, hists);
2593	browser->hb.c2c_filter = true;
2594	browser->hb.title = perf_c2c_cacheline_browser__title;
2595	browser->he = he;
2596	}
2597
2598	return browser;
2599	}
2600
2601	static int perf_c2c__browse_cacheline(struct hist_entry *he)
2602	{
2603	struct c2c_hist_entry *c2c_he;
2604	struct c2c_hists *c2c_hists;
2605	struct c2c_cacheline_browser *cl_browser;
2606	struct hist_browser *browser;
2607	int key = -1;
2608	static const char help[] =
2609	" ENTER Toggle callchains (if present) \n"
2610	" n Toggle Node details info \n"
2611	" s Toggle full length of symbol and source line columns \n"
2612	" q Return back to cacheline list \n";
2613
2614	if (!he)
2615	return 0;
2616
2617	/* Display compact version first. */
2618	c2c.symbol_full = false;
2619
2620	c2c_he = container_of(he, struct c2c_hist_entry, he);
2621	c2c_hists = c2c_he->hists;
2622
2623	cl_browser = c2c_cacheline_browser__new(&c2c_hists->hists, he);
2624	if (cl_browser == NULL)
2625	return -1;
2626
2627	browser = &cl_browser->hb;
2628
2629	/* reset abort key so that it can get Ctrl-C as a key */
2630	SLang_reset_tty();
2631	SLang_init_tty(0, 0, 0);
2632
2633	c2c_browser__update_nr_entries(browser);
2634
2635	while (1) {
2636	key = hist_browser__run(browser, "? - help", true, 0);
2637
2638	switch (key) {
2639	case 's':
2640	c2c.symbol_full = !c2c.symbol_full;
2641	break;
2642	case 'n':
2643	c2c.node_info = (c2c.node_info + 1) % 3;
2644	setup_nodes_header();
2645	break;
2646	case 'q':
2647	goto out;
2648	case '?':
2649	ui_browser__help_window(&browser->b, help);
2650	break;
2651	default:
2652	break;
2653	}
2654	}
2655
2656	out:
2657	free(cl_browser);
2658	return 0;
2659	}
2660
2661	static int perf_c2c_browser__title(struct hist_browser *browser,
2662	char *bf, size_t size)
2663	{
2664	scnprintf(bf, size,
2665	"Shared Data Cache Line Table "
2666	"(%lu entries, sorted on %s)",
2667	browser->nr_non_filtered_entries,
2668	display_str[c2c.display]);
2669	return 0;
2670	}
2671
2672	static struct hist_browser*
2673	perf_c2c_browser__new(struct hists *hists)
2674	{
2675	struct hist_browser *browser = hist_browser__new(hists);
2676
2677	if (browser) {
2678	browser->title = perf_c2c_browser__title;
2679	browser->c2c_filter = true;
2680	}
2681
2682	return browser;
2683	}
2684
2685	static int perf_c2c__hists_browse(struct hists *hists)
2686	{
2687	struct hist_browser *browser;
2688	int key = -1;
2689	static const char help[] =
2690	" d Display cacheline details \n"
2691	" ENTER Toggle callchains (if present) \n"
2692	" q Quit \n";
2693
2694	browser = perf_c2c_browser__new(hists);
2695	if (browser == NULL)
2696	return -1;
2697
2698	/* reset abort key so that it can get Ctrl-C as a key */
2699	SLang_reset_tty();
2700	SLang_init_tty(0, 0, 0);
2701
2702	c2c_browser__update_nr_entries(browser);
2703
2704	while (1) {
2705	key = hist_browser__run(browser, "? - help", true, 0);
2706
2707	switch (key) {
2708	case 'q':
2709	goto out;
2710	case 'd':
2711	perf_c2c__browse_cacheline(browser->he_selection);
2712	break;
2713	case '?':
2714	ui_browser__help_window(&browser->b, help);
2715	break;
2716	default:
2717	break;
2718	}
2719	}
2720
2721	out:
2722	hist_browser__delete(browser);
2723	return 0;
2724	}
2725
2726	static void perf_c2c_display(struct perf_session *session)
2727	{
2728	if (use_browser == 0)
2729	perf_c2c__hists_fprintf(stdout, session);
2730	else
2731	perf_c2c__hists_browse(&c2c.hists.hists);
2732	}
2733	#else
2734	static void perf_c2c_display(struct perf_session *session)
2735	{
2736	use_browser = 0;
2737	perf_c2c__hists_fprintf(stdout, session);
2738	}
2739	#endif /* HAVE_SLANG_SUPPORT */
2740
2741	static char *fill_line(const char *orig, int len)
2742	{
2743	int i, j, olen = strlen(orig);
2744	char *buf;
2745
2746	buf = zalloc(len + 1);
2747	if (!buf)
2748	return NULL;
2749
2750	j = len / 2 - olen / 2;
2751
2752	for (i = 0; i < j - 1; i++)
2753	buf[i] = '-';
2754
2755	buf[i++] = ' ';
2756
2757	strcpy(buf + i, orig);
2758
2759	i += olen;
2760
2761	buf[i++] = ' ';
2762
2763	for (; i < len; i++)
2764	buf[i] = '-';
2765
2766	return buf;
2767	}
2768
2769	static int ui_quirks(void)
2770	{
2771	const char *nodestr = "Data address";
2772	char *buf;
2773
2774	if (!c2c.use_stdio) {
2775	dim_offset.width = 5;
2776	dim_offset.header = header_offset_tui;
2777	nodestr = chk_double_cl ? "Double-CL" : "CL";
2778	}
2779
2780	dim_percent_costly_snoop.header = percent_costly_snoop_header[c2c.display];
2781
2782	/* Fix the zero line for dcacheline column. */
2783	buf = fill_line(chk_double_cl ? "Double-Cacheline" : "Cacheline",
2784	dim_dcacheline.width +
2785	dim_dcacheline_node.width +
2786	dim_dcacheline_count.width + 4);
2787	if (!buf)
2788	return -ENOMEM;
2789
2790	dim_dcacheline.header.line[0].text = buf;
2791
2792	/* Fix the zero line for offset column. */
2793	buf = fill_line(orig: nodestr, len: dim_offset.width +
2794	dim_offset_node.width +
2795	dim_dcacheline_count.width + 4);
2796	if (!buf)
2797	return -ENOMEM;
2798
2799	dim_offset.header.line[0].text = buf;
2800
2801	return 0;
2802	}
2803
2804	#define CALLCHAIN_DEFAULT_OPT "graph,0.5,caller,function,percent"
2805
2806	const char callchain_help[] = "Display call graph (stack chain/backtrace):\n\n"
2807	CALLCHAIN_REPORT_HELP
2808	"\n\t\t\t\tDefault: " CALLCHAIN_DEFAULT_OPT;
2809
2810	static int
2811	parse_callchain_opt(const struct option *opt, const char *arg, int unset)
2812	{
2813	struct callchain_param *callchain = opt->value;
2814
2815	callchain->enabled = !unset;
2816	/*
2817	* --no-call-graph
2818	*/
2819	if (unset) {
2820	symbol_conf.use_callchain = false;
2821	callchain->mode = CHAIN_NONE;
2822	return 0;
2823	}
2824
2825	return parse_callchain_report_opt(arg);
2826	}
2827
2828	static int setup_callchain(struct evlist *evlist)
2829	{
2830	u64 sample_type = evlist__combined_sample_type(evlist);
2831	enum perf_call_graph_mode mode = CALLCHAIN_NONE;
2832
2833	if ((sample_type & PERF_SAMPLE_REGS_USER) &&
2834	(sample_type & PERF_SAMPLE_STACK_USER)) {
2835	mode = CALLCHAIN_DWARF;
2836	dwarf_callchain_users = true;
2837	} else if (sample_type & PERF_SAMPLE_BRANCH_STACK)
2838	mode = CALLCHAIN_LBR;
2839	else if (sample_type & PERF_SAMPLE_CALLCHAIN)
2840	mode = CALLCHAIN_FP;
2841
2842	if (!callchain_param.enabled &&
2843	callchain_param.mode != CHAIN_NONE &&
2844	mode != CALLCHAIN_NONE) {
2845	symbol_conf.use_callchain = true;
2846	if (callchain_register_param(&callchain_param) < 0) {
2847	ui__error("Can't register callchain params.\n");
2848	return -EINVAL;
2849	}
2850	}
2851
2852	if (c2c.stitch_lbr && (mode != CALLCHAIN_LBR)) {
2853	ui__warning("Can't find LBR callchain. Switch off --stitch-lbr.\n"
2854	"Please apply --call-graph lbr when recording.\n");
2855	c2c.stitch_lbr = false;
2856	}
2857
2858	callchain_param.record_mode = mode;
2859	callchain_param.min_percent = 0;
2860	return 0;
2861	}
2862
2863	static int setup_display(const char *str)
2864	{
2865	const char *display = str;
2866
2867	if (!strcmp(display, "tot"))
2868	c2c.display = DISPLAY_TOT_HITM;
2869	else if (!strcmp(display, "rmt"))
2870	c2c.display = DISPLAY_RMT_HITM;
2871	else if (!strcmp(display, "lcl"))
2872	c2c.display = DISPLAY_LCL_HITM;
2873	else if (!strcmp(display, "peer"))
2874	c2c.display = DISPLAY_SNP_PEER;
2875	else {
2876	pr_err("failed: unknown display type: %s\n", str);
2877	return -1;
2878	}
2879
2880	return 0;
2881	}
2882
2883	#define for_each_token(__tok, __buf, __sep, __tmp) \
2884	for (__tok = strtok_r(__buf, __sep, &__tmp); __tok; \
2885	__tok = strtok_r(NULL, __sep, &__tmp))
2886
2887	static int build_cl_output(char *cl_sort, bool no_source)
2888	{
2889	char *tok, *tmp, *buf = strdup(cl_sort);
2890	bool add_pid = false;
2891	bool add_tid = false;
2892	bool add_iaddr = false;
2893	bool add_sym = false;
2894	bool add_dso = false;
2895	bool add_src = false;
2896	int ret = 0;
2897
2898	if (!buf)
2899	return -ENOMEM;
2900
2901	for_each_token(tok, buf, ",", tmp) {
2902	if (!strcmp(tok, "tid")) {
2903	add_tid = true;
2904	} else if (!strcmp(tok, "pid")) {
2905	add_pid = true;
2906	} else if (!strcmp(tok, "iaddr")) {
2907	add_iaddr = true;
2908	add_sym = true;
2909	add_dso = true;
2910	add_src = no_source ? false : true;
2911	} else if (!strcmp(tok, "dso")) {
2912	add_dso = true;
2913	} else if (strcmp(tok, "offset")) {
2914	pr_err("unrecognized sort token: %s\n", tok);
2915	ret = -EINVAL;
2916	goto err;
2917	}
2918	}
2919
2920	if (asprintf(ptr: &c2c.cl_output,
2921	fmt: "%s%s%s%s%s%s%s%s%s%s%s%s",
2922	c2c.use_stdio ? "cl_num_empty," : "",
2923	c2c.display == DISPLAY_SNP_PEER ? "percent_rmt_peer,"
2924	"percent_lcl_peer," :
2925	"percent_rmt_hitm,"
2926	"percent_lcl_hitm,",
2927	"percent_stores_l1hit,"
2928	"percent_stores_l1miss,"
2929	"percent_stores_na,"
2930	"offset,offset_node,dcacheline_count,",
2931	add_pid ? "pid," : "",
2932	add_tid ? "tid," : "",
2933	add_iaddr ? "iaddr," : "",
2934	c2c.display == DISPLAY_SNP_PEER ? "mean_rmt_peer,"
2935	"mean_lcl_peer," :
2936	"mean_rmt,"
2937	"mean_lcl,",
2938	"mean_load,"
2939	"tot_recs,"
2940	"cpucnt,",
2941	add_sym ? "symbol," : "",
2942	add_dso ? "dso," : "",
2943	add_src ? "cl_srcline," : "",
2944	"node") < 0) {
2945	ret = -ENOMEM;
2946	goto err;
2947	}
2948
2949	c2c.show_src = add_src;
2950	err:
2951	free(buf);
2952	return ret;
2953	}
2954
2955	static int setup_coalesce(const char *coalesce, bool no_source)
2956	{
2957	const char *c = coalesce ?: coalesce_default;
2958	const char *sort_str = NULL;
2959
2960	if (asprintf(ptr: &c2c.cl_sort, fmt: "offset,%s", c) < 0)
2961	return -ENOMEM;
2962
2963	if (build_cl_output(cl_sort: c2c.cl_sort, no_source))
2964	return -1;
2965
2966	if (c2c.display == DISPLAY_TOT_HITM)
2967	sort_str = "tot_hitm";
2968	else if (c2c.display == DISPLAY_RMT_HITM)
2969	sort_str = "rmt_hitm,lcl_hitm";
2970	else if (c2c.display == DISPLAY_LCL_HITM)
2971	sort_str = "lcl_hitm,rmt_hitm";
2972	else if (c2c.display == DISPLAY_SNP_PEER)
2973	sort_str = "tot_peer";
2974
2975	if (asprintf(ptr: &c2c.cl_resort, fmt: "offset,%s", sort_str) < 0)
2976	return -ENOMEM;
2977
2978	pr_debug("coalesce sort fields: %s\n", c2c.cl_sort);
2979	pr_debug("coalesce resort fields: %s\n", c2c.cl_resort);
2980	pr_debug("coalesce output fields: %s\n", c2c.cl_output);
2981	return 0;
2982	}
2983
2984	static int perf_c2c__report(int argc, const char **argv)
2985	{
2986	struct itrace_synth_opts itrace_synth_opts = {
2987	.set = true,
2988	.mem = true, /* Only enable memory event */
2989	.default_no_sample = true,
2990	};
2991
2992	struct perf_session *session;
2993	struct ui_progress prog;
2994	struct perf_data data = {
2995	.mode = PERF_DATA_MODE_READ,
2996	};
2997	char callchain_default_opt[] = CALLCHAIN_DEFAULT_OPT;
2998	const char *display = NULL;
2999	const char *coalesce = NULL;
3000	bool no_source = false;
3001	const struct option options[] = {
3002	OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
3003	"file", "vmlinux pathname"),
3004	OPT_STRING('i', "input", &input_name, "file",
3005	"the input file to process"),
3006	OPT_INCR('N', "node-info", &c2c.node_info,
3007	"show extra node info in report (repeat for more info)"),
3008	OPT_BOOLEAN(0, "stdio", &c2c.use_stdio, "Use the stdio interface"),
3009	OPT_BOOLEAN(0, "stats", &c2c.stats_only,
3010	"Display only statistic tables (implies --stdio)"),
3011	OPT_BOOLEAN(0, "full-symbols", &c2c.symbol_full,
3012	"Display full length of symbols"),
3013	OPT_BOOLEAN(0, "no-source", &no_source,
3014	"Do not display Source Line column"),
3015	OPT_BOOLEAN(0, "show-all", &c2c.show_all,
3016	"Show all captured HITM lines."),
3017	OPT_CALLBACK_DEFAULT('g', "call-graph", &callchain_param,
3018	"print_type,threshold[,print_limit],order,sort_key[,branch],value",
3019	callchain_help, &parse_callchain_opt,
3020	callchain_default_opt),
3021	OPT_STRING('d', "display", &display, "Switch HITM output type", "tot,lcl,rmt,peer"),
3022	OPT_STRING('c', "coalesce", &coalesce, "coalesce fields",
3023	"coalesce fields: pid,tid,iaddr,dso"),
3024	OPT_BOOLEAN('f', "force", &symbol_conf.force, "don't complain, do it"),
3025	OPT_BOOLEAN(0, "stitch-lbr", &c2c.stitch_lbr,
3026	"Enable LBR callgraph stitching approach"),
3027	OPT_BOOLEAN(0, "double-cl", &chk_double_cl, "Detect adjacent cacheline false sharing"),
3028	OPT_PARENT(c2c_options),
3029	OPT_END()
3030	};
3031	int err = 0;
3032	const char *output_str, *sort_str = NULL;
3033
3034	argc = parse_options(argc, argv, options, report_c2c_usage,
3035	PARSE_OPT_STOP_AT_NON_OPTION);
3036	if (argc)
3037	usage_with_options(report_c2c_usage, options);
3038
3039	#ifndef HAVE_SLANG_SUPPORT
3040	c2c.use_stdio = true;
3041	#endif
3042
3043	if (c2c.stats_only)
3044	c2c.use_stdio = true;
3045
3046	err = symbol__validate_sym_arguments();
3047	if (err)
3048	goto out;
3049
3050	if (!input_name || !strlen(input_name))
3051	input_name = "perf.data";
3052
3053	data.path = input_name;
3054	data.force = symbol_conf.force;
3055
3056	perf_tool__init(&c2c.tool, /*ordered_events=*/true);
3057	c2c.tool.sample = process_sample_event;
3058	c2c.tool.mmap = perf_event__process_mmap;
3059	c2c.tool.mmap2 = perf_event__process_mmap2;
3060	c2c.tool.comm = perf_event__process_comm;
3061	c2c.tool.exit = perf_event__process_exit;
3062	c2c.tool.fork = perf_event__process_fork;
3063	c2c.tool.lost = perf_event__process_lost;
3064	c2c.tool.attr = perf_event__process_attr;
3065	c2c.tool.auxtrace_info = perf_event__process_auxtrace_info;
3066	c2c.tool.auxtrace = perf_event__process_auxtrace;
3067	c2c.tool.auxtrace_error = perf_event__process_auxtrace_error;
3068	c2c.tool.ordering_requires_timestamps = true;
3069	session = perf_session__new(&data, &c2c.tool);
3070	if (IS_ERR(session)) {
3071	err = PTR_ERR(session);
3072	pr_debug("Error creating perf session\n");
3073	goto out;
3074	}
3075
3076	/*
3077	* Use the 'tot' as default display type if user doesn't specify it;
3078	* since Arm64 platform doesn't support HITMs flag, use 'peer' as the
3079	* default display type.
3080	*/
3081	if (!display) {
3082	if (!strcmp(perf_env__arch(&session->header.env), "arm64"))
3083	display = "peer";
3084	else
3085	display = "tot";
3086	}
3087
3088	err = setup_display(display);
3089	if (err)
3090	goto out_session;
3091
3092	err = setup_coalesce(coalesce, no_source);
3093	if (err) {
3094	pr_debug("Failed to initialize hists\n");
3095	goto out_session;
3096	}
3097
3098	err = c2c_hists__init(hists: &c2c.hists, sort: "dcacheline", nr_header_lines: 2);
3099	if (err) {
3100	pr_debug("Failed to initialize hists\n");
3101	goto out_session;
3102	}
3103
3104	session->itrace_synth_opts = &itrace_synth_opts;
3105
3106	err = setup_nodes(session);
3107	if (err) {
3108	pr_err("Failed setup nodes\n");
3109	goto out_session;
3110	}
3111
3112	err = mem2node__init(&c2c.mem2node, &session->header.env);
3113	if (err)
3114	goto out_session;
3115
3116	err = setup_callchain(session->evlist);
3117	if (err)
3118	goto out_mem2node;
3119
3120	if (symbol__init(&session->header.env) < 0)
3121	goto out_mem2node;
3122
3123	/* No pipe support at the moment. */
3124	if (perf_data__is_pipe(session->data)) {
3125	pr_debug("No pipe support at the moment.\n");
3126	goto out_mem2node;
3127	}
3128
3129	if (c2c.use_stdio)
3130	use_browser = 0;
3131	else
3132	use_browser = 1;
3133
3134	setup_browser(false);
3135
3136	err = perf_session__process_events(session);
3137	if (err) {
3138	pr_err("failed to process sample\n");
3139	goto out_mem2node;
3140	}
3141
3142	if (c2c.display != DISPLAY_SNP_PEER)
3143	output_str = "cl_idx,"
3144	"dcacheline,"
3145	"dcacheline_node,"
3146	"dcacheline_count,"
3147	"percent_costly_snoop,"
3148	"tot_hitm,lcl_hitm,rmt_hitm,"
3149	"tot_recs,"
3150	"tot_loads,"
3151	"tot_stores,"
3152	"stores_l1hit,stores_l1miss,stores_na,"
3153	"ld_fbhit,ld_l1hit,ld_l2hit,"
3154	"ld_lclhit,lcl_hitm,"
3155	"ld_rmthit,rmt_hitm,"
3156	"dram_lcl,dram_rmt";
3157	else
3158	output_str = "cl_idx,"
3159	"dcacheline,"
3160	"dcacheline_node,"
3161	"dcacheline_count,"
3162	"percent_costly_snoop,"
3163	"tot_peer,lcl_peer,rmt_peer,"
3164	"tot_recs,"
3165	"tot_loads,"
3166	"tot_stores,"
3167	"stores_l1hit,stores_l1miss,stores_na,"
3168	"ld_fbhit,ld_l1hit,ld_l2hit,"
3169	"ld_lclhit,lcl_hitm,"
3170	"ld_rmthit,rmt_hitm,"
3171	"dram_lcl,dram_rmt";
3172
3173	if (c2c.display == DISPLAY_TOT_HITM)
3174	sort_str = "tot_hitm";
3175	else if (c2c.display == DISPLAY_RMT_HITM)
3176	sort_str = "rmt_hitm";
3177	else if (c2c.display == DISPLAY_LCL_HITM)
3178	sort_str = "lcl_hitm";
3179	else if (c2c.display == DISPLAY_SNP_PEER)
3180	sort_str = "tot_peer";
3181
3182	c2c_hists__reinit(c2c_hists: &c2c.hists, output: output_str, sort: sort_str);
3183
3184	ui_progress__init(&prog, c2c.hists.hists.nr_entries, "Sorting...");
3185
3186	hists__collapse_resort(&c2c.hists.hists, NULL);
3187	hists__output_resort_cb(&c2c.hists.hists, &prog, resort_shared_cl_cb);
3188	hists__iterate_cb(&c2c.hists.hists, resort_cl_cb);
3189
3190	ui_progress__finish();
3191
3192	if (ui_quirks()) {
3193	pr_err("failed to setup UI\n");
3194	goto out_mem2node;
3195	}
3196
3197	perf_c2c_display(session);
3198
3199	out_mem2node:
3200	mem2node__exit(&c2c.mem2node);
3201	out_session:
3202	perf_session__delete(session);
3203	out:
3204	return err;
3205	}
3206
3207	static int parse_record_events(const struct option *opt,
3208	const char *str, int unset __maybe_unused)
3209	{
3210	bool *event_set = (bool *) opt->value;
3211	struct perf_pmu *pmu;
3212
3213	pmu = perf_mem_events_find_pmu();
3214	if (!pmu) {
3215	pr_err("failed: there is no PMU that supports perf c2c\n");
3216	exit(-1);
3217	}
3218
3219	if (!strcmp(str, "list")) {
3220	perf_pmu__mem_events_list(pmu);
3221	exit(0);
3222	}
3223	if (perf_pmu__mem_events_parse(pmu, str))
3224	exit(-1);
3225
3226	*event_set = true;
3227	return 0;
3228	}
3229
3230
3231	static const char * const __usage_record[] = {
3232	"perf c2c record [<options>] [<command>]",
3233	"perf c2c record [<options>] -- <command> [<options>]",
3234	NULL
3235	};
3236
3237	static const char * const *record_mem_usage = __usage_record;
3238
3239	static int perf_c2c__record(int argc, const char **argv)
3240	{
3241	int rec_argc, i = 0, j;
3242	const char **rec_argv;
3243	char *event_name_storage = NULL;
3244	int ret;
3245	bool all_user = false, all_kernel = false;
3246	bool event_set = false;
3247	struct perf_mem_event *e;
3248	struct perf_pmu *pmu;
3249	struct option options[] = {
3250	OPT_CALLBACK('e', "event", &event_set, "event",
3251	"event selector. Use 'perf c2c record -e list' to list available events",
3252	parse_record_events),
3253	OPT_BOOLEAN('u', "all-user", &all_user, "collect only user level data"),
3254	OPT_BOOLEAN('k', "all-kernel", &all_kernel, "collect only kernel level data"),
3255	OPT_UINTEGER('l', "ldlat", &perf_mem_events__loads_ldlat, "setup mem-loads latency"),
3256	OPT_PARENT(c2c_options),
3257	OPT_END()
3258	};
3259
3260	pmu = perf_mem_events_find_pmu();
3261	if (!pmu) {
3262	pr_err("failed: no PMU supports the memory events\n");
3263	return -1;
3264	}
3265
3266	if (perf_pmu__mem_events_init()) {
3267	pr_err("failed: memory events not supported\n");
3268	return -1;
3269	}
3270
3271	argc = parse_options(argc, argv, options, record_mem_usage,
3272	PARSE_OPT_KEEP_UNKNOWN);
3273
3274	/* Max number of arguments multiplied by number of PMUs that can support them. */
3275	rec_argc = argc + 11 * (perf_pmu__mem_events_num_mem_pmus(pmu) + 1);
3276
3277	rec_argv = calloc(rec_argc + 1, sizeof(char *));
3278	if (!rec_argv)
3279	return -1;
3280
3281	rec_argv[i++] = "record";
3282
3283	if (!event_set) {
3284	e = perf_pmu__mem_events_ptr(pmu, PERF_MEM_EVENTS__LOAD_STORE);
3285	/*
3286	* The load and store operations are required, use the event
3287	* PERF_MEM_EVENTS__LOAD_STORE if it is supported.
3288	*/
3289	if (e->tag) {
3290	perf_mem_record[PERF_MEM_EVENTS__LOAD_STORE] = true;
3291	rec_argv[i++] = "-W";
3292	} else {
3293	perf_mem_record[PERF_MEM_EVENTS__LOAD] = true;
3294	perf_mem_record[PERF_MEM_EVENTS__STORE] = true;
3295	}
3296	}
3297
3298	if (perf_mem_record[PERF_MEM_EVENTS__LOAD])
3299	rec_argv[i++] = "-W";
3300
3301	rec_argv[i++] = "-d";
3302	rec_argv[i++] = "--phys-data";
3303	rec_argv[i++] = "--sample-cpu";
3304
3305	ret = perf_mem_events__record_args(rec_argv, &i, &event_name_storage);
3306	if (ret)
3307	goto out;
3308
3309	if (all_user)
3310	rec_argv[i++] = "--all-user";
3311
3312	if (all_kernel)
3313	rec_argv[i++] = "--all-kernel";
3314
3315	for (j = 0; j < argc; j++, i++)
3316	rec_argv[i] = argv[j];
3317
3318	if (verbose > 0) {
3319	pr_debug("calling: ");
3320
3321	j = 0;
3322
3323	while (rec_argv[j]) {
3324	pr_debug("%s ", rec_argv[j]);
3325	j++;
3326	}
3327	pr_debug("\n");
3328	}
3329
3330	ret = cmd_record(argc: i, argv: rec_argv);
3331	out:
3332	free(event_name_storage);
3333	free(rec_argv);
3334	return ret;
3335	}
3336
3337	int cmd_c2c(int argc, const char **argv)
3338	{
3339	argc = parse_options(argc, argv, c2c_options, c2c_usage,
3340	PARSE_OPT_STOP_AT_NON_OPTION);
3341
3342	if (!argc)
3343	usage_with_options(c2c_usage, c2c_options);
3344
3345	if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) {
3346	return perf_c2c__record(argc, argv);
3347	} else if (strlen(argv[0]) > 2 && strstarts("report", argv[0])) {
3348	return perf_c2c__report(argc, argv);
3349	} else {
3350	usage_with_options(c2c_usage, c2c_options);
3351	}
3352
3353	return 0;
3354	}
3355