1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* This is rewrite of original c2c tool introduced in here:
4	* http://lwn.net/Articles/588866/
5	*
6	* The original tool was changed to fit in current perf state.
7	*
8	* Original authors:
9	* Don Zickus <dzickus@redhat.com>
10	* Dick Fowles <fowles@inreach.com>
11	* Joe Mario <jmario@redhat.com>
12	*/
13	#include <errno.h>
14	#include <inttypes.h>
15	#include <linux/compiler.h>
16	#include <linux/err.h>
17	#include <linux/kernel.h>
18	#include <linux/stringify.h>
19	#include <linux/zalloc.h>
20	#include <asm/bug.h>
21	#include <sys/param.h>
22	#include "debug.h"
23	#include "builtin.h"
24	#include <perf/cpumap.h>
25	#include <subcmd/pager.h>
26	#include <subcmd/parse-options.h>
27	#include "map_symbol.h"
28	#include "mem-events.h"
29	#include "session.h"
30	#include "hist.h"
31	#include "sort.h"
32	#include "tool.h"
33	#include "cacheline.h"
34	#include "data.h"
35	#include "event.h"
36	#include "evlist.h"
37	#include "evsel.h"
38	#include "ui/browsers/hists.h"
39	#include "thread.h"
40	#include "mem2node.h"
41	#include "mem-info.h"
42	#include "symbol.h"
43	#include "ui/ui.h"
44	#include "ui/progress.h"
45	#include "pmus.h"
46	#include "string2.h"
47	#include "util/util.h"
48	#include "util/symbol.h"
49	#include "util/annotate.h"
50
51	struct c2c_hists {
52	struct hists hists;
53	struct perf_hpp_list list;
54	struct c2c_stats stats;
55	};
56
57	struct compute_stats {
58	struct stats lcl_hitm;
59	struct stats rmt_hitm;
60	struct stats lcl_peer;
61	struct stats rmt_peer;
62	struct stats load;
63	};
64
65	struct c2c_hist_entry {
66	struct c2c_hists *hists;
67	struct evsel *evsel;
68	struct c2c_stats stats;
69	unsigned long *cpuset;
70	unsigned long *nodeset;
71	struct c2c_stats *node_stats;
72	unsigned int cacheline_idx;
73
74	struct compute_stats cstats;
75
76	unsigned long paddr;
77	unsigned long paddr_cnt;
78	bool paddr_zero;
79	char *nodestr;
80
81	/*
82	* must be at the end,
83	* because of its callchain dynamic entry
84	*/
85	struct hist_entry he;
86	};
87
88	static char const *coalesce_default = "iaddr";
89
90	struct perf_c2c {
91	struct perf_tool tool;
92	struct c2c_hists hists;
93	struct mem2node mem2node;
94
95	unsigned long **nodes;
96	int nodes_cnt;
97	int cpus_cnt;
98	int *cpu2node;
99	int node_info;
100
101	bool show_src;
102	bool show_all;
103	bool use_stdio;
104	bool stats_only;
105	bool symbol_full;
106	bool stitch_lbr;
107
108	/* Shared cache line stats */
109	struct c2c_stats shared_clines_stats;
110	int shared_clines;
111
112	int display;
113
114	const char *coalesce;
115	char *cl_sort;
116	char *cl_resort;
117	char *cl_output;
118	};
119
120	enum {
121	DISPLAY_LCL_HITM,
122	DISPLAY_RMT_HITM,
123	DISPLAY_TOT_HITM,
124	DISPLAY_SNP_PEER,
125	DISPLAY_MAX,
126	};
127
128	static const char *display_str[DISPLAY_MAX] = {
129	[DISPLAY_LCL_HITM] = "Local HITMs",
130	[DISPLAY_RMT_HITM] = "Remote HITMs",
131	[DISPLAY_TOT_HITM] = "Total HITMs",
132	[DISPLAY_SNP_PEER] = "Peer Snoop",
133	};
134
135	static const struct option c2c_options[] = {
136	OPT_INCR('v', "verbose", &verbose, "be more verbose (show counter open errors, etc)"),
137	OPT_END()
138	};
139
140	static struct perf_c2c c2c;
141
142	static void *c2c_he_zalloc(size_t size)
143	{
144	struct c2c_hist_entry *c2c_he;
145
146	c2c_he = zalloc(size + sizeof(*c2c_he));
147	if (!c2c_he)
148	return NULL;
149
150	c2c_he->cpuset = bitmap_zalloc(c2c.cpus_cnt);
151	if (!c2c_he->cpuset)
152	goto out_free;
153
154	c2c_he->nodeset = bitmap_zalloc(c2c.nodes_cnt);
155	if (!c2c_he->nodeset)
156	goto out_free;
157
158	c2c_he->node_stats = zalloc(c2c.nodes_cnt * sizeof(*c2c_he->node_stats));
159	if (!c2c_he->node_stats)
160	goto out_free;
161
162	init_stats(&c2c_he->cstats.lcl_hitm);
163	init_stats(&c2c_he->cstats.rmt_hitm);
164	init_stats(&c2c_he->cstats.lcl_peer);
165	init_stats(&c2c_he->cstats.rmt_peer);
166	init_stats(&c2c_he->cstats.load);
167
168	return &c2c_he->he;
169
170	out_free:
171	zfree(&c2c_he->nodeset);
172	zfree(&c2c_he->cpuset);
173	free(c2c_he);
174	return NULL;
175	}
176
177	static void c2c_he_free(void *he)
178	{
179	struct c2c_hist_entry *c2c_he;
180
181	c2c_he = container_of(he, struct c2c_hist_entry, he);
182	if (c2c_he->hists) {
183	hists__delete_entries(&c2c_he->hists->hists);
184	zfree(&c2c_he->hists);
185	}
186
187	zfree(&c2c_he->cpuset);
188	zfree(&c2c_he->nodeset);
189	zfree(&c2c_he->nodestr);
190	zfree(&c2c_he->node_stats);
191	free(c2c_he);
192	}
193
194	static struct hist_entry_ops c2c_entry_ops = {
195	.new = c2c_he_zalloc,
196	.free = c2c_he_free,
197	};
198
199	static int c2c_hists__init(struct c2c_hists *hists,
200	const char *sort,
201	int nr_header_lines,
202	struct perf_env *env);
203
204	static struct c2c_hists*
205	he__get_c2c_hists(struct hist_entry *he,
206	const char *sort,
207	int nr_header_lines,
208	struct perf_env *env)
209	{
210	struct c2c_hist_entry *c2c_he;
211	struct c2c_hists *hists;
212	int ret;
213
214	c2c_he = container_of(he, struct c2c_hist_entry, he);
215	if (c2c_he->hists)
216	return c2c_he->hists;
217
218	hists = c2c_he->hists = zalloc(sizeof(*hists));
219	if (!hists)
220	return NULL;
221
222	ret = c2c_hists__init(hists, sort, nr_header_lines, env);
223	if (ret) {
224	free(hists);
225	return NULL;
226	}
227
228	return hists;
229	}
230
231	static void c2c_he__set_evsel(struct c2c_hist_entry *c2c_he,
232	struct evsel *evsel)
233	{
234	c2c_he->evsel = evsel;
235	}
236
237	static void c2c_he__set_cpu(struct c2c_hist_entry *c2c_he,
238	struct perf_sample *sample)
239	{
240	if (WARN_ONCE(sample->cpu == (unsigned int) -1,
241	"WARNING: no sample cpu value"))
242	return;
243
244	__set_bit(sample->cpu, c2c_he->cpuset);
245	}
246
247	static void c2c_he__set_node(struct c2c_hist_entry *c2c_he,
248	struct perf_sample *sample)
249	{
250	int node;
251
252	if (!sample->phys_addr) {
253	c2c_he->paddr_zero = true;
254	return;
255	}
256
257	node = mem2node__node(&c2c.mem2node, sample->phys_addr);
258	if (WARN_ONCE(node < 0, "WARNING: failed to find node\n"))
259	return;
260
261	__set_bit(node, c2c_he->nodeset);
262
263	if (c2c_he->paddr != sample->phys_addr) {
264	c2c_he->paddr_cnt++;
265	c2c_he->paddr = sample->phys_addr;
266	}
267	}
268
269	static void compute_stats(struct c2c_hist_entry *c2c_he,
270	struct c2c_stats *stats,
271	u64 weight)
272	{
273	struct compute_stats *cstats = &c2c_he->cstats;
274
275	if (stats->rmt_hitm)
276	update_stats(&cstats->rmt_hitm, weight);
277	else if (stats->lcl_hitm)
278	update_stats(&cstats->lcl_hitm, weight);
279	else if (stats->rmt_peer)
280	update_stats(&cstats->rmt_peer, weight);
281	else if (stats->lcl_peer)
282	update_stats(&cstats->lcl_peer, weight);
283	else if (stats->load)
284	update_stats(&cstats->load, weight);
285	}
286
287	/*
288	* Return true if annotation is possible. When list is NULL,
289	* it means that we are called at the c2c_browser level,
290	* in that case we allow annotation to be initialized. When list
291	* is non-NULL, it means that we are called at the cacheline_browser
292	* level, in that case we allow annotation only if use_browser
293	* is set and symbol information is available.
294	*/
295	static bool perf_c2c__has_annotation(struct perf_hpp_list *list)
296	{
297	if (use_browser != 1)
298	return false;
299	return !list || list->sym;
300	}
301
302	static void perf_c2c__evsel_hists_inc_stats(struct evsel *evsel,
303	struct hist_entry *he,
304	struct perf_sample *sample)
305	{
306	struct hists *evsel_hists = evsel__hists(evsel);
307
308	hists__inc_nr_samples(evsel_hists, he->filtered);
309	evsel_hists->stats.total_period += sample->period;
310	if (!he->filtered)
311	evsel_hists->stats.total_non_filtered_period += sample->period;
312	}
313
314	static int process_sample_event(const struct perf_tool *tool __maybe_unused,
315	union perf_event *event,
316	struct perf_sample *sample,
317	struct evsel *evsel,
318	struct machine *machine)
319	{
320	struct c2c_hists *c2c_hists = &c2c.hists;
321	struct c2c_hist_entry *c2c_he;
322	struct c2c_stats stats = { .nr_entries = 0, };
323	struct hist_entry *he;
324	struct addr_location al;
325	struct mem_info *mi = NULL;
326	struct callchain_cursor *cursor;
327	int ret;
328
329	addr_location__init(al: &al);
330	if (machine__resolve(machine, &al, sample) < 0) {
331	pr_debug("problem processing %d event, skipping it.\n",
332	event->header.type);
333	ret = -1;
334	goto out;
335	}
336
337	if (c2c.stitch_lbr)
338	thread__set_lbr_stitch_enable(al.thread, true);
339
340	cursor = get_tls_callchain_cursor();
341	ret = sample__resolve_callchain(sample, cursor, NULL,
342	evsel, &al, sysctl_perf_event_max_stack);
343	if (ret)
344	goto out;
345
346	mi = sample__resolve_mem(sample, &al);
347	if (mi == NULL) {
348	ret = -ENOMEM;
349	goto out;
350	}
351
352	c2c_decode_stats(stats: &stats, mi);
353
354	he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops,
355	&al, NULL, NULL, mi, NULL,
356	sample, true);
357	if (he == NULL) {
358	ret = -ENOMEM;
359	goto out;
360	}
361
362	c2c_he = container_of(he, struct c2c_hist_entry, he);
363	c2c_add_stats(stats: &c2c_he->stats, add: &stats);
364	c2c_add_stats(stats: &c2c_hists->stats, add: &stats);
365
366	c2c_he__set_cpu(c2c_he, sample);
367	c2c_he__set_node(c2c_he, sample);
368	c2c_he__set_evsel(c2c_he, evsel);
369
370	hists__inc_nr_samples(&c2c_hists->hists, he->filtered);
371
372	if (perf_c2c__has_annotation(NULL)) {
373	perf_c2c__evsel_hists_inc_stats(evsel, he, sample);
374	addr_map_symbol__inc_samples(ams: mem_info__iaddr(mi), sample, evsel);
375	}
376
377	ret = hist_entry__append_callchain(he, sample);
378
379	if (!ret) {
380	/*
381	* There's already been warning about missing
382	* sample's cpu value. Let's account all to
383	* node 0 in this case, without any further
384	* warning.
385	*
386	* Doing node stats only for single callchain data.
387	*/
388	int cpu = sample->cpu == (unsigned int) -1 ? 0 : sample->cpu;
389	int node = c2c.cpu2node[cpu];
390
391	c2c_hists = he__get_c2c_hists(he, sort: c2c.cl_sort, nr_header_lines: 2, env: machine->env);
392	if (!c2c_hists) {
393	ret = -ENOMEM;
394	goto out;
395	}
396
397	he = hists__add_entry_ops(&c2c_hists->hists, &c2c_entry_ops,
398	&al, NULL, NULL, mi, NULL,
399	sample, true);
400	if (he == NULL) {
401	ret = -ENOMEM;
402	goto out;
403	}
404
405	c2c_he = container_of(he, struct c2c_hist_entry, he);
406	c2c_add_stats(stats: &c2c_he->stats, add: &stats);
407	c2c_add_stats(stats: &c2c_hists->stats, add: &stats);
408	c2c_add_stats(stats: &c2c_he->node_stats[node], add: &stats);
409
410	compute_stats(c2c_he, stats: &stats, weight: sample->weight);
411
412	c2c_he__set_cpu(c2c_he, sample);
413	c2c_he__set_node(c2c_he, sample);
414	c2c_he__set_evsel(c2c_he, evsel);
415
416	hists__inc_nr_samples(&c2c_hists->hists, he->filtered);
417	ret = hist_entry__append_callchain(he, sample);
418	}
419
420	out:
421	mem_info__put(mi);
422	addr_location__exit(al: &al);
423	return ret;
424	}
425
426	static const char * const c2c_usage[] = {
427	"perf c2c {record|report}",
428	NULL
429	};
430
431	static const char * const __usage_report[] = {
432	"perf c2c report",
433	NULL
434	};
435
436	static const char * const *report_c2c_usage = __usage_report;
437
438	#define C2C_HEADER_MAX 2
439
440	struct c2c_header {
441	struct {
442	const char *text;
443	int span;
444	} line[C2C_HEADER_MAX];
445	};
446
447	struct c2c_dimension {
448	struct c2c_header header;
449	const char *name;
450	int width;
451	struct sort_entry *se;
452
453	int64_t (*cmp)(struct perf_hpp_fmt *fmt,
454	struct hist_entry *, struct hist_entry *);
455	int (*entry)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
456	struct hist_entry *he);
457	int (*color)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
458	struct hist_entry *he);
459	};
460
461	struct c2c_fmt {
462	struct perf_hpp_fmt fmt;
463	struct c2c_dimension *dim;
464	};
465
466	#define SYMBOL_WIDTH 30
467
468	static struct c2c_dimension dim_symbol;
469	static struct c2c_dimension dim_srcline;
470
471	static int symbol_width(struct hists *hists, struct sort_entry *se)
472	{
473	int width = hists__col_len(hists, se->se_width_idx);
474
475	if (!c2c.symbol_full)
476	width = MIN(width, SYMBOL_WIDTH);
477
478	return width;
479	}
480
481	static int c2c_width(struct perf_hpp_fmt *fmt,
482	struct perf_hpp *hpp __maybe_unused,
483	struct hists *hists)
484	{
485	struct c2c_fmt *c2c_fmt;
486	struct c2c_dimension *dim;
487
488	c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
489	dim = c2c_fmt->dim;
490
491	if (dim == &dim_symbol || dim == &dim_srcline)
492	return symbol_width(hists, se: dim->se);
493
494	return dim->se ? hists__col_len(hists, dim->se->se_width_idx) :
495	c2c_fmt->dim->width;
496	}
497
498	static int c2c_header(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
499	struct hists *hists, int line, int *span)
500	{
501	struct perf_hpp_list *hpp_list = hists->hpp_list;
502	struct c2c_fmt *c2c_fmt;
503	struct c2c_dimension *dim;
504	const char *text = NULL;
505	int width = c2c_width(fmt, hpp, hists);
506
507	c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
508	dim = c2c_fmt->dim;
509
510	if (dim->se) {
511	text = dim->header.line[line].text;
512	/* Use the last line from sort_entry if not defined. */
513	if (!text && (line == hpp_list->nr_header_lines - 1))
514	text = dim->se->se_header;
515	} else {
516	text = dim->header.line[line].text;
517
518	if (*span) {
519	(*span)--;
520	return 0;
521	} else {
522	*span = dim->header.line[line].span;
523	}
524	}
525
526	if (text == NULL)
527	text = "";
528
529	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, text);
530	}
531
532	#define HEX_STR(__s, __v) \
533	({ \
534	scnprintf(__s, sizeof(__s), "0x%" PRIx64, __v); \
535	__s; \
536	})
537
538	static int64_t
539	dcacheline_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
540	struct hist_entry *left, struct hist_entry *right)
541	{
542	return sort__dcacheline_cmp(left, right);
543	}
544
545	static int dcacheline_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
546	struct hist_entry *he)
547	{
548	uint64_t addr = 0;
549	int width = c2c_width(fmt, hpp, hists: he->hists);
550	char buf[20];
551
552	if (he->mem_info)
553	addr = cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl);
554
555	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, HEX_STR(buf, addr));
556	}
557
558	static int
559	dcacheline_node_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
560	struct hist_entry *he)
561	{
562	struct c2c_hist_entry *c2c_he;
563	int width = c2c_width(fmt, hpp, hists: he->hists);
564
565	c2c_he = container_of(he, struct c2c_hist_entry, he);
566	if (WARN_ON_ONCE(!c2c_he->nodestr))
567	return 0;
568
569	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, c2c_he->nodestr);
570	}
571
572	static int
573	dcacheline_node_count(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
574	struct hist_entry *he)
575	{
576	struct c2c_hist_entry *c2c_he;
577	int width = c2c_width(fmt, hpp, hists: he->hists);
578
579	c2c_he = container_of(he, struct c2c_hist_entry, he);
580	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*lu", width, c2c_he->paddr_cnt);
581	}
582
583	static int offset_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
584	struct hist_entry *he)
585	{
586	uint64_t addr = 0;
587	int width = c2c_width(fmt, hpp, hists: he->hists);
588	char buf[20];
589
590	if (he->mem_info)
591	addr = cl_offset(mem_info__daddr(he->mem_info)->al_addr, chk_double_cl);
592
593	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, HEX_STR(buf, addr));
594	}
595
596	static int64_t
597	offset_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
598	struct hist_entry *left, struct hist_entry *right)
599	{
600	uint64_t l = 0, r = 0;
601
602	if (left->mem_info)
603	l = cl_offset(mem_info__daddr(left->mem_info)->addr, chk_double_cl);
604
605	if (right->mem_info)
606	r = cl_offset(mem_info__daddr(right->mem_info)->addr, chk_double_cl);
607
608	return (int64_t)(r - l);
609	}
610
611	static int
612	iaddr_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
613	struct hist_entry *he)
614	{
615	uint64_t addr = 0;
616	int width = c2c_width(fmt, hpp, hists: he->hists);
617	char buf[20];
618
619	if (he->mem_info)
620	addr = mem_info__iaddr(he->mem_info)->addr;
621
622	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, HEX_STR(buf, addr));
623	}
624
625	static int64_t
626	iaddr_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
627	struct hist_entry *left, struct hist_entry *right)
628	{
629	return sort__iaddr_cmp(left, right);
630	}
631
632	static int
633	tot_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
634	struct hist_entry *he)
635	{
636	struct c2c_hist_entry *c2c_he;
637	int width = c2c_width(fmt, hpp, hists: he->hists);
638	unsigned int tot_hitm;
639
640	c2c_he = container_of(he, struct c2c_hist_entry, he);
641	tot_hitm = c2c_he->stats.lcl_hitm + c2c_he->stats.rmt_hitm;
642
643	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*u", width, tot_hitm);
644	}
645
646	static int64_t
647	tot_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
648	struct hist_entry *left, struct hist_entry *right)
649	{
650	struct c2c_hist_entry *c2c_left;
651	struct c2c_hist_entry *c2c_right;
652	uint64_t tot_hitm_left;
653	uint64_t tot_hitm_right;
654
655	c2c_left = container_of(left, struct c2c_hist_entry, he);
656	c2c_right = container_of(right, struct c2c_hist_entry, he);
657
658	tot_hitm_left = c2c_left->stats.lcl_hitm + c2c_left->stats.rmt_hitm;
659	tot_hitm_right = c2c_right->stats.lcl_hitm + c2c_right->stats.rmt_hitm;
660
661	return tot_hitm_left - tot_hitm_right;
662	}
663
664	#define STAT_FN_ENTRY(__f) \
665	static int \
666	__f ## _entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, \
667	struct hist_entry *he) \
668	{ \
669	struct c2c_hist_entry *c2c_he; \
670	int width = c2c_width(fmt, hpp, he->hists); \
671	\
672	c2c_he = container_of(he, struct c2c_hist_entry, he); \
673	return scnprintf(hpp->buf, hpp->size, "%*u", width, \
674	c2c_he->stats.__f); \
675	}
676
677	#define STAT_FN_CMP(__f) \
678	static int64_t \
679	__f ## _cmp(struct perf_hpp_fmt *fmt __maybe_unused, \
680	struct hist_entry *left, struct hist_entry *right) \
681	{ \
682	struct c2c_hist_entry *c2c_left, *c2c_right; \
683	\
684	c2c_left = container_of(left, struct c2c_hist_entry, he); \
685	c2c_right = container_of(right, struct c2c_hist_entry, he); \
686	return (uint64_t) c2c_left->stats.__f - \
687	(uint64_t) c2c_right->stats.__f; \
688	}
689
690	#define STAT_FN(__f) \
691	STAT_FN_ENTRY(__f) \
692	STAT_FN_CMP(__f)
693
694	STAT_FN(rmt_hitm)
695	STAT_FN(lcl_hitm)
696	STAT_FN(rmt_peer)
697	STAT_FN(lcl_peer)
698	STAT_FN(tot_peer)
699	STAT_FN(store)
700	STAT_FN(st_l1hit)
701	STAT_FN(st_l1miss)
702	STAT_FN(st_na)
703	STAT_FN(ld_fbhit)
704	STAT_FN(ld_l1hit)
705	STAT_FN(ld_l2hit)
706	STAT_FN(ld_llchit)
707	STAT_FN(rmt_hit)
708
709	static uint64_t get_load_llc_misses(struct c2c_stats *stats)
710	{
711	return stats->lcl_dram +
712	stats->rmt_dram +
713	stats->rmt_hitm +
714	stats->rmt_hit;
715	}
716
717	static uint64_t get_load_cache_hits(struct c2c_stats *stats)
718	{
719	return stats->ld_fbhit +
720	stats->ld_l1hit +
721	stats->ld_l2hit +
722	stats->ld_llchit +
723	stats->lcl_hitm;
724	}
725
726	static uint64_t get_stores(struct c2c_stats *stats)
727	{
728	return stats->st_l1hit +
729	stats->st_l1miss +
730	stats->st_na;
731	}
732
733	static uint64_t total_records(struct c2c_stats *stats)
734	{
735	return get_load_llc_misses(stats) +
736	get_load_cache_hits(stats) +
737	get_stores(stats);
738	}
739
740	static int
741	tot_recs_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
742	struct hist_entry *he)
743	{
744	struct c2c_hist_entry *c2c_he;
745	int width = c2c_width(fmt, hpp, hists: he->hists);
746	uint64_t tot_recs;
747
748	c2c_he = container_of(he, struct c2c_hist_entry, he);
749	tot_recs = total_records(stats: &c2c_he->stats);
750
751	return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, tot_recs);
752	}
753
754	static int64_t
755	tot_recs_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
756	struct hist_entry *left, struct hist_entry *right)
757	{
758	struct c2c_hist_entry *c2c_left;
759	struct c2c_hist_entry *c2c_right;
760	uint64_t tot_recs_left;
761	uint64_t tot_recs_right;
762
763	c2c_left = container_of(left, struct c2c_hist_entry, he);
764	c2c_right = container_of(right, struct c2c_hist_entry, he);
765
766	tot_recs_left = total_records(stats: &c2c_left->stats);
767	tot_recs_right = total_records(stats: &c2c_right->stats);
768
769	return tot_recs_left - tot_recs_right;
770	}
771
772	static uint64_t total_loads(struct c2c_stats *stats)
773	{
774	return get_load_llc_misses(stats) +
775	get_load_cache_hits(stats);
776	}
777
778	static int
779	tot_loads_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
780	struct hist_entry *he)
781	{
782	struct c2c_hist_entry *c2c_he;
783	int width = c2c_width(fmt, hpp, hists: he->hists);
784	uint64_t tot_recs;
785
786	c2c_he = container_of(he, struct c2c_hist_entry, he);
787	tot_recs = total_loads(stats: &c2c_he->stats);
788
789	return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, tot_recs);
790	}
791
792	static int64_t
793	tot_loads_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
794	struct hist_entry *left, struct hist_entry *right)
795	{
796	struct c2c_hist_entry *c2c_left;
797	struct c2c_hist_entry *c2c_right;
798	uint64_t tot_recs_left;
799	uint64_t tot_recs_right;
800
801	c2c_left = container_of(left, struct c2c_hist_entry, he);
802	c2c_right = container_of(right, struct c2c_hist_entry, he);
803
804	tot_recs_left = total_loads(stats: &c2c_left->stats);
805	tot_recs_right = total_loads(stats: &c2c_right->stats);
806
807	return tot_recs_left - tot_recs_right;
808	}
809
810	typedef double (get_percent_cb)(struct c2c_hist_entry *);
811
812	static int
813	percent_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
814	struct hist_entry *he, get_percent_cb get_percent)
815	{
816	struct c2c_hist_entry *c2c_he;
817	int width = c2c_width(fmt, hpp, hists: he->hists);
818	double per;
819
820	c2c_he = container_of(he, struct c2c_hist_entry, he);
821	per = get_percent(c2c_he);
822
823	#ifdef HAVE_SLANG_SUPPORT
824	if (use_browser)
825	return __hpp__slsmg_color_printf(hpp, "%*.2f%%", width - 1, per);
826	#endif
827	return hpp_color_scnprintf(hpp, "%*.2f%%", width - 1, per);
828	}
829
830	static double percent_costly_snoop(struct c2c_hist_entry *c2c_he)
831	{
832	struct c2c_hists *hists;
833	struct c2c_stats *stats;
834	struct c2c_stats *total;
835	int tot = 0, st = 0;
836	double p;
837
838	hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);
839	stats = &c2c_he->stats;
840	total = &hists->stats;
841
842	switch (c2c.display) {
843	case DISPLAY_RMT_HITM:
844	st = stats->rmt_hitm;
845	tot = total->rmt_hitm;
846	break;
847	case DISPLAY_LCL_HITM:
848	st = stats->lcl_hitm;
849	tot = total->lcl_hitm;
850	break;
851	case DISPLAY_TOT_HITM:
852	st = stats->tot_hitm;
853	tot = total->tot_hitm;
854	break;
855	case DISPLAY_SNP_PEER:
856	st = stats->tot_peer;
857	tot = total->tot_peer;
858	break;
859	default:
860	break;
861	}
862
863	p = tot ? (double) st / tot : 0;
864
865	return 100 * p;
866	}
867
868	#define PERC_STR(__s, __v) \
869	({ \
870	scnprintf(__s, sizeof(__s), "%.2F%%", __v); \
871	__s; \
872	})
873
874	static int
875	percent_costly_snoop_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
876	struct hist_entry *he)
877	{
878	struct c2c_hist_entry *c2c_he;
879	int width = c2c_width(fmt, hpp, hists: he->hists);
880	char buf[10];
881	double per;
882
883	c2c_he = container_of(he, struct c2c_hist_entry, he);
884	per = percent_costly_snoop(c2c_he);
885	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, PERC_STR(buf, per));
886	}
887
888	static int
889	percent_costly_snoop_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
890	struct hist_entry *he)
891	{
892	return percent_color(fmt, hpp, he, get_percent: percent_costly_snoop);
893	}
894
895	static int64_t
896	percent_costly_snoop_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
897	struct hist_entry *left, struct hist_entry *right)
898	{
899	struct c2c_hist_entry *c2c_left;
900	struct c2c_hist_entry *c2c_right;
901	double per_left;
902	double per_right;
903
904	c2c_left = container_of(left, struct c2c_hist_entry, he);
905	c2c_right = container_of(right, struct c2c_hist_entry, he);
906
907	per_left = percent_costly_snoop(c2c_he: c2c_left);
908	per_right = percent_costly_snoop(c2c_he: c2c_right);
909
910	return per_left - per_right;
911	}
912
913	static struct c2c_stats *he_stats(struct hist_entry *he)
914	{
915	struct c2c_hist_entry *c2c_he;
916
917	c2c_he = container_of(he, struct c2c_hist_entry, he);
918	return &c2c_he->stats;
919	}
920
921	static struct c2c_stats *total_stats(struct hist_entry *he)
922	{
923	struct c2c_hists *hists;
924
925	hists = container_of(he->hists, struct c2c_hists, hists);
926	return &hists->stats;
927	}
928
929	static double percent(u32 st, u32 tot)
930	{
931	return tot ? 100. * (double) st / (double) tot : 0;
932	}
933
934	#define PERCENT(__h, __f) percent(he_stats(__h)->__f, total_stats(__h)->__f)
935
936	#define PERCENT_FN(__f) \
937	static double percent_ ## __f(struct c2c_hist_entry *c2c_he) \
938	{ \
939	struct c2c_hists *hists; \
940	\
941	hists = container_of(c2c_he->he.hists, struct c2c_hists, hists); \
942	return percent(c2c_he->stats.__f, hists->stats.__f); \
943	}
944
945	PERCENT_FN(rmt_hitm)
946	PERCENT_FN(lcl_hitm)
947	PERCENT_FN(rmt_peer)
948	PERCENT_FN(lcl_peer)
949	PERCENT_FN(st_l1hit)
950	PERCENT_FN(st_l1miss)
951	PERCENT_FN(st_na)
952
953	static int
954	percent_rmt_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
955	struct hist_entry *he)
956	{
957	int width = c2c_width(fmt, hpp, hists: he->hists);
958	double per = PERCENT(he, rmt_hitm);
959	char buf[10];
960
961	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, PERC_STR(buf, per));
962	}
963
964	static int
965	percent_rmt_hitm_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
966	struct hist_entry *he)
967	{
968	return percent_color(fmt, hpp, he, get_percent: percent_rmt_hitm);
969	}
970
971	static int64_t
972	percent_rmt_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
973	struct hist_entry *left, struct hist_entry *right)
974	{
975	double per_left;
976	double per_right;
977
978	per_left = PERCENT(left, rmt_hitm);
979	per_right = PERCENT(right, rmt_hitm);
980
981	return per_left - per_right;
982	}
983
984	static int
985	percent_lcl_hitm_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
986	struct hist_entry *he)
987	{
988	int width = c2c_width(fmt, hpp, hists: he->hists);
989	double per = PERCENT(he, lcl_hitm);
990	char buf[10];
991
992	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, PERC_STR(buf, per));
993	}
994
995	static int
996	percent_lcl_hitm_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
997	struct hist_entry *he)
998	{
999	return percent_color(fmt, hpp, he, get_percent: percent_lcl_hitm);
1000	}
1001
1002	static int64_t
1003	percent_lcl_hitm_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1004	struct hist_entry *left, struct hist_entry *right)
1005	{
1006	double per_left;
1007	double per_right;
1008
1009	per_left = PERCENT(left, lcl_hitm);
1010	per_right = PERCENT(right, lcl_hitm);
1011
1012	return per_left - per_right;
1013	}
1014
1015	static int
1016	percent_lcl_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1017	struct hist_entry *he)
1018	{
1019	int width = c2c_width(fmt, hpp, hists: he->hists);
1020	double per = PERCENT(he, lcl_peer);
1021	char buf[10];
1022
1023	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, PERC_STR(buf, per));
1024	}
1025
1026	static int
1027	percent_lcl_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1028	struct hist_entry *he)
1029	{
1030	return percent_color(fmt, hpp, he, get_percent: percent_lcl_peer);
1031	}
1032
1033	static int64_t
1034	percent_lcl_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1035	struct hist_entry *left, struct hist_entry *right)
1036	{
1037	double per_left;
1038	double per_right;
1039
1040	per_left = PERCENT(left, lcl_peer);
1041	per_right = PERCENT(right, lcl_peer);
1042
1043	return per_left - per_right;
1044	}
1045
1046	static int
1047	percent_rmt_peer_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1048	struct hist_entry *he)
1049	{
1050	int width = c2c_width(fmt, hpp, hists: he->hists);
1051	double per = PERCENT(he, rmt_peer);
1052	char buf[10];
1053
1054	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, PERC_STR(buf, per));
1055	}
1056
1057	static int
1058	percent_rmt_peer_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1059	struct hist_entry *he)
1060	{
1061	return percent_color(fmt, hpp, he, get_percent: percent_rmt_peer);
1062	}
1063
1064	static int64_t
1065	percent_rmt_peer_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1066	struct hist_entry *left, struct hist_entry *right)
1067	{
1068	double per_left;
1069	double per_right;
1070
1071	per_left = PERCENT(left, rmt_peer);
1072	per_right = PERCENT(right, rmt_peer);
1073
1074	return per_left - per_right;
1075	}
1076
1077	static int
1078	percent_stores_l1hit_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1079	struct hist_entry *he)
1080	{
1081	int width = c2c_width(fmt, hpp, hists: he->hists);
1082	double per = PERCENT(he, st_l1hit);
1083	char buf[10];
1084
1085	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, PERC_STR(buf, per));
1086	}
1087
1088	static int
1089	percent_stores_l1hit_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1090	struct hist_entry *he)
1091	{
1092	return percent_color(fmt, hpp, he, get_percent: percent_st_l1hit);
1093	}
1094
1095	static int64_t
1096	percent_stores_l1hit_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1097	struct hist_entry *left, struct hist_entry *right)
1098	{
1099	double per_left;
1100	double per_right;
1101
1102	per_left = PERCENT(left, st_l1hit);
1103	per_right = PERCENT(right, st_l1hit);
1104
1105	return per_left - per_right;
1106	}
1107
1108	static int
1109	percent_stores_l1miss_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1110	struct hist_entry *he)
1111	{
1112	int width = c2c_width(fmt, hpp, hists: he->hists);
1113	double per = PERCENT(he, st_l1miss);
1114	char buf[10];
1115
1116	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, PERC_STR(buf, per));
1117	}
1118
1119	static int
1120	percent_stores_l1miss_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1121	struct hist_entry *he)
1122	{
1123	return percent_color(fmt, hpp, he, get_percent: percent_st_l1miss);
1124	}
1125
1126	static int64_t
1127	percent_stores_l1miss_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1128	struct hist_entry *left, struct hist_entry *right)
1129	{
1130	double per_left;
1131	double per_right;
1132
1133	per_left = PERCENT(left, st_l1miss);
1134	per_right = PERCENT(right, st_l1miss);
1135
1136	return per_left - per_right;
1137	}
1138
1139	static int
1140	percent_stores_na_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1141	struct hist_entry *he)
1142	{
1143	int width = c2c_width(fmt, hpp, hists: he->hists);
1144	double per = PERCENT(he, st_na);
1145	char buf[10];
1146
1147	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, PERC_STR(buf, per));
1148	}
1149
1150	static int
1151	percent_stores_na_color(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1152	struct hist_entry *he)
1153	{
1154	return percent_color(fmt, hpp, he, get_percent: percent_st_na);
1155	}
1156
1157	static int64_t
1158	percent_stores_na_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1159	struct hist_entry *left, struct hist_entry *right)
1160	{
1161	double per_left;
1162	double per_right;
1163
1164	per_left = PERCENT(left, st_na);
1165	per_right = PERCENT(right, st_na);
1166
1167	return per_left - per_right;
1168	}
1169
1170	STAT_FN(lcl_dram)
1171	STAT_FN(rmt_dram)
1172
1173	static int
1174	pid_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1175	struct hist_entry *he)
1176	{
1177	int width = c2c_width(fmt, hpp, hists: he->hists);
1178
1179	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*d", width, thread__pid(he->thread));
1180	}
1181
1182	static int64_t
1183	pid_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1184	struct hist_entry *left, struct hist_entry *right)
1185	{
1186	return thread__pid(left->thread) - thread__pid(right->thread);
1187	}
1188
1189	static int64_t
1190	empty_cmp(struct perf_hpp_fmt *fmt __maybe_unused,
1191	struct hist_entry *left __maybe_unused,
1192	struct hist_entry *right __maybe_unused)
1193	{
1194	return 0;
1195	}
1196
1197	static int display_metrics(struct perf_hpp *hpp, u32 val, u32 sum)
1198	{
1199	int ret;
1200
1201	if (sum != 0)
1202	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%5.1f%% ",
1203	percent(st: val, tot: sum));
1204	else
1205	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%6s ", "n/a");
1206
1207	return ret;
1208	}
1209
1210	static int
1211	node_entry(struct perf_hpp_fmt *fmt __maybe_unused, struct perf_hpp *hpp,
1212	struct hist_entry *he)
1213	{
1214	struct c2c_hist_entry *c2c_he;
1215	bool first = true;
1216	int node;
1217	int ret = 0;
1218
1219	c2c_he = container_of(he, struct c2c_hist_entry, he);
1220
1221	for (node = 0; node < c2c.nodes_cnt; node++) {
1222	DECLARE_BITMAP(set, c2c.cpus_cnt);
1223
1224	bitmap_zero(dst: set, nbits: c2c.cpus_cnt);
1225	bitmap_and(dst: set, src1: c2c_he->cpuset, src2: c2c.nodes[node], nbits: c2c.cpus_cnt);
1226
1227	if (bitmap_empty(src: set, nbits: c2c.cpus_cnt)) {
1228	if (c2c.node_info == 1) {
1229	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%21s", " ");
1230	advance_hpp(hpp, ret);
1231	}
1232	continue;
1233	}
1234
1235	if (!first) {
1236	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: " ");
1237	advance_hpp(hpp, ret);
1238	}
1239
1240	switch (c2c.node_info) {
1241	case 0:
1242	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%2d", node);
1243	advance_hpp(hpp, ret);
1244	break;
1245	case 1:
1246	{
1247	int num = bitmap_weight(src: set, nbits: c2c.cpus_cnt);
1248	struct c2c_stats *stats = &c2c_he->node_stats[node];
1249
1250	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%2d{%2d ", node, num);
1251	advance_hpp(hpp, ret);
1252
1253	switch (c2c.display) {
1254	case DISPLAY_RMT_HITM:
1255	ret = display_metrics(hpp, val: stats->rmt_hitm,
1256	sum: c2c_he->stats.rmt_hitm);
1257	break;
1258	case DISPLAY_LCL_HITM:
1259	ret = display_metrics(hpp, val: stats->lcl_hitm,
1260	sum: c2c_he->stats.lcl_hitm);
1261	break;
1262	case DISPLAY_TOT_HITM:
1263	ret = display_metrics(hpp, val: stats->tot_hitm,
1264	sum: c2c_he->stats.tot_hitm);
1265	break;
1266	case DISPLAY_SNP_PEER:
1267	ret = display_metrics(hpp, val: stats->tot_peer,
1268	sum: c2c_he->stats.tot_peer);
1269	break;
1270	default:
1271	break;
1272	}
1273
1274	advance_hpp(hpp, ret);
1275
1276	if (c2c_he->stats.store > 0) {
1277	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%5.1f%%}",
1278	percent(st: stats->store, tot: c2c_he->stats.store));
1279	} else {
1280	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%6s}", "n/a");
1281	}
1282
1283	advance_hpp(hpp, ret);
1284	break;
1285	}
1286	case 2:
1287	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%2d{", node);
1288	advance_hpp(hpp, ret);
1289
1290	ret = bitmap_scnprintf(set, c2c.cpus_cnt, hpp->buf, hpp->size);
1291	advance_hpp(hpp, ret);
1292
1293	ret = scnprintf(buf: hpp->buf, size: hpp->size, fmt: "}");
1294	advance_hpp(hpp, ret);
1295	break;
1296	default:
1297	break;
1298	}
1299
1300	first = false;
1301	}
1302
1303	return 0;
1304	}
1305
1306	static int
1307	mean_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1308	struct hist_entry *he, double mean)
1309	{
1310	int width = c2c_width(fmt, hpp, hists: he->hists);
1311	char buf[10];
1312
1313	scnprintf(buf, size: 10, fmt: "%6.0f", mean);
1314	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, buf);
1315	}
1316
1317	#define MEAN_ENTRY(__func, __val) \
1318	static int \
1319	__func(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, struct hist_entry *he) \
1320	{ \
1321	struct c2c_hist_entry *c2c_he; \
1322	c2c_he = container_of(he, struct c2c_hist_entry, he); \
1323	return mean_entry(fmt, hpp, he, avg_stats(&c2c_he->cstats.__val)); \
1324	}
1325
1326	MEAN_ENTRY(mean_rmt_entry, rmt_hitm);
1327	MEAN_ENTRY(mean_lcl_entry, lcl_hitm);
1328	MEAN_ENTRY(mean_load_entry, load);
1329	MEAN_ENTRY(mean_rmt_peer_entry, rmt_peer);
1330	MEAN_ENTRY(mean_lcl_peer_entry, lcl_peer);
1331
1332	static int
1333	cpucnt_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1334	struct hist_entry *he)
1335	{
1336	struct c2c_hist_entry *c2c_he;
1337	int width = c2c_width(fmt, hpp, hists: he->hists);
1338	char buf[10];
1339
1340	c2c_he = container_of(he, struct c2c_hist_entry, he);
1341
1342	scnprintf(buf, size: 10, fmt: "%d", bitmap_weight(src: c2c_he->cpuset, nbits: c2c.cpus_cnt));
1343	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, buf);
1344	}
1345
1346	static int
1347	cl_idx_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1348	struct hist_entry *he)
1349	{
1350	struct c2c_hist_entry *c2c_he;
1351	int width = c2c_width(fmt, hpp, hists: he->hists);
1352	char buf[10];
1353
1354	c2c_he = container_of(he, struct c2c_hist_entry, he);
1355
1356	scnprintf(buf, size: 10, fmt: "%u", c2c_he->cacheline_idx);
1357	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, buf);
1358	}
1359
1360	static int
1361	cl_idx_empty_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1362	struct hist_entry *he)
1363	{
1364	int width = c2c_width(fmt, hpp, hists: he->hists);
1365
1366	return scnprintf(buf: hpp->buf, size: hpp->size, fmt: "%*s", width, "");
1367	}
1368
1369	#define HEADER_LOW(__h) \
1370	{ \
1371	.line[1] = { \
1372	.text = __h, \
1373	}, \
1374	}
1375
1376	#define HEADER_BOTH(__h0, __h1) \
1377	{ \
1378	.line[0] = { \
1379	.text = __h0, \
1380	}, \
1381	.line[1] = { \
1382	.text = __h1, \
1383	}, \
1384	}
1385
1386	#define HEADER_SPAN(__h0, __h1, __s) \
1387	{ \
1388	.line[0] = { \
1389	.text = __h0, \
1390	.span = __s, \
1391	}, \
1392	.line[1] = { \
1393	.text = __h1, \
1394	}, \
1395	}
1396
1397	#define HEADER_SPAN_LOW(__h) \
1398	{ \
1399	.line[1] = { \
1400	.text = __h, \
1401	}, \
1402	}
1403
1404	static struct c2c_dimension dim_dcacheline = {
1405	.header = HEADER_SPAN("--- Cacheline ----", "Address", 2),
1406	.name = "dcacheline",
1407	.cmp = dcacheline_cmp,
1408	.entry = dcacheline_entry,
1409	.width = 18,
1410	};
1411
1412	static struct c2c_dimension dim_dcacheline_node = {
1413	.header = HEADER_LOW("Node"),
1414	.name = "dcacheline_node",
1415	.cmp = empty_cmp,
1416	.entry = dcacheline_node_entry,
1417	.width = 4,
1418	};
1419
1420	static struct c2c_dimension dim_dcacheline_count = {
1421	.header = HEADER_LOW("PA cnt"),
1422	.name = "dcacheline_count",
1423	.cmp = empty_cmp,
1424	.entry = dcacheline_node_count,
1425	.width = 6,
1426	};
1427
1428	static struct c2c_header header_offset_tui = HEADER_SPAN("-----", "Off", 2);
1429
1430	static struct c2c_dimension dim_offset = {
1431	.header = HEADER_SPAN("--- Data address -", "Offset", 2),
1432	.name = "offset",
1433	.cmp = offset_cmp,
1434	.entry = offset_entry,
1435	.width = 18,
1436	};
1437
1438	static struct c2c_dimension dim_offset_node = {
1439	.header = HEADER_LOW("Node"),
1440	.name = "offset_node",
1441	.cmp = empty_cmp,
1442	.entry = dcacheline_node_entry,
1443	.width = 4,
1444	};
1445
1446	static struct c2c_dimension dim_iaddr = {
1447	.header = HEADER_LOW("Code address"),
1448	.name = "iaddr",
1449	.cmp = iaddr_cmp,
1450	.entry = iaddr_entry,
1451	.width = 18,
1452	};
1453
1454	static struct c2c_dimension dim_tot_hitm = {
1455	.header = HEADER_SPAN("------- Load Hitm -------", "Total", 2),
1456	.name = "tot_hitm",
1457	.cmp = tot_hitm_cmp,
1458	.entry = tot_hitm_entry,
1459	.width = 7,
1460	};
1461
1462	static struct c2c_dimension dim_lcl_hitm = {
1463	.header = HEADER_SPAN_LOW("LclHitm"),
1464	.name = "lcl_hitm",
1465	.cmp = lcl_hitm_cmp,
1466	.entry = lcl_hitm_entry,
1467	.width = 7,
1468	};
1469
1470	static struct c2c_dimension dim_rmt_hitm = {
1471	.header = HEADER_SPAN_LOW("RmtHitm"),
1472	.name = "rmt_hitm",
1473	.cmp = rmt_hitm_cmp,
1474	.entry = rmt_hitm_entry,
1475	.width = 7,
1476	};
1477
1478	static struct c2c_dimension dim_tot_peer = {
1479	.header = HEADER_SPAN("------- Load Peer -------", "Total", 2),
1480	.name = "tot_peer",
1481	.cmp = tot_peer_cmp,
1482	.entry = tot_peer_entry,
1483	.width = 7,
1484	};
1485
1486	static struct c2c_dimension dim_lcl_peer = {
1487	.header = HEADER_SPAN_LOW("Local"),
1488	.name = "lcl_peer",
1489	.cmp = lcl_peer_cmp,
1490	.entry = lcl_peer_entry,
1491	.width = 7,
1492	};
1493
1494	static struct c2c_dimension dim_rmt_peer = {
1495	.header = HEADER_SPAN_LOW("Remote"),
1496	.name = "rmt_peer",
1497	.cmp = rmt_peer_cmp,
1498	.entry = rmt_peer_entry,
1499	.width = 7,
1500	};
1501
1502	static struct c2c_dimension dim_cl_rmt_hitm = {
1503	.header = HEADER_SPAN("----- HITM -----", "Rmt", 1),
1504	.name = "cl_rmt_hitm",
1505	.cmp = rmt_hitm_cmp,
1506	.entry = rmt_hitm_entry,
1507	.width = 7,
1508	};
1509
1510	static struct c2c_dimension dim_cl_lcl_hitm = {
1511	.header = HEADER_SPAN_LOW("Lcl"),
1512	.name = "cl_lcl_hitm",
1513	.cmp = lcl_hitm_cmp,
1514	.entry = lcl_hitm_entry,
1515	.width = 7,
1516	};
1517
1518	static struct c2c_dimension dim_cl_rmt_peer = {
1519	.header = HEADER_SPAN("----- Peer -----", "Rmt", 1),
1520	.name = "cl_rmt_peer",
1521	.cmp = rmt_peer_cmp,
1522	.entry = rmt_peer_entry,
1523	.width = 7,
1524	};
1525
1526	static struct c2c_dimension dim_cl_lcl_peer = {
1527	.header = HEADER_SPAN_LOW("Lcl"),
1528	.name = "cl_lcl_peer",
1529	.cmp = lcl_peer_cmp,
1530	.entry = lcl_peer_entry,
1531	.width = 7,
1532	};
1533
1534	static struct c2c_dimension dim_tot_stores = {
1535	.header = HEADER_BOTH("Total", "Stores"),
1536	.name = "tot_stores",
1537	.cmp = store_cmp,
1538	.entry = store_entry,
1539	.width = 7,
1540	};
1541
1542	static struct c2c_dimension dim_stores_l1hit = {
1543	.header = HEADER_SPAN("--------- Stores --------", "L1Hit", 2),
1544	.name = "stores_l1hit",
1545	.cmp = st_l1hit_cmp,
1546	.entry = st_l1hit_entry,
1547	.width = 7,
1548	};
1549
1550	static struct c2c_dimension dim_stores_l1miss = {
1551	.header = HEADER_SPAN_LOW("L1Miss"),
1552	.name = "stores_l1miss",
1553	.cmp = st_l1miss_cmp,
1554	.entry = st_l1miss_entry,
1555	.width = 7,
1556	};
1557
1558	static struct c2c_dimension dim_stores_na = {
1559	.header = HEADER_SPAN_LOW("N/A"),
1560	.name = "stores_na",
1561	.cmp = st_na_cmp,
1562	.entry = st_na_entry,
1563	.width = 7,
1564	};
1565
1566	static struct c2c_dimension dim_cl_stores_l1hit = {
1567	.header = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2),
1568	.name = "cl_stores_l1hit",
1569	.cmp = st_l1hit_cmp,
1570	.entry = st_l1hit_entry,
1571	.width = 7,
1572	};
1573
1574	static struct c2c_dimension dim_cl_stores_l1miss = {
1575	.header = HEADER_SPAN_LOW("L1 Miss"),
1576	.name = "cl_stores_l1miss",
1577	.cmp = st_l1miss_cmp,
1578	.entry = st_l1miss_entry,
1579	.width = 7,
1580	};
1581
1582	static struct c2c_dimension dim_cl_stores_na = {
1583	.header = HEADER_SPAN_LOW("N/A"),
1584	.name = "cl_stores_na",
1585	.cmp = st_na_cmp,
1586	.entry = st_na_entry,
1587	.width = 7,
1588	};
1589
1590	static struct c2c_dimension dim_ld_fbhit = {
1591	.header = HEADER_SPAN("----- Core Load Hit -----", "FB", 2),
1592	.name = "ld_fbhit",
1593	.cmp = ld_fbhit_cmp,
1594	.entry = ld_fbhit_entry,
1595	.width = 7,
1596	};
1597
1598	static struct c2c_dimension dim_ld_l1hit = {
1599	.header = HEADER_SPAN_LOW("L1"),
1600	.name = "ld_l1hit",
1601	.cmp = ld_l1hit_cmp,
1602	.entry = ld_l1hit_entry,
1603	.width = 7,
1604	};
1605
1606	static struct c2c_dimension dim_ld_l2hit = {
1607	.header = HEADER_SPAN_LOW("L2"),
1608	.name = "ld_l2hit",
1609	.cmp = ld_l2hit_cmp,
1610	.entry = ld_l2hit_entry,
1611	.width = 7,
1612	};
1613
1614	static struct c2c_dimension dim_ld_llchit = {
1615	.header = HEADER_SPAN("- LLC Load Hit --", "LclHit", 1),
1616	.name = "ld_lclhit",
1617	.cmp = ld_llchit_cmp,
1618	.entry = ld_llchit_entry,
1619	.width = 8,
1620	};
1621
1622	static struct c2c_dimension dim_ld_rmthit = {
1623	.header = HEADER_SPAN("- RMT Load Hit --", "RmtHit", 1),
1624	.name = "ld_rmthit",
1625	.cmp = rmt_hit_cmp,
1626	.entry = rmt_hit_entry,
1627	.width = 8,
1628	};
1629
1630	static struct c2c_dimension dim_tot_recs = {
1631	.header = HEADER_BOTH("Total", "records"),
1632	.name = "tot_recs",
1633	.cmp = tot_recs_cmp,
1634	.entry = tot_recs_entry,
1635	.width = 7,
1636	};
1637
1638	static struct c2c_dimension dim_tot_loads = {
1639	.header = HEADER_BOTH("Total", "Loads"),
1640	.name = "tot_loads",
1641	.cmp = tot_loads_cmp,
1642	.entry = tot_loads_entry,
1643	.width = 7,
1644	};
1645
1646	static struct c2c_header percent_costly_snoop_header[] = {
1647	[DISPLAY_LCL_HITM] = HEADER_BOTH("Lcl", "Hitm"),
1648	[DISPLAY_RMT_HITM] = HEADER_BOTH("Rmt", "Hitm"),
1649	[DISPLAY_TOT_HITM] = HEADER_BOTH("Tot", "Hitm"),
1650	[DISPLAY_SNP_PEER] = HEADER_BOTH("Peer", "Snoop"),
1651	};
1652
1653	static struct c2c_dimension dim_percent_costly_snoop = {
1654	.name = "percent_costly_snoop",
1655	.cmp = percent_costly_snoop_cmp,
1656	.entry = percent_costly_snoop_entry,
1657	.color = percent_costly_snoop_color,
1658	.width = 7,
1659	};
1660
1661	static struct c2c_dimension dim_percent_rmt_hitm = {
1662	.header = HEADER_SPAN("----- HITM -----", "RmtHitm", 1),
1663	.name = "percent_rmt_hitm",
1664	.cmp = percent_rmt_hitm_cmp,
1665	.entry = percent_rmt_hitm_entry,
1666	.color = percent_rmt_hitm_color,
1667	.width = 7,
1668	};
1669
1670	static struct c2c_dimension dim_percent_lcl_hitm = {
1671	.header = HEADER_SPAN_LOW("LclHitm"),
1672	.name = "percent_lcl_hitm",
1673	.cmp = percent_lcl_hitm_cmp,
1674	.entry = percent_lcl_hitm_entry,
1675	.color = percent_lcl_hitm_color,
1676	.width = 7,
1677	};
1678
1679	static struct c2c_dimension dim_percent_rmt_peer = {
1680	.header = HEADER_SPAN("-- Peer Snoop --", "Rmt", 1),
1681	.name = "percent_rmt_peer",
1682	.cmp = percent_rmt_peer_cmp,
1683	.entry = percent_rmt_peer_entry,
1684	.color = percent_rmt_peer_color,
1685	.width = 7,
1686	};
1687
1688	static struct c2c_dimension dim_percent_lcl_peer = {
1689	.header = HEADER_SPAN_LOW("Lcl"),
1690	.name = "percent_lcl_peer",
1691	.cmp = percent_lcl_peer_cmp,
1692	.entry = percent_lcl_peer_entry,
1693	.color = percent_lcl_peer_color,
1694	.width = 7,
1695	};
1696
1697	static struct c2c_dimension dim_percent_stores_l1hit = {
1698	.header = HEADER_SPAN("------- Store Refs ------", "L1 Hit", 2),
1699	.name = "percent_stores_l1hit",
1700	.cmp = percent_stores_l1hit_cmp,
1701	.entry = percent_stores_l1hit_entry,
1702	.color = percent_stores_l1hit_color,
1703	.width = 7,
1704	};
1705
1706	static struct c2c_dimension dim_percent_stores_l1miss = {
1707	.header = HEADER_SPAN_LOW("L1 Miss"),
1708	.name = "percent_stores_l1miss",
1709	.cmp = percent_stores_l1miss_cmp,
1710	.entry = percent_stores_l1miss_entry,
1711	.color = percent_stores_l1miss_color,
1712	.width = 7,
1713	};
1714
1715	static struct c2c_dimension dim_percent_stores_na = {
1716	.header = HEADER_SPAN_LOW("N/A"),
1717	.name = "percent_stores_na",
1718	.cmp = percent_stores_na_cmp,
1719	.entry = percent_stores_na_entry,
1720	.color = percent_stores_na_color,
1721	.width = 7,
1722	};
1723
1724	static struct c2c_dimension dim_dram_lcl = {
1725	.header = HEADER_SPAN("--- Load Dram ----", "Lcl", 1),
1726	.name = "dram_lcl",
1727	.cmp = lcl_dram_cmp,
1728	.entry = lcl_dram_entry,
1729	.width = 8,
1730	};
1731
1732	static struct c2c_dimension dim_dram_rmt = {
1733	.header = HEADER_SPAN_LOW("Rmt"),
1734	.name = "dram_rmt",
1735	.cmp = rmt_dram_cmp,
1736	.entry = rmt_dram_entry,
1737	.width = 8,
1738	};
1739
1740	static struct c2c_dimension dim_pid = {
1741	.header = HEADER_LOW("Pid"),
1742	.name = "pid",
1743	.cmp = pid_cmp,
1744	.entry = pid_entry,
1745	.width = 7,
1746	};
1747
1748	static struct c2c_dimension dim_tid = {
1749	.header = HEADER_LOW("Tid"),
1750	.name = "tid",
1751	.se = &sort_thread,
1752	};
1753
1754	static struct c2c_dimension dim_symbol = {
1755	.name = "symbol",
1756	.se = &sort_sym,
1757	};
1758
1759	static struct c2c_dimension dim_dso = {
1760	.header = HEADER_BOTH("Shared", "Object"),
1761	.name = "dso",
1762	.se = &sort_dso,
1763	};
1764
1765	static struct c2c_dimension dim_node = {
1766	.name = "node",
1767	.cmp = empty_cmp,
1768	.entry = node_entry,
1769	.width = 4,
1770	};
1771
1772	static struct c2c_dimension dim_mean_rmt = {
1773	.header = HEADER_SPAN("---------- cycles ----------", "rmt hitm", 2),
1774	.name = "mean_rmt",
1775	.cmp = empty_cmp,
1776	.entry = mean_rmt_entry,
1777	.width = 8,
1778	};
1779
1780	static struct c2c_dimension dim_mean_lcl = {
1781	.header = HEADER_SPAN_LOW("lcl hitm"),
1782	.name = "mean_lcl",
1783	.cmp = empty_cmp,
1784	.entry = mean_lcl_entry,
1785	.width = 8,
1786	};
1787
1788	static struct c2c_dimension dim_mean_load = {
1789	.header = HEADER_SPAN_LOW("load"),
1790	.name = "mean_load",
1791	.cmp = empty_cmp,
1792	.entry = mean_load_entry,
1793	.width = 8,
1794	};
1795
1796	static struct c2c_dimension dim_mean_rmt_peer = {
1797	.header = HEADER_SPAN("---------- cycles ----------", "rmt peer", 2),
1798	.name = "mean_rmt_peer",
1799	.cmp = empty_cmp,
1800	.entry = mean_rmt_peer_entry,
1801	.width = 8,
1802	};
1803
1804	static struct c2c_dimension dim_mean_lcl_peer = {
1805	.header = HEADER_SPAN_LOW("lcl peer"),
1806	.name = "mean_lcl_peer",
1807	.cmp = empty_cmp,
1808	.entry = mean_lcl_peer_entry,
1809	.width = 8,
1810	};
1811
1812	static struct c2c_dimension dim_cpucnt = {
1813	.header = HEADER_BOTH("cpu", "cnt"),
1814	.name = "cpucnt",
1815	.cmp = empty_cmp,
1816	.entry = cpucnt_entry,
1817	.width = 8,
1818	};
1819
1820	static struct c2c_dimension dim_srcline = {
1821	.name = "cl_srcline",
1822	.se = &sort_srcline,
1823	};
1824
1825	static struct c2c_dimension dim_dcacheline_idx = {
1826	.header = HEADER_LOW("Index"),
1827	.name = "cl_idx",
1828	.cmp = empty_cmp,
1829	.entry = cl_idx_entry,
1830	.width = 5,
1831	};
1832
1833	static struct c2c_dimension dim_dcacheline_num = {
1834	.header = HEADER_LOW("Num"),
1835	.name = "cl_num",
1836	.cmp = empty_cmp,
1837	.entry = cl_idx_entry,
1838	.width = 5,
1839	};
1840
1841	static struct c2c_dimension dim_dcacheline_num_empty = {
1842	.header = HEADER_LOW("Num"),
1843	.name = "cl_num_empty",
1844	.cmp = empty_cmp,
1845	.entry = cl_idx_empty_entry,
1846	.width = 5,
1847	};
1848
1849	static struct c2c_dimension *dimensions[] = {
1850	&dim_dcacheline,
1851	&dim_dcacheline_node,
1852	&dim_dcacheline_count,
1853	&dim_offset,
1854	&dim_offset_node,
1855	&dim_iaddr,
1856	&dim_tot_hitm,
1857	&dim_lcl_hitm,
1858	&dim_rmt_hitm,
1859	&dim_tot_peer,
1860	&dim_lcl_peer,
1861	&dim_rmt_peer,
1862	&dim_cl_lcl_hitm,
1863	&dim_cl_rmt_hitm,
1864	&dim_cl_lcl_peer,
1865	&dim_cl_rmt_peer,
1866	&dim_tot_stores,
1867	&dim_stores_l1hit,
1868	&dim_stores_l1miss,
1869	&dim_stores_na,
1870	&dim_cl_stores_l1hit,
1871	&dim_cl_stores_l1miss,
1872	&dim_cl_stores_na,
1873	&dim_ld_fbhit,
1874	&dim_ld_l1hit,
1875	&dim_ld_l2hit,
1876	&dim_ld_llchit,
1877	&dim_ld_rmthit,
1878	&dim_tot_recs,
1879	&dim_tot_loads,
1880	&dim_percent_costly_snoop,
1881	&dim_percent_rmt_hitm,
1882	&dim_percent_lcl_hitm,
1883	&dim_percent_rmt_peer,
1884	&dim_percent_lcl_peer,
1885	&dim_percent_stores_l1hit,
1886	&dim_percent_stores_l1miss,
1887	&dim_percent_stores_na,
1888	&dim_dram_lcl,
1889	&dim_dram_rmt,
1890	&dim_pid,
1891	&dim_tid,
1892	&dim_symbol,
1893	&dim_dso,
1894	&dim_node,
1895	&dim_mean_rmt,
1896	&dim_mean_lcl,
1897	&dim_mean_rmt_peer,
1898	&dim_mean_lcl_peer,
1899	&dim_mean_load,
1900	&dim_cpucnt,
1901	&dim_srcline,
1902	&dim_dcacheline_idx,
1903	&dim_dcacheline_num,
1904	&dim_dcacheline_num_empty,
1905	NULL,
1906	};
1907
1908	static void fmt_free(struct perf_hpp_fmt *fmt)
1909	{
1910	struct c2c_fmt *c2c_fmt;
1911
1912	c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1913	free(c2c_fmt);
1914	}
1915
1916	static bool fmt_equal(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b)
1917	{
1918	struct c2c_fmt *c2c_a = container_of(a, struct c2c_fmt, fmt);
1919	struct c2c_fmt *c2c_b = container_of(b, struct c2c_fmt, fmt);
1920
1921	return c2c_a->dim == c2c_b->dim;
1922	}
1923
1924	static struct c2c_dimension *get_dimension(const char *name)
1925	{
1926	unsigned int i;
1927
1928	for (i = 0; dimensions[i]; i++) {
1929	struct c2c_dimension *dim = dimensions[i];
1930
1931	if (!strcmp(dim->name, name))
1932	return dim;
1933	}
1934
1935	return NULL;
1936	}
1937
1938	static int c2c_se_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
1939	struct hist_entry *he)
1940	{
1941	struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1942	struct c2c_dimension *dim = c2c_fmt->dim;
1943	size_t len = fmt->user_len;
1944
1945	if (!len) {
1946	len = hists__col_len(he->hists, dim->se->se_width_idx);
1947
1948	if (dim == &dim_symbol || dim == &dim_srcline)
1949	len = symbol_width(hists: he->hists, se: dim->se);
1950	}
1951
1952	return dim->se->se_snprintf(he, hpp->buf, hpp->size, len);
1953	}
1954
1955	static int64_t c2c_se_cmp(struct perf_hpp_fmt *fmt,
1956	struct hist_entry *a, struct hist_entry *b)
1957	{
1958	struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1959	struct c2c_dimension *dim = c2c_fmt->dim;
1960
1961	return dim->se->se_cmp(a, b);
1962	}
1963
1964	static int64_t c2c_se_collapse(struct perf_hpp_fmt *fmt,
1965	struct hist_entry *a, struct hist_entry *b)
1966	{
1967	struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt);
1968	struct c2c_dimension *dim = c2c_fmt->dim;
1969	int64_t (*collapse_fn)(struct hist_entry *, struct hist_entry *);
1970
1971	collapse_fn = dim->se->se_collapse ?: dim->se->se_cmp;
1972	return collapse_fn(a, b);
1973	}
1974
1975	static struct c2c_fmt *get_format(const char *name)
1976	{
1977	struct c2c_dimension *dim = get_dimension(name);
1978	struct c2c_fmt *c2c_fmt;
1979	struct perf_hpp_fmt *fmt;
1980
1981	if (!dim)
1982	return NULL;
1983
1984	c2c_fmt = zalloc(sizeof(*c2c_fmt));
1985	if (!c2c_fmt)
1986	return NULL;
1987
1988	c2c_fmt->dim = dim;
1989
1990	fmt = &c2c_fmt->fmt;
1991	INIT_LIST_HEAD(list: &fmt->list);
1992	INIT_LIST_HEAD(list: &fmt->sort_list);
1993
1994	fmt->cmp = dim->se ? c2c_se_cmp : dim->cmp;
1995	fmt->sort = dim->se ? c2c_se_cmp : dim->cmp;
1996	fmt->color = dim->se ? NULL : dim->color;
1997	fmt->entry = dim->se ? c2c_se_entry : dim->entry;
1998	fmt->header = c2c_header;
1999	fmt->width = c2c_width;
2000	fmt->collapse = dim->se ? c2c_se_collapse : dim->cmp;
2001	fmt->equal = fmt_equal;
2002	fmt->free = fmt_free;
2003
2004	return c2c_fmt;
2005	}
2006
2007	static int c2c_hists__init_output(struct perf_hpp_list *hpp_list, char *name,
2008	struct perf_env *env __maybe_unused)
2009	{
2010	struct c2c_fmt *c2c_fmt = get_format(name);
2011	int level = 0;
2012
2013	if (!c2c_fmt) {
2014	reset_dimensions();
2015	return output_field_add(hpp_list, name, &level);
2016	}
2017
2018	perf_hpp_list__column_register(hpp_list, &c2c_fmt->fmt);
2019	return 0;
2020	}
2021
2022	static int c2c_hists__init_sort(struct perf_hpp_list *hpp_list, char *name, struct perf_env *env)
2023	{
2024	struct c2c_fmt *c2c_fmt = get_format(name);
2025	struct c2c_dimension *dim;
2026
2027	if (!c2c_fmt) {
2028	reset_dimensions();
2029	return sort_dimension__add(hpp_list, name, /*evlist=*/NULL, env, /*level=*/0);
2030	}
2031
2032	dim = c2c_fmt->dim;
2033	if (dim == &dim_dso)
2034	hpp_list->dso = 1;
2035
2036	if (dim == &dim_symbol || dim == &dim_iaddr)
2037	hpp_list->sym = 1;
2038
2039	perf_hpp_list__register_sort_field(hpp_list, &c2c_fmt->fmt);
2040	return 0;
2041	}
2042
2043	#define PARSE_LIST(_list, _fn) \
2044	do { \
2045	char *tmp, *tok; \
2046	ret = 0; \
2047	\
2048	if (!_list) \
2049	break; \
2050	\
2051	for (tok = strtok_r((char *)_list, ", ", &tmp); \
2052	tok; tok = strtok_r(NULL, ", ", &tmp)) { \
2053	ret = _fn(hpp_list, tok, env); \
2054	if (ret == -EINVAL) { \
2055	pr_err("Invalid --fields key: `%s'", tok); \
2056	break; \
2057	} else if (ret == -ESRCH) { \
2058	pr_err("Unknown --fields key: `%s'", tok); \
2059	break; \
2060	} \
2061	} \
2062	} while (0)
2063
2064	static int hpp_list__parse(struct perf_hpp_list *hpp_list,
2065	const char *output_,
2066	const char *sort_,
2067	struct perf_env *env)
2068	{
2069	char *output = output_ ? strdup(output_) : NULL;
2070	char *sort = sort_ ? strdup(sort_) : NULL;
2071	int ret;
2072
2073	PARSE_LIST(output, c2c_hists__init_output);
2074	PARSE_LIST(sort, c2c_hists__init_sort);
2075
2076	/* copy sort keys to output fields */
2077	perf_hpp__setup_output_field(hpp_list);
2078
2079	/*
2080	* We don't need other sorting keys other than those
2081	* we already specified. It also really slows down
2082	* the processing a lot with big number of output
2083	* fields, so switching this off for c2c.
2084	*/
2085
2086	#if 0
2087	/* and then copy output fields to sort keys */
2088	perf_hpp__append_sort_keys(&hists->list);
2089	#endif
2090
2091	free(output);
2092	free(sort);
2093	return ret;
2094	}
2095
2096	static int c2c_hists__init(struct c2c_hists *hists,
2097	const char *sort,
2098	int nr_header_lines,
2099	struct perf_env *env)
2100	{
2101	__hists__init(&hists->hists, &hists->list);
2102
2103	/*
2104	* Initialize only with sort fields, we need to resort
2105	* later anyway, and that's where we add output fields
2106	* as well.
2107	*/
2108	perf_hpp_list__init(&hists->list);
2109
2110	/* Overload number of header lines.*/
2111	hists->list.nr_header_lines = nr_header_lines;
2112
2113	return hpp_list__parse(hpp_list: &hists->list, /*output=*/NULL, sort_: sort, env);
2114	}
2115
2116	static int c2c_hists__reinit(struct c2c_hists *c2c_hists,
2117	const char *output,
2118	const char *sort,
2119	struct perf_env *env)
2120	{
2121	perf_hpp__reset_output_field(&c2c_hists->list);
2122	return hpp_list__parse(hpp_list: &c2c_hists->list, output_: output, sort_: sort, env);
2123	}
2124
2125	#define DISPLAY_LINE_LIMIT 0.001
2126
2127	static u8 filter_display(u32 val, u32 sum)
2128	{
2129	if (sum == 0 || ((double)val / sum) < DISPLAY_LINE_LIMIT)
2130	return HIST_FILTER__C2C;
2131
2132	return 0;
2133	}
2134
2135	static bool he__display(struct hist_entry *he, struct c2c_stats *stats)
2136	{
2137	struct c2c_hist_entry *c2c_he;
2138
2139	if (c2c.show_all)
2140	return true;
2141
2142	c2c_he = container_of(he, struct c2c_hist_entry, he);
2143
2144	switch (c2c.display) {
2145	case DISPLAY_LCL_HITM:
2146	he->filtered = filter_display(val: c2c_he->stats.lcl_hitm,
2147	sum: stats->lcl_hitm);
2148	break;
2149	case DISPLAY_RMT_HITM:
2150	he->filtered = filter_display(val: c2c_he->stats.rmt_hitm,
2151	sum: stats->rmt_hitm);
2152	break;
2153	case DISPLAY_TOT_HITM:
2154	he->filtered = filter_display(val: c2c_he->stats.tot_hitm,
2155	sum: stats->tot_hitm);
2156	break;
2157	case DISPLAY_SNP_PEER:
2158	he->filtered = filter_display(val: c2c_he->stats.tot_peer,
2159	sum: stats->tot_peer);
2160	break;
2161	default:
2162	break;
2163	}
2164
2165	return he->filtered == 0;
2166	}
2167
2168	static inline bool is_valid_hist_entry(struct hist_entry *he)
2169	{
2170	struct c2c_hist_entry *c2c_he;
2171	bool has_record = false;
2172
2173	c2c_he = container_of(he, struct c2c_hist_entry, he);
2174
2175	/* It's a valid entry if contains stores */
2176	if (c2c_he->stats.store)
2177	return true;
2178
2179	switch (c2c.display) {
2180	case DISPLAY_LCL_HITM:
2181	has_record = !!c2c_he->stats.lcl_hitm;
2182	break;
2183	case DISPLAY_RMT_HITM:
2184	has_record = !!c2c_he->stats.rmt_hitm;
2185	break;
2186	case DISPLAY_TOT_HITM:
2187	has_record = !!c2c_he->stats.tot_hitm;
2188	break;
2189	case DISPLAY_SNP_PEER:
2190	has_record = !!c2c_he->stats.tot_peer;
2191	default:
2192	break;
2193	}
2194
2195	return has_record;
2196	}
2197
2198	static void set_node_width(struct c2c_hist_entry *c2c_he, int len)
2199	{
2200	struct c2c_dimension *dim;
2201
2202	dim = &c2c.hists == c2c_he->hists ?
2203	&dim_dcacheline_node : &dim_offset_node;
2204
2205	if (len > dim->width)
2206	dim->width = len;
2207	}
2208
2209	static int set_nodestr(struct c2c_hist_entry *c2c_he)
2210	{
2211	char buf[30];
2212	int len;
2213
2214	if (c2c_he->nodestr)
2215	return 0;
2216
2217	if (!bitmap_empty(src: c2c_he->nodeset, nbits: c2c.nodes_cnt)) {
2218	len = bitmap_scnprintf(c2c_he->nodeset, c2c.nodes_cnt,
2219	buf, sizeof(buf));
2220	} else {
2221	len = scnprintf(buf, size: sizeof(buf), fmt: "N/A");
2222	}
2223
2224	set_node_width(c2c_he, len);
2225	c2c_he->nodestr = strdup(buf);
2226	return c2c_he->nodestr ? 0 : -ENOMEM;
2227	}
2228
2229	static void calc_width(struct c2c_hist_entry *c2c_he)
2230	{
2231	struct c2c_hists *c2c_hists;
2232
2233	c2c_hists = container_of(c2c_he->he.hists, struct c2c_hists, hists);
2234	hists__calc_col_len(&c2c_hists->hists, &c2c_he->he);
2235	set_nodestr(c2c_he);
2236	}
2237
2238	static int filter_cb(struct hist_entry *he, void *arg __maybe_unused)
2239	{
2240	struct c2c_hist_entry *c2c_he;
2241
2242	c2c_he = container_of(he, struct c2c_hist_entry, he);
2243
2244	if (c2c.show_src && !he->srcline)
2245	he->srcline = hist_entry__srcline(he);
2246
2247	calc_width(c2c_he);
2248
2249	if (!is_valid_hist_entry(he))
2250	he->filtered = HIST_FILTER__C2C;
2251
2252	return 0;
2253	}
2254
2255	static int resort_cl_cb(struct hist_entry *he, void *arg)
2256	{
2257	struct perf_env *env = arg;
2258	struct c2c_hist_entry *c2c_he;
2259	struct c2c_hists *c2c_hists;
2260	bool display = he__display(he, stats: &c2c.shared_clines_stats);
2261
2262	c2c_he = container_of(he, struct c2c_hist_entry, he);
2263	c2c_hists = c2c_he->hists;
2264
2265	if (display && c2c_hists) {
2266	static unsigned int idx;
2267
2268	c2c_he->cacheline_idx = idx++;
2269	calc_width(c2c_he);
2270
2271	c2c_hists__reinit(c2c_hists, output: c2c.cl_output, sort: c2c.cl_resort, env);
2272
2273	hists__collapse_resort(&c2c_hists->hists, NULL);
2274	hists__output_resort_cb(&c2c_hists->hists, NULL, filter_cb);
2275	}
2276
2277	return 0;
2278	}
2279
2280	static struct c2c_header header_node_0 = HEADER_LOW("Node");
2281	static struct c2c_header header_node_1_hitms_stores =
2282	HEADER_LOW("Node{cpus %hitms %stores}");
2283	static struct c2c_header header_node_1_peers_stores =
2284	HEADER_LOW("Node{cpus %peers %stores}");
2285	static struct c2c_header header_node_2 = HEADER_LOW("Node{cpu list}");
2286
2287	static void setup_nodes_header(void)
2288	{
2289	switch (c2c.node_info) {
2290	case 0:
2291	dim_node.header = header_node_0;
2292	break;
2293	case 1:
2294	if (c2c.display == DISPLAY_SNP_PEER)
2295	dim_node.header = header_node_1_peers_stores;
2296	else
2297	dim_node.header = header_node_1_hitms_stores;
2298	break;
2299	case 2:
2300	dim_node.header = header_node_2;
2301	break;
2302	default:
2303	break;
2304	}
2305
2306	return;
2307	}
2308
2309	static int setup_nodes(struct perf_session *session)
2310	{
2311	struct numa_node *n;
2312	unsigned long **nodes;
2313	int node, idx;
2314	struct perf_cpu cpu;
2315	int *cpu2node;
2316	struct perf_env *env = perf_session__env(session);
2317
2318	if (c2c.node_info > 2)
2319	c2c.node_info = 2;
2320
2321	c2c.nodes_cnt = env->nr_numa_nodes;
2322	c2c.cpus_cnt = env->nr_cpus_avail;
2323
2324	n = env->numa_nodes;
2325	if (!n)
2326	return -EINVAL;
2327
2328	nodes = zalloc(sizeof(unsigned long *) * c2c.nodes_cnt);
2329	if (!nodes)
2330	return -ENOMEM;
2331
2332	c2c.nodes = nodes;
2333
2334	cpu2node = zalloc(sizeof(int) * c2c.cpus_cnt);
2335	if (!cpu2node)
2336	return -ENOMEM;
2337
2338	for (idx = 0; idx < c2c.cpus_cnt; idx++)
2339	cpu2node[idx] = -1;
2340
2341	c2c.cpu2node = cpu2node;
2342
2343	for (node = 0; node < c2c.nodes_cnt; node++) {
2344	struct perf_cpu_map *map = n[node].map;
2345	unsigned long *set;
2346
2347	set = bitmap_zalloc(c2c.cpus_cnt);
2348	if (!set)
2349	return -ENOMEM;
2350
2351	nodes[node] = set;
2352
2353	perf_cpu_map__for_each_cpu_skip_any(cpu, idx, map) {
2354	__set_bit(cpu.cpu, set);
2355
2356	if (WARN_ONCE(cpu2node[cpu.cpu] != -1, "node/cpu topology bug"))
2357	return -EINVAL;
2358
2359	cpu2node[cpu.cpu] = node;
2360	}
2361	}
2362
2363	setup_nodes_header();
2364	return 0;
2365	}
2366
2367	#define HAS_HITMS(__h) ((__h)->stats.lcl_hitm || (__h)->stats.rmt_hitm)
2368	#define HAS_PEER(__h) ((__h)->stats.lcl_peer || (__h)->stats.rmt_peer)
2369
2370	static int resort_shared_cl_cb(struct hist_entry *he, void *arg __maybe_unused)
2371	{
2372	struct c2c_hist_entry *c2c_he;
2373	c2c_he = container_of(he, struct c2c_hist_entry, he);
2374
2375	if (HAS_HITMS(c2c_he) || HAS_PEER(c2c_he)) {
2376	c2c.shared_clines++;
2377	c2c_add_stats(stats: &c2c.shared_clines_stats, add: &c2c_he->stats);
2378	}
2379
2380	return 0;
2381	}
2382
2383	static int hists__iterate_cb(struct hists *hists, hists__resort_cb_t cb, void *arg)
2384	{
2385	struct rb_node *next = rb_first_cached(&hists->entries);
2386	int ret = 0;
2387
2388	while (next) {
2389	struct hist_entry *he;
2390
2391	he = rb_entry(next, struct hist_entry, rb_node);
2392	ret = cb(he, arg);
2393	if (ret)
2394	break;
2395	next = rb_next(&he->rb_node);
2396	}
2397
2398	return ret;
2399	}
2400
2401	static void print_c2c__display_stats(FILE *out)
2402	{
2403	int llc_misses;
2404	struct c2c_stats *stats = &c2c.hists.stats;
2405
2406	llc_misses = get_load_llc_misses(stats);
2407
2408	fprintf(out, "=================================================\n");
2409	fprintf(out, " Trace Event Information \n");
2410	fprintf(out, "=================================================\n");
2411	fprintf(out, " Total records : %10d\n", stats->nr_entries);
2412	fprintf(out, " Locked Load/Store Operations : %10d\n", stats->locks);
2413	fprintf(out, " Load Operations : %10d\n", stats->load);
2414	fprintf(out, " Loads - uncacheable : %10d\n", stats->ld_uncache);
2415	fprintf(out, " Loads - IO : %10d\n", stats->ld_io);
2416	fprintf(out, " Loads - Miss : %10d\n", stats->ld_miss);
2417	fprintf(out, " Loads - no mapping : %10d\n", stats->ld_noadrs);
2418	fprintf(out, " Load Fill Buffer Hit : %10d\n", stats->ld_fbhit);
2419	fprintf(out, " Load L1D hit : %10d\n", stats->ld_l1hit);
2420	fprintf(out, " Load L2D hit : %10d\n", stats->ld_l2hit);
2421	fprintf(out, " Load LLC hit : %10d\n", stats->ld_llchit + stats->lcl_hitm);
2422	fprintf(out, " Load Local HITM : %10d\n", stats->lcl_hitm);
2423	fprintf(out, " Load Remote HITM : %10d\n", stats->rmt_hitm);
2424	fprintf(out, " Load Remote HIT : %10d\n", stats->rmt_hit);
2425	fprintf(out, " Load Local DRAM : %10d\n", stats->lcl_dram);
2426	fprintf(out, " Load Remote DRAM : %10d\n", stats->rmt_dram);
2427	fprintf(out, " Load MESI State Exclusive : %10d\n", stats->ld_excl);
2428	fprintf(out, " Load MESI State Shared : %10d\n", stats->ld_shared);
2429	fprintf(out, " Load LLC Misses : %10d\n", llc_misses);
2430	fprintf(out, " Load access blocked by data : %10d\n", stats->blk_data);
2431	fprintf(out, " Load access blocked by address : %10d\n", stats->blk_addr);
2432	fprintf(out, " Load HIT Local Peer : %10d\n", stats->lcl_peer);
2433	fprintf(out, " Load HIT Remote Peer : %10d\n", stats->rmt_peer);
2434	fprintf(out, " LLC Misses to Local DRAM : %10.1f%%\n", ((double)stats->lcl_dram/(double)llc_misses) * 100.);
2435	fprintf(out, " LLC Misses to Remote DRAM : %10.1f%%\n", ((double)stats->rmt_dram/(double)llc_misses) * 100.);
2436	fprintf(out, " LLC Misses to Remote cache (HIT) : %10.1f%%\n", ((double)stats->rmt_hit /(double)llc_misses) * 100.);
2437	fprintf(out, " LLC Misses to Remote cache (HITM) : %10.1f%%\n", ((double)stats->rmt_hitm/(double)llc_misses) * 100.);
2438	fprintf(out, " Store Operations : %10d\n", stats->store);
2439	fprintf(out, " Store - uncacheable : %10d\n", stats->st_uncache);
2440	fprintf(out, " Store - no mapping : %10d\n", stats->st_noadrs);
2441	fprintf(out, " Store L1D Hit : %10d\n", stats->st_l1hit);
2442	fprintf(out, " Store L1D Miss : %10d\n", stats->st_l1miss);
2443	fprintf(out, " Store No available memory level : %10d\n", stats->st_na);
2444	fprintf(out, " No Page Map Rejects : %10d\n", stats->nomap);
2445	fprintf(out, " Unable to parse data source : %10d\n", stats->noparse);
2446	}
2447
2448	static void print_shared_cacheline_info(FILE *out)
2449	{
2450	struct c2c_stats *stats = &c2c.shared_clines_stats;
2451	int hitm_cnt = stats->lcl_hitm + stats->rmt_hitm;
2452
2453	fprintf(out, "=================================================\n");
2454	fprintf(out, " Global Shared Cache Line Event Information \n");
2455	fprintf(out, "=================================================\n");
2456	fprintf(out, " Total Shared Cache Lines : %10d\n", c2c.shared_clines);
2457	fprintf(out, " Load HITs on shared lines : %10d\n", stats->load);
2458	fprintf(out, " Fill Buffer Hits on shared lines : %10d\n", stats->ld_fbhit);
2459	fprintf(out, " L1D hits on shared lines : %10d\n", stats->ld_l1hit);
2460	fprintf(out, " L2D hits on shared lines : %10d\n", stats->ld_l2hit);
2461	fprintf(out, " LLC hits on shared lines : %10d\n", stats->ld_llchit + stats->lcl_hitm);
2462	fprintf(out, " Load hits on peer cache or nodes : %10d\n", stats->lcl_peer + stats->rmt_peer);
2463	fprintf(out, " Locked Access on shared lines : %10d\n", stats->locks);
2464	fprintf(out, " Blocked Access on shared lines : %10d\n", stats->blk_data + stats->blk_addr);
2465	fprintf(out, " Store HITs on shared lines : %10d\n", stats->store);
2466	fprintf(out, " Store L1D hits on shared lines : %10d\n", stats->st_l1hit);
2467	fprintf(out, " Store No available memory level : %10d\n", stats->st_na);
2468	fprintf(out, " Total Merged records : %10d\n", hitm_cnt + stats->store);
2469	}
2470
2471	static void print_cacheline(struct c2c_hists *c2c_hists,
2472	struct hist_entry *he_cl,
2473	struct perf_hpp_list *hpp_list,
2474	FILE *out)
2475	{
2476	char bf[1000];
2477	struct perf_hpp hpp = {
2478	.buf = bf,
2479	.size = 1000,
2480	};
2481	static bool once;
2482
2483	if (!once) {
2484	hists__fprintf_headers(&c2c_hists->hists, out);
2485	once = true;
2486	} else {
2487	fprintf(out, "\n");
2488	}
2489
2490	fprintf(out, " ----------------------------------------------------------------------\n");
2491	__hist_entry__snprintf(he_cl, &hpp, hpp_list);
2492	fprintf(out, "%s\n", bf);
2493	fprintf(out, " ----------------------------------------------------------------------\n");
2494
2495	hists__fprintf(&c2c_hists->hists, false, 0, 0, 0, out, false);
2496	}
2497
2498	static void print_pareto(FILE *out, struct perf_env *env)
2499	{
2500	struct perf_hpp_list hpp_list;
2501	struct rb_node *nd;
2502	int ret;
2503	const char *cl_output;
2504
2505	if (c2c.display != DISPLAY_SNP_PEER)
2506	cl_output = "cl_num,"
2507	"cl_rmt_hitm,"
2508	"cl_lcl_hitm,"
2509	"cl_stores_l1hit,"
2510	"cl_stores_l1miss,"
2511	"cl_stores_na,"
2512	"dcacheline";
2513	else
2514	cl_output = "cl_num,"
2515	"cl_rmt_peer,"
2516	"cl_lcl_peer,"
2517	"cl_stores_l1hit,"
2518	"cl_stores_l1miss,"
2519	"cl_stores_na,"
2520	"dcacheline";
2521
2522	perf_hpp_list__init(&hpp_list);
2523	ret = hpp_list__parse(hpp_list: &hpp_list, output_: cl_output, /*evlist=*/NULL, env);
2524
2525	if (WARN_ONCE(ret, "failed to setup sort entries\n"))
2526	return;
2527
2528	nd = rb_first_cached(&c2c.hists.hists.entries);
2529
2530	for (; nd; nd = rb_next(nd)) {
2531	struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
2532	struct c2c_hist_entry *c2c_he;
2533
2534	if (he->filtered)
2535	continue;
2536
2537	c2c_he = container_of(he, struct c2c_hist_entry, he);
2538	print_cacheline(c2c_he->hists, he, &hpp_list, out);
2539	}
2540	}
2541
2542	static void print_c2c_info(FILE *out, struct perf_session *session)
2543	{
2544	struct evlist *evlist = session->evlist;
2545	struct evsel *evsel;
2546	bool first = true;
2547
2548	fprintf(out, "=================================================\n");
2549	fprintf(out, " c2c details \n");
2550	fprintf(out, "=================================================\n");
2551
2552	evlist__for_each_entry(evlist, evsel) {
2553	fprintf(out, "%-36s: %s\n", first ? " Events" : "", evsel__name(evsel));
2554	first = false;
2555	}
2556	fprintf(out, " Cachelines sort on : %s\n",
2557	display_str[c2c.display]);
2558	fprintf(out, " Cacheline data grouping : %s\n", c2c.cl_sort);
2559	}
2560
2561	static void perf_c2c__hists_fprintf(FILE *out, struct perf_session *session)
2562	{
2563	setup_pager();
2564
2565	print_c2c__display_stats(out);
2566	fprintf(out, "\n");
2567	print_shared_cacheline_info(out);
2568	fprintf(out, "\n");
2569	print_c2c_info(out, session);
2570
2571	if (c2c.stats_only)
2572	return;
2573
2574	fprintf(out, "\n");
2575	fprintf(out, "=================================================\n");
2576	fprintf(out, " Shared Data Cache Line Table \n");
2577	fprintf(out, "=================================================\n");
2578	fprintf(out, "#\n");
2579
2580	hists__fprintf(&c2c.hists.hists, true, 0, 0, 0, stdout, true);
2581
2582	fprintf(out, "\n");
2583	fprintf(out, "=================================================\n");
2584	fprintf(out, " Shared Cache Line Distribution Pareto \n");
2585	fprintf(out, "=================================================\n");
2586	fprintf(out, "#\n");
2587
2588	print_pareto(out, perf_session__env(session));
2589	}
2590
2591	#ifdef HAVE_SLANG_SUPPORT
2592
2593	static int perf_c2c__toggle_annotation(struct hist_browser *browser)
2594	{
2595	struct hist_entry *he = browser->he_selection;
2596	struct symbol *sym = NULL;
2597	struct annotated_source *src = NULL;
2598	struct c2c_hist_entry *c2c_he = NULL;
2599	u64 al_addr = NO_ADDR;
2600
2601	if (!perf_c2c__has_annotation(he->hists->hpp_list)) {
2602	ui_browser__help_window(&browser->b, "No annotation support");
2603	return 0;
2604	}
2605
2606	if (he == NULL) {
2607	ui_browser__help_window(&browser->b, "No entry selected for annotation");
2608	return 0;
2609	}
2610
2611	sym = he->ms.sym;
2612	if (sym == NULL) {
2613	ui_browser__help_window(&browser->b, "Can not annotate, no symbol found");
2614	return 0;
2615	}
2616
2617	src = symbol__hists(sym, 0);
2618	if (src == NULL) {
2619	ui_browser__help_window(&browser->b, "Failed to initialize annotation source");
2620	return 0;
2621	}
2622
2623	if (he->mem_info)
2624	al_addr = mem_info__iaddr(he->mem_info)->al_addr;
2625
2626	c2c_he = container_of(he, struct c2c_hist_entry, he);
2627	return hist_entry__tui_annotate(he, c2c_he->evsel, NULL, al_addr);
2628	}
2629
2630	static void c2c_browser__update_nr_entries(struct hist_browser *hb)
2631	{
2632	u64 nr_entries = 0;
2633	struct rb_node *nd = rb_first_cached(&hb->hists->entries);
2634
2635	while (nd) {
2636	struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
2637
2638	if (!he->filtered)
2639	nr_entries++;
2640
2641	nd = rb_next(nd);
2642	}
2643
2644	hb->nr_non_filtered_entries = nr_entries;
2645	}
2646
2647	struct c2c_cacheline_browser {
2648	struct hist_browser hb;
2649	struct hist_entry *he;
2650	};
2651
2652	static int
2653	perf_c2c_cacheline_browser__title(struct hist_browser *browser,
2654	char *bf, size_t size)
2655	{
2656	struct c2c_cacheline_browser *cl_browser;
2657	struct hist_entry *he;
2658	uint64_t addr = 0;
2659
2660	cl_browser = container_of(browser, struct c2c_cacheline_browser, hb);
2661	he = cl_browser->he;
2662
2663	if (he->mem_info)
2664	addr = cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl);
2665
2666	scnprintf(bf, size, "Cacheline 0x%lx", addr);
2667	return 0;
2668	}
2669
2670	static struct c2c_cacheline_browser*
2671	c2c_cacheline_browser__new(struct hists *hists, struct hist_entry *he)
2672	{
2673	struct c2c_cacheline_browser *browser;
2674
2675	browser = zalloc(sizeof(*browser));
2676	if (browser) {
2677	hist_browser__init(&browser->hb, hists);
2678	browser->hb.c2c_filter = true;
2679	browser->hb.title = perf_c2c_cacheline_browser__title;
2680	browser->he = he;
2681	}
2682
2683	return browser;
2684	}
2685
2686	static int perf_c2c__browse_cacheline(struct hist_entry *he)
2687	{
2688	struct c2c_hist_entry *c2c_he;
2689	struct c2c_hists *c2c_hists;
2690	struct c2c_cacheline_browser *cl_browser;
2691	struct hist_browser *browser;
2692	int key = -1;
2693	static const char help[] =
2694	" ENTER Toggle callchains (if present) \n"
2695	" n Toggle Node details info \n"
2696	" s Toggle full length of symbol and source line columns \n"
2697	" a Toggle annotation view \n"
2698	" q Return back to cacheline list \n";
2699
2700	if (!he)
2701	return 0;
2702
2703	/* Display compact version first. */
2704	c2c.symbol_full = false;
2705
2706	c2c_he = container_of(he, struct c2c_hist_entry, he);
2707	c2c_hists = c2c_he->hists;
2708
2709	cl_browser = c2c_cacheline_browser__new(&c2c_hists->hists, he);
2710	if (cl_browser == NULL)
2711	return -1;
2712
2713	browser = &cl_browser->hb;
2714
2715	/* reset abort key so that it can get Ctrl-C as a key */
2716	SLang_reset_tty();
2717	SLang_init_tty(0, 0, 0);
2718
2719	c2c_browser__update_nr_entries(browser);
2720
2721	while (1) {
2722	key = hist_browser__run(browser, "? - help", true, 0);
2723
2724	switch (key) {
2725	case 's':
2726	c2c.symbol_full = !c2c.symbol_full;
2727	break;
2728	case 'n':
2729	c2c.node_info = (c2c.node_info + 1) % 3;
2730	setup_nodes_header();
2731	break;
2732	case 'a':
2733	perf_c2c__toggle_annotation(browser);
2734	break;
2735	case 'q':
2736	goto out;
2737	case '?':
2738	ui_browser__help_window(&browser->b, help);
2739	break;
2740	default:
2741	break;
2742	}
2743	}
2744
2745	out:
2746	free(cl_browser);
2747	return 0;
2748	}
2749
2750	static int perf_c2c_browser__title(struct hist_browser *browser,
2751	char *bf, size_t size)
2752	{
2753	scnprintf(bf, size,
2754	"Shared Data Cache Line Table "
2755	"(%lu entries, sorted on %s)",
2756	browser->nr_non_filtered_entries,
2757	display_str[c2c.display]);
2758	return 0;
2759	}
2760
2761	static struct hist_browser*
2762	perf_c2c_browser__new(struct hists *hists)
2763	{
2764	struct hist_browser *browser = hist_browser__new(hists);
2765
2766	if (browser) {
2767	browser->title = perf_c2c_browser__title;
2768	browser->c2c_filter = true;
2769	}
2770
2771	return browser;
2772	}
2773
2774	static int perf_c2c__hists_browse(struct hists *hists)
2775	{
2776	struct hist_browser *browser;
2777	int key = -1;
2778	static const char help[] =
2779	" d Display cacheline details \n"
2780	" ENTER Toggle callchains (if present) \n"
2781	" q Quit \n";
2782
2783	browser = perf_c2c_browser__new(hists);
2784	if (browser == NULL)
2785	return -1;
2786
2787	/* reset abort key so that it can get Ctrl-C as a key */
2788	SLang_reset_tty();
2789	SLang_init_tty(0, 0, 0);
2790
2791	c2c_browser__update_nr_entries(browser);
2792
2793	while (1) {
2794	key = hist_browser__run(browser, "? - help", true, 0);
2795
2796	switch (key) {
2797	case 'q':
2798	goto out;
2799	case 'd':
2800	perf_c2c__browse_cacheline(browser->he_selection);
2801	break;
2802	case '?':
2803	ui_browser__help_window(&browser->b, help);
2804	break;
2805	default:
2806	break;
2807	}
2808	}
2809
2810	out:
2811	hist_browser__delete(browser);
2812	return 0;
2813	}
2814
2815	static void perf_c2c_display(struct perf_session *session)
2816	{
2817	if (use_browser == 0)
2818	perf_c2c__hists_fprintf(stdout, session);
2819	else
2820	perf_c2c__hists_browse(&c2c.hists.hists);
2821	}
2822	#else
2823	static void perf_c2c_display(struct perf_session *session)
2824	{
2825	use_browser = 0;
2826	perf_c2c__hists_fprintf(stdout, session);
2827	}
2828	#endif /* HAVE_SLANG_SUPPORT */
2829
2830	static char *fill_line(const char *orig, int len)
2831	{
2832	int i, j, olen = strlen(orig);
2833	char *buf;
2834
2835	buf = zalloc(len + 1);
2836	if (!buf)
2837	return NULL;
2838
2839	j = len / 2 - olen / 2;
2840
2841	for (i = 0; i < j - 1; i++)
2842	buf[i] = '-';
2843
2844	buf[i++] = ' ';
2845
2846	strcpy(p: buf + i, q: orig);
2847
2848	i += olen;
2849
2850	buf[i++] = ' ';
2851
2852	for (; i < len; i++)
2853	buf[i] = '-';
2854
2855	return buf;
2856	}
2857
2858	static int ui_quirks(void)
2859	{
2860	const char *nodestr = "Data address";
2861	char *buf;
2862
2863	if (!c2c.use_stdio) {
2864	dim_offset.width = 5;
2865	dim_offset.header = header_offset_tui;
2866	nodestr = chk_double_cl ? "Double-CL" : "CL";
2867	}
2868
2869	dim_percent_costly_snoop.header = percent_costly_snoop_header[c2c.display];
2870
2871	/* Fix the zero line for dcacheline column. */
2872	buf = fill_line(chk_double_cl ? "Double-Cacheline" : "Cacheline",
2873	dim_dcacheline.width +
2874	dim_dcacheline_node.width +
2875	dim_dcacheline_count.width + 4);
2876	if (!buf)
2877	return -ENOMEM;
2878
2879	dim_dcacheline.header.line[0].text = buf;
2880
2881	/* Fix the zero line for offset column. */
2882	buf = fill_line(orig: nodestr, len: dim_offset.width +
2883	dim_offset_node.width +
2884	dim_dcacheline_count.width + 4);
2885	if (!buf)
2886	return -ENOMEM;
2887
2888	dim_offset.header.line[0].text = buf;
2889
2890	return 0;
2891	}
2892
2893	#define CALLCHAIN_DEFAULT_OPT "graph,0.5,caller,function,percent"
2894
2895	const char callchain_help[] = "Display call graph (stack chain/backtrace):\n\n"
2896	CALLCHAIN_REPORT_HELP
2897	"\n\t\t\t\tDefault: " CALLCHAIN_DEFAULT_OPT;
2898
2899	static int
2900	parse_callchain_opt(const struct option *opt, const char *arg, int unset)
2901	{
2902	struct callchain_param *callchain = opt->value;
2903
2904	callchain->enabled = !unset;
2905	/*
2906	* --no-call-graph
2907	*/
2908	if (unset) {
2909	symbol_conf.use_callchain = false;
2910	callchain->mode = CHAIN_NONE;
2911	return 0;
2912	}
2913
2914	return parse_callchain_report_opt(arg);
2915	}
2916
2917	static int setup_callchain(struct evlist *evlist)
2918	{
2919	u64 sample_type = evlist__combined_sample_type(evlist);
2920	enum perf_call_graph_mode mode = CALLCHAIN_NONE;
2921
2922	if ((sample_type & PERF_SAMPLE_REGS_USER) &&
2923	(sample_type & PERF_SAMPLE_STACK_USER)) {
2924	mode = CALLCHAIN_DWARF;
2925	dwarf_callchain_users = true;
2926	} else if (sample_type & PERF_SAMPLE_BRANCH_STACK)
2927	mode = CALLCHAIN_LBR;
2928	else if (sample_type & PERF_SAMPLE_CALLCHAIN)
2929	mode = CALLCHAIN_FP;
2930
2931	if (!callchain_param.enabled &&
2932	callchain_param.mode != CHAIN_NONE &&
2933	mode != CALLCHAIN_NONE) {
2934	symbol_conf.use_callchain = true;
2935	if (callchain_register_param(&callchain_param) < 0) {
2936	ui__error("Can't register callchain params.\n");
2937	return -EINVAL;
2938	}
2939	}
2940
2941	if (c2c.stitch_lbr && (mode != CALLCHAIN_LBR)) {
2942	ui__warning("Can't find LBR callchain. Switch off --stitch-lbr.\n"
2943	"Please apply --call-graph lbr when recording.\n");
2944	c2c.stitch_lbr = false;
2945	}
2946
2947	callchain_param.record_mode = mode;
2948	callchain_param.min_percent = 0;
2949	return 0;
2950	}
2951
2952	static int setup_display(const char *str)
2953	{
2954	const char *display = str;
2955
2956	if (!strcmp(display, "tot"))
2957	c2c.display = DISPLAY_TOT_HITM;
2958	else if (!strcmp(display, "rmt"))
2959	c2c.display = DISPLAY_RMT_HITM;
2960	else if (!strcmp(display, "lcl"))
2961	c2c.display = DISPLAY_LCL_HITM;
2962	else if (!strcmp(display, "peer"))
2963	c2c.display = DISPLAY_SNP_PEER;
2964	else {
2965	pr_err("failed: unknown display type: %s\n", str);
2966	return -1;
2967	}
2968
2969	return 0;
2970	}
2971
2972	#define for_each_token(__tok, __buf, __sep, __tmp) \
2973	for (__tok = strtok_r(__buf, __sep, &__tmp); __tok; \
2974	__tok = strtok_r(NULL, __sep, &__tmp))
2975
2976	static int build_cl_output(char *cl_sort, bool no_source)
2977	{
2978	char *tok, *tmp, *buf = strdup(cl_sort);
2979	bool add_pid = false;
2980	bool add_tid = false;
2981	bool add_iaddr = false;
2982	bool add_sym = false;
2983	bool add_dso = false;
2984	bool add_src = false;
2985	int ret = 0;
2986
2987	if (!buf)
2988	return -ENOMEM;
2989
2990	for_each_token(tok, buf, ",", tmp) {
2991	if (!strcmp(tok, "tid")) {
2992	add_tid = true;
2993	} else if (!strcmp(tok, "pid")) {
2994	add_pid = true;
2995	} else if (!strcmp(tok, "iaddr")) {
2996	add_iaddr = true;
2997	add_sym = true;
2998	add_dso = true;
2999	add_src = no_source ? false : true;
3000	} else if (!strcmp(tok, "dso")) {
3001	add_dso = true;
3002	} else if (strcmp(tok, "offset")) {
3003	pr_err("unrecognized sort token: %s\n", tok);
3004	ret = -EINVAL;
3005	goto err;
3006	}
3007	}
3008
3009	if (asprintf(&c2c.cl_output,
3010	"%s%s%s%s%s%s%s%s%s%s%s%s",
3011	c2c.use_stdio ? "cl_num_empty," : "",
3012	c2c.display == DISPLAY_SNP_PEER ? "percent_rmt_peer,"
3013	"percent_lcl_peer," :
3014	"percent_rmt_hitm,"
3015	"percent_lcl_hitm,",
3016	"percent_stores_l1hit,"
3017	"percent_stores_l1miss,"
3018	"percent_stores_na,"
3019	"offset,offset_node,dcacheline_count,",
3020	add_pid ? "pid," : "",
3021	add_tid ? "tid," : "",
3022	add_iaddr ? "iaddr," : "",
3023	c2c.display == DISPLAY_SNP_PEER ? "mean_rmt_peer,"
3024	"mean_lcl_peer," :
3025	"mean_rmt,"
3026	"mean_lcl,",
3027	"mean_load,"
3028	"tot_recs,"
3029	"cpucnt,",
3030	add_sym ? "symbol," : "",
3031	add_dso ? "dso," : "",
3032	add_src ? "cl_srcline," : "",
3033	"node") < 0) {
3034	ret = -ENOMEM;
3035	goto err;
3036	}
3037
3038	c2c.show_src = add_src;
3039	err:
3040	free(buf);
3041	return ret;
3042	}
3043
3044	static int setup_coalesce(const char *coalesce, bool no_source)
3045	{
3046	const char *c = coalesce ?: coalesce_default;
3047	const char *sort_str = NULL;
3048
3049	if (asprintf(&c2c.cl_sort, "offset,%s", c) < 0)
3050	return -ENOMEM;
3051
3052	if (build_cl_output(cl_sort: c2c.cl_sort, no_source))
3053	return -1;
3054
3055	if (c2c.display == DISPLAY_TOT_HITM)
3056	sort_str = "tot_hitm";
3057	else if (c2c.display == DISPLAY_RMT_HITM)
3058	sort_str = "rmt_hitm,lcl_hitm";
3059	else if (c2c.display == DISPLAY_LCL_HITM)
3060	sort_str = "lcl_hitm,rmt_hitm";
3061	else if (c2c.display == DISPLAY_SNP_PEER)
3062	sort_str = "tot_peer";
3063
3064	if (asprintf(&c2c.cl_resort, "offset,%s", sort_str) < 0)
3065	return -ENOMEM;
3066
3067	pr_debug("coalesce sort fields: %s\n", c2c.cl_sort);
3068	pr_debug("coalesce resort fields: %s\n", c2c.cl_resort);
3069	pr_debug("coalesce output fields: %s\n", c2c.cl_output);
3070	return 0;
3071	}
3072
3073	static int perf_c2c__report(int argc, const char **argv)
3074	{
3075	struct itrace_synth_opts itrace_synth_opts = {
3076	.set = true,
3077	.mem = true, /* Only enable memory event */
3078	.default_no_sample = true,
3079	};
3080
3081	struct perf_session *session;
3082	struct ui_progress prog;
3083	struct perf_data data = {
3084	.mode = PERF_DATA_MODE_READ,
3085	};
3086	char callchain_default_opt[] = CALLCHAIN_DEFAULT_OPT;
3087	const char *display = NULL;
3088	const char *coalesce = NULL;
3089	bool no_source = false;
3090	const char *disassembler_style = NULL, *objdump_path = NULL;
3091	const struct option options[] = {
3092	OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
3093	"file", "vmlinux pathname"),
3094	OPT_STRING('i', "input", &input_name, "file",
3095	"the input file to process"),
3096	OPT_INCR('N', "node-info", &c2c.node_info,
3097	"show extra node info in report (repeat for more info)"),
3098	OPT_BOOLEAN(0, "stdio", &c2c.use_stdio, "Use the stdio interface"),
3099	OPT_BOOLEAN(0, "stats", &c2c.stats_only,
3100	"Display only statistic tables (implies --stdio)"),
3101	OPT_BOOLEAN(0, "full-symbols", &c2c.symbol_full,
3102	"Display full length of symbols"),
3103	OPT_BOOLEAN(0, "no-source", &no_source,
3104	"Do not display Source Line column"),
3105	OPT_BOOLEAN(0, "show-all", &c2c.show_all,
3106	"Show all captured HITM lines."),
3107	OPT_CALLBACK_DEFAULT('g', "call-graph", &callchain_param,
3108	"print_type,threshold[,print_limit],order,sort_key[,branch],value",
3109	callchain_help, &parse_callchain_opt,
3110	callchain_default_opt),
3111	OPT_STRING('d', "display", &display, "Switch HITM output type", "tot,lcl,rmt,peer"),
3112	OPT_STRING('c', "coalesce", &coalesce, "coalesce fields",
3113	"coalesce fields: pid,tid,iaddr,dso"),
3114	OPT_BOOLEAN('f', "force", &symbol_conf.force, "don't complain, do it"),
3115	OPT_BOOLEAN(0, "stitch-lbr", &c2c.stitch_lbr,
3116	"Enable LBR callgraph stitching approach"),
3117	OPT_BOOLEAN(0, "double-cl", &chk_double_cl, "Detect adjacent cacheline false sharing"),
3118	OPT_STRING('M', "disassembler-style", &disassembler_style, "disassembler style",
3119	"Specify disassembler style (e.g. -M intel for intel syntax)"),
3120	OPT_STRING(0, "objdump", &objdump_path, "path",
3121	"objdump binary to use for disassembly and annotations"),
3122	OPT_PARENT(c2c_options),
3123	OPT_END()
3124	};
3125	int err = 0;
3126	const char *output_str, *sort_str = NULL;
3127	struct perf_env *env;
3128
3129	annotation_options__init();
3130
3131	err = hists__init();
3132	if (err < 0)
3133	goto out;
3134
3135	argc = parse_options(argc, argv, options, report_c2c_usage,
3136	PARSE_OPT_STOP_AT_NON_OPTION);
3137	if (argc)
3138	usage_with_options(report_c2c_usage, options);
3139
3140	#ifndef HAVE_SLANG_SUPPORT
3141	c2c.use_stdio = true;
3142	#endif
3143
3144	if (c2c.stats_only)
3145	c2c.use_stdio = true;
3146
3147	/**
3148	* Annotation related options disassembler_style, objdump_path are set
3149	* in the c2c_options, so we can use them here.
3150	*/
3151	if (disassembler_style) {
3152	annotate_opts.disassembler_style = strdup(disassembler_style);
3153	if (!annotate_opts.disassembler_style) {
3154	err = -ENOMEM;
3155	pr_err("Failed to allocate memory for annotation options\n");
3156	goto out;
3157	}
3158	}
3159	if (objdump_path) {
3160	annotate_opts.objdump_path = strdup(objdump_path);
3161	if (!annotate_opts.objdump_path) {
3162	err = -ENOMEM;
3163	pr_err("Failed to allocate memory for annotation options\n");
3164	goto out;
3165	}
3166	}
3167
3168	err = symbol__validate_sym_arguments();
3169	if (err)
3170	goto out;
3171
3172	if (!input_name || !strlen(input_name))
3173	input_name = "perf.data";
3174
3175	data.path = input_name;
3176	data.force = symbol_conf.force;
3177
3178	perf_tool__init(&c2c.tool, /*ordered_events=*/true);
3179	c2c.tool.sample = process_sample_event;
3180	c2c.tool.mmap = perf_event__process_mmap;
3181	c2c.tool.mmap2 = perf_event__process_mmap2;
3182	c2c.tool.comm = perf_event__process_comm;
3183	c2c.tool.exit = perf_event__process_exit;
3184	c2c.tool.fork = perf_event__process_fork;
3185	c2c.tool.lost = perf_event__process_lost;
3186	c2c.tool.attr = perf_event__process_attr;
3187	c2c.tool.auxtrace_info = perf_event__process_auxtrace_info;
3188	c2c.tool.auxtrace = perf_event__process_auxtrace;
3189	c2c.tool.auxtrace_error = perf_event__process_auxtrace_error;
3190	c2c.tool.ordering_requires_timestamps = true;
3191	session = perf_session__new(&data, &c2c.tool);
3192	if (IS_ERR(ptr: session)) {
3193	err = PTR_ERR(ptr: session);
3194	pr_debug("Error creating perf session\n");
3195	goto out;
3196	}
3197	env = perf_session__env(session);
3198	/*
3199	* Use the 'tot' as default display type if user doesn't specify it;
3200	* since Arm64 platform doesn't support HITMs flag, use 'peer' as the
3201	* default display type.
3202	*/
3203	if (!display) {
3204	if (!strcmp(perf_env__arch(env), "arm64"))
3205	display = "peer";
3206	else
3207	display = "tot";
3208	}
3209
3210	err = setup_display(display);
3211	if (err)
3212	goto out_session;
3213
3214	err = setup_coalesce(coalesce, no_source);
3215	if (err) {
3216	pr_debug("Failed to initialize hists\n");
3217	goto out_session;
3218	}
3219
3220	err = c2c_hists__init(hists: &c2c.hists, sort: "dcacheline", nr_header_lines: 2, env: perf_session__env(session));
3221	if (err) {
3222	pr_debug("Failed to initialize hists\n");
3223	goto out_session;
3224	}
3225
3226	session->itrace_synth_opts = &itrace_synth_opts;
3227
3228	err = setup_nodes(session);
3229	if (err) {
3230	pr_err("Failed setup nodes\n");
3231	goto out_session;
3232	}
3233
3234	err = mem2node__init(&c2c.mem2node, env);
3235	if (err)
3236	goto out_session;
3237
3238	err = setup_callchain(session->evlist);
3239	if (err)
3240	goto out_mem2node;
3241
3242	if (c2c.use_stdio)
3243	use_browser = 0;
3244	else
3245	use_browser = 1;
3246
3247	/*
3248	* Only in the TUI browser we are doing integrated annotation,
3249	* so don't allocate extra space that won't be used in the stdio
3250	* implementation.
3251	*/
3252	if (perf_c2c__has_annotation(NULL)) {
3253	int ret = symbol__annotation_init();
3254
3255	if (ret < 0)
3256	goto out_mem2node;
3257	/*
3258	* For searching by name on the "Browse map details".
3259	* providing it only in verbose mode not to bloat too
3260	* much struct symbol.
3261	*/
3262	if (verbose > 0) {
3263	/*
3264	* XXX: Need to provide a less kludgy way to ask for
3265	* more space per symbol, the u32 is for the index on
3266	* the ui browser.
3267	* See symbol__browser_index.
3268	*/
3269	symbol_conf.priv_size += sizeof(u32);
3270	}
3271	annotation_config__init();
3272	}
3273
3274	if (symbol__init(env) < 0)
3275	goto out_mem2node;
3276
3277	/* No pipe support at the moment. */
3278	if (perf_data__is_pipe(session->data)) {
3279	pr_debug("No pipe support at the moment.\n");
3280	goto out_mem2node;
3281	}
3282
3283	setup_browser(false);
3284
3285	err = perf_session__process_events(session);
3286	if (err) {
3287	pr_err("failed to process sample\n");
3288	goto out_mem2node;
3289	}
3290
3291	if (c2c.display != DISPLAY_SNP_PEER)
3292	output_str = "cl_idx,"
3293	"dcacheline,"
3294	"dcacheline_node,"
3295	"dcacheline_count,"
3296	"percent_costly_snoop,"
3297	"tot_hitm,lcl_hitm,rmt_hitm,"
3298	"tot_recs,"
3299	"tot_loads,"
3300	"tot_stores,"
3301	"stores_l1hit,stores_l1miss,stores_na,"
3302	"ld_fbhit,ld_l1hit,ld_l2hit,"
3303	"ld_lclhit,lcl_hitm,"
3304	"ld_rmthit,rmt_hitm,"
3305	"dram_lcl,dram_rmt";
3306	else
3307	output_str = "cl_idx,"
3308	"dcacheline,"
3309	"dcacheline_node,"
3310	"dcacheline_count,"
3311	"percent_costly_snoop,"
3312	"tot_peer,lcl_peer,rmt_peer,"
3313	"tot_recs,"
3314	"tot_loads,"
3315	"tot_stores,"
3316	"stores_l1hit,stores_l1miss,stores_na,"
3317	"ld_fbhit,ld_l1hit,ld_l2hit,"
3318	"ld_lclhit,lcl_hitm,"
3319	"ld_rmthit,rmt_hitm,"
3320	"dram_lcl,dram_rmt";
3321
3322	if (c2c.display == DISPLAY_TOT_HITM)
3323	sort_str = "tot_hitm";
3324	else if (c2c.display == DISPLAY_RMT_HITM)
3325	sort_str = "rmt_hitm";
3326	else if (c2c.display == DISPLAY_LCL_HITM)
3327	sort_str = "lcl_hitm";
3328	else if (c2c.display == DISPLAY_SNP_PEER)
3329	sort_str = "tot_peer";
3330
3331	c2c_hists__reinit(c2c_hists: &c2c.hists, output: output_str, sort: sort_str, env: perf_session__env(session));
3332
3333	ui_progress__init(&prog, c2c.hists.hists.nr_entries, "Sorting...");
3334
3335	hists__collapse_resort(&c2c.hists.hists, NULL);
3336	hists__output_resort_cb(&c2c.hists.hists, &prog, resort_shared_cl_cb);
3337	hists__iterate_cb(&c2c.hists.hists, resort_cl_cb, perf_session__env(session));
3338
3339	ui_progress__finish();
3340
3341	if (ui_quirks()) {
3342	pr_err("failed to setup UI\n");
3343	goto out_mem2node;
3344	}
3345
3346	perf_c2c_display(session);
3347
3348	out_mem2node:
3349	mem2node__exit(&c2c.mem2node);
3350	out_session:
3351	perf_session__delete(session);
3352	out:
3353	annotation_options__exit();
3354	return err;
3355	}
3356
3357	static int parse_record_events(const struct option *opt,
3358	const char *str, int unset __maybe_unused)
3359	{
3360	bool *event_set = (bool *) opt->value;
3361	struct perf_pmu *pmu;
3362
3363	pmu = perf_mem_events_find_pmu();
3364	if (!pmu) {
3365	pr_err("failed: there is no PMU that supports perf c2c\n");
3366	exit(-1);
3367	}
3368
3369	if (!strcmp(str, "list")) {
3370	perf_pmu__mem_events_list(pmu);
3371	exit(0);
3372	}
3373	if (perf_pmu__mem_events_parse(pmu, str))
3374	exit(-1);
3375
3376	*event_set = true;
3377	return 0;
3378	}
3379
3380
3381	static const char * const __usage_record[] = {
3382	"perf c2c record [<options>] [<command>]",
3383	"perf c2c record [<options>] -- <command> [<options>]",
3384	NULL
3385	};
3386
3387	static const char * const *record_mem_usage = __usage_record;
3388
3389	static int perf_c2c__record(int argc, const char **argv)
3390	{
3391	int rec_argc, i = 0, j;
3392	const char **rec_argv;
3393	char *event_name_storage = NULL;
3394	int ret;
3395	bool all_user = false, all_kernel = false;
3396	bool event_set = false;
3397	struct perf_mem_event *e;
3398	struct perf_pmu *pmu;
3399	struct option options[] = {
3400	OPT_CALLBACK('e', "event", &event_set, "event",
3401	"event selector. Use 'perf c2c record -e list' to list available events",
3402	parse_record_events),
3403	OPT_BOOLEAN('u', "all-user", &all_user, "collect only user level data"),
3404	OPT_BOOLEAN('k', "all-kernel", &all_kernel, "collect only kernel level data"),
3405	OPT_UINTEGER('l', "ldlat", &perf_mem_events__loads_ldlat, "setup mem-loads latency"),
3406	OPT_PARENT(c2c_options),
3407	OPT_END()
3408	};
3409
3410	pmu = perf_mem_events_find_pmu();
3411	if (!pmu) {
3412	pr_err("failed: no PMU supports the memory events\n");
3413	return -1;
3414	}
3415
3416	if (perf_pmu__mem_events_init()) {
3417	pr_err("failed: memory events not supported\n");
3418	return -1;
3419	}
3420
3421	argc = parse_options(argc, argv, options, record_mem_usage,
3422	PARSE_OPT_KEEP_UNKNOWN);
3423
3424	/* Max number of arguments multiplied by number of PMUs that can support them. */
3425	rec_argc = argc + 11 * (perf_pmu__mem_events_num_mem_pmus(pmu) + 1);
3426
3427	rec_argv = calloc(rec_argc + 1, sizeof(char *));
3428	if (!rec_argv)
3429	return -1;
3430
3431	rec_argv[i++] = "record";
3432
3433	if (!event_set) {
3434	e = perf_pmu__mem_events_ptr(pmu, i: PERF_MEM_EVENTS__LOAD_STORE);
3435	/*
3436	* The load and store operations are required, use the event
3437	* PERF_MEM_EVENTS__LOAD_STORE if it is supported.
3438	*/
3439	if (e->tag) {
3440	perf_mem_record[PERF_MEM_EVENTS__LOAD_STORE] = true;
3441	rec_argv[i++] = "-W";
3442	} else {
3443	perf_mem_record[PERF_MEM_EVENTS__LOAD] = true;
3444	perf_mem_record[PERF_MEM_EVENTS__STORE] = true;
3445	}
3446	}
3447
3448	if (perf_mem_record[PERF_MEM_EVENTS__LOAD])
3449	rec_argv[i++] = "-W";
3450
3451	rec_argv[i++] = "-d";
3452	rec_argv[i++] = "--phys-data";
3453	rec_argv[i++] = "--sample-cpu";
3454
3455	ret = perf_mem_events__record_args(rec_argv, argv_nr: &i, event_name_storage_out: &event_name_storage);
3456	if (ret)
3457	goto out;
3458
3459	if (all_user)
3460	rec_argv[i++] = "--all-user";
3461
3462	if (all_kernel)
3463	rec_argv[i++] = "--all-kernel";
3464
3465	for (j = 0; j < argc; j++, i++)
3466	rec_argv[i] = argv[j];
3467
3468	if (verbose > 0) {
3469	pr_debug("calling: ");
3470
3471	j = 0;
3472
3473	while (rec_argv[j]) {
3474	pr_debug("%s ", rec_argv[j]);
3475	j++;
3476	}
3477	pr_debug("\n");
3478	}
3479
3480	ret = cmd_record(argc: i, argv: rec_argv);
3481	out:
3482	free(event_name_storage);
3483	free(rec_argv);
3484	return ret;
3485	}
3486
3487	int cmd_c2c(int argc, const char **argv)
3488	{
3489	argc = parse_options(argc, argv, c2c_options, c2c_usage,
3490	PARSE_OPT_STOP_AT_NON_OPTION);
3491
3492	if (!argc)
3493	usage_with_options(c2c_usage, c2c_options);
3494
3495	if (strlen(argv[0]) > 2 && strstarts(str: "record", prefix: argv[0])) {
3496	return perf_c2c__record(argc, argv);
3497	} else if (strlen(argv[0]) > 2 && strstarts(str: "report", prefix: argv[0])) {
3498	return perf_c2c__report(argc, argv);
3499	} else {
3500	usage_with_options(c2c_usage, c2c_options);
3501	}
3502
3503	return 0;
3504	}
3505