1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright(C) 2016 Linaro Limited. All rights reserved.
4	* Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5	*/
6
7	#include <linux/atomic.h>
8	#include <linux/coresight.h>
9	#include <linux/dma-mapping.h>
10	#include <linux/iommu.h>
11	#include <linux/idr.h>
12	#include <linux/mutex.h>
13	#include <linux/refcount.h>
14	#include <linux/slab.h>
15	#include <linux/types.h>
16	#include <linux/vmalloc.h>
17	#include "coresight-catu.h"
18	#include "coresight-etm-perf.h"
19	#include "coresight-priv.h"
20	#include "coresight-tmc.h"
21
22	struct etr_flat_buf {
23	struct device *dev;
24	dma_addr_t daddr;
25	void *vaddr;
26	size_t size;
27	};
28
29	struct etr_buf_hw {
30	bool has_iommu;
31	bool has_etr_sg;
32	bool has_catu;
33	};
34
35	/*
36	* etr_perf_buffer - Perf buffer used for ETR
37	* @drvdata - The ETR drvdaga this buffer has been allocated for.
38	* @etr_buf - Actual buffer used by the ETR
39	* @pid - The PID this etr_perf_buffer belongs to.
40	* @snaphost - Perf session mode
41	* @nr_pages - Number of pages in the ring buffer.
42	* @pages - Array of Pages in the ring buffer.
43	*/
44	struct etr_perf_buffer {
45	struct tmc_drvdata *drvdata;
46	struct etr_buf *etr_buf;
47	pid_t pid;
48	bool snapshot;
49	int nr_pages;
50	void **pages;
51	};
52
53	/* Convert the perf index to an offset within the ETR buffer */
54	#define PERF_IDX2OFF(idx, buf) \
55	((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
56
57	/* Lower limit for ETR hardware buffer */
58	#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M
59
60	/*
61	* The TMC ETR SG has a page size of 4K. The SG table contains pointers
62	* to 4KB buffers. However, the OS may use a PAGE_SIZE different from
63	* 4K (i.e, 16KB or 64KB). This implies that a single OS page could
64	* contain more than one SG buffer and tables.
65	*
66	* A table entry has the following format:
67	*
68	* ---Bit31------------Bit4-------Bit1-----Bit0--
69	* | Address[39:12] | SBZ | Entry Type |
70	* ----------------------------------------------
71	*
72	* Address: Bits [39:12] of a physical page address. Bits [11:0] are
73	* always zero.
74	*
75	* Entry type:
76	* b00 - Reserved.
77	* b01 - Last entry in the tables, points to 4K page buffer.
78	* b10 - Normal entry, points to 4K page buffer.
79	* b11 - Link. The address points to the base of next table.
80	*/
81
82	typedef u32 sgte_t;
83
84	#define ETR_SG_PAGE_SHIFT 12
85	#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
86	#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
87	#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
88	#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
89
90	#define ETR_SG_ET_MASK 0x3
91	#define ETR_SG_ET_LAST 0x1
92	#define ETR_SG_ET_NORMAL 0x2
93	#define ETR_SG_ET_LINK 0x3
94
95	#define ETR_SG_ADDR_SHIFT 4
96
97	#define ETR_SG_ENTRY(addr, type) \
98	(sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
99	(type & ETR_SG_ET_MASK))
100
101	#define ETR_SG_ADDR(entry) \
102	(((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
103	#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
104
105	/*
106	* struct etr_sg_table : ETR SG Table
107	* @sg_table: Generic SG Table holding the data/table pages.
108	* @hwaddr: hwaddress used by the TMC, which is the base
109	* address of the table.
110	*/
111	struct etr_sg_table {
112	struct tmc_sg_table *sg_table;
113	dma_addr_t hwaddr;
114	};
115
116	/*
117	* tmc_etr_sg_table_entries: Total number of table entries required to map
118	* @nr_pages system pages.
119	*
120	* We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
121	* Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
122	* with the last entry pointing to another page of table entries.
123	* If we spill over to a new page for mapping 1 entry, we could as
124	* well replace the link entry of the previous page with the last entry.
125	*/
126	static inline unsigned long __attribute_const__
127	tmc_etr_sg_table_entries(int nr_pages)
128	{
129	unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
130	unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
131	/*
132	* If we spill over to a new page for 1 entry, we could as well
133	* make it the LAST entry in the previous page, skipping the Link
134	* address.
135	*/
136	if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
137	nr_sglinks--;
138	return nr_sgpages + nr_sglinks;
139	}
140
141	/*
142	* tmc_pages_get_offset: Go through all the pages in the tmc_pages
143	* and map the device address @addr to an offset within the virtual
144	* contiguous buffer.
145	*/
146	static long
147	tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
148	{
149	int i;
150	dma_addr_t page_start;
151
152	for (i = 0; i < tmc_pages->nr_pages; i++) {
153	page_start = tmc_pages->daddrs[i];
154	if (addr >= page_start && addr < (page_start + PAGE_SIZE))
155	return i * PAGE_SIZE + (addr - page_start);
156	}
157
158	return -EINVAL;
159	}
160
161	/*
162	* tmc_pages_free : Unmap and free the pages used by tmc_pages.
163	* If the pages were not allocated in tmc_pages_alloc(), we would
164	* simply drop the refcount.
165	*/
166	static void tmc_pages_free(struct tmc_pages *tmc_pages,
167	struct device *dev, enum dma_data_direction dir)
168	{
169	int i;
170	struct device *real_dev = dev->parent;
171
172	for (i = 0; i < tmc_pages->nr_pages; i++) {
173	if (tmc_pages->daddrs && tmc_pages->daddrs[i])
174	dma_unmap_page(real_dev, tmc_pages->daddrs[i],
175	PAGE_SIZE, dir);
176	if (tmc_pages->pages && tmc_pages->pages[i])
177	__free_page(tmc_pages->pages[i]);
178	}
179
180	kfree(objp: tmc_pages->pages);
181	kfree(objp: tmc_pages->daddrs);
182	tmc_pages->pages = NULL;
183	tmc_pages->daddrs = NULL;
184	tmc_pages->nr_pages = 0;
185	}
186
187	/*
188	* tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
189	* If @pages is not NULL, the list of page virtual addresses are
190	* used as the data pages. The pages are then dma_map'ed for @dev
191	* with dma_direction @dir.
192	*
193	* Returns 0 upon success, else the error number.
194	*/
195	static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
196	struct device *dev, int node,
197	enum dma_data_direction dir, void **pages)
198	{
199	int i, nr_pages;
200	dma_addr_t paddr;
201	struct page *page;
202	struct device *real_dev = dev->parent;
203
204	nr_pages = tmc_pages->nr_pages;
205	tmc_pages->daddrs = kcalloc(n: nr_pages, size: sizeof(*tmc_pages->daddrs),
206	GFP_KERNEL);
207	if (!tmc_pages->daddrs)
208	return -ENOMEM;
209	tmc_pages->pages = kcalloc(n: nr_pages, size: sizeof(*tmc_pages->pages),
210	GFP_KERNEL);
211	if (!tmc_pages->pages) {
212	kfree(objp: tmc_pages->daddrs);
213	tmc_pages->daddrs = NULL;
214	return -ENOMEM;
215	}
216
217	for (i = 0; i < nr_pages; i++) {
218	if (pages && pages[i]) {
219	page = virt_to_page(pages[i]);
220	/* Hold a refcount on the page */
221	get_page(page);
222	} else {
223	page = alloc_pages_node(nid: node,
224	GFP_KERNEL | __GFP_ZERO, order: 0);
225	if (!page)
226	goto err;
227	}
228	paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
229	if (dma_mapping_error(dev: real_dev, dma_addr: paddr))
230	goto err;
231	tmc_pages->daddrs[i] = paddr;
232	tmc_pages->pages[i] = page;
233	}
234	return 0;
235	err:
236	tmc_pages_free(tmc_pages, dev, dir);
237	return -ENOMEM;
238	}
239
240	static inline long
241	tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
242	{
243	return tmc_pages_get_offset(tmc_pages: &sg_table->data_pages, addr);
244	}
245
246	static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
247	{
248	if (sg_table->table_vaddr)
249	vunmap(addr: sg_table->table_vaddr);
250	tmc_pages_free(tmc_pages: &sg_table->table_pages, dev: sg_table->dev, dir: DMA_TO_DEVICE);
251	}
252
253	static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
254	{
255	if (sg_table->data_vaddr)
256	vunmap(addr: sg_table->data_vaddr);
257	tmc_pages_free(tmc_pages: &sg_table->data_pages, dev: sg_table->dev, dir: DMA_FROM_DEVICE);
258	}
259
260	void tmc_free_sg_table(struct tmc_sg_table *sg_table)
261	{
262	tmc_free_table_pages(sg_table);
263	tmc_free_data_pages(sg_table);
264	}
265	EXPORT_SYMBOL_GPL(tmc_free_sg_table);
266
267	/*
268	* Alloc pages for the table. Since this will be used by the device,
269	* allocate the pages closer to the device (i.e, dev_to_node(dev)
270	* rather than the CPU node).
271	*/
272	static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
273	{
274	int rc;
275	struct tmc_pages *table_pages = &sg_table->table_pages;
276
277	rc = tmc_pages_alloc(tmc_pages: table_pages, dev: sg_table->dev,
278	node: dev_to_node(dev: sg_table->dev),
279	dir: DMA_TO_DEVICE, NULL);
280	if (rc)
281	return rc;
282	sg_table->table_vaddr = vmap(pages: table_pages->pages,
283	count: table_pages->nr_pages,
284	VM_MAP,
285	PAGE_KERNEL);
286	if (!sg_table->table_vaddr)
287	rc = -ENOMEM;
288	else
289	sg_table->table_daddr = table_pages->daddrs[0];
290	return rc;
291	}
292
293	static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
294	{
295	int rc;
296
297	/* Allocate data pages on the node requested by the caller */
298	rc = tmc_pages_alloc(tmc_pages: &sg_table->data_pages,
299	dev: sg_table->dev, node: sg_table->node,
300	dir: DMA_FROM_DEVICE, pages);
301	if (!rc) {
302	sg_table->data_vaddr = vmap(pages: sg_table->data_pages.pages,
303	count: sg_table->data_pages.nr_pages,
304	VM_MAP,
305	PAGE_KERNEL);
306	if (!sg_table->data_vaddr)
307	rc = -ENOMEM;
308	}
309	return rc;
310	}
311
312	/*
313	* tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
314	* and data buffers. TMC writes to the data buffers and reads from the SG
315	* Table pages.
316	*
317	* @dev - Coresight device to which page should be DMA mapped.
318	* @node - Numa node for mem allocations
319	* @nr_tpages - Number of pages for the table entries.
320	* @nr_dpages - Number of pages for Data buffer.
321	* @pages - Optional list of virtual address of pages.
322	*/
323	struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
324	int node,
325	int nr_tpages,
326	int nr_dpages,
327	void **pages)
328	{
329	long rc;
330	struct tmc_sg_table *sg_table;
331
332	sg_table = kzalloc(size: sizeof(*sg_table), GFP_KERNEL);
333	if (!sg_table)
334	return ERR_PTR(error: -ENOMEM);
335	sg_table->data_pages.nr_pages = nr_dpages;
336	sg_table->table_pages.nr_pages = nr_tpages;
337	sg_table->node = node;
338	sg_table->dev = dev;
339
340	rc = tmc_alloc_data_pages(sg_table, pages);
341	if (!rc)
342	rc = tmc_alloc_table_pages(sg_table);
343	if (rc) {
344	tmc_free_sg_table(sg_table);
345	kfree(objp: sg_table);
346	return ERR_PTR(error: rc);
347	}
348
349	return sg_table;
350	}
351	EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
352
353	/*
354	* tmc_sg_table_sync_data_range: Sync the data buffer written
355	* by the device from @offset upto a @size bytes.
356	*/
357	void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
358	u64 offset, u64 size)
359	{
360	int i, index, start;
361	int npages = DIV_ROUND_UP(size, PAGE_SIZE);
362	struct device *real_dev = table->dev->parent;
363	struct tmc_pages *data = &table->data_pages;
364
365	start = offset >> PAGE_SHIFT;
366	for (i = start; i < (start + npages); i++) {
367	index = i % data->nr_pages;
368	dma_sync_single_for_cpu(dev: real_dev, addr: data->daddrs[index],
369	PAGE_SIZE, dir: DMA_FROM_DEVICE);
370	}
371	}
372	EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
373
374	/* tmc_sg_sync_table: Sync the page table */
375	void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
376	{
377	int i;
378	struct device *real_dev = sg_table->dev->parent;
379	struct tmc_pages *table_pages = &sg_table->table_pages;
380
381	for (i = 0; i < table_pages->nr_pages; i++)
382	dma_sync_single_for_device(dev: real_dev, addr: table_pages->daddrs[i],
383	PAGE_SIZE, dir: DMA_TO_DEVICE);
384	}
385	EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
386
387	/*
388	* tmc_sg_table_get_data: Get the buffer pointer for data @offset
389	* in the SG buffer. The @bufpp is updated to point to the buffer.
390	* Returns :
391	* the length of linear data available at @offset.
392	* or
393	* <= 0 if no data is available.
394	*/
395	ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
396	u64 offset, size_t len, char **bufpp)
397	{
398	size_t size;
399	int pg_idx = offset >> PAGE_SHIFT;
400	int pg_offset = offset & (PAGE_SIZE - 1);
401	struct tmc_pages *data_pages = &sg_table->data_pages;
402
403	size = tmc_sg_table_buf_size(sg_table);
404	if (offset >= size)
405	return -EINVAL;
406
407	/* Make sure we don't go beyond the end */
408	len = (len < (size - offset)) ? len : size - offset;
409	/* Respect the page boundaries */
410	len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
411	if (len > 0)
412	*bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
413	return len;
414	}
415	EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
416
417	#ifdef ETR_SG_DEBUG
418	/* Map a dma address to virtual address */
419	static unsigned long
420	tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
421	dma_addr_t addr, bool table)
422	{
423	long offset;
424	unsigned long base;
425	struct tmc_pages *tmc_pages;
426
427	if (table) {
428	tmc_pages = &sg_table->table_pages;
429	base = (unsigned long)sg_table->table_vaddr;
430	} else {
431	tmc_pages = &sg_table->data_pages;
432	base = (unsigned long)sg_table->data_vaddr;
433	}
434
435	offset = tmc_pages_get_offset(tmc_pages, addr);
436	if (offset < 0)
437	return 0;
438	return base + offset;
439	}
440
441	/* Dump the given sg_table */
442	static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
443	{
444	sgte_t *ptr;
445	int i = 0;
446	dma_addr_t addr;
447	struct tmc_sg_table *sg_table = etr_table->sg_table;
448
449	ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
450	etr_table->hwaddr, true);
451	while (ptr) {
452	addr = ETR_SG_ADDR(*ptr);
453	switch (ETR_SG_ET(*ptr)) {
454	case ETR_SG_ET_NORMAL:
455	dev_dbg(sg_table->dev,
456	"%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
457	ptr++;
458	break;
459	case ETR_SG_ET_LINK:
460	dev_dbg(sg_table->dev,
461	"%05d: *** %p\t:{L} 0x%llx ***\n",
462	i, ptr, addr);
463	ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
464	addr, true);
465	break;
466	case ETR_SG_ET_LAST:
467	dev_dbg(sg_table->dev,
468	"%05d: ### %p\t:[L] 0x%llx ###\n",
469	i, ptr, addr);
470	return;
471	default:
472	dev_dbg(sg_table->dev,
473	"%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
474	i, ptr, addr);
475	return;
476	}
477	i++;
478	}
479	dev_dbg(sg_table->dev, "******* End of Table *****\n");
480	}
481	#else
482	static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
483	#endif
484
485	/*
486	* Populate the SG Table page table entries from table/data
487	* pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
488	* So does a Table page. So we keep track of indices of the tables
489	* in each system page and move the pointers accordingly.
490	*/
491	#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
492	static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
493	{
494	dma_addr_t paddr;
495	int i, type, nr_entries;
496	int tpidx = 0; /* index to the current system table_page */
497	int sgtidx = 0; /* index to the sg_table within the current syspage */
498	int sgtentry = 0; /* the entry within the sg_table */
499	int dpidx = 0; /* index to the current system data_page */
500	int spidx = 0; /* index to the SG page within the current data page */
501	sgte_t *ptr; /* pointer to the table entry to fill */
502	struct tmc_sg_table *sg_table = etr_table->sg_table;
503	dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
504	dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
505
506	nr_entries = tmc_etr_sg_table_entries(nr_pages: sg_table->data_pages.nr_pages);
507	/*
508	* Use the contiguous virtual address of the table to update entries.
509	*/
510	ptr = sg_table->table_vaddr;
511	/*
512	* Fill all the entries, except the last entry to avoid special
513	* checks within the loop.
514	*/
515	for (i = 0; i < nr_entries - 1; i++) {
516	if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
517	/*
518	* Last entry in a sg_table page is a link address to
519	* the next table page. If this sg_table is the last
520	* one in the system page, it links to the first
521	* sg_table in the next system page. Otherwise, it
522	* links to the next sg_table page within the system
523	* page.
524	*/
525	if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
526	paddr = table_daddrs[tpidx + 1];
527	} else {
528	paddr = table_daddrs[tpidx] +
529	(ETR_SG_PAGE_SIZE * (sgtidx + 1));
530	}
531	type = ETR_SG_ET_LINK;
532	} else {
533	/*
534	* Update the indices to the data_pages to point to the
535	* next sg_page in the data buffer.
536	*/
537	type = ETR_SG_ET_NORMAL;
538	paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
539	if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
540	dpidx++;
541	}
542	*ptr++ = ETR_SG_ENTRY(paddr, type);
543	/*
544	* Move to the next table pointer, moving the table page index
545	* if necessary
546	*/
547	if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
548	if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
549	tpidx++;
550	}
551	}
552
553	/* Set up the last entry, which is always a data pointer */
554	paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
555	*ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
556	}
557
558	/*
559	* tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
560	* populate the table.
561	*
562	* @dev - Device pointer for the TMC
563	* @node - NUMA node where the memory should be allocated
564	* @size - Total size of the data buffer
565	* @pages - Optional list of page virtual address
566	*/
567	static struct etr_sg_table *
568	tmc_init_etr_sg_table(struct device *dev, int node,
569	unsigned long size, void **pages)
570	{
571	int nr_entries, nr_tpages;
572	int nr_dpages = size >> PAGE_SHIFT;
573	struct tmc_sg_table *sg_table;
574	struct etr_sg_table *etr_table;
575
576	etr_table = kzalloc(size: sizeof(*etr_table), GFP_KERNEL);
577	if (!etr_table)
578	return ERR_PTR(error: -ENOMEM);
579	nr_entries = tmc_etr_sg_table_entries(nr_pages: nr_dpages);
580	nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
581
582	sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
583	if (IS_ERR(ptr: sg_table)) {
584	kfree(objp: etr_table);
585	return ERR_CAST(ptr: sg_table);
586	}
587
588	etr_table->sg_table = sg_table;
589	/* TMC should use table base address for DBA */
590	etr_table->hwaddr = sg_table->table_daddr;
591	tmc_etr_sg_table_populate(etr_table);
592	/* Sync the table pages for the HW */
593	tmc_sg_table_sync_table(sg_table);
594	tmc_etr_sg_table_dump(etr_table);
595
596	return etr_table;
597	}
598
599	/*
600	* tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
601	*/
602	static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
603	struct etr_buf *etr_buf, int node,
604	void **pages)
605	{
606	struct etr_flat_buf *flat_buf;
607	struct device *real_dev = drvdata->csdev->dev.parent;
608
609	/* We cannot reuse existing pages for flat buf */
610	if (pages)
611	return -EINVAL;
612
613	flat_buf = kzalloc(size: sizeof(*flat_buf), GFP_KERNEL);
614	if (!flat_buf)
615	return -ENOMEM;
616
617	flat_buf->vaddr = dma_alloc_noncoherent(dev: real_dev, size: etr_buf->size,
618	dma_handle: &flat_buf->daddr,
619	dir: DMA_FROM_DEVICE,
620	GFP_KERNEL | __GFP_NOWARN);
621	if (!flat_buf->vaddr) {
622	kfree(objp: flat_buf);
623	return -ENOMEM;
624	}
625
626	flat_buf->size = etr_buf->size;
627	flat_buf->dev = &drvdata->csdev->dev;
628	etr_buf->hwaddr = flat_buf->daddr;
629	etr_buf->mode = ETR_MODE_FLAT;
630	etr_buf->private = flat_buf;
631	return 0;
632	}
633
634	static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
635	{
636	struct etr_flat_buf *flat_buf = etr_buf->private;
637
638	if (flat_buf && flat_buf->daddr) {
639	struct device *real_dev = flat_buf->dev->parent;
640
641	dma_free_noncoherent(dev: real_dev, size: etr_buf->size,
642	vaddr: flat_buf->vaddr, dma_handle: flat_buf->daddr,
643	dir: DMA_FROM_DEVICE);
644	}
645	kfree(objp: flat_buf);
646	}
647
648	static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
649	{
650	struct etr_flat_buf *flat_buf = etr_buf->private;
651	struct device *real_dev = flat_buf->dev->parent;
652
653	/*
654	* Adjust the buffer to point to the beginning of the trace data
655	* and update the available trace data.
656	*/
657	etr_buf->offset = rrp - etr_buf->hwaddr;
658	if (etr_buf->full)
659	etr_buf->len = etr_buf->size;
660	else
661	etr_buf->len = rwp - rrp;
662
663	/*
664	* The driver always starts tracing at the beginning of the buffer,
665	* the only reason why we would get a wrap around is when the buffer
666	* is full. Sync the entire buffer in one go for this case.
667	*/
668	if (etr_buf->offset + etr_buf->len > etr_buf->size)
669	dma_sync_single_for_cpu(dev: real_dev, addr: flat_buf->daddr,
670	size: etr_buf->size, dir: DMA_FROM_DEVICE);
671	else
672	dma_sync_single_for_cpu(dev: real_dev,
673	addr: flat_buf->daddr + etr_buf->offset,
674	size: etr_buf->len, dir: DMA_FROM_DEVICE);
675	}
676
677	static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
678	u64 offset, size_t len, char **bufpp)
679	{
680	struct etr_flat_buf *flat_buf = etr_buf->private;
681
682	*bufpp = (char *)flat_buf->vaddr + offset;
683	/*
684	* tmc_etr_buf_get_data already adjusts the length to handle
685	* buffer wrapping around.
686	*/
687	return len;
688	}
689
690	static const struct etr_buf_operations etr_flat_buf_ops = {
691	.alloc = tmc_etr_alloc_flat_buf,
692	.free = tmc_etr_free_flat_buf,
693	.sync = tmc_etr_sync_flat_buf,
694	.get_data = tmc_etr_get_data_flat_buf,
695	};
696
697	/*
698	* tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
699	* appropriately.
700	*/
701	static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
702	struct etr_buf *etr_buf, int node,
703	void **pages)
704	{
705	struct etr_sg_table *etr_table;
706	struct device *dev = &drvdata->csdev->dev;
707
708	etr_table = tmc_init_etr_sg_table(dev, node,
709	size: etr_buf->size, pages);
710	if (IS_ERR(ptr: etr_table))
711	return -ENOMEM;
712	etr_buf->hwaddr = etr_table->hwaddr;
713	etr_buf->mode = ETR_MODE_ETR_SG;
714	etr_buf->private = etr_table;
715	return 0;
716	}
717
718	static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
719	{
720	struct etr_sg_table *etr_table = etr_buf->private;
721
722	if (etr_table) {
723	tmc_free_sg_table(etr_table->sg_table);
724	kfree(objp: etr_table);
725	}
726	}
727
728	static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
729	size_t len, char **bufpp)
730	{
731	struct etr_sg_table *etr_table = etr_buf->private;
732
733	return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
734	}
735
736	static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
737	{
738	long r_offset, w_offset;
739	struct etr_sg_table *etr_table = etr_buf->private;
740	struct tmc_sg_table *table = etr_table->sg_table;
741
742	/* Convert hw address to offset in the buffer */
743	r_offset = tmc_sg_get_data_page_offset(sg_table: table, addr: rrp);
744	if (r_offset < 0) {
745	dev_warn(table->dev,
746	"Unable to map RRP %llx to offset\n", rrp);
747	etr_buf->len = 0;
748	return;
749	}
750
751	w_offset = tmc_sg_get_data_page_offset(sg_table: table, addr: rwp);
752	if (w_offset < 0) {
753	dev_warn(table->dev,
754	"Unable to map RWP %llx to offset\n", rwp);
755	etr_buf->len = 0;
756	return;
757	}
758
759	etr_buf->offset = r_offset;
760	if (etr_buf->full)
761	etr_buf->len = etr_buf->size;
762	else
763	etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
764	w_offset - r_offset;
765	tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
766	}
767
768	static const struct etr_buf_operations etr_sg_buf_ops = {
769	.alloc = tmc_etr_alloc_sg_buf,
770	.free = tmc_etr_free_sg_buf,
771	.sync = tmc_etr_sync_sg_buf,
772	.get_data = tmc_etr_get_data_sg_buf,
773	};
774
775	/*
776	* TMC ETR could be connected to a CATU device, which can provide address
777	* translation service. This is represented by the Output port of the TMC
778	* (ETR) connected to the input port of the CATU.
779	*
780	* Returns : coresight_device ptr for the CATU device if a CATU is found.
781	* : NULL otherwise.
782	*/
783	struct coresight_device *
784	tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
785	{
786	struct coresight_device *etr = drvdata->csdev;
787	union coresight_dev_subtype catu_subtype = {
788	.helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CATU
789	};
790
791	if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
792	return NULL;
793
794	return coresight_find_output_type(pdata: etr->pdata, type: CORESIGHT_DEV_TYPE_HELPER,
795	subtype: catu_subtype);
796	}
797	EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
798
799	static const struct etr_buf_operations *etr_buf_ops[] = {
800	[ETR_MODE_FLAT] = &etr_flat_buf_ops,
801	[ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
802	[ETR_MODE_CATU] = NULL,
803	};
804
805	void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
806	{
807	etr_buf_ops[ETR_MODE_CATU] = catu;
808	}
809	EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
810
811	void tmc_etr_remove_catu_ops(void)
812	{
813	etr_buf_ops[ETR_MODE_CATU] = NULL;
814	}
815	EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
816
817	static inline int tmc_etr_mode_alloc_buf(int mode,
818	struct tmc_drvdata *drvdata,
819	struct etr_buf *etr_buf, int node,
820	void **pages)
821	{
822	int rc = -EINVAL;
823
824	switch (mode) {
825	case ETR_MODE_FLAT:
826	case ETR_MODE_ETR_SG:
827	case ETR_MODE_CATU:
828	if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
829	rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
830	node, pages);
831	if (!rc)
832	etr_buf->ops = etr_buf_ops[mode];
833	return rc;
834	default:
835	return -EINVAL;
836	}
837	}
838
839	static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw)
840	{
841	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: dev->parent);
842
843	buf_hw->has_iommu = iommu_get_domain_for_dev(dev: dev->parent);
844	buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
845	buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata);
846	}
847
848	static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size)
849	{
850	bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg;
851
852	return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M;
853	}
854
855	/*
856	* tmc_alloc_etr_buf: Allocate a buffer use by ETR.
857	* @drvdata : ETR device details.
858	* @size : size of the requested buffer.
859	* @flags : Required properties for the buffer.
860	* @node : Node for memory allocations.
861	* @pages : An optional list of pages.
862	*/
863	static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
864	ssize_t size, int flags,
865	int node, void **pages)
866	{
867	int rc = -ENOMEM;
868	struct etr_buf *etr_buf;
869	struct etr_buf_hw buf_hw;
870	struct device *dev = &drvdata->csdev->dev;
871
872	get_etr_buf_hw(dev, buf_hw: &buf_hw);
873	etr_buf = kzalloc(size: sizeof(*etr_buf), GFP_KERNEL);
874	if (!etr_buf)
875	return ERR_PTR(error: -ENOMEM);
876
877	etr_buf->size = size;
878
879	/* If there is user directive for buffer mode, try that first */
880	if (drvdata->etr_mode != ETR_MODE_AUTO)
881	rc = tmc_etr_mode_alloc_buf(mode: drvdata->etr_mode, drvdata,
882	etr_buf, node, pages);
883
884	/*
885	* If we have to use an existing list of pages, we cannot reliably
886	* use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
887	* we use the contiguous DMA memory if at least one of the following
888	* conditions is true:
889	* a) The ETR cannot use Scatter-Gather.
890	* b) we have a backing IOMMU
891	* c) The requested memory size is smaller (< 1M).
892	*
893	* Fallback to available mechanisms.
894	*
895	*/
896	if (rc && !pages && etr_can_use_flat_mode(buf_hw: &buf_hw, etr_buf_size: size))
897	rc = tmc_etr_mode_alloc_buf(mode: ETR_MODE_FLAT, drvdata,
898	etr_buf, node, pages);
899	if (rc && buf_hw.has_etr_sg)
900	rc = tmc_etr_mode_alloc_buf(mode: ETR_MODE_ETR_SG, drvdata,
901	etr_buf, node, pages);
902	if (rc && buf_hw.has_catu)
903	rc = tmc_etr_mode_alloc_buf(mode: ETR_MODE_CATU, drvdata,
904	etr_buf, node, pages);
905	if (rc) {
906	kfree(objp: etr_buf);
907	return ERR_PTR(error: rc);
908	}
909
910	refcount_set(r: &etr_buf->refcount, n: 1);
911	dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
912	(unsigned long)size >> 10, etr_buf->mode);
913	return etr_buf;
914	}
915
916	static void tmc_free_etr_buf(struct etr_buf *etr_buf)
917	{
918	WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
919	etr_buf->ops->free(etr_buf);
920	kfree(objp: etr_buf);
921	}
922
923	/*
924	* tmc_etr_buf_get_data: Get the pointer the trace data at @offset
925	* with a maximum of @len bytes.
926	* Returns: The size of the linear data available @pos, with *bufpp
927	* updated to point to the buffer.
928	*/
929	static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
930	u64 offset, size_t len, char **bufpp)
931	{
932	/* Adjust the length to limit this transaction to end of buffer */
933	len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
934
935	return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
936	}
937
938	static inline s64
939	tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
940	{
941	ssize_t len;
942	char *bufp;
943
944	len = tmc_etr_buf_get_data(etr_buf, offset,
945	CORESIGHT_BARRIER_PKT_SIZE, bufpp: &bufp);
946	if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE))
947	return -EINVAL;
948	coresight_insert_barrier_packet(buf: bufp);
949	return offset + CORESIGHT_BARRIER_PKT_SIZE;
950	}
951
952	/*
953	* tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
954	* Makes sure the trace data is synced to the memory for consumption.
955	* @etr_buf->offset will hold the offset to the beginning of the trace data
956	* within the buffer, with @etr_buf->len bytes to consume.
957	*/
958	static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
959	{
960	struct etr_buf *etr_buf = drvdata->etr_buf;
961	u64 rrp, rwp;
962	u32 status;
963
964	rrp = tmc_read_rrp(drvdata);
965	rwp = tmc_read_rwp(drvdata);
966	status = readl_relaxed(drvdata->base + TMC_STS);
967
968	/*
969	* If there were memory errors in the session, truncate the
970	* buffer.
971	*/
972	if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
973	dev_dbg(&drvdata->csdev->dev,
974	"tmc memory error detected, truncating buffer\n");
975	etr_buf->len = 0;
976	etr_buf->full = false;
977	return;
978	}
979
980	etr_buf->full = !!(status & TMC_STS_FULL);
981
982	WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
983
984	etr_buf->ops->sync(etr_buf, rrp, rwp);
985	}
986
987	static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
988	{
989	u32 axictl, sts;
990	struct etr_buf *etr_buf = drvdata->etr_buf;
991	int rc = 0;
992
993	CS_UNLOCK(addr: drvdata->base);
994
995	/* Wait for TMCSReady bit to be set */
996	rc = tmc_wait_for_tmcready(drvdata);
997	if (rc) {
998	dev_err(&drvdata->csdev->dev,
999	"Failed to enable : TMC not ready\n");
1000	CS_LOCK(addr: drvdata->base);
1001	return rc;
1002	}
1003
1004	writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
1005	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
1006
1007	axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
1008	axictl &= ~TMC_AXICTL_CLEAR_MASK;
1009	axictl |= TMC_AXICTL_PROT_CTL_B1;
1010	axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
1011	axictl |= TMC_AXICTL_AXCACHE_OS;
1012
1013	if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
1014	axictl &= ~TMC_AXICTL_ARCACHE_MASK;
1015	axictl |= TMC_AXICTL_ARCACHE_OS;
1016	}
1017
1018	if (etr_buf->mode == ETR_MODE_ETR_SG)
1019	axictl |= TMC_AXICTL_SCT_GAT_MODE;
1020
1021	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1022	tmc_write_dba(drvdata, val: etr_buf->hwaddr);
1023	/*
1024	* If the TMC pointers must be programmed before the session,
1025	* we have to set it properly (i.e, RRP/RWP to base address and
1026	* STS to "not full").
1027	*/
1028	if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1029	tmc_write_rrp(drvdata, val: etr_buf->hwaddr);
1030	tmc_write_rwp(drvdata, val: etr_buf->hwaddr);
1031	sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1032	writel_relaxed(sts, drvdata->base + TMC_STS);
1033	}
1034
1035	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
1036	TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
1037	TMC_FFCR_TRIGON_TRIGIN,
1038	drvdata->base + TMC_FFCR);
1039	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1040	tmc_enable_hw(drvdata);
1041
1042	CS_LOCK(addr: drvdata->base);
1043	return rc;
1044	}
1045
1046	static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1047	struct etr_buf *etr_buf)
1048	{
1049	int rc;
1050
1051	/* Callers should provide an appropriate buffer for use */
1052	if (WARN_ON(!etr_buf))
1053	return -EINVAL;
1054
1055	if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1056	WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1057	return -EINVAL;
1058
1059	if (WARN_ON(drvdata->etr_buf))
1060	return -EBUSY;
1061
1062	rc = coresight_claim_device(csdev: drvdata->csdev);
1063	if (!rc) {
1064	drvdata->etr_buf = etr_buf;
1065	rc = __tmc_etr_enable_hw(drvdata);
1066	if (rc) {
1067	drvdata->etr_buf = NULL;
1068	coresight_disclaim_device(csdev: drvdata->csdev);
1069	}
1070	}
1071
1072	return rc;
1073	}
1074
1075	/*
1076	* Return the available trace data in the buffer (starts at etr_buf->offset,
1077	* limited by etr_buf->len) from @pos, with a maximum limit of @len,
1078	* also updating the @bufpp on where to find it. Since the trace data
1079	* starts at anywhere in the buffer, depending on the RRP, we adjust the
1080	* @len returned to handle buffer wrapping around.
1081	*
1082	* We are protected here by drvdata->reading != 0, which ensures the
1083	* sysfs_buf stays alive.
1084	*/
1085	ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1086	loff_t pos, size_t len, char **bufpp)
1087	{
1088	s64 offset;
1089	ssize_t actual = len;
1090	struct etr_buf *etr_buf = drvdata->sysfs_buf;
1091
1092	if (pos + actual > etr_buf->len)
1093	actual = etr_buf->len - pos;
1094	if (actual <= 0)
1095	return actual;
1096
1097	/* Compute the offset from which we read the data */
1098	offset = etr_buf->offset + pos;
1099	if (offset >= etr_buf->size)
1100	offset -= etr_buf->size;
1101	return tmc_etr_buf_get_data(etr_buf, offset, len: actual, bufpp);
1102	}
1103
1104	static struct etr_buf *
1105	tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1106	{
1107	return tmc_alloc_etr_buf(drvdata, size: drvdata->size,
1108	flags: 0, cpu_to_node(cpu: 0), NULL);
1109	}
1110
1111	static void
1112	tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1113	{
1114	if (buf)
1115	tmc_free_etr_buf(etr_buf: buf);
1116	}
1117
1118	static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1119	{
1120	struct etr_buf *etr_buf = drvdata->etr_buf;
1121
1122	if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1123	tmc_etr_free_sysfs_buf(buf: drvdata->sysfs_buf);
1124	drvdata->sysfs_buf = NULL;
1125	} else {
1126	tmc_sync_etr_buf(drvdata);
1127	/*
1128	* Insert barrier packets at the beginning, if there was
1129	* an overflow.
1130	*/
1131	if (etr_buf->full)
1132	tmc_etr_buf_insert_barrier_packet(etr_buf,
1133	offset: etr_buf->offset);
1134	}
1135	}
1136
1137	static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1138	{
1139	CS_UNLOCK(addr: drvdata->base);
1140
1141	tmc_flush_and_stop(drvdata);
1142	/*
1143	* When operating in sysFS mode the content of the buffer needs to be
1144	* read before the TMC is disabled.
1145	*/
1146	if (coresight_get_mode(csdev: drvdata->csdev) == CS_MODE_SYSFS)
1147	tmc_etr_sync_sysfs_buf(drvdata);
1148
1149	tmc_disable_hw(drvdata);
1150
1151	CS_LOCK(addr: drvdata->base);
1152
1153	}
1154
1155	void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1156	{
1157	__tmc_etr_disable_hw(drvdata);
1158	coresight_disclaim_device(csdev: drvdata->csdev);
1159	/* Reset the ETR buf used by hardware */
1160	drvdata->etr_buf = NULL;
1161	}
1162
1163	static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1164	{
1165	int ret = 0;
1166	unsigned long flags;
1167	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1168	struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1169
1170	/*
1171	* If we are enabling the ETR from disabled state, we need to make
1172	* sure we have a buffer with the right size. The etr_buf is not reset
1173	* immediately after we stop the tracing in SYSFS mode as we wait for
1174	* the user to collect the data. We may be able to reuse the existing
1175	* buffer, provided the size matches. Any allocation has to be done
1176	* with the lock released.
1177	*/
1178	spin_lock_irqsave(&drvdata->spinlock, flags);
1179	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1180	if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1181	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1182
1183	/* Allocate memory with the locks released */
1184	free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1185	if (IS_ERR(ptr: new_buf))
1186	return new_buf;
1187
1188	/* Let's try again */
1189	spin_lock_irqsave(&drvdata->spinlock, flags);
1190	}
1191
1192	if (drvdata->reading || coresight_get_mode(csdev) == CS_MODE_PERF) {
1193	ret = -EBUSY;
1194	goto out;
1195	}
1196
1197	/*
1198	* If we don't have a buffer or it doesn't match the requested size,
1199	* use the buffer allocated above. Otherwise reuse the existing buffer.
1200	*/
1201	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1202	if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1203	free_buf = sysfs_buf;
1204	drvdata->sysfs_buf = new_buf;
1205	}
1206
1207	out:
1208	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1209
1210	/* Free memory outside the spinlock if need be */
1211	if (free_buf)
1212	tmc_etr_free_sysfs_buf(buf: free_buf);
1213	return ret ? ERR_PTR(error: ret) : drvdata->sysfs_buf;
1214	}
1215
1216	static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1217	{
1218	int ret = 0;
1219	unsigned long flags;
1220	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1221	struct etr_buf *sysfs_buf = tmc_etr_get_sysfs_buffer(csdev);
1222
1223	if (IS_ERR(ptr: sysfs_buf))
1224	return PTR_ERR(ptr: sysfs_buf);
1225
1226	spin_lock_irqsave(&drvdata->spinlock, flags);
1227
1228	/*
1229	* In sysFS mode we can have multiple writers per sink. Since this
1230	* sink is already enabled no memory is needed and the HW need not be
1231	* touched, even if the buffer size has changed.
1232	*/
1233	if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1234	csdev->refcnt++;
1235	goto out;
1236	}
1237
1238	ret = tmc_etr_enable_hw(drvdata, etr_buf: sysfs_buf);
1239	if (!ret) {
1240	coresight_set_mode(csdev, new_mode: CS_MODE_SYSFS);
1241	csdev->refcnt++;
1242	}
1243
1244	out:
1245	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1246
1247	if (!ret)
1248	dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1249
1250	return ret;
1251	}
1252
1253	struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,
1254	enum cs_mode mode, void *data)
1255	{
1256	struct perf_output_handle *handle = data;
1257	struct etr_perf_buffer *etr_perf;
1258
1259	switch (mode) {
1260	case CS_MODE_SYSFS:
1261	return tmc_etr_get_sysfs_buffer(csdev);
1262	case CS_MODE_PERF:
1263	etr_perf = etm_perf_sink_config(handle);
1264	if (WARN_ON(!etr_perf || !etr_perf->etr_buf))
1265	return ERR_PTR(error: -EINVAL);
1266	return etr_perf->etr_buf;
1267	default:
1268	return ERR_PTR(error: -EINVAL);
1269	}
1270	}
1271	EXPORT_SYMBOL_GPL(tmc_etr_get_buffer);
1272
1273	/*
1274	* alloc_etr_buf: Allocate ETR buffer for use by perf.
1275	* The size of the hardware buffer is dependent on the size configured
1276	* via sysfs and the perf ring buffer size. We prefer to allocate the
1277	* largest possible size, scaling down the size by half until it
1278	* reaches a minimum limit (1M), beyond which we give up.
1279	*/
1280	static struct etr_buf *
1281	alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1282	int nr_pages, void **pages, bool snapshot)
1283	{
1284	int node;
1285	struct etr_buf *etr_buf;
1286	unsigned long size;
1287
1288	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(cpu: event->cpu);
1289	/*
1290	* Try to match the perf ring buffer size if it is larger
1291	* than the size requested via sysfs.
1292	*/
1293	if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1294	etr_buf = tmc_alloc_etr_buf(drvdata, size: ((ssize_t)nr_pages << PAGE_SHIFT),
1295	flags: 0, node, NULL);
1296	if (!IS_ERR(ptr: etr_buf))
1297	goto done;
1298	}
1299
1300	/*
1301	* Else switch to configured size for this ETR
1302	* and scale down until we hit the minimum limit.
1303	*/
1304	size = drvdata->size;
1305	do {
1306	etr_buf = tmc_alloc_etr_buf(drvdata, size, flags: 0, node, NULL);
1307	if (!IS_ERR(ptr: etr_buf))
1308	goto done;
1309	size /= 2;
1310	} while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1311
1312	return ERR_PTR(error: -ENOMEM);
1313
1314	done:
1315	return etr_buf;
1316	}
1317
1318	static struct etr_buf *
1319	get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1320	struct perf_event *event, int nr_pages,
1321	void **pages, bool snapshot)
1322	{
1323	int ret;
1324	pid_t pid = task_pid_nr(tsk: event->owner);
1325	struct etr_buf *etr_buf;
1326
1327	retry:
1328	/*
1329	* An etr_perf_buffer is associated with an event and holds a reference
1330	* to the AUX ring buffer that was created for that event. In CPU-wide
1331	* N:1 mode multiple events (one per CPU), each with its own AUX ring
1332	* buffer, share a sink. As such an etr_perf_buffer is created for each
1333	* event but a single etr_buf associated with the ETR is shared between
1334	* them. The last event in a trace session will copy the content of the
1335	* etr_buf to its AUX ring buffer. Ring buffer associated to other
1336	* events are simply not used an freed as events are destoyed. We still
1337	* need to allocate a ring buffer for each event since we don't know
1338	* which event will be last.
1339	*/
1340
1341	/*
1342	* The first thing to do here is check if an etr_buf has already been
1343	* allocated for this session. If so it is shared with this event,
1344	* otherwise it is created.
1345	*/
1346	mutex_lock(&drvdata->idr_mutex);
1347	etr_buf = idr_find(&drvdata->idr, id: pid);
1348	if (etr_buf) {
1349	refcount_inc(r: &etr_buf->refcount);
1350	mutex_unlock(lock: &drvdata->idr_mutex);
1351	return etr_buf;
1352	}
1353
1354	/* If we made it here no buffer has been allocated, do so now. */
1355	mutex_unlock(lock: &drvdata->idr_mutex);
1356
1357	etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1358	if (IS_ERR(ptr: etr_buf))
1359	return etr_buf;
1360
1361	/* Now that we have a buffer, add it to the IDR. */
1362	mutex_lock(&drvdata->idr_mutex);
1363	ret = idr_alloc(&drvdata->idr, ptr: etr_buf, start: pid, end: pid + 1, GFP_KERNEL);
1364	mutex_unlock(lock: &drvdata->idr_mutex);
1365
1366	/* Another event with this session ID has allocated this buffer. */
1367	if (ret == -ENOSPC) {
1368	tmc_free_etr_buf(etr_buf);
1369	goto retry;
1370	}
1371
1372	/* The IDR can't allocate room for a new session, abandon ship. */
1373	if (ret == -ENOMEM) {
1374	tmc_free_etr_buf(etr_buf);
1375	return ERR_PTR(error: ret);
1376	}
1377
1378
1379	return etr_buf;
1380	}
1381
1382	static struct etr_buf *
1383	get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1384	struct perf_event *event, int nr_pages,
1385	void **pages, bool snapshot)
1386	{
1387	/*
1388	* In per-thread mode the etr_buf isn't shared, so just go ahead
1389	* with memory allocation.
1390	*/
1391	return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1392	}
1393
1394	static struct etr_buf *
1395	get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1396	int nr_pages, void **pages, bool snapshot)
1397	{
1398	if (event->cpu == -1)
1399	return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1400	pages, snapshot);
1401
1402	return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1403	pages, snapshot);
1404	}
1405
1406	static struct etr_perf_buffer *
1407	tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1408	int nr_pages, void **pages, bool snapshot)
1409	{
1410	int node;
1411	struct etr_buf *etr_buf;
1412	struct etr_perf_buffer *etr_perf;
1413
1414	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(cpu: event->cpu);
1415
1416	etr_perf = kzalloc_node(size: sizeof(*etr_perf), GFP_KERNEL, node);
1417	if (!etr_perf)
1418	return ERR_PTR(error: -ENOMEM);
1419
1420	etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1421	if (!IS_ERR(ptr: etr_buf))
1422	goto done;
1423
1424	kfree(objp: etr_perf);
1425	return ERR_PTR(error: -ENOMEM);
1426
1427	done:
1428	/*
1429	* Keep a reference to the ETR this buffer has been allocated for
1430	* in order to have access to the IDR in tmc_free_etr_buffer().
1431	*/
1432	etr_perf->drvdata = drvdata;
1433	etr_perf->etr_buf = etr_buf;
1434
1435	return etr_perf;
1436	}
1437
1438
1439	static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1440	struct perf_event *event, void **pages,
1441	int nr_pages, bool snapshot)
1442	{
1443	struct etr_perf_buffer *etr_perf;
1444	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1445
1446	etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1447	nr_pages, pages, snapshot);
1448	if (IS_ERR(ptr: etr_perf)) {
1449	dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1450	return NULL;
1451	}
1452
1453	etr_perf->pid = task_pid_nr(tsk: event->owner);
1454	etr_perf->snapshot = snapshot;
1455	etr_perf->nr_pages = nr_pages;
1456	etr_perf->pages = pages;
1457
1458	return etr_perf;
1459	}
1460
1461	static void tmc_free_etr_buffer(void *config)
1462	{
1463	struct etr_perf_buffer *etr_perf = config;
1464	struct tmc_drvdata *drvdata = etr_perf->drvdata;
1465	struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1466
1467	if (!etr_buf)
1468	goto free_etr_perf_buffer;
1469
1470	mutex_lock(&drvdata->idr_mutex);
1471	/* If we are not the last one to use the buffer, don't touch it. */
1472	if (!refcount_dec_and_test(r: &etr_buf->refcount)) {
1473	mutex_unlock(lock: &drvdata->idr_mutex);
1474	goto free_etr_perf_buffer;
1475	}
1476
1477	/* We are the last one, remove from the IDR and free the buffer. */
1478	buf = idr_remove(&drvdata->idr, id: etr_perf->pid);
1479	mutex_unlock(lock: &drvdata->idr_mutex);
1480
1481	/*
1482	* Something went very wrong if the buffer associated with this ID
1483	* is not the same in the IDR. Leak to avoid use after free.
1484	*/
1485	if (buf && WARN_ON(buf != etr_buf))
1486	goto free_etr_perf_buffer;
1487
1488	tmc_free_etr_buf(etr_buf: etr_perf->etr_buf);
1489
1490	free_etr_perf_buffer:
1491	kfree(objp: etr_perf);
1492	}
1493
1494	/*
1495	* tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1496	* buffer to the perf ring buffer.
1497	*/
1498	static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1499	unsigned long head,
1500	unsigned long src_offset,
1501	unsigned long to_copy)
1502	{
1503	long bytes;
1504	long pg_idx, pg_offset;
1505	char **dst_pages, *src_buf;
1506	struct etr_buf *etr_buf = etr_perf->etr_buf;
1507
1508	head = PERF_IDX2OFF(head, etr_perf);
1509	pg_idx = head >> PAGE_SHIFT;
1510	pg_offset = head & (PAGE_SIZE - 1);
1511	dst_pages = (char **)etr_perf->pages;
1512
1513	while (to_copy > 0) {
1514	/*
1515	* In one iteration, we can copy minimum of :
1516	* 1) what is available in the source buffer,
1517	* 2) what is available in the source buffer, before it
1518	* wraps around.
1519	* 3) what is available in the destination page.
1520	* in one iteration.
1521	*/
1522	if (src_offset >= etr_buf->size)
1523	src_offset -= etr_buf->size;
1524	bytes = tmc_etr_buf_get_data(etr_buf, offset: src_offset, len: to_copy,
1525	bufpp: &src_buf);
1526	if (WARN_ON_ONCE(bytes <= 0))
1527	break;
1528	bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1529
1530	memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1531
1532	to_copy -= bytes;
1533
1534	/* Move destination pointers */
1535	pg_offset += bytes;
1536	if (pg_offset == PAGE_SIZE) {
1537	pg_offset = 0;
1538	if (++pg_idx == etr_perf->nr_pages)
1539	pg_idx = 0;
1540	}
1541
1542	/* Move source pointers */
1543	src_offset += bytes;
1544	}
1545	}
1546
1547	/*
1548	* tmc_update_etr_buffer : Update the perf ring buffer with the
1549	* available trace data. We use software double buffering at the moment.
1550	*
1551	* TODO: Add support for reusing the perf ring buffer.
1552	*/
1553	static unsigned long
1554	tmc_update_etr_buffer(struct coresight_device *csdev,
1555	struct perf_output_handle *handle,
1556	void *config)
1557	{
1558	bool lost = false;
1559	unsigned long flags, offset, size = 0;
1560	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1561	struct etr_perf_buffer *etr_perf = config;
1562	struct etr_buf *etr_buf = etr_perf->etr_buf;
1563
1564	spin_lock_irqsave(&drvdata->spinlock, flags);
1565
1566	/* Don't do anything if another tracer is using this sink */
1567	if (csdev->refcnt != 1) {
1568	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1569	goto out;
1570	}
1571
1572	if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1573	lost = true;
1574	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1575	goto out;
1576	}
1577
1578	CS_UNLOCK(addr: drvdata->base);
1579
1580	tmc_flush_and_stop(drvdata);
1581	tmc_sync_etr_buf(drvdata);
1582
1583	CS_LOCK(addr: drvdata->base);
1584	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1585
1586	lost = etr_buf->full;
1587	offset = etr_buf->offset;
1588	size = etr_buf->len;
1589
1590	/*
1591	* The ETR buffer may be bigger than the space available in the
1592	* perf ring buffer (handle->size). If so advance the offset so that we
1593	* get the latest trace data. In snapshot mode none of that matters
1594	* since we are expected to clobber stale data in favour of the latest
1595	* traces.
1596	*/
1597	if (!etr_perf->snapshot && size > handle->size) {
1598	u32 mask = tmc_get_memwidth_mask(drvdata);
1599
1600	/*
1601	* Make sure the new size is aligned in accordance with the
1602	* requirement explained in function tmc_get_memwidth_mask().
1603	*/
1604	size = handle->size & mask;
1605	offset = etr_buf->offset + etr_buf->len - size;
1606
1607	if (offset >= etr_buf->size)
1608	offset -= etr_buf->size;
1609	lost = true;
1610	}
1611
1612	/* Insert barrier packets at the beginning, if there was an overflow */
1613	if (lost)
1614	tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1615	tmc_etr_sync_perf_buffer(etr_perf, head: handle->head, src_offset: offset, to_copy: size);
1616
1617	/*
1618	* In snapshot mode we simply increment the head by the number of byte
1619	* that were written. User space will figure out how many bytes to get
1620	* from the AUX buffer based on the position of the head.
1621	*/
1622	if (etr_perf->snapshot)
1623	handle->head += size;
1624
1625	/*
1626	* Ensure that the AUX trace data is visible before the aux_head
1627	* is updated via perf_aux_output_end(), as expected by the
1628	* perf ring buffer.
1629	*/
1630	smp_wmb();
1631
1632	out:
1633	/*
1634	* Don't set the TRUNCATED flag in snapshot mode because 1) the
1635	* captured buffer is expected to be truncated and 2) a full buffer
1636	* prevents the event from being re-enabled by the perf core,
1637	* resulting in stale data being send to user space.
1638	*/
1639	if (!etr_perf->snapshot && lost)
1640	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1641	return size;
1642	}
1643
1644	static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1645	{
1646	int rc = 0;
1647	pid_t pid;
1648	unsigned long flags;
1649	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1650	struct perf_output_handle *handle = data;
1651	struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1652
1653	spin_lock_irqsave(&drvdata->spinlock, flags);
1654	/* Don't use this sink if it is already claimed by sysFS */
1655	if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
1656	rc = -EBUSY;
1657	goto unlock_out;
1658	}
1659
1660	if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1661	rc = -EINVAL;
1662	goto unlock_out;
1663	}
1664
1665	/* Get a handle on the pid of the process to monitor */
1666	pid = etr_perf->pid;
1667
1668	/* Do not proceed if this device is associated with another session */
1669	if (drvdata->pid != -1 && drvdata->pid != pid) {
1670	rc = -EBUSY;
1671	goto unlock_out;
1672	}
1673
1674	/*
1675	* No HW configuration is needed if the sink is already in
1676	* use for this session.
1677	*/
1678	if (drvdata->pid == pid) {
1679	csdev->refcnt++;
1680	goto unlock_out;
1681	}
1682
1683	rc = tmc_etr_enable_hw(drvdata, etr_buf: etr_perf->etr_buf);
1684	if (!rc) {
1685	/* Associate with monitored process. */
1686	drvdata->pid = pid;
1687	coresight_set_mode(csdev, new_mode: CS_MODE_PERF);
1688	drvdata->perf_buf = etr_perf->etr_buf;
1689	csdev->refcnt++;
1690	}
1691
1692	unlock_out:
1693	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1694	return rc;
1695	}
1696
1697	static int tmc_enable_etr_sink(struct coresight_device *csdev,
1698	enum cs_mode mode, void *data)
1699	{
1700	switch (mode) {
1701	case CS_MODE_SYSFS:
1702	return tmc_enable_etr_sink_sysfs(csdev);
1703	case CS_MODE_PERF:
1704	return tmc_enable_etr_sink_perf(csdev, data);
1705	default:
1706	return -EINVAL;
1707	}
1708	}
1709
1710	static int tmc_disable_etr_sink(struct coresight_device *csdev)
1711	{
1712	unsigned long flags;
1713	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1714
1715	spin_lock_irqsave(&drvdata->spinlock, flags);
1716
1717	if (drvdata->reading) {
1718	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1719	return -EBUSY;
1720	}
1721
1722	csdev->refcnt--;
1723	if (csdev->refcnt) {
1724	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1725	return -EBUSY;
1726	}
1727
1728	/* Complain if we (somehow) got out of sync */
1729	WARN_ON_ONCE(coresight_get_mode(csdev) == CS_MODE_DISABLED);
1730	tmc_etr_disable_hw(drvdata);
1731	/* Dissociate from monitored process. */
1732	drvdata->pid = -1;
1733	coresight_set_mode(csdev, new_mode: CS_MODE_DISABLED);
1734	/* Reset perf specific data */
1735	drvdata->perf_buf = NULL;
1736
1737	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1738
1739	dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1740	return 0;
1741	}
1742
1743	static const struct coresight_ops_sink tmc_etr_sink_ops = {
1744	.enable = tmc_enable_etr_sink,
1745	.disable = tmc_disable_etr_sink,
1746	.alloc_buffer = tmc_alloc_etr_buffer,
1747	.update_buffer = tmc_update_etr_buffer,
1748	.free_buffer = tmc_free_etr_buffer,
1749	};
1750
1751	const struct coresight_ops tmc_etr_cs_ops = {
1752	.sink_ops = &tmc_etr_sink_ops,
1753	};
1754
1755	int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1756	{
1757	int ret = 0;
1758	unsigned long flags;
1759
1760	/* config types are set a boot time and never change */
1761	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1762	return -EINVAL;
1763
1764	spin_lock_irqsave(&drvdata->spinlock, flags);
1765	if (drvdata->reading) {
1766	ret = -EBUSY;
1767	goto out;
1768	}
1769
1770	/*
1771	* We can safely allow reads even if the ETR is operating in PERF mode,
1772	* since the sysfs session is captured in mode specific data.
1773	* If drvdata::sysfs_data is NULL the trace data has been read already.
1774	*/
1775	if (!drvdata->sysfs_buf) {
1776	ret = -EINVAL;
1777	goto out;
1778	}
1779
1780	/* Disable the TMC if we are trying to read from a running session. */
1781	if (coresight_get_mode(csdev: drvdata->csdev) == CS_MODE_SYSFS)
1782	__tmc_etr_disable_hw(drvdata);
1783
1784	drvdata->reading = true;
1785	out:
1786	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1787
1788	return ret;
1789	}
1790
1791	int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1792	{
1793	unsigned long flags;
1794	struct etr_buf *sysfs_buf = NULL;
1795
1796	/* config types are set a boot time and never change */
1797	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1798	return -EINVAL;
1799
1800	spin_lock_irqsave(&drvdata->spinlock, flags);
1801
1802	/* RE-enable the TMC if need be */
1803	if (coresight_get_mode(csdev: drvdata->csdev) == CS_MODE_SYSFS) {
1804	/*
1805	* The trace run will continue with the same allocated trace
1806	* buffer. Since the tracer is still enabled drvdata::buf can't
1807	* be NULL.
1808	*/
1809	__tmc_etr_enable_hw(drvdata);
1810	} else {
1811	/*
1812	* The ETR is not tracing and the buffer was just read.
1813	* As such prepare to free the trace buffer.
1814	*/
1815	sysfs_buf = drvdata->sysfs_buf;
1816	drvdata->sysfs_buf = NULL;
1817	}
1818
1819	drvdata->reading = false;
1820	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1821
1822	/* Free allocated memory out side of the spinlock */
1823	if (sysfs_buf)
1824	tmc_etr_free_sysfs_buf(buf: sysfs_buf);
1825
1826	return 0;
1827	}
1828
1829	static const char *const buf_modes_str[] = {
1830	[ETR_MODE_FLAT] = "flat",
1831	[ETR_MODE_ETR_SG] = "tmc-sg",
1832	[ETR_MODE_CATU] = "catu",
1833	[ETR_MODE_AUTO] = "auto",
1834	};
1835
1836	static ssize_t buf_modes_available_show(struct device *dev,
1837	struct device_attribute *attr, char *buf)
1838	{
1839	struct etr_buf_hw buf_hw;
1840	ssize_t size = 0;
1841
1842	get_etr_buf_hw(dev, buf_hw: &buf_hw);
1843	size += sysfs_emit(buf, fmt: "%s ", buf_modes_str[ETR_MODE_AUTO]);
1844	size += sysfs_emit_at(buf, at: size, fmt: "%s ", buf_modes_str[ETR_MODE_FLAT]);
1845	if (buf_hw.has_etr_sg)
1846	size += sysfs_emit_at(buf, at: size, fmt: "%s ", buf_modes_str[ETR_MODE_ETR_SG]);
1847
1848	if (buf_hw.has_catu)
1849	size += sysfs_emit_at(buf, at: size, fmt: "%s ", buf_modes_str[ETR_MODE_CATU]);
1850
1851	size += sysfs_emit_at(buf, at: size, fmt: "\n");
1852	return size;
1853	}
1854	static DEVICE_ATTR_RO(buf_modes_available);
1855
1856	static ssize_t buf_mode_preferred_show(struct device *dev,
1857	struct device_attribute *attr, char *buf)
1858	{
1859	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: dev->parent);
1860
1861	return sysfs_emit(buf, fmt: "%s\n", buf_modes_str[drvdata->etr_mode]);
1862	}
1863
1864	static ssize_t buf_mode_preferred_store(struct device *dev,
1865	struct device_attribute *attr,
1866	const char *buf, size_t size)
1867	{
1868	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: dev->parent);
1869	struct etr_buf_hw buf_hw;
1870
1871	get_etr_buf_hw(dev, buf_hw: &buf_hw);
1872	if (sysfs_streq(s1: buf, s2: buf_modes_str[ETR_MODE_FLAT]))
1873	drvdata->etr_mode = ETR_MODE_FLAT;
1874	else if (sysfs_streq(s1: buf, s2: buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg)
1875	drvdata->etr_mode = ETR_MODE_ETR_SG;
1876	else if (sysfs_streq(s1: buf, s2: buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu)
1877	drvdata->etr_mode = ETR_MODE_CATU;
1878	else if (sysfs_streq(s1: buf, s2: buf_modes_str[ETR_MODE_AUTO]))
1879	drvdata->etr_mode = ETR_MODE_AUTO;
1880	else
1881	return -EINVAL;
1882	return size;
1883	}
1884	static DEVICE_ATTR_RW(buf_mode_preferred);
1885
1886	static struct attribute *coresight_etr_attrs[] = {
1887	&dev_attr_buf_modes_available.attr,
1888	&dev_attr_buf_mode_preferred.attr,
1889	NULL,
1890	};
1891
1892	const struct attribute_group coresight_etr_group = {
1893	.attrs = coresight_etr_attrs,
1894	};
1895