1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */
3
4	/* TSN endpoint Ethernet MAC driver
5	*
6	* The TSN endpoint Ethernet MAC is a FPGA based network device for real-time
7	* communication. It is designed for endpoints within TSN (Time Sensitive
8	* Networking) networks; e.g., for PLCs in the industrial automation case.
9	*
10	* It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used
11	* by the driver.
12	*
13	* More information can be found here:
14	* - www.embedded-experts.at/tsn
15	* - www.engleder-embedded.com
16	*/
17
18	#include "tsnep.h"
19	#include "tsnep_hw.h"
20
21	#include <linux/module.h>
22	#include <linux/of.h>
23	#include <linux/of_net.h>
24	#include <linux/of_mdio.h>
25	#include <linux/interrupt.h>
26	#include <linux/etherdevice.h>
27	#include <linux/phy.h>
28	#include <linux/iopoll.h>
29	#include <linux/bpf.h>
30	#include <linux/bpf_trace.h>
31	#include <net/page_pool/helpers.h>
32	#include <net/xdp_sock_drv.h>
33
34	#define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)
35	#define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4)
36	#define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \
37	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
38	/* XSK buffer shall store at least Q-in-Q frame */
39	#define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \
40	ETH_FRAME_LEN + ETH_FCS_LEN + \
41	VLAN_HLEN * 2, 4))
42
43	#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
44	#define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF))
45	#else
46	#define DMA_ADDR_HIGH(dma_addr) ((u32)(0))
47	#endif
48	#define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF))
49
50	#define TSNEP_COALESCE_USECS_DEFAULT 64
51	#define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \
52	ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1)
53
54	/* mapping type */
55	#define TSNEP_TX_TYPE_MAP BIT(0)
56	#define TSNEP_TX_TYPE_MAP_PAGE BIT(1)
57	#define TSNEP_TX_TYPE_INLINE BIT(2)
58	/* buffer type */
59	#define TSNEP_TX_TYPE_SKB BIT(8)
60	#define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP)
61	#define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE)
62	#define TSNEP_TX_TYPE_SKB_FRAG BIT(9)
63	#define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE)
64	#define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE)
65	#define TSNEP_TX_TYPE_XDP_TX BIT(10)
66	#define TSNEP_TX_TYPE_XDP_NDO BIT(11)
67	#define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE)
68	#define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO)
69	#define TSNEP_TX_TYPE_XSK BIT(12)
70
71	#define TSNEP_XDP_TX BIT(0)
72	#define TSNEP_XDP_REDIRECT BIT(1)
73
74	static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask)
75	{
76	iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
77	}
78
79	static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask)
80	{
81	mask |= ECM_INT_DISABLE;
82	iowrite32(mask, adapter->addr + ECM_INT_ENABLE);
83	}
84
85	static irqreturn_t tsnep_irq(int irq, void *arg)
86	{
87	struct tsnep_adapter *adapter = arg;
88	u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE);
89
90	/* acknowledge interrupt */
91	if (active != 0)
92	iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE);
93
94	/* handle link interrupt */
95	if ((active & ECM_INT_LINK) != 0)
96	phy_mac_interrupt(phydev: adapter->netdev->phydev);
97
98	/* handle TX/RX queue 0 interrupt */
99	if ((active & adapter->queue[0].irq_mask) != 0) {
100	if (napi_schedule_prep(n: &adapter->queue[0].napi)) {
101	tsnep_disable_irq(adapter, mask: adapter->queue[0].irq_mask);
102	/* schedule after masking to avoid races */
103	__napi_schedule(n: &adapter->queue[0].napi);
104	}
105	}
106
107	return IRQ_HANDLED;
108	}
109
110	static irqreturn_t tsnep_irq_txrx(int irq, void *arg)
111	{
112	struct tsnep_queue *queue = arg;
113
114	/* handle TX/RX queue interrupt */
115	if (napi_schedule_prep(n: &queue->napi)) {
116	tsnep_disable_irq(adapter: queue->adapter, mask: queue->irq_mask);
117	/* schedule after masking to avoid races */
118	__napi_schedule(n: &queue->napi);
119	}
120
121	return IRQ_HANDLED;
122	}
123
124	int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs)
125	{
126	if (usecs > TSNEP_COALESCE_USECS_MAX)
127	return -ERANGE;
128
129	usecs /= ECM_INT_DELAY_BASE_US;
130	usecs <<= ECM_INT_DELAY_SHIFT;
131	usecs &= ECM_INT_DELAY_MASK;
132
133	queue->irq_delay &= ~ECM_INT_DELAY_MASK;
134	queue->irq_delay |= usecs;
135	iowrite8(queue->irq_delay, queue->irq_delay_addr);
136
137	return 0;
138	}
139
140	u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue)
141	{
142	u32 usecs;
143
144	usecs = (queue->irq_delay & ECM_INT_DELAY_MASK);
145	usecs >>= ECM_INT_DELAY_SHIFT;
146	usecs *= ECM_INT_DELAY_BASE_US;
147
148	return usecs;
149	}
150
151	static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum)
152	{
153	struct tsnep_adapter *adapter = bus->priv;
154	u32 md;
155	int retval;
156
157	md = ECM_MD_READ;
158	if (!adapter->suppress_preamble)
159	md |= ECM_MD_PREAMBLE;
160	md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
161	md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
162	iowrite32(md, adapter->addr + ECM_MD_CONTROL);
163	retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
164	!(md & ECM_MD_BUSY), 16, 1000);
165	if (retval != 0)
166	return retval;
167
168	return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT;
169	}
170
171	static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum,
172	u16 val)
173	{
174	struct tsnep_adapter *adapter = bus->priv;
175	u32 md;
176	int retval;
177
178	md = ECM_MD_WRITE;
179	if (!adapter->suppress_preamble)
180	md |= ECM_MD_PREAMBLE;
181	md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;
182	md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;
183	md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK;
184	iowrite32(md, adapter->addr + ECM_MD_CONTROL);
185	retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,
186	!(md & ECM_MD_BUSY), 16, 1000);
187	if (retval != 0)
188	return retval;
189
190	return 0;
191	}
192
193	static void tsnep_set_link_mode(struct tsnep_adapter *adapter)
194	{
195	u32 mode;
196
197	switch (adapter->phydev->speed) {
198	case SPEED_100:
199	mode = ECM_LINK_MODE_100;
200	break;
201	case SPEED_1000:
202	mode = ECM_LINK_MODE_1000;
203	break;
204	default:
205	mode = ECM_LINK_MODE_OFF;
206	break;
207	}
208	iowrite32(mode, adapter->addr + ECM_STATUS);
209	}
210
211	static void tsnep_phy_link_status_change(struct net_device *netdev)
212	{
213	struct tsnep_adapter *adapter = netdev_priv(dev: netdev);
214	struct phy_device *phydev = netdev->phydev;
215
216	if (phydev->link)
217	tsnep_set_link_mode(adapter);
218
219	phy_print_status(phydev: netdev->phydev);
220	}
221
222	static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable)
223	{
224	int retval;
225
226	retval = phy_loopback(phydev: adapter->phydev, enable);
227
228	/* PHY link state change is not signaled if loopback is enabled, it
229	* would delay a working loopback anyway, let's ensure that loopback
230	* is working immediately by setting link mode directly
231	*/
232	if (!retval && enable) {
233	netif_carrier_on(dev: adapter->netdev);
234	tsnep_set_link_mode(adapter);
235	}
236
237	return retval;
238	}
239
240	static int tsnep_phy_open(struct tsnep_adapter *adapter)
241	{
242	struct phy_device *phydev;
243	struct ethtool_keee ethtool_keee;
244	int retval;
245
246	retval = phy_connect_direct(dev: adapter->netdev, phydev: adapter->phydev,
247	handler: tsnep_phy_link_status_change,
248	interface: adapter->phy_mode);
249	if (retval)
250	return retval;
251	phydev = adapter->netdev->phydev;
252
253	/* MAC supports only 100Mbps|1000Mbps full duplex
254	* SPE (Single Pair Ethernet) is also an option but not implemented yet
255	*/
256	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_10baseT_Half_BIT);
257	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_10baseT_Full_BIT);
258	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_100baseT_Half_BIT);
259	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
260
261	/* disable EEE autoneg, EEE not supported by TSNEP */
262	memset(&ethtool_keee, 0, sizeof(ethtool_keee));
263	phy_ethtool_set_eee(phydev: adapter->phydev, data: &ethtool_keee);
264
265	adapter->phydev->irq = PHY_MAC_INTERRUPT;
266	phy_start(phydev: adapter->phydev);
267
268	return 0;
269	}
270
271	static void tsnep_phy_close(struct tsnep_adapter *adapter)
272	{
273	phy_stop(phydev: adapter->netdev->phydev);
274	phy_disconnect(phydev: adapter->netdev->phydev);
275	}
276
277	static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx)
278	{
279	struct device *dmadev = tx->adapter->dmadev;
280	int i;
281
282	memset(tx->entry, 0, sizeof(tx->entry));
283
284	for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
285	if (tx->page[i]) {
286	dma_free_coherent(dev: dmadev, PAGE_SIZE, cpu_addr: tx->page[i],
287	dma_handle: tx->page_dma[i]);
288	tx->page[i] = NULL;
289	tx->page_dma[i] = 0;
290	}
291	}
292	}
293
294	static int tsnep_tx_ring_create(struct tsnep_tx *tx)
295	{
296	struct device *dmadev = tx->adapter->dmadev;
297	struct tsnep_tx_entry *entry;
298	struct tsnep_tx_entry *next_entry;
299	int i, j;
300	int retval;
301
302	for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
303	tx->page[i] =
304	dma_alloc_coherent(dev: dmadev, PAGE_SIZE, dma_handle: &tx->page_dma[i],
305	GFP_KERNEL);
306	if (!tx->page[i]) {
307	retval = -ENOMEM;
308	goto alloc_failed;
309	}
310	for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
311	entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
312	entry->desc_wb = (struct tsnep_tx_desc_wb *)
313	(((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j);
314	entry->desc = (struct tsnep_tx_desc *)
315	(((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
316	entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j;
317	entry->owner_user_flag = false;
318	}
319	}
320	for (i = 0; i < TSNEP_RING_SIZE; i++) {
321	entry = &tx->entry[i];
322	next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK];
323	entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
324	}
325
326	return 0;
327
328	alloc_failed:
329	tsnep_tx_ring_cleanup(tx);
330	return retval;
331	}
332
333	static void tsnep_tx_init(struct tsnep_tx *tx)
334	{
335	dma_addr_t dma;
336
337	dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
338	iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW);
339	iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH);
340	tx->write = 0;
341	tx->read = 0;
342	tx->owner_counter = 1;
343	tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
344	}
345
346	static void tsnep_tx_enable(struct tsnep_tx *tx)
347	{
348	struct netdev_queue *nq;
349
350	nq = netdev_get_tx_queue(dev: tx->adapter->netdev, index: tx->queue_index);
351
352	__netif_tx_lock_bh(txq: nq);
353	netif_tx_wake_queue(dev_queue: nq);
354	__netif_tx_unlock_bh(txq: nq);
355	}
356
357	static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi)
358	{
359	struct netdev_queue *nq;
360	u32 val;
361
362	nq = netdev_get_tx_queue(dev: tx->adapter->netdev, index: tx->queue_index);
363
364	__netif_tx_lock_bh(txq: nq);
365	netif_tx_stop_queue(dev_queue: nq);
366	__netif_tx_unlock_bh(txq: nq);
367
368	/* wait until TX is done in hardware */
369	readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
370	((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
371	1000000);
372
373	/* wait until TX is also done in software */
374	while (READ_ONCE(tx->read) != tx->write) {
375	napi_schedule(n: napi);
376	napi_synchronize(n: napi);
377	}
378	}
379
380	static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,
381	bool last)
382	{
383	struct tsnep_tx_entry *entry = &tx->entry[index];
384
385	entry->properties = 0;
386	/* xdpf and zc are union with skb */
387	if (entry->skb) {
388	entry->properties = length & TSNEP_DESC_LENGTH_MASK;
389	entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
390	if ((entry->type & TSNEP_TX_TYPE_SKB) &&
391	(skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS))
392	entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG;
393
394	/* toggle user flag to prevent false acknowledge
395	*
396	* Only the first fragment is acknowledged. For all other
397	* fragments no acknowledge is done and the last written owner
398	* counter stays in the writeback descriptor. Therefore, it is
399	* possible that the last written owner counter is identical to
400	* the new incremented owner counter and a false acknowledge is
401	* detected before the real acknowledge has been done by
402	* hardware.
403	*
404	* The user flag is used to prevent this situation. The user
405	* flag is copied to the writeback descriptor by the hardware
406	* and is used as additional acknowledge data. By toggeling the
407	* user flag only for the first fragment (which is
408	* acknowledged), it is guaranteed that the last acknowledge
409	* done for this descriptor has used a different user flag and
410	* cannot be detected as false acknowledge.
411	*/
412	entry->owner_user_flag = !entry->owner_user_flag;
413	}
414	if (last)
415	entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG;
416	if (index == tx->increment_owner_counter) {
417	tx->owner_counter++;
418	if (tx->owner_counter == 4)
419	tx->owner_counter = 1;
420	tx->increment_owner_counter--;
421	if (tx->increment_owner_counter < 0)
422	tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
423	}
424	entry->properties |=
425	(tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
426	TSNEP_DESC_OWNER_COUNTER_MASK;
427	if (entry->owner_user_flag)
428	entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG;
429	entry->desc->more_properties =
430	__cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK);
431	if (entry->type & TSNEP_TX_TYPE_INLINE)
432	entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG;
433
434	/* descriptor properties shall be written last, because valid data is
435	* signaled there
436	*/
437	dma_wmb();
438
439	entry->desc->properties = __cpu_to_le32(entry->properties);
440	}
441
442	static int tsnep_tx_desc_available(struct tsnep_tx *tx)
443	{
444	if (tx->read <= tx->write)
445	return TSNEP_RING_SIZE - tx->write + tx->read - 1;
446	else
447	return tx->read - tx->write - 1;
448	}
449
450	static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry,
451	struct device *dmadev, dma_addr_t *dma)
452	{
453	unsigned int len;
454	int mapped;
455
456	len = skb_frag_size(frag);
457	if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
458	*dma = skb_frag_dma_map(dev: dmadev, frag, offset: 0, size: len, dir: DMA_TO_DEVICE);
459	if (dma_mapping_error(dev: dmadev, dma_addr: *dma))
460	return -ENOMEM;
461	entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE;
462	mapped = 1;
463	} else {
464	void *fragdata = skb_frag_address_safe(frag);
465
466	if (likely(fragdata)) {
467	memcpy(&entry->desc->tx, fragdata, len);
468	} else {
469	struct page *page = skb_frag_page(frag);
470
471	fragdata = kmap_local_page(page);
472	memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag),
473	len);
474	kunmap_local(fragdata);
475	}
476	entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE;
477	mapped = 0;
478	}
479
480	return mapped;
481	}
482
483	static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count)
484	{
485	struct device *dmadev = tx->adapter->dmadev;
486	struct tsnep_tx_entry *entry;
487	unsigned int len;
488	int map_len = 0;
489	dma_addr_t dma;
490	int i, mapped;
491
492	for (i = 0; i < count; i++) {
493	entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
494
495	if (!i) {
496	len = skb_headlen(skb);
497	if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {
498	dma = dma_map_single(dmadev, skb->data, len,
499	DMA_TO_DEVICE);
500	if (dma_mapping_error(dev: dmadev, dma_addr: dma))
501	return -ENOMEM;
502	entry->type = TSNEP_TX_TYPE_SKB_MAP;
503	mapped = 1;
504	} else {
505	memcpy(&entry->desc->tx, skb->data, len);
506	entry->type = TSNEP_TX_TYPE_SKB_INLINE;
507	mapped = 0;
508	}
509	} else {
510	skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
511
512	len = skb_frag_size(frag);
513	mapped = tsnep_tx_map_frag(frag, entry, dmadev, dma: &dma);
514	if (mapped < 0)
515	return mapped;
516	}
517
518	entry->len = len;
519	if (likely(mapped)) {
520	dma_unmap_addr_set(entry, dma, dma);
521	entry->desc->tx = __cpu_to_le64(dma);
522	}
523
524	map_len += len;
525	}
526
527	return map_len;
528	}
529
530	static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
531	{
532	struct device *dmadev = tx->adapter->dmadev;
533	struct tsnep_tx_entry *entry;
534	int map_len = 0;
535	int i;
536
537	for (i = 0; i < count; i++) {
538	entry = &tx->entry[(index + i) & TSNEP_RING_MASK];
539
540	if (entry->len) {
541	if (entry->type & TSNEP_TX_TYPE_MAP)
542	dma_unmap_single(dmadev,
543	dma_unmap_addr(entry, dma),
544	dma_unmap_len(entry, len),
545	DMA_TO_DEVICE);
546	else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE)
547	dma_unmap_page(dmadev,
548	dma_unmap_addr(entry, dma),
549	dma_unmap_len(entry, len),
550	DMA_TO_DEVICE);
551	map_len += entry->len;
552	entry->len = 0;
553	}
554	}
555
556	return map_len;
557	}
558
559	static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
560	struct tsnep_tx *tx)
561	{
562	int count = 1;
563	struct tsnep_tx_entry *entry;
564	int length;
565	int i;
566	int retval;
567
568	if (skb_shinfo(skb)->nr_frags > 0)
569	count += skb_shinfo(skb)->nr_frags;
570
571	if (tsnep_tx_desc_available(tx) < count) {
572	/* ring full, shall not happen because queue is stopped if full
573	* below
574	*/
575	netif_stop_subqueue(dev: tx->adapter->netdev, queue_index: tx->queue_index);
576
577	return NETDEV_TX_BUSY;
578	}
579
580	entry = &tx->entry[tx->write];
581	entry->skb = skb;
582
583	retval = tsnep_tx_map(skb, tx, count);
584	if (retval < 0) {
585	tsnep_tx_unmap(tx, index: tx->write, count);
586	dev_kfree_skb_any(skb: entry->skb);
587	entry->skb = NULL;
588
589	tx->dropped++;
590
591	return NETDEV_TX_OK;
592	}
593	length = retval;
594
595	if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
596	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
597
598	for (i = 0; i < count; i++)
599	tsnep_tx_activate(tx, index: (tx->write + i) & TSNEP_RING_MASK, length,
600	last: i == count - 1);
601	tx->write = (tx->write + count) & TSNEP_RING_MASK;
602
603	skb_tx_timestamp(skb);
604
605	/* descriptor properties shall be valid before hardware is notified */
606	dma_wmb();
607
608	iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
609
610	if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) {
611	/* ring can get full with next frame */
612	netif_stop_subqueue(dev: tx->adapter->netdev, queue_index: tx->queue_index);
613	}
614
615	return NETDEV_TX_OK;
616	}
617
618	static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx,
619	struct skb_shared_info *shinfo, int count, u32 type)
620	{
621	struct device *dmadev = tx->adapter->dmadev;
622	struct tsnep_tx_entry *entry;
623	struct page *page;
624	skb_frag_t *frag;
625	unsigned int len;
626	int map_len = 0;
627	dma_addr_t dma;
628	void *data;
629	int i;
630
631	frag = NULL;
632	len = xdpf->len;
633	for (i = 0; i < count; i++) {
634	entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];
635	if (type & TSNEP_TX_TYPE_XDP_NDO) {
636	data = unlikely(frag) ? skb_frag_address(frag) :
637	xdpf->data;
638	dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE);
639	if (dma_mapping_error(dev: dmadev, dma_addr: dma))
640	return -ENOMEM;
641
642	entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE;
643	} else {
644	page = unlikely(frag) ? skb_frag_page(frag) :
645	virt_to_page(xdpf->data);
646	dma = page_pool_get_dma_addr(page);
647	if (unlikely(frag))
648	dma += skb_frag_off(frag);
649	else
650	dma += sizeof(*xdpf) + xdpf->headroom;
651	dma_sync_single_for_device(dev: dmadev, addr: dma, size: len,
652	dir: DMA_BIDIRECTIONAL);
653
654	entry->type = TSNEP_TX_TYPE_XDP_TX;
655	}
656
657	entry->len = len;
658	dma_unmap_addr_set(entry, dma, dma);
659
660	entry->desc->tx = __cpu_to_le64(dma);
661
662	map_len += len;
663
664	if (i + 1 < count) {
665	frag = &shinfo->frags[i];
666	len = skb_frag_size(frag);
667	}
668	}
669
670	return map_len;
671	}
672
673	/* This function requires __netif_tx_lock is held by the caller. */
674	static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf,
675	struct tsnep_tx *tx, u32 type)
676	{
677	struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(frame: xdpf);
678	struct tsnep_tx_entry *entry;
679	int count, length, retval, i;
680
681	count = 1;
682	if (unlikely(xdp_frame_has_frags(xdpf)))
683	count += shinfo->nr_frags;
684
685	/* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
686	* will be available for normal TX path and queue is stopped there if
687	* necessary
688	*/
689	if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count))
690	return false;
691
692	entry = &tx->entry[tx->write];
693	entry->xdpf = xdpf;
694
695	retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type);
696	if (retval < 0) {
697	tsnep_tx_unmap(tx, index: tx->write, count);
698	entry->xdpf = NULL;
699
700	tx->dropped++;
701
702	return false;
703	}
704	length = retval;
705
706	for (i = 0; i < count; i++)
707	tsnep_tx_activate(tx, index: (tx->write + i) & TSNEP_RING_MASK, length,
708	last: i == count - 1);
709	tx->write = (tx->write + count) & TSNEP_RING_MASK;
710
711	/* descriptor properties shall be valid before hardware is notified */
712	dma_wmb();
713
714	return true;
715	}
716
717	static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx)
718	{
719	iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);
720	}
721
722	static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,
723	struct xdp_buff *xdp,
724	struct netdev_queue *tx_nq, struct tsnep_tx *tx,
725	bool zc)
726	{
727	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
728	bool xmit;
729	u32 type;
730
731	if (unlikely(!xdpf))
732	return false;
733
734	/* no page pool for zero copy */
735	if (zc)
736	type = TSNEP_TX_TYPE_XDP_NDO;
737	else
738	type = TSNEP_TX_TYPE_XDP_TX;
739
740	__netif_tx_lock(txq: tx_nq, smp_processor_id());
741
742	xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, type);
743
744	/* Avoid transmit queue timeout since we share it with the slow path */
745	if (xmit)
746	txq_trans_cond_update(txq: tx_nq);
747
748	__netif_tx_unlock(txq: tx_nq);
749
750	return xmit;
751	}
752
753	static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx)
754	{
755	struct tsnep_tx_entry *entry;
756	dma_addr_t dma;
757
758	entry = &tx->entry[tx->write];
759	entry->zc = true;
760
761	dma = xsk_buff_raw_get_dma(pool: tx->xsk_pool, addr: xdpd->addr);
762	xsk_buff_raw_dma_sync_for_device(pool: tx->xsk_pool, dma, size: xdpd->len);
763
764	entry->type = TSNEP_TX_TYPE_XSK;
765	entry->len = xdpd->len;
766
767	entry->desc->tx = __cpu_to_le64(dma);
768
769	return xdpd->len;
770	}
771
772	static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd,
773	struct tsnep_tx *tx)
774	{
775	int length;
776
777	length = tsnep_xdp_tx_map_zc(xdpd, tx);
778
779	tsnep_tx_activate(tx, index: tx->write, length, last: true);
780	tx->write = (tx->write + 1) & TSNEP_RING_MASK;
781	}
782
783	static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx)
784	{
785	int desc_available = tsnep_tx_desc_available(tx);
786	struct xdp_desc *descs = tx->xsk_pool->tx_descs;
787	int batch, i;
788
789	/* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
790	* will be available for normal TX path and queue is stopped there if
791	* necessary
792	*/
793	if (desc_available <= (MAX_SKB_FRAGS + 1))
794	return;
795	desc_available -= MAX_SKB_FRAGS + 1;
796
797	batch = xsk_tx_peek_release_desc_batch(pool: tx->xsk_pool, max: desc_available);
798	for (i = 0; i < batch; i++)
799	tsnep_xdp_xmit_frame_ring_zc(xdpd: &descs[i], tx);
800
801	if (batch) {
802	/* descriptor properties shall be valid before hardware is
803	* notified
804	*/
805	dma_wmb();
806
807	tsnep_xdp_xmit_flush(tx);
808	}
809	}
810
811	static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
812	{
813	struct tsnep_tx_entry *entry;
814	struct netdev_queue *nq;
815	int xsk_frames = 0;
816	int budget = 128;
817	int length;
818	int count;
819
820	nq = netdev_get_tx_queue(dev: tx->adapter->netdev, index: tx->queue_index);
821	__netif_tx_lock(txq: nq, smp_processor_id());
822
823	do {
824	if (tx->read == tx->write)
825	break;
826
827	entry = &tx->entry[tx->read];
828	if ((__le32_to_cpu(entry->desc_wb->properties) &
829	TSNEP_TX_DESC_OWNER_MASK) !=
830	(entry->properties & TSNEP_TX_DESC_OWNER_MASK))
831	break;
832
833	/* descriptor properties shall be read first, because valid data
834	* is signaled there
835	*/
836	dma_rmb();
837
838	count = 1;
839	if ((entry->type & TSNEP_TX_TYPE_SKB) &&
840	skb_shinfo(entry->skb)->nr_frags > 0)
841	count += skb_shinfo(entry->skb)->nr_frags;
842	else if ((entry->type & TSNEP_TX_TYPE_XDP) &&
843	xdp_frame_has_frags(frame: entry->xdpf))
844	count += xdp_get_shared_info_from_frame(frame: entry->xdpf)->nr_frags;
845
846	length = tsnep_tx_unmap(tx, index: tx->read, count);
847
848	if ((entry->type & TSNEP_TX_TYPE_SKB) &&
849	(skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) &&
850	(__le32_to_cpu(entry->desc_wb->properties) &
851	TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) {
852	struct skb_shared_hwtstamps hwtstamps;
853	u64 timestamp;
854
855	if (skb_shinfo(entry->skb)->tx_flags &
856	SKBTX_HW_TSTAMP_USE_CYCLES)
857	timestamp =
858	__le64_to_cpu(entry->desc_wb->counter);
859	else
860	timestamp =
861	__le64_to_cpu(entry->desc_wb->timestamp);
862
863	memset(&hwtstamps, 0, sizeof(hwtstamps));
864	hwtstamps.hwtstamp = ns_to_ktime(ns: timestamp);
865
866	skb_tstamp_tx(orig_skb: entry->skb, hwtstamps: &hwtstamps);
867	}
868
869	if (entry->type & TSNEP_TX_TYPE_SKB)
870	napi_consume_skb(skb: entry->skb, budget: napi_budget);
871	else if (entry->type & TSNEP_TX_TYPE_XDP)
872	xdp_return_frame_rx_napi(xdpf: entry->xdpf);
873	else
874	xsk_frames++;
875	/* xdpf and zc are union with skb */
876	entry->skb = NULL;
877
878	tx->read = (tx->read + count) & TSNEP_RING_MASK;
879
880	tx->packets++;
881	tx->bytes += length + ETH_FCS_LEN;
882
883	budget--;
884	} while (likely(budget));
885
886	if (tx->xsk_pool) {
887	if (xsk_frames)
888	xsk_tx_completed(pool: tx->xsk_pool, nb_entries: xsk_frames);
889	if (xsk_uses_need_wakeup(pool: tx->xsk_pool))
890	xsk_set_tx_need_wakeup(pool: tx->xsk_pool);
891	tsnep_xdp_xmit_zc(tx);
892	}
893
894	if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) &&
895	netif_tx_queue_stopped(dev_queue: nq)) {
896	netif_tx_wake_queue(dev_queue: nq);
897	}
898
899	__netif_tx_unlock(txq: nq);
900
901	return budget != 0;
902	}
903
904	static bool tsnep_tx_pending(struct tsnep_tx *tx)
905	{
906	struct tsnep_tx_entry *entry;
907	struct netdev_queue *nq;
908	bool pending = false;
909
910	nq = netdev_get_tx_queue(dev: tx->adapter->netdev, index: tx->queue_index);
911	__netif_tx_lock(txq: nq, smp_processor_id());
912
913	if (tx->read != tx->write) {
914	entry = &tx->entry[tx->read];
915	if ((__le32_to_cpu(entry->desc_wb->properties) &
916	TSNEP_TX_DESC_OWNER_MASK) ==
917	(entry->properties & TSNEP_TX_DESC_OWNER_MASK))
918	pending = true;
919	}
920
921	__netif_tx_unlock(txq: nq);
922
923	return pending;
924	}
925
926	static int tsnep_tx_open(struct tsnep_tx *tx)
927	{
928	int retval;
929
930	retval = tsnep_tx_ring_create(tx);
931	if (retval)
932	return retval;
933
934	tsnep_tx_init(tx);
935
936	return 0;
937	}
938
939	static void tsnep_tx_close(struct tsnep_tx *tx)
940	{
941	tsnep_tx_ring_cleanup(tx);
942	}
943
944	static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
945	{
946	struct device *dmadev = rx->adapter->dmadev;
947	struct tsnep_rx_entry *entry;
948	int i;
949
950	for (i = 0; i < TSNEP_RING_SIZE; i++) {
951	entry = &rx->entry[i];
952	if (!rx->xsk_pool && entry->page)
953	page_pool_put_full_page(pool: rx->page_pool, page: entry->page,
954	allow_direct: false);
955	if (rx->xsk_pool && entry->xdp)
956	xsk_buff_free(xdp: entry->xdp);
957	/* xdp is union with page */
958	entry->page = NULL;
959	}
960
961	if (rx->page_pool)
962	page_pool_destroy(pool: rx->page_pool);
963
964	memset(rx->entry, 0, sizeof(rx->entry));
965
966	for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
967	if (rx->page[i]) {
968	dma_free_coherent(dev: dmadev, PAGE_SIZE, cpu_addr: rx->page[i],
969	dma_handle: rx->page_dma[i]);
970	rx->page[i] = NULL;
971	rx->page_dma[i] = 0;
972	}
973	}
974	}
975
976	static int tsnep_rx_ring_create(struct tsnep_rx *rx)
977	{
978	struct device *dmadev = rx->adapter->dmadev;
979	struct tsnep_rx_entry *entry;
980	struct page_pool_params pp_params = { 0 };
981	struct tsnep_rx_entry *next_entry;
982	int i, j;
983	int retval;
984
985	for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
986	rx->page[i] =
987	dma_alloc_coherent(dev: dmadev, PAGE_SIZE, dma_handle: &rx->page_dma[i],
988	GFP_KERNEL);
989	if (!rx->page[i]) {
990	retval = -ENOMEM;
991	goto failed;
992	}
993	for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {
994	entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];
995	entry->desc_wb = (struct tsnep_rx_desc_wb *)
996	(((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j);
997	entry->desc = (struct tsnep_rx_desc *)
998	(((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);
999	entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;
1000	}
1001	}
1002
1003	pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
1004	pp_params.order = 0;
1005	pp_params.pool_size = TSNEP_RING_SIZE;
1006	pp_params.nid = dev_to_node(dev: dmadev);
1007	pp_params.dev = dmadev;
1008	pp_params.dma_dir = DMA_BIDIRECTIONAL;
1009	pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE;
1010	pp_params.offset = TSNEP_RX_OFFSET;
1011	rx->page_pool = page_pool_create(params: &pp_params);
1012	if (IS_ERR(ptr: rx->page_pool)) {
1013	retval = PTR_ERR(ptr: rx->page_pool);
1014	rx->page_pool = NULL;
1015	goto failed;
1016	}
1017
1018	for (i = 0; i < TSNEP_RING_SIZE; i++) {
1019	entry = &rx->entry[i];
1020	next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK];
1021	entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
1022	}
1023
1024	return 0;
1025
1026	failed:
1027	tsnep_rx_ring_cleanup(rx);
1028	return retval;
1029	}
1030
1031	static void tsnep_rx_init(struct tsnep_rx *rx)
1032	{
1033	dma_addr_t dma;
1034
1035	dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;
1036	iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);
1037	iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);
1038	rx->write = 0;
1039	rx->read = 0;
1040	rx->owner_counter = 1;
1041	rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1042	}
1043
1044	static void tsnep_rx_enable(struct tsnep_rx *rx)
1045	{
1046	/* descriptor properties shall be valid before hardware is notified */
1047	dma_wmb();
1048
1049	iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);
1050	}
1051
1052	static void tsnep_rx_disable(struct tsnep_rx *rx)
1053	{
1054	u32 val;
1055
1056	iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);
1057	readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,
1058	((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000,
1059	1000000);
1060	}
1061
1062	static int tsnep_rx_desc_available(struct tsnep_rx *rx)
1063	{
1064	if (rx->read <= rx->write)
1065	return TSNEP_RING_SIZE - rx->write + rx->read - 1;
1066	else
1067	return rx->read - rx->write - 1;
1068	}
1069
1070	static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx)
1071	{
1072	struct page **page;
1073
1074	/* last entry of page_buffer is always zero, because ring cannot be
1075	* filled completely
1076	*/
1077	page = rx->page_buffer;
1078	while (*page) {
1079	page_pool_put_full_page(pool: rx->page_pool, page: *page, allow_direct: false);
1080	*page = NULL;
1081	page++;
1082	}
1083	}
1084
1085	static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx)
1086	{
1087	int i;
1088
1089	/* alloc for all ring entries except the last one, because ring cannot
1090	* be filled completely
1091	*/
1092	for (i = 0; i < TSNEP_RING_SIZE - 1; i++) {
1093	rx->page_buffer[i] = page_pool_dev_alloc_pages(pool: rx->page_pool);
1094	if (!rx->page_buffer[i]) {
1095	tsnep_rx_free_page_buffer(rx);
1096
1097	return -ENOMEM;
1098	}
1099	}
1100
1101	return 0;
1102	}
1103
1104	static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1105	struct page *page)
1106	{
1107	entry->page = page;
1108	entry->len = TSNEP_MAX_RX_BUF_SIZE;
1109	entry->dma = page_pool_get_dma_addr(page: entry->page);
1110	entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET);
1111	}
1112
1113	static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index)
1114	{
1115	struct tsnep_rx_entry *entry = &rx->entry[index];
1116	struct page *page;
1117
1118	page = page_pool_dev_alloc_pages(pool: rx->page_pool);
1119	if (unlikely(!page))
1120	return -ENOMEM;
1121	tsnep_rx_set_page(rx, entry, page);
1122
1123	return 0;
1124	}
1125
1126	static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index)
1127	{
1128	struct tsnep_rx_entry *entry = &rx->entry[index];
1129	struct tsnep_rx_entry *read = &rx->entry[rx->read];
1130
1131	tsnep_rx_set_page(rx, entry, page: read->page);
1132	read->page = NULL;
1133	}
1134
1135	static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
1136	{
1137	struct tsnep_rx_entry *entry = &rx->entry[index];
1138
1139	/* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */
1140	entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK;
1141	entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
1142	if (index == rx->increment_owner_counter) {
1143	rx->owner_counter++;
1144	if (rx->owner_counter == 4)
1145	rx->owner_counter = 1;
1146	rx->increment_owner_counter--;
1147	if (rx->increment_owner_counter < 0)
1148	rx->increment_owner_counter = TSNEP_RING_SIZE - 1;
1149	}
1150	entry->properties |=
1151	(rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &
1152	TSNEP_DESC_OWNER_COUNTER_MASK;
1153
1154	/* descriptor properties shall be written last, because valid data is
1155	* signaled there
1156	*/
1157	dma_wmb();
1158
1159	entry->desc->properties = __cpu_to_le32(entry->properties);
1160	}
1161
1162	static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse)
1163	{
1164	bool alloc_failed = false;
1165	int i, index;
1166
1167	for (i = 0; i < count && !alloc_failed; i++) {
1168	index = (rx->write + i) & TSNEP_RING_MASK;
1169
1170	if (unlikely(tsnep_rx_alloc_buffer(rx, index))) {
1171	rx->alloc_failed++;
1172	alloc_failed = true;
1173
1174	/* reuse only if no other allocation was successful */
1175	if (i == 0 && reuse)
1176	tsnep_rx_reuse_buffer(rx, index);
1177	else
1178	break;
1179	}
1180
1181	tsnep_rx_activate(rx, index);
1182	}
1183
1184	if (i)
1185	rx->write = (rx->write + i) & TSNEP_RING_MASK;
1186
1187	return i;
1188	}
1189
1190	static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)
1191	{
1192	int desc_refilled;
1193
1194	desc_refilled = tsnep_rx_alloc(rx, count, reuse);
1195	if (desc_refilled)
1196	tsnep_rx_enable(rx);
1197
1198	return desc_refilled;
1199	}
1200
1201	static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,
1202	struct xdp_buff *xdp)
1203	{
1204	entry->xdp = xdp;
1205	entry->len = TSNEP_XSK_RX_BUF_SIZE;
1206	entry->dma = xsk_buff_xdp_get_dma(xdp: entry->xdp);
1207	entry->desc->rx = __cpu_to_le64(entry->dma);
1208	}
1209
1210	static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index)
1211	{
1212	struct tsnep_rx_entry *entry = &rx->entry[index];
1213	struct tsnep_rx_entry *read = &rx->entry[rx->read];
1214
1215	tsnep_rx_set_xdp(rx, entry, xdp: read->xdp);
1216	read->xdp = NULL;
1217	}
1218
1219	static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse)
1220	{
1221	u32 allocated;
1222	int i;
1223
1224	allocated = xsk_buff_alloc_batch(pool: rx->xsk_pool, xdp: rx->xdp_batch, max: count);
1225	for (i = 0; i < allocated; i++) {
1226	int index = (rx->write + i) & TSNEP_RING_MASK;
1227	struct tsnep_rx_entry *entry = &rx->entry[index];
1228
1229	tsnep_rx_set_xdp(rx, entry, xdp: rx->xdp_batch[i]);
1230	tsnep_rx_activate(rx, index);
1231	}
1232	if (i == 0) {
1233	rx->alloc_failed++;
1234
1235	if (reuse) {
1236	tsnep_rx_reuse_buffer_zc(rx, index: rx->write);
1237	tsnep_rx_activate(rx, index: rx->write);
1238	}
1239	}
1240
1241	if (i)
1242	rx->write = (rx->write + i) & TSNEP_RING_MASK;
1243
1244	return i;
1245	}
1246
1247	static void tsnep_rx_free_zc(struct tsnep_rx *rx)
1248	{
1249	int i;
1250
1251	for (i = 0; i < TSNEP_RING_SIZE; i++) {
1252	struct tsnep_rx_entry *entry = &rx->entry[i];
1253
1254	if (entry->xdp)
1255	xsk_buff_free(xdp: entry->xdp);
1256	entry->xdp = NULL;
1257	}
1258	}
1259
1260	static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse)
1261	{
1262	int desc_refilled;
1263
1264	desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse);
1265	if (desc_refilled)
1266	tsnep_rx_enable(rx);
1267
1268	return desc_refilled;
1269	}
1270
1271	static void tsnep_xsk_rx_need_wakeup(struct tsnep_rx *rx, int desc_available)
1272	{
1273	if (desc_available)
1274	xsk_set_rx_need_wakeup(pool: rx->xsk_pool);
1275	else
1276	xsk_clear_rx_need_wakeup(pool: rx->xsk_pool);
1277	}
1278
1279	static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,
1280	struct xdp_buff *xdp, int *status,
1281	struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1282	{
1283	unsigned int length;
1284	unsigned int sync;
1285	u32 act;
1286
1287	length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;
1288
1289	act = bpf_prog_run_xdp(prog, xdp);
1290	switch (act) {
1291	case XDP_PASS:
1292	return false;
1293	case XDP_TX:
1294	if (!tsnep_xdp_xmit_back(adapter: rx->adapter, xdp, tx_nq, tx, zc: false))
1295	goto out_failure;
1296	*status |= TSNEP_XDP_TX;
1297	return true;
1298	case XDP_REDIRECT:
1299	if (xdp_do_redirect(dev: rx->adapter->netdev, xdp, prog) < 0)
1300	goto out_failure;
1301	*status |= TSNEP_XDP_REDIRECT;
1302	return true;
1303	default:
1304	bpf_warn_invalid_xdp_action(dev: rx->adapter->netdev, prog, act);
1305	fallthrough;
1306	case XDP_ABORTED:
1307	out_failure:
1308	trace_xdp_exception(dev: rx->adapter->netdev, xdp: prog, act);
1309	fallthrough;
1310	case XDP_DROP:
1311	/* Due xdp_adjust_tail: DMA sync for_device cover max len CPU
1312	* touch
1313	*/
1314	sync = xdp->data_end - xdp->data_hard_start -
1315	XDP_PACKET_HEADROOM;
1316	sync = max(sync, length);
1317	page_pool_put_page(pool: rx->page_pool, page: virt_to_head_page(x: xdp->data),
1318	dma_sync_size: sync, allow_direct: true);
1319	return true;
1320	}
1321	}
1322
1323	static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog,
1324	struct xdp_buff *xdp, int *status,
1325	struct netdev_queue *tx_nq,
1326	struct tsnep_tx *tx)
1327	{
1328	u32 act;
1329
1330	act = bpf_prog_run_xdp(prog, xdp);
1331
1332	/* XDP_REDIRECT is the main action for zero-copy */
1333	if (likely(act == XDP_REDIRECT)) {
1334	if (xdp_do_redirect(dev: rx->adapter->netdev, xdp, prog) < 0)
1335	goto out_failure;
1336	*status |= TSNEP_XDP_REDIRECT;
1337	return true;
1338	}
1339
1340	switch (act) {
1341	case XDP_PASS:
1342	return false;
1343	case XDP_TX:
1344	if (!tsnep_xdp_xmit_back(adapter: rx->adapter, xdp, tx_nq, tx, zc: true))
1345	goto out_failure;
1346	*status |= TSNEP_XDP_TX;
1347	return true;
1348	default:
1349	bpf_warn_invalid_xdp_action(dev: rx->adapter->netdev, prog, act);
1350	fallthrough;
1351	case XDP_ABORTED:
1352	out_failure:
1353	trace_xdp_exception(dev: rx->adapter->netdev, xdp: prog, act);
1354	fallthrough;
1355	case XDP_DROP:
1356	xsk_buff_free(xdp);
1357	return true;
1358	}
1359	}
1360
1361	static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status,
1362	struct netdev_queue *tx_nq, struct tsnep_tx *tx)
1363	{
1364	if (status & TSNEP_XDP_TX) {
1365	__netif_tx_lock(txq: tx_nq, smp_processor_id());
1366	tsnep_xdp_xmit_flush(tx);
1367	__netif_tx_unlock(txq: tx_nq);
1368	}
1369
1370	if (status & TSNEP_XDP_REDIRECT)
1371	xdp_do_flush();
1372	}
1373
1374	static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,
1375	int length)
1376	{
1377	struct sk_buff *skb;
1378
1379	skb = napi_build_skb(page_address(page), PAGE_SIZE);
1380	if (unlikely(!skb))
1381	return NULL;
1382
1383	/* update pointers within the skb to store the data */
1384	skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE);
1385	__skb_put(skb, len: length - ETH_FCS_LEN);
1386
1387	if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) {
1388	struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
1389	struct tsnep_rx_inline *rx_inline =
1390	(struct tsnep_rx_inline *)(page_address(page) +
1391	TSNEP_RX_OFFSET);
1392
1393	skb_shinfo(skb)->tx_flags |=
1394	SKBTX_HW_TSTAMP_NETDEV;
1395	memset(hwtstamps, 0, sizeof(*hwtstamps));
1396	hwtstamps->netdev_data = rx_inline;
1397	}
1398
1399	skb_record_rx_queue(skb, rx_queue: rx->queue_index);
1400	skb->protocol = eth_type_trans(skb, dev: rx->adapter->netdev);
1401
1402	return skb;
1403	}
1404
1405	static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi,
1406	struct page *page, int length)
1407	{
1408	struct sk_buff *skb;
1409
1410	skb = tsnep_build_skb(rx, page, length);
1411	if (skb) {
1412	skb_mark_for_recycle(skb);
1413
1414	rx->packets++;
1415	rx->bytes += length;
1416	if (skb->pkt_type == PACKET_MULTICAST)
1417	rx->multicast++;
1418
1419	napi_gro_receive(napi, skb);
1420	} else {
1421	page_pool_recycle_direct(pool: rx->page_pool, page);
1422
1423	rx->dropped++;
1424	}
1425	}
1426
1427	static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
1428	int budget)
1429	{
1430	struct device *dmadev = rx->adapter->dmadev;
1431	enum dma_data_direction dma_dir;
1432	struct tsnep_rx_entry *entry;
1433	struct netdev_queue *tx_nq;
1434	struct bpf_prog *prog;
1435	struct xdp_buff xdp;
1436	struct tsnep_tx *tx;
1437	int desc_available;
1438	int xdp_status = 0;
1439	int done = 0;
1440	int length;
1441
1442	desc_available = tsnep_rx_desc_available(rx);
1443	dma_dir = page_pool_get_dma_dir(pool: rx->page_pool);
1444	prog = READ_ONCE(rx->adapter->xdp_prog);
1445	if (prog) {
1446	tx_nq = netdev_get_tx_queue(dev: rx->adapter->netdev,
1447	index: rx->tx_queue_index);
1448	tx = &rx->adapter->tx[rx->tx_queue_index];
1449
1450	xdp_init_buff(xdp: &xdp, PAGE_SIZE, rxq: &rx->xdp_rxq);
1451	}
1452
1453	while (likely(done < budget) && (rx->read != rx->write)) {
1454	entry = &rx->entry[rx->read];
1455	if ((__le32_to_cpu(entry->desc_wb->properties) &
1456	TSNEP_DESC_OWNER_COUNTER_MASK) !=
1457	(entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1458	break;
1459	done++;
1460
1461	if (desc_available >= TSNEP_RING_RX_REFILL) {
1462	bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1463
1464	desc_available -= tsnep_rx_refill(rx, count: desc_available,
1465	reuse);
1466	if (!entry->page) {
1467	/* buffer has been reused for refill to prevent
1468	* empty RX ring, thus buffer cannot be used for
1469	* RX processing
1470	*/
1471	rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1472	desc_available++;
1473
1474	rx->dropped++;
1475
1476	continue;
1477	}
1478	}
1479
1480	/* descriptor properties shall be read first, because valid data
1481	* is signaled there
1482	*/
1483	dma_rmb();
1484
1485	prefetch(page_address(entry->page) + TSNEP_RX_OFFSET);
1486	length = __le32_to_cpu(entry->desc_wb->properties) &
1487	TSNEP_DESC_LENGTH_MASK;
1488	dma_sync_single_range_for_cpu(dev: dmadev, addr: entry->dma,
1489	TSNEP_RX_OFFSET, size: length, dir: dma_dir);
1490
1491	/* RX metadata with timestamps is in front of actual data,
1492	* subtract metadata size to get length of actual data and
1493	* consider metadata size as offset of actual data during RX
1494	* processing
1495	*/
1496	length -= TSNEP_RX_INLINE_METADATA_SIZE;
1497
1498	rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1499	desc_available++;
1500
1501	if (prog) {
1502	bool consume;
1503
1504	xdp_prepare_buff(xdp: &xdp, page_address(entry->page),
1505	XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE,
1506	data_len: length - ETH_FCS_LEN, meta_valid: false);
1507
1508	consume = tsnep_xdp_run_prog(rx, prog, xdp: &xdp,
1509	status: &xdp_status, tx_nq, tx);
1510	if (consume) {
1511	rx->packets++;
1512	rx->bytes += length;
1513
1514	entry->page = NULL;
1515
1516	continue;
1517	}
1518	}
1519
1520	tsnep_rx_page(rx, napi, page: entry->page, length);
1521	entry->page = NULL;
1522	}
1523
1524	if (xdp_status)
1525	tsnep_finalize_xdp(adapter: rx->adapter, status: xdp_status, tx_nq, tx);
1526
1527	if (desc_available)
1528	tsnep_rx_refill(rx, count: desc_available, reuse: false);
1529
1530	return done;
1531	}
1532
1533	static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi,
1534	int budget)
1535	{
1536	struct tsnep_rx_entry *entry;
1537	struct netdev_queue *tx_nq;
1538	struct bpf_prog *prog;
1539	struct tsnep_tx *tx;
1540	int desc_available;
1541	int xdp_status = 0;
1542	struct page *page;
1543	int done = 0;
1544	int length;
1545
1546	desc_available = tsnep_rx_desc_available(rx);
1547	prog = READ_ONCE(rx->adapter->xdp_prog);
1548	if (prog) {
1549	tx_nq = netdev_get_tx_queue(dev: rx->adapter->netdev,
1550	index: rx->tx_queue_index);
1551	tx = &rx->adapter->tx[rx->tx_queue_index];
1552	}
1553
1554	while (likely(done < budget) && (rx->read != rx->write)) {
1555	entry = &rx->entry[rx->read];
1556	if ((__le32_to_cpu(entry->desc_wb->properties) &
1557	TSNEP_DESC_OWNER_COUNTER_MASK) !=
1558	(entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1559	break;
1560	done++;
1561
1562	if (desc_available >= TSNEP_RING_RX_REFILL) {
1563	bool reuse = desc_available >= TSNEP_RING_RX_REUSE;
1564
1565	desc_available -= tsnep_rx_refill_zc(rx, count: desc_available,
1566	reuse);
1567	if (!entry->xdp) {
1568	/* buffer has been reused for refill to prevent
1569	* empty RX ring, thus buffer cannot be used for
1570	* RX processing
1571	*/
1572	rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1573	desc_available++;
1574
1575	rx->dropped++;
1576
1577	continue;
1578	}
1579	}
1580
1581	/* descriptor properties shall be read first, because valid data
1582	* is signaled there
1583	*/
1584	dma_rmb();
1585
1586	prefetch(entry->xdp->data);
1587	length = __le32_to_cpu(entry->desc_wb->properties) &
1588	TSNEP_DESC_LENGTH_MASK;
1589	xsk_buff_set_size(xdp: entry->xdp, size: length - ETH_FCS_LEN);
1590	xsk_buff_dma_sync_for_cpu(xdp: entry->xdp, pool: rx->xsk_pool);
1591
1592	/* RX metadata with timestamps is in front of actual data,
1593	* subtract metadata size to get length of actual data and
1594	* consider metadata size as offset of actual data during RX
1595	* processing
1596	*/
1597	length -= TSNEP_RX_INLINE_METADATA_SIZE;
1598
1599	rx->read = (rx->read + 1) & TSNEP_RING_MASK;
1600	desc_available++;
1601
1602	if (prog) {
1603	bool consume;
1604
1605	entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE;
1606	entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE;
1607
1608	consume = tsnep_xdp_run_prog_zc(rx, prog, xdp: entry->xdp,
1609	status: &xdp_status, tx_nq, tx);
1610	if (consume) {
1611	rx->packets++;
1612	rx->bytes += length;
1613
1614	entry->xdp = NULL;
1615
1616	continue;
1617	}
1618	}
1619
1620	page = page_pool_dev_alloc_pages(pool: rx->page_pool);
1621	if (page) {
1622	memcpy(page_address(page) + TSNEP_RX_OFFSET,
1623	entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE,
1624	length + TSNEP_RX_INLINE_METADATA_SIZE);
1625	tsnep_rx_page(rx, napi, page, length);
1626	} else {
1627	rx->dropped++;
1628	}
1629	xsk_buff_free(xdp: entry->xdp);
1630	entry->xdp = NULL;
1631	}
1632
1633	if (xdp_status)
1634	tsnep_finalize_xdp(adapter: rx->adapter, status: xdp_status, tx_nq, tx);
1635
1636	if (desc_available)
1637	desc_available -= tsnep_rx_refill_zc(rx, count: desc_available, reuse: false);
1638
1639	if (xsk_uses_need_wakeup(pool: rx->xsk_pool)) {
1640	tsnep_xsk_rx_need_wakeup(rx, desc_available);
1641
1642	return done;
1643	}
1644
1645	return desc_available ? budget : done;
1646	}
1647
1648	static bool tsnep_rx_pending(struct tsnep_rx *rx)
1649	{
1650	struct tsnep_rx_entry *entry;
1651
1652	if (rx->read != rx->write) {
1653	entry = &rx->entry[rx->read];
1654	if ((__le32_to_cpu(entry->desc_wb->properties) &
1655	TSNEP_DESC_OWNER_COUNTER_MASK) ==
1656	(entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))
1657	return true;
1658	}
1659
1660	return false;
1661	}
1662
1663	static int tsnep_rx_open(struct tsnep_rx *rx)
1664	{
1665	int desc_available;
1666	int retval;
1667
1668	retval = tsnep_rx_ring_create(rx);
1669	if (retval)
1670	return retval;
1671
1672	tsnep_rx_init(rx);
1673
1674	desc_available = tsnep_rx_desc_available(rx);
1675	if (rx->xsk_pool)
1676	retval = tsnep_rx_alloc_zc(rx, count: desc_available, reuse: false);
1677	else
1678	retval = tsnep_rx_alloc(rx, count: desc_available, reuse: false);
1679	if (retval != desc_available) {
1680	retval = -ENOMEM;
1681
1682	goto alloc_failed;
1683	}
1684
1685	/* prealloc pages to prevent allocation failures when XSK pool is
1686	* disabled at runtime
1687	*/
1688	if (rx->xsk_pool) {
1689	retval = tsnep_rx_alloc_page_buffer(rx);
1690	if (retval)
1691	goto alloc_failed;
1692	}
1693
1694	return 0;
1695
1696	alloc_failed:
1697	tsnep_rx_ring_cleanup(rx);
1698	return retval;
1699	}
1700
1701	static void tsnep_rx_close(struct tsnep_rx *rx)
1702	{
1703	if (rx->xsk_pool)
1704	tsnep_rx_free_page_buffer(rx);
1705
1706	tsnep_rx_ring_cleanup(rx);
1707	}
1708
1709	static void tsnep_rx_reopen(struct tsnep_rx *rx)
1710	{
1711	struct page **page = rx->page_buffer;
1712	int i;
1713
1714	tsnep_rx_init(rx);
1715
1716	for (i = 0; i < TSNEP_RING_SIZE; i++) {
1717	struct tsnep_rx_entry *entry = &rx->entry[i];
1718
1719	/* defined initial values for properties are required for
1720	* correct owner counter checking
1721	*/
1722	entry->desc->properties = 0;
1723	entry->desc_wb->properties = 0;
1724
1725	/* prevent allocation failures by reusing kept pages */
1726	if (*page) {
1727	tsnep_rx_set_page(rx, entry, page: *page);
1728	tsnep_rx_activate(rx, index: rx->write);
1729	rx->write++;
1730
1731	*page = NULL;
1732	page++;
1733	}
1734	}
1735	}
1736
1737	static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx)
1738	{
1739	struct page **page = rx->page_buffer;
1740	u32 allocated;
1741	int i;
1742
1743	tsnep_rx_init(rx);
1744
1745	/* alloc all ring entries except the last one, because ring cannot be
1746	* filled completely, as many buffers as possible is enough as wakeup is
1747	* done if new buffers are available
1748	*/
1749	allocated = xsk_buff_alloc_batch(pool: rx->xsk_pool, xdp: rx->xdp_batch,
1750	TSNEP_RING_SIZE - 1);
1751
1752	for (i = 0; i < TSNEP_RING_SIZE; i++) {
1753	struct tsnep_rx_entry *entry = &rx->entry[i];
1754
1755	/* keep pages to prevent allocation failures when xsk is
1756	* disabled
1757	*/
1758	if (entry->page) {
1759	*page = entry->page;
1760	entry->page = NULL;
1761
1762	page++;
1763	}
1764
1765	/* defined initial values for properties are required for
1766	* correct owner counter checking
1767	*/
1768	entry->desc->properties = 0;
1769	entry->desc_wb->properties = 0;
1770
1771	if (allocated) {
1772	tsnep_rx_set_xdp(rx, entry,
1773	xdp: rx->xdp_batch[allocated - 1]);
1774	tsnep_rx_activate(rx, index: rx->write);
1775	rx->write++;
1776
1777	allocated--;
1778	}
1779	}
1780
1781	/* set need wakeup flag immediately if ring is not filled completely,
1782	* first polling would be too late as need wakeup signalisation would
1783	* be delayed for an indefinite time
1784	*/
1785	if (xsk_uses_need_wakeup(pool: rx->xsk_pool))
1786	tsnep_xsk_rx_need_wakeup(rx, desc_available: tsnep_rx_desc_available(rx));
1787	}
1788
1789	static bool tsnep_pending(struct tsnep_queue *queue)
1790	{
1791	if (queue->tx && tsnep_tx_pending(tx: queue->tx))
1792	return true;
1793
1794	if (queue->rx && tsnep_rx_pending(rx: queue->rx))
1795	return true;
1796
1797	return false;
1798	}
1799
1800	static int tsnep_poll(struct napi_struct *napi, int budget)
1801	{
1802	struct tsnep_queue *queue = container_of(napi, struct tsnep_queue,
1803	napi);
1804	bool complete = true;
1805	int done = 0;
1806
1807	if (queue->tx)
1808	complete = tsnep_tx_poll(tx: queue->tx, napi_budget: budget);
1809
1810	/* handle case where we are called by netpoll with a budget of 0 */
1811	if (unlikely(budget <= 0))
1812	return budget;
1813
1814	if (queue->rx) {
1815	done = queue->rx->xsk_pool ?
1816	tsnep_rx_poll_zc(rx: queue->rx, napi, budget) :
1817	tsnep_rx_poll(rx: queue->rx, napi, budget);
1818	if (done >= budget)
1819	complete = false;
1820	}
1821
1822	/* if all work not completed, return budget and keep polling */
1823	if (!complete)
1824	return budget;
1825
1826	if (likely(napi_complete_done(napi, done))) {
1827	tsnep_enable_irq(adapter: queue->adapter, mask: queue->irq_mask);
1828
1829	/* reschedule if work is already pending, prevent rotten packets
1830	* which are transmitted or received after polling but before
1831	* interrupt enable
1832	*/
1833	if (tsnep_pending(queue)) {
1834	tsnep_disable_irq(adapter: queue->adapter, mask: queue->irq_mask);
1835	napi_schedule(n: napi);
1836	}
1837	}
1838
1839	return min(done, budget - 1);
1840	}
1841
1842	static int tsnep_request_irq(struct tsnep_queue *queue, bool first)
1843	{
1844	const char *name = netdev_name(dev: queue->adapter->netdev);
1845	irq_handler_t handler;
1846	void *dev;
1847	int retval;
1848
1849	if (first) {
1850	sprintf(buf: queue->name, fmt: "%s-mac", name);
1851	handler = tsnep_irq;
1852	dev = queue->adapter;
1853	} else {
1854	if (queue->tx && queue->rx)
1855	snprintf(buf: queue->name, size: sizeof(queue->name), fmt: "%s-txrx-%d",
1856	name, queue->rx->queue_index);
1857	else if (queue->tx)
1858	snprintf(buf: queue->name, size: sizeof(queue->name), fmt: "%s-tx-%d",
1859	name, queue->tx->queue_index);
1860	else
1861	snprintf(buf: queue->name, size: sizeof(queue->name), fmt: "%s-rx-%d",
1862	name, queue->rx->queue_index);
1863	handler = tsnep_irq_txrx;
1864	dev = queue;
1865	}
1866
1867	retval = request_irq(irq: queue->irq, handler, flags: 0, name: queue->name, dev);
1868	if (retval) {
1869	/* if name is empty, then interrupt won't be freed */
1870	memset(queue->name, 0, sizeof(queue->name));
1871	}
1872
1873	return retval;
1874	}
1875
1876	static void tsnep_free_irq(struct tsnep_queue *queue, bool first)
1877	{
1878	void *dev;
1879
1880	if (!strlen(queue->name))
1881	return;
1882
1883	if (first)
1884	dev = queue->adapter;
1885	else
1886	dev = queue;
1887
1888	free_irq(queue->irq, dev);
1889	memset(queue->name, 0, sizeof(queue->name));
1890	}
1891
1892	static void tsnep_queue_close(struct tsnep_queue *queue, bool first)
1893	{
1894	struct tsnep_rx *rx = queue->rx;
1895
1896	tsnep_free_irq(queue, first);
1897
1898	if (rx) {
1899	if (xdp_rxq_info_is_reg(xdp_rxq: &rx->xdp_rxq))
1900	xdp_rxq_info_unreg(xdp_rxq: &rx->xdp_rxq);
1901	if (xdp_rxq_info_is_reg(xdp_rxq: &rx->xdp_rxq_zc))
1902	xdp_rxq_info_unreg(xdp_rxq: &rx->xdp_rxq_zc);
1903	}
1904
1905	netif_napi_del(napi: &queue->napi);
1906	}
1907
1908	static int tsnep_queue_open(struct tsnep_adapter *adapter,
1909	struct tsnep_queue *queue, bool first)
1910	{
1911	struct tsnep_rx *rx = queue->rx;
1912	struct tsnep_tx *tx = queue->tx;
1913	int retval;
1914
1915	netif_napi_add(dev: adapter->netdev, napi: &queue->napi, poll: tsnep_poll);
1916
1917	if (rx) {
1918	/* choose TX queue for XDP_TX */
1919	if (tx)
1920	rx->tx_queue_index = tx->queue_index;
1921	else if (rx->queue_index < adapter->num_tx_queues)
1922	rx->tx_queue_index = rx->queue_index;
1923	else
1924	rx->tx_queue_index = 0;
1925
1926	/* prepare both memory models to eliminate possible registration
1927	* errors when memory model is switched between page pool and
1928	* XSK pool during runtime
1929	*/
1930	retval = xdp_rxq_info_reg(xdp_rxq: &rx->xdp_rxq, dev: adapter->netdev,
1931	queue_index: rx->queue_index, napi_id: queue->napi.napi_id);
1932	if (retval)
1933	goto failed;
1934	retval = xdp_rxq_info_reg_mem_model(xdp_rxq: &rx->xdp_rxq,
1935	type: MEM_TYPE_PAGE_POOL,
1936	allocator: rx->page_pool);
1937	if (retval)
1938	goto failed;
1939	retval = xdp_rxq_info_reg(xdp_rxq: &rx->xdp_rxq_zc, dev: adapter->netdev,
1940	queue_index: rx->queue_index, napi_id: queue->napi.napi_id);
1941	if (retval)
1942	goto failed;
1943	retval = xdp_rxq_info_reg_mem_model(xdp_rxq: &rx->xdp_rxq_zc,
1944	type: MEM_TYPE_XSK_BUFF_POOL,
1945	NULL);
1946	if (retval)
1947	goto failed;
1948	if (rx->xsk_pool)
1949	xsk_pool_set_rxq_info(pool: rx->xsk_pool, rxq: &rx->xdp_rxq_zc);
1950	}
1951
1952	retval = tsnep_request_irq(queue, first);
1953	if (retval) {
1954	netif_err(adapter, drv, adapter->netdev,
1955	"can't get assigned irq %d.\n", queue->irq);
1956	goto failed;
1957	}
1958
1959	return 0;
1960
1961	failed:
1962	tsnep_queue_close(queue, first);
1963
1964	return retval;
1965	}
1966
1967	static void tsnep_queue_enable(struct tsnep_queue *queue)
1968	{
1969	napi_enable(n: &queue->napi);
1970	tsnep_enable_irq(adapter: queue->adapter, mask: queue->irq_mask);
1971
1972	if (queue->tx)
1973	tsnep_tx_enable(tx: queue->tx);
1974
1975	if (queue->rx)
1976	tsnep_rx_enable(rx: queue->rx);
1977	}
1978
1979	static void tsnep_queue_disable(struct tsnep_queue *queue)
1980	{
1981	if (queue->tx)
1982	tsnep_tx_disable(tx: queue->tx, napi: &queue->napi);
1983
1984	napi_disable(n: &queue->napi);
1985	tsnep_disable_irq(adapter: queue->adapter, mask: queue->irq_mask);
1986
1987	/* disable RX after NAPI polling has been disabled, because RX can be
1988	* enabled during NAPI polling
1989	*/
1990	if (queue->rx)
1991	tsnep_rx_disable(rx: queue->rx);
1992	}
1993
1994	static int tsnep_netdev_open(struct net_device *netdev)
1995	{
1996	struct tsnep_adapter *adapter = netdev_priv(dev: netdev);
1997	int i, retval;
1998
1999	for (i = 0; i < adapter->num_queues; i++) {
2000	if (adapter->queue[i].tx) {
2001	retval = tsnep_tx_open(tx: adapter->queue[i].tx);
2002	if (retval)
2003	goto failed;
2004	}
2005	if (adapter->queue[i].rx) {
2006	retval = tsnep_rx_open(rx: adapter->queue[i].rx);
2007	if (retval)
2008	goto failed;
2009	}
2010
2011	retval = tsnep_queue_open(adapter, queue: &adapter->queue[i], first: i == 0);
2012	if (retval)
2013	goto failed;
2014	}
2015
2016	retval = netif_set_real_num_tx_queues(dev: adapter->netdev,
2017	txq: adapter->num_tx_queues);
2018	if (retval)
2019	goto failed;
2020	retval = netif_set_real_num_rx_queues(dev: adapter->netdev,
2021	rxq: adapter->num_rx_queues);
2022	if (retval)
2023	goto failed;
2024
2025	tsnep_enable_irq(adapter, ECM_INT_LINK);
2026	retval = tsnep_phy_open(adapter);
2027	if (retval)
2028	goto phy_failed;
2029
2030	for (i = 0; i < adapter->num_queues; i++)
2031	tsnep_queue_enable(queue: &adapter->queue[i]);
2032
2033	return 0;
2034
2035	phy_failed:
2036	tsnep_disable_irq(adapter, ECM_INT_LINK);
2037	failed:
2038	for (i = 0; i < adapter->num_queues; i++) {
2039	tsnep_queue_close(queue: &adapter->queue[i], first: i == 0);
2040
2041	if (adapter->queue[i].rx)
2042	tsnep_rx_close(rx: adapter->queue[i].rx);
2043	if (adapter->queue[i].tx)
2044	tsnep_tx_close(tx: adapter->queue[i].tx);
2045	}
2046	return retval;
2047	}
2048
2049	static int tsnep_netdev_close(struct net_device *netdev)
2050	{
2051	struct tsnep_adapter *adapter = netdev_priv(dev: netdev);
2052	int i;
2053
2054	tsnep_disable_irq(adapter, ECM_INT_LINK);
2055	tsnep_phy_close(adapter);
2056
2057	for (i = 0; i < adapter->num_queues; i++) {
2058	tsnep_queue_disable(queue: &adapter->queue[i]);
2059
2060	tsnep_queue_close(queue: &adapter->queue[i], first: i == 0);
2061
2062	if (adapter->queue[i].rx)
2063	tsnep_rx_close(rx: adapter->queue[i].rx);
2064	if (adapter->queue[i].tx)
2065	tsnep_tx_close(tx: adapter->queue[i].tx);
2066	}
2067
2068	return 0;
2069	}
2070
2071	int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool)
2072	{
2073	bool running = netif_running(dev: queue->adapter->netdev);
2074	u32 frame_size;
2075
2076	frame_size = xsk_pool_get_rx_frame_size(pool);
2077	if (frame_size < TSNEP_XSK_RX_BUF_SIZE)
2078	return -EOPNOTSUPP;
2079
2080	queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE,
2081	size: sizeof(*queue->rx->page_buffer),
2082	GFP_KERNEL);
2083	if (!queue->rx->page_buffer)
2084	return -ENOMEM;
2085	queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE,
2086	size: sizeof(*queue->rx->xdp_batch),
2087	GFP_KERNEL);
2088	if (!queue->rx->xdp_batch) {
2089	kfree(objp: queue->rx->page_buffer);
2090	queue->rx->page_buffer = NULL;
2091
2092	return -ENOMEM;
2093	}
2094
2095	xsk_pool_set_rxq_info(pool, rxq: &queue->rx->xdp_rxq_zc);
2096
2097	if (running)
2098	tsnep_queue_disable(queue);
2099
2100	queue->tx->xsk_pool = pool;
2101	queue->rx->xsk_pool = pool;
2102
2103	if (running) {
2104	tsnep_rx_reopen_xsk(rx: queue->rx);
2105	tsnep_queue_enable(queue);
2106	}
2107
2108	return 0;
2109	}
2110
2111	void tsnep_disable_xsk(struct tsnep_queue *queue)
2112	{
2113	bool running = netif_running(dev: queue->adapter->netdev);
2114
2115	if (running)
2116	tsnep_queue_disable(queue);
2117
2118	tsnep_rx_free_zc(rx: queue->rx);
2119
2120	queue->rx->xsk_pool = NULL;
2121	queue->tx->xsk_pool = NULL;
2122
2123	if (running) {
2124	tsnep_rx_reopen(rx: queue->rx);
2125	tsnep_queue_enable(queue);
2126	}
2127
2128	kfree(objp: queue->rx->xdp_batch);
2129	queue->rx->xdp_batch = NULL;
2130	kfree(objp: queue->rx->page_buffer);
2131	queue->rx->page_buffer = NULL;
2132	}
2133
2134	static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb,
2135	struct net_device *netdev)
2136	{
2137	struct tsnep_adapter *adapter = netdev_priv(dev: netdev);
2138	u16 queue_mapping = skb_get_queue_mapping(skb);
2139
2140	if (queue_mapping >= adapter->num_tx_queues)
2141	queue_mapping = 0;
2142
2143	return tsnep_xmit_frame_ring(skb, tx: &adapter->tx[queue_mapping]);
2144	}
2145
2146	static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr,
2147	int cmd)
2148	{
2149	if (!netif_running(dev: netdev))
2150	return -EINVAL;
2151	if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP)
2152	return tsnep_ptp_ioctl(netdev, ifr, cmd);
2153	return phy_mii_ioctl(phydev: netdev->phydev, ifr, cmd);
2154	}
2155
2156	static void tsnep_netdev_set_multicast(struct net_device *netdev)
2157	{
2158	struct tsnep_adapter *adapter = netdev_priv(dev: netdev);
2159
2160	u16 rx_filter = 0;
2161
2162	/* configured MAC address and broadcasts are never filtered */
2163	if (netdev->flags & IFF_PROMISC) {
2164	rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2165	rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS;
2166	} else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) {
2167	rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;
2168	}
2169	iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER);
2170	}
2171
2172	static void tsnep_netdev_get_stats64(struct net_device *netdev,
2173	struct rtnl_link_stats64 *stats)
2174	{
2175	struct tsnep_adapter *adapter = netdev_priv(dev: netdev);
2176	u32 reg;
2177	u32 val;
2178	int i;
2179
2180	for (i = 0; i < adapter->num_tx_queues; i++) {
2181	stats->tx_packets += adapter->tx[i].packets;
2182	stats->tx_bytes += adapter->tx[i].bytes;
2183	stats->tx_dropped += adapter->tx[i].dropped;
2184	}
2185	for (i = 0; i < adapter->num_rx_queues; i++) {
2186	stats->rx_packets += adapter->rx[i].packets;
2187	stats->rx_bytes += adapter->rx[i].bytes;
2188	stats->rx_dropped += adapter->rx[i].dropped;
2189	stats->multicast += adapter->rx[i].multicast;
2190
2191	reg = ioread32(adapter->addr + TSNEP_QUEUE(i) +
2192	TSNEP_RX_STATISTIC);
2193	val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >>
2194	TSNEP_RX_STATISTIC_NO_DESC_SHIFT;
2195	stats->rx_dropped += val;
2196	val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >>
2197	TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT;
2198	stats->rx_dropped += val;
2199	val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >>
2200	TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT;
2201	stats->rx_errors += val;
2202	stats->rx_fifo_errors += val;
2203	val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >>
2204	TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT;
2205	stats->rx_errors += val;
2206	stats->rx_frame_errors += val;
2207	}
2208
2209	reg = ioread32(adapter->addr + ECM_STAT);
2210	val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT;
2211	stats->rx_errors += val;
2212	val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT;
2213	stats->rx_errors += val;
2214	stats->rx_crc_errors += val;
2215	val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT;
2216	stats->rx_errors += val;
2217	}
2218
2219	static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr)
2220	{
2221	iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2222	iowrite16(*(u16 *)(addr + sizeof(u32)),
2223	adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2224
2225	ether_addr_copy(dst: adapter->mac_address, src: addr);
2226	netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n",
2227	addr);
2228	}
2229
2230	static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr)
2231	{
2232	struct tsnep_adapter *adapter = netdev_priv(dev: netdev);
2233	struct sockaddr *sock_addr = addr;
2234	int retval;
2235
2236	retval = eth_prepare_mac_addr_change(dev: netdev, p: sock_addr);
2237	if (retval)
2238	return retval;
2239	eth_hw_addr_set(dev: netdev, addr: sock_addr->sa_data);
2240	tsnep_mac_set_address(adapter, addr: sock_addr->sa_data);
2241
2242	return 0;
2243	}
2244
2245	static int tsnep_netdev_set_features(struct net_device *netdev,
2246	netdev_features_t features)
2247	{
2248	struct tsnep_adapter *adapter = netdev_priv(dev: netdev);
2249	netdev_features_t changed = netdev->features ^ features;
2250	bool enable;
2251	int retval = 0;
2252
2253	if (changed & NETIF_F_LOOPBACK) {
2254	enable = !!(features & NETIF_F_LOOPBACK);
2255	retval = tsnep_phy_loopback(adapter, enable);
2256	}
2257
2258	return retval;
2259	}
2260
2261	static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev,
2262	const struct skb_shared_hwtstamps *hwtstamps,
2263	bool cycles)
2264	{
2265	struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data;
2266	u64 timestamp;
2267
2268	if (cycles)
2269	timestamp = __le64_to_cpu(rx_inline->counter);
2270	else
2271	timestamp = __le64_to_cpu(rx_inline->timestamp);
2272
2273	return ns_to_ktime(ns: timestamp);
2274	}
2275
2276	static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf)
2277	{
2278	struct tsnep_adapter *adapter = netdev_priv(dev);
2279
2280	switch (bpf->command) {
2281	case XDP_SETUP_PROG:
2282	return tsnep_xdp_setup_prog(adapter, prog: bpf->prog, extack: bpf->extack);
2283	case XDP_SETUP_XSK_POOL:
2284	return tsnep_xdp_setup_pool(adapter, pool: bpf->xsk.pool,
2285	queue_id: bpf->xsk.queue_id);
2286	default:
2287	return -EOPNOTSUPP;
2288	}
2289	}
2290
2291	static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu)
2292	{
2293	if (cpu >= TSNEP_MAX_QUEUES)
2294	cpu &= TSNEP_MAX_QUEUES - 1;
2295
2296	while (cpu >= adapter->num_tx_queues)
2297	cpu -= adapter->num_tx_queues;
2298
2299	return &adapter->tx[cpu];
2300	}
2301
2302	static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n,
2303	struct xdp_frame **xdp, u32 flags)
2304	{
2305	struct tsnep_adapter *adapter = netdev_priv(dev);
2306	u32 cpu = smp_processor_id();
2307	struct netdev_queue *nq;
2308	struct tsnep_tx *tx;
2309	int nxmit;
2310	bool xmit;
2311
2312	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2313	return -EINVAL;
2314
2315	tx = tsnep_xdp_get_tx(adapter, cpu);
2316	nq = netdev_get_tx_queue(dev: adapter->netdev, index: tx->queue_index);
2317
2318	__netif_tx_lock(txq: nq, cpu);
2319
2320	for (nxmit = 0; nxmit < n; nxmit++) {
2321	xmit = tsnep_xdp_xmit_frame_ring(xdpf: xdp[nxmit], tx,
2322	TSNEP_TX_TYPE_XDP_NDO);
2323	if (!xmit)
2324	break;
2325
2326	/* avoid transmit queue timeout since we share it with the slow
2327	* path
2328	*/
2329	txq_trans_cond_update(txq: nq);
2330	}
2331
2332	if (flags & XDP_XMIT_FLUSH)
2333	tsnep_xdp_xmit_flush(tx);
2334
2335	__netif_tx_unlock(txq: nq);
2336
2337	return nxmit;
2338	}
2339
2340	static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id,
2341	u32 flags)
2342	{
2343	struct tsnep_adapter *adapter = netdev_priv(dev);
2344	struct tsnep_queue *queue;
2345
2346	if (queue_id >= adapter->num_rx_queues ||
2347	queue_id >= adapter->num_tx_queues)
2348	return -EINVAL;
2349
2350	queue = &adapter->queue[queue_id];
2351
2352	if (!napi_if_scheduled_mark_missed(n: &queue->napi))
2353	napi_schedule(n: &queue->napi);
2354
2355	return 0;
2356	}
2357
2358	static const struct net_device_ops tsnep_netdev_ops = {
2359	.ndo_open = tsnep_netdev_open,
2360	.ndo_stop = tsnep_netdev_close,
2361	.ndo_start_xmit = tsnep_netdev_xmit_frame,
2362	.ndo_eth_ioctl = tsnep_netdev_ioctl,
2363	.ndo_set_rx_mode = tsnep_netdev_set_multicast,
2364	.ndo_get_stats64 = tsnep_netdev_get_stats64,
2365	.ndo_set_mac_address = tsnep_netdev_set_mac_address,
2366	.ndo_set_features = tsnep_netdev_set_features,
2367	.ndo_get_tstamp = tsnep_netdev_get_tstamp,
2368	.ndo_setup_tc = tsnep_tc_setup,
2369	.ndo_bpf = tsnep_netdev_bpf,
2370	.ndo_xdp_xmit = tsnep_netdev_xdp_xmit,
2371	.ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup,
2372	};
2373
2374	static int tsnep_mac_init(struct tsnep_adapter *adapter)
2375	{
2376	int retval;
2377
2378	/* initialize RX filtering, at least configured MAC address and
2379	* broadcast are not filtered
2380	*/
2381	iowrite16(0, adapter->addr + TSNEP_RX_FILTER);
2382
2383	/* try to get MAC address in the following order:
2384	* - device tree
2385	* - valid MAC address already set
2386	* - MAC address register if valid
2387	* - random MAC address
2388	*/
2389	retval = of_get_mac_address(np: adapter->pdev->dev.of_node,
2390	mac: adapter->mac_address);
2391	if (retval == -EPROBE_DEFER)
2392	return retval;
2393	if (retval && !is_valid_ether_addr(addr: adapter->mac_address)) {
2394	*(u32 *)adapter->mac_address =
2395	ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW);
2396	*(u16 *)(adapter->mac_address + sizeof(u32)) =
2397	ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH);
2398	if (!is_valid_ether_addr(addr: adapter->mac_address))
2399	eth_random_addr(addr: adapter->mac_address);
2400	}
2401
2402	tsnep_mac_set_address(adapter, addr: adapter->mac_address);
2403	eth_hw_addr_set(dev: adapter->netdev, addr: adapter->mac_address);
2404
2405	return 0;
2406	}
2407
2408	static int tsnep_mdio_init(struct tsnep_adapter *adapter)
2409	{
2410	struct device_node *np = adapter->pdev->dev.of_node;
2411	int retval;
2412
2413	if (np) {
2414	np = of_get_child_by_name(node: np, name: "mdio");
2415	if (!np)
2416	return 0;
2417
2418	adapter->suppress_preamble =
2419	of_property_read_bool(np, propname: "suppress-preamble");
2420	}
2421
2422	adapter->mdiobus = devm_mdiobus_alloc(dev: &adapter->pdev->dev);
2423	if (!adapter->mdiobus) {
2424	retval = -ENOMEM;
2425
2426	goto out;
2427	}
2428
2429	adapter->mdiobus->priv = (void *)adapter;
2430	adapter->mdiobus->parent = &adapter->pdev->dev;
2431	adapter->mdiobus->read = tsnep_mdiobus_read;
2432	adapter->mdiobus->write = tsnep_mdiobus_write;
2433	adapter->mdiobus->name = TSNEP "-mdiobus";
2434	snprintf(buf: adapter->mdiobus->id, MII_BUS_ID_SIZE, fmt: "%s",
2435	adapter->pdev->name);
2436
2437	/* do not scan broadcast address */
2438	adapter->mdiobus->phy_mask = 0x0000001;
2439
2440	retval = of_mdiobus_register(mdio: adapter->mdiobus, np);
2441
2442	out:
2443	of_node_put(node: np);
2444
2445	return retval;
2446	}
2447
2448	static int tsnep_phy_init(struct tsnep_adapter *adapter)
2449	{
2450	struct device_node *phy_node;
2451	int retval;
2452
2453	retval = of_get_phy_mode(np: adapter->pdev->dev.of_node,
2454	interface: &adapter->phy_mode);
2455	if (retval)
2456	adapter->phy_mode = PHY_INTERFACE_MODE_GMII;
2457
2458	phy_node = of_parse_phandle(np: adapter->pdev->dev.of_node, phandle_name: "phy-handle",
2459	index: 0);
2460	adapter->phydev = of_phy_find_device(phy_np: phy_node);
2461	of_node_put(node: phy_node);
2462	if (!adapter->phydev && adapter->mdiobus)
2463	adapter->phydev = phy_find_first(bus: adapter->mdiobus);
2464	if (!adapter->phydev)
2465	return -EIO;
2466
2467	return 0;
2468	}
2469
2470	static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)
2471	{
2472	u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0;
2473	char name[8];
2474	int i;
2475	int retval;
2476
2477	/* one TX/RX queue pair for netdev is mandatory */
2478	if (platform_irq_count(adapter->pdev) == 1)
2479	retval = platform_get_irq(adapter->pdev, 0);
2480	else
2481	retval = platform_get_irq_byname(adapter->pdev, "mac");
2482	if (retval < 0)
2483	return retval;
2484	adapter->num_tx_queues = 1;
2485	adapter->num_rx_queues = 1;
2486	adapter->num_queues = 1;
2487	adapter->queue[0].adapter = adapter;
2488	adapter->queue[0].irq = retval;
2489	adapter->queue[0].tx = &adapter->tx[0];
2490	adapter->queue[0].tx->adapter = adapter;
2491	adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0);
2492	adapter->queue[0].tx->queue_index = 0;
2493	adapter->queue[0].rx = &adapter->rx[0];
2494	adapter->queue[0].rx->adapter = adapter;
2495	adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0);
2496	adapter->queue[0].rx->queue_index = 0;
2497	adapter->queue[0].irq_mask = irq_mask;
2498	adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY;
2499	retval = tsnep_set_irq_coalesce(queue: &adapter->queue[0],
2500	TSNEP_COALESCE_USECS_DEFAULT);
2501	if (retval < 0)
2502	return retval;
2503
2504	adapter->netdev->irq = adapter->queue[0].irq;
2505
2506	/* add additional TX/RX queue pairs only if dedicated interrupt is
2507	* available
2508	*/
2509	for (i = 1; i < queue_count; i++) {
2510	sprintf(buf: name, fmt: "txrx-%d", i);
2511	retval = platform_get_irq_byname_optional(dev: adapter->pdev, name);
2512	if (retval < 0)
2513	break;
2514
2515	adapter->num_tx_queues++;
2516	adapter->num_rx_queues++;
2517	adapter->num_queues++;
2518	adapter->queue[i].adapter = adapter;
2519	adapter->queue[i].irq = retval;
2520	adapter->queue[i].tx = &adapter->tx[i];
2521	adapter->queue[i].tx->adapter = adapter;
2522	adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i);
2523	adapter->queue[i].tx->queue_index = i;
2524	adapter->queue[i].rx = &adapter->rx[i];
2525	adapter->queue[i].rx->adapter = adapter;
2526	adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i);
2527	adapter->queue[i].rx->queue_index = i;
2528	adapter->queue[i].irq_mask =
2529	irq_mask << (ECM_INT_TXRX_SHIFT * i);
2530	adapter->queue[i].irq_delay_addr =
2531	adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i;
2532	retval = tsnep_set_irq_coalesce(queue: &adapter->queue[i],
2533	TSNEP_COALESCE_USECS_DEFAULT);
2534	if (retval < 0)
2535	return retval;
2536	}
2537
2538	return 0;
2539	}
2540
2541	static int tsnep_probe(struct platform_device *pdev)
2542	{
2543	struct tsnep_adapter *adapter;
2544	struct net_device *netdev;
2545	struct resource *io;
2546	u32 type;
2547	int revision;
2548	int version;
2549	int queue_count;
2550	int retval;
2551
2552	netdev = devm_alloc_etherdev_mqs(dev: &pdev->dev,
2553	sizeof_priv: sizeof(struct tsnep_adapter),
2554	TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES);
2555	if (!netdev)
2556	return -ENODEV;
2557	SET_NETDEV_DEV(netdev, &pdev->dev);
2558	adapter = netdev_priv(dev: netdev);
2559	platform_set_drvdata(pdev, data: adapter);
2560	adapter->pdev = pdev;
2561	adapter->dmadev = &pdev->dev;
2562	adapter->netdev = netdev;
2563	adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |
2564	NETIF_MSG_LINK | NETIF_MSG_IFUP |
2565	NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;
2566
2567	netdev->min_mtu = ETH_MIN_MTU;
2568	netdev->max_mtu = TSNEP_MAX_FRAME_SIZE;
2569
2570	mutex_init(&adapter->gate_control_lock);
2571	mutex_init(&adapter->rxnfc_lock);
2572	INIT_LIST_HEAD(list: &adapter->rxnfc_rules);
2573
2574	adapter->addr = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &io);
2575	if (IS_ERR(ptr: adapter->addr))
2576	return PTR_ERR(ptr: adapter->addr);
2577	netdev->mem_start = io->start;
2578	netdev->mem_end = io->end;
2579
2580	type = ioread32(adapter->addr + ECM_TYPE);
2581	revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT;
2582	version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT;
2583	queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT;
2584	adapter->gate_control = type & ECM_GATE_CONTROL;
2585	adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT;
2586
2587	tsnep_disable_irq(adapter, ECM_INT_ALL);
2588
2589	retval = tsnep_queue_init(adapter, queue_count);
2590	if (retval)
2591	return retval;
2592
2593	retval = dma_set_mask_and_coherent(dev: &adapter->pdev->dev,
2594	DMA_BIT_MASK(64));
2595	if (retval) {
2596	dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n");
2597	return retval;
2598	}
2599
2600	retval = tsnep_mac_init(adapter);
2601	if (retval)
2602	return retval;
2603
2604	retval = tsnep_mdio_init(adapter);
2605	if (retval)
2606	goto mdio_init_failed;
2607
2608	retval = tsnep_phy_init(adapter);
2609	if (retval)
2610	goto phy_init_failed;
2611
2612	retval = tsnep_ptp_init(adapter);
2613	if (retval)
2614	goto ptp_init_failed;
2615
2616	retval = tsnep_tc_init(adapter);
2617	if (retval)
2618	goto tc_init_failed;
2619
2620	retval = tsnep_rxnfc_init(adapter);
2621	if (retval)
2622	goto rxnfc_init_failed;
2623
2624	netdev->netdev_ops = &tsnep_netdev_ops;
2625	netdev->ethtool_ops = &tsnep_ethtool_ops;
2626	netdev->features = NETIF_F_SG;
2627	netdev->hw_features = netdev->features | NETIF_F_LOOPBACK;
2628
2629	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
2630	NETDEV_XDP_ACT_NDO_XMIT |
2631	NETDEV_XDP_ACT_NDO_XMIT_SG |
2632	NETDEV_XDP_ACT_XSK_ZEROCOPY;
2633
2634	/* carrier off reporting is important to ethtool even BEFORE open */
2635	netif_carrier_off(dev: netdev);
2636
2637	retval = register_netdev(dev: netdev);
2638	if (retval)
2639	goto register_failed;
2640
2641	dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version,
2642	revision);
2643	if (adapter->gate_control)
2644	dev_info(&adapter->pdev->dev, "gate control detected\n");
2645
2646	return 0;
2647
2648	register_failed:
2649	tsnep_rxnfc_cleanup(adapter);
2650	rxnfc_init_failed:
2651	tsnep_tc_cleanup(adapter);
2652	tc_init_failed:
2653	tsnep_ptp_cleanup(adapter);
2654	ptp_init_failed:
2655	phy_init_failed:
2656	if (adapter->mdiobus)
2657	mdiobus_unregister(bus: adapter->mdiobus);
2658	mdio_init_failed:
2659	return retval;
2660	}
2661
2662	static void tsnep_remove(struct platform_device *pdev)
2663	{
2664	struct tsnep_adapter *adapter = platform_get_drvdata(pdev);
2665
2666	unregister_netdev(dev: adapter->netdev);
2667
2668	tsnep_rxnfc_cleanup(adapter);
2669
2670	tsnep_tc_cleanup(adapter);
2671
2672	tsnep_ptp_cleanup(adapter);
2673
2674	if (adapter->mdiobus)
2675	mdiobus_unregister(bus: adapter->mdiobus);
2676
2677	tsnep_disable_irq(adapter, ECM_INT_ALL);
2678	}
2679
2680	static const struct of_device_id tsnep_of_match[] = {
2681	{ .compatible = "engleder,tsnep", },
2682	{ },
2683	};
2684	MODULE_DEVICE_TABLE(of, tsnep_of_match);
2685
2686	static struct platform_driver tsnep_driver = {
2687	.driver = {
2688	.name = TSNEP,
2689	.of_match_table = tsnep_of_match,
2690	},
2691	.probe = tsnep_probe,
2692	.remove_new = tsnep_remove,
2693	};
2694	module_platform_driver(tsnep_driver);
2695
2696	MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>");
2697	MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver");
2698	MODULE_LICENSE("GPL");
2699