1	// SPDX-License-Identifier: GPL-2.0-only
2	/* 10G controller driver for Samsung SoCs
3	*
4	* Copyright (C) 2013 Samsung Electronics Co., Ltd.
5	* http://www.samsung.com
6	*
7	* Author: Siva Reddy Kallam <siva.kallam@samsung.com>
8	*/
9
10	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12	#include <linux/clk.h>
13	#include <linux/crc32.h>
14	#include <linux/dma-mapping.h>
15	#include <linux/etherdevice.h>
16	#include <linux/ethtool.h>
17	#include <linux/if.h>
18	#include <linux/if_ether.h>
19	#include <linux/if_vlan.h>
20	#include <linux/init.h>
21	#include <linux/interrupt.h>
22	#include <linux/ip.h>
23	#include <linux/kernel.h>
24	#include <linux/mii.h>
25	#include <linux/module.h>
26	#include <linux/net_tstamp.h>
27	#include <linux/netdevice.h>
28	#include <linux/phy.h>
29	#include <linux/platform_device.h>
30	#include <linux/prefetch.h>
31	#include <linux/skbuff.h>
32	#include <linux/slab.h>
33	#include <linux/tcp.h>
34	#include <linux/sxgbe_platform.h>
35
36	#include "sxgbe_common.h"
37	#include "sxgbe_desc.h"
38	#include "sxgbe_dma.h"
39	#include "sxgbe_mtl.h"
40	#include "sxgbe_reg.h"
41
42	#define SXGBE_ALIGN(x) L1_CACHE_ALIGN(x)
43	#define JUMBO_LEN 9000
44
45	/* Module parameters */
46	#define TX_TIMEO 5000
47	#define DMA_TX_SIZE 512
48	#define DMA_RX_SIZE 1024
49	#define TC_DEFAULT 64
50	#define DMA_BUFFER_SIZE BUF_SIZE_2KiB
51	/* The default timer value as per the sxgbe specification 1 sec(1000 ms) */
52	#define SXGBE_DEFAULT_LPI_TIMER 1000
53
54	static int debug = -1;
55	static int eee_timer = SXGBE_DEFAULT_LPI_TIMER;
56
57	module_param(eee_timer, int, 0644);
58
59	module_param(debug, int, 0644);
60	static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
61	NETIF_MSG_LINK | NETIF_MSG_IFUP |
62	NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
63
64	static irqreturn_t sxgbe_common_interrupt(int irq, void *dev_id);
65	static irqreturn_t sxgbe_tx_interrupt(int irq, void *dev_id);
66	static irqreturn_t sxgbe_rx_interrupt(int irq, void *dev_id);
67
68	#define SXGBE_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
69
70	#define SXGBE_LPI_TIMER(x) (jiffies + msecs_to_jiffies(x))
71
72	/**
73	* sxgbe_verify_args - verify the driver parameters.
74	* Description: it verifies if some wrong parameter is passed to the driver.
75	* Note that wrong parameters are replaced with the default values.
76	*/
77	static void sxgbe_verify_args(void)
78	{
79	if (unlikely(eee_timer < 0))
80	eee_timer = SXGBE_DEFAULT_LPI_TIMER;
81	}
82
83	static void sxgbe_enable_eee_mode(const struct sxgbe_priv_data *priv)
84	{
85	/* Check and enter in LPI mode */
86	if (!priv->tx_path_in_lpi_mode)
87	priv->hw->mac->set_eee_mode(priv->ioaddr);
88	}
89
90	void sxgbe_disable_eee_mode(struct sxgbe_priv_data * const priv)
91	{
92	/* Exit and disable EEE in case of we are in LPI state. */
93	priv->hw->mac->reset_eee_mode(priv->ioaddr);
94	del_timer_sync(timer: &priv->eee_ctrl_timer);
95	priv->tx_path_in_lpi_mode = false;
96	}
97
98	/**
99	* sxgbe_eee_ctrl_timer
100	* @t: timer list containing a data
101	* Description:
102	* If there is no data transfer and if we are not in LPI state,
103	* then MAC Transmitter can be moved to LPI state.
104	*/
105	static void sxgbe_eee_ctrl_timer(struct timer_list *t)
106	{
107	struct sxgbe_priv_data *priv = from_timer(priv, t, eee_ctrl_timer);
108
109	sxgbe_enable_eee_mode(priv);
110	mod_timer(timer: &priv->eee_ctrl_timer, SXGBE_LPI_TIMER(eee_timer));
111	}
112
113	/**
114	* sxgbe_eee_init
115	* @priv: private device pointer
116	* Description:
117	* If the EEE support has been enabled while configuring the driver,
118	* if the GMAC actually supports the EEE (from the HW cap reg) and the
119	* phy can also manage EEE, so enable the LPI state and start the timer
120	* to verify if the tx path can enter in LPI state.
121	*/
122	bool sxgbe_eee_init(struct sxgbe_priv_data * const priv)
123	{
124	struct net_device *ndev = priv->dev;
125	bool ret = false;
126
127	/* MAC core supports the EEE feature. */
128	if (priv->hw_cap.eee) {
129	/* Check if the PHY supports EEE */
130	if (phy_init_eee(phydev: ndev->phydev, clk_stop_enable: true))
131	return false;
132
133	timer_setup(&priv->eee_ctrl_timer, sxgbe_eee_ctrl_timer, 0);
134	priv->eee_ctrl_timer.expires = SXGBE_LPI_TIMER(eee_timer);
135	add_timer(timer: &priv->eee_ctrl_timer);
136
137	priv->hw->mac->set_eee_timer(priv->ioaddr,
138	SXGBE_DEFAULT_LPI_TIMER,
139	priv->tx_lpi_timer);
140
141	pr_info("Energy-Efficient Ethernet initialized\n");
142
143	ret = true;
144	}
145
146	return ret;
147	}
148
149	static void sxgbe_eee_adjust(const struct sxgbe_priv_data *priv)
150	{
151	struct net_device *ndev = priv->dev;
152
153	/* When the EEE has been already initialised we have to
154	* modify the PLS bit in the LPI ctrl & status reg according
155	* to the PHY link status. For this reason.
156	*/
157	if (priv->eee_enabled)
158	priv->hw->mac->set_eee_pls(priv->ioaddr, ndev->phydev->link);
159	}
160
161	/**
162	* sxgbe_clk_csr_set - dynamically set the MDC clock
163	* @priv: driver private structure
164	* Description: this is to dynamically set the MDC clock according to the csr
165	* clock input.
166	*/
167	static void sxgbe_clk_csr_set(struct sxgbe_priv_data *priv)
168	{
169	u32 clk_rate = clk_get_rate(clk: priv->sxgbe_clk);
170
171	/* assign the proper divider, this will be used during
172	* mdio communication
173	*/
174	if (clk_rate < SXGBE_CSR_F_150M)
175	priv->clk_csr = SXGBE_CSR_100_150M;
176	else if (clk_rate <= SXGBE_CSR_F_250M)
177	priv->clk_csr = SXGBE_CSR_150_250M;
178	else if (clk_rate <= SXGBE_CSR_F_300M)
179	priv->clk_csr = SXGBE_CSR_250_300M;
180	else if (clk_rate <= SXGBE_CSR_F_350M)
181	priv->clk_csr = SXGBE_CSR_300_350M;
182	else if (clk_rate <= SXGBE_CSR_F_400M)
183	priv->clk_csr = SXGBE_CSR_350_400M;
184	else if (clk_rate <= SXGBE_CSR_F_500M)
185	priv->clk_csr = SXGBE_CSR_400_500M;
186	}
187
188	/* minimum number of free TX descriptors required to wake up TX process */
189	#define SXGBE_TX_THRESH(x) (x->dma_tx_size/4)
190
191	static inline u32 sxgbe_tx_avail(struct sxgbe_tx_queue *queue, int tx_qsize)
192	{
193	return queue->dirty_tx + tx_qsize - queue->cur_tx - 1;
194	}
195
196	/**
197	* sxgbe_adjust_link
198	* @dev: net device structure
199	* Description: it adjusts the link parameters.
200	*/
201	static void sxgbe_adjust_link(struct net_device *dev)
202	{
203	struct sxgbe_priv_data *priv = netdev_priv(dev);
204	struct phy_device *phydev = dev->phydev;
205	u8 new_state = 0;
206	u8 speed = 0xff;
207
208	if (!phydev)
209	return;
210
211	/* SXGBE is not supporting auto-negotiation and
212	* half duplex mode. so, not handling duplex change
213	* in this function. only handling speed and link status
214	*/
215	if (phydev->link) {
216	if (phydev->speed != priv->speed) {
217	new_state = 1;
218	switch (phydev->speed) {
219	case SPEED_10000:
220	speed = SXGBE_SPEED_10G;
221	break;
222	case SPEED_2500:
223	speed = SXGBE_SPEED_2_5G;
224	break;
225	case SPEED_1000:
226	speed = SXGBE_SPEED_1G;
227	break;
228	default:
229	netif_err(priv, link, dev,
230	"Speed (%d) not supported\n",
231	phydev->speed);
232	}
233
234	priv->speed = phydev->speed;
235	priv->hw->mac->set_speed(priv->ioaddr, speed);
236	}
237
238	if (!priv->oldlink) {
239	new_state = 1;
240	priv->oldlink = 1;
241	}
242	} else if (priv->oldlink) {
243	new_state = 1;
244	priv->oldlink = 0;
245	priv->speed = SPEED_UNKNOWN;
246	}
247
248	if (new_state & netif_msg_link(priv))
249	phy_print_status(phydev);
250
251	/* Alter the MAC settings for EEE */
252	sxgbe_eee_adjust(priv);
253	}
254
255	/**
256	* sxgbe_init_phy - PHY initialization
257	* @ndev: net device structure
258	* Description: it initializes the driver's PHY state, and attaches the PHY
259	* to the mac driver.
260	* Return value:
261	* 0 on success
262	*/
263	static int sxgbe_init_phy(struct net_device *ndev)
264	{
265	char phy_id_fmt[MII_BUS_ID_SIZE + 3];
266	char bus_id[MII_BUS_ID_SIZE];
267	struct phy_device *phydev;
268	struct sxgbe_priv_data *priv = netdev_priv(dev: ndev);
269	int phy_iface = priv->plat->interface;
270
271	/* assign default link status */
272	priv->oldlink = 0;
273	priv->speed = SPEED_UNKNOWN;
274	priv->oldduplex = DUPLEX_UNKNOWN;
275
276	if (priv->plat->phy_bus_name)
277	snprintf(buf: bus_id, MII_BUS_ID_SIZE, fmt: "%s-%x",
278	priv->plat->phy_bus_name, priv->plat->bus_id);
279	else
280	snprintf(buf: bus_id, MII_BUS_ID_SIZE, fmt: "sxgbe-%x",
281	priv->plat->bus_id);
282
283	snprintf(buf: phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
284	priv->plat->phy_addr);
285	netdev_dbg(ndev, "%s: trying to attach to %s\n", __func__, phy_id_fmt);
286
287	phydev = phy_connect(dev: ndev, bus_id: phy_id_fmt, handler: &sxgbe_adjust_link, interface: phy_iface);
288
289	if (IS_ERR(ptr: phydev)) {
290	netdev_err(dev: ndev, format: "Could not attach to PHY\n");
291	return PTR_ERR(ptr: phydev);
292	}
293
294	/* Stop Advertising 1000BASE Capability if interface is not GMII */
295	if ((phy_iface == PHY_INTERFACE_MODE_MII) ||
296	(phy_iface == PHY_INTERFACE_MODE_RMII))
297	phy_set_max_speed(phydev, SPEED_1000);
298
299	if (phydev->phy_id == 0) {
300	phy_disconnect(phydev);
301	return -ENODEV;
302	}
303
304	netdev_dbg(ndev, "%s: attached to PHY (UID 0x%x) Link = %d\n",
305	__func__, phydev->phy_id, phydev->link);
306
307	return 0;
308	}
309
310	/**
311	* sxgbe_clear_descriptors: clear descriptors
312	* @priv: driver private structure
313	* Description: this function is called to clear the tx and rx descriptors
314	* in case of both basic and extended descriptors are used.
315	*/
316	static void sxgbe_clear_descriptors(struct sxgbe_priv_data *priv)
317	{
318	int i, j;
319	unsigned int txsize = priv->dma_tx_size;
320	unsigned int rxsize = priv->dma_rx_size;
321
322	/* Clear the Rx/Tx descriptors */
323	for (j = 0; j < SXGBE_RX_QUEUES; j++) {
324	for (i = 0; i < rxsize; i++)
325	priv->hw->desc->init_rx_desc(&priv->rxq[j]->dma_rx[i],
326	priv->use_riwt, priv->mode,
327	(i == rxsize - 1));
328	}
329
330	for (j = 0; j < SXGBE_TX_QUEUES; j++) {
331	for (i = 0; i < txsize; i++)
332	priv->hw->desc->init_tx_desc(&priv->txq[j]->dma_tx[i]);
333	}
334	}
335
336	static int sxgbe_init_rx_buffers(struct net_device *dev,
337	struct sxgbe_rx_norm_desc *p, int i,
338	unsigned int dma_buf_sz,
339	struct sxgbe_rx_queue *rx_ring)
340	{
341	struct sxgbe_priv_data *priv = netdev_priv(dev);
342	struct sk_buff *skb;
343
344	skb = __netdev_alloc_skb_ip_align(dev, length: dma_buf_sz, GFP_KERNEL);
345	if (!skb)
346	return -ENOMEM;
347
348	rx_ring->rx_skbuff[i] = skb;
349	rx_ring->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
350	dma_buf_sz, DMA_FROM_DEVICE);
351
352	if (dma_mapping_error(dev: priv->device, dma_addr: rx_ring->rx_skbuff_dma[i])) {
353	netdev_err(dev, format: "%s: DMA mapping error\n", __func__);
354	dev_kfree_skb_any(skb);
355	return -EINVAL;
356	}
357
358	p->rdes23.rx_rd_des23.buf2_addr = rx_ring->rx_skbuff_dma[i];
359
360	return 0;
361	}
362
363	/**
364	* sxgbe_free_rx_buffers - free what sxgbe_init_rx_buffers() allocated
365	* @dev: net device structure
366	* @p: dec pointer
367	* @i: index
368	* @dma_buf_sz: size
369	* @rx_ring: ring to be freed
370	*
371	* Description: this function initializes the DMA RX descriptor
372	*/
373	static void sxgbe_free_rx_buffers(struct net_device *dev,
374	struct sxgbe_rx_norm_desc *p, int i,
375	unsigned int dma_buf_sz,
376	struct sxgbe_rx_queue *rx_ring)
377	{
378	struct sxgbe_priv_data *priv = netdev_priv(dev);
379
380	kfree_skb(skb: rx_ring->rx_skbuff[i]);
381	dma_unmap_single(priv->device, rx_ring->rx_skbuff_dma[i],
382	dma_buf_sz, DMA_FROM_DEVICE);
383	}
384
385	/**
386	* init_tx_ring - init the TX descriptor ring
387	* @dev: net device structure
388	* @queue_no: queue
389	* @tx_ring: ring to be initialised
390	* @tx_rsize: ring size
391	* Description: this function initializes the DMA TX descriptor
392	*/
393	static int init_tx_ring(struct device *dev, u8 queue_no,
394	struct sxgbe_tx_queue *tx_ring, int tx_rsize)
395	{
396	/* TX ring is not allcoated */
397	if (!tx_ring) {
398	dev_err(dev, "No memory for TX queue of SXGBE\n");
399	return -ENOMEM;
400	}
401
402	/* allocate memory for TX descriptors */
403	tx_ring->dma_tx = dma_alloc_coherent(dev,
404	size: tx_rsize * sizeof(struct sxgbe_tx_norm_desc),
405	dma_handle: &tx_ring->dma_tx_phy, GFP_KERNEL);
406	if (!tx_ring->dma_tx)
407	return -ENOMEM;
408
409	/* allocate memory for TX skbuff array */
410	tx_ring->tx_skbuff_dma = devm_kcalloc(dev, n: tx_rsize,
411	size: sizeof(dma_addr_t), GFP_KERNEL);
412	if (!tx_ring->tx_skbuff_dma)
413	goto dmamem_err;
414
415	tx_ring->tx_skbuff = devm_kcalloc(dev, n: tx_rsize,
416	size: sizeof(struct sk_buff *), GFP_KERNEL);
417
418	if (!tx_ring->tx_skbuff)
419	goto dmamem_err;
420
421	/* assign queue number */
422	tx_ring->queue_no = queue_no;
423
424	/* initialise counters */
425	tx_ring->dirty_tx = 0;
426	tx_ring->cur_tx = 0;
427
428	return 0;
429
430	dmamem_err:
431	dma_free_coherent(dev, size: tx_rsize * sizeof(struct sxgbe_tx_norm_desc),
432	cpu_addr: tx_ring->dma_tx, dma_handle: tx_ring->dma_tx_phy);
433	return -ENOMEM;
434	}
435
436	/**
437	* free_rx_ring - free the RX descriptor ring
438	* @dev: net device structure
439	* @rx_ring: ring to be initialised
440	* @rx_rsize: ring size
441	* Description: this function initializes the DMA RX descriptor
442	*/
443	static void free_rx_ring(struct device *dev, struct sxgbe_rx_queue *rx_ring,
444	int rx_rsize)
445	{
446	dma_free_coherent(dev, size: rx_rsize * sizeof(struct sxgbe_rx_norm_desc),
447	cpu_addr: rx_ring->dma_rx, dma_handle: rx_ring->dma_rx_phy);
448	kfree(objp: rx_ring->rx_skbuff_dma);
449	kfree(objp: rx_ring->rx_skbuff);
450	}
451
452	/**
453	* init_rx_ring - init the RX descriptor ring
454	* @dev: net device structure
455	* @queue_no: queue
456	* @rx_ring: ring to be initialised
457	* @rx_rsize: ring size
458	* Description: this function initializes the DMA RX descriptor
459	*/
460	static int init_rx_ring(struct net_device *dev, u8 queue_no,
461	struct sxgbe_rx_queue *rx_ring, int rx_rsize)
462	{
463	struct sxgbe_priv_data *priv = netdev_priv(dev);
464	int desc_index;
465	unsigned int bfsize = 0;
466	unsigned int ret = 0;
467
468	/* Set the max buffer size according to the MTU. */
469	bfsize = ALIGN(dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN, 8);
470
471	netif_dbg(priv, probe, dev, "%s: bfsize %d\n", __func__, bfsize);
472
473	/* RX ring is not allcoated */
474	if (rx_ring == NULL) {
475	netdev_err(dev, format: "No memory for RX queue\n");
476	return -ENOMEM;
477	}
478
479	/* assign queue number */
480	rx_ring->queue_no = queue_no;
481
482	/* allocate memory for RX descriptors */
483	rx_ring->dma_rx = dma_alloc_coherent(dev: priv->device,
484	size: rx_rsize * sizeof(struct sxgbe_rx_norm_desc),
485	dma_handle: &rx_ring->dma_rx_phy, GFP_KERNEL);
486
487	if (rx_ring->dma_rx == NULL)
488	return -ENOMEM;
489
490	/* allocate memory for RX skbuff array */
491	rx_ring->rx_skbuff_dma = kmalloc_array(n: rx_rsize,
492	size: sizeof(dma_addr_t), GFP_KERNEL);
493	if (!rx_ring->rx_skbuff_dma) {
494	ret = -ENOMEM;
495	goto err_free_dma_rx;
496	}
497
498	rx_ring->rx_skbuff = kmalloc_array(n: rx_rsize,
499	size: sizeof(struct sk_buff *), GFP_KERNEL);
500	if (!rx_ring->rx_skbuff) {
501	ret = -ENOMEM;
502	goto err_free_skbuff_dma;
503	}
504
505	/* initialise the buffers */
506	for (desc_index = 0; desc_index < rx_rsize; desc_index++) {
507	struct sxgbe_rx_norm_desc *p;
508	p = rx_ring->dma_rx + desc_index;
509	ret = sxgbe_init_rx_buffers(dev, p, i: desc_index,
510	dma_buf_sz: bfsize, rx_ring);
511	if (ret)
512	goto err_free_rx_buffers;
513	}
514
515	/* initialise counters */
516	rx_ring->cur_rx = 0;
517	rx_ring->dirty_rx = (unsigned int)(desc_index - rx_rsize);
518	priv->dma_buf_sz = bfsize;
519
520	return 0;
521
522	err_free_rx_buffers:
523	while (--desc_index >= 0) {
524	struct sxgbe_rx_norm_desc *p;
525
526	p = rx_ring->dma_rx + desc_index;
527	sxgbe_free_rx_buffers(dev, p, i: desc_index, dma_buf_sz: bfsize, rx_ring);
528	}
529	kfree(objp: rx_ring->rx_skbuff);
530	err_free_skbuff_dma:
531	kfree(objp: rx_ring->rx_skbuff_dma);
532	err_free_dma_rx:
533	dma_free_coherent(dev: priv->device,
534	size: rx_rsize * sizeof(struct sxgbe_rx_norm_desc),
535	cpu_addr: rx_ring->dma_rx, dma_handle: rx_ring->dma_rx_phy);
536
537	return ret;
538	}
539	/**
540	* free_tx_ring - free the TX descriptor ring
541	* @dev: net device structure
542	* @tx_ring: ring to be initialised
543	* @tx_rsize: ring size
544	* Description: this function initializes the DMA TX descriptor
545	*/
546	static void free_tx_ring(struct device *dev, struct sxgbe_tx_queue *tx_ring,
547	int tx_rsize)
548	{
549	dma_free_coherent(dev, size: tx_rsize * sizeof(struct sxgbe_tx_norm_desc),
550	cpu_addr: tx_ring->dma_tx, dma_handle: tx_ring->dma_tx_phy);
551	}
552
553	/**
554	* init_dma_desc_rings - init the RX/TX descriptor rings
555	* @netd: net device structure
556	* Description: this function initializes the DMA RX/TX descriptors
557	* and allocates the socket buffers. It suppors the chained and ring
558	* modes.
559	*/
560	static int init_dma_desc_rings(struct net_device *netd)
561	{
562	int queue_num, ret;
563	struct sxgbe_priv_data *priv = netdev_priv(dev: netd);
564	int tx_rsize = priv->dma_tx_size;
565	int rx_rsize = priv->dma_rx_size;
566
567	/* Allocate memory for queue structures and TX descs */
568	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
569	ret = init_tx_ring(dev: priv->device, queue_no: queue_num,
570	tx_ring: priv->txq[queue_num], tx_rsize);
571	if (ret) {
572	dev_err(&netd->dev, "TX DMA ring allocation failed!\n");
573	goto txalloc_err;
574	}
575
576	/* save private pointer in each ring this
577	* pointer is needed during cleaing TX queue
578	*/
579	priv->txq[queue_num]->priv_ptr = priv;
580	}
581
582	/* Allocate memory for queue structures and RX descs */
583	SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
584	ret = init_rx_ring(dev: netd, queue_no: queue_num,
585	rx_ring: priv->rxq[queue_num], rx_rsize);
586	if (ret) {
587	netdev_err(dev: netd, format: "RX DMA ring allocation failed!!\n");
588	goto rxalloc_err;
589	}
590
591	/* save private pointer in each ring this
592	* pointer is needed during cleaing TX queue
593	*/
594	priv->rxq[queue_num]->priv_ptr = priv;
595	}
596
597	sxgbe_clear_descriptors(priv);
598
599	return 0;
600
601	txalloc_err:
602	while (queue_num--)
603	free_tx_ring(dev: priv->device, tx_ring: priv->txq[queue_num], tx_rsize);
604	return ret;
605
606	rxalloc_err:
607	while (queue_num--)
608	free_rx_ring(dev: priv->device, rx_ring: priv->rxq[queue_num], rx_rsize);
609	return ret;
610	}
611
612	static void tx_free_ring_skbufs(struct sxgbe_tx_queue *txqueue)
613	{
614	int dma_desc;
615	struct sxgbe_priv_data *priv = txqueue->priv_ptr;
616	int tx_rsize = priv->dma_tx_size;
617
618	for (dma_desc = 0; dma_desc < tx_rsize; dma_desc++) {
619	struct sxgbe_tx_norm_desc *tdesc = txqueue->dma_tx + dma_desc;
620
621	if (txqueue->tx_skbuff_dma[dma_desc])
622	dma_unmap_single(priv->device,
623	txqueue->tx_skbuff_dma[dma_desc],
624	priv->hw->desc->get_tx_len(tdesc),
625	DMA_TO_DEVICE);
626
627	dev_kfree_skb_any(skb: txqueue->tx_skbuff[dma_desc]);
628	txqueue->tx_skbuff[dma_desc] = NULL;
629	txqueue->tx_skbuff_dma[dma_desc] = 0;
630	}
631	}
632
633
634	static void dma_free_tx_skbufs(struct sxgbe_priv_data *priv)
635	{
636	int queue_num;
637
638	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
639	struct sxgbe_tx_queue *tqueue = priv->txq[queue_num];
640	tx_free_ring_skbufs(txqueue: tqueue);
641	}
642	}
643
644	static void free_dma_desc_resources(struct sxgbe_priv_data *priv)
645	{
646	int queue_num;
647	int tx_rsize = priv->dma_tx_size;
648	int rx_rsize = priv->dma_rx_size;
649
650	/* Release the DMA TX buffers */
651	dma_free_tx_skbufs(priv);
652
653	/* Release the TX ring memory also */
654	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
655	free_tx_ring(dev: priv->device, tx_ring: priv->txq[queue_num], tx_rsize);
656	}
657
658	/* Release the RX ring memory also */
659	SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
660	free_rx_ring(dev: priv->device, rx_ring: priv->rxq[queue_num], rx_rsize);
661	}
662	}
663
664	static int txring_mem_alloc(struct sxgbe_priv_data *priv)
665	{
666	int queue_num;
667
668	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
669	priv->txq[queue_num] = devm_kmalloc(dev: priv->device,
670	size: sizeof(struct sxgbe_tx_queue), GFP_KERNEL);
671	if (!priv->txq[queue_num])
672	return -ENOMEM;
673	}
674
675	return 0;
676	}
677
678	static int rxring_mem_alloc(struct sxgbe_priv_data *priv)
679	{
680	int queue_num;
681
682	SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
683	priv->rxq[queue_num] = devm_kmalloc(dev: priv->device,
684	size: sizeof(struct sxgbe_rx_queue), GFP_KERNEL);
685	if (!priv->rxq[queue_num])
686	return -ENOMEM;
687	}
688
689	return 0;
690	}
691
692	/**
693	* sxgbe_mtl_operation_mode - HW MTL operation mode
694	* @priv: driver private structure
695	* Description: it sets the MTL operation mode: tx/rx MTL thresholds
696	* or Store-And-Forward capability.
697	*/
698	static void sxgbe_mtl_operation_mode(struct sxgbe_priv_data *priv)
699	{
700	int queue_num;
701
702	/* TX/RX threshold control */
703	if (likely(priv->plat->force_sf_dma_mode)) {
704	/* set TC mode for TX QUEUES */
705	SXGBE_FOR_EACH_QUEUE(priv->hw_cap.tx_mtl_queues, queue_num)
706	priv->hw->mtl->set_tx_mtl_mode(priv->ioaddr, queue_num,
707	SXGBE_MTL_SFMODE);
708	priv->tx_tc = SXGBE_MTL_SFMODE;
709
710	/* set TC mode for RX QUEUES */
711	SXGBE_FOR_EACH_QUEUE(priv->hw_cap.rx_mtl_queues, queue_num)
712	priv->hw->mtl->set_rx_mtl_mode(priv->ioaddr, queue_num,
713	SXGBE_MTL_SFMODE);
714	priv->rx_tc = SXGBE_MTL_SFMODE;
715	} else if (unlikely(priv->plat->force_thresh_dma_mode)) {
716	/* set TC mode for TX QUEUES */
717	SXGBE_FOR_EACH_QUEUE(priv->hw_cap.tx_mtl_queues, queue_num)
718	priv->hw->mtl->set_tx_mtl_mode(priv->ioaddr, queue_num,
719	priv->tx_tc);
720	/* set TC mode for RX QUEUES */
721	SXGBE_FOR_EACH_QUEUE(priv->hw_cap.rx_mtl_queues, queue_num)
722	priv->hw->mtl->set_rx_mtl_mode(priv->ioaddr, queue_num,
723	priv->rx_tc);
724	} else {
725	pr_err("ERROR: %s: Invalid TX threshold mode\n", __func__);
726	}
727	}
728
729	/**
730	* sxgbe_tx_queue_clean:
731	* @tqueue: queue pointer
732	* Description: it reclaims resources after transmission completes.
733	*/
734	static void sxgbe_tx_queue_clean(struct sxgbe_tx_queue *tqueue)
735	{
736	struct sxgbe_priv_data *priv = tqueue->priv_ptr;
737	unsigned int tx_rsize = priv->dma_tx_size;
738	struct netdev_queue *dev_txq;
739	u8 queue_no = tqueue->queue_no;
740
741	dev_txq = netdev_get_tx_queue(dev: priv->dev, index: queue_no);
742
743	__netif_tx_lock(txq: dev_txq, smp_processor_id());
744
745	priv->xstats.tx_clean++;
746	while (tqueue->dirty_tx != tqueue->cur_tx) {
747	unsigned int entry = tqueue->dirty_tx % tx_rsize;
748	struct sk_buff *skb = tqueue->tx_skbuff[entry];
749	struct sxgbe_tx_norm_desc *p;
750
751	p = tqueue->dma_tx + entry;
752
753	/* Check if the descriptor is owned by the DMA. */
754	if (priv->hw->desc->get_tx_owner(p))
755	break;
756
757	if (netif_msg_tx_done(priv))
758	pr_debug("%s: curr %d, dirty %d\n",
759	__func__, tqueue->cur_tx, tqueue->dirty_tx);
760
761	if (likely(tqueue->tx_skbuff_dma[entry])) {
762	dma_unmap_single(priv->device,
763	tqueue->tx_skbuff_dma[entry],
764	priv->hw->desc->get_tx_len(p),
765	DMA_TO_DEVICE);
766	tqueue->tx_skbuff_dma[entry] = 0;
767	}
768
769	if (likely(skb)) {
770	dev_kfree_skb(skb);
771	tqueue->tx_skbuff[entry] = NULL;
772	}
773
774	priv->hw->desc->release_tx_desc(p);
775
776	tqueue->dirty_tx++;
777	}
778
779	/* wake up queue */
780	if (unlikely(netif_tx_queue_stopped(dev_txq) &&
781	sxgbe_tx_avail(tqueue, tx_rsize) > SXGBE_TX_THRESH(priv))) {
782	if (netif_msg_tx_done(priv))
783	pr_debug("%s: restart transmit\n", __func__);
784	netif_tx_wake_queue(dev_queue: dev_txq);
785	}
786
787	__netif_tx_unlock(txq: dev_txq);
788	}
789
790	/**
791	* sxgbe_tx_all_clean:
792	* @priv: driver private structure
793	* Description: it reclaims resources after transmission completes.
794	*/
795	static void sxgbe_tx_all_clean(struct sxgbe_priv_data * const priv)
796	{
797	u8 queue_num;
798
799	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
800	struct sxgbe_tx_queue *tqueue = priv->txq[queue_num];
801
802	sxgbe_tx_queue_clean(tqueue);
803	}
804
805	if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
806	sxgbe_enable_eee_mode(priv);
807	mod_timer(timer: &priv->eee_ctrl_timer, SXGBE_LPI_TIMER(eee_timer));
808	}
809	}
810
811	/**
812	* sxgbe_restart_tx_queue: irq tx error mng function
813	* @priv: driver private structure
814	* @queue_num: queue number
815	* Description: it cleans the descriptors and restarts the transmission
816	* in case of errors.
817	*/
818	static void sxgbe_restart_tx_queue(struct sxgbe_priv_data *priv, int queue_num)
819	{
820	struct sxgbe_tx_queue *tx_ring = priv->txq[queue_num];
821	struct netdev_queue *dev_txq = netdev_get_tx_queue(dev: priv->dev,
822	index: queue_num);
823
824	/* stop the queue */
825	netif_tx_stop_queue(dev_queue: dev_txq);
826
827	/* stop the tx dma */
828	priv->hw->dma->stop_tx_queue(priv->ioaddr, queue_num);
829
830	/* free the skbuffs of the ring */
831	tx_free_ring_skbufs(txqueue: tx_ring);
832
833	/* initialise counters */
834	tx_ring->cur_tx = 0;
835	tx_ring->dirty_tx = 0;
836
837	/* start the tx dma */
838	priv->hw->dma->start_tx_queue(priv->ioaddr, queue_num);
839
840	priv->dev->stats.tx_errors++;
841
842	/* wakeup the queue */
843	netif_tx_wake_queue(dev_queue: dev_txq);
844	}
845
846	/**
847	* sxgbe_reset_all_tx_queues: irq tx error mng function
848	* @priv: driver private structure
849	* Description: it cleans all the descriptors and
850	* restarts the transmission on all queues in case of errors.
851	*/
852	static void sxgbe_reset_all_tx_queues(struct sxgbe_priv_data *priv)
853	{
854	int queue_num;
855
856	/* On TX timeout of net device, resetting of all queues
857	* may not be proper way, revisit this later if needed
858	*/
859	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num)
860	sxgbe_restart_tx_queue(priv, queue_num);
861	}
862
863	/**
864	* sxgbe_get_hw_features: get XMAC capabilities from the HW cap. register.
865	* @priv: driver private structure
866	* Description:
867	* new GMAC chip generations have a new register to indicate the
868	* presence of the optional feature/functions.
869	* This can be also used to override the value passed through the
870	* platform and necessary for old MAC10/100 and GMAC chips.
871	*/
872	static int sxgbe_get_hw_features(struct sxgbe_priv_data * const priv)
873	{
874	int rval = 0;
875	struct sxgbe_hw_features *features = &priv->hw_cap;
876
877	/* Read First Capability Register CAP[0] */
878	rval = priv->hw->mac->get_hw_feature(priv->ioaddr, 0);
879	if (rval) {
880	features->pmt_remote_wake_up =
881	SXGBE_HW_FEAT_PMT_TEMOTE_WOP(rval);
882	features->pmt_magic_frame = SXGBE_HW_FEAT_PMT_MAGIC_PKT(rval);
883	features->atime_stamp = SXGBE_HW_FEAT_IEEE1500_2008(rval);
884	features->tx_csum_offload =
885	SXGBE_HW_FEAT_TX_CSUM_OFFLOAD(rval);
886	features->rx_csum_offload =
887	SXGBE_HW_FEAT_RX_CSUM_OFFLOAD(rval);
888	features->multi_macaddr = SXGBE_HW_FEAT_MACADDR_COUNT(rval);
889	features->tstamp_srcselect = SXGBE_HW_FEAT_TSTMAP_SRC(rval);
890	features->sa_vlan_insert = SXGBE_HW_FEAT_SRCADDR_VLAN(rval);
891	features->eee = SXGBE_HW_FEAT_EEE(rval);
892	}
893
894	/* Read First Capability Register CAP[1] */
895	rval = priv->hw->mac->get_hw_feature(priv->ioaddr, 1);
896	if (rval) {
897	features->rxfifo_size = SXGBE_HW_FEAT_RX_FIFO_SIZE(rval);
898	features->txfifo_size = SXGBE_HW_FEAT_TX_FIFO_SIZE(rval);
899	features->atstmap_hword = SXGBE_HW_FEAT_TX_FIFO_SIZE(rval);
900	features->dcb_enable = SXGBE_HW_FEAT_DCB(rval);
901	features->splithead_enable = SXGBE_HW_FEAT_SPLIT_HDR(rval);
902	features->tcpseg_offload = SXGBE_HW_FEAT_TSO(rval);
903	features->debug_mem = SXGBE_HW_FEAT_DEBUG_MEM_IFACE(rval);
904	features->rss_enable = SXGBE_HW_FEAT_RSS(rval);
905	features->hash_tsize = SXGBE_HW_FEAT_HASH_TABLE_SIZE(rval);
906	features->l3l4_filer_size = SXGBE_HW_FEAT_L3L4_FILTER_NUM(rval);
907	}
908
909	/* Read First Capability Register CAP[2] */
910	rval = priv->hw->mac->get_hw_feature(priv->ioaddr, 2);
911	if (rval) {
912	features->rx_mtl_queues = SXGBE_HW_FEAT_RX_MTL_QUEUES(rval);
913	features->tx_mtl_queues = SXGBE_HW_FEAT_TX_MTL_QUEUES(rval);
914	features->rx_dma_channels = SXGBE_HW_FEAT_RX_DMA_CHANNELS(rval);
915	features->tx_dma_channels = SXGBE_HW_FEAT_TX_DMA_CHANNELS(rval);
916	features->pps_output_count = SXGBE_HW_FEAT_PPS_OUTPUTS(rval);
917	features->aux_input_count = SXGBE_HW_FEAT_AUX_SNAPSHOTS(rval);
918	}
919
920	return rval;
921	}
922
923	/**
924	* sxgbe_check_ether_addr: check if the MAC addr is valid
925	* @priv: driver private structure
926	* Description:
927	* it is to verify if the MAC address is valid, in case of failures it
928	* generates a random MAC address
929	*/
930	static void sxgbe_check_ether_addr(struct sxgbe_priv_data *priv)
931	{
932	if (!is_valid_ether_addr(addr: priv->dev->dev_addr)) {
933	u8 addr[ETH_ALEN];
934
935	priv->hw->mac->get_umac_addr((void __iomem *)
936	priv->ioaddr, addr, 0);
937	if (is_valid_ether_addr(addr))
938	eth_hw_addr_set(dev: priv->dev, addr);
939	else
940	eth_hw_addr_random(dev: priv->dev);
941	}
942	dev_info(priv->device, "device MAC address %pM\n",
943	priv->dev->dev_addr);
944	}
945
946	/**
947	* sxgbe_init_dma_engine: DMA init.
948	* @priv: driver private structure
949	* Description:
950	* It inits the DMA invoking the specific SXGBE callback.
951	* Some DMA parameters can be passed from the platform;
952	* in case of these are not passed a default is kept for the MAC or GMAC.
953	*/
954	static int sxgbe_init_dma_engine(struct sxgbe_priv_data *priv)
955	{
956	int pbl = DEFAULT_DMA_PBL, fixed_burst = 0, burst_map = 0;
957	int queue_num;
958
959	if (priv->plat->dma_cfg) {
960	pbl = priv->plat->dma_cfg->pbl;
961	fixed_burst = priv->plat->dma_cfg->fixed_burst;
962	burst_map = priv->plat->dma_cfg->burst_map;
963	}
964
965	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num)
966	priv->hw->dma->cha_init(priv->ioaddr, queue_num,
967	fixed_burst, pbl,
968	(priv->txq[queue_num])->dma_tx_phy,
969	(priv->rxq[queue_num])->dma_rx_phy,
970	priv->dma_tx_size, priv->dma_rx_size);
971
972	return priv->hw->dma->init(priv->ioaddr, fixed_burst, burst_map);
973	}
974
975	/**
976	* sxgbe_init_mtl_engine: MTL init.
977	* @priv: driver private structure
978	* Description:
979	* It inits the MTL invoking the specific SXGBE callback.
980	*/
981	static void sxgbe_init_mtl_engine(struct sxgbe_priv_data *priv)
982	{
983	int queue_num;
984
985	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
986	priv->hw->mtl->mtl_set_txfifosize(priv->ioaddr, queue_num,
987	priv->hw_cap.tx_mtl_qsize);
988	priv->hw->mtl->mtl_enable_txqueue(priv->ioaddr, queue_num);
989	}
990	}
991
992	/**
993	* sxgbe_disable_mtl_engine: MTL disable.
994	* @priv: driver private structure
995	* Description:
996	* It disables the MTL queues by invoking the specific SXGBE callback.
997	*/
998	static void sxgbe_disable_mtl_engine(struct sxgbe_priv_data *priv)
999	{
1000	int queue_num;
1001
1002	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num)
1003	priv->hw->mtl->mtl_disable_txqueue(priv->ioaddr, queue_num);
1004	}
1005
1006
1007	/**
1008	* sxgbe_tx_timer: mitigation sw timer for tx.
1009	* @t: timer pointer
1010	* Description:
1011	* This is the timer handler to directly invoke the sxgbe_tx_clean.
1012	*/
1013	static void sxgbe_tx_timer(struct timer_list *t)
1014	{
1015	struct sxgbe_tx_queue *p = from_timer(p, t, txtimer);
1016	sxgbe_tx_queue_clean(tqueue: p);
1017	}
1018
1019	/**
1020	* sxgbe_tx_init_coalesce: init tx mitigation options.
1021	* @priv: driver private structure
1022	* Description:
1023	* This inits the transmit coalesce parameters: i.e. timer rate,
1024	* timer handler and default threshold used for enabling the
1025	* interrupt on completion bit.
1026	*/
1027	static void sxgbe_tx_init_coalesce(struct sxgbe_priv_data *priv)
1028	{
1029	u8 queue_num;
1030
1031	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
1032	struct sxgbe_tx_queue *p = priv->txq[queue_num];
1033	p->tx_coal_frames = SXGBE_TX_FRAMES;
1034	p->tx_coal_timer = SXGBE_COAL_TX_TIMER;
1035	timer_setup(&p->txtimer, sxgbe_tx_timer, 0);
1036	p->txtimer.expires = SXGBE_COAL_TIMER(p->tx_coal_timer);
1037	add_timer(timer: &p->txtimer);
1038	}
1039	}
1040
1041	static void sxgbe_tx_del_timer(struct sxgbe_priv_data *priv)
1042	{
1043	u8 queue_num;
1044
1045	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
1046	struct sxgbe_tx_queue *p = priv->txq[queue_num];
1047	del_timer_sync(timer: &p->txtimer);
1048	}
1049	}
1050
1051	/**
1052	* sxgbe_open - open entry point of the driver
1053	* @dev : pointer to the device structure.
1054	* Description:
1055	* This function is the open entry point of the driver.
1056	* Return value:
1057	* 0 on success and an appropriate (-)ve integer as defined in errno.h
1058	* file on failure.
1059	*/
1060	static int sxgbe_open(struct net_device *dev)
1061	{
1062	struct sxgbe_priv_data *priv = netdev_priv(dev);
1063	int ret, queue_num;
1064
1065	clk_prepare_enable(clk: priv->sxgbe_clk);
1066
1067	sxgbe_check_ether_addr(priv);
1068
1069	/* Init the phy */
1070	ret = sxgbe_init_phy(ndev: dev);
1071	if (ret) {
1072	netdev_err(dev, format: "%s: Cannot attach to PHY (error: %d)\n",
1073	__func__, ret);
1074	goto phy_error;
1075	}
1076
1077	/* Create and initialize the TX/RX descriptors chains. */
1078	priv->dma_tx_size = SXGBE_ALIGN(DMA_TX_SIZE);
1079	priv->dma_rx_size = SXGBE_ALIGN(DMA_RX_SIZE);
1080	priv->dma_buf_sz = SXGBE_ALIGN(DMA_BUFFER_SIZE);
1081	priv->tx_tc = TC_DEFAULT;
1082	priv->rx_tc = TC_DEFAULT;
1083	init_dma_desc_rings(netd: dev);
1084
1085	/* DMA initialization and SW reset */
1086	ret = sxgbe_init_dma_engine(priv);
1087	if (ret < 0) {
1088	netdev_err(dev, format: "%s: DMA initialization failed\n", __func__);
1089	goto init_error;
1090	}
1091
1092	/* MTL initialization */
1093	sxgbe_init_mtl_engine(priv);
1094
1095	/* Copy the MAC addr into the HW */
1096	priv->hw->mac->set_umac_addr(priv->ioaddr, dev->dev_addr, 0);
1097
1098	/* Initialize the MAC Core */
1099	priv->hw->mac->core_init(priv->ioaddr);
1100	SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
1101	priv->hw->mac->enable_rxqueue(priv->ioaddr, queue_num);
1102	}
1103
1104	/* Request the IRQ lines */
1105	ret = devm_request_irq(dev: priv->device, irq: priv->irq, handler: sxgbe_common_interrupt,
1106	IRQF_SHARED, devname: dev->name, dev_id: dev);
1107	if (unlikely(ret < 0)) {
1108	netdev_err(dev, format: "%s: ERROR: allocating the IRQ %d (error: %d)\n",
1109	__func__, priv->irq, ret);
1110	goto init_error;
1111	}
1112
1113	/* If the LPI irq is different from the mac irq
1114	* register a dedicated handler
1115	*/
1116	if (priv->lpi_irq != dev->irq) {
1117	ret = devm_request_irq(dev: priv->device, irq: priv->lpi_irq,
1118	handler: sxgbe_common_interrupt,
1119	IRQF_SHARED, devname: dev->name, dev_id: dev);
1120	if (unlikely(ret < 0)) {
1121	netdev_err(dev, format: "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
1122	__func__, priv->lpi_irq, ret);
1123	goto init_error;
1124	}
1125	}
1126
1127	/* Request TX DMA irq lines */
1128	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
1129	ret = devm_request_irq(dev: priv->device,
1130	irq: (priv->txq[queue_num])->irq_no,
1131	handler: sxgbe_tx_interrupt, irqflags: 0,
1132	devname: dev->name, dev_id: priv->txq[queue_num]);
1133	if (unlikely(ret < 0)) {
1134	netdev_err(dev, format: "%s: ERROR: allocating TX IRQ %d (error: %d)\n",
1135	__func__, priv->irq, ret);
1136	goto init_error;
1137	}
1138	}
1139
1140	/* Request RX DMA irq lines */
1141	SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
1142	ret = devm_request_irq(dev: priv->device,
1143	irq: (priv->rxq[queue_num])->irq_no,
1144	handler: sxgbe_rx_interrupt, irqflags: 0,
1145	devname: dev->name, dev_id: priv->rxq[queue_num]);
1146	if (unlikely(ret < 0)) {
1147	netdev_err(dev, format: "%s: ERROR: allocating TX IRQ %d (error: %d)\n",
1148	__func__, priv->irq, ret);
1149	goto init_error;
1150	}
1151	}
1152
1153	/* Enable the MAC Rx/Tx */
1154	priv->hw->mac->enable_tx(priv->ioaddr, true);
1155	priv->hw->mac->enable_rx(priv->ioaddr, true);
1156
1157	/* Set the HW DMA mode and the COE */
1158	sxgbe_mtl_operation_mode(priv);
1159
1160	/* Extra statistics */
1161	memset(&priv->xstats, 0, sizeof(struct sxgbe_extra_stats));
1162
1163	priv->xstats.tx_threshold = priv->tx_tc;
1164	priv->xstats.rx_threshold = priv->rx_tc;
1165
1166	/* Start the ball rolling... */
1167	netdev_dbg(dev, "DMA RX/TX processes started...\n");
1168	priv->hw->dma->start_tx(priv->ioaddr, SXGBE_TX_QUEUES);
1169	priv->hw->dma->start_rx(priv->ioaddr, SXGBE_RX_QUEUES);
1170
1171	if (dev->phydev)
1172	phy_start(phydev: dev->phydev);
1173
1174	/* initialise TX coalesce parameters */
1175	sxgbe_tx_init_coalesce(priv);
1176
1177	if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
1178	priv->rx_riwt = SXGBE_MAX_DMA_RIWT;
1179	priv->hw->dma->rx_watchdog(priv->ioaddr, SXGBE_MAX_DMA_RIWT);
1180	}
1181
1182	priv->tx_lpi_timer = SXGBE_DEFAULT_LPI_TIMER;
1183	priv->eee_enabled = sxgbe_eee_init(priv);
1184
1185	napi_enable(n: &priv->napi);
1186	netif_start_queue(dev);
1187
1188	return 0;
1189
1190	init_error:
1191	free_dma_desc_resources(priv);
1192	if (dev->phydev)
1193	phy_disconnect(phydev: dev->phydev);
1194	phy_error:
1195	clk_disable_unprepare(clk: priv->sxgbe_clk);
1196
1197	return ret;
1198	}
1199
1200	/**
1201	* sxgbe_release - close entry point of the driver
1202	* @dev : device pointer.
1203	* Description:
1204	* This is the stop entry point of the driver.
1205	*/
1206	static int sxgbe_release(struct net_device *dev)
1207	{
1208	struct sxgbe_priv_data *priv = netdev_priv(dev);
1209
1210	if (priv->eee_enabled)
1211	del_timer_sync(timer: &priv->eee_ctrl_timer);
1212
1213	/* Stop and disconnect the PHY */
1214	if (dev->phydev) {
1215	phy_stop(phydev: dev->phydev);
1216	phy_disconnect(phydev: dev->phydev);
1217	}
1218
1219	netif_tx_stop_all_queues(dev);
1220
1221	napi_disable(n: &priv->napi);
1222
1223	/* delete TX timers */
1224	sxgbe_tx_del_timer(priv);
1225
1226	/* Stop TX/RX DMA and clear the descriptors */
1227	priv->hw->dma->stop_tx(priv->ioaddr, SXGBE_TX_QUEUES);
1228	priv->hw->dma->stop_rx(priv->ioaddr, SXGBE_RX_QUEUES);
1229
1230	/* disable MTL queue */
1231	sxgbe_disable_mtl_engine(priv);
1232
1233	/* Release and free the Rx/Tx resources */
1234	free_dma_desc_resources(priv);
1235
1236	/* Disable the MAC Rx/Tx */
1237	priv->hw->mac->enable_tx(priv->ioaddr, false);
1238	priv->hw->mac->enable_rx(priv->ioaddr, false);
1239
1240	clk_disable_unprepare(clk: priv->sxgbe_clk);
1241
1242	return 0;
1243	}
1244	/* Prepare first Tx descriptor for doing TSO operation */
1245	static void sxgbe_tso_prepare(struct sxgbe_priv_data *priv,
1246	struct sxgbe_tx_norm_desc *first_desc,
1247	struct sk_buff *skb)
1248	{
1249	unsigned int total_hdr_len, tcp_hdr_len;
1250
1251	/* Write first Tx descriptor with appropriate value */
1252	tcp_hdr_len = tcp_hdrlen(skb);
1253	total_hdr_len = skb_transport_offset(skb) + tcp_hdr_len;
1254
1255	first_desc->tdes01 = dma_map_single(priv->device, skb->data,
1256	total_hdr_len, DMA_TO_DEVICE);
1257	if (dma_mapping_error(dev: priv->device, dma_addr: first_desc->tdes01))
1258	pr_err("%s: TX dma mapping failed!!\n", __func__);
1259
1260	first_desc->tdes23.tx_rd_des23.first_desc = 1;
1261	priv->hw->desc->tx_desc_enable_tse(first_desc, 1, total_hdr_len,
1262	tcp_hdr_len,
1263	skb->len - total_hdr_len);
1264	}
1265
1266	/**
1267	* sxgbe_xmit: Tx entry point of the driver
1268	* @skb : the socket buffer
1269	* @dev : device pointer
1270	* Description : this is the tx entry point of the driver.
1271	* It programs the chain or the ring and supports oversized frames
1272	* and SG feature.
1273	*/
1274	static netdev_tx_t sxgbe_xmit(struct sk_buff *skb, struct net_device *dev)
1275	{
1276	unsigned int entry, frag_num;
1277	int cksum_flag = 0;
1278	struct netdev_queue *dev_txq;
1279	unsigned txq_index = skb_get_queue_mapping(skb);
1280	struct sxgbe_priv_data *priv = netdev_priv(dev);
1281	unsigned int tx_rsize = priv->dma_tx_size;
1282	struct sxgbe_tx_queue *tqueue = priv->txq[txq_index];
1283	struct sxgbe_tx_norm_desc *tx_desc, *first_desc;
1284	struct sxgbe_tx_ctxt_desc *ctxt_desc = NULL;
1285	int nr_frags = skb_shinfo(skb)->nr_frags;
1286	int no_pagedlen = skb_headlen(skb);
1287	int is_jumbo = 0;
1288	u16 cur_mss = skb_shinfo(skb)->gso_size;
1289	u32 ctxt_desc_req = 0;
1290
1291	/* get the TX queue handle */
1292	dev_txq = netdev_get_tx_queue(dev, index: txq_index);
1293
1294	if (unlikely(skb_is_gso(skb) && tqueue->prev_mss != cur_mss))
1295	ctxt_desc_req = 1;
1296
1297	if (unlikely(skb_vlan_tag_present(skb) ||
1298	((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1299	tqueue->hwts_tx_en)))
1300	ctxt_desc_req = 1;
1301
1302	if (priv->tx_path_in_lpi_mode)
1303	sxgbe_disable_eee_mode(priv);
1304
1305	if (unlikely(sxgbe_tx_avail(tqueue, tx_rsize) < nr_frags + 1)) {
1306	if (!netif_tx_queue_stopped(dev_queue: dev_txq)) {
1307	netif_tx_stop_queue(dev_queue: dev_txq);
1308	netdev_err(dev, format: "%s: Tx Ring is full when %d queue is awake\n",
1309	__func__, txq_index);
1310	}
1311	return NETDEV_TX_BUSY;
1312	}
1313
1314	entry = tqueue->cur_tx % tx_rsize;
1315	tx_desc = tqueue->dma_tx + entry;
1316
1317	first_desc = tx_desc;
1318	if (ctxt_desc_req)
1319	ctxt_desc = (struct sxgbe_tx_ctxt_desc *)first_desc;
1320
1321	/* save the skb address */
1322	tqueue->tx_skbuff[entry] = skb;
1323
1324	if (!is_jumbo) {
1325	if (likely(skb_is_gso(skb))) {
1326	/* TSO support */
1327	if (unlikely(tqueue->prev_mss != cur_mss)) {
1328	priv->hw->desc->tx_ctxt_desc_set_mss(
1329	ctxt_desc, cur_mss);
1330	priv->hw->desc->tx_ctxt_desc_set_tcmssv(
1331	ctxt_desc);
1332	priv->hw->desc->tx_ctxt_desc_reset_ostc(
1333	ctxt_desc);
1334	priv->hw->desc->tx_ctxt_desc_set_ctxt(
1335	ctxt_desc);
1336	priv->hw->desc->tx_ctxt_desc_set_owner(
1337	ctxt_desc);
1338
1339	entry = (++tqueue->cur_tx) % tx_rsize;
1340	first_desc = tqueue->dma_tx + entry;
1341
1342	tqueue->prev_mss = cur_mss;
1343	}
1344	sxgbe_tso_prepare(priv, first_desc, skb);
1345	} else {
1346	tx_desc->tdes01 = dma_map_single(priv->device,
1347	skb->data, no_pagedlen, DMA_TO_DEVICE);
1348	if (dma_mapping_error(dev: priv->device, dma_addr: tx_desc->tdes01))
1349	netdev_err(dev, format: "%s: TX dma mapping failed!!\n",
1350	__func__);
1351
1352	priv->hw->desc->prepare_tx_desc(tx_desc, 1, no_pagedlen,
1353	no_pagedlen, cksum_flag);
1354	}
1355	}
1356
1357	for (frag_num = 0; frag_num < nr_frags; frag_num++) {
1358	const skb_frag_t *frag = &skb_shinfo(skb)->frags[frag_num];
1359	int len = skb_frag_size(frag);
1360
1361	entry = (++tqueue->cur_tx) % tx_rsize;
1362	tx_desc = tqueue->dma_tx + entry;
1363	tx_desc->tdes01 = skb_frag_dma_map(dev: priv->device, frag, offset: 0, size: len,
1364	dir: DMA_TO_DEVICE);
1365
1366	tqueue->tx_skbuff_dma[entry] = tx_desc->tdes01;
1367	tqueue->tx_skbuff[entry] = NULL;
1368
1369	/* prepare the descriptor */
1370	priv->hw->desc->prepare_tx_desc(tx_desc, 0, len,
1371	len, cksum_flag);
1372	/* memory barrier to flush descriptor */
1373	wmb();
1374
1375	/* set the owner */
1376	priv->hw->desc->set_tx_owner(tx_desc);
1377	}
1378
1379	/* close the descriptors */
1380	priv->hw->desc->close_tx_desc(tx_desc);
1381
1382	/* memory barrier to flush descriptor */
1383	wmb();
1384
1385	tqueue->tx_count_frames += nr_frags + 1;
1386	if (tqueue->tx_count_frames > tqueue->tx_coal_frames) {
1387	priv->hw->desc->clear_tx_ic(tx_desc);
1388	priv->xstats.tx_reset_ic_bit++;
1389	mod_timer(timer: &tqueue->txtimer,
1390	SXGBE_COAL_TIMER(tqueue->tx_coal_timer));
1391	} else {
1392	tqueue->tx_count_frames = 0;
1393	}
1394
1395	/* set owner for first desc */
1396	priv->hw->desc->set_tx_owner(first_desc);
1397
1398	/* memory barrier to flush descriptor */
1399	wmb();
1400
1401	tqueue->cur_tx++;
1402
1403	/* display current ring */
1404	netif_dbg(priv, pktdata, dev, "%s: curr %d dirty=%d entry=%d, first=%p, nfrags=%d\n",
1405	__func__, tqueue->cur_tx % tx_rsize,
1406	tqueue->dirty_tx % tx_rsize, entry,
1407	first_desc, nr_frags);
1408
1409	if (unlikely(sxgbe_tx_avail(tqueue, tx_rsize) <= (MAX_SKB_FRAGS + 1))) {
1410	netif_dbg(priv, hw, dev, "%s: stop transmitted packets\n",
1411	__func__);
1412	netif_tx_stop_queue(dev_queue: dev_txq);
1413	}
1414
1415	dev->stats.tx_bytes += skb->len;
1416
1417	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1418	tqueue->hwts_tx_en)) {
1419	/* declare that device is doing timestamping */
1420	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1421	priv->hw->desc->tx_enable_tstamp(first_desc);
1422	}
1423
1424	skb_tx_timestamp(skb);
1425
1426	priv->hw->dma->enable_dma_transmission(priv->ioaddr, txq_index);
1427
1428	return NETDEV_TX_OK;
1429	}
1430
1431	/**
1432	* sxgbe_rx_refill: refill used skb preallocated buffers
1433	* @priv: driver private structure
1434	* Description : this is to reallocate the skb for the reception process
1435	* that is based on zero-copy.
1436	*/
1437	static void sxgbe_rx_refill(struct sxgbe_priv_data *priv)
1438	{
1439	unsigned int rxsize = priv->dma_rx_size;
1440	int bfsize = priv->dma_buf_sz;
1441	u8 qnum = priv->cur_rx_qnum;
1442
1443	for (; priv->rxq[qnum]->cur_rx - priv->rxq[qnum]->dirty_rx > 0;
1444	priv->rxq[qnum]->dirty_rx++) {
1445	unsigned int entry = priv->rxq[qnum]->dirty_rx % rxsize;
1446	struct sxgbe_rx_norm_desc *p;
1447
1448	p = priv->rxq[qnum]->dma_rx + entry;
1449
1450	if (likely(priv->rxq[qnum]->rx_skbuff[entry] == NULL)) {
1451	struct sk_buff *skb;
1452
1453	skb = netdev_alloc_skb_ip_align(dev: priv->dev, length: bfsize);
1454
1455	if (unlikely(skb == NULL))
1456	break;
1457
1458	priv->rxq[qnum]->rx_skbuff[entry] = skb;
1459	priv->rxq[qnum]->rx_skbuff_dma[entry] =
1460	dma_map_single(priv->device, skb->data, bfsize,
1461	DMA_FROM_DEVICE);
1462
1463	p->rdes23.rx_rd_des23.buf2_addr =
1464	priv->rxq[qnum]->rx_skbuff_dma[entry];
1465	}
1466
1467	/* Added memory barrier for RX descriptor modification */
1468	wmb();
1469	priv->hw->desc->set_rx_owner(p);
1470	priv->hw->desc->set_rx_int_on_com(p);
1471	/* Added memory barrier for RX descriptor modification */
1472	wmb();
1473	}
1474	}
1475
1476	/**
1477	* sxgbe_rx: receive the frames from the remote host
1478	* @priv: driver private structure
1479	* @limit: napi bugget.
1480	* Description : this the function called by the napi poll method.
1481	* It gets all the frames inside the ring.
1482	*/
1483	static int sxgbe_rx(struct sxgbe_priv_data *priv, int limit)
1484	{
1485	u8 qnum = priv->cur_rx_qnum;
1486	unsigned int rxsize = priv->dma_rx_size;
1487	unsigned int entry = priv->rxq[qnum]->cur_rx;
1488	unsigned int next_entry = 0;
1489	unsigned int count = 0;
1490	int checksum;
1491	int status;
1492
1493	while (count < limit) {
1494	struct sxgbe_rx_norm_desc *p;
1495	struct sk_buff *skb;
1496	int frame_len;
1497
1498	p = priv->rxq[qnum]->dma_rx + entry;
1499
1500	if (priv->hw->desc->get_rx_owner(p))
1501	break;
1502
1503	count++;
1504
1505	next_entry = (++priv->rxq[qnum]->cur_rx) % rxsize;
1506	prefetch(priv->rxq[qnum]->dma_rx + next_entry);
1507
1508	/* Read the status of the incoming frame and also get checksum
1509	* value based on whether it is enabled in SXGBE hardware or
1510	* not.
1511	*/
1512	status = priv->hw->desc->rx_wbstatus(p, &priv->xstats,
1513	&checksum);
1514	if (unlikely(status < 0)) {
1515	entry = next_entry;
1516	continue;
1517	}
1518	if (unlikely(!priv->rxcsum_insertion))
1519	checksum = CHECKSUM_NONE;
1520
1521	skb = priv->rxq[qnum]->rx_skbuff[entry];
1522
1523	if (unlikely(!skb))
1524	netdev_err(dev: priv->dev, format: "rx descriptor is not consistent\n");
1525
1526	prefetch(skb->data - NET_IP_ALIGN);
1527	priv->rxq[qnum]->rx_skbuff[entry] = NULL;
1528
1529	frame_len = priv->hw->desc->get_rx_frame_len(p);
1530
1531	skb_put(skb, len: frame_len);
1532
1533	skb->ip_summed = checksum;
1534	if (checksum == CHECKSUM_NONE)
1535	netif_receive_skb(skb);
1536	else
1537	napi_gro_receive(napi: &priv->napi, skb);
1538
1539	entry = next_entry;
1540	}
1541
1542	sxgbe_rx_refill(priv);
1543
1544	return count;
1545	}
1546
1547	/**
1548	* sxgbe_poll - sxgbe poll method (NAPI)
1549	* @napi : pointer to the napi structure.
1550	* @budget : maximum number of packets that the current CPU can receive from
1551	* all interfaces.
1552	* Description :
1553	* To look at the incoming frames and clear the tx resources.
1554	*/
1555	static int sxgbe_poll(struct napi_struct *napi, int budget)
1556	{
1557	struct sxgbe_priv_data *priv = container_of(napi,
1558	struct sxgbe_priv_data, napi);
1559	int work_done = 0;
1560	u8 qnum = priv->cur_rx_qnum;
1561
1562	priv->xstats.napi_poll++;
1563	/* first, clean the tx queues */
1564	sxgbe_tx_all_clean(priv);
1565
1566	work_done = sxgbe_rx(priv, limit: budget);
1567	if (work_done < budget) {
1568	napi_complete_done(n: napi, work_done);
1569	priv->hw->dma->enable_dma_irq(priv->ioaddr, qnum);
1570	}
1571
1572	return work_done;
1573	}
1574
1575	/**
1576	* sxgbe_tx_timeout
1577	* @dev : Pointer to net device structure
1578	* @txqueue: index of the hanging queue
1579	* Description: this function is called when a packet transmission fails to
1580	* complete within a reasonable time. The driver will mark the error in the
1581	* netdev structure and arrange for the device to be reset to a sane state
1582	* in order to transmit a new packet.
1583	*/
1584	static void sxgbe_tx_timeout(struct net_device *dev, unsigned int txqueue)
1585	{
1586	struct sxgbe_priv_data *priv = netdev_priv(dev);
1587
1588	sxgbe_reset_all_tx_queues(priv);
1589	}
1590
1591	/**
1592	* sxgbe_common_interrupt - main ISR
1593	* @irq: interrupt number.
1594	* @dev_id: to pass the net device pointer.
1595	* Description: this is the main driver interrupt service routine.
1596	* It calls the DMA ISR and also the core ISR to manage PMT, MMC, LPI
1597	* interrupts.
1598	*/
1599	static irqreturn_t sxgbe_common_interrupt(int irq, void *dev_id)
1600	{
1601	struct net_device *netdev = (struct net_device *)dev_id;
1602	struct sxgbe_priv_data *priv = netdev_priv(dev: netdev);
1603	int status;
1604
1605	status = priv->hw->mac->host_irq_status(priv->ioaddr, &priv->xstats);
1606	/* For LPI we need to save the tx status */
1607	if (status & TX_ENTRY_LPI_MODE) {
1608	priv->xstats.tx_lpi_entry_n++;
1609	priv->tx_path_in_lpi_mode = true;
1610	}
1611	if (status & TX_EXIT_LPI_MODE) {
1612	priv->xstats.tx_lpi_exit_n++;
1613	priv->tx_path_in_lpi_mode = false;
1614	}
1615	if (status & RX_ENTRY_LPI_MODE)
1616	priv->xstats.rx_lpi_entry_n++;
1617	if (status & RX_EXIT_LPI_MODE)
1618	priv->xstats.rx_lpi_exit_n++;
1619
1620	return IRQ_HANDLED;
1621	}
1622
1623	/**
1624	* sxgbe_tx_interrupt - TX DMA ISR
1625	* @irq: interrupt number.
1626	* @dev_id: to pass the net device pointer.
1627	* Description: this is the tx dma interrupt service routine.
1628	*/
1629	static irqreturn_t sxgbe_tx_interrupt(int irq, void *dev_id)
1630	{
1631	int status;
1632	struct sxgbe_tx_queue *txq = (struct sxgbe_tx_queue *)dev_id;
1633	struct sxgbe_priv_data *priv = txq->priv_ptr;
1634
1635	/* get the channel status */
1636	status = priv->hw->dma->tx_dma_int_status(priv->ioaddr, txq->queue_no,
1637	&priv->xstats);
1638	/* check for normal path */
1639	if (likely((status & handle_tx)))
1640	napi_schedule(n: &priv->napi);
1641
1642	/* check for unrecoverable error */
1643	if (unlikely((status & tx_hard_error)))
1644	sxgbe_restart_tx_queue(priv, queue_num: txq->queue_no);
1645
1646	/* check for TC configuration change */
1647	if (unlikely((status & tx_bump_tc) &&
1648	(priv->tx_tc != SXGBE_MTL_SFMODE) &&
1649	(priv->tx_tc < 512))) {
1650	/* step of TX TC is 32 till 128, otherwise 64 */
1651	priv->tx_tc += (priv->tx_tc < 128) ? 32 : 64;
1652	priv->hw->mtl->set_tx_mtl_mode(priv->ioaddr,
1653	txq->queue_no, priv->tx_tc);
1654	priv->xstats.tx_threshold = priv->tx_tc;
1655	}
1656
1657	return IRQ_HANDLED;
1658	}
1659
1660	/**
1661	* sxgbe_rx_interrupt - RX DMA ISR
1662	* @irq: interrupt number.
1663	* @dev_id: to pass the net device pointer.
1664	* Description: this is the rx dma interrupt service routine.
1665	*/
1666	static irqreturn_t sxgbe_rx_interrupt(int irq, void *dev_id)
1667	{
1668	int status;
1669	struct sxgbe_rx_queue *rxq = (struct sxgbe_rx_queue *)dev_id;
1670	struct sxgbe_priv_data *priv = rxq->priv_ptr;
1671
1672	/* get the channel status */
1673	status = priv->hw->dma->rx_dma_int_status(priv->ioaddr, rxq->queue_no,
1674	&priv->xstats);
1675
1676	if (likely((status & handle_rx) && (napi_schedule_prep(&priv->napi)))) {
1677	priv->hw->dma->disable_dma_irq(priv->ioaddr, rxq->queue_no);
1678	__napi_schedule(n: &priv->napi);
1679	}
1680
1681	/* check for TC configuration change */
1682	if (unlikely((status & rx_bump_tc) &&
1683	(priv->rx_tc != SXGBE_MTL_SFMODE) &&
1684	(priv->rx_tc < 128))) {
1685	/* step of TC is 32 */
1686	priv->rx_tc += 32;
1687	priv->hw->mtl->set_rx_mtl_mode(priv->ioaddr,
1688	rxq->queue_no, priv->rx_tc);
1689	priv->xstats.rx_threshold = priv->rx_tc;
1690	}
1691
1692	return IRQ_HANDLED;
1693	}
1694
1695	static inline u64 sxgbe_get_stat64(void __iomem *ioaddr, int reg_lo, int reg_hi)
1696	{
1697	u64 val = readl(addr: ioaddr + reg_lo);
1698
1699	val |= ((u64)readl(addr: ioaddr + reg_hi)) << 32;
1700
1701	return val;
1702	}
1703
1704
1705	/* sxgbe_get_stats64 - entry point to see statistical information of device
1706	* @dev : device pointer.
1707	* @stats : pointer to hold all the statistical information of device.
1708	* Description:
1709	* This function is a driver entry point whenever ifconfig command gets
1710	* executed to see device statistics. Statistics are number of
1711	* bytes sent or received, errors occurred etc.
1712	*/
1713	static void sxgbe_get_stats64(struct net_device *dev,
1714	struct rtnl_link_stats64 *stats)
1715	{
1716	struct sxgbe_priv_data *priv = netdev_priv(dev);
1717	void __iomem *ioaddr = priv->ioaddr;
1718	u64 count;
1719
1720	spin_lock(lock: &priv->stats_lock);
1721	/* Freeze the counter registers before reading value otherwise it may
1722	* get updated by hardware while we are reading them
1723	*/
1724	writel(SXGBE_MMC_CTRL_CNT_FRZ, addr: ioaddr + SXGBE_MMC_CTL_REG);
1725
1726	stats->rx_bytes = sxgbe_get_stat64(ioaddr,
1727	SXGBE_MMC_RXOCTETLO_GCNT_REG,
1728	SXGBE_MMC_RXOCTETHI_GCNT_REG);
1729
1730	stats->rx_packets = sxgbe_get_stat64(ioaddr,
1731	SXGBE_MMC_RXFRAMELO_GBCNT_REG,
1732	SXGBE_MMC_RXFRAMEHI_GBCNT_REG);
1733
1734	stats->multicast = sxgbe_get_stat64(ioaddr,
1735	SXGBE_MMC_RXMULTILO_GCNT_REG,
1736	SXGBE_MMC_RXMULTIHI_GCNT_REG);
1737
1738	stats->rx_crc_errors = sxgbe_get_stat64(ioaddr,
1739	SXGBE_MMC_RXCRCERRLO_REG,
1740	SXGBE_MMC_RXCRCERRHI_REG);
1741
1742	stats->rx_length_errors = sxgbe_get_stat64(ioaddr,
1743	SXGBE_MMC_RXLENERRLO_REG,
1744	SXGBE_MMC_RXLENERRHI_REG);
1745
1746	stats->rx_missed_errors = sxgbe_get_stat64(ioaddr,
1747	SXGBE_MMC_RXFIFOOVERFLOWLO_GBCNT_REG,
1748	SXGBE_MMC_RXFIFOOVERFLOWHI_GBCNT_REG);
1749
1750	stats->tx_bytes = sxgbe_get_stat64(ioaddr,
1751	SXGBE_MMC_TXOCTETLO_GCNT_REG,
1752	SXGBE_MMC_TXOCTETHI_GCNT_REG);
1753
1754	count = sxgbe_get_stat64(ioaddr, SXGBE_MMC_TXFRAMELO_GBCNT_REG,
1755	SXGBE_MMC_TXFRAMEHI_GBCNT_REG);
1756
1757	stats->tx_errors = sxgbe_get_stat64(ioaddr, SXGBE_MMC_TXFRAMELO_GCNT_REG,
1758	SXGBE_MMC_TXFRAMEHI_GCNT_REG);
1759	stats->tx_errors = count - stats->tx_errors;
1760	stats->tx_packets = count;
1761	stats->tx_fifo_errors = sxgbe_get_stat64(ioaddr, SXGBE_MMC_TXUFLWLO_GBCNT_REG,
1762	SXGBE_MMC_TXUFLWHI_GBCNT_REG);
1763	writel(val: 0, addr: ioaddr + SXGBE_MMC_CTL_REG);
1764	spin_unlock(lock: &priv->stats_lock);
1765	}
1766
1767	/* sxgbe_set_features - entry point to set offload features of the device.
1768	* @dev : device pointer.
1769	* @features : features which are required to be set.
1770	* Description:
1771	* This function is a driver entry point and called by Linux kernel whenever
1772	* any device features are set or reset by user.
1773	* Return value:
1774	* This function returns 0 after setting or resetting device features.
1775	*/
1776	static int sxgbe_set_features(struct net_device *dev,
1777	netdev_features_t features)
1778	{
1779	struct sxgbe_priv_data *priv = netdev_priv(dev);
1780	netdev_features_t changed = dev->features ^ features;
1781
1782	if (changed & NETIF_F_RXCSUM) {
1783	if (features & NETIF_F_RXCSUM) {
1784	priv->hw->mac->enable_rx_csum(priv->ioaddr);
1785	priv->rxcsum_insertion = true;
1786	} else {
1787	priv->hw->mac->disable_rx_csum(priv->ioaddr);
1788	priv->rxcsum_insertion = false;
1789	}
1790	}
1791
1792	return 0;
1793	}
1794
1795	/* sxgbe_change_mtu - entry point to change MTU size for the device.
1796	* @dev : device pointer.
1797	* @new_mtu : the new MTU size for the device.
1798	* Description: the Maximum Transfer Unit (MTU) is used by the network layer
1799	* to drive packet transmission. Ethernet has an MTU of 1500 octets
1800	* (ETH_DATA_LEN). This value can be changed with ifconfig.
1801	* Return value:
1802	* 0 on success and an appropriate (-)ve integer as defined in errno.h
1803	* file on failure.
1804	*/
1805	static int sxgbe_change_mtu(struct net_device *dev, int new_mtu)
1806	{
1807	dev->mtu = new_mtu;
1808
1809	if (!netif_running(dev))
1810	return 0;
1811
1812	/* Recevice ring buffer size is needed to be set based on MTU. If MTU is
1813	* changed then reinitilisation of the receive ring buffers need to be
1814	* done. Hence bring interface down and bring interface back up
1815	*/
1816	sxgbe_release(dev);
1817	return sxgbe_open(dev);
1818	}
1819
1820	static void sxgbe_set_umac_addr(void __iomem *ioaddr, unsigned char *addr,
1821	unsigned int reg_n)
1822	{
1823	unsigned long data;
1824
1825	data = (addr[5] << 8) | addr[4];
1826	/* For MAC Addr registers se have to set the Address Enable (AE)
1827	* bit that has no effect on the High Reg 0 where the bit 31 (MO)
1828	* is RO.
1829	*/
1830	writel(val: data | SXGBE_HI_REG_AE, addr: ioaddr + SXGBE_ADDR_HIGH(reg_n));
1831	data = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0];
1832	writel(val: data, addr: ioaddr + SXGBE_ADDR_LOW(reg_n));
1833	}
1834
1835	/**
1836	* sxgbe_set_rx_mode - entry point for setting different receive mode of
1837	* a device. unicast, multicast addressing
1838	* @dev : pointer to the device structure
1839	* Description:
1840	* This function is a driver entry point which gets called by the kernel
1841	* whenever different receive mode like unicast, multicast and promiscuous
1842	* must be enabled/disabled.
1843	* Return value:
1844	* void.
1845	*/
1846	static void sxgbe_set_rx_mode(struct net_device *dev)
1847	{
1848	struct sxgbe_priv_data *priv = netdev_priv(dev);
1849	void __iomem *ioaddr = (void __iomem *)priv->ioaddr;
1850	unsigned int value = 0;
1851	u32 mc_filter[2];
1852	struct netdev_hw_addr *ha;
1853	int reg = 1;
1854
1855	netdev_dbg(dev, "%s: # mcasts %d, # unicast %d\n",
1856	__func__, netdev_mc_count(dev), netdev_uc_count(dev));
1857
1858	if (dev->flags & IFF_PROMISC) {
1859	value = SXGBE_FRAME_FILTER_PR;
1860
1861	} else if ((netdev_mc_count(dev) > SXGBE_HASH_TABLE_SIZE) ||
1862	(dev->flags & IFF_ALLMULTI)) {
1863	value = SXGBE_FRAME_FILTER_PM; /* pass all multi */
1864	writel(val: 0xffffffff, addr: ioaddr + SXGBE_HASH_HIGH);
1865	writel(val: 0xffffffff, addr: ioaddr + SXGBE_HASH_LOW);
1866
1867	} else if (!netdev_mc_empty(dev)) {
1868	/* Hash filter for multicast */
1869	value = SXGBE_FRAME_FILTER_HMC;
1870
1871	memset(mc_filter, 0, sizeof(mc_filter));
1872	netdev_for_each_mc_addr(ha, dev) {
1873	/* The upper 6 bits of the calculated CRC are used to
1874	* index the contens of the hash table
1875	*/
1876	int bit_nr = bitrev32(~crc32_le(~0, ha->addr, 6)) >> 26;
1877
1878	/* The most significant bit determines the register to
1879	* use (H/L) while the other 5 bits determine the bit
1880	* within the register.
1881	*/
1882	mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31);
1883	}
1884	writel(val: mc_filter[0], addr: ioaddr + SXGBE_HASH_LOW);
1885	writel(val: mc_filter[1], addr: ioaddr + SXGBE_HASH_HIGH);
1886	}
1887
1888	/* Handle multiple unicast addresses (perfect filtering) */
1889	if (netdev_uc_count(dev) > SXGBE_MAX_PERFECT_ADDRESSES)
1890	/* Switch to promiscuous mode if more than 16 addrs
1891	* are required
1892	*/
1893	value |= SXGBE_FRAME_FILTER_PR;
1894	else {
1895	netdev_for_each_uc_addr(ha, dev) {
1896	sxgbe_set_umac_addr(ioaddr, addr: ha->addr, reg_n: reg);
1897	reg++;
1898	}
1899	}
1900	#ifdef FRAME_FILTER_DEBUG
1901	/* Enable Receive all mode (to debug filtering_fail errors) */
1902	value |= SXGBE_FRAME_FILTER_RA;
1903	#endif
1904	writel(val: value, addr: ioaddr + SXGBE_FRAME_FILTER);
1905
1906	netdev_dbg(dev, "Filter: 0x%08x\n\tHash: HI 0x%08x, LO 0x%08x\n",
1907	readl(ioaddr + SXGBE_FRAME_FILTER),
1908	readl(ioaddr + SXGBE_HASH_HIGH),
1909	readl(ioaddr + SXGBE_HASH_LOW));
1910	}
1911
1912	#ifdef CONFIG_NET_POLL_CONTROLLER
1913	/**
1914	* sxgbe_poll_controller - entry point for polling receive by device
1915	* @dev : pointer to the device structure
1916	* Description:
1917	* This function is used by NETCONSOLE and other diagnostic tools
1918	* to allow network I/O with interrupts disabled.
1919	* Return value:
1920	* Void.
1921	*/
1922	static void sxgbe_poll_controller(struct net_device *dev)
1923	{
1924	struct sxgbe_priv_data *priv = netdev_priv(dev);
1925
1926	disable_irq(irq: priv->irq);
1927	sxgbe_rx_interrupt(irq: priv->irq, dev_id: dev);
1928	enable_irq(irq: priv->irq);
1929	}
1930	#endif
1931
1932	/* sxgbe_ioctl - Entry point for the Ioctl
1933	* @dev: Device pointer.
1934	* @rq: An IOCTL specefic structure, that can contain a pointer to
1935	* a proprietary structure used to pass information to the driver.
1936	* @cmd: IOCTL command
1937	* Description:
1938	* Currently it supports the phy_mii_ioctl(...) and HW time stamping.
1939	*/
1940	static int sxgbe_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1941	{
1942	int ret = -EOPNOTSUPP;
1943
1944	if (!netif_running(dev))
1945	return -EINVAL;
1946
1947	switch (cmd) {
1948	case SIOCGMIIPHY:
1949	case SIOCGMIIREG:
1950	case SIOCSMIIREG:
1951	ret = phy_do_ioctl(dev, ifr: rq, cmd);
1952	break;
1953	default:
1954	break;
1955	}
1956
1957	return ret;
1958	}
1959
1960	static const struct net_device_ops sxgbe_netdev_ops = {
1961	.ndo_open = sxgbe_open,
1962	.ndo_start_xmit = sxgbe_xmit,
1963	.ndo_stop = sxgbe_release,
1964	.ndo_get_stats64 = sxgbe_get_stats64,
1965	.ndo_change_mtu = sxgbe_change_mtu,
1966	.ndo_set_features = sxgbe_set_features,
1967	.ndo_set_rx_mode = sxgbe_set_rx_mode,
1968	.ndo_tx_timeout = sxgbe_tx_timeout,
1969	.ndo_eth_ioctl = sxgbe_ioctl,
1970	#ifdef CONFIG_NET_POLL_CONTROLLER
1971	.ndo_poll_controller = sxgbe_poll_controller,
1972	#endif
1973	.ndo_set_mac_address = eth_mac_addr,
1974	};
1975
1976	/* Get the hardware ops */
1977	static void sxgbe_get_ops(struct sxgbe_ops * const ops_ptr)
1978	{
1979	ops_ptr->mac = sxgbe_get_core_ops();
1980	ops_ptr->desc = sxgbe_get_desc_ops();
1981	ops_ptr->dma = sxgbe_get_dma_ops();
1982	ops_ptr->mtl = sxgbe_get_mtl_ops();
1983
1984	/* set the MDIO communication Address/Data regisers */
1985	ops_ptr->mii.addr = SXGBE_MDIO_SCMD_ADD_REG;
1986	ops_ptr->mii.data = SXGBE_MDIO_SCMD_DATA_REG;
1987
1988	/* Assigning the default link settings
1989	* no SXGBE defined default values to be set in registers,
1990	* so assigning as 0 for port and duplex
1991	*/
1992	ops_ptr->link.port = 0;
1993	ops_ptr->link.duplex = 0;
1994	ops_ptr->link.speed = SXGBE_SPEED_10G;
1995	}
1996
1997	/**
1998	* sxgbe_hw_init - Init the GMAC device
1999	* @priv: driver private structure
2000	* Description: this function checks the HW capability
2001	* (if supported) and sets the driver's features.
2002	*/
2003	static int sxgbe_hw_init(struct sxgbe_priv_data * const priv)
2004	{
2005	u32 ctrl_ids;
2006
2007	priv->hw = kmalloc(size: sizeof(*priv->hw), GFP_KERNEL);
2008	if(!priv->hw)
2009	return -ENOMEM;
2010
2011	/* get the hardware ops */
2012	sxgbe_get_ops(ops_ptr: priv->hw);
2013
2014	/* get the controller id */
2015	ctrl_ids = priv->hw->mac->get_controller_version(priv->ioaddr);
2016	priv->hw->ctrl_uid = (ctrl_ids & 0x00ff0000) >> 16;
2017	priv->hw->ctrl_id = (ctrl_ids & 0x000000ff);
2018	pr_info("user ID: 0x%x, Controller ID: 0x%x\n",
2019	priv->hw->ctrl_uid, priv->hw->ctrl_id);
2020
2021	/* get the H/W features */
2022	if (!sxgbe_get_hw_features(priv))
2023	pr_info("Hardware features not found\n");
2024
2025	if (priv->hw_cap.tx_csum_offload)
2026	pr_info("TX Checksum offload supported\n");
2027
2028	if (priv->hw_cap.rx_csum_offload)
2029	pr_info("RX Checksum offload supported\n");
2030
2031	return 0;
2032	}
2033
2034	static int sxgbe_sw_reset(void __iomem *addr)
2035	{
2036	int retry_count = 10;
2037
2038	writel(SXGBE_DMA_SOFT_RESET, addr: addr + SXGBE_DMA_MODE_REG);
2039	while (retry_count--) {
2040	if (!(readl(addr: addr + SXGBE_DMA_MODE_REG) &
2041	SXGBE_DMA_SOFT_RESET))
2042	break;
2043	mdelay(10);
2044	}
2045
2046	if (retry_count < 0)
2047	return -EBUSY;
2048
2049	return 0;
2050	}
2051
2052	/**
2053	* sxgbe_drv_probe
2054	* @device: device pointer
2055	* @plat_dat: platform data pointer
2056	* @addr: iobase memory address
2057	* Description: this is the main probe function used to
2058	* call the alloc_etherdev, allocate the priv structure.
2059	*/
2060	struct sxgbe_priv_data *sxgbe_drv_probe(struct device *device,
2061	struct sxgbe_plat_data *plat_dat,
2062	void __iomem *addr)
2063	{
2064	struct sxgbe_priv_data *priv;
2065	struct net_device *ndev;
2066	int ret;
2067	u8 queue_num;
2068
2069	ndev = alloc_etherdev_mqs(sizeof_priv: sizeof(struct sxgbe_priv_data),
2070	SXGBE_TX_QUEUES, SXGBE_RX_QUEUES);
2071	if (!ndev)
2072	return NULL;
2073
2074	SET_NETDEV_DEV(ndev, device);
2075
2076	priv = netdev_priv(dev: ndev);
2077	priv->device = device;
2078	priv->dev = ndev;
2079
2080	sxgbe_set_ethtool_ops(netdev: ndev);
2081	priv->plat = plat_dat;
2082	priv->ioaddr = addr;
2083
2084	ret = sxgbe_sw_reset(addr: priv->ioaddr);
2085	if (ret)
2086	goto error_free_netdev;
2087
2088	/* Verify driver arguments */
2089	sxgbe_verify_args();
2090
2091	/* Init MAC and get the capabilities */
2092	ret = sxgbe_hw_init(priv);
2093	if (ret)
2094	goto error_free_netdev;
2095
2096	/* allocate memory resources for Descriptor rings */
2097	ret = txring_mem_alloc(priv);
2098	if (ret)
2099	goto error_free_hw;
2100
2101	ret = rxring_mem_alloc(priv);
2102	if (ret)
2103	goto error_free_hw;
2104
2105	ndev->netdev_ops = &sxgbe_netdev_ops;
2106
2107	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2108	NETIF_F_RXCSUM | NETIF_F_TSO | NETIF_F_TSO6 |
2109	NETIF_F_GRO;
2110	ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
2111	ndev->watchdog_timeo = msecs_to_jiffies(TX_TIMEO);
2112
2113	/* assign filtering support */
2114	ndev->priv_flags |= IFF_UNICAST_FLT;
2115
2116	/* MTU range: 68 - 9000 */
2117	ndev->min_mtu = MIN_MTU;
2118	ndev->max_mtu = MAX_MTU;
2119
2120	priv->msg_enable = netif_msg_init(debug_value: debug, default_msg_enable_bits: default_msg_level);
2121
2122	/* Enable TCP segmentation offload for all DMA channels */
2123	if (priv->hw_cap.tcpseg_offload) {
2124	SXGBE_FOR_EACH_QUEUE(SXGBE_TX_QUEUES, queue_num) {
2125	priv->hw->dma->enable_tso(priv->ioaddr, queue_num);
2126	}
2127	}
2128
2129	/* Enable Rx checksum offload */
2130	if (priv->hw_cap.rx_csum_offload) {
2131	priv->hw->mac->enable_rx_csum(priv->ioaddr);
2132	priv->rxcsum_insertion = true;
2133	}
2134
2135	/* Initialise pause frame settings */
2136	priv->rx_pause = 1;
2137	priv->tx_pause = 1;
2138
2139	/* Rx Watchdog is available, enable depend on platform data */
2140	if (!priv->plat->riwt_off) {
2141	priv->use_riwt = 1;
2142	pr_info("Enable RX Mitigation via HW Watchdog Timer\n");
2143	}
2144
2145	netif_napi_add(dev: ndev, napi: &priv->napi, poll: sxgbe_poll);
2146
2147	spin_lock_init(&priv->stats_lock);
2148
2149	priv->sxgbe_clk = clk_get(dev: priv->device, SXGBE_RESOURCE_NAME);
2150	if (IS_ERR(ptr: priv->sxgbe_clk)) {
2151	netdev_warn(dev: ndev, format: "%s: warning: cannot get CSR clock\n",
2152	__func__);
2153	goto error_napi_del;
2154	}
2155
2156	/* If a specific clk_csr value is passed from the platform
2157	* this means that the CSR Clock Range selection cannot be
2158	* changed at run-time and it is fixed. Viceversa the driver'll try to
2159	* set the MDC clock dynamically according to the csr actual
2160	* clock input.
2161	*/
2162	if (!priv->plat->clk_csr)
2163	sxgbe_clk_csr_set(priv);
2164	else
2165	priv->clk_csr = priv->plat->clk_csr;
2166
2167	/* MDIO bus Registration */
2168	ret = sxgbe_mdio_register(ndev);
2169	if (ret < 0) {
2170	netdev_dbg(ndev, "%s: MDIO bus (id: %d) registration failed\n",
2171	__func__, priv->plat->bus_id);
2172	goto error_clk_put;
2173	}
2174
2175	ret = register_netdev(dev: ndev);
2176	if (ret) {
2177	pr_err("%s: ERROR %i registering the device\n", __func__, ret);
2178	goto error_mdio_unregister;
2179	}
2180
2181	sxgbe_check_ether_addr(priv);
2182
2183	return priv;
2184
2185	error_mdio_unregister:
2186	sxgbe_mdio_unregister(ndev);
2187	error_clk_put:
2188	clk_put(clk: priv->sxgbe_clk);
2189	error_napi_del:
2190	netif_napi_del(napi: &priv->napi);
2191	error_free_hw:
2192	kfree(objp: priv->hw);
2193	error_free_netdev:
2194	free_netdev(dev: ndev);
2195
2196	return NULL;
2197	}
2198
2199	/**
2200	* sxgbe_drv_remove
2201	* @ndev: net device pointer
2202	* Description: this function resets the TX/RX processes, disables the MAC RX/TX
2203	* changes the link status, releases the DMA descriptor rings.
2204	*/
2205	void sxgbe_drv_remove(struct net_device *ndev)
2206	{
2207	struct sxgbe_priv_data *priv = netdev_priv(dev: ndev);
2208	u8 queue_num;
2209
2210	netdev_info(dev: ndev, format: "%s: removing driver\n", __func__);
2211
2212	SXGBE_FOR_EACH_QUEUE(SXGBE_RX_QUEUES, queue_num) {
2213	priv->hw->mac->disable_rxqueue(priv->ioaddr, queue_num);
2214	}
2215
2216	priv->hw->dma->stop_rx(priv->ioaddr, SXGBE_RX_QUEUES);
2217	priv->hw->dma->stop_tx(priv->ioaddr, SXGBE_TX_QUEUES);
2218
2219	priv->hw->mac->enable_tx(priv->ioaddr, false);
2220	priv->hw->mac->enable_rx(priv->ioaddr, false);
2221
2222	unregister_netdev(dev: ndev);
2223
2224	sxgbe_mdio_unregister(ndev);
2225
2226	clk_put(clk: priv->sxgbe_clk);
2227
2228	netif_napi_del(napi: &priv->napi);
2229
2230	kfree(objp: priv->hw);
2231
2232	free_netdev(dev: ndev);
2233	}
2234
2235	#ifdef CONFIG_PM
2236	int sxgbe_suspend(struct net_device *ndev)
2237	{
2238	return 0;
2239	}
2240
2241	int sxgbe_resume(struct net_device *ndev)
2242	{
2243	return 0;
2244	}
2245
2246	int sxgbe_freeze(struct net_device *ndev)
2247	{
2248	return -ENOSYS;
2249	}
2250
2251	int sxgbe_restore(struct net_device *ndev)
2252	{
2253	return -ENOSYS;
2254	}
2255	#endif /* CONFIG_PM */
2256
2257	/* Driver is configured as Platform driver */
2258	static int __init sxgbe_init(void)
2259	{
2260	int ret;
2261
2262	ret = sxgbe_register_platform();
2263	if (ret)
2264	goto err;
2265	return 0;
2266	err:
2267	pr_err("driver registration failed\n");
2268	return ret;
2269	}
2270
2271	static void __exit sxgbe_exit(void)
2272	{
2273	sxgbe_unregister_platform();
2274	}
2275
2276	module_init(sxgbe_init);
2277	module_exit(sxgbe_exit);
2278
2279	#ifndef MODULE
2280	static int __init sxgbe_cmdline_opt(char *str)
2281	{
2282	char *opt;
2283
2284	if (!str || !*str)
2285	return 1;
2286	while ((opt = strsep(&str, ",")) != NULL) {
2287	if (!strncmp(opt, "eee_timer:", 10)) {
2288	if (kstrtoint(s: opt + 10, base: 0, res: &eee_timer))
2289	goto err;
2290	}
2291	}
2292	return 1;
2293
2294	err:
2295	pr_err("%s: ERROR broken module parameter conversion\n", __func__);
2296	return 1;
2297	}
2298
2299	__setup("sxgbeeth=", sxgbe_cmdline_opt);
2300	#endif /* MODULE */
2301
2302
2303
2304	MODULE_DESCRIPTION("Samsung 10G/2.5G/1G Ethernet PLATFORM driver");
2305
2306	MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
2307	MODULE_PARM_DESC(eee_timer, "EEE-LPI Default LS timer value");
2308
2309	MODULE_AUTHOR("Siva Reddy Kallam <siva.kallam@samsung.com>");
2310	MODULE_AUTHOR("ByungHo An <bh74.an@samsung.com>");
2311	MODULE_AUTHOR("Girish K S <ks.giri@samsung.com>");
2312	MODULE_AUTHOR("Vipul Pandya <vipul.pandya@samsung.com>");
2313
2314	MODULE_LICENSE("GPL");
2315