1	// SPDX-License-Identifier: GPL-2.0
2	/* Renesas Ethernet AVB device driver
3	*
4	* Copyright (C) 2014-2019 Renesas Electronics Corporation
5	* Copyright (C) 2015 Renesas Solutions Corp.
6	* Copyright (C) 2015-2016 Cogent Embedded, Inc. <source@cogentembedded.com>
7	*
8	* Based on the SuperH Ethernet driver
9	*/
10
11	#include <linux/cache.h>
12	#include <linux/clk.h>
13	#include <linux/delay.h>
14	#include <linux/dma-mapping.h>
15	#include <linux/err.h>
16	#include <linux/etherdevice.h>
17	#include <linux/ethtool.h>
18	#include <linux/if_vlan.h>
19	#include <linux/kernel.h>
20	#include <linux/list.h>
21	#include <linux/module.h>
22	#include <linux/net_tstamp.h>
23	#include <linux/of.h>
24	#include <linux/of_mdio.h>
25	#include <linux/of_net.h>
26	#include <linux/platform_device.h>
27	#include <linux/pm_runtime.h>
28	#include <linux/slab.h>
29	#include <linux/spinlock.h>
30	#include <linux/reset.h>
31	#include <linux/math64.h>
32	#include <net/ip.h>
33
34	#include "ravb.h"
35
36	#define RAVB_DEF_MSG_ENABLE \
37	(NETIF_MSG_LINK | \
38	NETIF_MSG_TIMER | \
39	NETIF_MSG_RX_ERR | \
40	NETIF_MSG_TX_ERR)
41
42	void ravb_modify(struct net_device *ndev, enum ravb_reg reg, u32 clear,
43	u32 set)
44	{
45	ravb_write(ndev, data: (ravb_read(ndev, reg) & ~clear) | set, reg);
46	}
47
48	int ravb_wait(struct net_device *ndev, enum ravb_reg reg, u32 mask, u32 value)
49	{
50	int i;
51
52	for (i = 0; i < 10000; i++) {
53	if ((ravb_read(ndev, reg) & mask) == value)
54	return 0;
55	udelay(10);
56	}
57	return -ETIMEDOUT;
58	}
59
60	static int ravb_set_opmode(struct net_device *ndev, u32 opmode)
61	{
62	u32 csr_ops = 1U << (opmode & CCC_OPC);
63	u32 ccc_mask = CCC_OPC;
64	int error;
65
66	/* If gPTP active in config mode is supported it needs to be configured
67	* along with CSEL and operating mode in the same access. This is a
68	* hardware limitation.
69	*/
70	if (opmode & CCC_GAC)
71	ccc_mask |= CCC_GAC | CCC_CSEL;
72
73	/* Set operating mode */
74	ravb_modify(ndev, reg: CCC, clear: ccc_mask, set: opmode);
75	/* Check if the operating mode is changed to the requested one */
76	error = ravb_wait(ndev, reg: CSR, mask: CSR_OPS, value: csr_ops);
77	if (error) {
78	netdev_err(dev: ndev, format: "failed to switch device to requested mode (%u)\n",
79	opmode & CCC_OPC);
80	}
81
82	return error;
83	}
84
85	static void ravb_set_rate_gbeth(struct net_device *ndev)
86	{
87	struct ravb_private *priv = netdev_priv(dev: ndev);
88
89	switch (priv->speed) {
90	case 10: /* 10BASE */
91	ravb_write(ndev, data: GBETH_GECMR_SPEED_10, reg: GECMR);
92	break;
93	case 100: /* 100BASE */
94	ravb_write(ndev, data: GBETH_GECMR_SPEED_100, reg: GECMR);
95	break;
96	case 1000: /* 1000BASE */
97	ravb_write(ndev, data: GBETH_GECMR_SPEED_1000, reg: GECMR);
98	break;
99	}
100	}
101
102	static void ravb_set_rate_rcar(struct net_device *ndev)
103	{
104	struct ravb_private *priv = netdev_priv(dev: ndev);
105
106	switch (priv->speed) {
107	case 100: /* 100BASE */
108	ravb_write(ndev, data: GECMR_SPEED_100, reg: GECMR);
109	break;
110	case 1000: /* 1000BASE */
111	ravb_write(ndev, data: GECMR_SPEED_1000, reg: GECMR);
112	break;
113	}
114	}
115
116	static struct sk_buff *
117	ravb_alloc_skb(struct net_device *ndev, const struct ravb_hw_info *info,
118	gfp_t gfp_mask)
119	{
120	struct sk_buff *skb;
121	u32 reserve;
122
123	skb = __netdev_alloc_skb(dev: ndev, length: info->rx_max_frame_size + RAVB_ALIGN - 1,
124	gfp_mask);
125	if (!skb)
126	return NULL;
127
128	reserve = (unsigned long)skb->data & (RAVB_ALIGN - 1);
129	if (reserve)
130	skb_reserve(skb, RAVB_ALIGN - reserve);
131
132	return skb;
133	}
134
135	/* Get MAC address from the MAC address registers
136	*
137	* Ethernet AVB device doesn't have ROM for MAC address.
138	* This function gets the MAC address that was used by a bootloader.
139	*/
140	static void ravb_read_mac_address(struct device_node *np,
141	struct net_device *ndev)
142	{
143	int ret;
144
145	ret = of_get_ethdev_address(np, dev: ndev);
146	if (ret) {
147	u32 mahr = ravb_read(ndev, reg: MAHR);
148	u32 malr = ravb_read(ndev, reg: MALR);
149	u8 addr[ETH_ALEN];
150
151	addr[0] = (mahr >> 24) & 0xFF;
152	addr[1] = (mahr >> 16) & 0xFF;
153	addr[2] = (mahr >> 8) & 0xFF;
154	addr[3] = (mahr >> 0) & 0xFF;
155	addr[4] = (malr >> 8) & 0xFF;
156	addr[5] = (malr >> 0) & 0xFF;
157	eth_hw_addr_set(dev: ndev, addr);
158	}
159	}
160
161	static void ravb_mdio_ctrl(struct mdiobb_ctrl *ctrl, u32 mask, int set)
162	{
163	struct ravb_private *priv = container_of(ctrl, struct ravb_private,
164	mdiobb);
165
166	ravb_modify(ndev: priv->ndev, reg: PIR, clear: mask, set: set ? mask : 0);
167	}
168
169	/* MDC pin control */
170	static void ravb_set_mdc(struct mdiobb_ctrl *ctrl, int level)
171	{
172	ravb_mdio_ctrl(ctrl, mask: PIR_MDC, set: level);
173	}
174
175	/* Data I/O pin control */
176	static void ravb_set_mdio_dir(struct mdiobb_ctrl *ctrl, int output)
177	{
178	ravb_mdio_ctrl(ctrl, mask: PIR_MMD, set: output);
179	}
180
181	/* Set data bit */
182	static void ravb_set_mdio_data(struct mdiobb_ctrl *ctrl, int value)
183	{
184	ravb_mdio_ctrl(ctrl, mask: PIR_MDO, set: value);
185	}
186
187	/* Get data bit */
188	static int ravb_get_mdio_data(struct mdiobb_ctrl *ctrl)
189	{
190	struct ravb_private *priv = container_of(ctrl, struct ravb_private,
191	mdiobb);
192
193	return (ravb_read(ndev: priv->ndev, reg: PIR) & PIR_MDI) != 0;
194	}
195
196	/* MDIO bus control struct */
197	static const struct mdiobb_ops bb_ops = {
198	.owner = THIS_MODULE,
199	.set_mdc = ravb_set_mdc,
200	.set_mdio_dir = ravb_set_mdio_dir,
201	.set_mdio_data = ravb_set_mdio_data,
202	.get_mdio_data = ravb_get_mdio_data,
203	};
204
205	static struct ravb_rx_desc *
206	ravb_rx_get_desc(struct ravb_private *priv, unsigned int q,
207	unsigned int i)
208	{
209	return priv->rx_ring[q].raw + priv->info->rx_desc_size * i;
210	}
211
212	/* Free TX skb function for AVB-IP */
213	static int ravb_tx_free(struct net_device *ndev, int q, bool free_txed_only)
214	{
215	struct ravb_private *priv = netdev_priv(dev: ndev);
216	struct net_device_stats *stats = &priv->stats[q];
217	unsigned int num_tx_desc = priv->num_tx_desc;
218	struct ravb_tx_desc *desc;
219	unsigned int entry;
220	int free_num = 0;
221	u32 size;
222
223	for (; priv->cur_tx[q] - priv->dirty_tx[q] > 0; priv->dirty_tx[q]++) {
224	bool txed;
225
226	entry = priv->dirty_tx[q] % (priv->num_tx_ring[q] *
227	num_tx_desc);
228	desc = &priv->tx_ring[q][entry];
229	txed = desc->die_dt == DT_FEMPTY;
230	if (free_txed_only && !txed)
231	break;
232	/* Descriptor type must be checked before all other reads */
233	dma_rmb();
234	size = le16_to_cpu(desc->ds_tagl) & TX_DS;
235	/* Free the original skb. */
236	if (priv->tx_skb[q][entry / num_tx_desc]) {
237	dma_unmap_single(ndev->dev.parent, le32_to_cpu(desc->dptr),
238	size, DMA_TO_DEVICE);
239	/* Last packet descriptor? */
240	if (entry % num_tx_desc == num_tx_desc - 1) {
241	entry /= num_tx_desc;
242	dev_kfree_skb_any(skb: priv->tx_skb[q][entry]);
243	priv->tx_skb[q][entry] = NULL;
244	if (txed)
245	stats->tx_packets++;
246	}
247	free_num++;
248	}
249	if (txed)
250	stats->tx_bytes += size;
251	desc->die_dt = DT_EEMPTY;
252	}
253	return free_num;
254	}
255
256	static void ravb_rx_ring_free(struct net_device *ndev, int q)
257	{
258	struct ravb_private *priv = netdev_priv(dev: ndev);
259	unsigned int ring_size;
260	unsigned int i;
261
262	if (!priv->rx_ring[q].raw)
263	return;
264
265	for (i = 0; i < priv->num_rx_ring[q]; i++) {
266	struct ravb_rx_desc *desc = ravb_rx_get_desc(priv, q, i);
267
268	if (!dma_mapping_error(dev: ndev->dev.parent,
269	le32_to_cpu(desc->dptr)))
270	dma_unmap_single(ndev->dev.parent,
271	le32_to_cpu(desc->dptr),
272	priv->info->rx_max_frame_size,
273	DMA_FROM_DEVICE);
274	}
275	ring_size = priv->info->rx_desc_size * (priv->num_rx_ring[q] + 1);
276	dma_free_coherent(dev: ndev->dev.parent, size: ring_size, cpu_addr: priv->rx_ring[q].raw,
277	dma_handle: priv->rx_desc_dma[q]);
278	priv->rx_ring[q].raw = NULL;
279	}
280
281	/* Free skb's and DMA buffers for Ethernet AVB */
282	static void ravb_ring_free(struct net_device *ndev, int q)
283	{
284	struct ravb_private *priv = netdev_priv(dev: ndev);
285	unsigned int num_tx_desc = priv->num_tx_desc;
286	unsigned int ring_size;
287	unsigned int i;
288
289	ravb_rx_ring_free(ndev, q);
290
291	if (priv->tx_ring[q]) {
292	ravb_tx_free(ndev, q, free_txed_only: false);
293
294	ring_size = sizeof(struct ravb_tx_desc) *
295	(priv->num_tx_ring[q] * num_tx_desc + 1);
296	dma_free_coherent(dev: ndev->dev.parent, size: ring_size, cpu_addr: priv->tx_ring[q],
297	dma_handle: priv->tx_desc_dma[q]);
298	priv->tx_ring[q] = NULL;
299	}
300
301	/* Free RX skb ringbuffer */
302	if (priv->rx_skb[q]) {
303	for (i = 0; i < priv->num_rx_ring[q]; i++)
304	dev_kfree_skb(priv->rx_skb[q][i]);
305	}
306	kfree(objp: priv->rx_skb[q]);
307	priv->rx_skb[q] = NULL;
308
309	/* Free aligned TX buffers */
310	kfree(objp: priv->tx_align[q]);
311	priv->tx_align[q] = NULL;
312
313	/* Free TX skb ringbuffer.
314	* SKBs are freed by ravb_tx_free() call above.
315	*/
316	kfree(objp: priv->tx_skb[q]);
317	priv->tx_skb[q] = NULL;
318	}
319
320	static void ravb_rx_ring_format(struct net_device *ndev, int q)
321	{
322	struct ravb_private *priv = netdev_priv(dev: ndev);
323	struct ravb_rx_desc *rx_desc;
324	unsigned int rx_ring_size;
325	dma_addr_t dma_addr;
326	unsigned int i;
327
328	rx_ring_size = priv->info->rx_desc_size * priv->num_rx_ring[q];
329	memset(priv->rx_ring[q].raw, 0, rx_ring_size);
330	/* Build RX ring buffer */
331	for (i = 0; i < priv->num_rx_ring[q]; i++) {
332	/* RX descriptor */
333	rx_desc = ravb_rx_get_desc(priv, q, i);
334	rx_desc->ds_cc = cpu_to_le16(priv->info->rx_max_desc_use);
335	dma_addr = dma_map_single(ndev->dev.parent, priv->rx_skb[q][i]->data,
336	priv->info->rx_max_frame_size,
337	DMA_FROM_DEVICE);
338	/* We just set the data size to 0 for a failed mapping which
339	* should prevent DMA from happening...
340	*/
341	if (dma_mapping_error(dev: ndev->dev.parent, dma_addr))
342	rx_desc->ds_cc = cpu_to_le16(0);
343	rx_desc->dptr = cpu_to_le32(dma_addr);
344	rx_desc->die_dt = DT_FEMPTY;
345	}
346	rx_desc = ravb_rx_get_desc(priv, q, i);
347	rx_desc->dptr = cpu_to_le32((u32)priv->rx_desc_dma[q]);
348	rx_desc->die_dt = DT_LINKFIX; /* type */
349	}
350
351	/* Format skb and descriptor buffer for Ethernet AVB */
352	static void ravb_ring_format(struct net_device *ndev, int q)
353	{
354	struct ravb_private *priv = netdev_priv(dev: ndev);
355	unsigned int num_tx_desc = priv->num_tx_desc;
356	struct ravb_tx_desc *tx_desc;
357	struct ravb_desc *desc;
358	unsigned int tx_ring_size = sizeof(*tx_desc) * priv->num_tx_ring[q] *
359	num_tx_desc;
360	unsigned int i;
361
362	priv->cur_rx[q] = 0;
363	priv->cur_tx[q] = 0;
364	priv->dirty_rx[q] = 0;
365	priv->dirty_tx[q] = 0;
366
367	ravb_rx_ring_format(ndev, q);
368
369	memset(priv->tx_ring[q], 0, tx_ring_size);
370	/* Build TX ring buffer */
371	for (i = 0, tx_desc = priv->tx_ring[q]; i < priv->num_tx_ring[q];
372	i++, tx_desc++) {
373	tx_desc->die_dt = DT_EEMPTY;
374	if (num_tx_desc > 1) {
375	tx_desc++;
376	tx_desc->die_dt = DT_EEMPTY;
377	}
378	}
379	tx_desc->dptr = cpu_to_le32((u32)priv->tx_desc_dma[q]);
380	tx_desc->die_dt = DT_LINKFIX; /* type */
381
382	/* RX descriptor base address for best effort */
383	desc = &priv->desc_bat[RX_QUEUE_OFFSET + q];
384	desc->die_dt = DT_LINKFIX; /* type */
385	desc->dptr = cpu_to_le32((u32)priv->rx_desc_dma[q]);
386
387	/* TX descriptor base address for best effort */
388	desc = &priv->desc_bat[q];
389	desc->die_dt = DT_LINKFIX; /* type */
390	desc->dptr = cpu_to_le32((u32)priv->tx_desc_dma[q]);
391	}
392
393	static void *ravb_alloc_rx_desc(struct net_device *ndev, int q)
394	{
395	struct ravb_private *priv = netdev_priv(dev: ndev);
396	unsigned int ring_size;
397
398	ring_size = priv->info->rx_desc_size * (priv->num_rx_ring[q] + 1);
399
400	priv->rx_ring[q].raw = dma_alloc_coherent(dev: ndev->dev.parent, size: ring_size,
401	dma_handle: &priv->rx_desc_dma[q],
402	GFP_KERNEL);
403
404	return priv->rx_ring[q].raw;
405	}
406
407	/* Init skb and descriptor buffer for Ethernet AVB */
408	static int ravb_ring_init(struct net_device *ndev, int q)
409	{
410	struct ravb_private *priv = netdev_priv(dev: ndev);
411	const struct ravb_hw_info *info = priv->info;
412	unsigned int num_tx_desc = priv->num_tx_desc;
413	unsigned int ring_size;
414	struct sk_buff *skb;
415	unsigned int i;
416
417	/* Allocate RX and TX skb rings */
418	priv->rx_skb[q] = kcalloc(n: priv->num_rx_ring[q],
419	size: sizeof(*priv->rx_skb[q]), GFP_KERNEL);
420	priv->tx_skb[q] = kcalloc(n: priv->num_tx_ring[q],
421	size: sizeof(*priv->tx_skb[q]), GFP_KERNEL);
422	if (!priv->rx_skb[q] || !priv->tx_skb[q])
423	goto error;
424
425	for (i = 0; i < priv->num_rx_ring[q]; i++) {
426	skb = ravb_alloc_skb(ndev, info, GFP_KERNEL);
427	if (!skb)
428	goto error;
429	priv->rx_skb[q][i] = skb;
430	}
431
432	if (num_tx_desc > 1) {
433	/* Allocate rings for the aligned buffers */
434	priv->tx_align[q] = kmalloc(DPTR_ALIGN * priv->num_tx_ring[q] +
435	DPTR_ALIGN - 1, GFP_KERNEL);
436	if (!priv->tx_align[q])
437	goto error;
438	}
439
440	/* Allocate all RX descriptors. */
441	if (!ravb_alloc_rx_desc(ndev, q))
442	goto error;
443
444	priv->dirty_rx[q] = 0;
445
446	/* Allocate all TX descriptors. */
447	ring_size = sizeof(struct ravb_tx_desc) *
448	(priv->num_tx_ring[q] * num_tx_desc + 1);
449	priv->tx_ring[q] = dma_alloc_coherent(dev: ndev->dev.parent, size: ring_size,
450	dma_handle: &priv->tx_desc_dma[q],
451	GFP_KERNEL);
452	if (!priv->tx_ring[q])
453	goto error;
454
455	return 0;
456
457	error:
458	ravb_ring_free(ndev, q);
459
460	return -ENOMEM;
461	}
462
463	static void ravb_csum_init_gbeth(struct net_device *ndev)
464	{
465	bool tx_enable = ndev->features & NETIF_F_HW_CSUM;
466	bool rx_enable = ndev->features & NETIF_F_RXCSUM;
467
468	if (!(tx_enable || rx_enable))
469	goto done;
470
471	ravb_write(ndev, data: 0, reg: CSR0);
472	if (ravb_wait(ndev, reg: CSR0, mask: CSR0_TPE | CSR0_RPE, value: 0)) {
473	netdev_err(dev: ndev, format: "Timeout enabling hardware checksum\n");
474
475	if (tx_enable)
476	ndev->features &= ~NETIF_F_HW_CSUM;
477
478	if (rx_enable)
479	ndev->features &= ~NETIF_F_RXCSUM;
480	} else {
481	if (tx_enable)
482	ravb_write(ndev, data: CSR1_TIP4 | CSR1_TTCP4 | CSR1_TUDP4, reg: CSR1);
483
484	if (rx_enable)
485	ravb_write(ndev, data: CSR2_RIP4 | CSR2_RTCP4 | CSR2_RUDP4 | CSR2_RICMP4,
486	reg: CSR2);
487	}
488
489	done:
490	ravb_write(ndev, data: CSR0_TPE | CSR0_RPE, reg: CSR0);
491	}
492
493	static void ravb_emac_init_gbeth(struct net_device *ndev)
494	{
495	struct ravb_private *priv = netdev_priv(dev: ndev);
496
497	if (priv->phy_interface == PHY_INTERFACE_MODE_MII) {
498	ravb_write(ndev, data: (1000 << 16) | CXR35_SEL_XMII_MII, reg: CXR35);
499	ravb_modify(ndev, reg: CXR31, clear: CXR31_SEL_LINK0 | CXR31_SEL_LINK1, set: 0);
500	} else {
501	ravb_write(ndev, data: (1000 << 16) | CXR35_SEL_XMII_RGMII, reg: CXR35);
502	ravb_modify(ndev, reg: CXR31, clear: CXR31_SEL_LINK0 | CXR31_SEL_LINK1,
503	set: CXR31_SEL_LINK0);
504	}
505
506	/* Receive frame limit set register */
507	ravb_write(ndev, data: priv->info->rx_max_frame_size + ETH_FCS_LEN, reg: RFLR);
508
509	/* EMAC Mode: PAUSE prohibition; Duplex; TX; RX; CRC Pass Through */
510	ravb_write(ndev, data: ECMR_ZPF | ((priv->duplex > 0) ? ECMR_DM : 0) |
511	ECMR_TE | ECMR_RE | ECMR_RCPT |
512	ECMR_TXF | ECMR_RXF, reg: ECMR);
513
514	ravb_set_rate_gbeth(ndev);
515
516	/* Set MAC address */
517	ravb_write(ndev,
518	data: (ndev->dev_addr[0] << 24) | (ndev->dev_addr[1] << 16) |
519	(ndev->dev_addr[2] << 8) | (ndev->dev_addr[3]), reg: MAHR);
520	ravb_write(ndev, data: (ndev->dev_addr[4] << 8) | (ndev->dev_addr[5]), reg: MALR);
521
522	/* E-MAC status register clear */
523	ravb_write(ndev, data: ECSR_ICD | ECSR_LCHNG | ECSR_PFRI, reg: ECSR);
524
525	ravb_csum_init_gbeth(ndev);
526
527	/* E-MAC interrupt enable register */
528	ravb_write(ndev, data: ECSIPR_ICDIP, reg: ECSIPR);
529	}
530
531	static void ravb_emac_init_rcar(struct net_device *ndev)
532	{
533	/* Receive frame limit set register */
534	ravb_write(ndev, data: ndev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN, reg: RFLR);
535
536	/* EMAC Mode: PAUSE prohibition; Duplex; RX Checksum; TX; RX */
537	ravb_write(ndev, data: ECMR_ZPF | ECMR_DM |
538	(ndev->features & NETIF_F_RXCSUM ? ECMR_RCSC : 0) |
539	ECMR_TE | ECMR_RE, reg: ECMR);
540
541	ravb_set_rate_rcar(ndev);
542
543	/* Set MAC address */
544	ravb_write(ndev,
545	data: (ndev->dev_addr[0] << 24) | (ndev->dev_addr[1] << 16) |
546	(ndev->dev_addr[2] << 8) | (ndev->dev_addr[3]), reg: MAHR);
547	ravb_write(ndev,
548	data: (ndev->dev_addr[4] << 8) | (ndev->dev_addr[5]), reg: MALR);
549
550	/* E-MAC status register clear */
551	ravb_write(ndev, data: ECSR_ICD | ECSR_MPD, reg: ECSR);
552
553	/* E-MAC interrupt enable register */
554	ravb_write(ndev, data: ECSIPR_ICDIP | ECSIPR_MPDIP | ECSIPR_LCHNGIP, reg: ECSIPR);
555	}
556
557	/* E-MAC init function */
558	static void ravb_emac_init(struct net_device *ndev)
559	{
560	struct ravb_private *priv = netdev_priv(dev: ndev);
561	const struct ravb_hw_info *info = priv->info;
562
563	info->emac_init(ndev);
564	}
565
566	static int ravb_dmac_init_gbeth(struct net_device *ndev)
567	{
568	struct ravb_private *priv = netdev_priv(dev: ndev);
569	int error;
570
571	error = ravb_ring_init(ndev, q: RAVB_BE);
572	if (error)
573	return error;
574
575	/* Descriptor format */
576	ravb_ring_format(ndev, q: RAVB_BE);
577
578	/* Set DMAC RX */
579	ravb_write(ndev, data: 0x60000000, reg: RCR);
580
581	/* Set Max Frame Length (RTC) */
582	ravb_write(ndev, data: 0x7ffc0000 | priv->info->rx_max_frame_size, reg: RTC);
583
584	/* Set FIFO size */
585	ravb_write(ndev, data: 0x00222200, reg: TGC);
586
587	ravb_write(ndev, data: 0, reg: TCCR);
588
589	/* Frame receive */
590	ravb_write(ndev, data: RIC0_FRE0, reg: RIC0);
591	/* Disable FIFO full warning */
592	ravb_write(ndev, data: 0x0, reg: RIC1);
593	/* Receive FIFO full error, descriptor empty */
594	ravb_write(ndev, data: RIC2_QFE0 | RIC2_RFFE, reg: RIC2);
595
596	ravb_write(ndev, data: TIC_FTE0, reg: TIC);
597
598	return 0;
599	}
600
601	static int ravb_dmac_init_rcar(struct net_device *ndev)
602	{
603	struct ravb_private *priv = netdev_priv(dev: ndev);
604	const struct ravb_hw_info *info = priv->info;
605	int error;
606
607	error = ravb_ring_init(ndev, q: RAVB_BE);
608	if (error)
609	return error;
610	error = ravb_ring_init(ndev, q: RAVB_NC);
611	if (error) {
612	ravb_ring_free(ndev, q: RAVB_BE);
613	return error;
614	}
615
616	/* Descriptor format */
617	ravb_ring_format(ndev, q: RAVB_BE);
618	ravb_ring_format(ndev, q: RAVB_NC);
619
620	/* Set AVB RX */
621	ravb_write(ndev,
622	data: RCR_EFFS | RCR_ENCF | RCR_ETS0 | RCR_ESF | 0x18000000, reg: RCR);
623
624	/* Set FIFO size */
625	ravb_write(ndev, data: TGC_TQP_AVBMODE1 | 0x00112200, reg: TGC);
626
627	/* Timestamp enable */
628	ravb_write(ndev, data: TCCR_TFEN, reg: TCCR);
629
630	/* Interrupt init: */
631	if (info->multi_irqs) {
632	/* Clear DIL.DPLx */
633	ravb_write(ndev, data: 0, reg: DIL);
634	/* Set queue specific interrupt */
635	ravb_write(ndev, data: CIE_CRIE | CIE_CTIE | CIE_CL0M, reg: CIE);
636	}
637	/* Frame receive */
638	ravb_write(ndev, data: RIC0_FRE0 | RIC0_FRE1, reg: RIC0);
639	/* Disable FIFO full warning */
640	ravb_write(ndev, data: 0, reg: RIC1);
641	/* Receive FIFO full error, descriptor empty */
642	ravb_write(ndev, data: RIC2_QFE0 | RIC2_QFE1 | RIC2_RFFE, reg: RIC2);
643	/* Frame transmitted, timestamp FIFO updated */
644	ravb_write(ndev, data: TIC_FTE0 | TIC_FTE1 | TIC_TFUE, reg: TIC);
645
646	return 0;
647	}
648
649	/* Device init function for Ethernet AVB */
650	static int ravb_dmac_init(struct net_device *ndev)
651	{
652	struct ravb_private *priv = netdev_priv(dev: ndev);
653	const struct ravb_hw_info *info = priv->info;
654	int error;
655
656	/* Set CONFIG mode */
657	error = ravb_set_opmode(ndev, opmode: CCC_OPC_CONFIG);
658	if (error)
659	return error;
660
661	error = info->dmac_init(ndev);
662	if (error)
663	return error;
664
665	/* Setting the control will start the AVB-DMAC process. */
666	return ravb_set_opmode(ndev, opmode: CCC_OPC_OPERATION);
667	}
668
669	static void ravb_get_tx_tstamp(struct net_device *ndev)
670	{
671	struct ravb_private *priv = netdev_priv(dev: ndev);
672	struct ravb_tstamp_skb *ts_skb, *ts_skb2;
673	struct skb_shared_hwtstamps shhwtstamps;
674	struct sk_buff *skb;
675	struct timespec64 ts;
676	u16 tag, tfa_tag;
677	int count;
678	u32 tfa2;
679
680	count = (ravb_read(ndev, reg: TSR) & TSR_TFFL) >> 8;
681	while (count--) {
682	tfa2 = ravb_read(ndev, reg: TFA2);
683	tfa_tag = (tfa2 & TFA2_TST) >> 16;
684	ts.tv_nsec = (u64)ravb_read(ndev, reg: TFA0);
685	ts.tv_sec = ((u64)(tfa2 & TFA2_TSV) << 32) |
686	ravb_read(ndev, reg: TFA1);
687	memset(&shhwtstamps, 0, sizeof(shhwtstamps));
688	shhwtstamps.hwtstamp = timespec64_to_ktime(ts);
689	list_for_each_entry_safe(ts_skb, ts_skb2, &priv->ts_skb_list,
690	list) {
691	skb = ts_skb->skb;
692	tag = ts_skb->tag;
693	list_del(entry: &ts_skb->list);
694	kfree(objp: ts_skb);
695	if (tag == tfa_tag) {
696	skb_tstamp_tx(orig_skb: skb, hwtstamps: &shhwtstamps);
697	dev_consume_skb_any(skb);
698	break;
699	} else {
700	dev_kfree_skb_any(skb);
701	}
702	}
703	ravb_modify(ndev, reg: TCCR, clear: TCCR_TFR, set: TCCR_TFR);
704	}
705	}
706
707	static void ravb_rx_csum_gbeth(struct sk_buff *skb)
708	{
709	__wsum csum_ip_hdr, csum_proto;
710	u8 *hw_csum;
711
712	/* The hardware checksum status is contained in sizeof(__sum16) * 2 = 4
713	* bytes appended to packet data. First 2 bytes is ip header checksum
714	* and last 2 bytes is protocol checksum.
715	*/
716	if (unlikely(skb->len < sizeof(__sum16) * 2))
717	return;
718
719	hw_csum = skb_tail_pointer(skb) - sizeof(__sum16);
720	csum_proto = csum_unfold(n: (__force __sum16)get_unaligned_le16(p: hw_csum));
721
722	hw_csum -= sizeof(__sum16);
723	csum_ip_hdr = csum_unfold(n: (__force __sum16)get_unaligned_le16(p: hw_csum));
724	skb_trim(skb, len: skb->len - 2 * sizeof(__sum16));
725
726	/* TODO: IPV6 Rx checksum */
727	if (skb->protocol == htons(ETH_P_IP) && !csum_ip_hdr && !csum_proto)
728	skb->ip_summed = CHECKSUM_UNNECESSARY;
729	}
730
731	static void ravb_rx_csum(struct sk_buff *skb)
732	{
733	u8 *hw_csum;
734
735	/* The hardware checksum is contained in sizeof(__sum16) (2) bytes
736	* appended to packet data
737	*/
738	if (unlikely(skb->len < sizeof(__sum16)))
739	return;
740	hw_csum = skb_tail_pointer(skb) - sizeof(__sum16);
741	skb->csum = csum_unfold(n: (__force __sum16)get_unaligned_le16(p: hw_csum));
742	skb->ip_summed = CHECKSUM_COMPLETE;
743	skb_trim(skb, len: skb->len - sizeof(__sum16));
744	}
745
746	static struct sk_buff *ravb_get_skb_gbeth(struct net_device *ndev, int entry,
747	struct ravb_rx_desc *desc)
748	{
749	struct ravb_private *priv = netdev_priv(dev: ndev);
750	struct sk_buff *skb;
751
752	skb = priv->rx_skb[RAVB_BE][entry];
753	priv->rx_skb[RAVB_BE][entry] = NULL;
754	dma_unmap_single(ndev->dev.parent, le32_to_cpu(desc->dptr),
755	ALIGN(priv->info->rx_max_frame_size, 16),
756	DMA_FROM_DEVICE);
757
758	return skb;
759	}
760
761	/* Packet receive function for Gigabit Ethernet */
762	static bool ravb_rx_gbeth(struct net_device *ndev, int *quota, int q)
763	{
764	struct ravb_private *priv = netdev_priv(dev: ndev);
765	const struct ravb_hw_info *info = priv->info;
766	struct net_device_stats *stats;
767	struct ravb_rx_desc *desc;
768	struct sk_buff *skb;
769	dma_addr_t dma_addr;
770	int rx_packets = 0;
771	u8 desc_status;
772	u16 desc_len;
773	u8 die_dt;
774	int entry;
775	int limit;
776	int i;
777
778	limit = priv->dirty_rx[q] + priv->num_rx_ring[q] - priv->cur_rx[q];
779	stats = &priv->stats[q];
780
781	for (i = 0; i < limit; i++, priv->cur_rx[q]++) {
782	entry = priv->cur_rx[q] % priv->num_rx_ring[q];
783	desc = &priv->rx_ring[q].desc[entry];
784	if (rx_packets == *quota || desc->die_dt == DT_FEMPTY)
785	break;
786
787	/* Descriptor type must be checked before all other reads */
788	dma_rmb();
789	desc_status = desc->msc;
790	desc_len = le16_to_cpu(desc->ds_cc) & RX_DS;
791
792	/* We use 0-byte descriptors to mark the DMA mapping errors */
793	if (!desc_len)
794	continue;
795
796	if (desc_status & MSC_MC)
797	stats->multicast++;
798
799	if (desc_status & (MSC_CRC | MSC_RFE | MSC_RTSF | MSC_RTLF | MSC_CEEF)) {
800	stats->rx_errors++;
801	if (desc_status & MSC_CRC)
802	stats->rx_crc_errors++;
803	if (desc_status & MSC_RFE)
804	stats->rx_frame_errors++;
805	if (desc_status & (MSC_RTLF | MSC_RTSF))
806	stats->rx_length_errors++;
807	if (desc_status & MSC_CEEF)
808	stats->rx_missed_errors++;
809	} else {
810	die_dt = desc->die_dt & 0xF0;
811	switch (die_dt) {
812	case DT_FSINGLE:
813	skb = ravb_get_skb_gbeth(ndev, entry, desc);
814	skb_put(skb, len: desc_len);
815	skb->protocol = eth_type_trans(skb, dev: ndev);
816	if (ndev->features & NETIF_F_RXCSUM)
817	ravb_rx_csum_gbeth(skb);
818	napi_gro_receive(napi: &priv->napi[q], skb);
819	rx_packets++;
820	stats->rx_bytes += desc_len;
821	break;
822	case DT_FSTART:
823	priv->rx_1st_skb = ravb_get_skb_gbeth(ndev, entry, desc);
824	skb_put(skb: priv->rx_1st_skb, len: desc_len);
825	break;
826	case DT_FMID:
827	skb = ravb_get_skb_gbeth(ndev, entry, desc);
828	skb_copy_to_linear_data_offset(skb: priv->rx_1st_skb,
829	offset: priv->rx_1st_skb->len,
830	from: skb->data,
831	len: desc_len);
832	skb_put(skb: priv->rx_1st_skb, len: desc_len);
833	dev_kfree_skb(skb);
834	break;
835	case DT_FEND:
836	skb = ravb_get_skb_gbeth(ndev, entry, desc);
837	skb_copy_to_linear_data_offset(skb: priv->rx_1st_skb,
838	offset: priv->rx_1st_skb->len,
839	from: skb->data,
840	len: desc_len);
841	skb_put(skb: priv->rx_1st_skb, len: desc_len);
842	dev_kfree_skb(skb);
843	priv->rx_1st_skb->protocol =
844	eth_type_trans(skb: priv->rx_1st_skb, dev: ndev);
845	if (ndev->features & NETIF_F_RXCSUM)
846	ravb_rx_csum_gbeth(skb: priv->rx_1st_skb);
847	stats->rx_bytes += priv->rx_1st_skb->len;
848	napi_gro_receive(napi: &priv->napi[q],
849	skb: priv->rx_1st_skb);
850	rx_packets++;
851	break;
852	}
853	}
854	}
855
856	/* Refill the RX ring buffers. */
857	for (; priv->cur_rx[q] - priv->dirty_rx[q] > 0; priv->dirty_rx[q]++) {
858	entry = priv->dirty_rx[q] % priv->num_rx_ring[q];
859	desc = &priv->rx_ring[q].desc[entry];
860	desc->ds_cc = cpu_to_le16(priv->info->rx_max_desc_use);
861
862	if (!priv->rx_skb[q][entry]) {
863	skb = ravb_alloc_skb(ndev, info, GFP_ATOMIC);
864	if (!skb)
865	break;
866	dma_addr = dma_map_single(ndev->dev.parent,
867	skb->data,
868	priv->info->rx_max_frame_size,
869	DMA_FROM_DEVICE);
870	skb_checksum_none_assert(skb);
871	/* We just set the data size to 0 for a failed mapping
872	* which should prevent DMA from happening...
873	*/
874	if (dma_mapping_error(dev: ndev->dev.parent, dma_addr))
875	desc->ds_cc = cpu_to_le16(0);
876	desc->dptr = cpu_to_le32(dma_addr);
877	priv->rx_skb[q][entry] = skb;
878	}
879	/* Descriptor type must be set after all the above writes */
880	dma_wmb();
881	desc->die_dt = DT_FEMPTY;
882	}
883
884	stats->rx_packets += rx_packets;
885	*quota -= rx_packets;
886	return *quota == 0;
887	}
888
889	/* Packet receive function for Ethernet AVB */
890	static bool ravb_rx_rcar(struct net_device *ndev, int *quota, int q)
891	{
892	struct ravb_private *priv = netdev_priv(dev: ndev);
893	const struct ravb_hw_info *info = priv->info;
894	struct net_device_stats *stats = &priv->stats[q];
895	struct ravb_ex_rx_desc *desc;
896	unsigned int limit, i;
897	struct sk_buff *skb;
898	dma_addr_t dma_addr;
899	struct timespec64 ts;
900	int rx_packets = 0;
901	u8 desc_status;
902	u16 pkt_len;
903	int entry;
904
905	limit = priv->dirty_rx[q] + priv->num_rx_ring[q] - priv->cur_rx[q];
906	for (i = 0; i < limit; i++, priv->cur_rx[q]++) {
907	entry = priv->cur_rx[q] % priv->num_rx_ring[q];
908	desc = &priv->rx_ring[q].ex_desc[entry];
909	if (rx_packets == *quota || desc->die_dt == DT_FEMPTY)
910	break;
911
912	/* Descriptor type must be checked before all other reads */
913	dma_rmb();
914	desc_status = desc->msc;
915	pkt_len = le16_to_cpu(desc->ds_cc) & RX_DS;
916
917	/* We use 0-byte descriptors to mark the DMA mapping errors */
918	if (!pkt_len)
919	continue;
920
921	if (desc_status & MSC_MC)
922	stats->multicast++;
923
924	if (desc_status & (MSC_CRC | MSC_RFE | MSC_RTSF | MSC_RTLF |
925	MSC_CEEF)) {
926	stats->rx_errors++;
927	if (desc_status & MSC_CRC)
928	stats->rx_crc_errors++;
929	if (desc_status & MSC_RFE)
930	stats->rx_frame_errors++;
931	if (desc_status & (MSC_RTLF | MSC_RTSF))
932	stats->rx_length_errors++;
933	if (desc_status & MSC_CEEF)
934	stats->rx_missed_errors++;
935	} else {
936	u32 get_ts = priv->tstamp_rx_ctrl & RAVB_RXTSTAMP_TYPE;
937
938	skb = priv->rx_skb[q][entry];
939	priv->rx_skb[q][entry] = NULL;
940	dma_unmap_single(ndev->dev.parent, le32_to_cpu(desc->dptr),
941	priv->info->rx_max_frame_size,
942	DMA_FROM_DEVICE);
943	get_ts &= (q == RAVB_NC) ?
944	RAVB_RXTSTAMP_TYPE_V2_L2_EVENT :
945	~RAVB_RXTSTAMP_TYPE_V2_L2_EVENT;
946	if (get_ts) {
947	struct skb_shared_hwtstamps *shhwtstamps;
948
949	shhwtstamps = skb_hwtstamps(skb);
950	memset(shhwtstamps, 0, sizeof(*shhwtstamps));
951	ts.tv_sec = ((u64) le16_to_cpu(desc->ts_sh) <<
952	32) | le32_to_cpu(desc->ts_sl);
953	ts.tv_nsec = le32_to_cpu(desc->ts_n);
954	shhwtstamps->hwtstamp = timespec64_to_ktime(ts);
955	}
956
957	skb_put(skb, len: pkt_len);
958	skb->protocol = eth_type_trans(skb, dev: ndev);
959	if (ndev->features & NETIF_F_RXCSUM)
960	ravb_rx_csum(skb);
961	napi_gro_receive(napi: &priv->napi[q], skb);
962	rx_packets++;
963	stats->rx_bytes += pkt_len;
964	}
965	}
966
967	/* Refill the RX ring buffers. */
968	for (; priv->cur_rx[q] - priv->dirty_rx[q] > 0; priv->dirty_rx[q]++) {
969	entry = priv->dirty_rx[q] % priv->num_rx_ring[q];
970	desc = &priv->rx_ring[q].ex_desc[entry];
971	desc->ds_cc = cpu_to_le16(priv->info->rx_max_desc_use);
972
973	if (!priv->rx_skb[q][entry]) {
974	skb = ravb_alloc_skb(ndev, info, GFP_ATOMIC);
975	if (!skb)
976	break; /* Better luck next round. */
977	dma_addr = dma_map_single(ndev->dev.parent, skb->data,
978	priv->info->rx_max_frame_size,
979	DMA_FROM_DEVICE);
980	skb_checksum_none_assert(skb);
981	/* We just set the data size to 0 for a failed mapping
982	* which should prevent DMA from happening...
983	*/
984	if (dma_mapping_error(dev: ndev->dev.parent, dma_addr))
985	desc->ds_cc = cpu_to_le16(0);
986	desc->dptr = cpu_to_le32(dma_addr);
987	priv->rx_skb[q][entry] = skb;
988	}
989	/* Descriptor type must be set after all the above writes */
990	dma_wmb();
991	desc->die_dt = DT_FEMPTY;
992	}
993
994	stats->rx_packets += rx_packets;
995	*quota -= rx_packets;
996	return *quota == 0;
997	}
998
999	/* Packet receive function for Ethernet AVB */
1000	static bool ravb_rx(struct net_device *ndev, int *quota, int q)
1001	{
1002	struct ravb_private *priv = netdev_priv(dev: ndev);
1003	const struct ravb_hw_info *info = priv->info;
1004
1005	return info->receive(ndev, quota, q);
1006	}
1007
1008	static void ravb_rcv_snd_disable(struct net_device *ndev)
1009	{
1010	/* Disable TX and RX */
1011	ravb_modify(ndev, reg: ECMR, clear: ECMR_RE | ECMR_TE, set: 0);
1012	}
1013
1014	static void ravb_rcv_snd_enable(struct net_device *ndev)
1015	{
1016	/* Enable TX and RX */
1017	ravb_modify(ndev, reg: ECMR, clear: ECMR_RE | ECMR_TE, set: ECMR_RE | ECMR_TE);
1018	}
1019
1020	/* function for waiting dma process finished */
1021	static int ravb_stop_dma(struct net_device *ndev)
1022	{
1023	struct ravb_private *priv = netdev_priv(dev: ndev);
1024	const struct ravb_hw_info *info = priv->info;
1025	int error;
1026
1027	/* Wait for stopping the hardware TX process */
1028	error = ravb_wait(ndev, reg: TCCR, mask: info->tccr_mask, value: 0);
1029
1030	if (error)
1031	return error;
1032
1033	error = ravb_wait(ndev, reg: CSR, mask: CSR_TPO0 | CSR_TPO1 | CSR_TPO2 | CSR_TPO3,
1034	value: 0);
1035	if (error)
1036	return error;
1037
1038	/* Stop the E-MAC's RX/TX processes. */
1039	ravb_rcv_snd_disable(ndev);
1040
1041	/* Wait for stopping the RX DMA process */
1042	error = ravb_wait(ndev, reg: CSR, mask: CSR_RPO, value: 0);
1043	if (error)
1044	return error;
1045
1046	/* Stop AVB-DMAC process */
1047	return ravb_set_opmode(ndev, opmode: CCC_OPC_CONFIG);
1048	}
1049
1050	/* E-MAC interrupt handler */
1051	static void ravb_emac_interrupt_unlocked(struct net_device *ndev)
1052	{
1053	struct ravb_private *priv = netdev_priv(dev: ndev);
1054	u32 ecsr, psr;
1055
1056	ecsr = ravb_read(ndev, reg: ECSR);
1057	ravb_write(ndev, data: ecsr, reg: ECSR); /* clear interrupt */
1058
1059	if (ecsr & ECSR_MPD)
1060	pm_wakeup_event(dev: &priv->pdev->dev, msec: 0);
1061	if (ecsr & ECSR_ICD)
1062	ndev->stats.tx_carrier_errors++;
1063	if (ecsr & ECSR_LCHNG) {
1064	/* Link changed */
1065	if (priv->no_avb_link)
1066	return;
1067	psr = ravb_read(ndev, reg: PSR);
1068	if (priv->avb_link_active_low)
1069	psr ^= PSR_LMON;
1070	if (!(psr & PSR_LMON)) {
1071	/* DIsable RX and TX */
1072	ravb_rcv_snd_disable(ndev);
1073	} else {
1074	/* Enable RX and TX */
1075	ravb_rcv_snd_enable(ndev);
1076	}
1077	}
1078	}
1079
1080	static irqreturn_t ravb_emac_interrupt(int irq, void *dev_id)
1081	{
1082	struct net_device *ndev = dev_id;
1083	struct ravb_private *priv = netdev_priv(dev: ndev);
1084	struct device *dev = &priv->pdev->dev;
1085	irqreturn_t result = IRQ_HANDLED;
1086
1087	pm_runtime_get_noresume(dev);
1088
1089	if (unlikely(!pm_runtime_active(dev))) {
1090	result = IRQ_NONE;
1091	goto out_rpm_put;
1092	}
1093
1094	spin_lock(lock: &priv->lock);
1095	ravb_emac_interrupt_unlocked(ndev);
1096	spin_unlock(lock: &priv->lock);
1097
1098	out_rpm_put:
1099	pm_runtime_put_noidle(dev);
1100	return result;
1101	}
1102
1103	/* Error interrupt handler */
1104	static void ravb_error_interrupt(struct net_device *ndev)
1105	{
1106	struct ravb_private *priv = netdev_priv(dev: ndev);
1107	u32 eis, ris2;
1108
1109	eis = ravb_read(ndev, reg: EIS);
1110	ravb_write(ndev, data: ~(EIS_QFS | EIS_RESERVED), reg: EIS);
1111	if (eis & EIS_QFS) {
1112	ris2 = ravb_read(ndev, reg: RIS2);
1113	ravb_write(ndev, data: ~(RIS2_QFF0 | RIS2_QFF1 | RIS2_RFFF | RIS2_RESERVED),
1114	reg: RIS2);
1115
1116	/* Receive Descriptor Empty int */
1117	if (ris2 & RIS2_QFF0)
1118	priv->stats[RAVB_BE].rx_over_errors++;
1119
1120	/* Receive Descriptor Empty int */
1121	if (ris2 & RIS2_QFF1)
1122	priv->stats[RAVB_NC].rx_over_errors++;
1123
1124	/* Receive FIFO Overflow int */
1125	if (ris2 & RIS2_RFFF)
1126	priv->rx_fifo_errors++;
1127	}
1128	}
1129
1130	static bool ravb_queue_interrupt(struct net_device *ndev, int q)
1131	{
1132	struct ravb_private *priv = netdev_priv(dev: ndev);
1133	const struct ravb_hw_info *info = priv->info;
1134	u32 ris0 = ravb_read(ndev, reg: RIS0);
1135	u32 ric0 = ravb_read(ndev, reg: RIC0);
1136	u32 tis = ravb_read(ndev, reg: TIS);
1137	u32 tic = ravb_read(ndev, reg: TIC);
1138
1139	if (((ris0 & ric0) & BIT(q)) || ((tis & tic) & BIT(q))) {
1140	if (napi_schedule_prep(n: &priv->napi[q])) {
1141	/* Mask RX and TX interrupts */
1142	if (!info->irq_en_dis) {
1143	ravb_write(ndev, data: ric0 & ~BIT(q), reg: RIC0);
1144	ravb_write(ndev, data: tic & ~BIT(q), reg: TIC);
1145	} else {
1146	ravb_write(ndev, BIT(q), reg: RID0);
1147	ravb_write(ndev, BIT(q), reg: TID);
1148	}
1149	__napi_schedule(n: &priv->napi[q]);
1150	} else {
1151	netdev_warn(dev: ndev,
1152	format: "ignoring interrupt, rx status 0x%08x, rx mask 0x%08x,\n",
1153	ris0, ric0);
1154	netdev_warn(dev: ndev,
1155	format: " tx status 0x%08x, tx mask 0x%08x.\n",
1156	tis, tic);
1157	}
1158	return true;
1159	}
1160	return false;
1161	}
1162
1163	static bool ravb_timestamp_interrupt(struct net_device *ndev)
1164	{
1165	u32 tis = ravb_read(ndev, reg: TIS);
1166
1167	if (tis & TIS_TFUF) {
1168	ravb_write(ndev, data: ~(TIS_TFUF | TIS_RESERVED), reg: TIS);
1169	ravb_get_tx_tstamp(ndev);
1170	return true;
1171	}
1172	return false;
1173	}
1174
1175	static irqreturn_t ravb_interrupt(int irq, void *dev_id)
1176	{
1177	struct net_device *ndev = dev_id;
1178	struct ravb_private *priv = netdev_priv(dev: ndev);
1179	const struct ravb_hw_info *info = priv->info;
1180	struct device *dev = &priv->pdev->dev;
1181	irqreturn_t result = IRQ_NONE;
1182	u32 iss;
1183
1184	pm_runtime_get_noresume(dev);
1185
1186	if (unlikely(!pm_runtime_active(dev)))
1187	goto out_rpm_put;
1188
1189	spin_lock(lock: &priv->lock);
1190	/* Get interrupt status */
1191	iss = ravb_read(ndev, reg: ISS);
1192
1193	/* Received and transmitted interrupts */
1194	if (iss & (ISS_FRS | ISS_FTS | ISS_TFUS)) {
1195	int q;
1196
1197	/* Timestamp updated */
1198	if (ravb_timestamp_interrupt(ndev))
1199	result = IRQ_HANDLED;
1200
1201	/* Network control and best effort queue RX/TX */
1202	if (info->nc_queues) {
1203	for (q = RAVB_NC; q >= RAVB_BE; q--) {
1204	if (ravb_queue_interrupt(ndev, q))
1205	result = IRQ_HANDLED;
1206	}
1207	} else {
1208	if (ravb_queue_interrupt(ndev, q: RAVB_BE))
1209	result = IRQ_HANDLED;
1210	}
1211	}
1212
1213	/* E-MAC status summary */
1214	if (iss & ISS_MS) {
1215	ravb_emac_interrupt_unlocked(ndev);
1216	result = IRQ_HANDLED;
1217	}
1218
1219	/* Error status summary */
1220	if (iss & ISS_ES) {
1221	ravb_error_interrupt(ndev);
1222	result = IRQ_HANDLED;
1223	}
1224
1225	/* gPTP interrupt status summary */
1226	if (iss & ISS_CGIS) {
1227	ravb_ptp_interrupt(ndev);
1228	result = IRQ_HANDLED;
1229	}
1230
1231	spin_unlock(lock: &priv->lock);
1232
1233	out_rpm_put:
1234	pm_runtime_put_noidle(dev);
1235	return result;
1236	}
1237
1238	/* Timestamp/Error/gPTP interrupt handler */
1239	static irqreturn_t ravb_multi_interrupt(int irq, void *dev_id)
1240	{
1241	struct net_device *ndev = dev_id;
1242	struct ravb_private *priv = netdev_priv(dev: ndev);
1243	struct device *dev = &priv->pdev->dev;
1244	irqreturn_t result = IRQ_NONE;
1245	u32 iss;
1246
1247	pm_runtime_get_noresume(dev);
1248
1249	if (unlikely(!pm_runtime_active(dev)))
1250	goto out_rpm_put;
1251
1252	spin_lock(lock: &priv->lock);
1253	/* Get interrupt status */
1254	iss = ravb_read(ndev, reg: ISS);
1255
1256	/* Timestamp updated */
1257	if ((iss & ISS_TFUS) && ravb_timestamp_interrupt(ndev))
1258	result = IRQ_HANDLED;
1259
1260	/* Error status summary */
1261	if (iss & ISS_ES) {
1262	ravb_error_interrupt(ndev);
1263	result = IRQ_HANDLED;
1264	}
1265
1266	/* gPTP interrupt status summary */
1267	if (iss & ISS_CGIS) {
1268	ravb_ptp_interrupt(ndev);
1269	result = IRQ_HANDLED;
1270	}
1271
1272	spin_unlock(lock: &priv->lock);
1273
1274	out_rpm_put:
1275	pm_runtime_put_noidle(dev);
1276	return result;
1277	}
1278
1279	static irqreturn_t ravb_dma_interrupt(int irq, void *dev_id, int q)
1280	{
1281	struct net_device *ndev = dev_id;
1282	struct ravb_private *priv = netdev_priv(dev: ndev);
1283	struct device *dev = &priv->pdev->dev;
1284	irqreturn_t result = IRQ_NONE;
1285
1286	pm_runtime_get_noresume(dev);
1287
1288	if (unlikely(!pm_runtime_active(dev)))
1289	goto out_rpm_put;
1290
1291	spin_lock(lock: &priv->lock);
1292
1293	/* Network control/Best effort queue RX/TX */
1294	if (ravb_queue_interrupt(ndev, q))
1295	result = IRQ_HANDLED;
1296
1297	spin_unlock(lock: &priv->lock);
1298
1299	out_rpm_put:
1300	pm_runtime_put_noidle(dev);
1301	return result;
1302	}
1303
1304	static irqreturn_t ravb_be_interrupt(int irq, void *dev_id)
1305	{
1306	return ravb_dma_interrupt(irq, dev_id, q: RAVB_BE);
1307	}
1308
1309	static irqreturn_t ravb_nc_interrupt(int irq, void *dev_id)
1310	{
1311	return ravb_dma_interrupt(irq, dev_id, q: RAVB_NC);
1312	}
1313
1314	static int ravb_poll(struct napi_struct *napi, int budget)
1315	{
1316	struct net_device *ndev = napi->dev;
1317	struct ravb_private *priv = netdev_priv(dev: ndev);
1318	const struct ravb_hw_info *info = priv->info;
1319	unsigned long flags;
1320	int q = napi - priv->napi;
1321	int mask = BIT(q);
1322	int quota = budget;
1323	bool unmask;
1324
1325	/* Processing RX Descriptor Ring */
1326	/* Clear RX interrupt */
1327	ravb_write(ndev, data: ~(mask | RIS0_RESERVED), reg: RIS0);
1328	unmask = !ravb_rx(ndev, quota: &quota, q);
1329
1330	/* Processing TX Descriptor Ring */
1331	spin_lock_irqsave(&priv->lock, flags);
1332	/* Clear TX interrupt */
1333	ravb_write(ndev, data: ~(mask | TIS_RESERVED), reg: TIS);
1334	ravb_tx_free(ndev, q, free_txed_only: true);
1335	netif_wake_subqueue(dev: ndev, queue_index: q);
1336	spin_unlock_irqrestore(lock: &priv->lock, flags);
1337
1338	/* Receive error message handling */
1339	priv->rx_over_errors = priv->stats[RAVB_BE].rx_over_errors;
1340	if (info->nc_queues)
1341	priv->rx_over_errors += priv->stats[RAVB_NC].rx_over_errors;
1342	if (priv->rx_over_errors != ndev->stats.rx_over_errors)
1343	ndev->stats.rx_over_errors = priv->rx_over_errors;
1344	if (priv->rx_fifo_errors != ndev->stats.rx_fifo_errors)
1345	ndev->stats.rx_fifo_errors = priv->rx_fifo_errors;
1346
1347	if (!unmask)
1348	goto out;
1349
1350	napi_complete(n: napi);
1351
1352	/* Re-enable RX/TX interrupts */
1353	spin_lock_irqsave(&priv->lock, flags);
1354	if (!info->irq_en_dis) {
1355	ravb_modify(ndev, reg: RIC0, clear: mask, set: mask);
1356	ravb_modify(ndev, reg: TIC, clear: mask, set: mask);
1357	} else {
1358	ravb_write(ndev, data: mask, reg: RIE0);
1359	ravb_write(ndev, data: mask, reg: TIE);
1360	}
1361	spin_unlock_irqrestore(lock: &priv->lock, flags);
1362
1363	out:
1364	return budget - quota;
1365	}
1366
1367	static void ravb_set_duplex_gbeth(struct net_device *ndev)
1368	{
1369	struct ravb_private *priv = netdev_priv(dev: ndev);
1370
1371	ravb_modify(ndev, reg: ECMR, clear: ECMR_DM, set: priv->duplex > 0 ? ECMR_DM : 0);
1372	}
1373
1374	/* PHY state control function */
1375	static void ravb_adjust_link(struct net_device *ndev)
1376	{
1377	struct ravb_private *priv = netdev_priv(dev: ndev);
1378	const struct ravb_hw_info *info = priv->info;
1379	struct phy_device *phydev = ndev->phydev;
1380	bool new_state = false;
1381	unsigned long flags;
1382
1383	spin_lock_irqsave(&priv->lock, flags);
1384
1385	/* Disable TX and RX right over here, if E-MAC change is ignored */
1386	if (priv->no_avb_link)
1387	ravb_rcv_snd_disable(ndev);
1388
1389	if (phydev->link) {
1390	if (info->half_duplex && phydev->duplex != priv->duplex) {
1391	new_state = true;
1392	priv->duplex = phydev->duplex;
1393	ravb_set_duplex_gbeth(ndev);
1394	}
1395
1396	if (phydev->speed != priv->speed) {
1397	new_state = true;
1398	priv->speed = phydev->speed;
1399	info->set_rate(ndev);
1400	}
1401	if (!priv->link) {
1402	ravb_modify(ndev, reg: ECMR, clear: ECMR_TXF, set: 0);
1403	new_state = true;
1404	priv->link = phydev->link;
1405	}
1406	} else if (priv->link) {
1407	new_state = true;
1408	priv->link = 0;
1409	priv->speed = 0;
1410	if (info->half_duplex)
1411	priv->duplex = -1;
1412	}
1413
1414	/* Enable TX and RX right over here, if E-MAC change is ignored */
1415	if (priv->no_avb_link && phydev->link)
1416	ravb_rcv_snd_enable(ndev);
1417
1418	spin_unlock_irqrestore(lock: &priv->lock, flags);
1419
1420	if (new_state && netif_msg_link(priv))
1421	phy_print_status(phydev);
1422	}
1423
1424	/* PHY init function */
1425	static int ravb_phy_init(struct net_device *ndev)
1426	{
1427	struct device_node *np = ndev->dev.parent->of_node;
1428	struct ravb_private *priv = netdev_priv(dev: ndev);
1429	const struct ravb_hw_info *info = priv->info;
1430	struct phy_device *phydev;
1431	struct device_node *pn;
1432	phy_interface_t iface;
1433	int err;
1434
1435	priv->link = 0;
1436	priv->speed = 0;
1437	priv->duplex = -1;
1438
1439	/* Try connecting to PHY */
1440	pn = of_parse_phandle(np, phandle_name: "phy-handle", index: 0);
1441	if (!pn) {
1442	/* In the case of a fixed PHY, the DT node associated
1443	* to the PHY is the Ethernet MAC DT node.
1444	*/
1445	if (of_phy_is_fixed_link(np)) {
1446	err = of_phy_register_fixed_link(np);
1447	if (err)
1448	return err;
1449	}
1450	pn = of_node_get(node: np);
1451	}
1452
1453	iface = priv->rgmii_override ? PHY_INTERFACE_MODE_RGMII
1454	: priv->phy_interface;
1455	phydev = of_phy_connect(dev: ndev, phy_np: pn, hndlr: ravb_adjust_link, flags: 0, iface);
1456	of_node_put(node: pn);
1457	if (!phydev) {
1458	netdev_err(dev: ndev, format: "failed to connect PHY\n");
1459	err = -ENOENT;
1460	goto err_deregister_fixed_link;
1461	}
1462
1463	if (!info->half_duplex) {
1464	/* 10BASE, Pause and Asym Pause is not supported */
1465	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_10baseT_Half_BIT);
1466	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_10baseT_Full_BIT);
1467	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_Pause_BIT);
1468	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_Asym_Pause_BIT);
1469
1470	/* Half Duplex is not supported */
1471	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
1472	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_100baseT_Half_BIT);
1473	}
1474
1475	phy_attached_info(phydev);
1476
1477	return 0;
1478
1479	err_deregister_fixed_link:
1480	if (of_phy_is_fixed_link(np))
1481	of_phy_deregister_fixed_link(np);
1482
1483	return err;
1484	}
1485
1486	/* PHY control start function */
1487	static int ravb_phy_start(struct net_device *ndev)
1488	{
1489	int error;
1490
1491	error = ravb_phy_init(ndev);
1492	if (error)
1493	return error;
1494
1495	phy_start(phydev: ndev->phydev);
1496
1497	return 0;
1498	}
1499
1500	static u32 ravb_get_msglevel(struct net_device *ndev)
1501	{
1502	struct ravb_private *priv = netdev_priv(dev: ndev);
1503
1504	return priv->msg_enable;
1505	}
1506
1507	static void ravb_set_msglevel(struct net_device *ndev, u32 value)
1508	{
1509	struct ravb_private *priv = netdev_priv(dev: ndev);
1510
1511	priv->msg_enable = value;
1512	}
1513
1514	static const char ravb_gstrings_stats_gbeth[][ETH_GSTRING_LEN] = {
1515	"rx_queue_0_current",
1516	"tx_queue_0_current",
1517	"rx_queue_0_dirty",
1518	"tx_queue_0_dirty",
1519	"rx_queue_0_packets",
1520	"tx_queue_0_packets",
1521	"rx_queue_0_bytes",
1522	"tx_queue_0_bytes",
1523	"rx_queue_0_mcast_packets",
1524	"rx_queue_0_errors",
1525	"rx_queue_0_crc_errors",
1526	"rx_queue_0_frame_errors",
1527	"rx_queue_0_length_errors",
1528	"rx_queue_0_csum_offload_errors",
1529	"rx_queue_0_over_errors",
1530	};
1531
1532	static const char ravb_gstrings_stats[][ETH_GSTRING_LEN] = {
1533	"rx_queue_0_current",
1534	"tx_queue_0_current",
1535	"rx_queue_0_dirty",
1536	"tx_queue_0_dirty",
1537	"rx_queue_0_packets",
1538	"tx_queue_0_packets",
1539	"rx_queue_0_bytes",
1540	"tx_queue_0_bytes",
1541	"rx_queue_0_mcast_packets",
1542	"rx_queue_0_errors",
1543	"rx_queue_0_crc_errors",
1544	"rx_queue_0_frame_errors",
1545	"rx_queue_0_length_errors",
1546	"rx_queue_0_missed_errors",
1547	"rx_queue_0_over_errors",
1548
1549	"rx_queue_1_current",
1550	"tx_queue_1_current",
1551	"rx_queue_1_dirty",
1552	"tx_queue_1_dirty",
1553	"rx_queue_1_packets",
1554	"tx_queue_1_packets",
1555	"rx_queue_1_bytes",
1556	"tx_queue_1_bytes",
1557	"rx_queue_1_mcast_packets",
1558	"rx_queue_1_errors",
1559	"rx_queue_1_crc_errors",
1560	"rx_queue_1_frame_errors",
1561	"rx_queue_1_length_errors",
1562	"rx_queue_1_missed_errors",
1563	"rx_queue_1_over_errors",
1564	};
1565
1566	static int ravb_get_sset_count(struct net_device *netdev, int sset)
1567	{
1568	struct ravb_private *priv = netdev_priv(dev: netdev);
1569	const struct ravb_hw_info *info = priv->info;
1570
1571	switch (sset) {
1572	case ETH_SS_STATS:
1573	return info->stats_len;
1574	default:
1575	return -EOPNOTSUPP;
1576	}
1577	}
1578
1579	static void ravb_get_ethtool_stats(struct net_device *ndev,
1580	struct ethtool_stats *estats, u64 *data)
1581	{
1582	struct ravb_private *priv = netdev_priv(dev: ndev);
1583	const struct ravb_hw_info *info = priv->info;
1584	int num_rx_q;
1585	int i = 0;
1586	int q;
1587
1588	num_rx_q = info->nc_queues ? NUM_RX_QUEUE : 1;
1589	/* Device-specific stats */
1590	for (q = RAVB_BE; q < num_rx_q; q++) {
1591	struct net_device_stats *stats = &priv->stats[q];
1592
1593	data[i++] = priv->cur_rx[q];
1594	data[i++] = priv->cur_tx[q];
1595	data[i++] = priv->dirty_rx[q];
1596	data[i++] = priv->dirty_tx[q];
1597	data[i++] = stats->rx_packets;
1598	data[i++] = stats->tx_packets;
1599	data[i++] = stats->rx_bytes;
1600	data[i++] = stats->tx_bytes;
1601	data[i++] = stats->multicast;
1602	data[i++] = stats->rx_errors;
1603	data[i++] = stats->rx_crc_errors;
1604	data[i++] = stats->rx_frame_errors;
1605	data[i++] = stats->rx_length_errors;
1606	data[i++] = stats->rx_missed_errors;
1607	data[i++] = stats->rx_over_errors;
1608	}
1609	}
1610
1611	static void ravb_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
1612	{
1613	struct ravb_private *priv = netdev_priv(dev: ndev);
1614	const struct ravb_hw_info *info = priv->info;
1615
1616	switch (stringset) {
1617	case ETH_SS_STATS:
1618	memcpy(data, info->gstrings_stats, info->gstrings_size);
1619	break;
1620	}
1621	}
1622
1623	static void ravb_get_ringparam(struct net_device *ndev,
1624	struct ethtool_ringparam *ring,
1625	struct kernel_ethtool_ringparam *kernel_ring,
1626	struct netlink_ext_ack *extack)
1627	{
1628	struct ravb_private *priv = netdev_priv(dev: ndev);
1629
1630	ring->rx_max_pending = BE_RX_RING_MAX;
1631	ring->tx_max_pending = BE_TX_RING_MAX;
1632	ring->rx_pending = priv->num_rx_ring[RAVB_BE];
1633	ring->tx_pending = priv->num_tx_ring[RAVB_BE];
1634	}
1635
1636	static int ravb_set_ringparam(struct net_device *ndev,
1637	struct ethtool_ringparam *ring,
1638	struct kernel_ethtool_ringparam *kernel_ring,
1639	struct netlink_ext_ack *extack)
1640	{
1641	struct ravb_private *priv = netdev_priv(dev: ndev);
1642	const struct ravb_hw_info *info = priv->info;
1643	int error;
1644
1645	if (ring->tx_pending > BE_TX_RING_MAX ||
1646	ring->rx_pending > BE_RX_RING_MAX ||
1647	ring->tx_pending < BE_TX_RING_MIN ||
1648	ring->rx_pending < BE_RX_RING_MIN)
1649	return -EINVAL;
1650	if (ring->rx_mini_pending || ring->rx_jumbo_pending)
1651	return -EINVAL;
1652
1653	if (netif_running(dev: ndev)) {
1654	netif_device_detach(dev: ndev);
1655	/* Stop PTP Clock driver */
1656	if (info->gptp)
1657	ravb_ptp_stop(ndev);
1658	/* Wait for DMA stopping */
1659	error = ravb_stop_dma(ndev);
1660	if (error) {
1661	netdev_err(dev: ndev,
1662	format: "cannot set ringparam! Any AVB processes are still running?\n");
1663	return error;
1664	}
1665	synchronize_irq(irq: ndev->irq);
1666
1667	/* Free all the skb's in the RX queue and the DMA buffers. */
1668	ravb_ring_free(ndev, q: RAVB_BE);
1669	if (info->nc_queues)
1670	ravb_ring_free(ndev, q: RAVB_NC);
1671	}
1672
1673	/* Set new parameters */
1674	priv->num_rx_ring[RAVB_BE] = ring->rx_pending;
1675	priv->num_tx_ring[RAVB_BE] = ring->tx_pending;
1676
1677	if (netif_running(dev: ndev)) {
1678	error = ravb_dmac_init(ndev);
1679	if (error) {
1680	netdev_err(dev: ndev,
1681	format: "%s: ravb_dmac_init() failed, error %d\n",
1682	__func__, error);
1683	return error;
1684	}
1685
1686	ravb_emac_init(ndev);
1687
1688	/* Initialise PTP Clock driver */
1689	if (info->gptp)
1690	ravb_ptp_init(ndev, pdev: priv->pdev);
1691
1692	netif_device_attach(dev: ndev);
1693	}
1694
1695	return 0;
1696	}
1697
1698	static int ravb_get_ts_info(struct net_device *ndev,
1699	struct ethtool_ts_info *info)
1700	{
1701	struct ravb_private *priv = netdev_priv(dev: ndev);
1702	const struct ravb_hw_info *hw_info = priv->info;
1703
1704	info->so_timestamping =
1705	SOF_TIMESTAMPING_TX_SOFTWARE |
1706	SOF_TIMESTAMPING_RX_SOFTWARE |
1707	SOF_TIMESTAMPING_SOFTWARE |
1708	SOF_TIMESTAMPING_TX_HARDWARE |
1709	SOF_TIMESTAMPING_RX_HARDWARE |
1710	SOF_TIMESTAMPING_RAW_HARDWARE;
1711	info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
1712	info->rx_filters =
1713	(1 << HWTSTAMP_FILTER_NONE) |
1714	(1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
1715	(1 << HWTSTAMP_FILTER_ALL);
1716	if (hw_info->gptp || hw_info->ccc_gac)
1717	info->phc_index = ptp_clock_index(ptp: priv->ptp.clock);
1718
1719	return 0;
1720	}
1721
1722	static void ravb_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
1723	{
1724	struct ravb_private *priv = netdev_priv(dev: ndev);
1725
1726	wol->supported = WAKE_MAGIC;
1727	wol->wolopts = priv->wol_enabled ? WAKE_MAGIC : 0;
1728	}
1729
1730	static int ravb_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
1731	{
1732	struct ravb_private *priv = netdev_priv(dev: ndev);
1733	const struct ravb_hw_info *info = priv->info;
1734
1735	if (!info->magic_pkt || (wol->wolopts & ~WAKE_MAGIC))
1736	return -EOPNOTSUPP;
1737
1738	priv->wol_enabled = !!(wol->wolopts & WAKE_MAGIC);
1739
1740	device_set_wakeup_enable(dev: &priv->pdev->dev, enable: priv->wol_enabled);
1741
1742	return 0;
1743	}
1744
1745	static const struct ethtool_ops ravb_ethtool_ops = {
1746	.nway_reset = phy_ethtool_nway_reset,
1747	.get_msglevel = ravb_get_msglevel,
1748	.set_msglevel = ravb_set_msglevel,
1749	.get_link = ethtool_op_get_link,
1750	.get_strings = ravb_get_strings,
1751	.get_ethtool_stats = ravb_get_ethtool_stats,
1752	.get_sset_count = ravb_get_sset_count,
1753	.get_ringparam = ravb_get_ringparam,
1754	.set_ringparam = ravb_set_ringparam,
1755	.get_ts_info = ravb_get_ts_info,
1756	.get_link_ksettings = phy_ethtool_get_link_ksettings,
1757	.set_link_ksettings = phy_ethtool_set_link_ksettings,
1758	.get_wol = ravb_get_wol,
1759	.set_wol = ravb_set_wol,
1760	};
1761
1762	static int ravb_set_config_mode(struct net_device *ndev)
1763	{
1764	struct ravb_private *priv = netdev_priv(dev: ndev);
1765	const struct ravb_hw_info *info = priv->info;
1766	int error;
1767
1768	if (info->gptp) {
1769	error = ravb_set_opmode(ndev, opmode: CCC_OPC_CONFIG);
1770	if (error)
1771	return error;
1772	/* Set CSEL value */
1773	ravb_modify(ndev, reg: CCC, clear: CCC_CSEL, set: CCC_CSEL_HPB);
1774	} else if (info->ccc_gac) {
1775	error = ravb_set_opmode(ndev, opmode: CCC_OPC_CONFIG | CCC_GAC | CCC_CSEL_HPB);
1776	} else {
1777	error = ravb_set_opmode(ndev, opmode: CCC_OPC_CONFIG);
1778	}
1779
1780	return error;
1781	}
1782
1783	static void ravb_set_gti(struct net_device *ndev)
1784	{
1785	struct ravb_private *priv = netdev_priv(dev: ndev);
1786	const struct ravb_hw_info *info = priv->info;
1787
1788	if (!(info->gptp || info->ccc_gac))
1789	return;
1790
1791	ravb_write(ndev, data: priv->gti_tiv, reg: GTI);
1792
1793	/* Request GTI loading */
1794	ravb_modify(ndev, reg: GCCR, clear: GCCR_LTI, set: GCCR_LTI);
1795	}
1796
1797	static int ravb_compute_gti(struct net_device *ndev)
1798	{
1799	struct ravb_private *priv = netdev_priv(dev: ndev);
1800	const struct ravb_hw_info *info = priv->info;
1801	struct device *dev = ndev->dev.parent;
1802	unsigned long rate;
1803	u64 inc;
1804
1805	if (!(info->gptp || info->ccc_gac))
1806	return 0;
1807
1808	if (info->gptp_ref_clk)
1809	rate = clk_get_rate(clk: priv->gptp_clk);
1810	else
1811	rate = clk_get_rate(clk: priv->clk);
1812	if (!rate)
1813	return -EINVAL;
1814
1815	inc = div64_ul(1000000000ULL << 20, rate);
1816
1817	if (inc < GTI_TIV_MIN || inc > GTI_TIV_MAX) {
1818	dev_err(dev, "gti.tiv increment 0x%llx is outside the range 0x%x - 0x%x\n",
1819	inc, GTI_TIV_MIN, GTI_TIV_MAX);
1820	return -EINVAL;
1821	}
1822	priv->gti_tiv = inc;
1823
1824	return 0;
1825	}
1826
1827	/* Set tx and rx clock internal delay modes */
1828	static void ravb_parse_delay_mode(struct device_node *np, struct net_device *ndev)
1829	{
1830	struct ravb_private *priv = netdev_priv(dev: ndev);
1831	bool explicit_delay = false;
1832	u32 delay;
1833
1834	if (!priv->info->internal_delay)
1835	return;
1836
1837	if (!of_property_read_u32(np, propname: "rx-internal-delay-ps", out_value: &delay)) {
1838	/* Valid values are 0 and 1800, according to DT bindings */
1839	priv->rxcidm = !!delay;
1840	explicit_delay = true;
1841	}
1842	if (!of_property_read_u32(np, propname: "tx-internal-delay-ps", out_value: &delay)) {
1843	/* Valid values are 0 and 2000, according to DT bindings */
1844	priv->txcidm = !!delay;
1845	explicit_delay = true;
1846	}
1847
1848	if (explicit_delay)
1849	return;
1850
1851	/* Fall back to legacy rgmii-*id behavior */
1852	if (priv->phy_interface == PHY_INTERFACE_MODE_RGMII_ID ||
1853	priv->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID) {
1854	priv->rxcidm = 1;
1855	priv->rgmii_override = 1;
1856	}
1857
1858	if (priv->phy_interface == PHY_INTERFACE_MODE_RGMII_ID ||
1859	priv->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID) {
1860	priv->txcidm = 1;
1861	priv->rgmii_override = 1;
1862	}
1863	}
1864
1865	static void ravb_set_delay_mode(struct net_device *ndev)
1866	{
1867	struct ravb_private *priv = netdev_priv(dev: ndev);
1868	u32 set = 0;
1869
1870	if (!priv->info->internal_delay)
1871	return;
1872
1873	if (priv->rxcidm)
1874	set |= APSR_RDM;
1875	if (priv->txcidm)
1876	set |= APSR_TDM;
1877	ravb_modify(ndev, reg: APSR, clear: APSR_RDM | APSR_TDM, set);
1878	}
1879
1880	/* Network device open function for Ethernet AVB */
1881	static int ravb_open(struct net_device *ndev)
1882	{
1883	struct ravb_private *priv = netdev_priv(dev: ndev);
1884	const struct ravb_hw_info *info = priv->info;
1885	struct device *dev = &priv->pdev->dev;
1886	int error;
1887
1888	napi_enable(n: &priv->napi[RAVB_BE]);
1889	if (info->nc_queues)
1890	napi_enable(n: &priv->napi[RAVB_NC]);
1891
1892	error = pm_runtime_resume_and_get(dev);
1893	if (error < 0)
1894	goto out_napi_off;
1895
1896	/* Set AVB config mode */
1897	error = ravb_set_config_mode(ndev);
1898	if (error)
1899	goto out_rpm_put;
1900
1901	ravb_set_delay_mode(ndev);
1902	ravb_write(ndev, data: priv->desc_bat_dma, reg: DBAT);
1903
1904	/* Device init */
1905	error = ravb_dmac_init(ndev);
1906	if (error)
1907	goto out_set_reset;
1908
1909	ravb_emac_init(ndev);
1910
1911	ravb_set_gti(ndev);
1912
1913	/* Initialise PTP Clock driver */
1914	if (info->gptp || info->ccc_gac)
1915	ravb_ptp_init(ndev, pdev: priv->pdev);
1916
1917	/* PHY control start */
1918	error = ravb_phy_start(ndev);
1919	if (error)
1920	goto out_ptp_stop;
1921
1922	netif_tx_start_all_queues(dev: ndev);
1923
1924	return 0;
1925
1926	out_ptp_stop:
1927	/* Stop PTP Clock driver */
1928	if (info->gptp || info->ccc_gac)
1929	ravb_ptp_stop(ndev);
1930	ravb_stop_dma(ndev);
1931	out_set_reset:
1932	ravb_set_opmode(ndev, opmode: CCC_OPC_RESET);
1933	out_rpm_put:
1934	pm_runtime_mark_last_busy(dev);
1935	pm_runtime_put_autosuspend(dev);
1936	out_napi_off:
1937	if (info->nc_queues)
1938	napi_disable(n: &priv->napi[RAVB_NC]);
1939	napi_disable(n: &priv->napi[RAVB_BE]);
1940	return error;
1941	}
1942
1943	/* Timeout function for Ethernet AVB */
1944	static void ravb_tx_timeout(struct net_device *ndev, unsigned int txqueue)
1945	{
1946	struct ravb_private *priv = netdev_priv(dev: ndev);
1947
1948	netif_err(priv, tx_err, ndev,
1949	"transmit timed out, status %08x, resetting...\n",
1950	ravb_read(ndev, ISS));
1951
1952	/* tx_errors count up */
1953	ndev->stats.tx_errors++;
1954
1955	schedule_work(work: &priv->work);
1956	}
1957
1958	static void ravb_tx_timeout_work(struct work_struct *work)
1959	{
1960	struct ravb_private *priv = container_of(work, struct ravb_private,
1961	work);
1962	const struct ravb_hw_info *info = priv->info;
1963	struct net_device *ndev = priv->ndev;
1964	int error;
1965
1966	if (!rtnl_trylock()) {
1967	usleep_range(min: 1000, max: 2000);
1968	schedule_work(work: &priv->work);
1969	return;
1970	}
1971
1972	netif_tx_stop_all_queues(dev: ndev);
1973
1974	/* Stop PTP Clock driver */
1975	if (info->gptp)
1976	ravb_ptp_stop(ndev);
1977
1978	/* Wait for DMA stopping */
1979	if (ravb_stop_dma(ndev)) {
1980	/* If ravb_stop_dma() fails, the hardware is still operating
1981	* for TX and/or RX. So, this should not call the following
1982	* functions because ravb_dmac_init() is possible to fail too.
1983	* Also, this should not retry ravb_stop_dma() again and again
1984	* here because it's possible to wait forever. So, this just
1985	* re-enables the TX and RX and skip the following
1986	* re-initialization procedure.
1987	*/
1988	ravb_rcv_snd_enable(ndev);
1989	goto out;
1990	}
1991
1992	ravb_ring_free(ndev, q: RAVB_BE);
1993	if (info->nc_queues)
1994	ravb_ring_free(ndev, q: RAVB_NC);
1995
1996	/* Device init */
1997	error = ravb_dmac_init(ndev);
1998	if (error) {
1999	/* If ravb_dmac_init() fails, descriptors are freed. So, this
2000	* should return here to avoid re-enabling the TX and RX in
2001	* ravb_emac_init().
2002	*/
2003	netdev_err(dev: ndev, format: "%s: ravb_dmac_init() failed, error %d\n",
2004	__func__, error);
2005	goto out_unlock;
2006	}
2007	ravb_emac_init(ndev);
2008
2009	out:
2010	/* Initialise PTP Clock driver */
2011	if (info->gptp)
2012	ravb_ptp_init(ndev, pdev: priv->pdev);
2013
2014	netif_tx_start_all_queues(dev: ndev);
2015
2016	out_unlock:
2017	rtnl_unlock();
2018	}
2019
2020	static bool ravb_can_tx_csum_gbeth(struct sk_buff *skb)
2021	{
2022	struct iphdr *ip = ip_hdr(skb);
2023
2024	/* TODO: Need to add support for VLAN tag 802.1Q */
2025	if (skb_vlan_tag_present(skb))
2026	return false;
2027
2028	/* TODO: Need to add hardware checksum for IPv6 */
2029	if (skb->protocol != htons(ETH_P_IP))
2030	return false;
2031
2032	switch (ip->protocol) {
2033	case IPPROTO_TCP:
2034	break;
2035	case IPPROTO_UDP:
2036	/* If the checksum value in the UDP header field is 0, TOE does
2037	* not calculate checksum for UDP part of this frame as it is
2038	* optional function as per standards.
2039	*/
2040	if (udp_hdr(skb)->check == 0)
2041	return false;
2042	break;
2043	default:
2044	return false;
2045	}
2046
2047	return true;
2048	}
2049
2050	/* Packet transmit function for Ethernet AVB */
2051	static netdev_tx_t ravb_start_xmit(struct sk_buff *skb, struct net_device *ndev)
2052	{
2053	struct ravb_private *priv = netdev_priv(dev: ndev);
2054	const struct ravb_hw_info *info = priv->info;
2055	unsigned int num_tx_desc = priv->num_tx_desc;
2056	u16 q = skb_get_queue_mapping(skb);
2057	struct ravb_tstamp_skb *ts_skb;
2058	struct ravb_tx_desc *desc;
2059	unsigned long flags;
2060	dma_addr_t dma_addr;
2061	void *buffer;
2062	u32 entry;
2063	u32 len;
2064
2065	if (skb->ip_summed == CHECKSUM_PARTIAL && !ravb_can_tx_csum_gbeth(skb))
2066	skb_checksum_help(skb);
2067
2068	spin_lock_irqsave(&priv->lock, flags);
2069	if (priv->cur_tx[q] - priv->dirty_tx[q] > (priv->num_tx_ring[q] - 1) *
2070	num_tx_desc) {
2071	netif_err(priv, tx_queued, ndev,
2072	"still transmitting with the full ring!\n");
2073	netif_stop_subqueue(dev: ndev, queue_index: q);
2074	spin_unlock_irqrestore(lock: &priv->lock, flags);
2075	return NETDEV_TX_BUSY;
2076	}
2077
2078	if (skb_put_padto(skb, ETH_ZLEN))
2079	goto exit;
2080
2081	entry = priv->cur_tx[q] % (priv->num_tx_ring[q] * num_tx_desc);
2082	priv->tx_skb[q][entry / num_tx_desc] = skb;
2083
2084	if (num_tx_desc > 1) {
2085	buffer = PTR_ALIGN(priv->tx_align[q], DPTR_ALIGN) +
2086	entry / num_tx_desc * DPTR_ALIGN;
2087	len = PTR_ALIGN(skb->data, DPTR_ALIGN) - skb->data;
2088
2089	/* Zero length DMA descriptors are problematic as they seem
2090	* to terminate DMA transfers. Avoid them by simply using a
2091	* length of DPTR_ALIGN (4) when skb data is aligned to
2092	* DPTR_ALIGN.
2093	*
2094	* As skb is guaranteed to have at least ETH_ZLEN (60)
2095	* bytes of data by the call to skb_put_padto() above this
2096	* is safe with respect to both the length of the first DMA
2097	* descriptor (len) overflowing the available data and the
2098	* length of the second DMA descriptor (skb->len - len)
2099	* being negative.
2100	*/
2101	if (len == 0)
2102	len = DPTR_ALIGN;
2103
2104	memcpy(buffer, skb->data, len);
2105	dma_addr = dma_map_single(ndev->dev.parent, buffer, len,
2106	DMA_TO_DEVICE);
2107	if (dma_mapping_error(dev: ndev->dev.parent, dma_addr))
2108	goto drop;
2109
2110	desc = &priv->tx_ring[q][entry];
2111	desc->ds_tagl = cpu_to_le16(len);
2112	desc->dptr = cpu_to_le32(dma_addr);
2113
2114	buffer = skb->data + len;
2115	len = skb->len - len;
2116	dma_addr = dma_map_single(ndev->dev.parent, buffer, len,
2117	DMA_TO_DEVICE);
2118	if (dma_mapping_error(dev: ndev->dev.parent, dma_addr))
2119	goto unmap;
2120
2121	desc++;
2122	} else {
2123	desc = &priv->tx_ring[q][entry];
2124	len = skb->len;
2125	dma_addr = dma_map_single(ndev->dev.parent, skb->data, skb->len,
2126	DMA_TO_DEVICE);
2127	if (dma_mapping_error(dev: ndev->dev.parent, dma_addr))
2128	goto drop;
2129	}
2130	desc->ds_tagl = cpu_to_le16(len);
2131	desc->dptr = cpu_to_le32(dma_addr);
2132
2133	/* TX timestamp required */
2134	if (info->gptp || info->ccc_gac) {
2135	if (q == RAVB_NC) {
2136	ts_skb = kmalloc(size: sizeof(*ts_skb), GFP_ATOMIC);
2137	if (!ts_skb) {
2138	if (num_tx_desc > 1) {
2139	desc--;
2140	dma_unmap_single(ndev->dev.parent, dma_addr,
2141	len, DMA_TO_DEVICE);
2142	}
2143	goto unmap;
2144	}
2145	ts_skb->skb = skb_get(skb);
2146	ts_skb->tag = priv->ts_skb_tag++;
2147	priv->ts_skb_tag &= 0x3ff;
2148	list_add_tail(new: &ts_skb->list, head: &priv->ts_skb_list);
2149
2150	/* TAG and timestamp required flag */
2151	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2152	desc->tagh_tsr = (ts_skb->tag >> 4) | TX_TSR;
2153	desc->ds_tagl |= cpu_to_le16(ts_skb->tag << 12);
2154	}
2155
2156	skb_tx_timestamp(skb);
2157	}
2158	/* Descriptor type must be set after all the above writes */
2159	dma_wmb();
2160	if (num_tx_desc > 1) {
2161	desc->die_dt = DT_FEND;
2162	desc--;
2163	desc->die_dt = DT_FSTART;
2164	} else {
2165	desc->die_dt = DT_FSINGLE;
2166	}
2167	ravb_modify(ndev, reg: TCCR, clear: TCCR_TSRQ0 << q, set: TCCR_TSRQ0 << q);
2168
2169	priv->cur_tx[q] += num_tx_desc;
2170	if (priv->cur_tx[q] - priv->dirty_tx[q] >
2171	(priv->num_tx_ring[q] - 1) * num_tx_desc &&
2172	!ravb_tx_free(ndev, q, free_txed_only: true))
2173	netif_stop_subqueue(dev: ndev, queue_index: q);
2174
2175	exit:
2176	spin_unlock_irqrestore(lock: &priv->lock, flags);
2177	return NETDEV_TX_OK;
2178
2179	unmap:
2180	dma_unmap_single(ndev->dev.parent, le32_to_cpu(desc->dptr),
2181	le16_to_cpu(desc->ds_tagl), DMA_TO_DEVICE);
2182	drop:
2183	dev_kfree_skb_any(skb);
2184	priv->tx_skb[q][entry / num_tx_desc] = NULL;
2185	goto exit;
2186	}
2187
2188	static u16 ravb_select_queue(struct net_device *ndev, struct sk_buff *skb,
2189	struct net_device *sb_dev)
2190	{
2191	/* If skb needs TX timestamp, it is handled in network control queue */
2192	return (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) ? RAVB_NC :
2193	RAVB_BE;
2194
2195	}
2196
2197	static struct net_device_stats *ravb_get_stats(struct net_device *ndev)
2198	{
2199	struct ravb_private *priv = netdev_priv(dev: ndev);
2200	const struct ravb_hw_info *info = priv->info;
2201	struct net_device_stats *nstats, *stats0, *stats1;
2202	struct device *dev = &priv->pdev->dev;
2203
2204	nstats = &ndev->stats;
2205
2206	pm_runtime_get_noresume(dev);
2207
2208	if (!pm_runtime_active(dev))
2209	goto out_rpm_put;
2210
2211	stats0 = &priv->stats[RAVB_BE];
2212
2213	if (info->tx_counters) {
2214	nstats->tx_dropped += ravb_read(ndev, reg: TROCR);
2215	ravb_write(ndev, data: 0, reg: TROCR); /* (write clear) */
2216	}
2217
2218	if (info->carrier_counters) {
2219	nstats->collisions += ravb_read(ndev, reg: CXR41);
2220	ravb_write(ndev, data: 0, reg: CXR41); /* (write clear) */
2221	nstats->tx_carrier_errors += ravb_read(ndev, reg: CXR42);
2222	ravb_write(ndev, data: 0, reg: CXR42); /* (write clear) */
2223	}
2224
2225	nstats->rx_packets = stats0->rx_packets;
2226	nstats->tx_packets = stats0->tx_packets;
2227	nstats->rx_bytes = stats0->rx_bytes;
2228	nstats->tx_bytes = stats0->tx_bytes;
2229	nstats->multicast = stats0->multicast;
2230	nstats->rx_errors = stats0->rx_errors;
2231	nstats->rx_crc_errors = stats0->rx_crc_errors;
2232	nstats->rx_frame_errors = stats0->rx_frame_errors;
2233	nstats->rx_length_errors = stats0->rx_length_errors;
2234	nstats->rx_missed_errors = stats0->rx_missed_errors;
2235	nstats->rx_over_errors = stats0->rx_over_errors;
2236	if (info->nc_queues) {
2237	stats1 = &priv->stats[RAVB_NC];
2238
2239	nstats->rx_packets += stats1->rx_packets;
2240	nstats->tx_packets += stats1->tx_packets;
2241	nstats->rx_bytes += stats1->rx_bytes;
2242	nstats->tx_bytes += stats1->tx_bytes;
2243	nstats->multicast += stats1->multicast;
2244	nstats->rx_errors += stats1->rx_errors;
2245	nstats->rx_crc_errors += stats1->rx_crc_errors;
2246	nstats->rx_frame_errors += stats1->rx_frame_errors;
2247	nstats->rx_length_errors += stats1->rx_length_errors;
2248	nstats->rx_missed_errors += stats1->rx_missed_errors;
2249	nstats->rx_over_errors += stats1->rx_over_errors;
2250	}
2251
2252	out_rpm_put:
2253	pm_runtime_put_noidle(dev);
2254	return nstats;
2255	}
2256
2257	/* Update promiscuous bit */
2258	static void ravb_set_rx_mode(struct net_device *ndev)
2259	{
2260	struct ravb_private *priv = netdev_priv(dev: ndev);
2261	unsigned long flags;
2262
2263	spin_lock_irqsave(&priv->lock, flags);
2264	ravb_modify(ndev, reg: ECMR, clear: ECMR_PRM,
2265	set: ndev->flags & IFF_PROMISC ? ECMR_PRM : 0);
2266	spin_unlock_irqrestore(lock: &priv->lock, flags);
2267	}
2268
2269	/* Device close function for Ethernet AVB */
2270	static int ravb_close(struct net_device *ndev)
2271	{
2272	struct device_node *np = ndev->dev.parent->of_node;
2273	struct ravb_private *priv = netdev_priv(dev: ndev);
2274	const struct ravb_hw_info *info = priv->info;
2275	struct ravb_tstamp_skb *ts_skb, *ts_skb2;
2276	struct device *dev = &priv->pdev->dev;
2277	int error;
2278
2279	netif_tx_stop_all_queues(dev: ndev);
2280
2281	/* Disable interrupts by clearing the interrupt masks. */
2282	ravb_write(ndev, data: 0, reg: RIC0);
2283	ravb_write(ndev, data: 0, reg: RIC2);
2284	ravb_write(ndev, data: 0, reg: TIC);
2285
2286	/* PHY disconnect */
2287	if (ndev->phydev) {
2288	phy_stop(phydev: ndev->phydev);
2289	phy_disconnect(phydev: ndev->phydev);
2290	if (of_phy_is_fixed_link(np))
2291	of_phy_deregister_fixed_link(np);
2292	}
2293
2294	/* Stop PTP Clock driver */
2295	if (info->gptp || info->ccc_gac)
2296	ravb_ptp_stop(ndev);
2297
2298	/* Set the config mode to stop the AVB-DMAC's processes */
2299	if (ravb_stop_dma(ndev) < 0)
2300	netdev_err(dev: ndev,
2301	format: "device will be stopped after h/w processes are done.\n");
2302
2303	/* Clear the timestamp list */
2304	if (info->gptp || info->ccc_gac) {
2305	list_for_each_entry_safe(ts_skb, ts_skb2, &priv->ts_skb_list, list) {
2306	list_del(entry: &ts_skb->list);
2307	kfree_skb(skb: ts_skb->skb);
2308	kfree(objp: ts_skb);
2309	}
2310	}
2311
2312	cancel_work_sync(work: &priv->work);
2313
2314	if (info->nc_queues)
2315	napi_disable(n: &priv->napi[RAVB_NC]);
2316	napi_disable(n: &priv->napi[RAVB_BE]);
2317
2318	/* Free all the skb's in the RX queue and the DMA buffers. */
2319	ravb_ring_free(ndev, q: RAVB_BE);
2320	if (info->nc_queues)
2321	ravb_ring_free(ndev, q: RAVB_NC);
2322
2323	/* Update statistics. */
2324	ravb_get_stats(ndev);
2325
2326	/* Set reset mode. */
2327	error = ravb_set_opmode(ndev, opmode: CCC_OPC_RESET);
2328	if (error)
2329	return error;
2330
2331	pm_runtime_mark_last_busy(dev);
2332	pm_runtime_put_autosuspend(dev);
2333
2334	return 0;
2335	}
2336
2337	static int ravb_hwtstamp_get(struct net_device *ndev, struct ifreq *req)
2338	{
2339	struct ravb_private *priv = netdev_priv(dev: ndev);
2340	struct hwtstamp_config config;
2341
2342	config.flags = 0;
2343	config.tx_type = priv->tstamp_tx_ctrl ? HWTSTAMP_TX_ON :
2344	HWTSTAMP_TX_OFF;
2345	switch (priv->tstamp_rx_ctrl & RAVB_RXTSTAMP_TYPE) {
2346	case RAVB_RXTSTAMP_TYPE_V2_L2_EVENT:
2347	config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
2348	break;
2349	case RAVB_RXTSTAMP_TYPE_ALL:
2350	config.rx_filter = HWTSTAMP_FILTER_ALL;
2351	break;
2352	default:
2353	config.rx_filter = HWTSTAMP_FILTER_NONE;
2354	}
2355
2356	return copy_to_user(to: req->ifr_data, from: &config, n: sizeof(config)) ?
2357	-EFAULT : 0;
2358	}
2359
2360	/* Control hardware time stamping */
2361	static int ravb_hwtstamp_set(struct net_device *ndev, struct ifreq *req)
2362	{
2363	struct ravb_private *priv = netdev_priv(dev: ndev);
2364	struct hwtstamp_config config;
2365	u32 tstamp_rx_ctrl = RAVB_RXTSTAMP_ENABLED;
2366	u32 tstamp_tx_ctrl;
2367
2368	if (copy_from_user(to: &config, from: req->ifr_data, n: sizeof(config)))
2369	return -EFAULT;
2370
2371	switch (config.tx_type) {
2372	case HWTSTAMP_TX_OFF:
2373	tstamp_tx_ctrl = 0;
2374	break;
2375	case HWTSTAMP_TX_ON:
2376	tstamp_tx_ctrl = RAVB_TXTSTAMP_ENABLED;
2377	break;
2378	default:
2379	return -ERANGE;
2380	}
2381
2382	switch (config.rx_filter) {
2383	case HWTSTAMP_FILTER_NONE:
2384	tstamp_rx_ctrl = 0;
2385	break;
2386	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2387	tstamp_rx_ctrl |= RAVB_RXTSTAMP_TYPE_V2_L2_EVENT;
2388	break;
2389	default:
2390	config.rx_filter = HWTSTAMP_FILTER_ALL;
2391	tstamp_rx_ctrl |= RAVB_RXTSTAMP_TYPE_ALL;
2392	}
2393
2394	priv->tstamp_tx_ctrl = tstamp_tx_ctrl;
2395	priv->tstamp_rx_ctrl = tstamp_rx_ctrl;
2396
2397	return copy_to_user(to: req->ifr_data, from: &config, n: sizeof(config)) ?
2398	-EFAULT : 0;
2399	}
2400
2401	/* ioctl to device function */
2402	static int ravb_do_ioctl(struct net_device *ndev, struct ifreq *req, int cmd)
2403	{
2404	struct phy_device *phydev = ndev->phydev;
2405
2406	if (!netif_running(dev: ndev))
2407	return -EINVAL;
2408
2409	if (!phydev)
2410	return -ENODEV;
2411
2412	switch (cmd) {
2413	case SIOCGHWTSTAMP:
2414	return ravb_hwtstamp_get(ndev, req);
2415	case SIOCSHWTSTAMP:
2416	return ravb_hwtstamp_set(ndev, req);
2417	}
2418
2419	return phy_mii_ioctl(phydev, ifr: req, cmd);
2420	}
2421
2422	static int ravb_change_mtu(struct net_device *ndev, int new_mtu)
2423	{
2424	struct ravb_private *priv = netdev_priv(dev: ndev);
2425
2426	ndev->mtu = new_mtu;
2427
2428	if (netif_running(dev: ndev)) {
2429	synchronize_irq(irq: priv->emac_irq);
2430	ravb_emac_init(ndev);
2431	}
2432
2433	netdev_update_features(dev: ndev);
2434
2435	return 0;
2436	}
2437
2438	static void ravb_set_rx_csum(struct net_device *ndev, bool enable)
2439	{
2440	struct ravb_private *priv = netdev_priv(dev: ndev);
2441	unsigned long flags;
2442
2443	spin_lock_irqsave(&priv->lock, flags);
2444
2445	/* Disable TX and RX */
2446	ravb_rcv_snd_disable(ndev);
2447
2448	/* Modify RX Checksum setting */
2449	ravb_modify(ndev, reg: ECMR, clear: ECMR_RCSC, set: enable ? ECMR_RCSC : 0);
2450
2451	/* Enable TX and RX */
2452	ravb_rcv_snd_enable(ndev);
2453
2454	spin_unlock_irqrestore(lock: &priv->lock, flags);
2455	}
2456
2457	static int ravb_endisable_csum_gbeth(struct net_device *ndev, enum ravb_reg reg,
2458	u32 val, u32 mask)
2459	{
2460	u32 csr0 = CSR0_TPE | CSR0_RPE;
2461	int ret;
2462
2463	ravb_write(ndev, data: csr0 & ~mask, reg: CSR0);
2464	ret = ravb_wait(ndev, reg: CSR0, mask, value: 0);
2465	if (!ret)
2466	ravb_write(ndev, data: val, reg);
2467
2468	ravb_write(ndev, data: csr0, reg: CSR0);
2469
2470	return ret;
2471	}
2472
2473	static int ravb_set_features_gbeth(struct net_device *ndev,
2474	netdev_features_t features)
2475	{
2476	netdev_features_t changed = ndev->features ^ features;
2477	struct ravb_private *priv = netdev_priv(dev: ndev);
2478	unsigned long flags;
2479	int ret = 0;
2480	u32 val;
2481
2482	spin_lock_irqsave(&priv->lock, flags);
2483	if (changed & NETIF_F_RXCSUM) {
2484	if (features & NETIF_F_RXCSUM)
2485	val = CSR2_RIP4 | CSR2_RTCP4 | CSR2_RUDP4 | CSR2_RICMP4;
2486	else
2487	val = 0;
2488
2489	ret = ravb_endisable_csum_gbeth(ndev, reg: CSR2, val, mask: CSR0_RPE);
2490	if (ret)
2491	goto done;
2492	}
2493
2494	if (changed & NETIF_F_HW_CSUM) {
2495	if (features & NETIF_F_HW_CSUM)
2496	val = CSR1_TIP4 | CSR1_TTCP4 | CSR1_TUDP4;
2497	else
2498	val = 0;
2499
2500	ret = ravb_endisable_csum_gbeth(ndev, reg: CSR1, val, mask: CSR0_TPE);
2501	if (ret)
2502	goto done;
2503	}
2504
2505	done:
2506	spin_unlock_irqrestore(lock: &priv->lock, flags);
2507
2508	return ret;
2509	}
2510
2511	static int ravb_set_features_rcar(struct net_device *ndev,
2512	netdev_features_t features)
2513	{
2514	netdev_features_t changed = ndev->features ^ features;
2515
2516	if (changed & NETIF_F_RXCSUM)
2517	ravb_set_rx_csum(ndev, enable: features & NETIF_F_RXCSUM);
2518
2519	return 0;
2520	}
2521
2522	static int ravb_set_features(struct net_device *ndev,
2523	netdev_features_t features)
2524	{
2525	struct ravb_private *priv = netdev_priv(dev: ndev);
2526	const struct ravb_hw_info *info = priv->info;
2527	struct device *dev = &priv->pdev->dev;
2528	int ret;
2529
2530	pm_runtime_get_noresume(dev);
2531
2532	if (pm_runtime_active(dev))
2533	ret = info->set_feature(ndev, features);
2534	else
2535	ret = 0;
2536
2537	pm_runtime_put_noidle(dev);
2538
2539	if (ret)
2540	return ret;
2541
2542	ndev->features = features;
2543
2544	return 0;
2545	}
2546
2547	static const struct net_device_ops ravb_netdev_ops = {
2548	.ndo_open = ravb_open,
2549	.ndo_stop = ravb_close,
2550	.ndo_start_xmit = ravb_start_xmit,
2551	.ndo_select_queue = ravb_select_queue,
2552	.ndo_get_stats = ravb_get_stats,
2553	.ndo_set_rx_mode = ravb_set_rx_mode,
2554	.ndo_tx_timeout = ravb_tx_timeout,
2555	.ndo_eth_ioctl = ravb_do_ioctl,
2556	.ndo_change_mtu = ravb_change_mtu,
2557	.ndo_validate_addr = eth_validate_addr,
2558	.ndo_set_mac_address = eth_mac_addr,
2559	.ndo_set_features = ravb_set_features,
2560	};
2561
2562	/* MDIO bus init function */
2563	static int ravb_mdio_init(struct ravb_private *priv)
2564	{
2565	struct platform_device *pdev = priv->pdev;
2566	struct device *dev = &pdev->dev;
2567	struct phy_device *phydev;
2568	struct device_node *pn;
2569	int error;
2570
2571	/* Bitbang init */
2572	priv->mdiobb.ops = &bb_ops;
2573
2574	/* MII controller setting */
2575	priv->mii_bus = alloc_mdio_bitbang(ctrl: &priv->mdiobb);
2576	if (!priv->mii_bus)
2577	return -ENOMEM;
2578
2579	/* Hook up MII support for ethtool */
2580	priv->mii_bus->name = "ravb_mii";
2581	priv->mii_bus->parent = dev;
2582	snprintf(buf: priv->mii_bus->id, MII_BUS_ID_SIZE, fmt: "%s-%x",
2583	pdev->name, pdev->id);
2584
2585	/* Register MDIO bus */
2586	error = of_mdiobus_register(mdio: priv->mii_bus, np: dev->of_node);
2587	if (error)
2588	goto out_free_bus;
2589
2590	pn = of_parse_phandle(np: dev->of_node, phandle_name: "phy-handle", index: 0);
2591	phydev = of_phy_find_device(phy_np: pn);
2592	if (phydev) {
2593	phydev->mac_managed_pm = true;
2594	put_device(dev: &phydev->mdio.dev);
2595	}
2596	of_node_put(node: pn);
2597
2598	return 0;
2599
2600	out_free_bus:
2601	free_mdio_bitbang(bus: priv->mii_bus);
2602	return error;
2603	}
2604
2605	/* MDIO bus release function */
2606	static int ravb_mdio_release(struct ravb_private *priv)
2607	{
2608	/* Unregister mdio bus */
2609	mdiobus_unregister(bus: priv->mii_bus);
2610
2611	/* Free bitbang info */
2612	free_mdio_bitbang(bus: priv->mii_bus);
2613
2614	return 0;
2615	}
2616
2617	static const struct ravb_hw_info ravb_gen3_hw_info = {
2618	.receive = ravb_rx_rcar,
2619	.set_rate = ravb_set_rate_rcar,
2620	.set_feature = ravb_set_features_rcar,
2621	.dmac_init = ravb_dmac_init_rcar,
2622	.emac_init = ravb_emac_init_rcar,
2623	.gstrings_stats = ravb_gstrings_stats,
2624	.gstrings_size = sizeof(ravb_gstrings_stats),
2625	.net_hw_features = NETIF_F_RXCSUM,
2626	.net_features = NETIF_F_RXCSUM,
2627	.stats_len = ARRAY_SIZE(ravb_gstrings_stats),
2628	.tccr_mask = TCCR_TSRQ0 | TCCR_TSRQ1 | TCCR_TSRQ2 | TCCR_TSRQ3,
2629	.rx_max_frame_size = SZ_2K,
2630	.rx_max_desc_use = SZ_2K - ETH_FCS_LEN + sizeof(__sum16),
2631	.rx_desc_size = sizeof(struct ravb_ex_rx_desc),
2632	.internal_delay = 1,
2633	.tx_counters = 1,
2634	.multi_irqs = 1,
2635	.irq_en_dis = 1,
2636	.ccc_gac = 1,
2637	.nc_queues = 1,
2638	.magic_pkt = 1,
2639	};
2640
2641	static const struct ravb_hw_info ravb_gen2_hw_info = {
2642	.receive = ravb_rx_rcar,
2643	.set_rate = ravb_set_rate_rcar,
2644	.set_feature = ravb_set_features_rcar,
2645	.dmac_init = ravb_dmac_init_rcar,
2646	.emac_init = ravb_emac_init_rcar,
2647	.gstrings_stats = ravb_gstrings_stats,
2648	.gstrings_size = sizeof(ravb_gstrings_stats),
2649	.net_hw_features = NETIF_F_RXCSUM,
2650	.net_features = NETIF_F_RXCSUM,
2651	.stats_len = ARRAY_SIZE(ravb_gstrings_stats),
2652	.tccr_mask = TCCR_TSRQ0 | TCCR_TSRQ1 | TCCR_TSRQ2 | TCCR_TSRQ3,
2653	.rx_max_frame_size = SZ_2K,
2654	.rx_max_desc_use = SZ_2K - ETH_FCS_LEN + sizeof(__sum16),
2655	.rx_desc_size = sizeof(struct ravb_ex_rx_desc),
2656	.aligned_tx = 1,
2657	.gptp = 1,
2658	.nc_queues = 1,
2659	.magic_pkt = 1,
2660	};
2661
2662	static const struct ravb_hw_info ravb_rzv2m_hw_info = {
2663	.receive = ravb_rx_rcar,
2664	.set_rate = ravb_set_rate_rcar,
2665	.set_feature = ravb_set_features_rcar,
2666	.dmac_init = ravb_dmac_init_rcar,
2667	.emac_init = ravb_emac_init_rcar,
2668	.gstrings_stats = ravb_gstrings_stats,
2669	.gstrings_size = sizeof(ravb_gstrings_stats),
2670	.net_hw_features = NETIF_F_RXCSUM,
2671	.net_features = NETIF_F_RXCSUM,
2672	.stats_len = ARRAY_SIZE(ravb_gstrings_stats),
2673	.tccr_mask = TCCR_TSRQ0 | TCCR_TSRQ1 | TCCR_TSRQ2 | TCCR_TSRQ3,
2674	.rx_max_frame_size = SZ_2K,
2675	.rx_max_desc_use = SZ_2K - ETH_FCS_LEN + sizeof(__sum16),
2676	.rx_desc_size = sizeof(struct ravb_ex_rx_desc),
2677	.multi_irqs = 1,
2678	.err_mgmt_irqs = 1,
2679	.gptp = 1,
2680	.gptp_ref_clk = 1,
2681	.nc_queues = 1,
2682	.magic_pkt = 1,
2683	};
2684
2685	static const struct ravb_hw_info gbeth_hw_info = {
2686	.receive = ravb_rx_gbeth,
2687	.set_rate = ravb_set_rate_gbeth,
2688	.set_feature = ravb_set_features_gbeth,
2689	.dmac_init = ravb_dmac_init_gbeth,
2690	.emac_init = ravb_emac_init_gbeth,
2691	.gstrings_stats = ravb_gstrings_stats_gbeth,
2692	.gstrings_size = sizeof(ravb_gstrings_stats_gbeth),
2693	.net_hw_features = NETIF_F_RXCSUM | NETIF_F_HW_CSUM,
2694	.net_features = NETIF_F_RXCSUM | NETIF_F_HW_CSUM,
2695	.stats_len = ARRAY_SIZE(ravb_gstrings_stats_gbeth),
2696	.tccr_mask = TCCR_TSRQ0,
2697	.rx_max_frame_size = SZ_8K,
2698	.rx_max_desc_use = 4080,
2699	.rx_desc_size = sizeof(struct ravb_rx_desc),
2700	.aligned_tx = 1,
2701	.tx_counters = 1,
2702	.carrier_counters = 1,
2703	.half_duplex = 1,
2704	};
2705
2706	static const struct of_device_id ravb_match_table[] = {
2707	{ .compatible = "renesas,etheravb-r8a7790", .data = &ravb_gen2_hw_info },
2708	{ .compatible = "renesas,etheravb-r8a7794", .data = &ravb_gen2_hw_info },
2709	{ .compatible = "renesas,etheravb-rcar-gen2", .data = &ravb_gen2_hw_info },
2710	{ .compatible = "renesas,etheravb-r8a7795", .data = &ravb_gen3_hw_info },
2711	{ .compatible = "renesas,etheravb-rcar-gen3", .data = &ravb_gen3_hw_info },
2712	{ .compatible = "renesas,etheravb-rcar-gen4", .data = &ravb_gen3_hw_info },
2713	{ .compatible = "renesas,etheravb-rzv2m", .data = &ravb_rzv2m_hw_info },
2714	{ .compatible = "renesas,rzg2l-gbeth", .data = &gbeth_hw_info },
2715	{ }
2716	};
2717	MODULE_DEVICE_TABLE(of, ravb_match_table);
2718
2719	static int ravb_setup_irq(struct ravb_private *priv, const char *irq_name,
2720	const char *ch, int *irq, irq_handler_t handler)
2721	{
2722	struct platform_device *pdev = priv->pdev;
2723	struct net_device *ndev = priv->ndev;
2724	struct device *dev = &pdev->dev;
2725	const char *dev_name;
2726	unsigned long flags;
2727	int error, irq_num;
2728
2729	if (irq_name) {
2730	dev_name = devm_kasprintf(dev, GFP_KERNEL, fmt: "%s:%s", ndev->name, ch);
2731	if (!dev_name)
2732	return -ENOMEM;
2733
2734	irq_num = platform_get_irq_byname(pdev, irq_name);
2735	flags = 0;
2736	} else {
2737	dev_name = ndev->name;
2738	irq_num = platform_get_irq(pdev, 0);
2739	flags = IRQF_SHARED;
2740	}
2741	if (irq_num < 0)
2742	return irq_num;
2743
2744	if (irq)
2745	*irq = irq_num;
2746
2747	error = devm_request_irq(dev, irq: irq_num, handler, irqflags: flags, devname: dev_name, dev_id: ndev);
2748	if (error)
2749	netdev_err(dev: ndev, format: "cannot request IRQ %s\n", dev_name);
2750
2751	return error;
2752	}
2753
2754	static int ravb_setup_irqs(struct ravb_private *priv)
2755	{
2756	const struct ravb_hw_info *info = priv->info;
2757	struct net_device *ndev = priv->ndev;
2758	const char *irq_name, *emac_irq_name;
2759	int error;
2760
2761	if (!info->multi_irqs)
2762	return ravb_setup_irq(priv, NULL, NULL, irq: &ndev->irq, handler: ravb_interrupt);
2763
2764	if (info->err_mgmt_irqs) {
2765	irq_name = "dia";
2766	emac_irq_name = "line3";
2767	} else {
2768	irq_name = "ch22";
2769	emac_irq_name = "ch24";
2770	}
2771
2772	error = ravb_setup_irq(priv, irq_name, ch: "ch22:multi", irq: &ndev->irq, handler: ravb_multi_interrupt);
2773	if (error)
2774	return error;
2775
2776	error = ravb_setup_irq(priv, irq_name: emac_irq_name, ch: "ch24:emac", irq: &priv->emac_irq,
2777	handler: ravb_emac_interrupt);
2778	if (error)
2779	return error;
2780
2781	if (info->err_mgmt_irqs) {
2782	error = ravb_setup_irq(priv, irq_name: "err_a", ch: "err_a", NULL, handler: ravb_multi_interrupt);
2783	if (error)
2784	return error;
2785
2786	error = ravb_setup_irq(priv, irq_name: "mgmt_a", ch: "mgmt_a", NULL, handler: ravb_multi_interrupt);
2787	if (error)
2788	return error;
2789	}
2790
2791	error = ravb_setup_irq(priv, irq_name: "ch0", ch: "ch0:rx_be", NULL, handler: ravb_be_interrupt);
2792	if (error)
2793	return error;
2794
2795	error = ravb_setup_irq(priv, irq_name: "ch1", ch: "ch1:rx_nc", NULL, handler: ravb_nc_interrupt);
2796	if (error)
2797	return error;
2798
2799	error = ravb_setup_irq(priv, irq_name: "ch18", ch: "ch18:tx_be", NULL, handler: ravb_be_interrupt);
2800	if (error)
2801	return error;
2802
2803	return ravb_setup_irq(priv, irq_name: "ch19", ch: "ch19:tx_nc", NULL, handler: ravb_nc_interrupt);
2804	}
2805
2806	static int ravb_probe(struct platform_device *pdev)
2807	{
2808	struct device_node *np = pdev->dev.of_node;
2809	const struct ravb_hw_info *info;
2810	struct reset_control *rstc;
2811	struct ravb_private *priv;
2812	struct net_device *ndev;
2813	struct resource *res;
2814	int error, q;
2815
2816	if (!np) {
2817	dev_err(&pdev->dev,
2818	"this driver is required to be instantiated from device tree\n");
2819	return -EINVAL;
2820	}
2821
2822	rstc = devm_reset_control_get_exclusive(dev: &pdev->dev, NULL);
2823	if (IS_ERR(ptr: rstc))
2824	return dev_err_probe(dev: &pdev->dev, err: PTR_ERR(ptr: rstc),
2825	fmt: "failed to get cpg reset\n");
2826
2827	ndev = alloc_etherdev_mqs(sizeof_priv: sizeof(struct ravb_private),
2828	NUM_TX_QUEUE, NUM_RX_QUEUE);
2829	if (!ndev)
2830	return -ENOMEM;
2831
2832	info = of_device_get_match_data(dev: &pdev->dev);
2833
2834	ndev->features = info->net_features;
2835	ndev->hw_features = info->net_hw_features;
2836
2837	error = reset_control_deassert(rstc);
2838	if (error)
2839	goto out_free_netdev;
2840
2841	SET_NETDEV_DEV(ndev, &pdev->dev);
2842
2843	priv = netdev_priv(dev: ndev);
2844	priv->info = info;
2845	priv->rstc = rstc;
2846	priv->ndev = ndev;
2847	priv->pdev = pdev;
2848	priv->num_tx_ring[RAVB_BE] = BE_TX_RING_SIZE;
2849	priv->num_rx_ring[RAVB_BE] = BE_RX_RING_SIZE;
2850	if (info->nc_queues) {
2851	priv->num_tx_ring[RAVB_NC] = NC_TX_RING_SIZE;
2852	priv->num_rx_ring[RAVB_NC] = NC_RX_RING_SIZE;
2853	}
2854
2855	error = ravb_setup_irqs(priv);
2856	if (error)
2857	goto out_reset_assert;
2858
2859	priv->clk = devm_clk_get(dev: &pdev->dev, NULL);
2860	if (IS_ERR(ptr: priv->clk)) {
2861	error = PTR_ERR(ptr: priv->clk);
2862	goto out_reset_assert;
2863	}
2864
2865	if (info->gptp_ref_clk) {
2866	priv->gptp_clk = devm_clk_get(dev: &pdev->dev, id: "gptp");
2867	if (IS_ERR(ptr: priv->gptp_clk)) {
2868	error = PTR_ERR(ptr: priv->gptp_clk);
2869	goto out_reset_assert;
2870	}
2871	}
2872
2873	priv->refclk = devm_clk_get_optional(dev: &pdev->dev, id: "refclk");
2874	if (IS_ERR(ptr: priv->refclk)) {
2875	error = PTR_ERR(ptr: priv->refclk);
2876	goto out_reset_assert;
2877	}
2878	clk_prepare(clk: priv->refclk);
2879
2880	platform_set_drvdata(pdev, data: ndev);
2881	pm_runtime_set_autosuspend_delay(dev: &pdev->dev, delay: 100);
2882	pm_runtime_use_autosuspend(dev: &pdev->dev);
2883	pm_runtime_enable(dev: &pdev->dev);
2884	error = pm_runtime_resume_and_get(dev: &pdev->dev);
2885	if (error < 0)
2886	goto out_rpm_disable;
2887
2888	priv->addr = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res);
2889	if (IS_ERR(ptr: priv->addr)) {
2890	error = PTR_ERR(ptr: priv->addr);
2891	goto out_rpm_put;
2892	}
2893
2894	/* The Ether-specific entries in the device structure. */
2895	ndev->base_addr = res->start;
2896
2897	spin_lock_init(&priv->lock);
2898	INIT_WORK(&priv->work, ravb_tx_timeout_work);
2899
2900	error = of_get_phy_mode(np, interface: &priv->phy_interface);
2901	if (error && error != -ENODEV)
2902	goto out_rpm_put;
2903
2904	priv->no_avb_link = of_property_read_bool(np, propname: "renesas,no-ether-link");
2905	priv->avb_link_active_low =
2906	of_property_read_bool(np, propname: "renesas,ether-link-active-low");
2907
2908	ndev->max_mtu = info->rx_max_frame_size -
2909	(ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN);
2910	ndev->min_mtu = ETH_MIN_MTU;
2911
2912	/* FIXME: R-Car Gen2 has 4byte alignment restriction for tx buffer
2913	* Use two descriptor to handle such situation. First descriptor to
2914	* handle aligned data buffer and second descriptor to handle the
2915	* overflow data because of alignment.
2916	*/
2917	priv->num_tx_desc = info->aligned_tx ? 2 : 1;
2918
2919	/* Set function */
2920	ndev->netdev_ops = &ravb_netdev_ops;
2921	ndev->ethtool_ops = &ravb_ethtool_ops;
2922
2923	error = ravb_compute_gti(ndev);
2924	if (error)
2925	goto out_rpm_put;
2926
2927	ravb_parse_delay_mode(np, ndev);
2928
2929	/* Allocate descriptor base address table */
2930	priv->desc_bat_size = sizeof(struct ravb_desc) * DBAT_ENTRY_NUM;
2931	priv->desc_bat = dma_alloc_coherent(dev: ndev->dev.parent, size: priv->desc_bat_size,
2932	dma_handle: &priv->desc_bat_dma, GFP_KERNEL);
2933	if (!priv->desc_bat) {
2934	dev_err(&pdev->dev,
2935	"Cannot allocate desc base address table (size %d bytes)\n",
2936	priv->desc_bat_size);
2937	error = -ENOMEM;
2938	goto out_rpm_put;
2939	}
2940	for (q = RAVB_BE; q < DBAT_ENTRY_NUM; q++)
2941	priv->desc_bat[q].die_dt = DT_EOS;
2942
2943	/* Initialise HW timestamp list */
2944	INIT_LIST_HEAD(list: &priv->ts_skb_list);
2945
2946	/* Debug message level */
2947	priv->msg_enable = RAVB_DEF_MSG_ENABLE;
2948
2949	/* Set config mode as this is needed for PHY initialization. */
2950	error = ravb_set_opmode(ndev, opmode: CCC_OPC_CONFIG);
2951	if (error)
2952	goto out_rpm_put;
2953
2954	/* Read and set MAC address */
2955	ravb_read_mac_address(np, ndev);
2956	if (!is_valid_ether_addr(addr: ndev->dev_addr)) {
2957	dev_warn(&pdev->dev,
2958	"no valid MAC address supplied, using a random one\n");
2959	eth_hw_addr_random(dev: ndev);
2960	}
2961
2962	/* MDIO bus init */
2963	error = ravb_mdio_init(priv);
2964	if (error) {
2965	dev_err(&pdev->dev, "failed to initialize MDIO\n");
2966	goto out_reset_mode;
2967	}
2968
2969	/* Undo previous switch to config opmode. */
2970	error = ravb_set_opmode(ndev, opmode: CCC_OPC_RESET);
2971	if (error)
2972	goto out_mdio_release;
2973
2974	netif_napi_add(dev: ndev, napi: &priv->napi[RAVB_BE], poll: ravb_poll);
2975	if (info->nc_queues)
2976	netif_napi_add(dev: ndev, napi: &priv->napi[RAVB_NC], poll: ravb_poll);
2977
2978	/* Network device register */
2979	error = register_netdev(dev: ndev);
2980	if (error)
2981	goto out_napi_del;
2982
2983	device_set_wakeup_capable(dev: &pdev->dev, capable: 1);
2984
2985	/* Print device information */
2986	netdev_info(dev: ndev, format: "Base address at %#x, %pM, IRQ %d.\n",
2987	(u32)ndev->base_addr, ndev->dev_addr, ndev->irq);
2988
2989	pm_runtime_mark_last_busy(dev: &pdev->dev);
2990	pm_runtime_put_autosuspend(dev: &pdev->dev);
2991
2992	return 0;
2993
2994	out_napi_del:
2995	if (info->nc_queues)
2996	netif_napi_del(napi: &priv->napi[RAVB_NC]);
2997
2998	netif_napi_del(napi: &priv->napi[RAVB_BE]);
2999	out_mdio_release:
3000	ravb_mdio_release(priv);
3001	out_reset_mode:
3002	ravb_set_opmode(ndev, opmode: CCC_OPC_RESET);
3003	dma_free_coherent(dev: ndev->dev.parent, size: priv->desc_bat_size, cpu_addr: priv->desc_bat,
3004	dma_handle: priv->desc_bat_dma);
3005	out_rpm_put:
3006	pm_runtime_put(dev: &pdev->dev);
3007	out_rpm_disable:
3008	pm_runtime_disable(dev: &pdev->dev);
3009	pm_runtime_dont_use_autosuspend(dev: &pdev->dev);
3010	clk_unprepare(clk: priv->refclk);
3011	out_reset_assert:
3012	reset_control_assert(rstc);
3013	out_free_netdev:
3014	free_netdev(dev: ndev);
3015	return error;
3016	}
3017
3018	static void ravb_remove(struct platform_device *pdev)
3019	{
3020	struct net_device *ndev = platform_get_drvdata(pdev);
3021	struct ravb_private *priv = netdev_priv(dev: ndev);
3022	const struct ravb_hw_info *info = priv->info;
3023	struct device *dev = &priv->pdev->dev;
3024	int error;
3025
3026	error = pm_runtime_resume_and_get(dev);
3027	if (error < 0)
3028	return;
3029
3030	unregister_netdev(dev: ndev);
3031	if (info->nc_queues)
3032	netif_napi_del(napi: &priv->napi[RAVB_NC]);
3033	netif_napi_del(napi: &priv->napi[RAVB_BE]);
3034
3035	ravb_mdio_release(priv);
3036
3037	dma_free_coherent(dev: ndev->dev.parent, size: priv->desc_bat_size, cpu_addr: priv->desc_bat,
3038	dma_handle: priv->desc_bat_dma);
3039
3040	pm_runtime_put_sync_suspend(dev: &pdev->dev);
3041	pm_runtime_disable(dev: &pdev->dev);
3042	pm_runtime_dont_use_autosuspend(dev);
3043	clk_unprepare(clk: priv->refclk);
3044	reset_control_assert(rstc: priv->rstc);
3045	free_netdev(dev: ndev);
3046	platform_set_drvdata(pdev, NULL);
3047	}
3048
3049	static int ravb_wol_setup(struct net_device *ndev)
3050	{
3051	struct ravb_private *priv = netdev_priv(dev: ndev);
3052	const struct ravb_hw_info *info = priv->info;
3053
3054	/* Disable interrupts by clearing the interrupt masks. */
3055	ravb_write(ndev, data: 0, reg: RIC0);
3056	ravb_write(ndev, data: 0, reg: RIC2);
3057	ravb_write(ndev, data: 0, reg: TIC);
3058
3059	/* Only allow ECI interrupts */
3060	synchronize_irq(irq: priv->emac_irq);
3061	if (info->nc_queues)
3062	napi_disable(n: &priv->napi[RAVB_NC]);
3063	napi_disable(n: &priv->napi[RAVB_BE]);
3064	ravb_write(ndev, data: ECSIPR_MPDIP, reg: ECSIPR);
3065
3066	/* Enable MagicPacket */
3067	ravb_modify(ndev, reg: ECMR, clear: ECMR_MPDE, set: ECMR_MPDE);
3068
3069	if (priv->info->ccc_gac)
3070	ravb_ptp_stop(ndev);
3071
3072	return enable_irq_wake(irq: priv->emac_irq);
3073	}
3074
3075	static int ravb_wol_restore(struct net_device *ndev)
3076	{
3077	struct ravb_private *priv = netdev_priv(dev: ndev);
3078	const struct ravb_hw_info *info = priv->info;
3079	int error;
3080
3081	/* Set reset mode to rearm the WoL logic. */
3082	error = ravb_set_opmode(ndev, opmode: CCC_OPC_RESET);
3083	if (error)
3084	return error;
3085
3086	/* Set AVB config mode. */
3087	error = ravb_set_config_mode(ndev);
3088	if (error)
3089	return error;
3090
3091	if (priv->info->ccc_gac)
3092	ravb_ptp_init(ndev, pdev: priv->pdev);
3093
3094	if (info->nc_queues)
3095	napi_enable(n: &priv->napi[RAVB_NC]);
3096	napi_enable(n: &priv->napi[RAVB_BE]);
3097
3098	/* Disable MagicPacket */
3099	ravb_modify(ndev, reg: ECMR, clear: ECMR_MPDE, set: 0);
3100
3101	ravb_close(ndev);
3102
3103	return disable_irq_wake(irq: priv->emac_irq);
3104	}
3105
3106	static int ravb_suspend(struct device *dev)
3107	{
3108	struct net_device *ndev = dev_get_drvdata(dev);
3109	struct ravb_private *priv = netdev_priv(dev: ndev);
3110	int ret;
3111
3112	if (!netif_running(dev: ndev))
3113	goto reset_assert;
3114
3115	netif_device_detach(dev: ndev);
3116
3117	if (priv->wol_enabled)
3118	return ravb_wol_setup(ndev);
3119
3120	ret = ravb_close(ndev);
3121	if (ret)
3122	return ret;
3123
3124	ret = pm_runtime_force_suspend(dev: &priv->pdev->dev);
3125	if (ret)
3126	return ret;
3127
3128	reset_assert:
3129	return reset_control_assert(rstc: priv->rstc);
3130	}
3131
3132	static int ravb_resume(struct device *dev)
3133	{
3134	struct net_device *ndev = dev_get_drvdata(dev);
3135	struct ravb_private *priv = netdev_priv(dev: ndev);
3136	int ret;
3137
3138	ret = reset_control_deassert(rstc: priv->rstc);
3139	if (ret)
3140	return ret;
3141
3142	if (!netif_running(dev: ndev))
3143	return 0;
3144
3145	/* If WoL is enabled restore the interface. */
3146	if (priv->wol_enabled) {
3147	ret = ravb_wol_restore(ndev);
3148	if (ret)
3149	return ret;
3150	} else {
3151	ret = pm_runtime_force_resume(dev);
3152	if (ret)
3153	return ret;
3154	}
3155
3156	/* Reopening the interface will restore the device to the working state. */
3157	ret = ravb_open(ndev);
3158	if (ret < 0)
3159	goto out_rpm_put;
3160
3161	ravb_set_rx_mode(ndev);
3162	netif_device_attach(dev: ndev);
3163
3164	return 0;
3165
3166	out_rpm_put:
3167	if (!priv->wol_enabled) {
3168	pm_runtime_mark_last_busy(dev);
3169	pm_runtime_put_autosuspend(dev);
3170	}
3171
3172	return ret;
3173	}
3174
3175	static int ravb_runtime_suspend(struct device *dev)
3176	{
3177	struct net_device *ndev = dev_get_drvdata(dev);
3178	struct ravb_private *priv = netdev_priv(dev: ndev);
3179
3180	clk_disable(clk: priv->refclk);
3181
3182	return 0;
3183	}
3184
3185	static int ravb_runtime_resume(struct device *dev)
3186	{
3187	struct net_device *ndev = dev_get_drvdata(dev);
3188	struct ravb_private *priv = netdev_priv(dev: ndev);
3189
3190	return clk_enable(clk: priv->refclk);
3191	}
3192
3193	static const struct dev_pm_ops ravb_dev_pm_ops = {
3194	SYSTEM_SLEEP_PM_OPS(ravb_suspend, ravb_resume)
3195	RUNTIME_PM_OPS(ravb_runtime_suspend, ravb_runtime_resume, NULL)
3196	};
3197
3198	static struct platform_driver ravb_driver = {
3199	.probe = ravb_probe,
3200	.remove_new = ravb_remove,
3201	.driver = {
3202	.name = "ravb",
3203	.pm = pm_ptr(&ravb_dev_pm_ops),
3204	.of_match_table = ravb_match_table,
3205	},
3206	};
3207
3208	module_platform_driver(ravb_driver);
3209
3210	MODULE_AUTHOR("Mitsuhiro Kimura, Masaru Nagai");
3211	MODULE_DESCRIPTION("Renesas Ethernet AVB driver");
3212	MODULE_LICENSE("GPL v2");
3213