1	// SPDX-License-Identifier: GPL-2.0
2	/* Texas Instruments K3 AM65 Ethernet Switch SubSystem Driver
3	*
4	* Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/
5	*
6	*/
7
8	#include <linux/clk.h>
9	#include <linux/etherdevice.h>
10	#include <linux/if_vlan.h>
11	#include <linux/interrupt.h>
12	#include <linux/irqdomain.h>
13	#include <linux/kernel.h>
14	#include <linux/kmemleak.h>
15	#include <linux/module.h>
16	#include <linux/netdevice.h>
17	#include <linux/net_tstamp.h>
18	#include <linux/of.h>
19	#include <linux/of_mdio.h>
20	#include <linux/of_net.h>
21	#include <linux/of_device.h>
22	#include <linux/of_platform.h>
23	#include <linux/phylink.h>
24	#include <linux/phy/phy.h>
25	#include <linux/platform_device.h>
26	#include <linux/pm_runtime.h>
27	#include <linux/regmap.h>
28	#include <linux/rtnetlink.h>
29	#include <linux/mfd/syscon.h>
30	#include <linux/sys_soc.h>
31	#include <linux/dma/ti-cppi5.h>
32	#include <linux/dma/k3-udma-glue.h>
33	#include <net/switchdev.h>
34
35	#include "cpsw_ale.h"
36	#include "cpsw_sl.h"
37	#include "am65-cpsw-nuss.h"
38	#include "am65-cpsw-switchdev.h"
39	#include "k3-cppi-desc-pool.h"
40	#include "am65-cpts.h"
41
42	#define AM65_CPSW_SS_BASE 0x0
43	#define AM65_CPSW_SGMII_BASE 0x100
44	#define AM65_CPSW_XGMII_BASE 0x2100
45	#define AM65_CPSW_CPSW_NU_BASE 0x20000
46	#define AM65_CPSW_NU_PORTS_BASE 0x1000
47	#define AM65_CPSW_NU_FRAM_BASE 0x12000
48	#define AM65_CPSW_NU_STATS_BASE 0x1a000
49	#define AM65_CPSW_NU_ALE_BASE 0x1e000
50	#define AM65_CPSW_NU_CPTS_BASE 0x1d000
51
52	#define AM65_CPSW_NU_PORTS_OFFSET 0x1000
53	#define AM65_CPSW_NU_STATS_PORT_OFFSET 0x200
54	#define AM65_CPSW_NU_FRAM_PORT_OFFSET 0x200
55
56	#define AM65_CPSW_MAX_PORTS 8
57
58	#define AM65_CPSW_MIN_PACKET_SIZE VLAN_ETH_ZLEN
59	#define AM65_CPSW_MAX_PACKET_SIZE 2024
60
61	#define AM65_CPSW_REG_CTL 0x004
62	#define AM65_CPSW_REG_STAT_PORT_EN 0x014
63	#define AM65_CPSW_REG_PTYPE 0x018
64
65	#define AM65_CPSW_P0_REG_CTL 0x004
66	#define AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET 0x008
67
68	#define AM65_CPSW_PORT_REG_PRI_CTL 0x01c
69	#define AM65_CPSW_PORT_REG_RX_PRI_MAP 0x020
70	#define AM65_CPSW_PORT_REG_RX_MAXLEN 0x024
71
72	#define AM65_CPSW_PORTN_REG_SA_L 0x308
73	#define AM65_CPSW_PORTN_REG_SA_H 0x30c
74	#define AM65_CPSW_PORTN_REG_TS_CTL 0x310
75	#define AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG 0x314
76	#define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG 0x318
77	#define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 0x31C
78
79	#define AM65_CPSW_SGMII_CONTROL_REG 0x010
80	#define AM65_CPSW_SGMII_MR_ADV_ABILITY_REG 0x018
81	#define AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE BIT(0)
82
83	#define AM65_CPSW_CTL_VLAN_AWARE BIT(1)
84	#define AM65_CPSW_CTL_P0_ENABLE BIT(2)
85	#define AM65_CPSW_CTL_P0_TX_CRC_REMOVE BIT(13)
86	#define AM65_CPSW_CTL_P0_RX_PAD BIT(14)
87
88	/* AM65_CPSW_P0_REG_CTL */
89	#define AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN BIT(0)
90	#define AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN BIT(16)
91
92	/* AM65_CPSW_PORT_REG_PRI_CTL */
93	#define AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN BIT(8)
94
95	/* AM65_CPSW_PN_TS_CTL register fields */
96	#define AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN BIT(4)
97	#define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN BIT(5)
98	#define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT2_EN BIT(6)
99	#define AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN BIT(7)
100	#define AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN BIT(10)
101	#define AM65_CPSW_PN_TS_CTL_TX_HOST_TS_EN BIT(11)
102	#define AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT 16
103
104	/* AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG register fields */
105	#define AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT 16
106
107	/* AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 */
108	#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 BIT(16)
109	#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 BIT(17)
110	#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 BIT(18)
111	#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 BIT(19)
112	#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 BIT(20)
113	#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 BIT(21)
114	#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 BIT(22)
115	#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO BIT(23)
116
117	/* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
118	#define AM65_CPSW_TS_EVENT_MSG_TYPE_BITS (BIT(0) | BIT(1) | BIT(2) | BIT(3))
119
120	#define AM65_CPSW_TS_SEQ_ID_OFFSET (0x1e)
121
122	#define AM65_CPSW_TS_TX_ANX_ALL_EN \
123	(AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN | \
124	AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN | \
125	AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN)
126
127	#define AM65_CPSW_ALE_AGEOUT_DEFAULT 30
128	/* Number of TX/RX descriptors */
129	#define AM65_CPSW_MAX_TX_DESC 500
130	#define AM65_CPSW_MAX_RX_DESC 500
131
132	#define AM65_CPSW_NAV_PS_DATA_SIZE 16
133	#define AM65_CPSW_NAV_SW_DATA_SIZE 16
134
135	#define AM65_CPSW_DEBUG (NETIF_MSG_HW | NETIF_MSG_DRV | NETIF_MSG_LINK | \
136	NETIF_MSG_IFUP | NETIF_MSG_PROBE | NETIF_MSG_IFDOWN | \
137	NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
138
139	#define AM65_CPSW_DEFAULT_TX_CHNS 8
140
141	static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave,
142	const u8 *dev_addr)
143	{
144	u32 mac_hi = (dev_addr[0] << 0) | (dev_addr[1] << 8) |
145	(dev_addr[2] << 16) | (dev_addr[3] << 24);
146	u32 mac_lo = (dev_addr[4] << 0) | (dev_addr[5] << 8);
147
148	writel(val: mac_hi, addr: slave->port_base + AM65_CPSW_PORTN_REG_SA_H);
149	writel(val: mac_lo, addr: slave->port_base + AM65_CPSW_PORTN_REG_SA_L);
150	}
151
152	static void am65_cpsw_sl_ctl_reset(struct am65_cpsw_port *port)
153	{
154	cpsw_sl_reset(sl: port->slave.mac_sl, tmo: 100);
155	/* Max length register has to be restored after MAC SL reset */
156	writel(AM65_CPSW_MAX_PACKET_SIZE,
157	addr: port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
158	}
159
160	static void am65_cpsw_nuss_get_ver(struct am65_cpsw_common *common)
161	{
162	common->nuss_ver = readl(addr: common->ss_base);
163	common->cpsw_ver = readl(addr: common->cpsw_base);
164	dev_info(common->dev,
165	"initializing am65 cpsw nuss version 0x%08X, cpsw version 0x%08X Ports: %u quirks:%08x\n",
166	common->nuss_ver,
167	common->cpsw_ver,
168	common->port_num + 1,
169	common->pdata.quirks);
170	}
171
172	static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev,
173	__be16 proto, u16 vid)
174	{
175	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
176	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
177	u32 port_mask, unreg_mcast = 0;
178	int ret;
179
180	if (!common->is_emac_mode)
181	return 0;
182
183	if (!netif_running(dev: ndev) || !vid)
184	return 0;
185
186	ret = pm_runtime_resume_and_get(dev: common->dev);
187	if (ret < 0)
188	return ret;
189
190	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
191	if (!vid)
192	unreg_mcast = port_mask;
193	dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid);
194	ret = cpsw_ale_vlan_add_modify(ale: common->ale, vid, port_mask,
195	untag_mask: unreg_mcast, reg_mcast: port_mask, unreg_mcast: 0);
196
197	pm_runtime_put(dev: common->dev);
198	return ret;
199	}
200
201	static int am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device *ndev,
202	__be16 proto, u16 vid)
203	{
204	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
205	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
206	int ret;
207
208	if (!common->is_emac_mode)
209	return 0;
210
211	if (!netif_running(dev: ndev) || !vid)
212	return 0;
213
214	ret = pm_runtime_resume_and_get(dev: common->dev);
215	if (ret < 0)
216	return ret;
217
218	dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid);
219	ret = cpsw_ale_del_vlan(ale: common->ale, vid,
220	BIT(port->port_id) | ALE_PORT_HOST);
221
222	pm_runtime_put(dev: common->dev);
223	return ret;
224	}
225
226	static void am65_cpsw_slave_set_promisc(struct am65_cpsw_port *port,
227	bool promisc)
228	{
229	struct am65_cpsw_common *common = port->common;
230
231	if (promisc && !common->is_emac_mode) {
232	dev_dbg(common->dev, "promisc mode requested in switch mode");
233	return;
234	}
235
236	if (promisc) {
237	/* Enable promiscuous mode */
238	cpsw_ale_control_set(ale: common->ale, port: port->port_id,
239	control: ALE_PORT_MACONLY_CAF, value: 1);
240	dev_dbg(common->dev, "promisc enabled\n");
241	} else {
242	/* Disable promiscuous mode */
243	cpsw_ale_control_set(ale: common->ale, port: port->port_id,
244	control: ALE_PORT_MACONLY_CAF, value: 0);
245	dev_dbg(common->dev, "promisc disabled\n");
246	}
247	}
248
249	static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev)
250	{
251	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
252	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
253	u32 port_mask;
254	bool promisc;
255
256	promisc = !!(ndev->flags & IFF_PROMISC);
257	am65_cpsw_slave_set_promisc(port, promisc);
258
259	if (promisc)
260	return;
261
262	/* Restore allmulti on vlans if necessary */
263	cpsw_ale_set_allmulti(ale: common->ale,
264	allmulti: ndev->flags & IFF_ALLMULTI, port: port->port_id);
265
266	port_mask = ALE_PORT_HOST;
267	/* Clear all mcast from ALE */
268	cpsw_ale_flush_multicast(ale: common->ale, port_mask, vid: -1);
269
270	if (!netdev_mc_empty(ndev)) {
271	struct netdev_hw_addr *ha;
272
273	/* program multicast address list into ALE register */
274	netdev_for_each_mc_addr(ha, ndev) {
275	cpsw_ale_add_mcast(ale: common->ale, addr: ha->addr,
276	port_mask, flags: 0, vid: 0, mcast_state: 0);
277	}
278	}
279	}
280
281	static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev,
282	unsigned int txqueue)
283	{
284	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
285	struct am65_cpsw_tx_chn *tx_chn;
286	struct netdev_queue *netif_txq;
287	unsigned long trans_start;
288
289	netif_txq = netdev_get_tx_queue(dev: ndev, index: txqueue);
290	tx_chn = &common->tx_chns[txqueue];
291	trans_start = READ_ONCE(netif_txq->trans_start);
292
293	netdev_err(dev: ndev, format: "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n",
294	txqueue,
295	netif_tx_queue_stopped(dev_queue: netif_txq),
296	jiffies_to_msecs(j: jiffies - trans_start),
297	netdev_queue_dql_avail(txq: netif_txq),
298	k3_cppi_desc_pool_avail(pool: tx_chn->desc_pool));
299
300	if (netif_tx_queue_stopped(dev_queue: netif_txq)) {
301	/* try recover if stopped by us */
302	txq_trans_update(txq: netif_txq);
303	netif_tx_wake_queue(dev_queue: netif_txq);
304	}
305	}
306
307	static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common,
308	struct sk_buff *skb)
309	{
310	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
311	struct cppi5_host_desc_t *desc_rx;
312	struct device *dev = common->dev;
313	u32 pkt_len = skb_tailroom(skb);
314	dma_addr_t desc_dma;
315	dma_addr_t buf_dma;
316	void *swdata;
317
318	desc_rx = k3_cppi_desc_pool_alloc(pool: rx_chn->desc_pool);
319	if (!desc_rx) {
320	dev_err(dev, "Failed to allocate RXFDQ descriptor\n");
321	return -ENOMEM;
322	}
323	desc_dma = k3_cppi_desc_pool_virt2dma(pool: rx_chn->desc_pool, addr: desc_rx);
324
325	buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len,
326	DMA_FROM_DEVICE);
327	if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {
328	k3_cppi_desc_pool_free(pool: rx_chn->desc_pool, addr: desc_rx);
329	dev_err(dev, "Failed to map rx skb buffer\n");
330	return -EINVAL;
331	}
332
333	cppi5_hdesc_init(desc: desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
334	AM65_CPSW_NAV_PS_DATA_SIZE);
335	k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn: rx_chn->rx_chn, addr: &buf_dma);
336	cppi5_hdesc_attach_buf(desc: desc_rx, buf: buf_dma, buf_data_len: skb_tailroom(skb), obuf: buf_dma, obuf_len: skb_tailroom(skb));
337	swdata = cppi5_hdesc_get_swdata(desc: desc_rx);
338	*((void **)swdata) = skb;
339
340	return k3_udma_glue_push_rx_chn(rx_chn: rx_chn->rx_chn, flow_num: 0, desc_tx: desc_rx, desc_dma);
341	}
342
343	void am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common *common)
344	{
345	struct am65_cpsw_host *host_p = am65_common_get_host(common);
346	u32 val, pri_map;
347
348	/* P0 set Receive Priority Type */
349	val = readl(addr: host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
350
351	if (common->pf_p0_rx_ptype_rrobin) {
352	val |= AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
353	/* Enet Ports fifos works in fixed priority mode only, so
354	* reset P0_Rx_Pri_Map so all packet will go in Enet fifo 0
355	*/
356	pri_map = 0x0;
357	} else {
358	val &= ~AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
359	/* restore P0_Rx_Pri_Map */
360	pri_map = 0x76543210;
361	}
362
363	writel(val: pri_map, addr: host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP);
364	writel(val, addr: host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
365	}
366
367	static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common);
368	static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common);
369	static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port);
370	static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port);
371
372	static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
373	{
374	struct am65_cpsw_rx_chn *rx_chn = data;
375	struct cppi5_host_desc_t *desc_rx;
376	struct sk_buff *skb;
377	dma_addr_t buf_dma;
378	u32 buf_dma_len;
379	void **swdata;
380
381	desc_rx = k3_cppi_desc_pool_dma2virt(pool: rx_chn->desc_pool, dma: desc_dma);
382	swdata = cppi5_hdesc_get_swdata(desc: desc_rx);
383	skb = *swdata;
384	cppi5_hdesc_get_obuf(desc: desc_rx, obuf: &buf_dma, obuf_len: &buf_dma_len);
385	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn: rx_chn->rx_chn, addr: &buf_dma);
386
387	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
388	k3_cppi_desc_pool_free(pool: rx_chn->desc_pool, addr: desc_rx);
389
390	dev_kfree_skb_any(skb);
391	}
392
393	static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
394	struct cppi5_host_desc_t *desc)
395	{
396	struct cppi5_host_desc_t *first_desc, *next_desc;
397	dma_addr_t buf_dma, next_desc_dma;
398	u32 buf_dma_len;
399
400	first_desc = desc;
401	next_desc = first_desc;
402
403	cppi5_hdesc_get_obuf(desc: first_desc, obuf: &buf_dma, obuf_len: &buf_dma_len);
404	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn: tx_chn->tx_chn, addr: &buf_dma);
405
406	dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);
407
408	next_desc_dma = cppi5_hdesc_get_next_hbdesc(desc: first_desc);
409	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn: tx_chn->tx_chn, addr: &next_desc_dma);
410	while (next_desc_dma) {
411	next_desc = k3_cppi_desc_pool_dma2virt(pool: tx_chn->desc_pool,
412	dma: next_desc_dma);
413	cppi5_hdesc_get_obuf(desc: next_desc, obuf: &buf_dma, obuf_len: &buf_dma_len);
414	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn: tx_chn->tx_chn, addr: &buf_dma);
415
416	dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
417	DMA_TO_DEVICE);
418
419	next_desc_dma = cppi5_hdesc_get_next_hbdesc(desc: next_desc);
420	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn: tx_chn->tx_chn, addr: &next_desc_dma);
421
422	k3_cppi_desc_pool_free(pool: tx_chn->desc_pool, addr: next_desc);
423	}
424
425	k3_cppi_desc_pool_free(pool: tx_chn->desc_pool, addr: first_desc);
426	}
427
428	static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
429	{
430	struct am65_cpsw_tx_chn *tx_chn = data;
431	struct cppi5_host_desc_t *desc_tx;
432	struct sk_buff *skb;
433	void **swdata;
434
435	desc_tx = k3_cppi_desc_pool_dma2virt(pool: tx_chn->desc_pool, dma: desc_dma);
436	swdata = cppi5_hdesc_get_swdata(desc: desc_tx);
437	skb = *(swdata);
438	am65_cpsw_nuss_xmit_free(tx_chn, desc: desc_tx);
439
440	dev_kfree_skb_any(skb);
441	}
442
443	static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
444	{
445	struct am65_cpsw_host *host_p = am65_common_get_host(common);
446	int port_idx, i, ret, tx;
447	struct sk_buff *skb;
448	u32 val, port_mask;
449
450	if (common->usage_count)
451	return 0;
452
453	/* Control register */
454	writel(AM65_CPSW_CTL_P0_ENABLE | AM65_CPSW_CTL_P0_TX_CRC_REMOVE |
455	AM65_CPSW_CTL_VLAN_AWARE | AM65_CPSW_CTL_P0_RX_PAD,
456	addr: common->cpsw_base + AM65_CPSW_REG_CTL);
457	/* Max length register */
458	writel(AM65_CPSW_MAX_PACKET_SIZE,
459	addr: host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
460	/* set base flow_id */
461	writel(val: common->rx_flow_id_base,
462	addr: host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET);
463	writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN | AM65_CPSW_P0_REG_CTL_RX_REMAP_VLAN,
464	addr: host_p->port_base + AM65_CPSW_P0_REG_CTL);
465
466	am65_cpsw_nuss_set_p0_ptype(common);
467
468	/* enable statistic */
469	val = BIT(HOST_PORT_NUM);
470	for (port_idx = 0; port_idx < common->port_num; port_idx++) {
471	struct am65_cpsw_port *port = &common->ports[port_idx];
472
473	if (!port->disabled)
474	val |= BIT(port->port_id);
475	}
476	writel(val, addr: common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
477
478	/* disable priority elevation */
479	writel(val: 0, addr: common->cpsw_base + AM65_CPSW_REG_PTYPE);
480
481	cpsw_ale_start(ale: common->ale);
482
483	/* limit to one RX flow only */
484	cpsw_ale_control_set(ale: common->ale, HOST_PORT_NUM,
485	control: ALE_DEFAULT_THREAD_ID, value: 0);
486	cpsw_ale_control_set(ale: common->ale, HOST_PORT_NUM,
487	control: ALE_DEFAULT_THREAD_ENABLE, value: 1);
488	/* switch to vlan unaware mode */
489	cpsw_ale_control_set(ale: common->ale, HOST_PORT_NUM, control: ALE_VLAN_AWARE, value: 1);
490	cpsw_ale_control_set(ale: common->ale, HOST_PORT_NUM,
491	control: ALE_PORT_STATE, value: ALE_PORT_STATE_FORWARD);
492
493	/* default vlan cfg: create mask based on enabled ports */
494	port_mask = GENMASK(common->port_num, 0) &
495	~common->disabled_ports_mask;
496
497	cpsw_ale_add_vlan(ale: common->ale, vid: 0, port: port_mask,
498	untag: port_mask, reg_mcast: port_mask,
499	unreg_mcast: port_mask & ~ALE_PORT_HOST);
500
501	if (common->is_emac_mode)
502	am65_cpsw_init_host_port_emac(common);
503	else
504	am65_cpsw_init_host_port_switch(common);
505
506	am65_cpsw_qos_tx_p0_rate_init(common);
507
508	for (i = 0; i < common->rx_chns.descs_num; i++) {
509	skb = __netdev_alloc_skb_ip_align(NULL,
510	AM65_CPSW_MAX_PACKET_SIZE,
511	GFP_KERNEL);
512	if (!skb) {
513	ret = -ENOMEM;
514	dev_err(common->dev, "cannot allocate skb\n");
515	if (i)
516	goto fail_rx;
517
518	return ret;
519	}
520
521	ret = am65_cpsw_nuss_rx_push(common, skb);
522	if (ret < 0) {
523	dev_err(common->dev,
524	"cannot submit skb to channel rx, error %d\n",
525	ret);
526	kfree_skb(skb);
527	if (i)
528	goto fail_rx;
529
530	return ret;
531	}
532	}
533
534	ret = k3_udma_glue_enable_rx_chn(rx_chn: common->rx_chns.rx_chn);
535	if (ret) {
536	dev_err(common->dev, "couldn't enable rx chn: %d\n", ret);
537	goto fail_rx;
538	}
539
540	for (tx = 0; tx < common->tx_ch_num; tx++) {
541	ret = k3_udma_glue_enable_tx_chn(tx_chn: common->tx_chns[tx].tx_chn);
542	if (ret) {
543	dev_err(common->dev, "couldn't enable tx chn %d: %d\n",
544	tx, ret);
545	tx--;
546	goto fail_tx;
547	}
548	napi_enable(n: &common->tx_chns[tx].napi_tx);
549	}
550
551	napi_enable(n: &common->napi_rx);
552	if (common->rx_irq_disabled) {
553	common->rx_irq_disabled = false;
554	enable_irq(irq: common->rx_chns.irq);
555	}
556
557	dev_dbg(common->dev, "cpsw_nuss started\n");
558	return 0;
559
560	fail_tx:
561	while (tx >= 0) {
562	napi_disable(n: &common->tx_chns[tx].napi_tx);
563	k3_udma_glue_disable_tx_chn(tx_chn: common->tx_chns[tx].tx_chn);
564	tx--;
565	}
566
567	k3_udma_glue_disable_rx_chn(rx_chn: common->rx_chns.rx_chn);
568
569	fail_rx:
570	k3_udma_glue_reset_rx_chn(rx_chn: common->rx_chns.rx_chn, flow_num: 0,
571	data: &common->rx_chns,
572	cleanup: am65_cpsw_nuss_rx_cleanup, skip_fdq: 0);
573	return ret;
574	}
575
576	static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
577	{
578	int i;
579
580	if (common->usage_count != 1)
581	return 0;
582
583	cpsw_ale_control_set(ale: common->ale, HOST_PORT_NUM,
584	control: ALE_PORT_STATE, value: ALE_PORT_STATE_DISABLE);
585
586	/* shutdown tx channels */
587	atomic_set(v: &common->tdown_cnt, i: common->tx_ch_num);
588	/* ensure new tdown_cnt value is visible */
589	smp_mb__after_atomic();
590	reinit_completion(x: &common->tdown_complete);
591
592	for (i = 0; i < common->tx_ch_num; i++)
593	k3_udma_glue_tdown_tx_chn(tx_chn: common->tx_chns[i].tx_chn, sync: false);
594
595	i = wait_for_completion_timeout(x: &common->tdown_complete,
596	timeout: msecs_to_jiffies(m: 1000));
597	if (!i)
598	dev_err(common->dev, "tx timeout\n");
599	for (i = 0; i < common->tx_ch_num; i++) {
600	napi_disable(n: &common->tx_chns[i].napi_tx);
601	hrtimer_cancel(timer: &common->tx_chns[i].tx_hrtimer);
602	}
603
604	for (i = 0; i < common->tx_ch_num; i++) {
605	k3_udma_glue_reset_tx_chn(tx_chn: common->tx_chns[i].tx_chn,
606	data: &common->tx_chns[i],
607	cleanup: am65_cpsw_nuss_tx_cleanup);
608	k3_udma_glue_disable_tx_chn(tx_chn: common->tx_chns[i].tx_chn);
609	}
610
611	reinit_completion(x: &common->tdown_complete);
612	k3_udma_glue_tdown_rx_chn(rx_chn: common->rx_chns.rx_chn, sync: true);
613
614	if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ) {
615	i = wait_for_completion_timeout(x: &common->tdown_complete, timeout: msecs_to_jiffies(m: 1000));
616	if (!i)
617	dev_err(common->dev, "rx teardown timeout\n");
618	}
619
620	napi_disable(n: &common->napi_rx);
621	hrtimer_cancel(timer: &common->rx_hrtimer);
622
623	for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++)
624	k3_udma_glue_reset_rx_chn(rx_chn: common->rx_chns.rx_chn, flow_num: i,
625	data: &common->rx_chns,
626	cleanup: am65_cpsw_nuss_rx_cleanup, skip_fdq: !!i);
627
628	k3_udma_glue_disable_rx_chn(rx_chn: common->rx_chns.rx_chn);
629
630	cpsw_ale_stop(ale: common->ale);
631
632	writel(val: 0, addr: common->cpsw_base + AM65_CPSW_REG_CTL);
633	writel(val: 0, addr: common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
634
635	dev_dbg(common->dev, "cpsw_nuss stopped\n");
636	return 0;
637	}
638
639	static int am65_cpsw_nuss_ndo_slave_stop(struct net_device *ndev)
640	{
641	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
642	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
643	int ret;
644
645	phylink_stop(port->slave.phylink);
646
647	netif_tx_stop_all_queues(dev: ndev);
648
649	phylink_disconnect_phy(port->slave.phylink);
650
651	ret = am65_cpsw_nuss_common_stop(common);
652	if (ret)
653	return ret;
654
655	common->usage_count--;
656	pm_runtime_put(dev: common->dev);
657	return 0;
658	}
659
660	static int cpsw_restore_vlans(struct net_device *vdev, int vid, void *arg)
661	{
662	struct am65_cpsw_port *port = arg;
663
664	if (!vdev)
665	return 0;
666
667	return am65_cpsw_nuss_ndo_slave_add_vid(ndev: port->ndev, proto: 0, vid);
668	}
669
670	static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
671	{
672	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
673	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
674	int ret, i;
675	u32 reg;
676
677	ret = pm_runtime_resume_and_get(dev: common->dev);
678	if (ret < 0)
679	return ret;
680
681	/* Idle MAC port */
682	cpsw_sl_ctl_set(sl: port->slave.mac_sl, ctl_funcs: CPSW_SL_CTL_CMD_IDLE);
683	cpsw_sl_wait_for_idle(sl: port->slave.mac_sl, tmo: 100);
684	cpsw_sl_ctl_reset(sl: port->slave.mac_sl);
685
686	/* soft reset MAC */
687	cpsw_sl_reg_write(sl: port->slave.mac_sl, reg: CPSW_SL_SOFT_RESET, val: 1);
688	mdelay(1);
689	reg = cpsw_sl_reg_read(sl: port->slave.mac_sl, reg: CPSW_SL_SOFT_RESET);
690	if (reg) {
691	dev_err(common->dev, "soft RESET didn't complete\n");
692	ret = -ETIMEDOUT;
693	goto runtime_put;
694	}
695
696	/* Notify the stack of the actual queue counts. */
697	ret = netif_set_real_num_tx_queues(dev: ndev, txq: common->tx_ch_num);
698	if (ret) {
699	dev_err(common->dev, "cannot set real number of tx queues\n");
700	goto runtime_put;
701	}
702
703	ret = netif_set_real_num_rx_queues(dev: ndev, AM65_CPSW_MAX_RX_QUEUES);
704	if (ret) {
705	dev_err(common->dev, "cannot set real number of rx queues\n");
706	goto runtime_put;
707	}
708
709	for (i = 0; i < common->tx_ch_num; i++) {
710	struct netdev_queue *txq = netdev_get_tx_queue(dev: ndev, index: i);
711
712	netdev_tx_reset_queue(q: txq);
713	txq->tx_maxrate = common->tx_chns[i].rate_mbps;
714	}
715
716	ret = am65_cpsw_nuss_common_open(common);
717	if (ret)
718	goto runtime_put;
719
720	common->usage_count++;
721
722	am65_cpsw_port_set_sl_mac(slave: port, dev_addr: ndev->dev_addr);
723
724	if (common->is_emac_mode)
725	am65_cpsw_init_port_emac_ale(port);
726	else
727	am65_cpsw_init_port_switch_ale(port);
728
729	/* mac_sl should be configured via phy-link interface */
730	am65_cpsw_sl_ctl_reset(port);
731
732	ret = phylink_of_phy_connect(port->slave.phylink, port->slave.phy_node, flags: 0);
733	if (ret)
734	goto error_cleanup;
735
736	/* restore vlan configurations */
737	vlan_for_each(dev: ndev, action: cpsw_restore_vlans, arg: port);
738
739	phylink_start(port->slave.phylink);
740
741	return 0;
742
743	error_cleanup:
744	am65_cpsw_nuss_ndo_slave_stop(ndev);
745	return ret;
746
747	runtime_put:
748	pm_runtime_put(dev: common->dev);
749	return ret;
750	}
751
752	static void am65_cpsw_nuss_rx_ts(struct sk_buff *skb, u32 *psdata)
753	{
754	struct skb_shared_hwtstamps *ssh;
755	u64 ns;
756
757	ns = ((u64)psdata[1] << 32) | psdata[0];
758
759	ssh = skb_hwtstamps(skb);
760	memset(ssh, 0, sizeof(*ssh));
761	ssh->hwtstamp = ns_to_ktime(ns);
762	}
763
764	/* RX psdata[2] word format - checksum information */
765	#define AM65_CPSW_RX_PSD_CSUM_ADD GENMASK(15, 0)
766	#define AM65_CPSW_RX_PSD_CSUM_ERR BIT(16)
767	#define AM65_CPSW_RX_PSD_IS_FRAGMENT BIT(17)
768	#define AM65_CPSW_RX_PSD_IS_TCP BIT(18)
769	#define AM65_CPSW_RX_PSD_IPV6_VALID BIT(19)
770	#define AM65_CPSW_RX_PSD_IPV4_VALID BIT(20)
771
772	static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info)
773	{
774	/* HW can verify IPv4/IPv6 TCP/UDP packets checksum
775	* csum information provides in psdata[2] word:
776	* AM65_CPSW_RX_PSD_CSUM_ERR bit - indicates csum error
777	* AM65_CPSW_RX_PSD_IPV6_VALID and AM65_CPSW_RX_PSD_IPV4_VALID
778	* bits - indicates IPv4/IPv6 packet
779	* AM65_CPSW_RX_PSD_IS_FRAGMENT bit - indicates fragmented packet
780	* AM65_CPSW_RX_PSD_CSUM_ADD has value 0xFFFF for non fragmented packets
781	* or csum value for fragmented packets if !AM65_CPSW_RX_PSD_CSUM_ERR
782	*/
783	skb_checksum_none_assert(skb);
784
785	if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
786	return;
787
788	if ((csum_info & (AM65_CPSW_RX_PSD_IPV6_VALID |
789	AM65_CPSW_RX_PSD_IPV4_VALID)) &&
790	!(csum_info & AM65_CPSW_RX_PSD_CSUM_ERR)) {
791	/* csum for fragmented packets is unsupported */
792	if (!(csum_info & AM65_CPSW_RX_PSD_IS_FRAGMENT))
793	skb->ip_summed = CHECKSUM_UNNECESSARY;
794	}
795	}
796
797	static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
798	u32 flow_idx)
799	{
800	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
801	u32 buf_dma_len, pkt_len, port_id = 0, csum_info;
802	struct am65_cpsw_ndev_priv *ndev_priv;
803	struct am65_cpsw_ndev_stats *stats;
804	struct cppi5_host_desc_t *desc_rx;
805	struct device *dev = common->dev;
806	struct sk_buff *skb, *new_skb;
807	dma_addr_t desc_dma, buf_dma;
808	struct am65_cpsw_port *port;
809	struct net_device *ndev;
810	void **swdata;
811	u32 *psdata;
812	int ret = 0;
813
814	ret = k3_udma_glue_pop_rx_chn(rx_chn: rx_chn->rx_chn, flow_num: flow_idx, desc_dma: &desc_dma);
815	if (ret) {
816	if (ret != -ENODATA)
817	dev_err(dev, "RX: pop chn fail %d\n", ret);
818	return ret;
819	}
820
821	if (cppi5_desc_is_tdcm(paddr: desc_dma)) {
822	dev_dbg(dev, "%s RX tdown flow: %u\n", __func__, flow_idx);
823	if (common->pdata.quirks & AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ)
824	complete(&common->tdown_complete);
825	return 0;
826	}
827
828	desc_rx = k3_cppi_desc_pool_dma2virt(pool: rx_chn->desc_pool, dma: desc_dma);
829	dev_dbg(dev, "%s flow_idx: %u desc %pad\n",
830	__func__, flow_idx, &desc_dma);
831
832	swdata = cppi5_hdesc_get_swdata(desc: desc_rx);
833	skb = *swdata;
834	cppi5_hdesc_get_obuf(desc: desc_rx, obuf: &buf_dma, obuf_len: &buf_dma_len);
835	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn: rx_chn->rx_chn, addr: &buf_dma);
836	pkt_len = cppi5_hdesc_get_pktlen(desc: desc_rx);
837	cppi5_desc_get_tags_ids(desc_hdr: &desc_rx->hdr, src_tag_id: &port_id, NULL);
838	dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id);
839	port = am65_common_get_port(common, port_id);
840	ndev = port->ndev;
841	skb->dev = ndev;
842
843	psdata = cppi5_hdesc_get_psdata(desc: desc_rx);
844	/* add RX timestamp */
845	if (port->rx_ts_enabled)
846	am65_cpsw_nuss_rx_ts(skb, psdata);
847	csum_info = psdata[2];
848	dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info);
849
850	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
851
852	k3_cppi_desc_pool_free(pool: rx_chn->desc_pool, addr: desc_rx);
853
854	new_skb = netdev_alloc_skb_ip_align(dev: ndev, AM65_CPSW_MAX_PACKET_SIZE);
855	if (new_skb) {
856	ndev_priv = netdev_priv(dev: ndev);
857	am65_cpsw_nuss_set_offload_fwd_mark(skb, val: ndev_priv->offload_fwd_mark);
858	skb_put(skb, len: pkt_len);
859	skb->protocol = eth_type_trans(skb, dev: ndev);
860	am65_cpsw_nuss_rx_csum(skb, csum_info);
861	napi_gro_receive(napi: &common->napi_rx, skb);
862
863	stats = this_cpu_ptr(ndev_priv->stats);
864
865	u64_stats_update_begin(syncp: &stats->syncp);
866	stats->rx_packets++;
867	stats->rx_bytes += pkt_len;
868	u64_stats_update_end(syncp: &stats->syncp);
869	kmemleak_not_leak(ptr: new_skb);
870	} else {
871	ndev->stats.rx_dropped++;
872	new_skb = skb;
873	}
874
875	if (netif_dormant(dev: ndev)) {
876	dev_kfree_skb_any(skb: new_skb);
877	ndev->stats.rx_dropped++;
878	return 0;
879	}
880
881	ret = am65_cpsw_nuss_rx_push(common, skb: new_skb);
882	if (WARN_ON(ret < 0)) {
883	dev_kfree_skb_any(skb: new_skb);
884	ndev->stats.rx_errors++;
885	ndev->stats.rx_dropped++;
886	}
887
888	return ret;
889	}
890
891	static enum hrtimer_restart am65_cpsw_nuss_rx_timer_callback(struct hrtimer *timer)
892	{
893	struct am65_cpsw_common *common =
894	container_of(timer, struct am65_cpsw_common, rx_hrtimer);
895
896	enable_irq(irq: common->rx_chns.irq);
897	return HRTIMER_NORESTART;
898	}
899
900	static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
901	{
902	struct am65_cpsw_common *common = am65_cpsw_napi_to_common(napi_rx);
903	int flow = AM65_CPSW_MAX_RX_FLOWS;
904	int cur_budget, ret;
905	int num_rx = 0;
906
907	/* process every flow */
908	while (flow--) {
909	cur_budget = budget - num_rx;
910
911	while (cur_budget--) {
912	ret = am65_cpsw_nuss_rx_packets(common, flow_idx: flow);
913	if (ret)
914	break;
915	num_rx++;
916	}
917
918	if (num_rx >= budget)
919	break;
920	}
921
922	dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);
923
924	if (num_rx < budget && napi_complete_done(n: napi_rx, work_done: num_rx)) {
925	if (common->rx_irq_disabled) {
926	common->rx_irq_disabled = false;
927	if (unlikely(common->rx_pace_timeout)) {
928	hrtimer_start(timer: &common->rx_hrtimer,
929	tim: ns_to_ktime(ns: common->rx_pace_timeout),
930	mode: HRTIMER_MODE_REL_PINNED);
931	} else {
932	enable_irq(irq: common->rx_chns.irq);
933	}
934	}
935	}
936
937	return num_rx;
938	}
939
940	static struct sk_buff *
941	am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn,
942	dma_addr_t desc_dma)
943	{
944	struct am65_cpsw_ndev_priv *ndev_priv;
945	struct am65_cpsw_ndev_stats *stats;
946	struct cppi5_host_desc_t *desc_tx;
947	struct net_device *ndev;
948	struct sk_buff *skb;
949	void **swdata;
950
951	desc_tx = k3_cppi_desc_pool_dma2virt(pool: tx_chn->desc_pool,
952	dma: desc_dma);
953	swdata = cppi5_hdesc_get_swdata(desc: desc_tx);
954	skb = *(swdata);
955	am65_cpsw_nuss_xmit_free(tx_chn, desc: desc_tx);
956
957	ndev = skb->dev;
958
959	am65_cpts_tx_timestamp(cpts: tx_chn->common->cpts, skb);
960
961	ndev_priv = netdev_priv(dev: ndev);
962	stats = this_cpu_ptr(ndev_priv->stats);
963	u64_stats_update_begin(syncp: &stats->syncp);
964	stats->tx_packets++;
965	stats->tx_bytes += skb->len;
966	u64_stats_update_end(syncp: &stats->syncp);
967
968	return skb;
969	}
970
971	static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,
972	struct netdev_queue *netif_txq)
973	{
974	if (netif_tx_queue_stopped(dev_queue: netif_txq)) {
975	/* Check whether the queue is stopped due to stalled
976	* tx dma, if the queue is stopped then wake the queue
977	* as we have free desc for tx
978	*/
979	__netif_tx_lock(txq: netif_txq, smp_processor_id());
980	if (netif_running(dev: ndev) &&
981	(k3_cppi_desc_pool_avail(pool: tx_chn->desc_pool) >= MAX_SKB_FRAGS))
982	netif_tx_wake_queue(dev_queue: netif_txq);
983
984	__netif_tx_unlock(txq: netif_txq);
985	}
986	}
987
988	static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
989	int chn, unsigned int budget, bool *tdown)
990	{
991	struct device *dev = common->dev;
992	struct am65_cpsw_tx_chn *tx_chn;
993	struct netdev_queue *netif_txq;
994	unsigned int total_bytes = 0;
995	struct net_device *ndev;
996	struct sk_buff *skb;
997	dma_addr_t desc_dma;
998	int res, num_tx = 0;
999
1000	tx_chn = &common->tx_chns[chn];
1001
1002	while (true) {
1003	spin_lock(lock: &tx_chn->lock);
1004	res = k3_udma_glue_pop_tx_chn(tx_chn: tx_chn->tx_chn, desc_dma: &desc_dma);
1005	spin_unlock(lock: &tx_chn->lock);
1006	if (res == -ENODATA)
1007	break;
1008
1009	if (cppi5_desc_is_tdcm(paddr: desc_dma)) {
1010	if (atomic_dec_and_test(v: &common->tdown_cnt))
1011	complete(&common->tdown_complete);
1012	*tdown = true;
1013	break;
1014	}
1015
1016	skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);
1017	total_bytes = skb->len;
1018	ndev = skb->dev;
1019	napi_consume_skb(skb, budget);
1020	num_tx++;
1021
1022	netif_txq = netdev_get_tx_queue(dev: ndev, index: chn);
1023
1024	netdev_tx_completed_queue(dev_queue: netif_txq, pkts: num_tx, bytes: total_bytes);
1025
1026	am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1027	}
1028
1029	dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1030
1031	return num_tx;
1032	}
1033
1034	static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common,
1035	int chn, unsigned int budget, bool *tdown)
1036	{
1037	struct device *dev = common->dev;
1038	struct am65_cpsw_tx_chn *tx_chn;
1039	struct netdev_queue *netif_txq;
1040	unsigned int total_bytes = 0;
1041	struct net_device *ndev;
1042	struct sk_buff *skb;
1043	dma_addr_t desc_dma;
1044	int res, num_tx = 0;
1045
1046	tx_chn = &common->tx_chns[chn];
1047
1048	while (true) {
1049	res = k3_udma_glue_pop_tx_chn(tx_chn: tx_chn->tx_chn, desc_dma: &desc_dma);
1050	if (res == -ENODATA)
1051	break;
1052
1053	if (cppi5_desc_is_tdcm(paddr: desc_dma)) {
1054	if (atomic_dec_and_test(v: &common->tdown_cnt))
1055	complete(&common->tdown_complete);
1056	*tdown = true;
1057	break;
1058	}
1059
1060	skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);
1061
1062	ndev = skb->dev;
1063	total_bytes += skb->len;
1064	napi_consume_skb(skb, budget);
1065	num_tx++;
1066	}
1067
1068	if (!num_tx)
1069	return 0;
1070
1071	netif_txq = netdev_get_tx_queue(dev: ndev, index: chn);
1072
1073	netdev_tx_completed_queue(dev_queue: netif_txq, pkts: num_tx, bytes: total_bytes);
1074
1075	am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1076
1077	dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1078
1079	return num_tx;
1080	}
1081
1082	static enum hrtimer_restart am65_cpsw_nuss_tx_timer_callback(struct hrtimer *timer)
1083	{
1084	struct am65_cpsw_tx_chn *tx_chns =
1085	container_of(timer, struct am65_cpsw_tx_chn, tx_hrtimer);
1086
1087	enable_irq(irq: tx_chns->irq);
1088	return HRTIMER_NORESTART;
1089	}
1090
1091	static int am65_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget)
1092	{
1093	struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx);
1094	bool tdown = false;
1095	int num_tx;
1096
1097	if (AM65_CPSW_IS_CPSW2G(tx_chn->common))
1098	num_tx = am65_cpsw_nuss_tx_compl_packets_2g(common: tx_chn->common, chn: tx_chn->id,
1099	budget, tdown: &tdown);
1100	else
1101	num_tx = am65_cpsw_nuss_tx_compl_packets(common: tx_chn->common,
1102	chn: tx_chn->id, budget, tdown: &tdown);
1103
1104	if (num_tx >= budget)
1105	return budget;
1106
1107	if (napi_complete_done(n: napi_tx, work_done: num_tx)) {
1108	if (unlikely(tx_chn->tx_pace_timeout && !tdown)) {
1109	hrtimer_start(timer: &tx_chn->tx_hrtimer,
1110	tim: ns_to_ktime(ns: tx_chn->tx_pace_timeout),
1111	mode: HRTIMER_MODE_REL_PINNED);
1112	} else {
1113	enable_irq(irq: tx_chn->irq);
1114	}
1115	}
1116
1117	return 0;
1118	}
1119
1120	static irqreturn_t am65_cpsw_nuss_rx_irq(int irq, void *dev_id)
1121	{
1122	struct am65_cpsw_common *common = dev_id;
1123
1124	common->rx_irq_disabled = true;
1125	disable_irq_nosync(irq);
1126	napi_schedule(n: &common->napi_rx);
1127
1128	return IRQ_HANDLED;
1129	}
1130
1131	static irqreturn_t am65_cpsw_nuss_tx_irq(int irq, void *dev_id)
1132	{
1133	struct am65_cpsw_tx_chn *tx_chn = dev_id;
1134
1135	disable_irq_nosync(irq);
1136	napi_schedule(n: &tx_chn->napi_tx);
1137
1138	return IRQ_HANDLED;
1139	}
1140
1141	static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
1142	struct net_device *ndev)
1143	{
1144	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1145	struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
1146	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1147	struct device *dev = common->dev;
1148	struct am65_cpsw_tx_chn *tx_chn;
1149	struct netdev_queue *netif_txq;
1150	dma_addr_t desc_dma, buf_dma;
1151	int ret, q_idx, i;
1152	void **swdata;
1153	u32 *psdata;
1154	u32 pkt_len;
1155
1156	/* padding enabled in hw */
1157	pkt_len = skb_headlen(skb);
1158
1159	/* SKB TX timestamp */
1160	if (port->tx_ts_enabled)
1161	am65_cpts_prep_tx_timestamp(cpts: common->cpts, skb);
1162
1163	q_idx = skb_get_queue_mapping(skb);
1164	dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx);
1165
1166	tx_chn = &common->tx_chns[q_idx];
1167	netif_txq = netdev_get_tx_queue(dev: ndev, index: q_idx);
1168
1169	/* Map the linear buffer */
1170	buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len,
1171	DMA_TO_DEVICE);
1172	if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1173	dev_err(dev, "Failed to map tx skb buffer\n");
1174	ndev->stats.tx_errors++;
1175	goto err_free_skb;
1176	}
1177
1178	first_desc = k3_cppi_desc_pool_alloc(pool: tx_chn->desc_pool);
1179	if (!first_desc) {
1180	dev_dbg(dev, "Failed to allocate descriptor\n");
1181	dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len,
1182	DMA_TO_DEVICE);
1183	goto busy_stop_q;
1184	}
1185
1186	cppi5_hdesc_init(desc: first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
1187	AM65_CPSW_NAV_PS_DATA_SIZE);
1188	cppi5_desc_set_pktids(desc_hdr: &first_desc->hdr, pkt_id: 0, flow_id: 0x3FFF);
1189	cppi5_hdesc_set_pkttype(desc: first_desc, pkt_type: 0x7);
1190	cppi5_desc_set_tags_ids(desc_hdr: &first_desc->hdr, src_tag_id: 0, dst_tag_id: port->port_id);
1191
1192	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn: tx_chn->tx_chn, addr: &buf_dma);
1193	cppi5_hdesc_attach_buf(desc: first_desc, buf: buf_dma, buf_data_len: pkt_len, obuf: buf_dma, obuf_len: pkt_len);
1194	swdata = cppi5_hdesc_get_swdata(desc: first_desc);
1195	*(swdata) = skb;
1196	psdata = cppi5_hdesc_get_psdata(desc: first_desc);
1197
1198	/* HW csum offload if enabled */
1199	psdata[2] = 0;
1200	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1201	unsigned int cs_start, cs_offset;
1202
1203	cs_start = skb_transport_offset(skb);
1204	cs_offset = cs_start + skb->csum_offset;
1205	/* HW numerates bytes starting from 1 */
1206	psdata[2] = ((cs_offset + 1) << 24) |
1207	((cs_start + 1) << 16) | (skb->len - cs_start);
1208	dev_dbg(dev, "%s tx psdata:%#x\n", __func__, psdata[2]);
1209	}
1210
1211	if (!skb_is_nonlinear(skb))
1212	goto done_tx;
1213
1214	dev_dbg(dev, "fragmented SKB\n");
1215
1216	/* Handle the case where skb is fragmented in pages */
1217	cur_desc = first_desc;
1218	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1219	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1220	u32 frag_size = skb_frag_size(frag);
1221
1222	next_desc = k3_cppi_desc_pool_alloc(pool: tx_chn->desc_pool);
1223	if (!next_desc) {
1224	dev_err(dev, "Failed to allocate descriptor\n");
1225	goto busy_free_descs;
1226	}
1227
1228	buf_dma = skb_frag_dma_map(dev: tx_chn->dma_dev, frag, offset: 0, size: frag_size,
1229	dir: DMA_TO_DEVICE);
1230	if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1231	dev_err(dev, "Failed to map tx skb page\n");
1232	k3_cppi_desc_pool_free(pool: tx_chn->desc_pool, addr: next_desc);
1233	ndev->stats.tx_errors++;
1234	goto err_free_descs;
1235	}
1236
1237	cppi5_hdesc_reset_hbdesc(desc: next_desc);
1238	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn: tx_chn->tx_chn, addr: &buf_dma);
1239	cppi5_hdesc_attach_buf(desc: next_desc,
1240	buf: buf_dma, buf_data_len: frag_size, obuf: buf_dma, obuf_len: frag_size);
1241
1242	desc_dma = k3_cppi_desc_pool_virt2dma(pool: tx_chn->desc_pool,
1243	addr: next_desc);
1244	k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn: tx_chn->tx_chn, addr: &desc_dma);
1245	cppi5_hdesc_link_hbdesc(desc: cur_desc, hbuf_desc: desc_dma);
1246
1247	pkt_len += frag_size;
1248	cur_desc = next_desc;
1249	}
1250	WARN_ON(pkt_len != skb->len);
1251
1252	done_tx:
1253	skb_tx_timestamp(skb);
1254
1255	/* report bql before sending packet */
1256	netdev_tx_sent_queue(dev_queue: netif_txq, bytes: pkt_len);
1257
1258	cppi5_hdesc_set_pktlen(desc: first_desc, pkt_len);
1259	desc_dma = k3_cppi_desc_pool_virt2dma(pool: tx_chn->desc_pool, addr: first_desc);
1260	if (AM65_CPSW_IS_CPSW2G(common)) {
1261	ret = k3_udma_glue_push_tx_chn(tx_chn: tx_chn->tx_chn, desc_tx: first_desc, desc_dma);
1262	} else {
1263	spin_lock_bh(lock: &tx_chn->lock);
1264	ret = k3_udma_glue_push_tx_chn(tx_chn: tx_chn->tx_chn, desc_tx: first_desc, desc_dma);
1265	spin_unlock_bh(lock: &tx_chn->lock);
1266	}
1267	if (ret) {
1268	dev_err(dev, "can't push desc %d\n", ret);
1269	/* inform bql */
1270	netdev_tx_completed_queue(dev_queue: netif_txq, pkts: 1, bytes: pkt_len);
1271	ndev->stats.tx_errors++;
1272	goto err_free_descs;
1273	}
1274
1275	if (k3_cppi_desc_pool_avail(pool: tx_chn->desc_pool) < MAX_SKB_FRAGS) {
1276	netif_tx_stop_queue(dev_queue: netif_txq);
1277	/* Barrier, so that stop_queue visible to other cpus */
1278	smp_mb__after_atomic();
1279	dev_dbg(dev, "netif_tx_stop_queue %d\n", q_idx);
1280
1281	/* re-check for smp */
1282	if (k3_cppi_desc_pool_avail(pool: tx_chn->desc_pool) >=
1283	MAX_SKB_FRAGS) {
1284	netif_tx_wake_queue(dev_queue: netif_txq);
1285	dev_dbg(dev, "netif_tx_wake_queue %d\n", q_idx);
1286	}
1287	}
1288
1289	return NETDEV_TX_OK;
1290
1291	err_free_descs:
1292	am65_cpsw_nuss_xmit_free(tx_chn, desc: first_desc);
1293	err_free_skb:
1294	ndev->stats.tx_dropped++;
1295	dev_kfree_skb_any(skb);
1296	return NETDEV_TX_OK;
1297
1298	busy_free_descs:
1299	am65_cpsw_nuss_xmit_free(tx_chn, desc: first_desc);
1300	busy_stop_q:
1301	netif_tx_stop_queue(dev_queue: netif_txq);
1302	return NETDEV_TX_BUSY;
1303	}
1304
1305	static int am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device *ndev,
1306	void *addr)
1307	{
1308	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1309	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1310	struct sockaddr *sockaddr = (struct sockaddr *)addr;
1311	int ret;
1312
1313	ret = eth_prepare_mac_addr_change(dev: ndev, p: addr);
1314	if (ret < 0)
1315	return ret;
1316
1317	ret = pm_runtime_resume_and_get(dev: common->dev);
1318	if (ret < 0)
1319	return ret;
1320
1321	cpsw_ale_del_ucast(ale: common->ale, addr: ndev->dev_addr,
1322	HOST_PORT_NUM, flags: 0, vid: 0);
1323	cpsw_ale_add_ucast(ale: common->ale, addr: sockaddr->sa_data,
1324	HOST_PORT_NUM, ALE_SECURE, vid: 0);
1325
1326	am65_cpsw_port_set_sl_mac(slave: port, dev_addr: addr);
1327	eth_commit_mac_addr_change(dev: ndev, p: sockaddr);
1328
1329	pm_runtime_put(dev: common->dev);
1330
1331	return 0;
1332	}
1333
1334	static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev,
1335	struct ifreq *ifr)
1336	{
1337	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1338	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1339	u32 ts_ctrl, seq_id, ts_ctrl_ltype2, ts_vlan_ltype;
1340	struct hwtstamp_config cfg;
1341
1342	if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1343	return -EOPNOTSUPP;
1344
1345	if (copy_from_user(to: &cfg, from: ifr->ifr_data, n: sizeof(cfg)))
1346	return -EFAULT;
1347
1348	/* TX HW timestamp */
1349	switch (cfg.tx_type) {
1350	case HWTSTAMP_TX_OFF:
1351	case HWTSTAMP_TX_ON:
1352	break;
1353	default:
1354	return -ERANGE;
1355	}
1356
1357	switch (cfg.rx_filter) {
1358	case HWTSTAMP_FILTER_NONE:
1359	port->rx_ts_enabled = false;
1360	break;
1361	case HWTSTAMP_FILTER_ALL:
1362	case HWTSTAMP_FILTER_SOME:
1363	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1364	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1365	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1366	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1367	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1368	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1369	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1370	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1371	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1372	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1373	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1374	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1375	case HWTSTAMP_FILTER_NTP_ALL:
1376	port->rx_ts_enabled = true;
1377	cfg.rx_filter = HWTSTAMP_FILTER_ALL;
1378	break;
1379	default:
1380	return -ERANGE;
1381	}
1382
1383	port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON);
1384
1385	/* cfg TX timestamp */
1386	seq_id = (AM65_CPSW_TS_SEQ_ID_OFFSET <<
1387	AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT) | ETH_P_1588;
1388
1389	ts_vlan_ltype = ETH_P_8021Q;
1390
1391	ts_ctrl_ltype2 = ETH_P_1588 |
1392	AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 |
1393	AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 |
1394	AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 |
1395	AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 |
1396	AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 |
1397	AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 |
1398	AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 |
1399	AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO;
1400
1401	ts_ctrl = AM65_CPSW_TS_EVENT_MSG_TYPE_BITS <<
1402	AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT;
1403
1404	if (port->tx_ts_enabled)
1405	ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN |
1406	AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN;
1407
1408	writel(val: seq_id, addr: port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG);
1409	writel(val: ts_vlan_ltype, addr: port->port_base +
1410	AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG);
1411	writel(val: ts_ctrl_ltype2, addr: port->port_base +
1412	AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2);
1413	writel(val: ts_ctrl, addr: port->port_base + AM65_CPSW_PORTN_REG_TS_CTL);
1414
1415	/* en/dis RX timestamp */
1416	am65_cpts_rx_enable(cpts: common->cpts, en: port->rx_ts_enabled);
1417
1418	return copy_to_user(to: ifr->ifr_data, from: &cfg, n: sizeof(cfg)) ? -EFAULT : 0;
1419	}
1420
1421	static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev,
1422	struct ifreq *ifr)
1423	{
1424	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1425	struct hwtstamp_config cfg;
1426
1427	if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1428	return -EOPNOTSUPP;
1429
1430	cfg.flags = 0;
1431	cfg.tx_type = port->tx_ts_enabled ?
1432	HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1433	cfg.rx_filter = port->rx_ts_enabled ?
1434	HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1435
1436	return copy_to_user(to: ifr->ifr_data, from: &cfg, n: sizeof(cfg)) ? -EFAULT : 0;
1437	}
1438
1439	static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev,
1440	struct ifreq *req, int cmd)
1441	{
1442	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1443
1444	if (!netif_running(dev: ndev))
1445	return -EINVAL;
1446
1447	switch (cmd) {
1448	case SIOCSHWTSTAMP:
1449	return am65_cpsw_nuss_hwtstamp_set(ndev, ifr: req);
1450	case SIOCGHWTSTAMP:
1451	return am65_cpsw_nuss_hwtstamp_get(ndev, ifr: req);
1452	}
1453
1454	return phylink_mii_ioctl(port->slave.phylink, req, cmd);
1455	}
1456
1457	static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev,
1458	struct rtnl_link_stats64 *stats)
1459	{
1460	struct am65_cpsw_ndev_priv *ndev_priv = netdev_priv(dev);
1461	unsigned int start;
1462	int cpu;
1463
1464	for_each_possible_cpu(cpu) {
1465	struct am65_cpsw_ndev_stats *cpu_stats;
1466	u64 rx_packets;
1467	u64 rx_bytes;
1468	u64 tx_packets;
1469	u64 tx_bytes;
1470
1471	cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu);
1472	do {
1473	start = u64_stats_fetch_begin(syncp: &cpu_stats->syncp);
1474	rx_packets = cpu_stats->rx_packets;
1475	rx_bytes = cpu_stats->rx_bytes;
1476	tx_packets = cpu_stats->tx_packets;
1477	tx_bytes = cpu_stats->tx_bytes;
1478	} while (u64_stats_fetch_retry(syncp: &cpu_stats->syncp, start));
1479
1480	stats->rx_packets += rx_packets;
1481	stats->rx_bytes += rx_bytes;
1482	stats->tx_packets += tx_packets;
1483	stats->tx_bytes += tx_bytes;
1484	}
1485
1486	stats->rx_errors = dev->stats.rx_errors;
1487	stats->rx_dropped = dev->stats.rx_dropped;
1488	stats->tx_dropped = dev->stats.tx_dropped;
1489	}
1490
1491	static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
1492	.ndo_open = am65_cpsw_nuss_ndo_slave_open,
1493	.ndo_stop = am65_cpsw_nuss_ndo_slave_stop,
1494	.ndo_start_xmit = am65_cpsw_nuss_ndo_slave_xmit,
1495	.ndo_set_rx_mode = am65_cpsw_nuss_ndo_slave_set_rx_mode,
1496	.ndo_get_stats64 = am65_cpsw_nuss_ndo_get_stats,
1497	.ndo_validate_addr = eth_validate_addr,
1498	.ndo_set_mac_address = am65_cpsw_nuss_ndo_slave_set_mac_address,
1499	.ndo_tx_timeout = am65_cpsw_nuss_ndo_host_tx_timeout,
1500	.ndo_vlan_rx_add_vid = am65_cpsw_nuss_ndo_slave_add_vid,
1501	.ndo_vlan_rx_kill_vid = am65_cpsw_nuss_ndo_slave_kill_vid,
1502	.ndo_eth_ioctl = am65_cpsw_nuss_ndo_slave_ioctl,
1503	.ndo_setup_tc = am65_cpsw_qos_ndo_setup_tc,
1504	.ndo_set_tx_maxrate = am65_cpsw_qos_ndo_tx_p0_set_maxrate,
1505	};
1506
1507	static void am65_cpsw_disable_phy(struct phy *phy)
1508	{
1509	phy_power_off(phy);
1510	phy_exit(phy);
1511	}
1512
1513	static int am65_cpsw_enable_phy(struct phy *phy)
1514	{
1515	int ret;
1516
1517	ret = phy_init(phy);
1518	if (ret < 0)
1519	return ret;
1520
1521	ret = phy_power_on(phy);
1522	if (ret < 0) {
1523	phy_exit(phy);
1524	return ret;
1525	}
1526
1527	return 0;
1528	}
1529
1530	static void am65_cpsw_disable_serdes_phy(struct am65_cpsw_common *common)
1531	{
1532	struct am65_cpsw_port *port;
1533	struct phy *phy;
1534	int i;
1535
1536	for (i = 0; i < common->port_num; i++) {
1537	port = &common->ports[i];
1538	phy = port->slave.serdes_phy;
1539	if (phy)
1540	am65_cpsw_disable_phy(phy);
1541	}
1542	}
1543
1544	static int am65_cpsw_init_serdes_phy(struct device *dev, struct device_node *port_np,
1545	struct am65_cpsw_port *port)
1546	{
1547	const char *name = "serdes";
1548	struct phy *phy;
1549	int ret;
1550
1551	phy = devm_of_phy_optional_get(dev, np: port_np, con_id: name);
1552	if (IS_ERR_OR_NULL(ptr: phy))
1553	return PTR_ERR_OR_ZERO(ptr: phy);
1554
1555	/* Serdes PHY exists. Store it. */
1556	port->slave.serdes_phy = phy;
1557
1558	ret = am65_cpsw_enable_phy(phy);
1559	if (ret < 0)
1560	goto err_phy;
1561
1562	return 0;
1563
1564	err_phy:
1565	devm_phy_put(dev, phy);
1566	return ret;
1567	}
1568
1569	static void am65_cpsw_nuss_mac_config(struct phylink_config *config, unsigned int mode,
1570	const struct phylink_link_state *state)
1571	{
1572	struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
1573	phylink_config);
1574	struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
1575	struct am65_cpsw_common *common = port->common;
1576
1577	if (common->pdata.extra_modes & BIT(state->interface)) {
1578	if (state->interface == PHY_INTERFACE_MODE_SGMII) {
1579	writel(ADVERTISE_SGMII,
1580	addr: port->sgmii_base + AM65_CPSW_SGMII_MR_ADV_ABILITY_REG);
1581	cpsw_sl_ctl_set(sl: port->slave.mac_sl, ctl_funcs: CPSW_SL_CTL_EXT_EN);
1582	} else {
1583	cpsw_sl_ctl_clr(sl: port->slave.mac_sl, ctl_funcs: CPSW_SL_CTL_EXT_EN);
1584	}
1585
1586	if (state->interface == PHY_INTERFACE_MODE_USXGMII) {
1587	cpsw_sl_ctl_set(sl: port->slave.mac_sl,
1588	ctl_funcs: CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
1589	} else {
1590	cpsw_sl_ctl_clr(sl: port->slave.mac_sl,
1591	ctl_funcs: CPSW_SL_CTL_XGIG | CPSW_SL_CTL_XGMII_EN);
1592	}
1593
1594	writel(AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE,
1595	addr: port->sgmii_base + AM65_CPSW_SGMII_CONTROL_REG);
1596	}
1597	}
1598
1599	static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned int mode,
1600	phy_interface_t interface)
1601	{
1602	struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
1603	phylink_config);
1604	struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
1605	struct am65_cpsw_common *common = port->common;
1606	struct net_device *ndev = port->ndev;
1607	u32 mac_control;
1608	int tmo;
1609
1610	/* disable forwarding */
1611	cpsw_ale_control_set(ale: common->ale, port: port->port_id, control: ALE_PORT_STATE, value: ALE_PORT_STATE_DISABLE);
1612
1613	cpsw_sl_ctl_set(sl: port->slave.mac_sl, ctl_funcs: CPSW_SL_CTL_CMD_IDLE);
1614
1615	tmo = cpsw_sl_wait_for_idle(sl: port->slave.mac_sl, tmo: 100);
1616	dev_dbg(common->dev, "down msc_sl %08x tmo %d\n",
1617	cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS), tmo);
1618
1619	/* All the bits that am65_cpsw_nuss_mac_link_up() can possibly set */
1620	mac_control = CPSW_SL_CTL_GMII_EN | CPSW_SL_CTL_GIG | CPSW_SL_CTL_IFCTL_A |
1621	CPSW_SL_CTL_FULLDUPLEX | CPSW_SL_CTL_RX_FLOW_EN | CPSW_SL_CTL_TX_FLOW_EN;
1622	/* If interface mode is RGMII, CPSW_SL_CTL_EXT_EN might have been set for 10 Mbps */
1623	if (phy_interface_mode_is_rgmii(mode: interface))
1624	mac_control |= CPSW_SL_CTL_EXT_EN;
1625	/* Only clear those bits that can be set by am65_cpsw_nuss_mac_link_up() */
1626	cpsw_sl_ctl_clr(sl: port->slave.mac_sl, ctl_funcs: mac_control);
1627
1628	am65_cpsw_qos_link_down(ndev);
1629	netif_tx_stop_all_queues(dev: ndev);
1630	}
1631
1632	static void am65_cpsw_nuss_mac_link_up(struct phylink_config *config, struct phy_device *phy,
1633	unsigned int mode, phy_interface_t interface, int speed,
1634	int duplex, bool tx_pause, bool rx_pause)
1635	{
1636	struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
1637	phylink_config);
1638	struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
1639	struct am65_cpsw_common *common = port->common;
1640	u32 mac_control = CPSW_SL_CTL_GMII_EN;
1641	struct net_device *ndev = port->ndev;
1642
1643	/* Bring the port out of idle state */
1644	cpsw_sl_ctl_clr(sl: port->slave.mac_sl, ctl_funcs: CPSW_SL_CTL_CMD_IDLE);
1645
1646	if (speed == SPEED_1000)
1647	mac_control |= CPSW_SL_CTL_GIG;
1648	/* TODO: Verify whether in-band is necessary for 10 Mbps RGMII */
1649	if (speed == SPEED_10 && phy_interface_mode_is_rgmii(mode: interface))
1650	/* Can be used with in band mode only */
1651	mac_control |= CPSW_SL_CTL_EXT_EN;
1652	if (speed == SPEED_100 && interface == PHY_INTERFACE_MODE_RMII)
1653	mac_control |= CPSW_SL_CTL_IFCTL_A;
1654	if (duplex)
1655	mac_control |= CPSW_SL_CTL_FULLDUPLEX;
1656
1657	/* rx_pause/tx_pause */
1658	if (rx_pause)
1659	mac_control |= CPSW_SL_CTL_TX_FLOW_EN;
1660
1661	if (tx_pause)
1662	mac_control |= CPSW_SL_CTL_RX_FLOW_EN;
1663
1664	cpsw_sl_ctl_set(sl: port->slave.mac_sl, ctl_funcs: mac_control);
1665
1666	/* enable forwarding */
1667	cpsw_ale_control_set(ale: common->ale, port: port->port_id, control: ALE_PORT_STATE, value: ALE_PORT_STATE_FORWARD);
1668
1669	am65_cpsw_qos_link_up(ndev, link_speed: speed);
1670	netif_tx_wake_all_queues(dev: ndev);
1671	}
1672
1673	static const struct phylink_mac_ops am65_cpsw_phylink_mac_ops = {
1674	.mac_config = am65_cpsw_nuss_mac_config,
1675	.mac_link_down = am65_cpsw_nuss_mac_link_down,
1676	.mac_link_up = am65_cpsw_nuss_mac_link_up,
1677	};
1678
1679	static void am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port *port)
1680	{
1681	struct am65_cpsw_common *common = port->common;
1682
1683	if (!port->disabled)
1684	return;
1685
1686	cpsw_ale_control_set(ale: common->ale, port: port->port_id,
1687	control: ALE_PORT_STATE, value: ALE_PORT_STATE_DISABLE);
1688
1689	cpsw_sl_reset(sl: port->slave.mac_sl, tmo: 100);
1690	cpsw_sl_ctl_reset(sl: port->slave.mac_sl);
1691	}
1692
1693	static void am65_cpsw_nuss_free_tx_chns(void *data)
1694	{
1695	struct am65_cpsw_common *common = data;
1696	int i;
1697
1698	for (i = 0; i < common->tx_ch_num; i++) {
1699	struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1700
1701	if (!IS_ERR_OR_NULL(ptr: tx_chn->desc_pool))
1702	k3_cppi_desc_pool_destroy(pool: tx_chn->desc_pool);
1703
1704	if (!IS_ERR_OR_NULL(ptr: tx_chn->tx_chn))
1705	k3_udma_glue_release_tx_chn(tx_chn: tx_chn->tx_chn);
1706
1707	memset(tx_chn, 0, sizeof(*tx_chn));
1708	}
1709	}
1710
1711	void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common)
1712	{
1713	struct device *dev = common->dev;
1714	int i;
1715
1716	devm_remove_action(dev, action: am65_cpsw_nuss_free_tx_chns, data: common);
1717
1718	common->tx_ch_rate_msk = 0;
1719	for (i = 0; i < common->tx_ch_num; i++) {
1720	struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1721
1722	if (tx_chn->irq)
1723	devm_free_irq(dev, irq: tx_chn->irq, dev_id: tx_chn);
1724
1725	netif_napi_del(napi: &tx_chn->napi_tx);
1726
1727	if (!IS_ERR_OR_NULL(ptr: tx_chn->desc_pool))
1728	k3_cppi_desc_pool_destroy(pool: tx_chn->desc_pool);
1729
1730	if (!IS_ERR_OR_NULL(ptr: tx_chn->tx_chn))
1731	k3_udma_glue_release_tx_chn(tx_chn: tx_chn->tx_chn);
1732
1733	memset(tx_chn, 0, sizeof(*tx_chn));
1734	}
1735	}
1736
1737	static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common)
1738	{
1739	struct device *dev = common->dev;
1740	int i, ret = 0;
1741
1742	for (i = 0; i < common->tx_ch_num; i++) {
1743	struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1744
1745	netif_napi_add_tx(dev: common->dma_ndev, napi: &tx_chn->napi_tx,
1746	poll: am65_cpsw_nuss_tx_poll);
1747	hrtimer_init(timer: &tx_chn->tx_hrtimer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL_PINNED);
1748	tx_chn->tx_hrtimer.function = &am65_cpsw_nuss_tx_timer_callback;
1749
1750	ret = devm_request_irq(dev, irq: tx_chn->irq,
1751	handler: am65_cpsw_nuss_tx_irq,
1752	IRQF_TRIGGER_HIGH,
1753	devname: tx_chn->tx_chn_name, dev_id: tx_chn);
1754	if (ret) {
1755	dev_err(dev, "failure requesting tx%u irq %u, %d\n",
1756	tx_chn->id, tx_chn->irq, ret);
1757	goto err;
1758	}
1759	}
1760
1761	err:
1762	return ret;
1763	}
1764
1765	static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common)
1766	{
1767	u32 max_desc_num = ALIGN(AM65_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS);
1768	struct k3_udma_glue_tx_channel_cfg tx_cfg = { 0 };
1769	struct device *dev = common->dev;
1770	struct k3_ring_cfg ring_cfg = {
1771	.elm_size = K3_RINGACC_RING_ELSIZE_8,
1772	.mode = K3_RINGACC_RING_MODE_RING,
1773	.flags = 0
1774	};
1775	u32 hdesc_size;
1776	int i, ret = 0;
1777
1778	hdesc_size = cppi5_hdesc_calc_size(epib: true, AM65_CPSW_NAV_PS_DATA_SIZE,
1779	AM65_CPSW_NAV_SW_DATA_SIZE);
1780
1781	tx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
1782	tx_cfg.tx_cfg = ring_cfg;
1783	tx_cfg.txcq_cfg = ring_cfg;
1784	tx_cfg.tx_cfg.size = max_desc_num;
1785	tx_cfg.txcq_cfg.size = max_desc_num;
1786
1787	for (i = 0; i < common->tx_ch_num; i++) {
1788	struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1789
1790	snprintf(buf: tx_chn->tx_chn_name,
1791	size: sizeof(tx_chn->tx_chn_name), fmt: "tx%d", i);
1792
1793	spin_lock_init(&tx_chn->lock);
1794	tx_chn->common = common;
1795	tx_chn->id = i;
1796	tx_chn->descs_num = max_desc_num;
1797
1798	tx_chn->tx_chn =
1799	k3_udma_glue_request_tx_chn(dev,
1800	name: tx_chn->tx_chn_name,
1801	cfg: &tx_cfg);
1802	if (IS_ERR(ptr: tx_chn->tx_chn)) {
1803	ret = dev_err_probe(dev, err: PTR_ERR(ptr: tx_chn->tx_chn),
1804	fmt: "Failed to request tx dma channel\n");
1805	goto err;
1806	}
1807	tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn: tx_chn->tx_chn);
1808
1809	tx_chn->desc_pool = k3_cppi_desc_pool_create_name(dev: tx_chn->dma_dev,
1810	size: tx_chn->descs_num,
1811	desc_size: hdesc_size,
1812	name: tx_chn->tx_chn_name);
1813	if (IS_ERR(ptr: tx_chn->desc_pool)) {
1814	ret = PTR_ERR(ptr: tx_chn->desc_pool);
1815	dev_err(dev, "Failed to create poll %d\n", ret);
1816	goto err;
1817	}
1818
1819	tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn: tx_chn->tx_chn);
1820	if (tx_chn->irq < 0) {
1821	dev_err(dev, "Failed to get tx dma irq %d\n",
1822	tx_chn->irq);
1823	ret = tx_chn->irq;
1824	goto err;
1825	}
1826
1827	snprintf(buf: tx_chn->tx_chn_name,
1828	size: sizeof(tx_chn->tx_chn_name), fmt: "%s-tx%d",
1829	dev_name(dev), tx_chn->id);
1830	}
1831
1832	ret = am65_cpsw_nuss_ndev_add_tx_napi(common);
1833	if (ret) {
1834	dev_err(dev, "Failed to add tx NAPI %d\n", ret);
1835	goto err;
1836	}
1837
1838	err:
1839	i = devm_add_action(dev, am65_cpsw_nuss_free_tx_chns, common);
1840	if (i) {
1841	dev_err(dev, "Failed to add free_tx_chns action %d\n", i);
1842	return i;
1843	}
1844
1845	return ret;
1846	}
1847
1848	static void am65_cpsw_nuss_free_rx_chns(void *data)
1849	{
1850	struct am65_cpsw_common *common = data;
1851	struct am65_cpsw_rx_chn *rx_chn;
1852
1853	rx_chn = &common->rx_chns;
1854
1855	if (!IS_ERR_OR_NULL(ptr: rx_chn->desc_pool))
1856	k3_cppi_desc_pool_destroy(pool: rx_chn->desc_pool);
1857
1858	if (!IS_ERR_OR_NULL(ptr: rx_chn->rx_chn))
1859	k3_udma_glue_release_rx_chn(rx_chn: rx_chn->rx_chn);
1860	}
1861
1862	static void am65_cpsw_nuss_remove_rx_chns(void *data)
1863	{
1864	struct am65_cpsw_common *common = data;
1865	struct am65_cpsw_rx_chn *rx_chn;
1866	struct device *dev = common->dev;
1867
1868	rx_chn = &common->rx_chns;
1869	devm_remove_action(dev, action: am65_cpsw_nuss_free_rx_chns, data: common);
1870
1871	if (!(rx_chn->irq < 0))
1872	devm_free_irq(dev, irq: rx_chn->irq, dev_id: common);
1873
1874	netif_napi_del(napi: &common->napi_rx);
1875
1876	if (!IS_ERR_OR_NULL(ptr: rx_chn->desc_pool))
1877	k3_cppi_desc_pool_destroy(pool: rx_chn->desc_pool);
1878
1879	if (!IS_ERR_OR_NULL(ptr: rx_chn->rx_chn))
1880	k3_udma_glue_release_rx_chn(rx_chn: rx_chn->rx_chn);
1881
1882	common->rx_flow_id_base = -1;
1883	}
1884
1885	static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common)
1886	{
1887	struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
1888	struct k3_udma_glue_rx_channel_cfg rx_cfg = { 0 };
1889	u32 max_desc_num = AM65_CPSW_MAX_RX_DESC;
1890	struct device *dev = common->dev;
1891	u32 hdesc_size;
1892	u32 fdqring_id;
1893	int i, ret = 0;
1894
1895	hdesc_size = cppi5_hdesc_calc_size(epib: true, AM65_CPSW_NAV_PS_DATA_SIZE,
1896	AM65_CPSW_NAV_SW_DATA_SIZE);
1897
1898	rx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
1899	rx_cfg.flow_id_num = AM65_CPSW_MAX_RX_FLOWS;
1900	rx_cfg.flow_id_base = common->rx_flow_id_base;
1901
1902	/* init all flows */
1903	rx_chn->dev = dev;
1904	rx_chn->descs_num = max_desc_num;
1905
1906	rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, name: "rx", cfg: &rx_cfg);
1907	if (IS_ERR(ptr: rx_chn->rx_chn)) {
1908	ret = dev_err_probe(dev, err: PTR_ERR(ptr: rx_chn->rx_chn),
1909	fmt: "Failed to request rx dma channel\n");
1910	goto err;
1911	}
1912	rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn: rx_chn->rx_chn);
1913
1914	rx_chn->desc_pool = k3_cppi_desc_pool_create_name(dev: rx_chn->dma_dev,
1915	size: rx_chn->descs_num,
1916	desc_size: hdesc_size, name: "rx");
1917	if (IS_ERR(ptr: rx_chn->desc_pool)) {
1918	ret = PTR_ERR(ptr: rx_chn->desc_pool);
1919	dev_err(dev, "Failed to create rx poll %d\n", ret);
1920	goto err;
1921	}
1922
1923	common->rx_flow_id_base =
1924	k3_udma_glue_rx_get_flow_id_base(rx_chn: rx_chn->rx_chn);
1925	dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base);
1926
1927	fdqring_id = K3_RINGACC_RING_ID_ANY;
1928	for (i = 0; i < rx_cfg.flow_id_num; i++) {
1929	struct k3_ring_cfg rxring_cfg = {
1930	.elm_size = K3_RINGACC_RING_ELSIZE_8,
1931	.mode = K3_RINGACC_RING_MODE_RING,
1932	.flags = 0,
1933	};
1934	struct k3_ring_cfg fdqring_cfg = {
1935	.elm_size = K3_RINGACC_RING_ELSIZE_8,
1936	.flags = K3_RINGACC_RING_SHARED,
1937	};
1938	struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = {
1939	.rx_cfg = rxring_cfg,
1940	.rxfdq_cfg = fdqring_cfg,
1941	.ring_rxq_id = K3_RINGACC_RING_ID_ANY,
1942	.src_tag_lo_sel =
1943	K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
1944	};
1945
1946	rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
1947	rx_flow_cfg.rx_cfg.size = max_desc_num;
1948	rx_flow_cfg.rxfdq_cfg.size = max_desc_num;
1949	rx_flow_cfg.rxfdq_cfg.mode = common->pdata.fdqring_mode;
1950
1951	ret = k3_udma_glue_rx_flow_init(rx_chn: rx_chn->rx_chn,
1952	flow_idx: i, flow_cfg: &rx_flow_cfg);
1953	if (ret) {
1954	dev_err(dev, "Failed to init rx flow%d %d\n", i, ret);
1955	goto err;
1956	}
1957	if (!i)
1958	fdqring_id =
1959	k3_udma_glue_rx_flow_get_fdq_id(rx_chn: rx_chn->rx_chn,
1960	flow_idx: i);
1961
1962	rx_chn->irq = k3_udma_glue_rx_get_irq(rx_chn: rx_chn->rx_chn, flow_num: i);
1963
1964	if (rx_chn->irq <= 0) {
1965	dev_err(dev, "Failed to get rx dma irq %d\n",
1966	rx_chn->irq);
1967	ret = -ENXIO;
1968	goto err;
1969	}
1970	}
1971
1972	netif_napi_add(dev: common->dma_ndev, napi: &common->napi_rx,
1973	poll: am65_cpsw_nuss_rx_poll);
1974	hrtimer_init(timer: &common->rx_hrtimer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL_PINNED);
1975	common->rx_hrtimer.function = &am65_cpsw_nuss_rx_timer_callback;
1976
1977	ret = devm_request_irq(dev, irq: rx_chn->irq,
1978	handler: am65_cpsw_nuss_rx_irq,
1979	IRQF_TRIGGER_HIGH, devname: dev_name(dev), dev_id: common);
1980	if (ret) {
1981	dev_err(dev, "failure requesting rx irq %u, %d\n",
1982	rx_chn->irq, ret);
1983	goto err;
1984	}
1985
1986	err:
1987	i = devm_add_action(dev, am65_cpsw_nuss_free_rx_chns, common);
1988	if (i) {
1989	dev_err(dev, "Failed to add free_rx_chns action %d\n", i);
1990	return i;
1991	}
1992
1993	return ret;
1994	}
1995
1996	static int am65_cpsw_nuss_init_host_p(struct am65_cpsw_common *common)
1997	{
1998	struct am65_cpsw_host *host_p = am65_common_get_host(common);
1999
2000	host_p->common = common;
2001	host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE;
2002	host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE;
2003
2004	return 0;
2005	}
2006
2007	static int am65_cpsw_am654_get_efuse_macid(struct device_node *of_node,
2008	int slave, u8 *mac_addr)
2009	{
2010	u32 mac_lo, mac_hi, offset;
2011	struct regmap *syscon;
2012	int ret;
2013
2014	syscon = syscon_regmap_lookup_by_phandle(np: of_node, property: "ti,syscon-efuse");
2015	if (IS_ERR(ptr: syscon)) {
2016	if (PTR_ERR(ptr: syscon) == -ENODEV)
2017	return 0;
2018	return PTR_ERR(ptr: syscon);
2019	}
2020
2021	ret = of_property_read_u32_index(np: of_node, propname: "ti,syscon-efuse", index: 1,
2022	out_value: &offset);
2023	if (ret)
2024	return ret;
2025
2026	regmap_read(map: syscon, reg: offset, val: &mac_lo);
2027	regmap_read(map: syscon, reg: offset + 4, val: &mac_hi);
2028
2029	mac_addr[0] = (mac_hi >> 8) & 0xff;
2030	mac_addr[1] = mac_hi & 0xff;
2031	mac_addr[2] = (mac_lo >> 24) & 0xff;
2032	mac_addr[3] = (mac_lo >> 16) & 0xff;
2033	mac_addr[4] = (mac_lo >> 8) & 0xff;
2034	mac_addr[5] = mac_lo & 0xff;
2035
2036	return 0;
2037	}
2038
2039	static int am65_cpsw_init_cpts(struct am65_cpsw_common *common)
2040	{
2041	struct device *dev = common->dev;
2042	struct device_node *node;
2043	struct am65_cpts *cpts;
2044	void __iomem *reg_base;
2045
2046	if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
2047	return 0;
2048
2049	node = of_get_child_by_name(node: dev->of_node, name: "cpts");
2050	if (!node) {
2051	dev_err(dev, "%s cpts not found\n", __func__);
2052	return -ENOENT;
2053	}
2054
2055	reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE;
2056	cpts = am65_cpts_create(dev, regs: reg_base, node);
2057	if (IS_ERR(ptr: cpts)) {
2058	int ret = PTR_ERR(ptr: cpts);
2059
2060	of_node_put(node);
2061	dev_err(dev, "cpts create err %d\n", ret);
2062	return ret;
2063	}
2064	common->cpts = cpts;
2065	/* Forbid PM runtime if CPTS is running.
2066	* K3 CPSWxG modules may completely lose context during ON->OFF
2067	* transitions depending on integration.
2068	* AM65x/J721E MCU CPSW2G: false
2069	* J721E MAIN_CPSW9G: true
2070	*/
2071	pm_runtime_forbid(dev);
2072
2073	return 0;
2074	}
2075
2076	static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common)
2077	{
2078	struct device_node *node, *port_np;
2079	struct device *dev = common->dev;
2080	int ret;
2081
2082	node = of_get_child_by_name(node: dev->of_node, name: "ethernet-ports");
2083	if (!node)
2084	return -ENOENT;
2085
2086	for_each_child_of_node(node, port_np) {
2087	struct am65_cpsw_port *port;
2088	u32 port_id;
2089
2090	/* it is not a slave port node, continue */
2091	if (strcmp(port_np->name, "port"))
2092	continue;
2093
2094	ret = of_property_read_u32(np: port_np, propname: "reg", out_value: &port_id);
2095	if (ret < 0) {
2096	dev_err(dev, "%pOF error reading port_id %d\n",
2097	port_np, ret);
2098	goto of_node_put;
2099	}
2100
2101	if (!port_id || port_id > common->port_num) {
2102	dev_err(dev, "%pOF has invalid port_id %u %s\n",
2103	port_np, port_id, port_np->name);
2104	ret = -EINVAL;
2105	goto of_node_put;
2106	}
2107
2108	port = am65_common_get_port(common, port_id);
2109	port->port_id = port_id;
2110	port->common = common;
2111	port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE +
2112	AM65_CPSW_NU_PORTS_OFFSET * (port_id);
2113	if (common->pdata.extra_modes)
2114	port->sgmii_base = common->ss_base + AM65_CPSW_SGMII_BASE * (port_id);
2115	port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE +
2116	(AM65_CPSW_NU_STATS_PORT_OFFSET * port_id);
2117	port->name = of_get_property(node: port_np, name: "label", NULL);
2118	port->fetch_ram_base =
2119	common->cpsw_base + AM65_CPSW_NU_FRAM_BASE +
2120	(AM65_CPSW_NU_FRAM_PORT_OFFSET * (port_id - 1));
2121
2122	port->slave.mac_sl = cpsw_sl_get(device_id: "am65", dev, sl_base: port->port_base);
2123	if (IS_ERR(ptr: port->slave.mac_sl)) {
2124	ret = PTR_ERR(ptr: port->slave.mac_sl);
2125	goto of_node_put;
2126	}
2127
2128	port->disabled = !of_device_is_available(device: port_np);
2129	if (port->disabled) {
2130	common->disabled_ports_mask |= BIT(port->port_id);
2131	continue;
2132	}
2133
2134	port->slave.ifphy = devm_of_phy_get(dev, np: port_np, NULL);
2135	if (IS_ERR(ptr: port->slave.ifphy)) {
2136	ret = PTR_ERR(ptr: port->slave.ifphy);
2137	dev_err(dev, "%pOF error retrieving port phy: %d\n",
2138	port_np, ret);
2139	goto of_node_put;
2140	}
2141
2142	/* Initialize the Serdes PHY for the port */
2143	ret = am65_cpsw_init_serdes_phy(dev, port_np, port);
2144	if (ret)
2145	goto of_node_put;
2146
2147	port->slave.mac_only =
2148	of_property_read_bool(np: port_np, propname: "ti,mac-only");
2149
2150	/* get phy/link info */
2151	port->slave.phy_node = port_np;
2152	ret = of_get_phy_mode(np: port_np, interface: &port->slave.phy_if);
2153	if (ret) {
2154	dev_err(dev, "%pOF read phy-mode err %d\n",
2155	port_np, ret);
2156	goto of_node_put;
2157	}
2158
2159	ret = phy_set_mode_ext(phy: port->slave.ifphy, mode: PHY_MODE_ETHERNET, submode: port->slave.phy_if);
2160	if (ret)
2161	goto of_node_put;
2162
2163	ret = of_get_mac_address(np: port_np, mac: port->slave.mac_addr);
2164	if (ret) {
2165	am65_cpsw_am654_get_efuse_macid(of_node: port_np,
2166	slave: port->port_id,
2167	mac_addr: port->slave.mac_addr);
2168	if (!is_valid_ether_addr(addr: port->slave.mac_addr)) {
2169	eth_random_addr(addr: port->slave.mac_addr);
2170	dev_err(dev, "Use random MAC address\n");
2171	}
2172	}
2173
2174	/* Reset all Queue priorities to 0 */
2175	writel(val: 0, addr: port->port_base + AM65_CPSW_PN_REG_TX_PRI_MAP);
2176	}
2177	of_node_put(node);
2178
2179	/* is there at least one ext.port */
2180	if (!(~common->disabled_ports_mask & GENMASK(common->port_num, 1))) {
2181	dev_err(dev, "No Ext. port are available\n");
2182	return -ENODEV;
2183	}
2184
2185	return 0;
2186
2187	of_node_put:
2188	of_node_put(node: port_np);
2189	of_node_put(node);
2190	return ret;
2191	}
2192
2193	static void am65_cpsw_pcpu_stats_free(void *data)
2194	{
2195	struct am65_cpsw_ndev_stats __percpu *stats = data;
2196
2197	free_percpu(pdata: stats);
2198	}
2199
2200	static void am65_cpsw_nuss_phylink_cleanup(struct am65_cpsw_common *common)
2201	{
2202	struct am65_cpsw_port *port;
2203	int i;
2204
2205	for (i = 0; i < common->port_num; i++) {
2206	port = &common->ports[i];
2207	if (port->slave.phylink)
2208	phylink_destroy(port->slave.phylink);
2209	}
2210	}
2211
2212	static int
2213	am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
2214	{
2215	struct am65_cpsw_ndev_priv *ndev_priv;
2216	struct device *dev = common->dev;
2217	struct am65_cpsw_port *port;
2218	struct phylink *phylink;
2219	int ret;
2220
2221	port = &common->ports[port_idx];
2222
2223	if (port->disabled)
2224	return 0;
2225
2226	/* alloc netdev */
2227	port->ndev = devm_alloc_etherdev_mqs(dev: common->dev,
2228	sizeof_priv: sizeof(struct am65_cpsw_ndev_priv),
2229	AM65_CPSW_MAX_TX_QUEUES,
2230	AM65_CPSW_MAX_RX_QUEUES);
2231	if (!port->ndev) {
2232	dev_err(dev, "error allocating slave net_device %u\n",
2233	port->port_id);
2234	return -ENOMEM;
2235	}
2236
2237	ndev_priv = netdev_priv(dev: port->ndev);
2238	ndev_priv->port = port;
2239	ndev_priv->msg_enable = AM65_CPSW_DEBUG;
2240	mutex_init(&ndev_priv->mm_lock);
2241	port->qos.link_speed = SPEED_UNKNOWN;
2242	SET_NETDEV_DEV(port->ndev, dev);
2243
2244	eth_hw_addr_set(dev: port->ndev, addr: port->slave.mac_addr);
2245
2246	port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE;
2247	port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE -
2248	(VLAN_ETH_HLEN + ETH_FCS_LEN);
2249	port->ndev->hw_features = NETIF_F_SG |
2250	NETIF_F_RXCSUM |
2251	NETIF_F_HW_CSUM |
2252	NETIF_F_HW_TC;
2253	port->ndev->features = port->ndev->hw_features |
2254	NETIF_F_HW_VLAN_CTAG_FILTER;
2255	port->ndev->vlan_features |= NETIF_F_SG;
2256	port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops;
2257	port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave;
2258
2259	/* Configuring Phylink */
2260	port->slave.phylink_config.dev = &port->ndev->dev;
2261	port->slave.phylink_config.type = PHYLINK_NETDEV;
2262	port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 |
2263	MAC_1000FD | MAC_5000FD;
2264	port->slave.phylink_config.mac_managed_pm = true; /* MAC does PM */
2265
2266	switch (port->slave.phy_if) {
2267	case PHY_INTERFACE_MODE_RGMII:
2268	case PHY_INTERFACE_MODE_RGMII_ID:
2269	case PHY_INTERFACE_MODE_RGMII_RXID:
2270	case PHY_INTERFACE_MODE_RGMII_TXID:
2271	phy_interface_set_rgmii(intf: port->slave.phylink_config.supported_interfaces);
2272	break;
2273
2274	case PHY_INTERFACE_MODE_RMII:
2275	__set_bit(PHY_INTERFACE_MODE_RMII,
2276	port->slave.phylink_config.supported_interfaces);
2277	break;
2278
2279	case PHY_INTERFACE_MODE_QSGMII:
2280	case PHY_INTERFACE_MODE_SGMII:
2281	case PHY_INTERFACE_MODE_USXGMII:
2282	if (common->pdata.extra_modes & BIT(port->slave.phy_if)) {
2283	__set_bit(port->slave.phy_if,
2284	port->slave.phylink_config.supported_interfaces);
2285	} else {
2286	dev_err(dev, "selected phy-mode is not supported\n");
2287	return -EOPNOTSUPP;
2288	}
2289	break;
2290
2291	default:
2292	dev_err(dev, "selected phy-mode is not supported\n");
2293	return -EOPNOTSUPP;
2294	}
2295
2296	phylink = phylink_create(&port->slave.phylink_config,
2297	of_node_to_fwnode(node: port->slave.phy_node),
2298	port->slave.phy_if,
2299	&am65_cpsw_phylink_mac_ops);
2300	if (IS_ERR(ptr: phylink))
2301	return PTR_ERR(ptr: phylink);
2302
2303	port->slave.phylink = phylink;
2304
2305	/* Disable TX checksum offload by default due to HW bug */
2306	if (common->pdata.quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM)
2307	port->ndev->features &= ~NETIF_F_HW_CSUM;
2308
2309	ndev_priv->stats = netdev_alloc_pcpu_stats(struct am65_cpsw_ndev_stats);
2310	if (!ndev_priv->stats)
2311	return -ENOMEM;
2312
2313	ret = devm_add_action_or_reset(dev, am65_cpsw_pcpu_stats_free,
2314	ndev_priv->stats);
2315	if (ret)
2316	dev_err(dev, "failed to add percpu stat free action %d\n", ret);
2317
2318	if (!common->dma_ndev)
2319	common->dma_ndev = port->ndev;
2320
2321	return ret;
2322	}
2323
2324	static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common)
2325	{
2326	int ret;
2327	int i;
2328
2329	for (i = 0; i < common->port_num; i++) {
2330	ret = am65_cpsw_nuss_init_port_ndev(common, port_idx: i);
2331	if (ret)
2332	return ret;
2333	}
2334
2335	return ret;
2336	}
2337
2338	static void am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common *common)
2339	{
2340	struct am65_cpsw_port *port;
2341	int i;
2342
2343	for (i = 0; i < common->port_num; i++) {
2344	port = &common->ports[i];
2345	if (port->ndev && port->ndev->reg_state == NETREG_REGISTERED)
2346	unregister_netdev(dev: port->ndev);
2347	}
2348	}
2349
2350	static void am65_cpsw_port_offload_fwd_mark_update(struct am65_cpsw_common *common)
2351	{
2352	int set_val = 0;
2353	int i;
2354
2355	if (common->br_members == (GENMASK(common->port_num, 1) & ~common->disabled_ports_mask))
2356	set_val = 1;
2357
2358	dev_dbg(common->dev, "set offload_fwd_mark %d\n", set_val);
2359
2360	for (i = 1; i <= common->port_num; i++) {
2361	struct am65_cpsw_port *port = am65_common_get_port(common, i);
2362	struct am65_cpsw_ndev_priv *priv;
2363
2364	if (!port->ndev)
2365	continue;
2366
2367	priv = am65_ndev_to_priv(port->ndev);
2368	priv->offload_fwd_mark = set_val;
2369	}
2370	}
2371
2372	bool am65_cpsw_port_dev_check(const struct net_device *ndev)
2373	{
2374	if (ndev->netdev_ops == &am65_cpsw_nuss_netdev_ops) {
2375	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2376
2377	return !common->is_emac_mode;
2378	}
2379
2380	return false;
2381	}
2382
2383	static int am65_cpsw_netdevice_port_link(struct net_device *ndev,
2384	struct net_device *br_ndev,
2385	struct netlink_ext_ack *extack)
2386	{
2387	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2388	struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2389	int err;
2390
2391	if (!common->br_members) {
2392	common->hw_bridge_dev = br_ndev;
2393	} else {
2394	/* This is adding the port to a second bridge, this is
2395	* unsupported
2396	*/
2397	if (common->hw_bridge_dev != br_ndev)
2398	return -EOPNOTSUPP;
2399	}
2400
2401	err = switchdev_bridge_port_offload(brport_dev: ndev, dev: ndev, NULL, NULL, NULL,
2402	tx_fwd_offload: false, extack);
2403	if (err)
2404	return err;
2405
2406	common->br_members |= BIT(priv->port->port_id);
2407
2408	am65_cpsw_port_offload_fwd_mark_update(common);
2409
2410	return NOTIFY_DONE;
2411	}
2412
2413	static void am65_cpsw_netdevice_port_unlink(struct net_device *ndev)
2414	{
2415	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2416	struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2417
2418	switchdev_bridge_port_unoffload(brport_dev: ndev, NULL, NULL, NULL);
2419
2420	common->br_members &= ~BIT(priv->port->port_id);
2421
2422	am65_cpsw_port_offload_fwd_mark_update(common);
2423
2424	if (!common->br_members)
2425	common->hw_bridge_dev = NULL;
2426	}
2427
2428	/* netdev notifier */
2429	static int am65_cpsw_netdevice_event(struct notifier_block *unused,
2430	unsigned long event, void *ptr)
2431	{
2432	struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(info: ptr);
2433	struct net_device *ndev = netdev_notifier_info_to_dev(info: ptr);
2434	struct netdev_notifier_changeupper_info *info;
2435	int ret = NOTIFY_DONE;
2436
2437	if (!am65_cpsw_port_dev_check(ndev))
2438	return NOTIFY_DONE;
2439
2440	switch (event) {
2441	case NETDEV_CHANGEUPPER:
2442	info = ptr;
2443
2444	if (netif_is_bridge_master(dev: info->upper_dev)) {
2445	if (info->linking)
2446	ret = am65_cpsw_netdevice_port_link(ndev,
2447	br_ndev: info->upper_dev,
2448	extack);
2449	else
2450	am65_cpsw_netdevice_port_unlink(ndev);
2451	}
2452	break;
2453	default:
2454	return NOTIFY_DONE;
2455	}
2456
2457	return notifier_from_errno(err: ret);
2458	}
2459
2460	static int am65_cpsw_register_notifiers(struct am65_cpsw_common *cpsw)
2461	{
2462	int ret = 0;
2463
2464	if (AM65_CPSW_IS_CPSW2G(cpsw) ||
2465	!IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2466	return 0;
2467
2468	cpsw->am65_cpsw_netdevice_nb.notifier_call = &am65_cpsw_netdevice_event;
2469	ret = register_netdevice_notifier(nb: &cpsw->am65_cpsw_netdevice_nb);
2470	if (ret) {
2471	dev_err(cpsw->dev, "can't register netdevice notifier\n");
2472	return ret;
2473	}
2474
2475	ret = am65_cpsw_switchdev_register_notifiers(cpsw);
2476	if (ret)
2477	unregister_netdevice_notifier(nb: &cpsw->am65_cpsw_netdevice_nb);
2478
2479	return ret;
2480	}
2481
2482	static void am65_cpsw_unregister_notifiers(struct am65_cpsw_common *cpsw)
2483	{
2484	if (AM65_CPSW_IS_CPSW2G(cpsw) ||
2485	!IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2486	return;
2487
2488	am65_cpsw_switchdev_unregister_notifiers(cpsw);
2489	unregister_netdevice_notifier(nb: &cpsw->am65_cpsw_netdevice_nb);
2490	}
2491
2492	static const struct devlink_ops am65_cpsw_devlink_ops = {};
2493
2494	static void am65_cpsw_init_stp_ale_entry(struct am65_cpsw_common *cpsw)
2495	{
2496	cpsw_ale_add_mcast(ale: cpsw->ale, eth_stp_addr, ALE_PORT_HOST, ALE_SUPER, vid: 0,
2497	ALE_MCAST_BLOCK_LEARN_FWD);
2498	}
2499
2500	static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common)
2501	{
2502	struct am65_cpsw_host *host = am65_common_get_host(common);
2503
2504	writel(val: common->default_vlan, addr: host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2505
2506	am65_cpsw_init_stp_ale_entry(cpsw: common);
2507
2508	cpsw_ale_control_set(ale: common->ale, HOST_PORT_NUM, control: ALE_P0_UNI_FLOOD, value: 1);
2509	dev_dbg(common->dev, "Set P0_UNI_FLOOD\n");
2510	cpsw_ale_control_set(ale: common->ale, HOST_PORT_NUM, control: ALE_PORT_NOLEARN, value: 0);
2511	}
2512
2513	static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common)
2514	{
2515	struct am65_cpsw_host *host = am65_common_get_host(common);
2516
2517	writel(val: 0, addr: host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2518
2519	cpsw_ale_control_set(ale: common->ale, HOST_PORT_NUM, control: ALE_P0_UNI_FLOOD, value: 0);
2520	dev_dbg(common->dev, "unset P0_UNI_FLOOD\n");
2521
2522	/* learning make no sense in multi-mac mode */
2523	cpsw_ale_control_set(ale: common->ale, HOST_PORT_NUM, control: ALE_PORT_NOLEARN, value: 1);
2524	}
2525
2526	static int am65_cpsw_dl_switch_mode_get(struct devlink *dl, u32 id,
2527	struct devlink_param_gset_ctx *ctx)
2528	{
2529	struct am65_cpsw_devlink *dl_priv = devlink_priv(devlink: dl);
2530	struct am65_cpsw_common *common = dl_priv->common;
2531
2532	dev_dbg(common->dev, "%s id:%u\n", __func__, id);
2533
2534	if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
2535	return -EOPNOTSUPP;
2536
2537	ctx->val.vbool = !common->is_emac_mode;
2538
2539	return 0;
2540	}
2541
2542	static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port)
2543	{
2544	struct am65_cpsw_slave_data *slave = &port->slave;
2545	struct am65_cpsw_common *common = port->common;
2546	u32 port_mask;
2547
2548	writel(val: slave->port_vlan, addr: port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2549
2550	if (slave->mac_only)
2551	/* enable mac-only mode on port */
2552	cpsw_ale_control_set(ale: common->ale, port: port->port_id,
2553	control: ALE_PORT_MACONLY, value: 1);
2554
2555	cpsw_ale_control_set(ale: common->ale, port: port->port_id, control: ALE_PORT_NOLEARN, value: 1);
2556
2557	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
2558
2559	cpsw_ale_add_ucast(ale: common->ale, addr: port->ndev->dev_addr,
2560	HOST_PORT_NUM, ALE_SECURE, vid: slave->port_vlan);
2561	cpsw_ale_add_mcast(ale: common->ale, addr: port->ndev->broadcast,
2562	port_mask, ALE_VLAN, vid: slave->port_vlan, ALE_MCAST_FWD_2);
2563	}
2564
2565	static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port)
2566	{
2567	struct am65_cpsw_slave_data *slave = &port->slave;
2568	struct am65_cpsw_common *cpsw = port->common;
2569	u32 port_mask;
2570
2571	cpsw_ale_control_set(ale: cpsw->ale, port: port->port_id,
2572	control: ALE_PORT_NOLEARN, value: 0);
2573
2574	cpsw_ale_add_ucast(ale: cpsw->ale, addr: port->ndev->dev_addr,
2575	HOST_PORT_NUM, ALE_SECURE | ALE_BLOCKED | ALE_VLAN,
2576	vid: slave->port_vlan);
2577
2578	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
2579
2580	cpsw_ale_add_mcast(ale: cpsw->ale, addr: port->ndev->broadcast,
2581	port_mask, ALE_VLAN, vid: slave->port_vlan,
2582	ALE_MCAST_FWD_2);
2583
2584	writel(val: slave->port_vlan, addr: port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2585
2586	cpsw_ale_control_set(ale: cpsw->ale, port: port->port_id,
2587	control: ALE_PORT_MACONLY, value: 0);
2588	}
2589
2590	static int am65_cpsw_dl_switch_mode_set(struct devlink *dl, u32 id,
2591	struct devlink_param_gset_ctx *ctx)
2592	{
2593	struct am65_cpsw_devlink *dl_priv = devlink_priv(devlink: dl);
2594	struct am65_cpsw_common *cpsw = dl_priv->common;
2595	bool switch_en = ctx->val.vbool;
2596	bool if_running = false;
2597	int i;
2598
2599	dev_dbg(cpsw->dev, "%s id:%u\n", __func__, id);
2600
2601	if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
2602	return -EOPNOTSUPP;
2603
2604	if (switch_en == !cpsw->is_emac_mode)
2605	return 0;
2606
2607	if (!switch_en && cpsw->br_members) {
2608	dev_err(cpsw->dev, "Remove ports from bridge before disabling switch mode\n");
2609	return -EINVAL;
2610	}
2611
2612	rtnl_lock();
2613
2614	cpsw->is_emac_mode = !switch_en;
2615
2616	for (i = 0; i < cpsw->port_num; i++) {
2617	struct net_device *sl_ndev = cpsw->ports[i].ndev;
2618
2619	if (!sl_ndev || !netif_running(dev: sl_ndev))
2620	continue;
2621
2622	if_running = true;
2623	}
2624
2625	if (!if_running) {
2626	/* all ndevs are down */
2627	for (i = 0; i < cpsw->port_num; i++) {
2628	struct net_device *sl_ndev = cpsw->ports[i].ndev;
2629	struct am65_cpsw_slave_data *slave;
2630
2631	if (!sl_ndev)
2632	continue;
2633
2634	slave = am65_ndev_to_slave(sl_ndev);
2635	if (switch_en)
2636	slave->port_vlan = cpsw->default_vlan;
2637	else
2638	slave->port_vlan = 0;
2639	}
2640
2641	goto exit;
2642	}
2643
2644	cpsw_ale_control_set(ale: cpsw->ale, port: 0, control: ALE_BYPASS, value: 1);
2645	/* clean up ALE table */
2646	cpsw_ale_control_set(ale: cpsw->ale, HOST_PORT_NUM, control: ALE_CLEAR, value: 1);
2647	cpsw_ale_control_get(ale: cpsw->ale, HOST_PORT_NUM, control: ALE_AGEOUT);
2648
2649	if (switch_en) {
2650	dev_info(cpsw->dev, "Enable switch mode\n");
2651
2652	am65_cpsw_init_host_port_switch(common: cpsw);
2653
2654	for (i = 0; i < cpsw->port_num; i++) {
2655	struct net_device *sl_ndev = cpsw->ports[i].ndev;
2656	struct am65_cpsw_slave_data *slave;
2657	struct am65_cpsw_port *port;
2658
2659	if (!sl_ndev)
2660	continue;
2661
2662	port = am65_ndev_to_port(sl_ndev);
2663	slave = am65_ndev_to_slave(sl_ndev);
2664	slave->port_vlan = cpsw->default_vlan;
2665
2666	if (netif_running(dev: sl_ndev))
2667	am65_cpsw_init_port_switch_ale(port);
2668	}
2669
2670	} else {
2671	dev_info(cpsw->dev, "Disable switch mode\n");
2672
2673	am65_cpsw_init_host_port_emac(common: cpsw);
2674
2675	for (i = 0; i < cpsw->port_num; i++) {
2676	struct net_device *sl_ndev = cpsw->ports[i].ndev;
2677	struct am65_cpsw_port *port;
2678
2679	if (!sl_ndev)
2680	continue;
2681
2682	port = am65_ndev_to_port(sl_ndev);
2683	port->slave.port_vlan = 0;
2684	if (netif_running(dev: sl_ndev))
2685	am65_cpsw_init_port_emac_ale(port);
2686	}
2687	}
2688	cpsw_ale_control_set(ale: cpsw->ale, HOST_PORT_NUM, control: ALE_BYPASS, value: 0);
2689	exit:
2690	rtnl_unlock();
2691
2692	return 0;
2693	}
2694
2695	static const struct devlink_param am65_cpsw_devlink_params[] = {
2696	DEVLINK_PARAM_DRIVER(AM65_CPSW_DL_PARAM_SWITCH_MODE, "switch_mode",
2697	DEVLINK_PARAM_TYPE_BOOL,
2698	BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2699	am65_cpsw_dl_switch_mode_get,
2700	am65_cpsw_dl_switch_mode_set, NULL),
2701	};
2702
2703	static int am65_cpsw_nuss_register_devlink(struct am65_cpsw_common *common)
2704	{
2705	struct devlink_port_attrs attrs = {};
2706	struct am65_cpsw_devlink *dl_priv;
2707	struct device *dev = common->dev;
2708	struct devlink_port *dl_port;
2709	struct am65_cpsw_port *port;
2710	int ret = 0;
2711	int i;
2712
2713	common->devlink =
2714	devlink_alloc(ops: &am65_cpsw_devlink_ops, priv_size: sizeof(*dl_priv), dev);
2715	if (!common->devlink)
2716	return -ENOMEM;
2717
2718	dl_priv = devlink_priv(devlink: common->devlink);
2719	dl_priv->common = common;
2720
2721	/* Provide devlink hook to switch mode when multiple external ports
2722	* are present NUSS switchdev driver is enabled.
2723	*/
2724	if (!AM65_CPSW_IS_CPSW2G(common) &&
2725	IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) {
2726	ret = devlink_params_register(devlink: common->devlink,
2727	params: am65_cpsw_devlink_params,
2728	ARRAY_SIZE(am65_cpsw_devlink_params));
2729	if (ret) {
2730	dev_err(dev, "devlink params reg fail ret:%d\n", ret);
2731	goto dl_unreg;
2732	}
2733	}
2734
2735	for (i = 1; i <= common->port_num; i++) {
2736	port = am65_common_get_port(common, i);
2737	dl_port = &port->devlink_port;
2738
2739	if (port->ndev)
2740	attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
2741	else
2742	attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED;
2743	attrs.phys.port_number = port->port_id;
2744	attrs.switch_id.id_len = sizeof(resource_size_t);
2745	memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len);
2746	devlink_port_attrs_set(devlink_port: dl_port, devlink_port_attrs: &attrs);
2747
2748	ret = devlink_port_register(devlink: common->devlink, devlink_port: dl_port, port_index: port->port_id);
2749	if (ret) {
2750	dev_err(dev, "devlink_port reg fail for port %d, ret:%d\n",
2751	port->port_id, ret);
2752	goto dl_port_unreg;
2753	}
2754	}
2755	devlink_register(devlink: common->devlink);
2756	return ret;
2757
2758	dl_port_unreg:
2759	for (i = i - 1; i >= 1; i--) {
2760	port = am65_common_get_port(common, i);
2761	dl_port = &port->devlink_port;
2762
2763	devlink_port_unregister(devlink_port: dl_port);
2764	}
2765	dl_unreg:
2766	devlink_free(devlink: common->devlink);
2767	return ret;
2768	}
2769
2770	static void am65_cpsw_unregister_devlink(struct am65_cpsw_common *common)
2771	{
2772	struct devlink_port *dl_port;
2773	struct am65_cpsw_port *port;
2774	int i;
2775
2776	devlink_unregister(devlink: common->devlink);
2777
2778	for (i = 1; i <= common->port_num; i++) {
2779	port = am65_common_get_port(common, i);
2780	dl_port = &port->devlink_port;
2781
2782	devlink_port_unregister(devlink_port: dl_port);
2783	}
2784
2785	if (!AM65_CPSW_IS_CPSW2G(common) &&
2786	IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2787	devlink_params_unregister(devlink: common->devlink,
2788	params: am65_cpsw_devlink_params,
2789	ARRAY_SIZE(am65_cpsw_devlink_params));
2790
2791	devlink_free(devlink: common->devlink);
2792	}
2793
2794	static int am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common *common)
2795	{
2796	struct am65_cpsw_rx_chn *rx_chan = &common->rx_chns;
2797	struct am65_cpsw_tx_chn *tx_chan = common->tx_chns;
2798	struct device *dev = common->dev;
2799	struct am65_cpsw_port *port;
2800	int ret = 0, i;
2801
2802	/* init tx channels */
2803	ret = am65_cpsw_nuss_init_tx_chns(common);
2804	if (ret)
2805	return ret;
2806	ret = am65_cpsw_nuss_init_rx_chns(common);
2807	if (ret)
2808	return ret;
2809
2810	/* The DMA Channels are not guaranteed to be in a clean state.
2811	* Reset and disable them to ensure that they are back to the
2812	* clean state and ready to be used.
2813	*/
2814	for (i = 0; i < common->tx_ch_num; i++) {
2815	k3_udma_glue_reset_tx_chn(tx_chn: tx_chan[i].tx_chn, data: &tx_chan[i],
2816	cleanup: am65_cpsw_nuss_tx_cleanup);
2817	k3_udma_glue_disable_tx_chn(tx_chn: tx_chan[i].tx_chn);
2818	}
2819
2820	for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++)
2821	k3_udma_glue_reset_rx_chn(rx_chn: rx_chan->rx_chn, flow_num: i, data: rx_chan,
2822	cleanup: am65_cpsw_nuss_rx_cleanup, skip_fdq: !!i);
2823
2824	k3_udma_glue_disable_rx_chn(rx_chn: rx_chan->rx_chn);
2825
2826	ret = am65_cpsw_nuss_register_devlink(common);
2827	if (ret)
2828	return ret;
2829
2830	for (i = 0; i < common->port_num; i++) {
2831	port = &common->ports[i];
2832
2833	if (!port->ndev)
2834	continue;
2835
2836	SET_NETDEV_DEVLINK_PORT(port->ndev, &port->devlink_port);
2837
2838	ret = register_netdev(dev: port->ndev);
2839	if (ret) {
2840	dev_err(dev, "error registering slave net device%i %d\n",
2841	i, ret);
2842	goto err_cleanup_ndev;
2843	}
2844	}
2845
2846	ret = am65_cpsw_register_notifiers(cpsw: common);
2847	if (ret)
2848	goto err_cleanup_ndev;
2849
2850	/* can't auto unregister ndev using devm_add_action() due to
2851	* devres release sequence in DD core for DMA
2852	*/
2853
2854	return 0;
2855
2856	err_cleanup_ndev:
2857	am65_cpsw_nuss_cleanup_ndev(common);
2858	am65_cpsw_unregister_devlink(common);
2859
2860	return ret;
2861	}
2862
2863	int am65_cpsw_nuss_update_tx_chns(struct am65_cpsw_common *common, int num_tx)
2864	{
2865	int ret;
2866
2867	common->tx_ch_num = num_tx;
2868	ret = am65_cpsw_nuss_init_tx_chns(common);
2869
2870	return ret;
2871	}
2872
2873	struct am65_cpsw_soc_pdata {
2874	u32 quirks_dis;
2875	};
2876
2877	static const struct am65_cpsw_soc_pdata am65x_soc_sr2_0 = {
2878	.quirks_dis = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
2879	};
2880
2881	static const struct soc_device_attribute am65_cpsw_socinfo[] = {
2882	{ .family = "AM65X",
2883	.revision = "SR2.0",
2884	.data = &am65x_soc_sr2_0
2885	},
2886	{/* sentinel */}
2887	};
2888
2889	static const struct am65_cpsw_pdata am65x_sr1_0 = {
2890	.quirks = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
2891	.ale_dev_id = "am65x-cpsw2g",
2892	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2893	};
2894
2895	static const struct am65_cpsw_pdata j721e_pdata = {
2896	.quirks = 0,
2897	.ale_dev_id = "am65x-cpsw2g",
2898	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2899	};
2900
2901	static const struct am65_cpsw_pdata am64x_cpswxg_pdata = {
2902	.quirks = AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ,
2903	.ale_dev_id = "am64-cpswxg",
2904	.fdqring_mode = K3_RINGACC_RING_MODE_RING,
2905	};
2906
2907	static const struct am65_cpsw_pdata j7200_cpswxg_pdata = {
2908	.quirks = 0,
2909	.ale_dev_id = "am64-cpswxg",
2910	.fdqring_mode = K3_RINGACC_RING_MODE_RING,
2911	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII),
2912	};
2913
2914	static const struct am65_cpsw_pdata j721e_cpswxg_pdata = {
2915	.quirks = 0,
2916	.ale_dev_id = "am64-cpswxg",
2917	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2918	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_SGMII),
2919	};
2920
2921	static const struct am65_cpsw_pdata j784s4_cpswxg_pdata = {
2922	.quirks = 0,
2923	.ale_dev_id = "am64-cpswxg",
2924	.fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2925	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII) | BIT(PHY_INTERFACE_MODE_USXGMII),
2926	};
2927
2928	static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
2929	{ .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0},
2930	{ .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata},
2931	{ .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata},
2932	{ .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata},
2933	{ .compatible = "ti,j721e-cpswxg-nuss", .data = &j721e_cpswxg_pdata},
2934	{ .compatible = "ti,j784s4-cpswxg-nuss", .data = &j784s4_cpswxg_pdata},
2935	{ /* sentinel */ },
2936	};
2937	MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
2938
2939	static void am65_cpsw_nuss_apply_socinfo(struct am65_cpsw_common *common)
2940	{
2941	const struct soc_device_attribute *soc;
2942
2943	soc = soc_device_match(matches: am65_cpsw_socinfo);
2944	if (soc && soc->data) {
2945	const struct am65_cpsw_soc_pdata *socdata = soc->data;
2946
2947	/* disable quirks */
2948	common->pdata.quirks &= ~socdata->quirks_dis;
2949	}
2950	}
2951
2952	static int am65_cpsw_nuss_probe(struct platform_device *pdev)
2953	{
2954	struct cpsw_ale_params ale_params = { 0 };
2955	const struct of_device_id *of_id;
2956	struct device *dev = &pdev->dev;
2957	struct am65_cpsw_common *common;
2958	struct device_node *node;
2959	struct resource *res;
2960	struct clk *clk;
2961	u64 id_temp;
2962	int ret, i;
2963	int ale_entries;
2964
2965	common = devm_kzalloc(dev, size: sizeof(struct am65_cpsw_common), GFP_KERNEL);
2966	if (!common)
2967	return -ENOMEM;
2968	common->dev = dev;
2969
2970	of_id = of_match_device(matches: am65_cpsw_nuss_of_mtable, dev);
2971	if (!of_id)
2972	return -EINVAL;
2973	common->pdata = *(const struct am65_cpsw_pdata *)of_id->data;
2974
2975	am65_cpsw_nuss_apply_socinfo(common);
2976
2977	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpsw_nuss");
2978	common->ss_base = devm_ioremap_resource(dev: &pdev->dev, res);
2979	if (IS_ERR(ptr: common->ss_base))
2980	return PTR_ERR(ptr: common->ss_base);
2981	common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE;
2982	/* Use device's physical base address as switch id */
2983	id_temp = cpu_to_be64(res->start);
2984	memcpy(common->switch_id, &id_temp, sizeof(res->start));
2985
2986	node = of_get_child_by_name(node: dev->of_node, name: "ethernet-ports");
2987	if (!node)
2988	return -ENOENT;
2989	common->port_num = of_get_child_count(np: node);
2990	of_node_put(node);
2991	if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS)
2992	return -ENOENT;
2993
2994	common->rx_flow_id_base = -1;
2995	init_completion(x: &common->tdown_complete);
2996	common->tx_ch_num = AM65_CPSW_DEFAULT_TX_CHNS;
2997	common->pf_p0_rx_ptype_rrobin = false;
2998	common->default_vlan = 1;
2999
3000	common->ports = devm_kcalloc(dev, n: common->port_num,
3001	size: sizeof(*common->ports),
3002	GFP_KERNEL);
3003	if (!common->ports)
3004	return -ENOMEM;
3005
3006	clk = devm_clk_get(dev, id: "fck");
3007	if (IS_ERR(ptr: clk))
3008	return dev_err_probe(dev, err: PTR_ERR(ptr: clk), fmt: "getting fck clock\n");
3009	common->bus_freq = clk_get_rate(clk);
3010
3011	pm_runtime_enable(dev);
3012	ret = pm_runtime_resume_and_get(dev);
3013	if (ret < 0) {
3014	pm_runtime_disable(dev);
3015	return ret;
3016	}
3017
3018	node = of_get_child_by_name(node: dev->of_node, name: "mdio");
3019	if (!node) {
3020	dev_warn(dev, "MDIO node not found\n");
3021	} else if (of_device_is_available(device: node)) {
3022	struct platform_device *mdio_pdev;
3023
3024	mdio_pdev = of_platform_device_create(np: node, NULL, parent: dev);
3025	if (!mdio_pdev) {
3026	ret = -ENODEV;
3027	goto err_pm_clear;
3028	}
3029
3030	common->mdio_dev = &mdio_pdev->dev;
3031	}
3032	of_node_put(node);
3033
3034	am65_cpsw_nuss_get_ver(common);
3035
3036	ret = am65_cpsw_nuss_init_host_p(common);
3037	if (ret)
3038	goto err_of_clear;
3039
3040	ret = am65_cpsw_nuss_init_slave_ports(common);
3041	if (ret)
3042	goto err_of_clear;
3043
3044	/* init common data */
3045	ale_params.dev = dev;
3046	ale_params.ale_ageout = AM65_CPSW_ALE_AGEOUT_DEFAULT;
3047	ale_params.ale_ports = common->port_num + 1;
3048	ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE;
3049	ale_params.dev_id = common->pdata.ale_dev_id;
3050	ale_params.bus_freq = common->bus_freq;
3051
3052	common->ale = cpsw_ale_create(params: &ale_params);
3053	if (IS_ERR(ptr: common->ale)) {
3054	dev_err(dev, "error initializing ale engine\n");
3055	ret = PTR_ERR(ptr: common->ale);
3056	goto err_of_clear;
3057	}
3058
3059	ale_entries = common->ale->params.ale_entries;
3060	common->ale_context = devm_kzalloc(dev,
3061	size: ale_entries * ALE_ENTRY_WORDS * sizeof(u32),
3062	GFP_KERNEL);
3063	ret = am65_cpsw_init_cpts(common);
3064	if (ret)
3065	goto err_of_clear;
3066
3067	/* init ports */
3068	for (i = 0; i < common->port_num; i++)
3069	am65_cpsw_nuss_slave_disable_unused(port: &common->ports[i]);
3070
3071	dev_set_drvdata(dev, data: common);
3072
3073	common->is_emac_mode = true;
3074
3075	ret = am65_cpsw_nuss_init_ndevs(common);
3076	if (ret)
3077	goto err_free_phylink;
3078
3079	ret = am65_cpsw_nuss_register_ndevs(common);
3080	if (ret)
3081	goto err_free_phylink;
3082
3083	pm_runtime_put(dev);
3084	return 0;
3085
3086	err_free_phylink:
3087	am65_cpsw_nuss_phylink_cleanup(common);
3088	am65_cpts_release(cpts: common->cpts);
3089	err_of_clear:
3090	if (common->mdio_dev)
3091	of_platform_device_destroy(dev: common->mdio_dev, NULL);
3092	err_pm_clear:
3093	pm_runtime_put_sync(dev);
3094	pm_runtime_disable(dev);
3095	return ret;
3096	}
3097
3098	static void am65_cpsw_nuss_remove(struct platform_device *pdev)
3099	{
3100	struct device *dev = &pdev->dev;
3101	struct am65_cpsw_common *common;
3102	int ret;
3103
3104	common = dev_get_drvdata(dev);
3105
3106	ret = pm_runtime_resume_and_get(dev: &pdev->dev);
3107	if (ret < 0) {
3108	/* Note, if this error path is taken, we're leaking some
3109	* resources.
3110	*/
3111	dev_err(&pdev->dev, "Failed to resume device (%pe)\n",
3112	ERR_PTR(ret));
3113	return;
3114	}
3115
3116	am65_cpsw_unregister_devlink(common);
3117	am65_cpsw_unregister_notifiers(cpsw: common);
3118
3119	/* must unregister ndevs here because DD release_driver routine calls
3120	* dma_deconfigure(dev) before devres_release_all(dev)
3121	*/
3122	am65_cpsw_nuss_cleanup_ndev(common);
3123	am65_cpsw_nuss_phylink_cleanup(common);
3124	am65_cpts_release(cpts: common->cpts);
3125	am65_cpsw_disable_serdes_phy(common);
3126
3127	if (common->mdio_dev)
3128	of_platform_device_destroy(dev: common->mdio_dev, NULL);
3129
3130	pm_runtime_put_sync(dev: &pdev->dev);
3131	pm_runtime_disable(dev: &pdev->dev);
3132	}
3133
3134	static int am65_cpsw_nuss_suspend(struct device *dev)
3135	{
3136	struct am65_cpsw_common *common = dev_get_drvdata(dev);
3137	struct am65_cpsw_host *host_p = am65_common_get_host(common);
3138	struct am65_cpsw_port *port;
3139	struct net_device *ndev;
3140	int i, ret;
3141
3142	cpsw_ale_dump(ale: common->ale, data: common->ale_context);
3143	host_p->vid_context = readl(addr: host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3144	for (i = 0; i < common->port_num; i++) {
3145	port = &common->ports[i];
3146	ndev = port->ndev;
3147
3148	if (!ndev)
3149	continue;
3150
3151	port->vid_context = readl(addr: port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3152	netif_device_detach(dev: ndev);
3153	if (netif_running(dev: ndev)) {
3154	rtnl_lock();
3155	ret = am65_cpsw_nuss_ndo_slave_stop(ndev);
3156	rtnl_unlock();
3157	if (ret < 0) {
3158	netdev_err(dev: ndev, format: "failed to stop: %d", ret);
3159	return ret;
3160	}
3161	}
3162	}
3163
3164	am65_cpts_suspend(cpts: common->cpts);
3165
3166	am65_cpsw_nuss_remove_rx_chns(data: common);
3167	am65_cpsw_nuss_remove_tx_chns(common);
3168
3169	return 0;
3170	}
3171
3172	static int am65_cpsw_nuss_resume(struct device *dev)
3173	{
3174	struct am65_cpsw_common *common = dev_get_drvdata(dev);
3175	struct am65_cpsw_port *port;
3176	struct net_device *ndev;
3177	int i, ret;
3178	struct am65_cpsw_host *host_p = am65_common_get_host(common);
3179
3180	ret = am65_cpsw_nuss_init_tx_chns(common);
3181	if (ret)
3182	return ret;
3183	ret = am65_cpsw_nuss_init_rx_chns(common);
3184	if (ret)
3185	return ret;
3186
3187	/* If RX IRQ was disabled before suspend, keep it disabled */
3188	if (common->rx_irq_disabled)
3189	disable_irq(irq: common->rx_chns.irq);
3190
3191	am65_cpts_resume(cpts: common->cpts);
3192
3193	for (i = 0; i < common->port_num; i++) {
3194	port = &common->ports[i];
3195	ndev = port->ndev;
3196
3197	if (!ndev)
3198	continue;
3199
3200	if (netif_running(dev: ndev)) {
3201	rtnl_lock();
3202	ret = am65_cpsw_nuss_ndo_slave_open(ndev);
3203	rtnl_unlock();
3204	if (ret < 0) {
3205	netdev_err(dev: ndev, format: "failed to start: %d", ret);
3206	return ret;
3207	}
3208	}
3209
3210	netif_device_attach(dev: ndev);
3211	writel(val: port->vid_context, addr: port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3212	}
3213
3214	writel(val: host_p->vid_context, addr: host_p->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
3215	cpsw_ale_restore(ale: common->ale, data: common->ale_context);
3216
3217	return 0;
3218	}
3219
3220	static const struct dev_pm_ops am65_cpsw_nuss_dev_pm_ops = {
3221	SYSTEM_SLEEP_PM_OPS(am65_cpsw_nuss_suspend, am65_cpsw_nuss_resume)
3222	};
3223
3224	static struct platform_driver am65_cpsw_nuss_driver = {
3225	.driver = {
3226	.name = AM65_CPSW_DRV_NAME,
3227	.of_match_table = am65_cpsw_nuss_of_mtable,
3228	.pm = &am65_cpsw_nuss_dev_pm_ops,
3229	},
3230	.probe = am65_cpsw_nuss_probe,
3231	.remove_new = am65_cpsw_nuss_remove,
3232	};
3233
3234	module_platform_driver(am65_cpsw_nuss_driver);
3235
3236	MODULE_LICENSE("GPL v2");
3237	MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
3238	MODULE_DESCRIPTION("TI AM65 CPSW Ethernet driver");
3239