1	// SPDX-License-Identifier: GPL-2.0-only
2	/*******************************************************************************
3	This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
4	ST Ethernet IPs are built around a Synopsys IP Core.
5
6	Copyright(C) 2007-2011 STMicroelectronics Ltd
7
8
9	Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
10
11	Documentation available at:
12	http://www.stlinux.com
13	Support available at:
14	https://bugzilla.stlinux.com/
15	*******************************************************************************/
16
17	#include <linux/clk.h>
18	#include <linux/kernel.h>
19	#include <linux/interrupt.h>
20	#include <linux/ip.h>
21	#include <linux/tcp.h>
22	#include <linux/skbuff.h>
23	#include <linux/ethtool.h>
24	#include <linux/if_ether.h>
25	#include <linux/crc32.h>
26	#include <linux/mii.h>
27	#include <linux/if.h>
28	#include <linux/if_vlan.h>
29	#include <linux/dma-mapping.h>
30	#include <linux/slab.h>
31	#include <linux/pm_runtime.h>
32	#include <linux/pm_wakeirq.h>
33	#include <linux/prefetch.h>
34	#include <linux/pinctrl/consumer.h>
35	#ifdef CONFIG_DEBUG_FS
36	#include <linux/debugfs.h>
37	#include <linux/seq_file.h>
38	#endif /* CONFIG_DEBUG_FS */
39	#include <linux/net_tstamp.h>
40	#include <linux/phylink.h>
41	#include <linux/udp.h>
42	#include <linux/bpf_trace.h>
43	#include <net/devlink.h>
44	#include <net/page_pool/helpers.h>
45	#include <net/pkt_cls.h>
46	#include <net/xdp_sock_drv.h>
47	#include "stmmac_ptp.h"
48	#include "stmmac_fpe.h"
49	#include "stmmac.h"
50	#include "stmmac_pcs.h"
51	#include "stmmac_xdp.h"
52	#include <linux/reset.h>
53	#include <linux/of_mdio.h>
54	#include "dwmac1000.h"
55	#include "dwxgmac2.h"
56	#include "hwif.h"
57
58	/* As long as the interface is active, we keep the timestamping counter enabled
59	* with fine resolution and binary rollover. This avoid non-monotonic behavior
60	* (clock jumps) when changing timestamping settings at runtime.
61	*/
62	#define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCTRLSSR)
63
64	#define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
65	#define TSO_MAX_BUFF_SIZE (SZ_16K - 1)
66
67	/* Module parameters */
68	#define TX_TIMEO 5000
69	static int watchdog = TX_TIMEO;
70	module_param(watchdog, int, 0644);
71	MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
72
73	static int debug = -1;
74	module_param(debug, int, 0644);
75	MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
76
77	static int phyaddr = -1;
78	module_param(phyaddr, int, 0444);
79	MODULE_PARM_DESC(phyaddr, "Physical device address");
80
81	#define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4)
82
83	/* Limit to make sure XDP TX and slow path can coexist */
84	#define STMMAC_XSK_TX_BUDGET_MAX 256
85	#define STMMAC_TX_XSK_AVAIL 16
86	#define STMMAC_RX_FILL_BATCH 16
87
88	#define STMMAC_XDP_PASS 0
89	#define STMMAC_XDP_CONSUMED BIT(0)
90	#define STMMAC_XDP_TX BIT(1)
91	#define STMMAC_XDP_REDIRECT BIT(2)
92	#define STMMAC_XSK_CONSUMED BIT(3)
93
94	static int flow_ctrl = 0xdead;
95	module_param(flow_ctrl, int, 0644);
96	MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off] (obsolete)");
97
98	static int pause = PAUSE_TIME;
99	module_param(pause, int, 0644);
100	MODULE_PARM_DESC(pause, "Flow Control Pause Time (units of 512 bit times)");
101
102	#define TC_DEFAULT 64
103	static int tc = TC_DEFAULT;
104	module_param(tc, int, 0644);
105	MODULE_PARM_DESC(tc, "DMA threshold control value");
106
107	/* This is unused */
108	#define DEFAULT_BUFSIZE 1536
109	static int buf_sz = DEFAULT_BUFSIZE;
110	module_param(buf_sz, int, 0644);
111	MODULE_PARM_DESC(buf_sz, "DMA buffer size");
112
113	static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
114	NETIF_MSG_LINK | NETIF_MSG_IFUP |
115	NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
116
117	#define STMMAC_DEFAULT_LPI_TIMER 1000
118	static unsigned int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
119	module_param(eee_timer, uint, 0644);
120	MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
121	#define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x))
122
123	/* By default the driver will use the ring mode to manage tx and rx descriptors,
124	* but allow user to force to use the chain instead of the ring
125	*/
126	static unsigned int chain_mode;
127	module_param(chain_mode, int, 0444);
128	MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
129
130	static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
131	/* For MSI interrupts handling */
132	static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id);
133	static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id);
134	static irqreturn_t stmmac_msi_intr_tx(int irq, void *data);
135	static irqreturn_t stmmac_msi_intr_rx(int irq, void *data);
136	static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue);
137	static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue);
138	static void stmmac_reset_queues_param(struct stmmac_priv *priv);
139	static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue);
140	static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue);
141	static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
142	u32 rxmode, u32 chan);
143
144	#ifdef CONFIG_DEBUG_FS
145	static const struct net_device_ops stmmac_netdev_ops;
146	static void stmmac_init_fs(struct net_device *dev);
147	static void stmmac_exit_fs(struct net_device *dev);
148	#endif
149
150	#define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC))
151
152	struct stmmac_devlink_priv {
153	struct stmmac_priv *stmmac_priv;
154	};
155
156	enum stmmac_dl_param_id {
157	STMMAC_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
158	STMMAC_DEVLINK_PARAM_ID_TS_COARSE,
159	};
160
161	/**
162	* stmmac_set_clk_tx_rate() - set the clock rate for the MAC transmit clock
163	* @bsp_priv: BSP private data structure (unused)
164	* @clk_tx_i: the transmit clock
165	* @interface: the selected interface mode
166	* @speed: the speed that the MAC will be operating at
167	*
168	* Set the transmit clock rate for the MAC, normally 2.5MHz for 10Mbps,
169	* 25MHz for 100Mbps and 125MHz for 1Gbps. This is suitable for at least
170	* MII, GMII, RGMII and RMII interface modes. Platforms can hook this into
171	* the plat_data->set_clk_tx_rate method directly, call it via their own
172	* implementation, or implement their own method should they have more
173	* complex requirements. It is intended to only be used in this method.
174	*
175	* plat_data->clk_tx_i must be filled in.
176	*/
177	int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i,
178	phy_interface_t interface, int speed)
179	{
180	long rate = rgmii_clock(speed);
181
182	/* Silently ignore unsupported speeds as rgmii_clock() only
183	* supports 10, 100 and 1000Mbps. We do not want to spit
184	* errors for 2500 and higher speeds here.
185	*/
186	if (rate < 0)
187	return 0;
188
189	return clk_set_rate(clk: clk_tx_i, rate);
190	}
191	EXPORT_SYMBOL_GPL(stmmac_set_clk_tx_rate);
192
193	/**
194	* stmmac_axi_blen_to_mask() - convert a burst length array to reg value
195	* @regval: pointer to a u32 for the resulting register value
196	* @blen: pointer to an array of u32 containing the burst length values in bytes
197	* @len: the number of entries in the @blen array
198	*/
199	void stmmac_axi_blen_to_mask(u32 *regval, const u32 *blen, size_t len)
200	{
201	size_t i;
202	u32 val;
203
204	for (val = i = 0; i < len; i++) {
205	u32 burst = blen[i];
206
207	/* Burst values of zero must be skipped. */
208	if (!burst)
209	continue;
210
211	/* The valid range for the burst length is 4 to 256 inclusive,
212	* and it must be a power of two.
213	*/
214	if (burst < 4 || burst > 256 || !is_power_of_2(n: burst)) {
215	pr_err("stmmac: invalid burst length %u at index %zu\n",
216	burst, i);
217	continue;
218	}
219
220	/* Since burst is a power of two, and the register field starts
221	* with burst = 4, shift right by two bits so bit 0 of the field
222	* corresponds with the minimum value.
223	*/
224	val |= burst >> 2;
225	}
226
227	*regval = FIELD_PREP(DMA_AXI_BLEN_MASK, val);
228	}
229	EXPORT_SYMBOL_GPL(stmmac_axi_blen_to_mask);
230
231	/**
232	* stmmac_verify_args - verify the driver parameters.
233	* Description: it checks the driver parameters and set a default in case of
234	* errors.
235	*/
236	static void stmmac_verify_args(void)
237	{
238	if (unlikely(watchdog < 0))
239	watchdog = TX_TIMEO;
240	if (unlikely((pause < 0) || (pause > 0xffff)))
241	pause = PAUSE_TIME;
242
243	if (flow_ctrl != 0xdead)
244	pr_warn("stmmac: module parameter 'flow_ctrl' is obsolete - please remove from your module configuration\n");
245	}
246
247	static void __stmmac_disable_all_queues(struct stmmac_priv *priv)
248	{
249	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
250	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
251	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
252	u32 queue;
253
254	for (queue = 0; queue < maxq; queue++) {
255	struct stmmac_channel *ch = &priv->channel[queue];
256
257	if (stmmac_xdp_is_enabled(priv) &&
258	test_bit(queue, priv->af_xdp_zc_qps)) {
259	napi_disable(n: &ch->rxtx_napi);
260	continue;
261	}
262
263	if (queue < rx_queues_cnt)
264	napi_disable(n: &ch->rx_napi);
265	if (queue < tx_queues_cnt)
266	napi_disable(n: &ch->tx_napi);
267	}
268	}
269
270	/**
271	* stmmac_disable_all_queues - Disable all queues
272	* @priv: driver private structure
273	*/
274	static void stmmac_disable_all_queues(struct stmmac_priv *priv)
275	{
276	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
277	struct stmmac_rx_queue *rx_q;
278	u32 queue;
279
280	/* synchronize_rcu() needed for pending XDP buffers to drain */
281	for (queue = 0; queue < rx_queues_cnt; queue++) {
282	rx_q = &priv->dma_conf.rx_queue[queue];
283	if (rx_q->xsk_pool) {
284	synchronize_rcu();
285	break;
286	}
287	}
288
289	__stmmac_disable_all_queues(priv);
290	}
291
292	/**
293	* stmmac_enable_all_queues - Enable all queues
294	* @priv: driver private structure
295	*/
296	static void stmmac_enable_all_queues(struct stmmac_priv *priv)
297	{
298	u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
299	u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
300	u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
301	u32 queue;
302
303	for (queue = 0; queue < maxq; queue++) {
304	struct stmmac_channel *ch = &priv->channel[queue];
305
306	if (stmmac_xdp_is_enabled(priv) &&
307	test_bit(queue, priv->af_xdp_zc_qps)) {
308	napi_enable(n: &ch->rxtx_napi);
309	continue;
310	}
311
312	if (queue < rx_queues_cnt)
313	napi_enable(n: &ch->rx_napi);
314	if (queue < tx_queues_cnt)
315	napi_enable(n: &ch->tx_napi);
316	}
317	}
318
319	static void stmmac_service_event_schedule(struct stmmac_priv *priv)
320	{
321	if (!test_bit(STMMAC_DOWN, &priv->state) &&
322	!test_and_set_bit(nr: STMMAC_SERVICE_SCHED, addr: &priv->state))
323	queue_work(wq: priv->wq, work: &priv->service_task);
324	}
325
326	static void stmmac_global_err(struct stmmac_priv *priv)
327	{
328	netif_carrier_off(dev: priv->dev);
329	set_bit(nr: STMMAC_RESET_REQUESTED, addr: &priv->state);
330	stmmac_service_event_schedule(priv);
331	}
332
333	static void print_pkt(unsigned char *buf, int len)
334	{
335	pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
336	print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
337	}
338
339	static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
340	{
341	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
342	u32 avail;
343
344	if (tx_q->dirty_tx > tx_q->cur_tx)
345	avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
346	else
347	avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1;
348
349	return avail;
350	}
351
352	/**
353	* stmmac_rx_dirty - Get RX queue dirty
354	* @priv: driver private structure
355	* @queue: RX queue index
356	*/
357	static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
358	{
359	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
360	u32 dirty;
361
362	if (rx_q->dirty_rx <= rx_q->cur_rx)
363	dirty = rx_q->cur_rx - rx_q->dirty_rx;
364	else
365	dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx;
366
367	return dirty;
368	}
369
370	static bool stmmac_eee_tx_busy(struct stmmac_priv *priv)
371	{
372	u32 tx_cnt = priv->plat->tx_queues_to_use;
373	u32 queue;
374
375	/* check if all TX queues have the work finished */
376	for (queue = 0; queue < tx_cnt; queue++) {
377	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
378
379	if (tx_q->dirty_tx != tx_q->cur_tx)
380	return true; /* still unfinished work */
381	}
382
383	return false;
384	}
385
386	static void stmmac_restart_sw_lpi_timer(struct stmmac_priv *priv)
387	{
388	mod_timer(timer: &priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
389	}
390
391	/**
392	* stmmac_try_to_start_sw_lpi - check and enter in LPI mode
393	* @priv: driver private structure
394	* Description: this function is to verify and enter in LPI mode in case of
395	* EEE.
396	*/
397	static void stmmac_try_to_start_sw_lpi(struct stmmac_priv *priv)
398	{
399	if (stmmac_eee_tx_busy(priv)) {
400	stmmac_restart_sw_lpi_timer(priv);
401	return;
402	}
403
404	/* Check and enter in LPI mode */
405	if (!priv->tx_path_in_lpi_mode)
406	stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_FORCED,
407	priv->tx_lpi_clk_stop, 0);
408	}
409
410	/**
411	* stmmac_stop_sw_lpi - stop transmitting LPI
412	* @priv: driver private structure
413	* Description: When using software-controlled LPI, stop transmitting LPI state.
414	*/
415	static void stmmac_stop_sw_lpi(struct stmmac_priv *priv)
416	{
417	timer_delete_sync(timer: &priv->eee_ctrl_timer);
418	stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0);
419	priv->tx_path_in_lpi_mode = false;
420	}
421
422	/**
423	* stmmac_eee_ctrl_timer - EEE TX SW timer.
424	* @t: timer_list struct containing private info
425	* Description:
426	* if there is no data transfer and if we are not in LPI state,
427	* then MAC Transmitter can be moved to LPI state.
428	*/
429	static void stmmac_eee_ctrl_timer(struct timer_list *t)
430	{
431	struct stmmac_priv *priv = timer_container_of(priv, t, eee_ctrl_timer);
432
433	stmmac_try_to_start_sw_lpi(priv);
434	}
435
436	/* stmmac_get_tx_hwtstamp - get HW TX timestamps
437	* @priv: driver private structure
438	* @p : descriptor pointer
439	* @skb : the socket buffer
440	* Description :
441	* This function will read timestamp from the descriptor & pass it to stack.
442	* and also perform some sanity checks.
443	*/
444	static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
445	struct dma_desc *p, struct sk_buff *skb)
446	{
447	struct skb_shared_hwtstamps shhwtstamp;
448	bool found = false;
449	u64 ns = 0;
450
451	if (!priv->hwts_tx_en)
452	return;
453
454	/* exit if skb doesn't support hw tstamp */
455	if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
456	return;
457
458	/* check tx tstamp status */
459	if (stmmac_get_tx_timestamp_status(priv, p)) {
460	stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
461	found = true;
462	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
463	found = true;
464	}
465
466	if (found) {
467	ns -= priv->plat->cdc_error_adj;
468
469	memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
470	shhwtstamp.hwtstamp = ns_to_ktime(ns);
471
472	netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
473	/* pass tstamp to stack */
474	skb_tstamp_tx(orig_skb: skb, hwtstamps: &shhwtstamp);
475	}
476	}
477
478	/* stmmac_get_rx_hwtstamp - get HW RX timestamps
479	* @priv: driver private structure
480	* @p : descriptor pointer
481	* @np : next descriptor pointer
482	* @skb : the socket buffer
483	* Description :
484	* This function will read received packet's timestamp from the descriptor
485	* and pass it to stack. It also perform some sanity checks.
486	*/
487	static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
488	struct dma_desc *np, struct sk_buff *skb)
489	{
490	struct skb_shared_hwtstamps *shhwtstamp = NULL;
491	struct dma_desc *desc = p;
492	u64 ns = 0;
493
494	if (!priv->hwts_rx_en)
495	return;
496	/* For GMAC4, the valid timestamp is from CTX next desc. */
497	if (dwmac_is_xmac(core_type: priv->plat->core_type))
498	desc = np;
499
500	/* Check if timestamp is available */
501	if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
502	stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
503
504	ns -= priv->plat->cdc_error_adj;
505
506	netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
507	shhwtstamp = skb_hwtstamps(skb);
508	memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
509	shhwtstamp->hwtstamp = ns_to_ktime(ns);
510	} else {
511	netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
512	}
513	}
514
515	static void stmmac_update_subsecond_increment(struct stmmac_priv *priv)
516	{
517	bool xmac = dwmac_is_xmac(core_type: priv->plat->core_type);
518	u32 sec_inc = 0;
519	u64 temp = 0;
520
521	stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags);
522
523	/* program Sub Second Increment reg */
524	stmmac_config_sub_second_increment(priv, priv->ptpaddr,
525	priv->plat->clk_ptp_rate,
526	xmac, &sec_inc);
527	temp = div_u64(dividend: 1000000000ULL, divisor: sec_inc);
528
529	/* Store sub second increment for later use */
530	priv->sub_second_inc = sec_inc;
531
532	/* calculate default added value:
533	* formula is :
534	* addend = (2^32)/freq_div_ratio;
535	* where, freq_div_ratio = 1e9ns/sec_inc
536	*/
537	temp = (u64)(temp << 32);
538	priv->default_addend = div_u64(dividend: temp, divisor: priv->plat->clk_ptp_rate);
539	stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
540	}
541
542	/**
543	* stmmac_hwtstamp_set - control hardware timestamping.
544	* @dev: device pointer.
545	* @config: the timestamping configuration.
546	* @extack: netlink extended ack structure for error reporting.
547	* Description:
548	* This function configures the MAC to enable/disable both outgoing(TX)
549	* and incoming(RX) packets time stamping based on user input.
550	* Return Value:
551	* 0 on success and an appropriate -ve integer on failure.
552	*/
553	static int stmmac_hwtstamp_set(struct net_device *dev,
554	struct kernel_hwtstamp_config *config,
555	struct netlink_ext_ack *extack)
556	{
557	struct stmmac_priv *priv = netdev_priv(dev);
558	u32 ptp_v2 = 0;
559	u32 tstamp_all = 0;
560	u32 ptp_over_ipv4_udp = 0;
561	u32 ptp_over_ipv6_udp = 0;
562	u32 ptp_over_ethernet = 0;
563	u32 snap_type_sel = 0;
564	u32 ts_master_en = 0;
565	u32 ts_event_en = 0;
566
567	if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
568	NL_SET_ERR_MSG_MOD(extack, "No support for HW time stamping");
569	priv->hwts_tx_en = 0;
570	priv->hwts_rx_en = 0;
571
572	return -EOPNOTSUPP;
573	}
574
575	if (!netif_running(dev)) {
576	NL_SET_ERR_MSG_MOD(extack,
577	"Cannot change timestamping configuration while down");
578	return -ENODEV;
579	}
580
581	netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
582	__func__, config->flags, config->tx_type, config->rx_filter);
583
584	if (config->tx_type != HWTSTAMP_TX_OFF &&
585	config->tx_type != HWTSTAMP_TX_ON)
586	return -ERANGE;
587
588	if (priv->adv_ts) {
589	switch (config->rx_filter) {
590	case HWTSTAMP_FILTER_NONE:
591	/* time stamp no incoming packet at all */
592	config->rx_filter = HWTSTAMP_FILTER_NONE;
593	break;
594
595	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
596	/* PTP v1, UDP, any kind of event packet */
597	config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
598	/* 'xmac' hardware can support Sync, Pdelay_Req and
599	* Pdelay_resp by setting bit14 and bits17/16 to 01
600	* This leaves Delay_Req timestamps out.
601	* Enable all events *and* general purpose message
602	* timestamping
603	*/
604	snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
605	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
606	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
607	break;
608
609	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
610	/* PTP v1, UDP, Sync packet */
611	config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
612	/* take time stamp for SYNC messages only */
613	ts_event_en = PTP_TCR_TSEVNTENA;
614
615	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
616	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
617	break;
618
619	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
620	/* PTP v1, UDP, Delay_req packet */
621	config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
622	/* take time stamp for Delay_Req messages only */
623	ts_master_en = PTP_TCR_TSMSTRENA;
624	ts_event_en = PTP_TCR_TSEVNTENA;
625
626	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
627	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
628	break;
629
630	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
631	/* PTP v2, UDP, any kind of event packet */
632	config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
633	ptp_v2 = PTP_TCR_TSVER2ENA;
634	/* take time stamp for all event messages */
635	snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
636
637	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
638	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
639	break;
640
641	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
642	/* PTP v2, UDP, Sync packet */
643	config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
644	ptp_v2 = PTP_TCR_TSVER2ENA;
645	/* take time stamp for SYNC messages only */
646	ts_event_en = PTP_TCR_TSEVNTENA;
647
648	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
649	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
650	break;
651
652	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
653	/* PTP v2, UDP, Delay_req packet */
654	config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
655	ptp_v2 = PTP_TCR_TSVER2ENA;
656	/* take time stamp for Delay_Req messages only */
657	ts_master_en = PTP_TCR_TSMSTRENA;
658	ts_event_en = PTP_TCR_TSEVNTENA;
659
660	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
661	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
662	break;
663
664	case HWTSTAMP_FILTER_PTP_V2_EVENT:
665	/* PTP v2/802.AS1 any layer, any kind of event packet */
666	config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
667	ptp_v2 = PTP_TCR_TSVER2ENA;
668	snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
669	if (priv->synopsys_id < DWMAC_CORE_4_10)
670	ts_event_en = PTP_TCR_TSEVNTENA;
671	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
672	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
673	ptp_over_ethernet = PTP_TCR_TSIPENA;
674	break;
675
676	case HWTSTAMP_FILTER_PTP_V2_SYNC:
677	/* PTP v2/802.AS1, any layer, Sync packet */
678	config->rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
679	ptp_v2 = PTP_TCR_TSVER2ENA;
680	/* take time stamp for SYNC messages only */
681	ts_event_en = PTP_TCR_TSEVNTENA;
682
683	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
684	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
685	ptp_over_ethernet = PTP_TCR_TSIPENA;
686	break;
687
688	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
689	/* PTP v2/802.AS1, any layer, Delay_req packet */
690	config->rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
691	ptp_v2 = PTP_TCR_TSVER2ENA;
692	/* take time stamp for Delay_Req messages only */
693	ts_master_en = PTP_TCR_TSMSTRENA;
694	ts_event_en = PTP_TCR_TSEVNTENA;
695
696	ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
697	ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
698	ptp_over_ethernet = PTP_TCR_TSIPENA;
699	break;
700
701	case HWTSTAMP_FILTER_NTP_ALL:
702	case HWTSTAMP_FILTER_ALL:
703	/* time stamp any incoming packet */
704	config->rx_filter = HWTSTAMP_FILTER_ALL;
705	tstamp_all = PTP_TCR_TSENALL;
706	break;
707
708	default:
709	return -ERANGE;
710	}
711	} else {
712	switch (config->rx_filter) {
713	case HWTSTAMP_FILTER_NONE:
714	config->rx_filter = HWTSTAMP_FILTER_NONE;
715	break;
716	default:
717	/* PTP v1, UDP, any kind of event packet */
718	config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
719	break;
720	}
721	}
722	priv->hwts_rx_en = config->rx_filter != HWTSTAMP_FILTER_NONE;
723	priv->hwts_tx_en = config->tx_type == HWTSTAMP_TX_ON;
724
725	priv->systime_flags = STMMAC_HWTS_ACTIVE;
726	if (!priv->tsfupdt_coarse)
727	priv->systime_flags |= PTP_TCR_TSCFUPDT;
728
729	if (priv->hwts_tx_en || priv->hwts_rx_en) {
730	priv->systime_flags |= tstamp_all | ptp_v2 |
731	ptp_over_ethernet | ptp_over_ipv6_udp |
732	ptp_over_ipv4_udp | ts_event_en |
733	ts_master_en | snap_type_sel;
734	}
735
736	stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags);
737
738	priv->tstamp_config = *config;
739
740	return 0;
741	}
742
743	/**
744	* stmmac_hwtstamp_get - read hardware timestamping.
745	* @dev: device pointer.
746	* @config: the timestamping configuration.
747	* Description:
748	* This function obtain the current hardware timestamping settings
749	* as requested.
750	*/
751	static int stmmac_hwtstamp_get(struct net_device *dev,
752	struct kernel_hwtstamp_config *config)
753	{
754	struct stmmac_priv *priv = netdev_priv(dev);
755
756	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
757	return -EOPNOTSUPP;
758
759	*config = priv->tstamp_config;
760
761	return 0;
762	}
763
764	/**
765	* stmmac_init_tstamp_counter - init hardware timestamping counter
766	* @priv: driver private structure
767	* @systime_flags: timestamping flags
768	* Description:
769	* Initialize hardware counter for packet timestamping.
770	* This is valid as long as the interface is open and not suspended.
771	* Will be rerun after resuming from suspend, case in which the timestamping
772	* flags updated by stmmac_hwtstamp_set() also need to be restored.
773	*/
774	static int stmmac_init_tstamp_counter(struct stmmac_priv *priv,
775	u32 systime_flags)
776	{
777	struct timespec64 now;
778
779	if (!priv->plat->clk_ptp_rate) {
780	netdev_err(dev: priv->dev, format: "Invalid PTP clock rate");
781	return -EINVAL;
782	}
783
784	stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags);
785	priv->systime_flags = systime_flags;
786
787	stmmac_update_subsecond_increment(priv);
788
789	/* initialize system time */
790	ktime_get_real_ts64(tv: &now);
791
792	/* lower 32 bits of tv_sec are safe until y2106 */
793	stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec);
794
795	return 0;
796	}
797
798	/**
799	* stmmac_init_timestamping - initialise timestamping
800	* @priv: driver private structure
801	* Description: this is to verify if the HW supports the PTPv1 or PTPv2.
802	* This is done by looking at the HW cap. register.
803	* This function also registers the ptp driver.
804	*/
805	static int stmmac_init_timestamping(struct stmmac_priv *priv)
806	{
807	bool xmac = dwmac_is_xmac(core_type: priv->plat->core_type);
808	int ret;
809
810	if (priv->plat->ptp_clk_freq_config)
811	priv->plat->ptp_clk_freq_config(priv);
812
813	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) {
814	netdev_info(dev: priv->dev, format: "PTP not supported by HW\n");
815	return -EOPNOTSUPP;
816	}
817
818	ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE |
819	PTP_TCR_TSCFUPDT);
820	if (ret) {
821	netdev_warn(dev: priv->dev, format: "PTP init failed\n");
822	return ret;
823	}
824
825	priv->adv_ts = 0;
826	/* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
827	if (xmac && priv->dma_cap.atime_stamp)
828	priv->adv_ts = 1;
829	/* Dwmac 3.x core with extend_desc can support adv_ts */
830	else if (priv->extend_desc && priv->dma_cap.atime_stamp)
831	priv->adv_ts = 1;
832
833	if (priv->dma_cap.time_stamp)
834	netdev_info(dev: priv->dev, format: "IEEE 1588-2002 Timestamp supported\n");
835
836	if (priv->adv_ts)
837	netdev_info(dev: priv->dev,
838	format: "IEEE 1588-2008 Advanced Timestamp supported\n");
839
840	priv->hwts_tx_en = 0;
841	priv->hwts_rx_en = 0;
842
843	if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY)
844	stmmac_hwtstamp_correct_latency(priv, priv);
845
846	return 0;
847	}
848
849	static void stmmac_setup_ptp(struct stmmac_priv *priv)
850	{
851	int ret;
852
853	ret = clk_prepare_enable(clk: priv->plat->clk_ptp_ref);
854	if (ret < 0)
855	netdev_warn(dev: priv->dev,
856	format: "failed to enable PTP reference clock: %pe\n",
857	ERR_PTR(error: ret));
858
859	if (stmmac_init_timestamping(priv) == 0)
860	stmmac_ptp_register(priv);
861	}
862
863	static void stmmac_release_ptp(struct stmmac_priv *priv)
864	{
865	stmmac_ptp_unregister(priv);
866	clk_disable_unprepare(clk: priv->plat->clk_ptp_ref);
867	}
868
869	/**
870	* stmmac_mac_flow_ctrl - Configure flow control in all queues
871	* @priv: driver private structure
872	* @duplex: duplex passed to the next function
873	* @flow_ctrl: desired flow control modes
874	* Description: It is used for configuring the flow control in all queues
875	*/
876	static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex,
877	unsigned int flow_ctrl)
878	{
879	u32 tx_cnt = priv->plat->tx_queues_to_use;
880
881	stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time,
882	tx_cnt);
883	}
884
885	static unsigned long stmmac_mac_get_caps(struct phylink_config *config,
886	phy_interface_t interface)
887	{
888	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
889
890	/* Refresh the MAC-specific capabilities */
891	stmmac_mac_update_caps(priv);
892
893	config->mac_capabilities = priv->hw->link.caps;
894
895	if (priv->plat->max_speed)
896	phylink_limit_mac_speed(config, max_speed: priv->plat->max_speed);
897
898	return config->mac_capabilities;
899	}
900
901	static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config,
902	phy_interface_t interface)
903	{
904	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
905	struct phylink_pcs *pcs;
906
907	if (priv->plat->select_pcs) {
908	pcs = priv->plat->select_pcs(priv, interface);
909	if (!IS_ERR(ptr: pcs))
910	return pcs;
911	}
912
913	/* The PCS control register is only relevant for SGMII, TBI and RTBI
914	* modes. We no longer support TBI or RTBI, so only configure this
915	* register when operating in SGMII mode with the integrated PCS.
916	*/
917	if (priv->hw->pcs & STMMAC_PCS_SGMII && priv->integrated_pcs)
918	return &priv->integrated_pcs->pcs;
919
920	return NULL;
921	}
922
923	static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
924	const struct phylink_link_state *state)
925	{
926	/* Nothing to do, xpcs_config() handles everything */
927	}
928
929	static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode,
930	phy_interface_t interface)
931	{
932	struct net_device *ndev = to_net_dev(config->dev);
933	struct stmmac_priv *priv = netdev_priv(dev: ndev);
934
935	if (priv->plat->mac_finish)
936	priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, interface);
937
938	return 0;
939	}
940
941	static void stmmac_mac_link_down(struct phylink_config *config,
942	unsigned int mode, phy_interface_t interface)
943	{
944	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
945
946	stmmac_mac_set(priv, priv->ioaddr, false);
947	if (priv->dma_cap.eee)
948	stmmac_set_eee_pls(priv, priv->hw, false);
949
950	if (stmmac_fpe_supported(priv))
951	ethtool_mmsv_link_state_handle(mmsv: &priv->fpe_cfg.mmsv, up: false);
952	}
953
954	static void stmmac_mac_link_up(struct phylink_config *config,
955	struct phy_device *phy,
956	unsigned int mode, phy_interface_t interface,
957	int speed, int duplex,
958	bool tx_pause, bool rx_pause)
959	{
960	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
961	unsigned int flow_ctrl;
962	u32 old_ctrl, ctrl;
963	int ret;
964
965	if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&
966	priv->plat->serdes_powerup)
967	priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv);
968
969	old_ctrl = readl(addr: priv->ioaddr + MAC_CTRL_REG);
970	ctrl = old_ctrl & ~priv->hw->link.speed_mask;
971
972	if (interface == PHY_INTERFACE_MODE_USXGMII) {
973	switch (speed) {
974	case SPEED_10000:
975	ctrl |= priv->hw->link.xgmii.speed10000;
976	break;
977	case SPEED_5000:
978	ctrl |= priv->hw->link.xgmii.speed5000;
979	break;
980	case SPEED_2500:
981	ctrl |= priv->hw->link.xgmii.speed2500;
982	break;
983	default:
984	return;
985	}
986	} else if (interface == PHY_INTERFACE_MODE_XLGMII) {
987	switch (speed) {
988	case SPEED_100000:
989	ctrl |= priv->hw->link.xlgmii.speed100000;
990	break;
991	case SPEED_50000:
992	ctrl |= priv->hw->link.xlgmii.speed50000;
993	break;
994	case SPEED_40000:
995	ctrl |= priv->hw->link.xlgmii.speed40000;
996	break;
997	case SPEED_25000:
998	ctrl |= priv->hw->link.xlgmii.speed25000;
999	break;
1000	case SPEED_10000:
1001	ctrl |= priv->hw->link.xgmii.speed10000;
1002	break;
1003	case SPEED_2500:
1004	ctrl |= priv->hw->link.speed2500;
1005	break;
1006	case SPEED_1000:
1007	ctrl |= priv->hw->link.speed1000;
1008	break;
1009	default:
1010	return;
1011	}
1012	} else {
1013	switch (speed) {
1014	case SPEED_2500:
1015	ctrl |= priv->hw->link.speed2500;
1016	break;
1017	case SPEED_1000:
1018	ctrl |= priv->hw->link.speed1000;
1019	break;
1020	case SPEED_100:
1021	ctrl |= priv->hw->link.speed100;
1022	break;
1023	case SPEED_10:
1024	ctrl |= priv->hw->link.speed10;
1025	break;
1026	default:
1027	return;
1028	}
1029	}
1030
1031	if (priv->plat->fix_mac_speed)
1032	priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode);
1033
1034	if (!duplex)
1035	ctrl &= ~priv->hw->link.duplex;
1036	else
1037	ctrl |= priv->hw->link.duplex;
1038
1039	/* Flow Control operation */
1040	if (rx_pause && tx_pause)
1041	flow_ctrl = FLOW_AUTO;
1042	else if (rx_pause && !tx_pause)
1043	flow_ctrl = FLOW_RX;
1044	else if (!rx_pause && tx_pause)
1045	flow_ctrl = FLOW_TX;
1046	else
1047	flow_ctrl = FLOW_OFF;
1048
1049	stmmac_mac_flow_ctrl(priv, duplex, flow_ctrl);
1050
1051	if (ctrl != old_ctrl)
1052	writel(val: ctrl, addr: priv->ioaddr + MAC_CTRL_REG);
1053
1054	if (priv->plat->set_clk_tx_rate) {
1055	ret = priv->plat->set_clk_tx_rate(priv->plat->bsp_priv,
1056	priv->plat->clk_tx_i,
1057	interface, speed);
1058	if (ret < 0)
1059	netdev_err(dev: priv->dev,
1060	format: "failed to configure %s transmit clock for %dMbps: %pe\n",
1061	phy_modes(interface), speed, ERR_PTR(error: ret));
1062	}
1063
1064	stmmac_mac_set(priv, priv->ioaddr, true);
1065	if (priv->dma_cap.eee)
1066	stmmac_set_eee_pls(priv, priv->hw, true);
1067
1068	if (stmmac_fpe_supported(priv))
1069	ethtool_mmsv_link_state_handle(mmsv: &priv->fpe_cfg.mmsv, up: true);
1070
1071	if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY)
1072	stmmac_hwtstamp_correct_latency(priv, priv);
1073	}
1074
1075	static void stmmac_mac_disable_tx_lpi(struct phylink_config *config)
1076	{
1077	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
1078
1079	priv->eee_active = false;
1080
1081	mutex_lock(&priv->lock);
1082
1083	priv->eee_enabled = false;
1084
1085	netdev_dbg(priv->dev, "disable EEE\n");
1086	priv->eee_sw_timer_en = false;
1087	timer_delete_sync(timer: &priv->eee_ctrl_timer);
1088	stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0);
1089	priv->tx_path_in_lpi_mode = false;
1090
1091	stmmac_set_eee_timer(priv, priv->hw, 0, STMMAC_DEFAULT_TWT_LS);
1092	mutex_unlock(lock: &priv->lock);
1093	}
1094
1095	static int stmmac_mac_enable_tx_lpi(struct phylink_config *config, u32 timer,
1096	bool tx_clk_stop)
1097	{
1098	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
1099	int ret;
1100
1101	priv->tx_lpi_timer = timer;
1102	priv->eee_active = true;
1103
1104	mutex_lock(&priv->lock);
1105
1106	priv->eee_enabled = true;
1107
1108	/* Update the transmit clock stop according to PHY capability if
1109	* the platform allows
1110	*/
1111	if (priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP)
1112	priv->tx_lpi_clk_stop = tx_clk_stop;
1113
1114	stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
1115	STMMAC_DEFAULT_TWT_LS);
1116
1117	/* Try to cnfigure the hardware timer. */
1118	ret = stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_TIMER,
1119	priv->tx_lpi_clk_stop, priv->tx_lpi_timer);
1120
1121	if (ret) {
1122	/* Hardware timer mode not supported, or value out of range.
1123	* Fall back to using software LPI mode
1124	*/
1125	priv->eee_sw_timer_en = true;
1126	stmmac_restart_sw_lpi_timer(priv);
1127	}
1128
1129	mutex_unlock(lock: &priv->lock);
1130	netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
1131
1132	return 0;
1133	}
1134
1135	static int stmmac_mac_wol_set(struct phylink_config *config, u32 wolopts,
1136	const u8 *sopass)
1137	{
1138	struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
1139
1140	device_set_wakeup_enable(dev: priv->device, enable: !!wolopts);
1141
1142	mutex_lock(&priv->lock);
1143	priv->wolopts = wolopts;
1144	mutex_unlock(lock: &priv->lock);
1145
1146	return 0;
1147	}
1148
1149	static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
1150	.mac_get_caps = stmmac_mac_get_caps,
1151	.mac_select_pcs = stmmac_mac_select_pcs,
1152	.mac_config = stmmac_mac_config,
1153	.mac_finish = stmmac_mac_finish,
1154	.mac_link_down = stmmac_mac_link_down,
1155	.mac_link_up = stmmac_mac_link_up,
1156	.mac_disable_tx_lpi = stmmac_mac_disable_tx_lpi,
1157	.mac_enable_tx_lpi = stmmac_mac_enable_tx_lpi,
1158	.mac_wol_set = stmmac_mac_wol_set,
1159	};
1160
1161	/**
1162	* stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
1163	* @priv: driver private structure
1164	* Description: this is to verify if the HW supports the PCS.
1165	* Physical Coding Sublayer (PCS) interface that can be used when the MAC is
1166	* configured for the TBI, RTBI, or SGMII PHY interface.
1167	*/
1168	static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
1169	{
1170	int interface = priv->plat->phy_interface;
1171	int speed = priv->plat->mac_port_sel_speed;
1172
1173	if (priv->dma_cap.pcs && interface == PHY_INTERFACE_MODE_SGMII) {
1174	netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
1175	priv->hw->pcs = STMMAC_PCS_SGMII;
1176
1177	switch (speed) {
1178	case SPEED_10:
1179	case SPEED_100:
1180	case SPEED_1000:
1181	priv->hw->reverse_sgmii_enable = true;
1182	break;
1183
1184	default:
1185	dev_warn(priv->device, "invalid port speed\n");
1186	fallthrough;
1187	case 0:
1188	priv->hw->reverse_sgmii_enable = false;
1189	break;
1190	}
1191	}
1192	}
1193
1194	/**
1195	* stmmac_init_phy - PHY initialization
1196	* @dev: net device structure
1197	* Description: it initializes the driver's PHY state, and attaches the PHY
1198	* to the mac driver.
1199	* Return value:
1200	* 0 on success
1201	*/
1202	static int stmmac_init_phy(struct net_device *dev)
1203	{
1204	struct stmmac_priv *priv = netdev_priv(dev);
1205	int mode = priv->plat->phy_interface;
1206	struct fwnode_handle *phy_fwnode;
1207	struct fwnode_handle *fwnode;
1208	struct ethtool_keee eee;
1209	int ret;
1210
1211	if (!phylink_expects_phy(pl: priv->phylink))
1212	return 0;
1213
1214	if (priv->hw->xpcs &&
1215	xpcs_get_an_mode(xpcs: priv->hw->xpcs, interface: mode) == DW_AN_C73)
1216	return 0;
1217
1218	fwnode = priv->plat->port_node;
1219	if (!fwnode)
1220	fwnode = dev_fwnode(priv->device);
1221
1222	if (fwnode)
1223	phy_fwnode = fwnode_get_phy_node(fwnode);
1224	else
1225	phy_fwnode = NULL;
1226
1227	/* Some DT bindings do not set-up the PHY handle. Let's try to
1228	* manually parse it
1229	*/
1230	if (!phy_fwnode || IS_ERR(ptr: phy_fwnode)) {
1231	int addr = priv->plat->phy_addr;
1232	struct phy_device *phydev;
1233
1234	if (addr < 0) {
1235	netdev_err(dev: priv->dev, format: "no phy found\n");
1236	return -ENODEV;
1237	}
1238
1239	phydev = mdiobus_get_phy(bus: priv->mii, addr);
1240	if (!phydev) {
1241	netdev_err(dev: priv->dev, format: "no phy at addr %d\n", addr);
1242	return -ENODEV;
1243	}
1244
1245	ret = phylink_connect_phy(priv->phylink, phydev);
1246	} else {
1247	fwnode_handle_put(fwnode: phy_fwnode);
1248	ret = phylink_fwnode_phy_connect(pl: priv->phylink, fwnode, flags: 0);
1249	}
1250
1251	if (ret) {
1252	netdev_err(dev: priv->dev, format: "cannot attach to PHY (error: %pe)\n",
1253	ERR_PTR(error: ret));
1254	return ret;
1255	}
1256
1257	/* Configure phylib's copy of the LPI timer. Normally,
1258	* phylink_config.lpi_timer_default would do this, but there is a
1259	* chance that userspace could change the eee_timer setting via sysfs
1260	* before the first open. Thus, preserve existing behaviour.
1261	*/
1262	if (!phylink_ethtool_get_eee(link: priv->phylink, eee: &eee)) {
1263	eee.tx_lpi_timer = priv->tx_lpi_timer;
1264	phylink_ethtool_set_eee(link: priv->phylink, eee: &eee);
1265	}
1266
1267	return 0;
1268	}
1269
1270	static int stmmac_phylink_setup(struct stmmac_priv *priv)
1271	{
1272	struct stmmac_mdio_bus_data *mdio_bus_data;
1273	struct phylink_config *config;
1274	struct fwnode_handle *fwnode;
1275	struct phylink_pcs *pcs;
1276	struct phylink *phylink;
1277
1278	config = &priv->phylink_config;
1279
1280	config->dev = &priv->dev->dev;
1281	config->type = PHYLINK_NETDEV;
1282	config->mac_managed_pm = true;
1283
1284	/* Stmmac always requires an RX clock for hardware initialization */
1285	config->mac_requires_rxc = true;
1286
1287	/* Disable EEE RX clock stop to ensure VLAN register access works
1288	* correctly.
1289	*/
1290	if (!(priv->plat->flags & STMMAC_FLAG_RX_CLK_RUNS_IN_LPI) &&
1291	!(priv->dev->features & NETIF_F_VLAN_FEATURES))
1292	config->eee_rx_clk_stop_enable = true;
1293
1294	/* Set the default transmit clock stop bit based on the platform glue */
1295	priv->tx_lpi_clk_stop = priv->plat->flags &
1296	STMMAC_FLAG_EN_TX_LPI_CLOCKGATING;
1297
1298	mdio_bus_data = priv->plat->mdio_bus_data;
1299	if (mdio_bus_data)
1300	config->default_an_inband = mdio_bus_data->default_an_inband;
1301
1302	/* Get the PHY interface modes (at the PHY end of the link) that
1303	* are supported by the platform.
1304	*/
1305	if (priv->plat->get_interfaces)
1306	priv->plat->get_interfaces(priv, priv->plat->bsp_priv,
1307	config->supported_interfaces);
1308
1309	/* Set the platform/firmware specified interface mode if the
1310	* supported interfaces have not already been provided using
1311	* phy_interface as a last resort.
1312	*/
1313	if (phy_interface_empty(intf: config->supported_interfaces))
1314	__set_bit(priv->plat->phy_interface,
1315	config->supported_interfaces);
1316
1317	/* If we have an xpcs, it defines which PHY interfaces are supported. */
1318	if (priv->hw->xpcs)
1319	pcs = xpcs_to_phylink_pcs(xpcs: priv->hw->xpcs);
1320	else
1321	pcs = priv->hw->phylink_pcs;
1322
1323	if (pcs)
1324	phy_interface_or(dst: config->supported_interfaces,
1325	a: config->supported_interfaces,
1326	b: pcs->supported_interfaces);
1327
1328	if (priv->dma_cap.eee) {
1329	/* Assume all supported interfaces also support LPI */
1330	memcpy(config->lpi_interfaces, config->supported_interfaces,
1331	sizeof(config->lpi_interfaces));
1332
1333	/* All full duplex speeds above 100Mbps are supported */
1334	config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD;
1335	config->lpi_timer_default = eee_timer * 1000;
1336	config->eee_enabled_default = true;
1337	}
1338
1339	config->wol_phy_speed_ctrl = true;
1340	if (priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL) {
1341	config->wol_phy_legacy = true;
1342	} else {
1343	if (priv->dma_cap.pmt_remote_wake_up)
1344	config->wol_mac_support |= WAKE_UCAST;
1345	if (priv->dma_cap.pmt_magic_frame)
1346	config->wol_mac_support |= WAKE_MAGIC;
1347	}
1348
1349	fwnode = priv->plat->port_node;
1350	if (!fwnode)
1351	fwnode = dev_fwnode(priv->device);
1352
1353	phylink = phylink_create(config, fwnode, priv->plat->phy_interface,
1354	&stmmac_phylink_mac_ops);
1355	if (IS_ERR(ptr: phylink))
1356	return PTR_ERR(ptr: phylink);
1357
1358	priv->phylink = phylink;
1359	return 0;
1360	}
1361
1362	static void stmmac_display_rx_rings(struct stmmac_priv *priv,
1363	struct stmmac_dma_conf *dma_conf)
1364	{
1365	u32 rx_cnt = priv->plat->rx_queues_to_use;
1366	unsigned int desc_size;
1367	void *head_rx;
1368	u32 queue;
1369
1370	/* Display RX rings */
1371	for (queue = 0; queue < rx_cnt; queue++) {
1372	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
1373
1374	pr_info("\tRX Queue %u rings\n", queue);
1375
1376	if (priv->extend_desc) {
1377	head_rx = (void *)rx_q->dma_erx;
1378	desc_size = sizeof(struct dma_extended_desc);
1379	} else {
1380	head_rx = (void *)rx_q->dma_rx;
1381	desc_size = sizeof(struct dma_desc);
1382	}
1383
1384	/* Display RX ring */
1385	stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true,
1386	rx_q->dma_rx_phy, desc_size);
1387	}
1388	}
1389
1390	static void stmmac_display_tx_rings(struct stmmac_priv *priv,
1391	struct stmmac_dma_conf *dma_conf)
1392	{
1393	u32 tx_cnt = priv->plat->tx_queues_to_use;
1394	unsigned int desc_size;
1395	void *head_tx;
1396	u32 queue;
1397
1398	/* Display TX rings */
1399	for (queue = 0; queue < tx_cnt; queue++) {
1400	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
1401
1402	pr_info("\tTX Queue %d rings\n", queue);
1403
1404	if (priv->extend_desc) {
1405	head_tx = (void *)tx_q->dma_etx;
1406	desc_size = sizeof(struct dma_extended_desc);
1407	} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
1408	head_tx = (void *)tx_q->dma_entx;
1409	desc_size = sizeof(struct dma_edesc);
1410	} else {
1411	head_tx = (void *)tx_q->dma_tx;
1412	desc_size = sizeof(struct dma_desc);
1413	}
1414
1415	stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false,
1416	tx_q->dma_tx_phy, desc_size);
1417	}
1418	}
1419
1420	static void stmmac_display_rings(struct stmmac_priv *priv,
1421	struct stmmac_dma_conf *dma_conf)
1422	{
1423	/* Display RX ring */
1424	stmmac_display_rx_rings(priv, dma_conf);
1425
1426	/* Display TX ring */
1427	stmmac_display_tx_rings(priv, dma_conf);
1428	}
1429
1430	static unsigned int stmmac_rx_offset(struct stmmac_priv *priv)
1431	{
1432	if (stmmac_xdp_is_enabled(priv))
1433	return XDP_PACKET_HEADROOM;
1434
1435	return NET_SKB_PAD;
1436	}
1437
1438	static int stmmac_set_bfsize(int mtu)
1439	{
1440	int ret;
1441
1442	if (mtu >= BUF_SIZE_8KiB)
1443	ret = BUF_SIZE_16KiB;
1444	else if (mtu >= BUF_SIZE_4KiB)
1445	ret = BUF_SIZE_8KiB;
1446	else if (mtu >= BUF_SIZE_2KiB)
1447	ret = BUF_SIZE_4KiB;
1448	else if (mtu > DEFAULT_BUFSIZE)
1449	ret = BUF_SIZE_2KiB;
1450	else
1451	ret = DEFAULT_BUFSIZE;
1452
1453	return ret;
1454	}
1455
1456	/**
1457	* stmmac_clear_rx_descriptors - clear RX descriptors
1458	* @priv: driver private structure
1459	* @dma_conf: structure to take the dma data
1460	* @queue: RX queue index
1461	* Description: this function is called to clear the RX descriptors
1462	* in case of both basic and extended descriptors are used.
1463	*/
1464	static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv,
1465	struct stmmac_dma_conf *dma_conf,
1466	u32 queue)
1467	{
1468	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
1469	int i;
1470
1471	/* Clear the RX descriptors */
1472	for (i = 0; i < dma_conf->dma_rx_size; i++)
1473	if (priv->extend_desc)
1474	stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
1475	priv->use_riwt, priv->mode,
1476	(i == dma_conf->dma_rx_size - 1),
1477	dma_conf->dma_buf_sz);
1478	else
1479	stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
1480	priv->use_riwt, priv->mode,
1481	(i == dma_conf->dma_rx_size - 1),
1482	dma_conf->dma_buf_sz);
1483	}
1484
1485	/**
1486	* stmmac_clear_tx_descriptors - clear tx descriptors
1487	* @priv: driver private structure
1488	* @dma_conf: structure to take the dma data
1489	* @queue: TX queue index.
1490	* Description: this function is called to clear the TX descriptors
1491	* in case of both basic and extended descriptors are used.
1492	*/
1493	static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv,
1494	struct stmmac_dma_conf *dma_conf,
1495	u32 queue)
1496	{
1497	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
1498	int i;
1499
1500	/* Clear the TX descriptors */
1501	for (i = 0; i < dma_conf->dma_tx_size; i++) {
1502	int last = (i == (dma_conf->dma_tx_size - 1));
1503	struct dma_desc *p;
1504
1505	if (priv->extend_desc)
1506	p = &tx_q->dma_etx[i].basic;
1507	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1508	p = &tx_q->dma_entx[i].basic;
1509	else
1510	p = &tx_q->dma_tx[i];
1511
1512	stmmac_init_tx_desc(priv, p, priv->mode, last);
1513	}
1514	}
1515
1516	/**
1517	* stmmac_clear_descriptors - clear descriptors
1518	* @priv: driver private structure
1519	* @dma_conf: structure to take the dma data
1520	* Description: this function is called to clear the TX and RX descriptors
1521	* in case of both basic and extended descriptors are used.
1522	*/
1523	static void stmmac_clear_descriptors(struct stmmac_priv *priv,
1524	struct stmmac_dma_conf *dma_conf)
1525	{
1526	u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1527	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1528	u32 queue;
1529
1530	/* Clear the RX descriptors */
1531	for (queue = 0; queue < rx_queue_cnt; queue++)
1532	stmmac_clear_rx_descriptors(priv, dma_conf, queue);
1533
1534	/* Clear the TX descriptors */
1535	for (queue = 0; queue < tx_queue_cnt; queue++)
1536	stmmac_clear_tx_descriptors(priv, dma_conf, queue);
1537	}
1538
1539	/**
1540	* stmmac_init_rx_buffers - init the RX descriptor buffer.
1541	* @priv: driver private structure
1542	* @dma_conf: structure to take the dma data
1543	* @p: descriptor pointer
1544	* @i: descriptor index
1545	* @flags: gfp flag
1546	* @queue: RX queue index
1547	* Description: this function is called to allocate a receive buffer, perform
1548	* the DMA mapping and init the descriptor.
1549	*/
1550	static int stmmac_init_rx_buffers(struct stmmac_priv *priv,
1551	struct stmmac_dma_conf *dma_conf,
1552	struct dma_desc *p,
1553	int i, gfp_t flags, u32 queue)
1554	{
1555	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
1556	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1557	gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
1558
1559	if (priv->dma_cap.host_dma_width <= 32)
1560	gfp |= GFP_DMA32;
1561
1562	if (!buf->page) {
1563	buf->page = page_pool_alloc_pages(pool: rx_q->page_pool, gfp);
1564	if (!buf->page)
1565	return -ENOMEM;
1566	buf->page_offset = stmmac_rx_offset(priv);
1567	}
1568
1569	if (priv->sph_active && !buf->sec_page) {
1570	buf->sec_page = page_pool_alloc_pages(pool: rx_q->page_pool, gfp);
1571	if (!buf->sec_page)
1572	return -ENOMEM;
1573
1574	buf->sec_addr = page_pool_get_dma_addr(page: buf->sec_page);
1575	stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
1576	} else {
1577	buf->sec_page = NULL;
1578	stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
1579	}
1580
1581	buf->addr = page_pool_get_dma_addr(page: buf->page) + buf->page_offset;
1582
1583	stmmac_set_desc_addr(priv, p, buf->addr);
1584	if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB)
1585	stmmac_init_desc3(priv, p);
1586
1587	return 0;
1588	}
1589
1590	/**
1591	* stmmac_free_rx_buffer - free RX dma buffers
1592	* @priv: private structure
1593	* @rx_q: RX queue
1594	* @i: buffer index.
1595	*/
1596	static void stmmac_free_rx_buffer(struct stmmac_priv *priv,
1597	struct stmmac_rx_queue *rx_q,
1598	int i)
1599	{
1600	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1601
1602	if (buf->page)
1603	page_pool_put_full_page(pool: rx_q->page_pool, page: buf->page, allow_direct: false);
1604	buf->page = NULL;
1605
1606	if (buf->sec_page)
1607	page_pool_put_full_page(pool: rx_q->page_pool, page: buf->sec_page, allow_direct: false);
1608	buf->sec_page = NULL;
1609	}
1610
1611	/**
1612	* stmmac_free_tx_buffer - free RX dma buffers
1613	* @priv: private structure
1614	* @dma_conf: structure to take the dma data
1615	* @queue: RX queue index
1616	* @i: buffer index.
1617	*/
1618	static void stmmac_free_tx_buffer(struct stmmac_priv *priv,
1619	struct stmmac_dma_conf *dma_conf,
1620	u32 queue, int i)
1621	{
1622	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
1623
1624	if (tx_q->tx_skbuff_dma[i].buf &&
1625	tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) {
1626	if (tx_q->tx_skbuff_dma[i].map_as_page)
1627	dma_unmap_page(priv->device,
1628	tx_q->tx_skbuff_dma[i].buf,
1629	tx_q->tx_skbuff_dma[i].len,
1630	DMA_TO_DEVICE);
1631	else
1632	dma_unmap_single(priv->device,
1633	tx_q->tx_skbuff_dma[i].buf,
1634	tx_q->tx_skbuff_dma[i].len,
1635	DMA_TO_DEVICE);
1636	}
1637
1638	if (tx_q->xdpf[i] &&
1639	(tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX ||
1640	tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) {
1641	xdp_return_frame(xdpf: tx_q->xdpf[i]);
1642	tx_q->xdpf[i] = NULL;
1643	}
1644
1645	if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX)
1646	tx_q->xsk_frames_done++;
1647
1648	if (tx_q->tx_skbuff[i] &&
1649	tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) {
1650	dev_kfree_skb_any(skb: tx_q->tx_skbuff[i]);
1651	tx_q->tx_skbuff[i] = NULL;
1652	}
1653
1654	tx_q->tx_skbuff_dma[i].buf = 0;
1655	tx_q->tx_skbuff_dma[i].map_as_page = false;
1656	}
1657
1658	/**
1659	* dma_free_rx_skbufs - free RX dma buffers
1660	* @priv: private structure
1661	* @dma_conf: structure to take the dma data
1662	* @queue: RX queue index
1663	*/
1664	static void dma_free_rx_skbufs(struct stmmac_priv *priv,
1665	struct stmmac_dma_conf *dma_conf,
1666	u32 queue)
1667	{
1668	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
1669	int i;
1670
1671	for (i = 0; i < dma_conf->dma_rx_size; i++)
1672	stmmac_free_rx_buffer(priv, rx_q, i);
1673	}
1674
1675	static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv,
1676	struct stmmac_dma_conf *dma_conf,
1677	u32 queue, gfp_t flags)
1678	{
1679	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
1680	int i;
1681
1682	for (i = 0; i < dma_conf->dma_rx_size; i++) {
1683	struct dma_desc *p;
1684	int ret;
1685
1686	if (priv->extend_desc)
1687	p = &((rx_q->dma_erx + i)->basic);
1688	else
1689	p = rx_q->dma_rx + i;
1690
1691	ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags,
1692	queue);
1693	if (ret)
1694	return ret;
1695
1696	rx_q->buf_alloc_num++;
1697	}
1698
1699	return 0;
1700	}
1701
1702	/**
1703	* dma_free_rx_xskbufs - free RX dma buffers from XSK pool
1704	* @priv: private structure
1705	* @dma_conf: structure to take the dma data
1706	* @queue: RX queue index
1707	*/
1708	static void dma_free_rx_xskbufs(struct stmmac_priv *priv,
1709	struct stmmac_dma_conf *dma_conf,
1710	u32 queue)
1711	{
1712	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
1713	int i;
1714
1715	for (i = 0; i < dma_conf->dma_rx_size; i++) {
1716	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1717
1718	if (!buf->xdp)
1719	continue;
1720
1721	xsk_buff_free(xdp: buf->xdp);
1722	buf->xdp = NULL;
1723	}
1724	}
1725
1726	static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv,
1727	struct stmmac_dma_conf *dma_conf,
1728	u32 queue)
1729	{
1730	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
1731	int i;
1732
1733	/* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes)
1734	* in struct xdp_buff_xsk to stash driver specific information. Thus,
1735	* use this macro to make sure no size violations.
1736	*/
1737	XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff);
1738
1739	for (i = 0; i < dma_conf->dma_rx_size; i++) {
1740	struct stmmac_rx_buffer *buf;
1741	dma_addr_t dma_addr;
1742	struct dma_desc *p;
1743
1744	if (priv->extend_desc)
1745	p = (struct dma_desc *)(rx_q->dma_erx + i);
1746	else
1747	p = rx_q->dma_rx + i;
1748
1749	buf = &rx_q->buf_pool[i];
1750
1751	buf->xdp = xsk_buff_alloc(pool: rx_q->xsk_pool);
1752	if (!buf->xdp)
1753	return -ENOMEM;
1754
1755	dma_addr = xsk_buff_xdp_get_dma(xdp: buf->xdp);
1756	stmmac_set_desc_addr(priv, p, dma_addr);
1757	rx_q->buf_alloc_num++;
1758	}
1759
1760	return 0;
1761	}
1762
1763	static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue)
1764	{
1765	if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps))
1766	return NULL;
1767
1768	return xsk_get_pool_from_qid(dev: priv->dev, queue_id: queue);
1769	}
1770
1771	/**
1772	* __init_dma_rx_desc_rings - init the RX descriptor ring (per queue)
1773	* @priv: driver private structure
1774	* @dma_conf: structure to take the dma data
1775	* @queue: RX queue index
1776	* @flags: gfp flag.
1777	* Description: this function initializes the DMA RX descriptors
1778	* and allocates the socket buffers. It supports the chained and ring
1779	* modes.
1780	*/
1781	static int __init_dma_rx_desc_rings(struct stmmac_priv *priv,
1782	struct stmmac_dma_conf *dma_conf,
1783	u32 queue, gfp_t flags)
1784	{
1785	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
1786	int ret;
1787
1788	netif_dbg(priv, probe, priv->dev,
1789	"(%s) dma_rx_phy=0x%08x\n", __func__,
1790	(u32)rx_q->dma_rx_phy);
1791
1792	stmmac_clear_rx_descriptors(priv, dma_conf, queue);
1793
1794	xdp_rxq_info_unreg_mem_model(xdp_rxq: &rx_q->xdp_rxq);
1795
1796	rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);
1797
1798	if (rx_q->xsk_pool) {
1799	WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq,
1800	MEM_TYPE_XSK_BUFF_POOL,
1801	NULL));
1802	netdev_info(dev: priv->dev,
1803	format: "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n",
1804	rx_q->queue_index);
1805	xsk_pool_set_rxq_info(pool: rx_q->xsk_pool, rxq: &rx_q->xdp_rxq);
1806	} else {
1807	WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq,
1808	MEM_TYPE_PAGE_POOL,
1809	rx_q->page_pool));
1810	netdev_info(dev: priv->dev,
1811	format: "Register MEM_TYPE_PAGE_POOL RxQ-%d\n",
1812	rx_q->queue_index);
1813	}
1814
1815	if (rx_q->xsk_pool) {
1816	/* RX XDP ZC buffer pool may not be populated, e.g.
1817	* xdpsock TX-only.
1818	*/
1819	stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue);
1820	} else {
1821	ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags);
1822	if (ret < 0)
1823	return -ENOMEM;
1824	}
1825
1826	/* Setup the chained descriptor addresses */
1827	if (priv->mode == STMMAC_CHAIN_MODE) {
1828	if (priv->extend_desc)
1829	stmmac_mode_init(priv, rx_q->dma_erx,
1830	rx_q->dma_rx_phy,
1831	dma_conf->dma_rx_size, 1);
1832	else
1833	stmmac_mode_init(priv, rx_q->dma_rx,
1834	rx_q->dma_rx_phy,
1835	dma_conf->dma_rx_size, 0);
1836	}
1837
1838	return 0;
1839	}
1840
1841	static int init_dma_rx_desc_rings(struct net_device *dev,
1842	struct stmmac_dma_conf *dma_conf,
1843	gfp_t flags)
1844	{
1845	struct stmmac_priv *priv = netdev_priv(dev);
1846	u32 rx_count = priv->plat->rx_queues_to_use;
1847	int queue;
1848	int ret;
1849
1850	/* RX INITIALIZATION */
1851	netif_dbg(priv, probe, priv->dev,
1852	"SKB addresses:\nskb\t\tskb data\tdma data\n");
1853
1854	for (queue = 0; queue < rx_count; queue++) {
1855	ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags);
1856	if (ret)
1857	goto err_init_rx_buffers;
1858	}
1859
1860	return 0;
1861
1862	err_init_rx_buffers:
1863	while (queue >= 0) {
1864	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
1865
1866	if (rx_q->xsk_pool)
1867	dma_free_rx_xskbufs(priv, dma_conf, queue);
1868	else
1869	dma_free_rx_skbufs(priv, dma_conf, queue);
1870
1871	rx_q->buf_alloc_num = 0;
1872	rx_q->xsk_pool = NULL;
1873
1874	queue--;
1875	}
1876
1877	return ret;
1878	}
1879
1880	/**
1881	* __init_dma_tx_desc_rings - init the TX descriptor ring (per queue)
1882	* @priv: driver private structure
1883	* @dma_conf: structure to take the dma data
1884	* @queue: TX queue index
1885	* Description: this function initializes the DMA TX descriptors
1886	* and allocates the socket buffers. It supports the chained and ring
1887	* modes.
1888	*/
1889	static int __init_dma_tx_desc_rings(struct stmmac_priv *priv,
1890	struct stmmac_dma_conf *dma_conf,
1891	u32 queue)
1892	{
1893	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
1894	int i;
1895
1896	netif_dbg(priv, probe, priv->dev,
1897	"(%s) dma_tx_phy=0x%08x\n", __func__,
1898	(u32)tx_q->dma_tx_phy);
1899
1900	/* Setup the chained descriptor addresses */
1901	if (priv->mode == STMMAC_CHAIN_MODE) {
1902	if (priv->extend_desc)
1903	stmmac_mode_init(priv, tx_q->dma_etx,
1904	tx_q->dma_tx_phy,
1905	dma_conf->dma_tx_size, 1);
1906	else if (!(tx_q->tbs & STMMAC_TBS_AVAIL))
1907	stmmac_mode_init(priv, tx_q->dma_tx,
1908	tx_q->dma_tx_phy,
1909	dma_conf->dma_tx_size, 0);
1910	}
1911
1912	tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);
1913
1914	for (i = 0; i < dma_conf->dma_tx_size; i++) {
1915	struct dma_desc *p;
1916
1917	if (priv->extend_desc)
1918	p = &((tx_q->dma_etx + i)->basic);
1919	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1920	p = &((tx_q->dma_entx + i)->basic);
1921	else
1922	p = tx_q->dma_tx + i;
1923
1924	stmmac_clear_desc(priv, p);
1925
1926	tx_q->tx_skbuff_dma[i].buf = 0;
1927	tx_q->tx_skbuff_dma[i].map_as_page = false;
1928	tx_q->tx_skbuff_dma[i].len = 0;
1929	tx_q->tx_skbuff_dma[i].last_segment = false;
1930	tx_q->tx_skbuff[i] = NULL;
1931	}
1932
1933	return 0;
1934	}
1935
1936	static int init_dma_tx_desc_rings(struct net_device *dev,
1937	struct stmmac_dma_conf *dma_conf)
1938	{
1939	struct stmmac_priv *priv = netdev_priv(dev);
1940	u32 tx_queue_cnt;
1941	u32 queue;
1942
1943	tx_queue_cnt = priv->plat->tx_queues_to_use;
1944
1945	for (queue = 0; queue < tx_queue_cnt; queue++)
1946	__init_dma_tx_desc_rings(priv, dma_conf, queue);
1947
1948	return 0;
1949	}
1950
1951	/**
1952	* init_dma_desc_rings - init the RX/TX descriptor rings
1953	* @dev: net device structure
1954	* @dma_conf: structure to take the dma data
1955	* @flags: gfp flag.
1956	* Description: this function initializes the DMA RX/TX descriptors
1957	* and allocates the socket buffers. It supports the chained and ring
1958	* modes.
1959	*/
1960	static int init_dma_desc_rings(struct net_device *dev,
1961	struct stmmac_dma_conf *dma_conf,
1962	gfp_t flags)
1963	{
1964	struct stmmac_priv *priv = netdev_priv(dev);
1965	int ret;
1966
1967	ret = init_dma_rx_desc_rings(dev, dma_conf, flags);
1968	if (ret)
1969	return ret;
1970
1971	ret = init_dma_tx_desc_rings(dev, dma_conf);
1972
1973	stmmac_clear_descriptors(priv, dma_conf);
1974
1975	if (netif_msg_hw(priv))
1976	stmmac_display_rings(priv, dma_conf);
1977
1978	return ret;
1979	}
1980
1981	/**
1982	* dma_free_tx_skbufs - free TX dma buffers
1983	* @priv: private structure
1984	* @dma_conf: structure to take the dma data
1985	* @queue: TX queue index
1986	*/
1987	static void dma_free_tx_skbufs(struct stmmac_priv *priv,
1988	struct stmmac_dma_conf *dma_conf,
1989	u32 queue)
1990	{
1991	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
1992	int i;
1993
1994	tx_q->xsk_frames_done = 0;
1995
1996	for (i = 0; i < dma_conf->dma_tx_size; i++)
1997	stmmac_free_tx_buffer(priv, dma_conf, queue, i);
1998
1999	if (tx_q->xsk_pool && tx_q->xsk_frames_done) {
2000	xsk_tx_completed(pool: tx_q->xsk_pool, nb_entries: tx_q->xsk_frames_done);
2001	tx_q->xsk_frames_done = 0;
2002	tx_q->xsk_pool = NULL;
2003	}
2004	}
2005
2006	/**
2007	* stmmac_free_tx_skbufs - free TX skb buffers
2008	* @priv: private structure
2009	*/
2010	static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)
2011	{
2012	u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
2013	u32 queue;
2014
2015	for (queue = 0; queue < tx_queue_cnt; queue++)
2016	dma_free_tx_skbufs(priv, dma_conf: &priv->dma_conf, queue);
2017	}
2018
2019	/**
2020	* __free_dma_rx_desc_resources - free RX dma desc resources (per queue)
2021	* @priv: private structure
2022	* @dma_conf: structure to take the dma data
2023	* @queue: RX queue index
2024	*/
2025	static void __free_dma_rx_desc_resources(struct stmmac_priv *priv,
2026	struct stmmac_dma_conf *dma_conf,
2027	u32 queue)
2028	{
2029	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
2030
2031	/* Release the DMA RX socket buffers */
2032	if (rx_q->xsk_pool)
2033	dma_free_rx_xskbufs(priv, dma_conf, queue);
2034	else
2035	dma_free_rx_skbufs(priv, dma_conf, queue);
2036
2037	rx_q->buf_alloc_num = 0;
2038	rx_q->xsk_pool = NULL;
2039
2040	/* Free DMA regions of consistent memory previously allocated */
2041	if (!priv->extend_desc)
2042	dma_free_coherent(dev: priv->device, size: dma_conf->dma_rx_size *
2043	sizeof(struct dma_desc),
2044	cpu_addr: rx_q->dma_rx, dma_handle: rx_q->dma_rx_phy);
2045	else
2046	dma_free_coherent(dev: priv->device, size: dma_conf->dma_rx_size *
2047	sizeof(struct dma_extended_desc),
2048	cpu_addr: rx_q->dma_erx, dma_handle: rx_q->dma_rx_phy);
2049
2050	if (xdp_rxq_info_is_reg(xdp_rxq: &rx_q->xdp_rxq))
2051	xdp_rxq_info_unreg(xdp_rxq: &rx_q->xdp_rxq);
2052
2053	kfree(objp: rx_q->buf_pool);
2054	if (rx_q->page_pool)
2055	page_pool_destroy(pool: rx_q->page_pool);
2056	}
2057
2058	static void free_dma_rx_desc_resources(struct stmmac_priv *priv,
2059	struct stmmac_dma_conf *dma_conf)
2060	{
2061	u32 rx_count = priv->plat->rx_queues_to_use;
2062	u32 queue;
2063
2064	/* Free RX queue resources */
2065	for (queue = 0; queue < rx_count; queue++)
2066	__free_dma_rx_desc_resources(priv, dma_conf, queue);
2067	}
2068
2069	/**
2070	* __free_dma_tx_desc_resources - free TX dma desc resources (per queue)
2071	* @priv: private structure
2072	* @dma_conf: structure to take the dma data
2073	* @queue: TX queue index
2074	*/
2075	static void __free_dma_tx_desc_resources(struct stmmac_priv *priv,
2076	struct stmmac_dma_conf *dma_conf,
2077	u32 queue)
2078	{
2079	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
2080	size_t size;
2081	void *addr;
2082
2083	/* Release the DMA TX socket buffers */
2084	dma_free_tx_skbufs(priv, dma_conf, queue);
2085
2086	if (priv->extend_desc) {
2087	size = sizeof(struct dma_extended_desc);
2088	addr = tx_q->dma_etx;
2089	} else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
2090	size = sizeof(struct dma_edesc);
2091	addr = tx_q->dma_entx;
2092	} else {
2093	size = sizeof(struct dma_desc);
2094	addr = tx_q->dma_tx;
2095	}
2096
2097	size *= dma_conf->dma_tx_size;
2098
2099	dma_free_coherent(dev: priv->device, size, cpu_addr: addr, dma_handle: tx_q->dma_tx_phy);
2100
2101	kfree(objp: tx_q->tx_skbuff_dma);
2102	kfree(objp: tx_q->tx_skbuff);
2103	}
2104
2105	static void free_dma_tx_desc_resources(struct stmmac_priv *priv,
2106	struct stmmac_dma_conf *dma_conf)
2107	{
2108	u32 tx_count = priv->plat->tx_queues_to_use;
2109	u32 queue;
2110
2111	/* Free TX queue resources */
2112	for (queue = 0; queue < tx_count; queue++)
2113	__free_dma_tx_desc_resources(priv, dma_conf, queue);
2114	}
2115
2116	/**
2117	* __alloc_dma_rx_desc_resources - alloc RX resources (per queue).
2118	* @priv: private structure
2119	* @dma_conf: structure to take the dma data
2120	* @queue: RX queue index
2121	* Description: according to which descriptor can be used (extend or basic)
2122	* this function allocates the resources for TX and RX paths. In case of
2123	* reception, for example, it pre-allocated the RX socket buffer in order to
2124	* allow zero-copy mechanism.
2125	*/
2126	static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
2127	struct stmmac_dma_conf *dma_conf,
2128	u32 queue)
2129	{
2130	struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue];
2131	struct stmmac_channel *ch = &priv->channel[queue];
2132	bool xdp_prog = stmmac_xdp_is_enabled(priv);
2133	struct page_pool_params pp_params = { 0 };
2134	unsigned int dma_buf_sz_pad, num_pages;
2135	unsigned int napi_id;
2136	int ret;
2137
2138	dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz +
2139	SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2140	num_pages = DIV_ROUND_UP(dma_buf_sz_pad, PAGE_SIZE);
2141
2142	rx_q->queue_index = queue;
2143	rx_q->priv_data = priv;
2144	rx_q->napi_skb_frag_size = num_pages * PAGE_SIZE;
2145
2146	pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
2147	pp_params.pool_size = dma_conf->dma_rx_size;
2148	pp_params.order = order_base_2(num_pages);
2149	pp_params.nid = dev_to_node(dev: priv->device);
2150	pp_params.dev = priv->device;
2151	pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
2152	pp_params.offset = stmmac_rx_offset(priv);
2153	pp_params.max_len = dma_conf->dma_buf_sz;
2154
2155	if (priv->sph_active) {
2156	pp_params.offset = 0;
2157	pp_params.max_len += stmmac_rx_offset(priv);
2158	}
2159
2160	rx_q->page_pool = page_pool_create(params: &pp_params);
2161	if (IS_ERR(ptr: rx_q->page_pool)) {
2162	ret = PTR_ERR(ptr: rx_q->page_pool);
2163	rx_q->page_pool = NULL;
2164	return ret;
2165	}
2166
2167	rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size,
2168	sizeof(*rx_q->buf_pool),
2169	GFP_KERNEL);
2170	if (!rx_q->buf_pool)
2171	return -ENOMEM;
2172
2173	if (priv->extend_desc) {
2174	rx_q->dma_erx = dma_alloc_coherent(dev: priv->device,
2175	size: dma_conf->dma_rx_size *
2176	sizeof(struct dma_extended_desc),
2177	dma_handle: &rx_q->dma_rx_phy,
2178	GFP_KERNEL);
2179	if (!rx_q->dma_erx)
2180	return -ENOMEM;
2181
2182	} else {
2183	rx_q->dma_rx = dma_alloc_coherent(dev: priv->device,
2184	size: dma_conf->dma_rx_size *
2185	sizeof(struct dma_desc),
2186	dma_handle: &rx_q->dma_rx_phy,
2187	GFP_KERNEL);
2188	if (!rx_q->dma_rx)
2189	return -ENOMEM;
2190	}
2191
2192	if (stmmac_xdp_is_enabled(priv) &&
2193	test_bit(queue, priv->af_xdp_zc_qps))
2194	napi_id = ch->rxtx_napi.napi_id;
2195	else
2196	napi_id = ch->rx_napi.napi_id;
2197
2198	ret = xdp_rxq_info_reg(xdp_rxq: &rx_q->xdp_rxq, dev: priv->dev,
2199	queue_index: rx_q->queue_index,
2200	napi_id);
2201	if (ret) {
2202	netdev_err(dev: priv->dev, format: "Failed to register xdp rxq info\n");
2203	return -EINVAL;
2204	}
2205
2206	return 0;
2207	}
2208
2209	static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv,
2210	struct stmmac_dma_conf *dma_conf)
2211	{
2212	u32 rx_count = priv->plat->rx_queues_to_use;
2213	u32 queue;
2214	int ret;
2215
2216	/* RX queues buffers and DMA */
2217	for (queue = 0; queue < rx_count; queue++) {
2218	ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue);
2219	if (ret)
2220	goto err_dma;
2221	}
2222
2223	return 0;
2224
2225	err_dma:
2226	free_dma_rx_desc_resources(priv, dma_conf);
2227
2228	return ret;
2229	}
2230
2231	/**
2232	* __alloc_dma_tx_desc_resources - alloc TX resources (per queue).
2233	* @priv: private structure
2234	* @dma_conf: structure to take the dma data
2235	* @queue: TX queue index
2236	* Description: according to which descriptor can be used (extend or basic)
2237	* this function allocates the resources for TX and RX paths. In case of
2238	* reception, for example, it pre-allocated the RX socket buffer in order to
2239	* allow zero-copy mechanism.
2240	*/
2241	static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv,
2242	struct stmmac_dma_conf *dma_conf,
2243	u32 queue)
2244	{
2245	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue];
2246	size_t size;
2247	void *addr;
2248
2249	tx_q->queue_index = queue;
2250	tx_q->priv_data = priv;
2251
2252	tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size,
2253	sizeof(*tx_q->tx_skbuff_dma),
2254	GFP_KERNEL);
2255	if (!tx_q->tx_skbuff_dma)
2256	return -ENOMEM;
2257
2258	tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size,
2259	sizeof(struct sk_buff *),
2260	GFP_KERNEL);
2261	if (!tx_q->tx_skbuff)
2262	return -ENOMEM;
2263
2264	if (priv->extend_desc)
2265	size = sizeof(struct dma_extended_desc);
2266	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2267	size = sizeof(struct dma_edesc);
2268	else
2269	size = sizeof(struct dma_desc);
2270
2271	size *= dma_conf->dma_tx_size;
2272
2273	addr = dma_alloc_coherent(dev: priv->device, size,
2274	dma_handle: &tx_q->dma_tx_phy, GFP_KERNEL);
2275	if (!addr)
2276	return -ENOMEM;
2277
2278	if (priv->extend_desc)
2279	tx_q->dma_etx = addr;
2280	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2281	tx_q->dma_entx = addr;
2282	else
2283	tx_q->dma_tx = addr;
2284
2285	return 0;
2286	}
2287
2288	static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv,
2289	struct stmmac_dma_conf *dma_conf)
2290	{
2291	u32 tx_count = priv->plat->tx_queues_to_use;
2292	u32 queue;
2293	int ret;
2294
2295	/* TX queues buffers and DMA */
2296	for (queue = 0; queue < tx_count; queue++) {
2297	ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue);
2298	if (ret)
2299	goto err_dma;
2300	}
2301
2302	return 0;
2303
2304	err_dma:
2305	free_dma_tx_desc_resources(priv, dma_conf);
2306	return ret;
2307	}
2308
2309	/**
2310	* alloc_dma_desc_resources - alloc TX/RX resources.
2311	* @priv: private structure
2312	* @dma_conf: structure to take the dma data
2313	* Description: according to which descriptor can be used (extend or basic)
2314	* this function allocates the resources for TX and RX paths. In case of
2315	* reception, for example, it pre-allocated the RX socket buffer in order to
2316	* allow zero-copy mechanism.
2317	*/
2318	static int alloc_dma_desc_resources(struct stmmac_priv *priv,
2319	struct stmmac_dma_conf *dma_conf)
2320	{
2321	/* RX Allocation */
2322	int ret = alloc_dma_rx_desc_resources(priv, dma_conf);
2323
2324	if (ret)
2325	return ret;
2326
2327	ret = alloc_dma_tx_desc_resources(priv, dma_conf);
2328
2329	return ret;
2330	}
2331
2332	/**
2333	* free_dma_desc_resources - free dma desc resources
2334	* @priv: private structure
2335	* @dma_conf: structure to take the dma data
2336	*/
2337	static void free_dma_desc_resources(struct stmmac_priv *priv,
2338	struct stmmac_dma_conf *dma_conf)
2339	{
2340	/* Release the DMA TX socket buffers */
2341	free_dma_tx_desc_resources(priv, dma_conf);
2342
2343	/* Release the DMA RX socket buffers later
2344	* to ensure all pending XDP_TX buffers are returned.
2345	*/
2346	free_dma_rx_desc_resources(priv, dma_conf);
2347	}
2348
2349	/**
2350	* stmmac_mac_enable_rx_queues - Enable MAC rx queues
2351	* @priv: driver private structure
2352	* Description: It is used for enabling the rx queues in the MAC
2353	*/
2354	static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
2355	{
2356	u32 rx_queues_count = priv->plat->rx_queues_to_use;
2357	int queue;
2358	u8 mode;
2359
2360	for (queue = 0; queue < rx_queues_count; queue++) {
2361	mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
2362	stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
2363	}
2364	}
2365
2366	/**
2367	* stmmac_start_rx_dma - start RX DMA channel
2368	* @priv: driver private structure
2369	* @chan: RX channel index
2370	* Description:
2371	* This starts a RX DMA channel
2372	*/
2373	static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
2374	{
2375	netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
2376	stmmac_start_rx(priv, priv->ioaddr, chan);
2377	}
2378
2379	/**
2380	* stmmac_start_tx_dma - start TX DMA channel
2381	* @priv: driver private structure
2382	* @chan: TX channel index
2383	* Description:
2384	* This starts a TX DMA channel
2385	*/
2386	static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
2387	{
2388	netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
2389	stmmac_start_tx(priv, priv->ioaddr, chan);
2390	}
2391
2392	/**
2393	* stmmac_stop_rx_dma - stop RX DMA channel
2394	* @priv: driver private structure
2395	* @chan: RX channel index
2396	* Description:
2397	* This stops a RX DMA channel
2398	*/
2399	static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
2400	{
2401	netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
2402	stmmac_stop_rx(priv, priv->ioaddr, chan);
2403	}
2404
2405	/**
2406	* stmmac_stop_tx_dma - stop TX DMA channel
2407	* @priv: driver private structure
2408	* @chan: TX channel index
2409	* Description:
2410	* This stops a TX DMA channel
2411	*/
2412	static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
2413	{
2414	netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
2415	stmmac_stop_tx(priv, priv->ioaddr, chan);
2416	}
2417
2418	static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv)
2419	{
2420	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2421	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2422	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
2423	u32 chan;
2424
2425	for (chan = 0; chan < dma_csr_ch; chan++) {
2426	struct stmmac_channel *ch = &priv->channel[chan];
2427	unsigned long flags;
2428
2429	spin_lock_irqsave(&ch->lock, flags);
2430	stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
2431	spin_unlock_irqrestore(lock: &ch->lock, flags);
2432	}
2433	}
2434
2435	/**
2436	* stmmac_start_all_dma - start all RX and TX DMA channels
2437	* @priv: driver private structure
2438	* Description:
2439	* This starts all the RX and TX DMA channels
2440	*/
2441	static void stmmac_start_all_dma(struct stmmac_priv *priv)
2442	{
2443	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2444	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2445	u32 chan = 0;
2446
2447	for (chan = 0; chan < rx_channels_count; chan++)
2448	stmmac_start_rx_dma(priv, chan);
2449
2450	for (chan = 0; chan < tx_channels_count; chan++)
2451	stmmac_start_tx_dma(priv, chan);
2452	}
2453
2454	/**
2455	* stmmac_stop_all_dma - stop all RX and TX DMA channels
2456	* @priv: driver private structure
2457	* Description:
2458	* This stops the RX and TX DMA channels
2459	*/
2460	static void stmmac_stop_all_dma(struct stmmac_priv *priv)
2461	{
2462	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2463	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2464	u32 chan = 0;
2465
2466	for (chan = 0; chan < rx_channels_count; chan++)
2467	stmmac_stop_rx_dma(priv, chan);
2468
2469	for (chan = 0; chan < tx_channels_count; chan++)
2470	stmmac_stop_tx_dma(priv, chan);
2471	}
2472
2473	/**
2474	* stmmac_dma_operation_mode - HW DMA operation mode
2475	* @priv: driver private structure
2476	* Description: it is used for configuring the DMA operation mode register in
2477	* order to program the tx/rx DMA thresholds or Store-And-Forward mode.
2478	*/
2479	static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
2480	{
2481	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2482	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2483	int rxfifosz = priv->plat->rx_fifo_size;
2484	int txfifosz = priv->plat->tx_fifo_size;
2485	u32 txmode = 0;
2486	u32 rxmode = 0;
2487	u32 chan = 0;
2488	u8 qmode = 0;
2489
2490	if (rxfifosz == 0)
2491	rxfifosz = priv->dma_cap.rx_fifo_size;
2492	if (txfifosz == 0)
2493	txfifosz = priv->dma_cap.tx_fifo_size;
2494
2495	/* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */
2496	if (dwmac_is_xmac(core_type: priv->plat->core_type)) {
2497	rxfifosz /= rx_channels_count;
2498	txfifosz /= tx_channels_count;
2499	}
2500
2501	if (priv->plat->force_thresh_dma_mode) {
2502	txmode = tc;
2503	rxmode = tc;
2504	} else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
2505	/*
2506	* In case of GMAC, SF mode can be enabled
2507	* to perform the TX COE in HW. This depends on:
2508	* 1) TX COE if actually supported
2509	* 2) There is no bugged Jumbo frame support
2510	* that needs to not insert csum in the TDES.
2511	*/
2512	txmode = SF_DMA_MODE;
2513	rxmode = SF_DMA_MODE;
2514	priv->xstats.threshold = SF_DMA_MODE;
2515	} else {
2516	txmode = tc;
2517	rxmode = SF_DMA_MODE;
2518	}
2519
2520	/* configure all channels */
2521	for (chan = 0; chan < rx_channels_count; chan++) {
2522	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan];
2523	u32 buf_size;
2524
2525	qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2526
2527	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
2528	rxfifosz, qmode);
2529
2530	if (rx_q->xsk_pool) {
2531	buf_size = xsk_pool_get_rx_frame_size(pool: rx_q->xsk_pool);
2532	stmmac_set_dma_bfsize(priv, priv->ioaddr,
2533	buf_size,
2534	chan);
2535	} else {
2536	stmmac_set_dma_bfsize(priv, priv->ioaddr,
2537	priv->dma_conf.dma_buf_sz,
2538	chan);
2539	}
2540	}
2541
2542	for (chan = 0; chan < tx_channels_count; chan++) {
2543	qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2544
2545	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
2546	txfifosz, qmode);
2547	}
2548	}
2549
2550	static void stmmac_xsk_request_timestamp(void *_priv)
2551	{
2552	struct stmmac_metadata_request *meta_req = _priv;
2553
2554	stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc);
2555	*meta_req->set_ic = true;
2556	}
2557
2558	static u64 stmmac_xsk_fill_timestamp(void *_priv)
2559	{
2560	struct stmmac_xsk_tx_complete *tx_compl = _priv;
2561	struct stmmac_priv *priv = tx_compl->priv;
2562	struct dma_desc *desc = tx_compl->desc;
2563	bool found = false;
2564	u64 ns = 0;
2565
2566	if (!priv->hwts_tx_en)
2567	return 0;
2568
2569	/* check tx tstamp status */
2570	if (stmmac_get_tx_timestamp_status(priv, desc)) {
2571	stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
2572	found = true;
2573	} else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
2574	found = true;
2575	}
2576
2577	if (found) {
2578	ns -= priv->plat->cdc_error_adj;
2579	return ns_to_ktime(ns);
2580	}
2581
2582	return 0;
2583	}
2584
2585	static void stmmac_xsk_request_launch_time(u64 launch_time, void *_priv)
2586	{
2587	struct timespec64 ts = ns_to_timespec64(nsec: launch_time);
2588	struct stmmac_metadata_request *meta_req = _priv;
2589
2590	if (meta_req->tbs & STMMAC_TBS_EN)
2591	stmmac_set_desc_tbs(meta_req->priv, meta_req->edesc, ts.tv_sec,
2592	ts.tv_nsec);
2593	}
2594
2595	static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = {
2596	.tmo_request_timestamp = stmmac_xsk_request_timestamp,
2597	.tmo_fill_timestamp = stmmac_xsk_fill_timestamp,
2598	.tmo_request_launch_time = stmmac_xsk_request_launch_time,
2599	};
2600
2601	static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
2602	{
2603	struct netdev_queue *nq = netdev_get_tx_queue(dev: priv->dev, index: queue);
2604	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
2605	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];
2606	bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported;
2607	struct xsk_buff_pool *pool = tx_q->xsk_pool;
2608	unsigned int entry = tx_q->cur_tx;
2609	struct dma_desc *tx_desc = NULL;
2610	struct xdp_desc xdp_desc;
2611	bool work_done = true;
2612	u32 tx_set_ic_bit = 0;
2613
2614	/* Avoids TX time-out as we are sharing with slow path */
2615	txq_trans_cond_update(txq: nq);
2616
2617	budget = min(budget, stmmac_tx_avail(priv, queue));
2618
2619	for (; budget > 0; budget--) {
2620	struct stmmac_metadata_request meta_req;
2621	struct xsk_tx_metadata *meta = NULL;
2622	dma_addr_t dma_addr;
2623	bool set_ic;
2624
2625	/* We are sharing with slow path and stop XSK TX desc submission when
2626	* available TX ring is less than threshold.
2627	*/
2628	if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) ||
2629	!netif_carrier_ok(dev: priv->dev)) {
2630	work_done = false;
2631	break;
2632	}
2633
2634	if (!xsk_tx_peek_desc(pool, desc: &xdp_desc))
2635	break;
2636
2637	if (priv->est && priv->est->enable &&
2638	priv->est->max_sdu[queue] &&
2639	xdp_desc.len > priv->est->max_sdu[queue]) {
2640	priv->xstats.max_sdu_txq_drop[queue]++;
2641	continue;
2642	}
2643
2644	if (likely(priv->extend_desc))
2645	tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
2646	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2647	tx_desc = &tx_q->dma_entx[entry].basic;
2648	else
2649	tx_desc = tx_q->dma_tx + entry;
2650
2651	dma_addr = xsk_buff_raw_get_dma(pool, addr: xdp_desc.addr);
2652	meta = xsk_buff_get_metadata(pool, addr: xdp_desc.addr);
2653	xsk_buff_raw_dma_sync_for_device(pool, dma: dma_addr, size: xdp_desc.len);
2654
2655	tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX;
2656
2657	/* To return XDP buffer to XSK pool, we simple call
2658	* xsk_tx_completed(), so we don't need to fill up
2659	* 'buf' and 'xdpf'.
2660	*/
2661	tx_q->tx_skbuff_dma[entry].buf = 0;
2662	tx_q->xdpf[entry] = NULL;
2663
2664	tx_q->tx_skbuff_dma[entry].map_as_page = false;
2665	tx_q->tx_skbuff_dma[entry].len = xdp_desc.len;
2666	tx_q->tx_skbuff_dma[entry].last_segment = true;
2667	tx_q->tx_skbuff_dma[entry].is_jumbo = false;
2668
2669	stmmac_set_desc_addr(priv, tx_desc, dma_addr);
2670
2671	tx_q->tx_count_frames++;
2672
2673	if (!priv->tx_coal_frames[queue])
2674	set_ic = false;
2675	else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)
2676	set_ic = true;
2677	else
2678	set_ic = false;
2679
2680	meta_req.priv = priv;
2681	meta_req.tx_desc = tx_desc;
2682	meta_req.set_ic = &set_ic;
2683	meta_req.tbs = tx_q->tbs;
2684	meta_req.edesc = &tx_q->dma_entx[entry];
2685	xsk_tx_metadata_request(meta, ops: &stmmac_xsk_tx_metadata_ops,
2686	priv: &meta_req);
2687	if (set_ic) {
2688	tx_q->tx_count_frames = 0;
2689	stmmac_set_tx_ic(priv, tx_desc);
2690	tx_set_ic_bit++;
2691	}
2692
2693	stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len,
2694	csum, priv->mode, true, true,
2695	xdp_desc.len);
2696
2697	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);
2698
2699	xsk_tx_metadata_to_compl(meta,
2700	compl: &tx_q->tx_skbuff_dma[entry].xsk_meta);
2701
2702	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
2703	entry = tx_q->cur_tx;
2704	}
2705	u64_stats_update_begin(syncp: &txq_stats->napi_syncp);
2706	u64_stats_add(p: &txq_stats->napi.tx_set_ic_bit, val: tx_set_ic_bit);
2707	u64_stats_update_end(syncp: &txq_stats->napi_syncp);
2708
2709	if (tx_desc) {
2710	stmmac_flush_tx_descriptors(priv, queue);
2711	xsk_tx_release(pool);
2712	}
2713
2714	/* Return true if all of the 3 conditions are met
2715	* a) TX Budget is still available
2716	* b) work_done = true when XSK TX desc peek is empty (no more
2717	* pending XSK TX for transmission)
2718	*/
2719	return !!budget && work_done;
2720	}
2721
2722	static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan)
2723	{
2724	if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) {
2725	tc += 64;
2726
2727	if (priv->plat->force_thresh_dma_mode)
2728	stmmac_set_dma_operation_mode(priv, txmode: tc, rxmode: tc, chan);
2729	else
2730	stmmac_set_dma_operation_mode(priv, txmode: tc, SF_DMA_MODE,
2731	chan);
2732
2733	priv->xstats.threshold = tc;
2734	}
2735	}
2736
2737	/**
2738	* stmmac_tx_clean - to manage the transmission completion
2739	* @priv: driver private structure
2740	* @budget: napi budget limiting this functions packet handling
2741	* @queue: TX queue index
2742	* @pending_packets: signal to arm the TX coal timer
2743	* Description: it reclaims the transmit resources after transmission completes.
2744	* If some packets still needs to be handled, due to TX coalesce, set
2745	* pending_packets to true to make NAPI arm the TX coal timer.
2746	*/
2747	static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
2748	bool *pending_packets)
2749	{
2750	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
2751	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];
2752	unsigned int bytes_compl = 0, pkts_compl = 0;
2753	unsigned int entry, xmits = 0, count = 0;
2754	u32 tx_packets = 0, tx_errors = 0;
2755
2756	__netif_tx_lock_bh(txq: netdev_get_tx_queue(dev: priv->dev, index: queue));
2757
2758	tx_q->xsk_frames_done = 0;
2759
2760	entry = tx_q->dirty_tx;
2761
2762	/* Try to clean all TX complete frame in 1 shot */
2763	while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) {
2764	struct xdp_frame *xdpf;
2765	struct sk_buff *skb;
2766	struct dma_desc *p;
2767	int status;
2768
2769	if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX ||
2770	tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) {
2771	xdpf = tx_q->xdpf[entry];
2772	skb = NULL;
2773	} else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
2774	xdpf = NULL;
2775	skb = tx_q->tx_skbuff[entry];
2776	} else {
2777	xdpf = NULL;
2778	skb = NULL;
2779	}
2780
2781	if (priv->extend_desc)
2782	p = (struct dma_desc *)(tx_q->dma_etx + entry);
2783	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2784	p = &tx_q->dma_entx[entry].basic;
2785	else
2786	p = tx_q->dma_tx + entry;
2787
2788	status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr);
2789	/* Check if the descriptor is owned by the DMA */
2790	if (unlikely(status & tx_dma_own))
2791	break;
2792
2793	count++;
2794
2795	/* Make sure descriptor fields are read after reading
2796	* the own bit.
2797	*/
2798	dma_rmb();
2799
2800	/* Just consider the last segment and ...*/
2801	if (likely(!(status & tx_not_ls))) {
2802	/* ... verify the status error condition */
2803	if (unlikely(status & tx_err)) {
2804	tx_errors++;
2805	if (unlikely(status & tx_err_bump_tc))
2806	stmmac_bump_dma_threshold(priv, chan: queue);
2807	} else {
2808	tx_packets++;
2809	}
2810	if (skb) {
2811	stmmac_get_tx_hwtstamp(priv, p, skb);
2812	} else if (tx_q->xsk_pool &&
2813	xp_tx_metadata_enabled(pool: tx_q->xsk_pool)) {
2814	struct stmmac_xsk_tx_complete tx_compl = {
2815	.priv = priv,
2816	.desc = p,
2817	};
2818
2819	xsk_tx_metadata_complete(compl: &tx_q->tx_skbuff_dma[entry].xsk_meta,
2820	ops: &stmmac_xsk_tx_metadata_ops,
2821	priv: &tx_compl);
2822	}
2823	}
2824
2825	if (likely(tx_q->tx_skbuff_dma[entry].buf &&
2826	tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) {
2827	if (tx_q->tx_skbuff_dma[entry].map_as_page)
2828	dma_unmap_page(priv->device,
2829	tx_q->tx_skbuff_dma[entry].buf,
2830	tx_q->tx_skbuff_dma[entry].len,
2831	DMA_TO_DEVICE);
2832	else
2833	dma_unmap_single(priv->device,
2834	tx_q->tx_skbuff_dma[entry].buf,
2835	tx_q->tx_skbuff_dma[entry].len,
2836	DMA_TO_DEVICE);
2837	tx_q->tx_skbuff_dma[entry].buf = 0;
2838	tx_q->tx_skbuff_dma[entry].len = 0;
2839	tx_q->tx_skbuff_dma[entry].map_as_page = false;
2840	}
2841
2842	stmmac_clean_desc3(priv, tx_q, p);
2843
2844	tx_q->tx_skbuff_dma[entry].last_segment = false;
2845	tx_q->tx_skbuff_dma[entry].is_jumbo = false;
2846
2847	if (xdpf &&
2848	tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) {
2849	xdp_return_frame_rx_napi(xdpf);
2850	tx_q->xdpf[entry] = NULL;
2851	}
2852
2853	if (xdpf &&
2854	tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) {
2855	xdp_return_frame(xdpf);
2856	tx_q->xdpf[entry] = NULL;
2857	}
2858
2859	if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX)
2860	tx_q->xsk_frames_done++;
2861
2862	if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
2863	if (likely(skb)) {
2864	pkts_compl++;
2865	bytes_compl += skb->len;
2866	dev_consume_skb_any(skb);
2867	tx_q->tx_skbuff[entry] = NULL;
2868	}
2869	}
2870
2871	stmmac_release_tx_desc(priv, p, priv->mode);
2872
2873	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
2874	}
2875	tx_q->dirty_tx = entry;
2876
2877	netdev_tx_completed_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: queue),
2878	pkts: pkts_compl, bytes: bytes_compl);
2879
2880	if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
2881	queue))) &&
2882	stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) {
2883
2884	netif_dbg(priv, tx_done, priv->dev,
2885	"%s: restart transmit\n", __func__);
2886	netif_tx_wake_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: queue));
2887	}
2888
2889	if (tx_q->xsk_pool) {
2890	bool work_done;
2891
2892	if (tx_q->xsk_frames_done)
2893	xsk_tx_completed(pool: tx_q->xsk_pool, nb_entries: tx_q->xsk_frames_done);
2894
2895	if (xsk_uses_need_wakeup(pool: tx_q->xsk_pool))
2896	xsk_set_tx_need_wakeup(pool: tx_q->xsk_pool);
2897
2898	/* For XSK TX, we try to send as many as possible.
2899	* If XSK work done (XSK TX desc empty and budget still
2900	* available), return "budget - 1" to reenable TX IRQ.
2901	* Else, return "budget" to make NAPI continue polling.
2902	*/
2903	work_done = stmmac_xdp_xmit_zc(priv, queue,
2904	STMMAC_XSK_TX_BUDGET_MAX);
2905	if (work_done)
2906	xmits = budget - 1;
2907	else
2908	xmits = budget;
2909	}
2910
2911	if (priv->eee_sw_timer_en && !priv->tx_path_in_lpi_mode)
2912	stmmac_restart_sw_lpi_timer(priv);
2913
2914	/* We still have pending packets, let's call for a new scheduling */
2915	if (tx_q->dirty_tx != tx_q->cur_tx)
2916	*pending_packets = true;
2917
2918	u64_stats_update_begin(syncp: &txq_stats->napi_syncp);
2919	u64_stats_add(p: &txq_stats->napi.tx_packets, val: tx_packets);
2920	u64_stats_add(p: &txq_stats->napi.tx_pkt_n, val: tx_packets);
2921	u64_stats_inc(p: &txq_stats->napi.tx_clean);
2922	u64_stats_update_end(syncp: &txq_stats->napi_syncp);
2923
2924	priv->xstats.tx_errors += tx_errors;
2925
2926	__netif_tx_unlock_bh(txq: netdev_get_tx_queue(dev: priv->dev, index: queue));
2927
2928	/* Combine decisions from TX clean and XSK TX */
2929	return max(count, xmits);
2930	}
2931
2932	/**
2933	* stmmac_tx_err - to manage the tx error
2934	* @priv: driver private structure
2935	* @chan: channel index
2936	* Description: it cleans the descriptors and restarts the transmission
2937	* in case of transmission errors.
2938	*/
2939	static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
2940	{
2941	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
2942
2943	netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: chan));
2944
2945	stmmac_stop_tx_dma(priv, chan);
2946	dma_free_tx_skbufs(priv, dma_conf: &priv->dma_conf, queue: chan);
2947	stmmac_clear_tx_descriptors(priv, dma_conf: &priv->dma_conf, queue: chan);
2948	stmmac_reset_tx_queue(priv, queue: chan);
2949	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2950	tx_q->dma_tx_phy, chan);
2951	stmmac_start_tx_dma(priv, chan);
2952
2953	priv->xstats.tx_errors++;
2954	netif_tx_wake_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: chan));
2955	}
2956
2957	/**
2958	* stmmac_set_dma_operation_mode - Set DMA operation mode by channel
2959	* @priv: driver private structure
2960	* @txmode: TX operating mode
2961	* @rxmode: RX operating mode
2962	* @chan: channel index
2963	* Description: it is used for configuring of the DMA operation mode in
2964	* runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
2965	* mode.
2966	*/
2967	static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
2968	u32 rxmode, u32 chan)
2969	{
2970	u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2971	u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2972	u32 rx_channels_count = priv->plat->rx_queues_to_use;
2973	u32 tx_channels_count = priv->plat->tx_queues_to_use;
2974	int rxfifosz = priv->plat->rx_fifo_size;
2975	int txfifosz = priv->plat->tx_fifo_size;
2976
2977	if (rxfifosz == 0)
2978	rxfifosz = priv->dma_cap.rx_fifo_size;
2979	if (txfifosz == 0)
2980	txfifosz = priv->dma_cap.tx_fifo_size;
2981
2982	/* Adjust for real per queue fifo size */
2983	rxfifosz /= rx_channels_count;
2984	txfifosz /= tx_channels_count;
2985
2986	stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
2987	stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
2988	}
2989
2990	static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
2991	{
2992	int ret;
2993
2994	ret = stmmac_safety_feat_irq_status(priv, priv->dev,
2995	priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
2996	if (ret && (ret != -EINVAL)) {
2997	stmmac_global_err(priv);
2998	return true;
2999	}
3000
3001	return false;
3002	}
3003
3004	static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir)
3005	{
3006	int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
3007	&priv->xstats, chan, dir);
3008	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan];
3009	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
3010	struct stmmac_channel *ch = &priv->channel[chan];
3011	struct napi_struct *rx_napi;
3012	struct napi_struct *tx_napi;
3013	unsigned long flags;
3014
3015	rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi;
3016	tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
3017
3018	if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
3019	if (napi_schedule_prep(n: rx_napi)) {
3020	spin_lock_irqsave(&ch->lock, flags);
3021	stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
3022	spin_unlock_irqrestore(lock: &ch->lock, flags);
3023	__napi_schedule(n: rx_napi);
3024	}
3025	}
3026
3027	if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {
3028	if (napi_schedule_prep(n: tx_napi)) {
3029	spin_lock_irqsave(&ch->lock, flags);
3030	stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
3031	spin_unlock_irqrestore(lock: &ch->lock, flags);
3032	__napi_schedule(n: tx_napi);
3033	}
3034	}
3035
3036	return status;
3037	}
3038
3039	/**
3040	* stmmac_dma_interrupt - DMA ISR
3041	* @priv: driver private structure
3042	* Description: this is the DMA ISR. It is called by the main ISR.
3043	* It calls the dwmac dma routine and schedule poll method in case of some
3044	* work can be done.
3045	*/
3046	static void stmmac_dma_interrupt(struct stmmac_priv *priv)
3047	{
3048	u32 tx_channel_count = priv->plat->tx_queues_to_use;
3049	u32 rx_channel_count = priv->plat->rx_queues_to_use;
3050	u32 channels_to_check = tx_channel_count > rx_channel_count ?
3051	tx_channel_count : rx_channel_count;
3052	u32 chan;
3053	int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
3054
3055	/* Make sure we never check beyond our status buffer. */
3056	if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
3057	channels_to_check = ARRAY_SIZE(status);
3058
3059	for (chan = 0; chan < channels_to_check; chan++)
3060	status[chan] = stmmac_napi_check(priv, chan,
3061	dir: DMA_DIR_RXTX);
3062
3063	for (chan = 0; chan < tx_channel_count; chan++) {
3064	if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
3065	/* Try to bump up the dma threshold on this failure */
3066	stmmac_bump_dma_threshold(priv, chan);
3067	} else if (unlikely(status[chan] == tx_hard_error)) {
3068	stmmac_tx_err(priv, chan);
3069	}
3070	}
3071	}
3072
3073	/**
3074	* stmmac_mmc_setup: setup the Mac Management Counters (MMC)
3075	* @priv: driver private structure
3076	* Description: this masks the MMC irq, in fact, the counters are managed in SW.
3077	*/
3078	static void stmmac_mmc_setup(struct stmmac_priv *priv)
3079	{
3080	unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
3081	MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
3082
3083	stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
3084
3085	if (priv->dma_cap.rmon) {
3086	stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
3087	memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
3088	} else
3089	netdev_info(dev: priv->dev, format: "No MAC Management Counters available\n");
3090	}
3091
3092	/**
3093	* stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
3094	* @priv: driver private structure
3095	* Description:
3096	* new GMAC chip generations have a new register to indicate the
3097	* presence of the optional feature/functions.
3098	* This can be also used to override the value passed through the
3099	* platform and necessary for old MAC10/100 and GMAC chips.
3100	*/
3101	static int stmmac_get_hw_features(struct stmmac_priv *priv)
3102	{
3103	return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
3104	}
3105
3106	/**
3107	* stmmac_check_ether_addr - check if the MAC addr is valid
3108	* @priv: driver private structure
3109	* Description:
3110	* it is to verify if the MAC address is valid, in case of failures it
3111	* generates a random MAC address
3112	*/
3113	static void stmmac_check_ether_addr(struct stmmac_priv *priv)
3114	{
3115	u8 addr[ETH_ALEN];
3116
3117	if (!is_valid_ether_addr(addr: priv->dev->dev_addr)) {
3118	stmmac_get_umac_addr(priv, priv->hw, addr, 0);
3119	if (is_valid_ether_addr(addr))
3120	eth_hw_addr_set(dev: priv->dev, addr);
3121	else
3122	eth_hw_addr_random(dev: priv->dev);
3123	dev_info(priv->device, "device MAC address %pM\n",
3124	priv->dev->dev_addr);
3125	}
3126	}
3127
3128	int stmmac_get_phy_intf_sel(phy_interface_t interface)
3129	{
3130	int phy_intf_sel = -EINVAL;
3131
3132	if (interface == PHY_INTERFACE_MODE_MII ||
3133	interface == PHY_INTERFACE_MODE_GMII)
3134	phy_intf_sel = PHY_INTF_SEL_GMII_MII;
3135	else if (phy_interface_mode_is_rgmii(mode: interface))
3136	phy_intf_sel = PHY_INTF_SEL_RGMII;
3137	else if (interface == PHY_INTERFACE_MODE_SGMII)
3138	phy_intf_sel = PHY_INTF_SEL_SGMII;
3139	else if (interface == PHY_INTERFACE_MODE_RMII)
3140	phy_intf_sel = PHY_INTF_SEL_RMII;
3141	else if (interface == PHY_INTERFACE_MODE_REVMII)
3142	phy_intf_sel = PHY_INTF_SEL_REVMII;
3143
3144	return phy_intf_sel;
3145	}
3146	EXPORT_SYMBOL_GPL(stmmac_get_phy_intf_sel);
3147
3148	static int stmmac_prereset_configure(struct stmmac_priv *priv)
3149	{
3150	struct plat_stmmacenet_data *plat_dat = priv->plat;
3151	phy_interface_t interface;
3152	int phy_intf_sel, ret;
3153
3154	if (!plat_dat->set_phy_intf_sel)
3155	return 0;
3156
3157	interface = plat_dat->phy_interface;
3158	phy_intf_sel = stmmac_get_phy_intf_sel(interface);
3159	if (phy_intf_sel < 0) {
3160	netdev_err(dev: priv->dev,
3161	format: "failed to get phy_intf_sel for %s: %pe\n",
3162	phy_modes(interface), ERR_PTR(error: phy_intf_sel));
3163	return phy_intf_sel;
3164	}
3165
3166	ret = plat_dat->set_phy_intf_sel(plat_dat->bsp_priv, phy_intf_sel);
3167	if (ret == -EINVAL)
3168	netdev_err(dev: priv->dev, format: "platform does not support %s\n",
3169	phy_modes(interface));
3170	else if (ret < 0)
3171	netdev_err(dev: priv->dev,
3172	format: "platform failed to set interface %s: %pe\n",
3173	phy_modes(interface), ERR_PTR(error: ret));
3174
3175	return ret;
3176	}
3177
3178	/**
3179	* stmmac_init_dma_engine - DMA init.
3180	* @priv: driver private structure
3181	* Description:
3182	* It inits the DMA invoking the specific MAC/GMAC callback.
3183	* Some DMA parameters can be passed from the platform;
3184	* in case of these are not passed a default is kept for the MAC or GMAC.
3185	*/
3186	static int stmmac_init_dma_engine(struct stmmac_priv *priv)
3187	{
3188	u32 rx_channels_count = priv->plat->rx_queues_to_use;
3189	u32 tx_channels_count = priv->plat->tx_queues_to_use;
3190	u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
3191	struct stmmac_rx_queue *rx_q;
3192	struct stmmac_tx_queue *tx_q;
3193	u32 chan = 0;
3194	int ret = 0;
3195
3196	if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
3197	netdev_err(dev: priv->dev, format: "Invalid DMA configuration\n");
3198	return -EINVAL;
3199	}
3200
3201	if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
3202	priv->plat->dma_cfg->atds = 1;
3203
3204	ret = stmmac_prereset_configure(priv);
3205	if (ret)
3206	return ret;
3207
3208	ret = stmmac_reset(priv);
3209	if (ret) {
3210	netdev_err(dev: priv->dev, format: "Failed to reset the dma\n");
3211	return ret;
3212	}
3213
3214	/* DMA Configuration */
3215	stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg);
3216
3217	if (priv->plat->axi)
3218	stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
3219
3220	/* DMA CSR Channel configuration */
3221	for (chan = 0; chan < dma_csr_ch; chan++) {
3222	stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
3223	stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
3224	}
3225
3226	/* DMA RX Channel Configuration */
3227	for (chan = 0; chan < rx_channels_count; chan++) {
3228	rx_q = &priv->dma_conf.rx_queue[chan];
3229
3230	stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
3231	rx_q->dma_rx_phy, chan);
3232
3233	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
3234	(rx_q->buf_alloc_num *
3235	sizeof(struct dma_desc));
3236	stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
3237	rx_q->rx_tail_addr, chan);
3238	}
3239
3240	/* DMA TX Channel Configuration */
3241	for (chan = 0; chan < tx_channels_count; chan++) {
3242	tx_q = &priv->dma_conf.tx_queue[chan];
3243
3244	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
3245	tx_q->dma_tx_phy, chan);
3246
3247	tx_q->tx_tail_addr = tx_q->dma_tx_phy;
3248	stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
3249	tx_q->tx_tail_addr, chan);
3250	}
3251
3252	return ret;
3253	}
3254
3255	static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
3256	{
3257	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
3258	u32 tx_coal_timer = priv->tx_coal_timer[queue];
3259	struct stmmac_channel *ch;
3260	struct napi_struct *napi;
3261
3262	if (!tx_coal_timer)
3263	return;
3264
3265	ch = &priv->channel[tx_q->queue_index];
3266	napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
3267
3268	/* Arm timer only if napi is not already scheduled.
3269	* Try to cancel any timer if napi is scheduled, timer will be armed
3270	* again in the next scheduled napi.
3271	*/
3272	if (unlikely(!napi_is_scheduled(napi)))
3273	hrtimer_start(timer: &tx_q->txtimer,
3274	STMMAC_COAL_TIMER(tx_coal_timer),
3275	mode: HRTIMER_MODE_REL);
3276	else
3277	hrtimer_try_to_cancel(timer: &tx_q->txtimer);
3278	}
3279
3280	/**
3281	* stmmac_tx_timer - mitigation sw timer for tx.
3282	* @t: data pointer
3283	* Description:
3284	* This is the timer handler to directly invoke the stmmac_tx_clean.
3285	*/
3286	static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t)
3287	{
3288	struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer);
3289	struct stmmac_priv *priv = tx_q->priv_data;
3290	struct stmmac_channel *ch;
3291	struct napi_struct *napi;
3292
3293	ch = &priv->channel[tx_q->queue_index];
3294	napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
3295
3296	if (likely(napi_schedule_prep(napi))) {
3297	unsigned long flags;
3298
3299	spin_lock_irqsave(&ch->lock, flags);
3300	stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);
3301	spin_unlock_irqrestore(lock: &ch->lock, flags);
3302	__napi_schedule(n: napi);
3303	}
3304
3305	return HRTIMER_NORESTART;
3306	}
3307
3308	/**
3309	* stmmac_init_coalesce - init mitigation options.
3310	* @priv: driver private structure
3311	* Description:
3312	* This inits the coalesce parameters: i.e. timer rate,
3313	* timer handler and default threshold used for enabling the
3314	* interrupt on completion bit.
3315	*/
3316	static void stmmac_init_coalesce(struct stmmac_priv *priv)
3317	{
3318	u32 tx_channel_count = priv->plat->tx_queues_to_use;
3319	u32 rx_channel_count = priv->plat->rx_queues_to_use;
3320	u32 chan;
3321
3322	for (chan = 0; chan < tx_channel_count; chan++) {
3323	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
3324
3325	priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES;
3326	priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER;
3327
3328	hrtimer_setup(timer: &tx_q->txtimer, function: stmmac_tx_timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
3329	}
3330
3331	for (chan = 0; chan < rx_channel_count; chan++)
3332	priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES;
3333	}
3334
3335	static void stmmac_set_rings_length(struct stmmac_priv *priv)
3336	{
3337	u32 rx_channels_count = priv->plat->rx_queues_to_use;
3338	u32 tx_channels_count = priv->plat->tx_queues_to_use;
3339	u32 chan;
3340
3341	/* set TX ring length */
3342	for (chan = 0; chan < tx_channels_count; chan++)
3343	stmmac_set_tx_ring_len(priv, priv->ioaddr,
3344	(priv->dma_conf.dma_tx_size - 1), chan);
3345
3346	/* set RX ring length */
3347	for (chan = 0; chan < rx_channels_count; chan++)
3348	stmmac_set_rx_ring_len(priv, priv->ioaddr,
3349	(priv->dma_conf.dma_rx_size - 1), chan);
3350	}
3351
3352	/**
3353	* stmmac_set_tx_queue_weight - Set TX queue weight
3354	* @priv: driver private structure
3355	* Description: It is used for setting TX queues weight
3356	*/
3357	static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
3358	{
3359	u32 tx_queues_count = priv->plat->tx_queues_to_use;
3360	u32 weight;
3361	u32 queue;
3362
3363	for (queue = 0; queue < tx_queues_count; queue++) {
3364	weight = priv->plat->tx_queues_cfg[queue].weight;
3365	stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
3366	}
3367	}
3368
3369	/**
3370	* stmmac_configure_cbs - Configure CBS in TX queue
3371	* @priv: driver private structure
3372	* Description: It is used for configuring CBS in AVB TX queues
3373	*/
3374	static void stmmac_configure_cbs(struct stmmac_priv *priv)
3375	{
3376	u32 tx_queues_count = priv->plat->tx_queues_to_use;
3377	u32 mode_to_use;
3378	u32 queue;
3379
3380	/* queue 0 is reserved for legacy traffic */
3381	for (queue = 1; queue < tx_queues_count; queue++) {
3382	mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
3383	if (mode_to_use == MTL_QUEUE_DCB)
3384	continue;
3385
3386	stmmac_config_cbs(priv, priv->hw,
3387	priv->plat->tx_queues_cfg[queue].send_slope,
3388	priv->plat->tx_queues_cfg[queue].idle_slope,
3389	priv->plat->tx_queues_cfg[queue].high_credit,
3390	priv->plat->tx_queues_cfg[queue].low_credit,
3391	queue);
3392	}
3393	}
3394
3395	/**
3396	* stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
3397	* @priv: driver private structure
3398	* Description: It is used for mapping RX queues to RX dma channels
3399	*/
3400	static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
3401	{
3402	u32 rx_queues_count = priv->plat->rx_queues_to_use;
3403	u32 queue;
3404	u32 chan;
3405
3406	for (queue = 0; queue < rx_queues_count; queue++) {
3407	chan = priv->plat->rx_queues_cfg[queue].chan;
3408	stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
3409	}
3410	}
3411
3412	/**
3413	* stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
3414	* @priv: driver private structure
3415	* Description: It is used for configuring the RX Queue Priority
3416	*/
3417	static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
3418	{
3419	u32 rx_queues_count = priv->plat->rx_queues_to_use;
3420	u32 queue;
3421	u32 prio;
3422
3423	for (queue = 0; queue < rx_queues_count; queue++) {
3424	if (!priv->plat->rx_queues_cfg[queue].use_prio)
3425	continue;
3426
3427	prio = priv->plat->rx_queues_cfg[queue].prio;
3428	stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
3429	}
3430	}
3431
3432	/**
3433	* stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
3434	* @priv: driver private structure
3435	* Description: It is used for configuring the TX Queue Priority
3436	*/
3437	static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
3438	{
3439	u32 tx_queues_count = priv->plat->tx_queues_to_use;
3440	u32 queue;
3441	u32 prio;
3442
3443	for (queue = 0; queue < tx_queues_count; queue++) {
3444	if (!priv->plat->tx_queues_cfg[queue].use_prio)
3445	continue;
3446
3447	prio = priv->plat->tx_queues_cfg[queue].prio;
3448	stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
3449	}
3450	}
3451
3452	/**
3453	* stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
3454	* @priv: driver private structure
3455	* Description: It is used for configuring the RX queue routing
3456	*/
3457	static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
3458	{
3459	u32 rx_queues_count = priv->plat->rx_queues_to_use;
3460	u32 queue;
3461	u8 packet;
3462
3463	for (queue = 0; queue < rx_queues_count; queue++) {
3464	/* no specific packet type routing specified for the queue */
3465	if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
3466	continue;
3467
3468	packet = priv->plat->rx_queues_cfg[queue].pkt_route;
3469	stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
3470	}
3471	}
3472
3473	static void stmmac_mac_config_rss(struct stmmac_priv *priv)
3474	{
3475	if (!priv->dma_cap.rssen || !priv->plat->rss_en) {
3476	priv->rss.enable = false;
3477	return;
3478	}
3479
3480	if (priv->dev->features & NETIF_F_RXHASH)
3481	priv->rss.enable = true;
3482	else
3483	priv->rss.enable = false;
3484
3485	stmmac_rss_configure(priv, priv->hw, &priv->rss,
3486	priv->plat->rx_queues_to_use);
3487	}
3488
3489	/**
3490	* stmmac_mtl_configuration - Configure MTL
3491	* @priv: driver private structure
3492	* Description: It is used for configurring MTL
3493	*/
3494	static void stmmac_mtl_configuration(struct stmmac_priv *priv)
3495	{
3496	u32 rx_queues_count = priv->plat->rx_queues_to_use;
3497	u32 tx_queues_count = priv->plat->tx_queues_to_use;
3498
3499	if (tx_queues_count > 1)
3500	stmmac_set_tx_queue_weight(priv);
3501
3502	/* Configure MTL RX algorithms */
3503	if (rx_queues_count > 1)
3504	stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
3505	priv->plat->rx_sched_algorithm);
3506
3507	/* Configure MTL TX algorithms */
3508	if (tx_queues_count > 1)
3509	stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
3510	priv->plat->tx_sched_algorithm);
3511
3512	/* Configure CBS in AVB TX queues */
3513	if (tx_queues_count > 1)
3514	stmmac_configure_cbs(priv);
3515
3516	/* Map RX MTL to DMA channels */
3517	stmmac_rx_queue_dma_chan_map(priv);
3518
3519	/* Enable MAC RX Queues */
3520	stmmac_mac_enable_rx_queues(priv);
3521
3522	/* Set RX priorities */
3523	if (rx_queues_count > 1)
3524	stmmac_mac_config_rx_queues_prio(priv);
3525
3526	/* Set TX priorities */
3527	if (tx_queues_count > 1)
3528	stmmac_mac_config_tx_queues_prio(priv);
3529
3530	/* Set RX routing */
3531	if (rx_queues_count > 1)
3532	stmmac_mac_config_rx_queues_routing(priv);
3533
3534	/* Receive Side Scaling */
3535	if (rx_queues_count > 1)
3536	stmmac_mac_config_rss(priv);
3537	}
3538
3539	static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
3540	{
3541	if (priv->dma_cap.asp) {
3542	netdev_info(dev: priv->dev, format: "Enabling Safety Features\n");
3543	stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp,
3544	priv->plat->safety_feat_cfg);
3545	} else {
3546	netdev_info(dev: priv->dev, format: "No Safety Features support found\n");
3547	}
3548	}
3549
3550	/**
3551	* stmmac_hw_setup - setup mac in a usable state.
3552	* @dev : pointer to the device structure.
3553	* Description:
3554	* this is the main function to setup the HW in a usable state because the
3555	* dma engine is reset, the core registers are configured (e.g. AXI,
3556	* Checksum features, timers). The DMA is ready to start receiving and
3557	* transmitting.
3558	* Return value:
3559	* 0 on success and an appropriate (-)ve integer as defined in errno.h
3560	* file on failure.
3561	*/
3562	static int stmmac_hw_setup(struct net_device *dev)
3563	{
3564	struct stmmac_priv *priv = netdev_priv(dev);
3565	u32 rx_cnt = priv->plat->rx_queues_to_use;
3566	u32 tx_cnt = priv->plat->tx_queues_to_use;
3567	bool sph_en;
3568	u32 chan;
3569	int ret;
3570
3571	/* Make sure RX clock is enabled */
3572	if (priv->hw->phylink_pcs)
3573	phylink_pcs_pre_init(pl: priv->phylink, pcs: priv->hw->phylink_pcs);
3574
3575	/* Note that clk_rx_i must be running for reset to complete. This
3576	* clock may also be required when setting the MAC address.
3577	*
3578	* Block the receive clock stop for LPI mode at the PHY in case
3579	* the link is established with EEE mode active.
3580	*/
3581	phylink_rx_clk_stop_block(priv->phylink);
3582
3583	/* DMA initialization and SW reset */
3584	ret = stmmac_init_dma_engine(priv);
3585	if (ret < 0) {
3586	phylink_rx_clk_stop_unblock(priv->phylink);
3587	netdev_err(dev: priv->dev, format: "%s: DMA engine initialization failed\n",
3588	__func__);
3589	return ret;
3590	}
3591
3592	/* Copy the MAC addr into the HW */
3593	stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
3594	phylink_rx_clk_stop_unblock(priv->phylink);
3595
3596	/* Initialize the MAC Core */
3597	stmmac_core_init(priv, priv->hw, dev);
3598
3599	/* Initialize MTL*/
3600	stmmac_mtl_configuration(priv);
3601
3602	/* Initialize Safety Features */
3603	stmmac_safety_feat_configuration(priv);
3604
3605	ret = stmmac_rx_ipc(priv, priv->hw);
3606	if (!ret) {
3607	netdev_warn(dev: priv->dev, format: "RX IPC Checksum Offload disabled\n");
3608	priv->plat->rx_coe = STMMAC_RX_COE_NONE;
3609	priv->hw->rx_csum = 0;
3610	}
3611
3612	/* Enable the MAC Rx/Tx */
3613	stmmac_mac_set(priv, priv->ioaddr, true);
3614
3615	/* Set the HW DMA mode and the COE */
3616	stmmac_dma_operation_mode(priv);
3617
3618	stmmac_mmc_setup(priv);
3619
3620	if (priv->use_riwt) {
3621	u32 queue;
3622
3623	for (queue = 0; queue < rx_cnt; queue++) {
3624	if (!priv->rx_riwt[queue])
3625	priv->rx_riwt[queue] = DEF_DMA_RIWT;
3626
3627	stmmac_rx_watchdog(priv, priv->ioaddr,
3628	priv->rx_riwt[queue], queue);
3629	}
3630	}
3631
3632	/* set TX and RX rings length */
3633	stmmac_set_rings_length(priv);
3634
3635	/* Enable TSO */
3636	if (priv->tso) {
3637	for (chan = 0; chan < tx_cnt; chan++) {
3638	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
3639
3640	/* TSO and TBS cannot co-exist */
3641	if (tx_q->tbs & STMMAC_TBS_AVAIL)
3642	continue;
3643
3644	stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
3645	}
3646	}
3647
3648	/* Enable Split Header */
3649	sph_en = (priv->hw->rx_csum > 0) && priv->sph_active;
3650	for (chan = 0; chan < rx_cnt; chan++)
3651	stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
3652
3653
3654	/* VLAN Tag Insertion */
3655	if (priv->dma_cap.vlins)
3656	stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
3657
3658	/* TBS */
3659	for (chan = 0; chan < tx_cnt; chan++) {
3660	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan];
3661	int enable = tx_q->tbs & STMMAC_TBS_AVAIL;
3662
3663	stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);
3664	}
3665
3666	/* Configure real RX and TX queues */
3667	netif_set_real_num_rx_queues(dev, rxq: priv->plat->rx_queues_to_use);
3668	netif_set_real_num_tx_queues(dev, txq: priv->plat->tx_queues_to_use);
3669
3670	/* Start the ball rolling... */
3671	stmmac_start_all_dma(priv);
3672
3673	phylink_rx_clk_stop_block(priv->phylink);
3674	stmmac_set_hw_vlan_mode(priv, priv->hw);
3675	phylink_rx_clk_stop_unblock(priv->phylink);
3676
3677	return 0;
3678	}
3679
3680	static void stmmac_free_irq(struct net_device *dev,
3681	enum request_irq_err irq_err, int irq_idx)
3682	{
3683	struct stmmac_priv *priv = netdev_priv(dev);
3684	int j;
3685
3686	switch (irq_err) {
3687	case REQ_IRQ_ERR_ALL:
3688	irq_idx = priv->plat->tx_queues_to_use;
3689	fallthrough;
3690	case REQ_IRQ_ERR_TX:
3691	for (j = irq_idx - 1; j >= 0; j--) {
3692	if (priv->tx_irq[j] > 0) {
3693	irq_set_affinity_hint(irq: priv->tx_irq[j], NULL);
3694	free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]);
3695	}
3696	}
3697	irq_idx = priv->plat->rx_queues_to_use;
3698	fallthrough;
3699	case REQ_IRQ_ERR_RX:
3700	for (j = irq_idx - 1; j >= 0; j--) {
3701	if (priv->rx_irq[j] > 0) {
3702	irq_set_affinity_hint(irq: priv->rx_irq[j], NULL);
3703	free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]);
3704	}
3705	}
3706
3707	if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq)
3708	free_irq(priv->sfty_ue_irq, dev);
3709	fallthrough;
3710	case REQ_IRQ_ERR_SFTY_UE:
3711	if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq)
3712	free_irq(priv->sfty_ce_irq, dev);
3713	fallthrough;
3714	case REQ_IRQ_ERR_SFTY_CE:
3715	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq)
3716	free_irq(priv->lpi_irq, dev);
3717	fallthrough;
3718	case REQ_IRQ_ERR_LPI:
3719	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq)
3720	free_irq(priv->wol_irq, dev);
3721	fallthrough;
3722	case REQ_IRQ_ERR_SFTY:
3723	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq)
3724	free_irq(priv->sfty_irq, dev);
3725	fallthrough;
3726	case REQ_IRQ_ERR_WOL:
3727	free_irq(dev->irq, dev);
3728	fallthrough;
3729	case REQ_IRQ_ERR_MAC:
3730	case REQ_IRQ_ERR_NO:
3731	/* If MAC IRQ request error, no more IRQ to free */
3732	break;
3733	}
3734	}
3735
3736	static int stmmac_request_irq_multi_msi(struct net_device *dev)
3737	{
3738	struct stmmac_priv *priv = netdev_priv(dev);
3739	enum request_irq_err irq_err;
3740	int irq_idx = 0;
3741	char *int_name;
3742	int ret;
3743	int i;
3744
3745	/* For common interrupt */
3746	int_name = priv->int_name_mac;
3747	sprintf(buf: int_name, fmt: "%s:%s", dev->name, "mac");
3748	ret = request_irq(irq: dev->irq, handler: stmmac_mac_interrupt,
3749	flags: 0, name: int_name, dev);
3750	if (unlikely(ret < 0)) {
3751	netdev_err(dev: priv->dev,
3752	format: "%s: alloc mac MSI %d (error: %d)\n",
3753	__func__, dev->irq, ret);
3754	irq_err = REQ_IRQ_ERR_MAC;
3755	goto irq_error;
3756	}
3757
3758	/* Request the Wake IRQ in case of another line
3759	* is used for WoL
3760	*/
3761	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {
3762	int_name = priv->int_name_wol;
3763	sprintf(buf: int_name, fmt: "%s:%s", dev->name, "wol");
3764	ret = request_irq(irq: priv->wol_irq,
3765	handler: stmmac_mac_interrupt,
3766	flags: 0, name: int_name, dev);
3767	if (unlikely(ret < 0)) {
3768	netdev_err(dev: priv->dev,
3769	format: "%s: alloc wol MSI %d (error: %d)\n",
3770	__func__, priv->wol_irq, ret);
3771	irq_err = REQ_IRQ_ERR_WOL;
3772	goto irq_error;
3773	}
3774	}
3775
3776	/* Request the LPI IRQ in case of another line
3777	* is used for LPI
3778	*/
3779	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {
3780	int_name = priv->int_name_lpi;
3781	sprintf(buf: int_name, fmt: "%s:%s", dev->name, "lpi");
3782	ret = request_irq(irq: priv->lpi_irq,
3783	handler: stmmac_mac_interrupt,
3784	flags: 0, name: int_name, dev);
3785	if (unlikely(ret < 0)) {
3786	netdev_err(dev: priv->dev,
3787	format: "%s: alloc lpi MSI %d (error: %d)\n",
3788	__func__, priv->lpi_irq, ret);
3789	irq_err = REQ_IRQ_ERR_LPI;
3790	goto irq_error;
3791	}
3792	}
3793
3794	/* Request the common Safety Feature Correctible/Uncorrectible
3795	* Error line in case of another line is used
3796	*/
3797	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) {
3798	int_name = priv->int_name_sfty;
3799	sprintf(buf: int_name, fmt: "%s:%s", dev->name, "safety");
3800	ret = request_irq(irq: priv->sfty_irq, handler: stmmac_safety_interrupt,
3801	flags: 0, name: int_name, dev);
3802	if (unlikely(ret < 0)) {
3803	netdev_err(dev: priv->dev,
3804	format: "%s: alloc sfty MSI %d (error: %d)\n",
3805	__func__, priv->sfty_irq, ret);
3806	irq_err = REQ_IRQ_ERR_SFTY;
3807	goto irq_error;
3808	}
3809	}
3810
3811	/* Request the Safety Feature Correctible Error line in
3812	* case of another line is used
3813	*/
3814	if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) {
3815	int_name = priv->int_name_sfty_ce;
3816	sprintf(buf: int_name, fmt: "%s:%s", dev->name, "safety-ce");
3817	ret = request_irq(irq: priv->sfty_ce_irq,
3818	handler: stmmac_safety_interrupt,
3819	flags: 0, name: int_name, dev);
3820	if (unlikely(ret < 0)) {
3821	netdev_err(dev: priv->dev,
3822	format: "%s: alloc sfty ce MSI %d (error: %d)\n",
3823	__func__, priv->sfty_ce_irq, ret);
3824	irq_err = REQ_IRQ_ERR_SFTY_CE;
3825	goto irq_error;
3826	}
3827	}
3828
3829	/* Request the Safety Feature Uncorrectible Error line in
3830	* case of another line is used
3831	*/
3832	if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) {
3833	int_name = priv->int_name_sfty_ue;
3834	sprintf(buf: int_name, fmt: "%s:%s", dev->name, "safety-ue");
3835	ret = request_irq(irq: priv->sfty_ue_irq,
3836	handler: stmmac_safety_interrupt,
3837	flags: 0, name: int_name, dev);
3838	if (unlikely(ret < 0)) {
3839	netdev_err(dev: priv->dev,
3840	format: "%s: alloc sfty ue MSI %d (error: %d)\n",
3841	__func__, priv->sfty_ue_irq, ret);
3842	irq_err = REQ_IRQ_ERR_SFTY_UE;
3843	goto irq_error;
3844	}
3845	}
3846
3847	/* Request Rx MSI irq */
3848	for (i = 0; i < priv->plat->rx_queues_to_use; i++) {
3849	if (i >= MTL_MAX_RX_QUEUES)
3850	break;
3851	if (priv->rx_irq[i] == 0)
3852	continue;
3853
3854	int_name = priv->int_name_rx_irq[i];
3855	sprintf(buf: int_name, fmt: "%s:%s-%d", dev->name, "rx", i);
3856	ret = request_irq(irq: priv->rx_irq[i],
3857	handler: stmmac_msi_intr_rx,
3858	flags: 0, name: int_name, dev: &priv->dma_conf.rx_queue[i]);
3859	if (unlikely(ret < 0)) {
3860	netdev_err(dev: priv->dev,
3861	format: "%s: alloc rx-%d MSI %d (error: %d)\n",
3862	__func__, i, priv->rx_irq[i], ret);
3863	irq_err = REQ_IRQ_ERR_RX;
3864	irq_idx = i;
3865	goto irq_error;
3866	}
3867	irq_set_affinity_hint(irq: priv->rx_irq[i],
3868	cpumask_of(i % num_online_cpus()));
3869	}
3870
3871	/* Request Tx MSI irq */
3872	for (i = 0; i < priv->plat->tx_queues_to_use; i++) {
3873	if (i >= MTL_MAX_TX_QUEUES)
3874	break;
3875	if (priv->tx_irq[i] == 0)
3876	continue;
3877
3878	int_name = priv->int_name_tx_irq[i];
3879	sprintf(buf: int_name, fmt: "%s:%s-%d", dev->name, "tx", i);
3880	ret = request_irq(irq: priv->tx_irq[i],
3881	handler: stmmac_msi_intr_tx,
3882	flags: 0, name: int_name, dev: &priv->dma_conf.tx_queue[i]);
3883	if (unlikely(ret < 0)) {
3884	netdev_err(dev: priv->dev,
3885	format: "%s: alloc tx-%d MSI %d (error: %d)\n",
3886	__func__, i, priv->tx_irq[i], ret);
3887	irq_err = REQ_IRQ_ERR_TX;
3888	irq_idx = i;
3889	goto irq_error;
3890	}
3891	irq_set_affinity_hint(irq: priv->tx_irq[i],
3892	cpumask_of(i % num_online_cpus()));
3893	}
3894
3895	return 0;
3896
3897	irq_error:
3898	stmmac_free_irq(dev, irq_err, irq_idx);
3899	return ret;
3900	}
3901
3902	static int stmmac_request_irq_single(struct net_device *dev)
3903	{
3904	struct stmmac_priv *priv = netdev_priv(dev);
3905	enum request_irq_err irq_err;
3906	int ret;
3907
3908	ret = request_irq(irq: dev->irq, handler: stmmac_interrupt,
3909	IRQF_SHARED, name: dev->name, dev);
3910	if (unlikely(ret < 0)) {
3911	netdev_err(dev: priv->dev,
3912	format: "%s: ERROR: allocating the IRQ %d (error: %d)\n",
3913	__func__, dev->irq, ret);
3914	irq_err = REQ_IRQ_ERR_MAC;
3915	goto irq_error;
3916	}
3917
3918	/* Request the Wake IRQ in case of another line
3919	* is used for WoL
3920	*/
3921	if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {
3922	ret = request_irq(irq: priv->wol_irq, handler: stmmac_interrupt,
3923	IRQF_SHARED, name: dev->name, dev);
3924	if (unlikely(ret < 0)) {
3925	netdev_err(dev: priv->dev,
3926	format: "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
3927	__func__, priv->wol_irq, ret);
3928	irq_err = REQ_IRQ_ERR_WOL;
3929	goto irq_error;
3930	}
3931	}
3932
3933	/* Request the IRQ lines */
3934	if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {
3935	ret = request_irq(irq: priv->lpi_irq, handler: stmmac_interrupt,
3936	IRQF_SHARED, name: dev->name, dev);
3937	if (unlikely(ret < 0)) {
3938	netdev_err(dev: priv->dev,
3939	format: "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
3940	__func__, priv->lpi_irq, ret);
3941	irq_err = REQ_IRQ_ERR_LPI;
3942	goto irq_error;
3943	}
3944	}
3945
3946	/* Request the common Safety Feature Correctible/Uncorrectible
3947	* Error line in case of another line is used
3948	*/
3949	if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) {
3950	ret = request_irq(irq: priv->sfty_irq, handler: stmmac_safety_interrupt,
3951	IRQF_SHARED, name: dev->name, dev);
3952	if (unlikely(ret < 0)) {
3953	netdev_err(dev: priv->dev,
3954	format: "%s: ERROR: allocating the sfty IRQ %d (%d)\n",
3955	__func__, priv->sfty_irq, ret);
3956	irq_err = REQ_IRQ_ERR_SFTY;
3957	goto irq_error;
3958	}
3959	}
3960
3961	return 0;
3962
3963	irq_error:
3964	stmmac_free_irq(dev, irq_err, irq_idx: 0);
3965	return ret;
3966	}
3967
3968	static int stmmac_request_irq(struct net_device *dev)
3969	{
3970	struct stmmac_priv *priv = netdev_priv(dev);
3971	int ret;
3972
3973	/* Request the IRQ lines */
3974	if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN)
3975	ret = stmmac_request_irq_multi_msi(dev);
3976	else
3977	ret = stmmac_request_irq_single(dev);
3978
3979	return ret;
3980	}
3981
3982	/**
3983	* stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue
3984	* @priv: driver private structure
3985	* @mtu: MTU to setup the dma queue and buf with
3986	* Description: Allocate and generate a dma_conf based on the provided MTU.
3987	* Allocate the Tx/Rx DMA queue and init them.
3988	* Return value:
3989	* the dma_conf allocated struct on success and an appropriate ERR_PTR on failure.
3990	*/
3991	static struct stmmac_dma_conf *
3992	stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu)
3993	{
3994	struct stmmac_dma_conf *dma_conf;
3995	int chan, bfsize, ret;
3996
3997	dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL);
3998	if (!dma_conf) {
3999	netdev_err(dev: priv->dev, format: "%s: DMA conf allocation failed\n",
4000	__func__);
4001	return ERR_PTR(error: -ENOMEM);
4002	}
4003
4004	/* Returns 0 or BUF_SIZE_16KiB if mtu > 8KiB and dwmac4 or ring mode */
4005	bfsize = stmmac_set_16kib_bfsize(priv, mtu);
4006	if (bfsize < 0)
4007	bfsize = 0;
4008
4009	if (bfsize < BUF_SIZE_16KiB)
4010	bfsize = stmmac_set_bfsize(mtu);
4011
4012	dma_conf->dma_buf_sz = bfsize;
4013	/* Chose the tx/rx size from the already defined one in the
4014	* priv struct. (if defined)
4015	*/
4016	dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size;
4017	dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size;
4018
4019	if (!dma_conf->dma_tx_size)
4020	dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE;
4021	if (!dma_conf->dma_rx_size)
4022	dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE;
4023
4024	/* Earlier check for TBS */
4025	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {
4026	struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan];
4027	int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;
4028
4029	/* Setup per-TXQ tbs flag before TX descriptor alloc */
4030	tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0;
4031	}
4032
4033	ret = alloc_dma_desc_resources(priv, dma_conf);
4034	if (ret < 0) {
4035	netdev_err(dev: priv->dev, format: "%s: DMA descriptors allocation failed\n",
4036	__func__);
4037	goto alloc_error;
4038	}
4039
4040	ret = init_dma_desc_rings(dev: priv->dev, dma_conf, GFP_KERNEL);
4041	if (ret < 0) {
4042	netdev_err(dev: priv->dev, format: "%s: DMA descriptors initialization failed\n",
4043	__func__);
4044	goto init_error;
4045	}
4046
4047	return dma_conf;
4048
4049	init_error:
4050	free_dma_desc_resources(priv, dma_conf);
4051	alloc_error:
4052	kfree(objp: dma_conf);
4053	return ERR_PTR(error: ret);
4054	}
4055
4056	/**
4057	* __stmmac_open - open entry point of the driver
4058	* @dev : pointer to the device structure.
4059	* @dma_conf : structure to take the dma data
4060	* Description:
4061	* This function is the open entry point of the driver.
4062	* Return value:
4063	* 0 on success and an appropriate (-)ve integer as defined in errno.h
4064	* file on failure.
4065	*/
4066	static int __stmmac_open(struct net_device *dev,
4067	struct stmmac_dma_conf *dma_conf)
4068	{
4069	struct stmmac_priv *priv = netdev_priv(dev);
4070	u32 chan;
4071	int ret;
4072
4073	for (int i = 0; i < MTL_MAX_TX_QUEUES; i++)
4074	if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN)
4075	dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs;
4076	memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf));
4077
4078	stmmac_reset_queues_param(priv);
4079
4080	if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&
4081	priv->plat->serdes_powerup) {
4082	ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv);
4083	if (ret < 0) {
4084	netdev_err(dev: priv->dev, format: "%s: Serdes powerup failed\n",
4085	__func__);
4086	goto init_error;
4087	}
4088	}
4089
4090	ret = stmmac_hw_setup(dev);
4091	if (ret < 0) {
4092	netdev_err(dev: priv->dev, format: "%s: Hw setup failed\n", __func__);
4093	goto init_error;
4094	}
4095
4096	stmmac_setup_ptp(priv);
4097
4098	stmmac_init_coalesce(priv);
4099
4100	phylink_start(priv->phylink);
4101
4102	ret = stmmac_request_irq(dev);
4103	if (ret)
4104	goto irq_error;
4105
4106	stmmac_enable_all_queues(priv);
4107	netif_tx_start_all_queues(dev: priv->dev);
4108	stmmac_enable_all_dma_irq(priv);
4109
4110	return 0;
4111
4112	irq_error:
4113	phylink_stop(priv->phylink);
4114
4115	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
4116	hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer);
4117
4118	stmmac_release_ptp(priv);
4119	init_error:
4120	return ret;
4121	}
4122
4123	static int stmmac_open(struct net_device *dev)
4124	{
4125	struct stmmac_priv *priv = netdev_priv(dev);
4126	struct stmmac_dma_conf *dma_conf;
4127	int ret;
4128
4129	/* Initialise the tx lpi timer, converting from msec to usec */
4130	if (!priv->tx_lpi_timer)
4131	priv->tx_lpi_timer = eee_timer * 1000;
4132
4133	dma_conf = stmmac_setup_dma_desc(priv, mtu: dev->mtu);
4134	if (IS_ERR(ptr: dma_conf))
4135	return PTR_ERR(ptr: dma_conf);
4136
4137	ret = pm_runtime_resume_and_get(dev: priv->device);
4138	if (ret < 0)
4139	goto err_dma_resources;
4140
4141	ret = stmmac_init_phy(dev);
4142	if (ret)
4143	goto err_runtime_pm;
4144
4145	ret = __stmmac_open(dev, dma_conf);
4146	if (ret)
4147	goto err_disconnect_phy;
4148
4149	kfree(objp: dma_conf);
4150
4151	/* We may have called phylink_speed_down before */
4152	phylink_speed_up(pl: priv->phylink);
4153
4154	return ret;
4155
4156	err_disconnect_phy:
4157	phylink_disconnect_phy(priv->phylink);
4158	err_runtime_pm:
4159	pm_runtime_put(dev: priv->device);
4160	err_dma_resources:
4161	free_dma_desc_resources(priv, dma_conf);
4162	kfree(objp: dma_conf);
4163	return ret;
4164	}
4165
4166	static void __stmmac_release(struct net_device *dev)
4167	{
4168	struct stmmac_priv *priv = netdev_priv(dev);
4169	u32 chan;
4170
4171	/* Stop and disconnect the PHY */
4172	phylink_stop(priv->phylink);
4173
4174	stmmac_disable_all_queues(priv);
4175
4176	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
4177	hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer);
4178
4179	netif_tx_disable(dev);
4180
4181	/* Free the IRQ lines */
4182	stmmac_free_irq(dev, irq_err: REQ_IRQ_ERR_ALL, irq_idx: 0);
4183
4184	/* Stop TX/RX DMA and clear the descriptors */
4185	stmmac_stop_all_dma(priv);
4186
4187	/* Release and free the Rx/Tx resources */
4188	free_dma_desc_resources(priv, dma_conf: &priv->dma_conf);
4189
4190	/* Powerdown Serdes if there is */
4191	if (priv->plat->serdes_powerdown)
4192	priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv);
4193
4194	stmmac_release_ptp(priv);
4195
4196	if (stmmac_fpe_supported(priv))
4197	ethtool_mmsv_stop(mmsv: &priv->fpe_cfg.mmsv);
4198	}
4199
4200	/**
4201	* stmmac_release - close entry point of the driver
4202	* @dev : device pointer.
4203	* Description:
4204	* This is the stop entry point of the driver.
4205	*/
4206	static int stmmac_release(struct net_device *dev)
4207	{
4208	struct stmmac_priv *priv = netdev_priv(dev);
4209
4210	/* If the PHY or MAC has WoL enabled, then the PHY will not be
4211	* suspended when phylink_stop() is called below. Set the PHY
4212	* to its slowest speed to save power.
4213	*/
4214	if (device_may_wakeup(dev: priv->device))
4215	phylink_speed_down(pl: priv->phylink, sync: false);
4216
4217	__stmmac_release(dev);
4218
4219	phylink_disconnect_phy(priv->phylink);
4220	pm_runtime_put(dev: priv->device);
4221
4222	return 0;
4223	}
4224
4225	static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
4226	struct stmmac_tx_queue *tx_q)
4227	{
4228	struct dma_desc *p;
4229	u16 tag = 0x0;
4230
4231	if (!priv->dma_cap.vlins || !skb_vlan_tag_present(skb))
4232	return false;
4233
4234	tag = skb_vlan_tag_get(skb);
4235
4236	if (tx_q->tbs & STMMAC_TBS_AVAIL)
4237	p = &tx_q->dma_entx[tx_q->cur_tx].basic;
4238	else
4239	p = &tx_q->dma_tx[tx_q->cur_tx];
4240
4241	if (stmmac_set_desc_vlan_tag(priv, p, tag, 0x0, 0x0))
4242	return false;
4243
4244	stmmac_set_tx_owner(priv, p);
4245	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
4246	return true;
4247	}
4248
4249	/**
4250	* stmmac_tso_allocator - close entry point of the driver
4251	* @priv: driver private structure
4252	* @des: buffer start address
4253	* @total_len: total length to fill in descriptors
4254	* @last_segment: condition for the last descriptor
4255	* @queue: TX queue index
4256	* Description:
4257	* This function fills descriptor and request new descriptors according to
4258	* buffer length to fill
4259	*/
4260	static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
4261	int total_len, bool last_segment, u32 queue)
4262	{
4263	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
4264	struct dma_desc *desc;
4265	u32 buff_size;
4266	int tmp_len;
4267
4268	tmp_len = total_len;
4269
4270	while (tmp_len > 0) {
4271	dma_addr_t curr_addr;
4272
4273	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
4274	priv->dma_conf.dma_tx_size);
4275	WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
4276
4277	if (tx_q->tbs & STMMAC_TBS_AVAIL)
4278	desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
4279	else
4280	desc = &tx_q->dma_tx[tx_q->cur_tx];
4281
4282	curr_addr = des + (total_len - tmp_len);
4283	stmmac_set_desc_addr(priv, desc, curr_addr);
4284	buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
4285	TSO_MAX_BUFF_SIZE : tmp_len;
4286
4287	stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
4288	0, 1,
4289	(last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
4290	0, 0);
4291
4292	tmp_len -= TSO_MAX_BUFF_SIZE;
4293	}
4294	}
4295
4296	static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue)
4297	{
4298	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
4299	int desc_size;
4300
4301	if (likely(priv->extend_desc))
4302	desc_size = sizeof(struct dma_extended_desc);
4303	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4304	desc_size = sizeof(struct dma_edesc);
4305	else
4306	desc_size = sizeof(struct dma_desc);
4307
4308	/* The own bit must be the latest setting done when prepare the
4309	* descriptor and then barrier is needed to make sure that
4310	* all is coherent before granting the DMA engine.
4311	*/
4312	wmb();
4313
4314	tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
4315	stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
4316	}
4317
4318	/**
4319	* stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
4320	* @skb : the socket buffer
4321	* @dev : device pointer
4322	* Description: this is the transmit function that is called on TSO frames
4323	* (support available on GMAC4 and newer chips).
4324	* Diagram below show the ring programming in case of TSO frames:
4325	*
4326	* First Descriptor
4327	* --------
4328	* | DES0 |---> buffer1 = L2/L3/L4 header
4329	* | DES1 |---> can be used as buffer2 for TCP Payload if the DMA AXI address
4330	* | | width is 32-bit, but we never use it.
4331	* | | Also can be used as the most-significant 8-bits or 16-bits of
4332	* | | buffer1 address pointer if the DMA AXI address width is 40-bit
4333	* | | or 48-bit, and we always use it.
4334	* | DES2 |---> buffer1 len
4335	* | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
4336	* --------
4337	* --------
4338	* | DES0 |---> buffer1 = TCP Payload (can continue on next descr...)
4339	* | DES1 |---> same as the First Descriptor
4340	* | DES2 |---> buffer1 len
4341	* | DES3 |
4342	* --------
4343	* |
4344	* ...
4345	* |
4346	* --------
4347	* | DES0 |---> buffer1 = Split TCP Payload
4348	* | DES1 |---> same as the First Descriptor
4349	* | DES2 |---> buffer1 len
4350	* | DES3 |
4351	* --------
4352	*
4353	* mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
4354	*/
4355	static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
4356	{
4357	struct dma_desc *desc, *first, *mss_desc = NULL;
4358	struct stmmac_priv *priv = netdev_priv(dev);
4359	unsigned int first_entry, tx_packets;
4360	struct stmmac_txq_stats *txq_stats;
4361	struct stmmac_tx_queue *tx_q;
4362	bool set_ic, is_last_segment;
4363	u32 pay_len, mss, queue;
4364	int i, first_tx, nfrags;
4365	u8 proto_hdr_len, hdr;
4366	dma_addr_t des;
4367
4368	/* Always insert VLAN tag to SKB payload for TSO frames.
4369	*
4370	* Never insert VLAN tag by HW, since segments splited by
4371	* TSO engine will be un-tagged by mistake.
4372	*/
4373	if (skb_vlan_tag_present(skb)) {
4374	skb = __vlan_hwaccel_push_inside(skb);
4375	if (unlikely(!skb)) {
4376	priv->xstats.tx_dropped++;
4377	return NETDEV_TX_OK;
4378	}
4379	}
4380
4381	nfrags = skb_shinfo(skb)->nr_frags;
4382	queue = skb_get_queue_mapping(skb);
4383
4384	tx_q = &priv->dma_conf.tx_queue[queue];
4385	txq_stats = &priv->xstats.txq_stats[queue];
4386	first_tx = tx_q->cur_tx;
4387
4388	/* Compute header lengths */
4389	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
4390	proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
4391	hdr = sizeof(struct udphdr);
4392	} else {
4393	proto_hdr_len = skb_tcp_all_headers(skb);
4394	hdr = tcp_hdrlen(skb);
4395	}
4396
4397	/* Desc availability based on threshold should be enough safe */
4398	if (unlikely(stmmac_tx_avail(priv, queue) <
4399	(((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
4400	if (!netif_tx_queue_stopped(dev_queue: netdev_get_tx_queue(dev, index: queue))) {
4401	netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev,
4402	index: queue));
4403	/* This is a hard error, log it. */
4404	netdev_err(dev: priv->dev,
4405	format: "%s: Tx Ring full when queue awake\n",
4406	__func__);
4407	}
4408	return NETDEV_TX_BUSY;
4409	}
4410
4411	pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
4412
4413	mss = skb_shinfo(skb)->gso_size;
4414
4415	/* set new MSS value if needed */
4416	if (mss != tx_q->mss) {
4417	if (tx_q->tbs & STMMAC_TBS_AVAIL)
4418	mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
4419	else
4420	mss_desc = &tx_q->dma_tx[tx_q->cur_tx];
4421
4422	stmmac_set_mss(priv, mss_desc, mss);
4423	tx_q->mss = mss;
4424	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
4425	priv->dma_conf.dma_tx_size);
4426	WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
4427	}
4428
4429	if (netif_msg_tx_queued(priv)) {
4430	pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
4431	__func__, hdr, proto_hdr_len, pay_len, mss);
4432	pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
4433	skb->data_len);
4434	}
4435
4436	first_entry = tx_q->cur_tx;
4437	WARN_ON(tx_q->tx_skbuff[first_entry]);
4438
4439	if (tx_q->tbs & STMMAC_TBS_AVAIL)
4440	desc = &tx_q->dma_entx[first_entry].basic;
4441	else
4442	desc = &tx_q->dma_tx[first_entry];
4443	first = desc;
4444
4445	/* first descriptor: fill Headers on Buf1 */
4446	des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
4447	DMA_TO_DEVICE);
4448	if (dma_mapping_error(dev: priv->device, dma_addr: des))
4449	goto dma_map_err;
4450
4451	stmmac_set_desc_addr(priv, first, des);
4452	stmmac_tso_allocator(priv, des: des + proto_hdr_len, total_len: pay_len,
4453	last_segment: (nfrags == 0), queue);
4454
4455	/* In case two or more DMA transmit descriptors are allocated for this
4456	* non-paged SKB data, the DMA buffer address should be saved to
4457	* tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor,
4458	* and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee
4459	* that stmmac_tx_clean() does not unmap the entire DMA buffer too early
4460	* since the tail areas of the DMA buffer can be accessed by DMA engine
4461	* sooner or later.
4462	* By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf
4463	* corresponding to the last descriptor, stmmac_tx_clean() will unmap
4464	* this DMA buffer right after the DMA engine completely finishes the
4465	* full buffer transmission.
4466	*/
4467	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
4468	tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb);
4469	tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false;
4470	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
4471
4472	/* Prepare fragments */
4473	for (i = 0; i < nfrags; i++) {
4474	const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4475
4476	des = skb_frag_dma_map(priv->device, frag, 0,
4477	skb_frag_size(frag),
4478	DMA_TO_DEVICE);
4479	if (dma_mapping_error(dev: priv->device, dma_addr: des))
4480	goto dma_map_err;
4481
4482	stmmac_tso_allocator(priv, des, total_len: skb_frag_size(frag),
4483	last_segment: (i == nfrags - 1), queue);
4484
4485	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
4486	tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
4487	tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
4488	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
4489	}
4490
4491	tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
4492
4493	/* Only the last descriptor gets to point to the skb. */
4494	tx_q->tx_skbuff[tx_q->cur_tx] = skb;
4495	tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
4496
4497	/* Manage tx mitigation */
4498	tx_packets = (tx_q->cur_tx + 1) - first_tx;
4499	tx_q->tx_count_frames += tx_packets;
4500
4501	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
4502	set_ic = true;
4503	else if (!priv->tx_coal_frames[queue])
4504	set_ic = false;
4505	else if (tx_packets > priv->tx_coal_frames[queue])
4506	set_ic = true;
4507	else if ((tx_q->tx_count_frames %
4508	priv->tx_coal_frames[queue]) < tx_packets)
4509	set_ic = true;
4510	else
4511	set_ic = false;
4512
4513	if (set_ic) {
4514	if (tx_q->tbs & STMMAC_TBS_AVAIL)
4515	desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
4516	else
4517	desc = &tx_q->dma_tx[tx_q->cur_tx];
4518
4519	tx_q->tx_count_frames = 0;
4520	stmmac_set_tx_ic(priv, desc);
4521	}
4522
4523	/* We've used all descriptors we need for this skb, however,
4524	* advance cur_tx so that it references a fresh descriptor.
4525	* ndo_start_xmit will fill this descriptor the next time it's
4526	* called and stmmac_tx_clean may clean up to this descriptor.
4527	*/
4528	tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
4529
4530	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
4531	netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
4532	__func__);
4533	netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: queue));
4534	}
4535
4536	u64_stats_update_begin(syncp: &txq_stats->q_syncp);
4537	u64_stats_add(p: &txq_stats->q.tx_bytes, val: skb->len);
4538	u64_stats_inc(p: &txq_stats->q.tx_tso_frames);
4539	u64_stats_add(p: &txq_stats->q.tx_tso_nfrags, val: nfrags);
4540	if (set_ic)
4541	u64_stats_inc(p: &txq_stats->q.tx_set_ic_bit);
4542	u64_stats_update_end(syncp: &txq_stats->q_syncp);
4543
4544	if (priv->sarc_type)
4545	stmmac_set_desc_sarc(priv, first, priv->sarc_type);
4546
4547	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
4548	priv->hwts_tx_en)) {
4549	/* declare that device is doing timestamping */
4550	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4551	stmmac_enable_tx_timestamp(priv, first);
4552	}
4553
4554	/* If we only have one entry used, then the first entry is the last
4555	* segment.
4556	*/
4557	is_last_segment = ((tx_q->cur_tx - first_entry) &
4558	(priv->dma_conf.dma_tx_size - 1)) == 1;
4559
4560	/* Complete the first descriptor before granting the DMA */
4561	stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1,
4562	is_last_segment, hdr / 4,
4563	skb->len - proto_hdr_len);
4564
4565	/* If context desc is used to change MSS */
4566	if (mss_desc) {
4567	/* Make sure that first descriptor has been completely
4568	* written, including its own bit. This is because MSS is
4569	* actually before first descriptor, so we need to make
4570	* sure that MSS's own bit is the last thing written.
4571	*/
4572	dma_wmb();
4573	stmmac_set_tx_owner(priv, mss_desc);
4574	}
4575
4576	if (netif_msg_pktdata(priv)) {
4577	pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
4578	__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
4579	tx_q->cur_tx, first, nfrags);
4580	pr_info(">>> frame to be transmitted: ");
4581	print_pkt(buf: skb->data, len: skb_headlen(skb));
4582	}
4583
4584	netdev_tx_sent_queue(dev_queue: netdev_get_tx_queue(dev, index: queue), bytes: skb->len);
4585	skb_tx_timestamp(skb);
4586
4587	stmmac_flush_tx_descriptors(priv, queue);
4588	stmmac_tx_timer_arm(priv, queue);
4589
4590	return NETDEV_TX_OK;
4591
4592	dma_map_err:
4593	dev_err(priv->device, "Tx dma map failed\n");
4594	dev_kfree_skb(skb);
4595	priv->xstats.tx_dropped++;
4596	return NETDEV_TX_OK;
4597	}
4598
4599	/**
4600	* stmmac_has_ip_ethertype() - Check if packet has IP ethertype
4601	* @skb: socket buffer to check
4602	*
4603	* Check if a packet has an ethertype that will trigger the IP header checks
4604	* and IP/TCP checksum engine of the stmmac core.
4605	*
4606	* Return: true if the ethertype can trigger the checksum engine, false
4607	* otherwise
4608	*/
4609	static bool stmmac_has_ip_ethertype(struct sk_buff *skb)
4610	{
4611	int depth = 0;
4612	__be16 proto;
4613
4614	proto = __vlan_get_protocol(skb, type: eth_header_parse_protocol(skb),
4615	depth: &depth);
4616
4617	return (depth <= ETH_HLEN) &&
4618	(proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6));
4619	}
4620
4621	/**
4622	* stmmac_xmit - Tx entry point of the driver
4623	* @skb : the socket buffer
4624	* @dev : device pointer
4625	* Description : this is the tx entry point of the driver.
4626	* It programs the chain or the ring and supports oversized frames
4627	* and SG feature.
4628	*/
4629	static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
4630	{
4631	bool enh_desc, has_vlan, set_ic, is_jumbo = false;
4632	struct stmmac_priv *priv = netdev_priv(dev);
4633	unsigned int nopaged_len = skb_headlen(skb);
4634	u32 queue = skb_get_queue_mapping(skb);
4635	int nfrags = skb_shinfo(skb)->nr_frags;
4636	unsigned int first_entry, tx_packets;
4637	int gso = skb_shinfo(skb)->gso_type;
4638	struct stmmac_txq_stats *txq_stats;
4639	struct dma_edesc *tbs_desc = NULL;
4640	struct dma_desc *desc, *first;
4641	struct stmmac_tx_queue *tx_q;
4642	int i, csum_insertion = 0;
4643	int entry, first_tx;
4644	dma_addr_t des;
4645	u32 sdu_len;
4646
4647	tx_q = &priv->dma_conf.tx_queue[queue];
4648	txq_stats = &priv->xstats.txq_stats[queue];
4649	first_tx = tx_q->cur_tx;
4650
4651	if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en)
4652	stmmac_stop_sw_lpi(priv);
4653
4654	/* Manage oversized TCP frames for GMAC4 device */
4655	if (skb_is_gso(skb) && priv->tso) {
4656	if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
4657	return stmmac_tso_xmit(skb, dev);
4658	if (priv->plat->core_type == DWMAC_CORE_GMAC4 &&
4659	(gso & SKB_GSO_UDP_L4))
4660	return stmmac_tso_xmit(skb, dev);
4661	}
4662
4663	if (priv->est && priv->est->enable &&
4664	priv->est->max_sdu[queue]) {
4665	sdu_len = skb->len;
4666	/* Add VLAN tag length if VLAN tag insertion offload is requested */
4667	if (priv->dma_cap.vlins && skb_vlan_tag_present(skb))
4668	sdu_len += VLAN_HLEN;
4669	if (sdu_len > priv->est->max_sdu[queue]) {
4670	priv->xstats.max_sdu_txq_drop[queue]++;
4671	goto max_sdu_err;
4672	}
4673	}
4674
4675	if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
4676	if (!netif_tx_queue_stopped(dev_queue: netdev_get_tx_queue(dev, index: queue))) {
4677	netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev,
4678	index: queue));
4679	/* This is a hard error, log it. */
4680	netdev_err(dev: priv->dev,
4681	format: "%s: Tx Ring full when queue awake\n",
4682	__func__);
4683	}
4684	return NETDEV_TX_BUSY;
4685	}
4686
4687	/* Check if VLAN can be inserted by HW */
4688	has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
4689
4690	entry = tx_q->cur_tx;
4691	first_entry = entry;
4692	WARN_ON(tx_q->tx_skbuff[first_entry]);
4693
4694	csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
4695	/* DWMAC IPs can be synthesized to support tx coe only for a few tx
4696	* queues. In that case, checksum offloading for those queues that don't
4697	* support tx coe needs to fallback to software checksum calculation.
4698	*
4699	* Packets that won't trigger the COE e.g. most DSA-tagged packets will
4700	* also have to be checksummed in software.
4701	*/
4702	if (csum_insertion &&
4703	(priv->plat->tx_queues_cfg[queue].coe_unsupported ||
4704	!stmmac_has_ip_ethertype(skb))) {
4705	if (unlikely(skb_checksum_help(skb)))
4706	goto dma_map_err;
4707	csum_insertion = !csum_insertion;
4708	}
4709
4710	if (likely(priv->extend_desc))
4711	desc = (struct dma_desc *)(tx_q->dma_etx + entry);
4712	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4713	desc = &tx_q->dma_entx[entry].basic;
4714	else
4715	desc = tx_q->dma_tx + entry;
4716
4717	first = desc;
4718
4719	if (has_vlan)
4720	stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
4721
4722	enh_desc = priv->plat->enh_desc;
4723	/* To program the descriptors according to the size of the frame */
4724	if (enh_desc)
4725	is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
4726
4727	if (unlikely(is_jumbo)) {
4728	entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
4729	if (unlikely(entry < 0) && (entry != -EINVAL))
4730	goto dma_map_err;
4731	}
4732
4733	for (i = 0; i < nfrags; i++) {
4734	const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4735	int len = skb_frag_size(frag);
4736	bool last_segment = (i == (nfrags - 1));
4737
4738	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
4739	WARN_ON(tx_q->tx_skbuff[entry]);
4740
4741	if (likely(priv->extend_desc))
4742	desc = (struct dma_desc *)(tx_q->dma_etx + entry);
4743	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4744	desc = &tx_q->dma_entx[entry].basic;
4745	else
4746	desc = tx_q->dma_tx + entry;
4747
4748	des = skb_frag_dma_map(priv->device, frag, 0, len,
4749	DMA_TO_DEVICE);
4750	if (dma_mapping_error(dev: priv->device, dma_addr: des))
4751	goto dma_map_err; /* should reuse desc w/o issues */
4752
4753	tx_q->tx_skbuff_dma[entry].buf = des;
4754
4755	stmmac_set_desc_addr(priv, desc, des);
4756
4757	tx_q->tx_skbuff_dma[entry].map_as_page = true;
4758	tx_q->tx_skbuff_dma[entry].len = len;
4759	tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
4760	tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
4761
4762	/* Prepare the descriptor and set the own bit too */
4763	stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
4764	priv->mode, 1, last_segment, skb->len);
4765	}
4766
4767	/* Only the last descriptor gets to point to the skb. */
4768	tx_q->tx_skbuff[entry] = skb;
4769	tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
4770
4771	/* According to the coalesce parameter the IC bit for the latest
4772	* segment is reset and the timer re-started to clean the tx status.
4773	* This approach takes care about the fragments: desc is the first
4774	* element in case of no SG.
4775	*/
4776	tx_packets = (entry + 1) - first_tx;
4777	tx_q->tx_count_frames += tx_packets;
4778
4779	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
4780	set_ic = true;
4781	else if (!priv->tx_coal_frames[queue])
4782	set_ic = false;
4783	else if (tx_packets > priv->tx_coal_frames[queue])
4784	set_ic = true;
4785	else if ((tx_q->tx_count_frames %
4786	priv->tx_coal_frames[queue]) < tx_packets)
4787	set_ic = true;
4788	else
4789	set_ic = false;
4790
4791	if (set_ic) {
4792	if (likely(priv->extend_desc))
4793	desc = &tx_q->dma_etx[entry].basic;
4794	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4795	desc = &tx_q->dma_entx[entry].basic;
4796	else
4797	desc = &tx_q->dma_tx[entry];
4798
4799	tx_q->tx_count_frames = 0;
4800	stmmac_set_tx_ic(priv, desc);
4801	}
4802
4803	/* We've used all descriptors we need for this skb, however,
4804	* advance cur_tx so that it references a fresh descriptor.
4805	* ndo_start_xmit will fill this descriptor the next time it's
4806	* called and stmmac_tx_clean may clean up to this descriptor.
4807	*/
4808	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
4809	tx_q->cur_tx = entry;
4810
4811	if (netif_msg_pktdata(priv)) {
4812	netdev_dbg(priv->dev,
4813	"%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
4814	__func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
4815	entry, first, nfrags);
4816
4817	netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
4818	print_pkt(buf: skb->data, len: skb->len);
4819	}
4820
4821	if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
4822	netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
4823	__func__);
4824	netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: queue));
4825	}
4826
4827	u64_stats_update_begin(syncp: &txq_stats->q_syncp);
4828	u64_stats_add(p: &txq_stats->q.tx_bytes, val: skb->len);
4829	if (set_ic)
4830	u64_stats_inc(p: &txq_stats->q.tx_set_ic_bit);
4831	u64_stats_update_end(syncp: &txq_stats->q_syncp);
4832
4833	if (priv->sarc_type)
4834	stmmac_set_desc_sarc(priv, first, priv->sarc_type);
4835
4836	/* Ready to fill the first descriptor and set the OWN bit w/o any
4837	* problems because all the descriptors are actually ready to be
4838	* passed to the DMA engine.
4839	*/
4840	if (likely(!is_jumbo)) {
4841	bool last_segment = (nfrags == 0);
4842
4843	des = dma_map_single(priv->device, skb->data,
4844	nopaged_len, DMA_TO_DEVICE);
4845	if (dma_mapping_error(dev: priv->device, dma_addr: des))
4846	goto dma_map_err;
4847
4848	tx_q->tx_skbuff_dma[first_entry].buf = des;
4849	tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB;
4850	tx_q->tx_skbuff_dma[first_entry].map_as_page = false;
4851
4852	stmmac_set_desc_addr(priv, first, des);
4853
4854	tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
4855	tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
4856
4857	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
4858	priv->hwts_tx_en)) {
4859	/* declare that device is doing timestamping */
4860	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4861	stmmac_enable_tx_timestamp(priv, first);
4862	}
4863
4864	/* Prepare the first descriptor setting the OWN bit too */
4865	stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
4866	csum_insertion, priv->mode, 0, last_segment,
4867	skb->len);
4868	}
4869
4870	if (tx_q->tbs & STMMAC_TBS_EN) {
4871	struct timespec64 ts = ns_to_timespec64(nsec: skb->tstamp);
4872
4873	tbs_desc = &tx_q->dma_entx[first_entry];
4874	stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);
4875	}
4876
4877	stmmac_set_tx_owner(priv, first);
4878
4879	netdev_tx_sent_queue(dev_queue: netdev_get_tx_queue(dev, index: queue), bytes: skb->len);
4880
4881	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);
4882	skb_tx_timestamp(skb);
4883	stmmac_flush_tx_descriptors(priv, queue);
4884	stmmac_tx_timer_arm(priv, queue);
4885
4886	return NETDEV_TX_OK;
4887
4888	dma_map_err:
4889	netdev_err(dev: priv->dev, format: "Tx DMA map failed\n");
4890	max_sdu_err:
4891	dev_kfree_skb(skb);
4892	priv->xstats.tx_dropped++;
4893	return NETDEV_TX_OK;
4894	}
4895
4896	static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
4897	{
4898	struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb);
4899	__be16 vlan_proto = veth->h_vlan_proto;
4900	u16 vlanid;
4901
4902	if ((vlan_proto == htons(ETH_P_8021Q) &&
4903	dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
4904	(vlan_proto == htons(ETH_P_8021AD) &&
4905	dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
4906	/* pop the vlan tag */
4907	vlanid = ntohs(veth->h_vlan_TCI);
4908	memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
4909	skb_pull(skb, VLAN_HLEN);
4910	__vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci: vlanid);
4911	}
4912	}
4913
4914	/**
4915	* stmmac_rx_refill - refill used skb preallocated buffers
4916	* @priv: driver private structure
4917	* @queue: RX queue index
4918	* Description : this is to reallocate the skb for the reception process
4919	* that is based on zero-copy.
4920	*/
4921	static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
4922	{
4923	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
4924	int dirty = stmmac_rx_dirty(priv, queue);
4925	unsigned int entry = rx_q->dirty_rx;
4926	gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
4927
4928	if (priv->dma_cap.host_dma_width <= 32)
4929	gfp |= GFP_DMA32;
4930
4931	while (dirty-- > 0) {
4932	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
4933	struct dma_desc *p;
4934	bool use_rx_wd;
4935
4936	if (priv->extend_desc)
4937	p = (struct dma_desc *)(rx_q->dma_erx + entry);
4938	else
4939	p = rx_q->dma_rx + entry;
4940
4941	if (!buf->page) {
4942	buf->page = page_pool_alloc_pages(pool: rx_q->page_pool, gfp);
4943	if (!buf->page)
4944	break;
4945	}
4946
4947	if (priv->sph_active && !buf->sec_page) {
4948	buf->sec_page = page_pool_alloc_pages(pool: rx_q->page_pool, gfp);
4949	if (!buf->sec_page)
4950	break;
4951
4952	buf->sec_addr = page_pool_get_dma_addr(page: buf->sec_page);
4953	}
4954
4955	buf->addr = page_pool_get_dma_addr(page: buf->page) + buf->page_offset;
4956
4957	stmmac_set_desc_addr(priv, p, buf->addr);
4958	if (priv->sph_active)
4959	stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
4960	else
4961	stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
4962	stmmac_refill_desc3(priv, rx_q, p);
4963
4964	rx_q->rx_count_frames++;
4965	rx_q->rx_count_frames += priv->rx_coal_frames[queue];
4966	if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])
4967	rx_q->rx_count_frames = 0;
4968
4969	use_rx_wd = !priv->rx_coal_frames[queue];
4970	use_rx_wd |= rx_q->rx_count_frames > 0;
4971	if (!priv->use_riwt)
4972	use_rx_wd = false;
4973
4974	dma_wmb();
4975	stmmac_set_rx_owner(priv, p, use_rx_wd);
4976
4977	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);
4978	}
4979	rx_q->dirty_rx = entry;
4980	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
4981	(rx_q->dirty_rx * sizeof(struct dma_desc));
4982	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
4983	/* Wake up Rx DMA from the suspend state if required */
4984	stmmac_enable_dma_reception(priv, priv->ioaddr, queue);
4985	}
4986
4987	static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
4988	struct dma_desc *p,
4989	int status, unsigned int len)
4990	{
4991	unsigned int plen = 0, hlen = 0;
4992	int coe = priv->hw->rx_csum;
4993
4994	/* Not first descriptor, buffer is always zero */
4995	if (priv->sph_active && len)
4996	return 0;
4997
4998	/* First descriptor, get split header length */
4999	stmmac_get_rx_header_len(priv, p, &hlen);
5000	if (priv->sph_active && hlen) {
5001	priv->xstats.rx_split_hdr_pkt_n++;
5002	return hlen;
5003	}
5004
5005	/* First descriptor, not last descriptor and not split header */
5006	if (status & rx_not_ls)
5007	return priv->dma_conf.dma_buf_sz;
5008
5009	plen = stmmac_get_rx_frame_len(priv, p, coe);
5010
5011	/* First descriptor and last descriptor and not split header */
5012	return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen);
5013	}
5014
5015	static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
5016	struct dma_desc *p,
5017	int status, unsigned int len)
5018	{
5019	int coe = priv->hw->rx_csum;
5020	unsigned int plen = 0;
5021
5022	/* Not split header, buffer is not available */
5023	if (!priv->sph_active)
5024	return 0;
5025
5026	/* Not last descriptor */
5027	if (status & rx_not_ls)
5028	return priv->dma_conf.dma_buf_sz;
5029
5030	plen = stmmac_get_rx_frame_len(priv, p, coe);
5031
5032	/* Last descriptor */
5033	return plen - len;
5034	}
5035
5036	static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,
5037	struct xdp_frame *xdpf, bool dma_map)
5038	{
5039	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue];
5040	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
5041	bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported;
5042	unsigned int entry = tx_q->cur_tx;
5043	struct dma_desc *tx_desc;
5044	dma_addr_t dma_addr;
5045	bool set_ic;
5046
5047	if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv))
5048	return STMMAC_XDP_CONSUMED;
5049
5050	if (priv->est && priv->est->enable &&
5051	priv->est->max_sdu[queue] &&
5052	xdpf->len > priv->est->max_sdu[queue]) {
5053	priv->xstats.max_sdu_txq_drop[queue]++;
5054	return STMMAC_XDP_CONSUMED;
5055	}
5056
5057	if (likely(priv->extend_desc))
5058	tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
5059	else if (tx_q->tbs & STMMAC_TBS_AVAIL)
5060	tx_desc = &tx_q->dma_entx[entry].basic;
5061	else
5062	tx_desc = tx_q->dma_tx + entry;
5063
5064	if (dma_map) {
5065	dma_addr = dma_map_single(priv->device, xdpf->data,
5066	xdpf->len, DMA_TO_DEVICE);
5067	if (dma_mapping_error(dev: priv->device, dma_addr))
5068	return STMMAC_XDP_CONSUMED;
5069
5070	tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO;
5071	} else {
5072	struct page *page = virt_to_page(xdpf->data);
5073
5074	dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) +
5075	xdpf->headroom;
5076	dma_sync_single_for_device(dev: priv->device, addr: dma_addr,
5077	size: xdpf->len, dir: DMA_BIDIRECTIONAL);
5078
5079	tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX;
5080	}
5081
5082	tx_q->tx_skbuff_dma[entry].buf = dma_addr;
5083	tx_q->tx_skbuff_dma[entry].map_as_page = false;
5084	tx_q->tx_skbuff_dma[entry].len = xdpf->len;
5085	tx_q->tx_skbuff_dma[entry].last_segment = true;
5086	tx_q->tx_skbuff_dma[entry].is_jumbo = false;
5087
5088	tx_q->xdpf[entry] = xdpf;
5089
5090	stmmac_set_desc_addr(priv, tx_desc, dma_addr);
5091
5092	stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len,
5093	csum, priv->mode, true, true,
5094	xdpf->len);
5095
5096	tx_q->tx_count_frames++;
5097
5098	if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)
5099	set_ic = true;
5100	else
5101	set_ic = false;
5102
5103	if (set_ic) {
5104	tx_q->tx_count_frames = 0;
5105	stmmac_set_tx_ic(priv, tx_desc);
5106	u64_stats_update_begin(syncp: &txq_stats->q_syncp);
5107	u64_stats_inc(p: &txq_stats->q.tx_set_ic_bit);
5108	u64_stats_update_end(syncp: &txq_stats->q_syncp);
5109	}
5110
5111	stmmac_enable_dma_transmission(priv, priv->ioaddr, queue);
5112
5113	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size);
5114	tx_q->cur_tx = entry;
5115
5116	return STMMAC_XDP_TX;
5117	}
5118
5119	static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv,
5120	int cpu)
5121	{
5122	int index = cpu;
5123
5124	if (unlikely(index < 0))
5125	index = 0;
5126
5127	while (index >= priv->plat->tx_queues_to_use)
5128	index -= priv->plat->tx_queues_to_use;
5129
5130	return index;
5131	}
5132
5133	static int stmmac_xdp_xmit_back(struct stmmac_priv *priv,
5134	struct xdp_buff *xdp)
5135	{
5136	bool zc = !!(xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL);
5137	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
5138	int cpu = smp_processor_id();
5139	struct netdev_queue *nq;
5140	int queue;
5141	int res;
5142
5143	if (unlikely(!xdpf))
5144	return STMMAC_XDP_CONSUMED;
5145
5146	queue = stmmac_xdp_get_tx_queue(priv, cpu);
5147	nq = netdev_get_tx_queue(dev: priv->dev, index: queue);
5148
5149	__netif_tx_lock(txq: nq, cpu);
5150	/* Avoids TX time-out as we are sharing with slow path */
5151	txq_trans_cond_update(txq: nq);
5152
5153	/* For zero copy XDP_TX action, dma_map is true */
5154	res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, dma_map: zc);
5155	if (res == STMMAC_XDP_TX) {
5156	stmmac_flush_tx_descriptors(priv, queue);
5157	} else if (res == STMMAC_XDP_CONSUMED && zc) {
5158	/* xdp has been freed by xdp_convert_buff_to_frame(),
5159	* no need to call xsk_buff_free() again, so return
5160	* STMMAC_XSK_CONSUMED.
5161	*/
5162	res = STMMAC_XSK_CONSUMED;
5163	xdp_return_frame(xdpf);
5164	}
5165
5166	__netif_tx_unlock(txq: nq);
5167
5168	return res;
5169	}
5170
5171	static int __stmmac_xdp_run_prog(struct stmmac_priv *priv,
5172	struct bpf_prog *prog,
5173	struct xdp_buff *xdp)
5174	{
5175	u32 act;
5176	int res;
5177
5178	act = bpf_prog_run_xdp(prog, xdp);
5179	switch (act) {
5180	case XDP_PASS:
5181	res = STMMAC_XDP_PASS;
5182	break;
5183	case XDP_TX:
5184	res = stmmac_xdp_xmit_back(priv, xdp);
5185	break;
5186	case XDP_REDIRECT:
5187	if (xdp_do_redirect(dev: priv->dev, xdp, prog) < 0)
5188	res = STMMAC_XDP_CONSUMED;
5189	else
5190	res = STMMAC_XDP_REDIRECT;
5191	break;
5192	default:
5193	bpf_warn_invalid_xdp_action(dev: priv->dev, prog, act);
5194	fallthrough;
5195	case XDP_ABORTED:
5196	trace_xdp_exception(dev: priv->dev, xdp: prog, act);
5197	fallthrough;
5198	case XDP_DROP:
5199	res = STMMAC_XDP_CONSUMED;
5200	break;
5201	}
5202
5203	return res;
5204	}
5205
5206	static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv,
5207	struct xdp_buff *xdp)
5208	{
5209	struct bpf_prog *prog;
5210	int res;
5211
5212	prog = READ_ONCE(priv->xdp_prog);
5213	if (!prog) {
5214	res = STMMAC_XDP_PASS;
5215	goto out;
5216	}
5217
5218	res = __stmmac_xdp_run_prog(priv, prog, xdp);
5219	out:
5220	return ERR_PTR(error: -res);
5221	}
5222
5223	static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv,
5224	int xdp_status)
5225	{
5226	int cpu = smp_processor_id();
5227	int queue;
5228
5229	queue = stmmac_xdp_get_tx_queue(priv, cpu);
5230
5231	if (xdp_status & STMMAC_XDP_TX)
5232	stmmac_tx_timer_arm(priv, queue);
5233
5234	if (xdp_status & STMMAC_XDP_REDIRECT)
5235	xdp_do_flush();
5236	}
5237
5238	static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch,
5239	struct xdp_buff *xdp)
5240	{
5241	unsigned int metasize = xdp->data - xdp->data_meta;
5242	unsigned int datasize = xdp->data_end - xdp->data;
5243	struct sk_buff *skb;
5244
5245	skb = napi_alloc_skb(napi: &ch->rxtx_napi,
5246	length: xdp->data_end - xdp->data_hard_start);
5247	if (unlikely(!skb))
5248	return NULL;
5249
5250	skb_reserve(skb, len: xdp->data - xdp->data_hard_start);
5251	memcpy(__skb_put(skb, datasize), xdp->data, datasize);
5252	if (metasize)
5253	skb_metadata_set(skb, meta_len: metasize);
5254
5255	return skb;
5256	}
5257
5258	static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue,
5259	struct dma_desc *p, struct dma_desc *np,
5260	struct xdp_buff *xdp)
5261	{
5262	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];
5263	struct stmmac_channel *ch = &priv->channel[queue];
5264	unsigned int len = xdp->data_end - xdp->data;
5265	enum pkt_hash_types hash_type;
5266	int coe = priv->hw->rx_csum;
5267	struct sk_buff *skb;
5268	u32 hash;
5269
5270	skb = stmmac_construct_skb_zc(ch, xdp);
5271	if (!skb) {
5272	priv->xstats.rx_dropped++;
5273	return;
5274	}
5275
5276	stmmac_get_rx_hwtstamp(priv, p, np, skb);
5277	if (priv->hw->hw_vlan_en)
5278	/* MAC level stripping. */
5279	stmmac_rx_hw_vlan(priv, priv->hw, p, skb);
5280	else
5281	/* Driver level stripping. */
5282	stmmac_rx_vlan(dev: priv->dev, skb);
5283	skb->protocol = eth_type_trans(skb, dev: priv->dev);
5284
5285	if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb))
5286	skb_checksum_none_assert(skb);
5287	else
5288	skb->ip_summed = CHECKSUM_UNNECESSARY;
5289
5290	if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
5291	skb_set_hash(skb, hash, type: hash_type);
5292
5293	skb_record_rx_queue(skb, rx_queue: queue);
5294	napi_gro_receive(napi: &ch->rxtx_napi, skb);
5295
5296	u64_stats_update_begin(syncp: &rxq_stats->napi_syncp);
5297	u64_stats_inc(p: &rxq_stats->napi.rx_pkt_n);
5298	u64_stats_add(p: &rxq_stats->napi.rx_bytes, val: len);
5299	u64_stats_update_end(syncp: &rxq_stats->napi_syncp);
5300	}
5301
5302	static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
5303	{
5304	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
5305	unsigned int entry = rx_q->dirty_rx;
5306	struct dma_desc *rx_desc = NULL;
5307	bool ret = true;
5308
5309	budget = min(budget, stmmac_rx_dirty(priv, queue));
5310
5311	while (budget-- > 0 && entry != rx_q->cur_rx) {
5312	struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
5313	dma_addr_t dma_addr;
5314	bool use_rx_wd;
5315
5316	if (!buf->xdp) {
5317	buf->xdp = xsk_buff_alloc(pool: rx_q->xsk_pool);
5318	if (!buf->xdp) {
5319	ret = false;
5320	break;
5321	}
5322	}
5323
5324	if (priv->extend_desc)
5325	rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry);
5326	else
5327	rx_desc = rx_q->dma_rx + entry;
5328
5329	dma_addr = xsk_buff_xdp_get_dma(xdp: buf->xdp);
5330	stmmac_set_desc_addr(priv, rx_desc, dma_addr);
5331	stmmac_set_desc_sec_addr(priv, rx_desc, 0, false);
5332	stmmac_refill_desc3(priv, rx_q, rx_desc);
5333
5334	rx_q->rx_count_frames++;
5335	rx_q->rx_count_frames += priv->rx_coal_frames[queue];
5336	if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])
5337	rx_q->rx_count_frames = 0;
5338
5339	use_rx_wd = !priv->rx_coal_frames[queue];
5340	use_rx_wd |= rx_q->rx_count_frames > 0;
5341	if (!priv->use_riwt)
5342	use_rx_wd = false;
5343
5344	dma_wmb();
5345	stmmac_set_rx_owner(priv, rx_desc, use_rx_wd);
5346
5347	entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size);
5348	}
5349
5350	if (rx_desc) {
5351	rx_q->dirty_rx = entry;
5352	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
5353	(rx_q->dirty_rx * sizeof(struct dma_desc));
5354	stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
5355	}
5356
5357	return ret;
5358	}
5359
5360	static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp)
5361	{
5362	/* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used
5363	* to represent incoming packet, whereas cb field in the same structure
5364	* is used to store driver specific info. Thus, struct stmmac_xdp_buff
5365	* is laid on top of xdp and cb fields of struct xdp_buff_xsk.
5366	*/
5367	return (struct stmmac_xdp_buff *)xdp;
5368	}
5369
5370	static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
5371	{
5372	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];
5373	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
5374	unsigned int count = 0, error = 0, len = 0;
5375	int dirty = stmmac_rx_dirty(priv, queue);
5376	unsigned int next_entry = rx_q->cur_rx;
5377	u32 rx_errors = 0, rx_dropped = 0;
5378	unsigned int desc_size;
5379	struct bpf_prog *prog;
5380	bool failure = false;
5381	int xdp_status = 0;
5382	int status = 0;
5383
5384	if (netif_msg_rx_status(priv)) {
5385	void *rx_head;
5386
5387	netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
5388	if (priv->extend_desc) {
5389	rx_head = (void *)rx_q->dma_erx;
5390	desc_size = sizeof(struct dma_extended_desc);
5391	} else {
5392	rx_head = (void *)rx_q->dma_rx;
5393	desc_size = sizeof(struct dma_desc);
5394	}
5395
5396	stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,
5397	rx_q->dma_rx_phy, desc_size);
5398	}
5399	while (count < limit) {
5400	struct stmmac_rx_buffer *buf;
5401	struct stmmac_xdp_buff *ctx;
5402	unsigned int buf1_len = 0;
5403	struct dma_desc *np, *p;
5404	int entry;
5405	int res;
5406
5407	if (!count && rx_q->state_saved) {
5408	error = rx_q->state.error;
5409	len = rx_q->state.len;
5410	} else {
5411	rx_q->state_saved = false;
5412	error = 0;
5413	len = 0;
5414	}
5415
5416	read_again:
5417	if (count >= limit)
5418	break;
5419
5420	buf1_len = 0;
5421	entry = next_entry;
5422	buf = &rx_q->buf_pool[entry];
5423
5424	if (dirty >= STMMAC_RX_FILL_BATCH) {
5425	failure = failure ||
5426	!stmmac_rx_refill_zc(priv, queue, budget: dirty);
5427	dirty = 0;
5428	}
5429
5430	if (priv->extend_desc)
5431	p = (struct dma_desc *)(rx_q->dma_erx + entry);
5432	else
5433	p = rx_q->dma_rx + entry;
5434
5435	/* read the status of the incoming frame */
5436	status = stmmac_rx_status(priv, &priv->xstats, p);
5437	/* check if managed by the DMA otherwise go ahead */
5438	if (unlikely(status & dma_own))
5439	break;
5440
5441	/* Prefetch the next RX descriptor */
5442	rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
5443	priv->dma_conf.dma_rx_size);
5444	next_entry = rx_q->cur_rx;
5445
5446	if (priv->extend_desc)
5447	np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
5448	else
5449	np = rx_q->dma_rx + next_entry;
5450
5451	prefetch(np);
5452
5453	/* Ensure a valid XSK buffer before proceed */
5454	if (!buf->xdp)
5455	break;
5456
5457	if (priv->extend_desc)
5458	stmmac_rx_extended_status(priv, &priv->xstats,
5459	rx_q->dma_erx + entry);
5460	if (unlikely(status == discard_frame)) {
5461	xsk_buff_free(xdp: buf->xdp);
5462	buf->xdp = NULL;
5463	dirty++;
5464	error = 1;
5465	if (!priv->hwts_rx_en)
5466	rx_errors++;
5467	}
5468
5469	if (unlikely(error && (status & rx_not_ls)))
5470	goto read_again;
5471	if (unlikely(error)) {
5472	count++;
5473	continue;
5474	}
5475
5476	/* XSK pool expects RX frame 1:1 mapped to XSK buffer */
5477	if (likely(status & rx_not_ls)) {
5478	xsk_buff_free(xdp: buf->xdp);
5479	buf->xdp = NULL;
5480	dirty++;
5481	count++;
5482	goto read_again;
5483	}
5484
5485	ctx = xsk_buff_to_stmmac_ctx(xdp: buf->xdp);
5486	ctx->priv = priv;
5487	ctx->desc = p;
5488	ctx->ndesc = np;
5489
5490	/* XDP ZC Frame only support primary buffers for now */
5491	buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
5492	len += buf1_len;
5493
5494	/* ACS is disabled; strip manually. */
5495	if (likely(!(status & rx_not_ls))) {
5496	buf1_len -= ETH_FCS_LEN;
5497	len -= ETH_FCS_LEN;
5498	}
5499
5500	/* RX buffer is good and fit into a XSK pool buffer */
5501	buf->xdp->data_end = buf->xdp->data + buf1_len;
5502	xsk_buff_dma_sync_for_cpu(xdp: buf->xdp);
5503
5504	prog = READ_ONCE(priv->xdp_prog);
5505	res = __stmmac_xdp_run_prog(priv, prog, xdp: buf->xdp);
5506
5507	switch (res) {
5508	case STMMAC_XDP_PASS:
5509	stmmac_dispatch_skb_zc(priv, queue, p, np, xdp: buf->xdp);
5510	xsk_buff_free(xdp: buf->xdp);
5511	break;
5512	case STMMAC_XDP_CONSUMED:
5513	xsk_buff_free(xdp: buf->xdp);
5514	fallthrough;
5515	case STMMAC_XSK_CONSUMED:
5516	rx_dropped++;
5517	break;
5518	case STMMAC_XDP_TX:
5519	case STMMAC_XDP_REDIRECT:
5520	xdp_status |= res;
5521	break;
5522	}
5523
5524	buf->xdp = NULL;
5525	dirty++;
5526	count++;
5527	}
5528
5529	if (status & rx_not_ls) {
5530	rx_q->state_saved = true;
5531	rx_q->state.error = error;
5532	rx_q->state.len = len;
5533	}
5534
5535	stmmac_finalize_xdp_rx(priv, xdp_status);
5536
5537	u64_stats_update_begin(syncp: &rxq_stats->napi_syncp);
5538	u64_stats_add(p: &rxq_stats->napi.rx_pkt_n, val: count);
5539	u64_stats_update_end(syncp: &rxq_stats->napi_syncp);
5540
5541	priv->xstats.rx_dropped += rx_dropped;
5542	priv->xstats.rx_errors += rx_errors;
5543
5544	if (xsk_uses_need_wakeup(pool: rx_q->xsk_pool)) {
5545	if (failure || stmmac_rx_dirty(priv, queue) > 0)
5546	xsk_set_rx_need_wakeup(pool: rx_q->xsk_pool);
5547	else
5548	xsk_clear_rx_need_wakeup(pool: rx_q->xsk_pool);
5549
5550	return (int)count;
5551	}
5552
5553	return failure ? limit : (int)count;
5554	}
5555
5556	/**
5557	* stmmac_rx - manage the receive process
5558	* @priv: driver private structure
5559	* @limit: napi bugget
5560	* @queue: RX queue index.
5561	* Description : this the function called by the napi poll method.
5562	* It gets all the frames inside the ring.
5563	*/
5564	static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
5565	{
5566	u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0;
5567	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue];
5568	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
5569	struct stmmac_channel *ch = &priv->channel[queue];
5570	unsigned int count = 0, error = 0, len = 0;
5571	int status = 0, coe = priv->hw->rx_csum;
5572	unsigned int next_entry = rx_q->cur_rx;
5573	enum dma_data_direction dma_dir;
5574	unsigned int desc_size;
5575	struct sk_buff *skb = NULL;
5576	struct stmmac_xdp_buff ctx;
5577	int xdp_status = 0;
5578	int bufsz;
5579
5580	dma_dir = page_pool_get_dma_dir(pool: rx_q->page_pool);
5581	bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
5582	limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit);
5583
5584	if (netif_msg_rx_status(priv)) {
5585	void *rx_head;
5586
5587	netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
5588	if (priv->extend_desc) {
5589	rx_head = (void *)rx_q->dma_erx;
5590	desc_size = sizeof(struct dma_extended_desc);
5591	} else {
5592	rx_head = (void *)rx_q->dma_rx;
5593	desc_size = sizeof(struct dma_desc);
5594	}
5595
5596	stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true,
5597	rx_q->dma_rx_phy, desc_size);
5598	}
5599	while (count < limit) {
5600	unsigned int buf1_len = 0, buf2_len = 0;
5601	enum pkt_hash_types hash_type;
5602	struct stmmac_rx_buffer *buf;
5603	struct dma_desc *np, *p;
5604	int entry;
5605	u32 hash;
5606
5607	if (!count && rx_q->state_saved) {
5608	skb = rx_q->state.skb;
5609	error = rx_q->state.error;
5610	len = rx_q->state.len;
5611	} else {
5612	rx_q->state_saved = false;
5613	skb = NULL;
5614	error = 0;
5615	len = 0;
5616	}
5617
5618	read_again:
5619	if (count >= limit)
5620	break;
5621
5622	buf1_len = 0;
5623	buf2_len = 0;
5624	entry = next_entry;
5625	buf = &rx_q->buf_pool[entry];
5626
5627	if (priv->extend_desc)
5628	p = (struct dma_desc *)(rx_q->dma_erx + entry);
5629	else
5630	p = rx_q->dma_rx + entry;
5631
5632	/* read the status of the incoming frame */
5633	status = stmmac_rx_status(priv, &priv->xstats, p);
5634	/* check if managed by the DMA otherwise go ahead */
5635	if (unlikely(status & dma_own))
5636	break;
5637
5638	rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
5639	priv->dma_conf.dma_rx_size);
5640	next_entry = rx_q->cur_rx;
5641
5642	if (priv->extend_desc)
5643	np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
5644	else
5645	np = rx_q->dma_rx + next_entry;
5646
5647	prefetch(np);
5648
5649	if (priv->extend_desc)
5650	stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry);
5651	if (unlikely(status == discard_frame)) {
5652	page_pool_put_page(pool: rx_q->page_pool, page: buf->page, dma_sync_size: 0, allow_direct: true);
5653	buf->page = NULL;
5654	error = 1;
5655	if (!priv->hwts_rx_en)
5656	rx_errors++;
5657	}
5658
5659	if (unlikely(error && (status & rx_not_ls)))
5660	goto read_again;
5661	if (unlikely(error)) {
5662	dev_kfree_skb(skb);
5663	skb = NULL;
5664	count++;
5665	continue;
5666	}
5667
5668	/* Buffer is good. Go on. */
5669
5670	buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
5671	len += buf1_len;
5672	buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
5673	len += buf2_len;
5674
5675	/* ACS is disabled; strip manually. */
5676	if (likely(!(status & rx_not_ls))) {
5677	if (buf2_len) {
5678	buf2_len -= ETH_FCS_LEN;
5679	len -= ETH_FCS_LEN;
5680	} else if (buf1_len) {
5681	buf1_len -= ETH_FCS_LEN;
5682	len -= ETH_FCS_LEN;
5683	}
5684	}
5685
5686	if (!skb) {
5687	unsigned int pre_len, sync_len;
5688
5689	dma_sync_single_for_cpu(dev: priv->device, addr: buf->addr,
5690	size: buf1_len, dir: dma_dir);
5691	net_prefetch(page_address(buf->page) +
5692	buf->page_offset);
5693
5694	xdp_init_buff(xdp: &ctx.xdp, frame_sz: bufsz, rxq: &rx_q->xdp_rxq);
5695	xdp_prepare_buff(xdp: &ctx.xdp, page_address(buf->page),
5696	headroom: buf->page_offset, data_len: buf1_len, meta_valid: true);
5697
5698	pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start -
5699	buf->page_offset;
5700
5701	ctx.priv = priv;
5702	ctx.desc = p;
5703	ctx.ndesc = np;
5704
5705	skb = stmmac_xdp_run_prog(priv, xdp: &ctx.xdp);
5706	/* Due xdp_adjust_tail: DMA sync for_device
5707	* cover max len CPU touch
5708	*/
5709	sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start -
5710	buf->page_offset;
5711	sync_len = max(sync_len, pre_len);
5712
5713	/* For Not XDP_PASS verdict */
5714	if (IS_ERR(ptr: skb)) {
5715	unsigned int xdp_res = -PTR_ERR(ptr: skb);
5716
5717	if (xdp_res & STMMAC_XDP_CONSUMED) {
5718	page_pool_put_page(pool: rx_q->page_pool,
5719	page: virt_to_head_page(x: ctx.xdp.data),
5720	dma_sync_size: sync_len, allow_direct: true);
5721	buf->page = NULL;
5722	rx_dropped++;
5723
5724	/* Clear skb as it was set as
5725	* status by XDP program.
5726	*/
5727	skb = NULL;
5728
5729	if (unlikely((status & rx_not_ls)))
5730	goto read_again;
5731
5732	count++;
5733	continue;
5734	} else if (xdp_res & (STMMAC_XDP_TX |
5735	STMMAC_XDP_REDIRECT)) {
5736	xdp_status |= xdp_res;
5737	buf->page = NULL;
5738	skb = NULL;
5739	count++;
5740	continue;
5741	}
5742	}
5743	}
5744
5745	if (!skb) {
5746	unsigned int head_pad_len;
5747
5748	/* XDP program may expand or reduce tail */
5749	buf1_len = ctx.xdp.data_end - ctx.xdp.data;
5750
5751	skb = napi_build_skb(page_address(buf->page),
5752	frag_size: rx_q->napi_skb_frag_size);
5753	if (!skb) {
5754	page_pool_recycle_direct(pool: rx_q->page_pool,
5755	page: buf->page);
5756	rx_dropped++;
5757	count++;
5758	goto drain_data;
5759	}
5760
5761	/* XDP program may adjust header */
5762	head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start;
5763	skb_reserve(skb, len: head_pad_len);
5764	skb_put(skb, len: buf1_len);
5765	skb_mark_for_recycle(skb);
5766	buf->page = NULL;
5767	} else if (buf1_len) {
5768	dma_sync_single_for_cpu(dev: priv->device, addr: buf->addr,
5769	size: buf1_len, dir: dma_dir);
5770	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
5771	page: buf->page, off: buf->page_offset, size: buf1_len,
5772	truesize: priv->dma_conf.dma_buf_sz);
5773	buf->page = NULL;
5774	}
5775
5776	if (buf2_len) {
5777	dma_sync_single_for_cpu(dev: priv->device, addr: buf->sec_addr,
5778	size: buf2_len, dir: dma_dir);
5779	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
5780	page: buf->sec_page, off: 0, size: buf2_len,
5781	truesize: priv->dma_conf.dma_buf_sz);
5782	buf->sec_page = NULL;
5783	}
5784
5785	drain_data:
5786	if (likely(status & rx_not_ls))
5787	goto read_again;
5788	if (!skb)
5789	continue;
5790
5791	/* Got entire packet into SKB. Finish it. */
5792
5793	stmmac_get_rx_hwtstamp(priv, p, np, skb);
5794
5795	if (priv->hw->hw_vlan_en)
5796	/* MAC level stripping. */
5797	stmmac_rx_hw_vlan(priv, priv->hw, p, skb);
5798	else
5799	/* Driver level stripping. */
5800	stmmac_rx_vlan(dev: priv->dev, skb);
5801
5802	skb->protocol = eth_type_trans(skb, dev: priv->dev);
5803
5804	if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) ||
5805	(status & csum_none))
5806	skb_checksum_none_assert(skb);
5807	else
5808	skb->ip_summed = CHECKSUM_UNNECESSARY;
5809
5810	if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
5811	skb_set_hash(skb, hash, type: hash_type);
5812
5813	skb_record_rx_queue(skb, rx_queue: queue);
5814	napi_gro_receive(napi: &ch->rx_napi, skb);
5815	skb = NULL;
5816
5817	rx_packets++;
5818	rx_bytes += len;
5819	count++;
5820	}
5821
5822	if (status & rx_not_ls || skb) {
5823	rx_q->state_saved = true;
5824	rx_q->state.skb = skb;
5825	rx_q->state.error = error;
5826	rx_q->state.len = len;
5827	}
5828
5829	stmmac_finalize_xdp_rx(priv, xdp_status);
5830
5831	stmmac_rx_refill(priv, queue);
5832
5833	u64_stats_update_begin(syncp: &rxq_stats->napi_syncp);
5834	u64_stats_add(p: &rxq_stats->napi.rx_packets, val: rx_packets);
5835	u64_stats_add(p: &rxq_stats->napi.rx_bytes, val: rx_bytes);
5836	u64_stats_add(p: &rxq_stats->napi.rx_pkt_n, val: count);
5837	u64_stats_update_end(syncp: &rxq_stats->napi_syncp);
5838
5839	priv->xstats.rx_dropped += rx_dropped;
5840	priv->xstats.rx_errors += rx_errors;
5841
5842	return count;
5843	}
5844
5845	static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
5846	{
5847	struct stmmac_channel *ch =
5848	container_of(napi, struct stmmac_channel, rx_napi);
5849	struct stmmac_priv *priv = ch->priv_data;
5850	struct stmmac_rxq_stats *rxq_stats;
5851	u32 chan = ch->index;
5852	int work_done;
5853
5854	rxq_stats = &priv->xstats.rxq_stats[chan];
5855	u64_stats_update_begin(syncp: &rxq_stats->napi_syncp);
5856	u64_stats_inc(p: &rxq_stats->napi.poll);
5857	u64_stats_update_end(syncp: &rxq_stats->napi_syncp);
5858
5859	work_done = stmmac_rx(priv, limit: budget, queue: chan);
5860	if (work_done < budget && napi_complete_done(n: napi, work_done)) {
5861	unsigned long flags;
5862
5863	spin_lock_irqsave(&ch->lock, flags);
5864	stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
5865	spin_unlock_irqrestore(lock: &ch->lock, flags);
5866	}
5867
5868	return work_done;
5869	}
5870
5871	static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
5872	{
5873	struct stmmac_channel *ch =
5874	container_of(napi, struct stmmac_channel, tx_napi);
5875	struct stmmac_priv *priv = ch->priv_data;
5876	struct stmmac_txq_stats *txq_stats;
5877	bool pending_packets = false;
5878	u32 chan = ch->index;
5879	int work_done;
5880
5881	txq_stats = &priv->xstats.txq_stats[chan];
5882	u64_stats_update_begin(syncp: &txq_stats->napi_syncp);
5883	u64_stats_inc(p: &txq_stats->napi.poll);
5884	u64_stats_update_end(syncp: &txq_stats->napi_syncp);
5885
5886	work_done = stmmac_tx_clean(priv, budget, queue: chan, pending_packets: &pending_packets);
5887	work_done = min(work_done, budget);
5888
5889	if (work_done < budget && napi_complete_done(n: napi, work_done)) {
5890	unsigned long flags;
5891
5892	spin_lock_irqsave(&ch->lock, flags);
5893	stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
5894	spin_unlock_irqrestore(lock: &ch->lock, flags);
5895	}
5896
5897	/* TX still have packet to handle, check if we need to arm tx timer */
5898	if (pending_packets)
5899	stmmac_tx_timer_arm(priv, queue: chan);
5900
5901	return work_done;
5902	}
5903
5904	static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget)
5905	{
5906	struct stmmac_channel *ch =
5907	container_of(napi, struct stmmac_channel, rxtx_napi);
5908	struct stmmac_priv *priv = ch->priv_data;
5909	bool tx_pending_packets = false;
5910	int rx_done, tx_done, rxtx_done;
5911	struct stmmac_rxq_stats *rxq_stats;
5912	struct stmmac_txq_stats *txq_stats;
5913	u32 chan = ch->index;
5914
5915	rxq_stats = &priv->xstats.rxq_stats[chan];
5916	u64_stats_update_begin(syncp: &rxq_stats->napi_syncp);
5917	u64_stats_inc(p: &rxq_stats->napi.poll);
5918	u64_stats_update_end(syncp: &rxq_stats->napi_syncp);
5919
5920	txq_stats = &priv->xstats.txq_stats[chan];
5921	u64_stats_update_begin(syncp: &txq_stats->napi_syncp);
5922	u64_stats_inc(p: &txq_stats->napi.poll);
5923	u64_stats_update_end(syncp: &txq_stats->napi_syncp);
5924
5925	tx_done = stmmac_tx_clean(priv, budget, queue: chan, pending_packets: &tx_pending_packets);
5926	tx_done = min(tx_done, budget);
5927
5928	rx_done = stmmac_rx_zc(priv, limit: budget, queue: chan);
5929
5930	rxtx_done = max(tx_done, rx_done);
5931
5932	/* If either TX or RX work is not complete, return budget
5933	* and keep pooling
5934	*/
5935	if (rxtx_done >= budget)
5936	return budget;
5937
5938	/* all work done, exit the polling mode */
5939	if (napi_complete_done(n: napi, work_done: rxtx_done)) {
5940	unsigned long flags;
5941
5942	spin_lock_irqsave(&ch->lock, flags);
5943	/* Both RX and TX work done are compelte,
5944	* so enable both RX & TX IRQs.
5945	*/
5946	stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
5947	spin_unlock_irqrestore(lock: &ch->lock, flags);
5948	}
5949
5950	/* TX still have packet to handle, check if we need to arm tx timer */
5951	if (tx_pending_packets)
5952	stmmac_tx_timer_arm(priv, queue: chan);
5953
5954	return min(rxtx_done, budget - 1);
5955	}
5956
5957	/**
5958	* stmmac_tx_timeout
5959	* @dev : Pointer to net device structure
5960	* @txqueue: the index of the hanging transmit queue
5961	* Description: this function is called when a packet transmission fails to
5962	* complete within a reasonable time. The driver will mark the error in the
5963	* netdev structure and arrange for the device to be reset to a sane state
5964	* in order to transmit a new packet.
5965	*/
5966	static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue)
5967	{
5968	struct stmmac_priv *priv = netdev_priv(dev);
5969
5970	stmmac_global_err(priv);
5971	}
5972
5973	/**
5974	* stmmac_set_rx_mode - entry point for multicast addressing
5975	* @dev : pointer to the device structure
5976	* Description:
5977	* This function is a driver entry point which gets called by the kernel
5978	* whenever multicast addresses must be enabled/disabled.
5979	* Return value:
5980	* void.
5981	*
5982	* FIXME: This may need RXC to be running, but it may be called with BH
5983	* disabled, which means we can't call phylink_rx_clk_stop*().
5984	*/
5985	static void stmmac_set_rx_mode(struct net_device *dev)
5986	{
5987	struct stmmac_priv *priv = netdev_priv(dev);
5988
5989	stmmac_set_filter(priv, priv->hw, dev);
5990	}
5991
5992	/**
5993	* stmmac_change_mtu - entry point to change MTU size for the device.
5994	* @dev : device pointer.
5995	* @new_mtu : the new MTU size for the device.
5996	* Description: the Maximum Transfer Unit (MTU) is used by the network layer
5997	* to drive packet transmission. Ethernet has an MTU of 1500 octets
5998	* (ETH_DATA_LEN). This value can be changed with ifconfig.
5999	* Return value:
6000	* 0 on success and an appropriate (-)ve integer as defined in errno.h
6001	* file on failure.
6002	*/
6003	static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
6004	{
6005	struct stmmac_priv *priv = netdev_priv(dev);
6006	int txfifosz = priv->plat->tx_fifo_size;
6007	struct stmmac_dma_conf *dma_conf;
6008	const int mtu = new_mtu;
6009	int ret;
6010
6011	if (txfifosz == 0)
6012	txfifosz = priv->dma_cap.tx_fifo_size;
6013
6014	txfifosz /= priv->plat->tx_queues_to_use;
6015
6016	if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) {
6017	netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n");
6018	return -EINVAL;
6019	}
6020
6021	new_mtu = STMMAC_ALIGN(new_mtu);
6022
6023	/* If condition true, FIFO is too small or MTU too large */
6024	if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB))
6025	return -EINVAL;
6026
6027	if (netif_running(dev)) {
6028	netdev_dbg(priv->dev, "restarting interface to change its MTU\n");
6029	/* Try to allocate the new DMA conf with the new mtu */
6030	dma_conf = stmmac_setup_dma_desc(priv, mtu);
6031	if (IS_ERR(ptr: dma_conf)) {
6032	netdev_err(dev: priv->dev, format: "failed allocating new dma conf for new MTU %d\n",
6033	mtu);
6034	return PTR_ERR(ptr: dma_conf);
6035	}
6036
6037	__stmmac_release(dev);
6038
6039	ret = __stmmac_open(dev, dma_conf);
6040	if (ret) {
6041	free_dma_desc_resources(priv, dma_conf);
6042	kfree(objp: dma_conf);
6043	netdev_err(dev: priv->dev, format: "failed reopening the interface after MTU change\n");
6044	return ret;
6045	}
6046
6047	kfree(objp: dma_conf);
6048
6049	stmmac_set_rx_mode(dev);
6050	}
6051
6052	WRITE_ONCE(dev->mtu, mtu);
6053	netdev_update_features(dev);
6054
6055	return 0;
6056	}
6057
6058	static netdev_features_t stmmac_fix_features(struct net_device *dev,
6059	netdev_features_t features)
6060	{
6061	struct stmmac_priv *priv = netdev_priv(dev);
6062
6063	if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
6064	features &= ~NETIF_F_RXCSUM;
6065
6066	if (!priv->plat->tx_coe)
6067	features &= ~NETIF_F_CSUM_MASK;
6068
6069	/* Some GMAC devices have a bugged Jumbo frame support that
6070	* needs to have the Tx COE disabled for oversized frames
6071	* (due to limited buffer sizes). In this case we disable
6072	* the TX csum insertion in the TDES and not use SF.
6073	*/
6074	if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
6075	features &= ~NETIF_F_CSUM_MASK;
6076
6077	/* Disable tso if asked by ethtool */
6078	if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) {
6079	if (features & NETIF_F_TSO)
6080	priv->tso = true;
6081	else
6082	priv->tso = false;
6083	}
6084
6085	return features;
6086	}
6087
6088	static int stmmac_set_features(struct net_device *netdev,
6089	netdev_features_t features)
6090	{
6091	struct stmmac_priv *priv = netdev_priv(dev: netdev);
6092
6093	/* Keep the COE Type in case of csum is supporting */
6094	if (features & NETIF_F_RXCSUM)
6095	priv->hw->rx_csum = priv->plat->rx_coe;
6096	else
6097	priv->hw->rx_csum = 0;
6098	/* No check needed because rx_coe has been set before and it will be
6099	* fixed in case of issue.
6100	*/
6101	stmmac_rx_ipc(priv, priv->hw);
6102
6103	if (priv->sph_capable) {
6104	bool sph_en = (priv->hw->rx_csum > 0) && priv->sph_active;
6105	u32 chan;
6106
6107	for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
6108	stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
6109	}
6110
6111	if (features & NETIF_F_HW_VLAN_CTAG_RX)
6112	priv->hw->hw_vlan_en = true;
6113	else
6114	priv->hw->hw_vlan_en = false;
6115
6116	phylink_rx_clk_stop_block(priv->phylink);
6117	stmmac_set_hw_vlan_mode(priv, priv->hw);
6118	phylink_rx_clk_stop_unblock(priv->phylink);
6119
6120	return 0;
6121	}
6122
6123	static void stmmac_common_interrupt(struct stmmac_priv *priv)
6124	{
6125	u32 rx_cnt = priv->plat->rx_queues_to_use;
6126	u32 tx_cnt = priv->plat->tx_queues_to_use;
6127	u32 queues_count;
6128	u32 queue;
6129	bool xmac;
6130
6131	xmac = dwmac_is_xmac(core_type: priv->plat->core_type);
6132	queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
6133
6134	if (priv->irq_wake)
6135	pm_wakeup_event(dev: priv->device, msec: 0);
6136
6137	if (priv->dma_cap.estsel)
6138	stmmac_est_irq_status(priv, priv, priv->dev,
6139	&priv->xstats, tx_cnt);
6140
6141	if (stmmac_fpe_supported(priv))
6142	stmmac_fpe_irq_status(priv);
6143
6144	/* To handle GMAC own interrupts */
6145	if (priv->plat->core_type == DWMAC_CORE_GMAC || xmac) {
6146	int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
6147
6148	if (unlikely(status)) {
6149	/* For LPI we need to save the tx status */
6150	if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
6151	priv->tx_path_in_lpi_mode = true;
6152	if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
6153	priv->tx_path_in_lpi_mode = false;
6154	}
6155
6156	for (queue = 0; queue < queues_count; queue++)
6157	stmmac_host_mtl_irq_status(priv, priv->hw, queue);
6158
6159	stmmac_timestamp_interrupt(priv, priv);
6160	}
6161	}
6162
6163	/**
6164	* stmmac_interrupt - main ISR
6165	* @irq: interrupt number.
6166	* @dev_id: to pass the net device pointer.
6167	* Description: this is the main driver interrupt service routine.
6168	* It can call:
6169	* o DMA service routine (to manage incoming frame reception and transmission
6170	* status)
6171	* o Core interrupts to manage: remote wake-up, management counter, LPI
6172	* interrupts.
6173	*/
6174	static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
6175	{
6176	struct net_device *dev = (struct net_device *)dev_id;
6177	struct stmmac_priv *priv = netdev_priv(dev);
6178
6179	/* Check if adapter is up */
6180	if (test_bit(STMMAC_DOWN, &priv->state))
6181	return IRQ_HANDLED;
6182
6183	/* Check ASP error if it isn't delivered via an individual IRQ */
6184	if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv))
6185	return IRQ_HANDLED;
6186
6187	/* To handle Common interrupts */
6188	stmmac_common_interrupt(priv);
6189
6190	/* To handle DMA interrupts */
6191	stmmac_dma_interrupt(priv);
6192
6193	return IRQ_HANDLED;
6194	}
6195
6196	static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id)
6197	{
6198	struct net_device *dev = (struct net_device *)dev_id;
6199	struct stmmac_priv *priv = netdev_priv(dev);
6200
6201	/* Check if adapter is up */
6202	if (test_bit(STMMAC_DOWN, &priv->state))
6203	return IRQ_HANDLED;
6204
6205	/* To handle Common interrupts */
6206	stmmac_common_interrupt(priv);
6207
6208	return IRQ_HANDLED;
6209	}
6210
6211	static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id)
6212	{
6213	struct net_device *dev = (struct net_device *)dev_id;
6214	struct stmmac_priv *priv = netdev_priv(dev);
6215
6216	/* Check if adapter is up */
6217	if (test_bit(STMMAC_DOWN, &priv->state))
6218	return IRQ_HANDLED;
6219
6220	/* Check if a fatal error happened */
6221	stmmac_safety_feat_interrupt(priv);
6222
6223	return IRQ_HANDLED;
6224	}
6225
6226	static irqreturn_t stmmac_msi_intr_tx(int irq, void *data)
6227	{
6228	struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data;
6229	struct stmmac_dma_conf *dma_conf;
6230	int chan = tx_q->queue_index;
6231	struct stmmac_priv *priv;
6232	int status;
6233
6234	dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]);
6235	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);
6236
6237	/* Check if adapter is up */
6238	if (test_bit(STMMAC_DOWN, &priv->state))
6239	return IRQ_HANDLED;
6240
6241	status = stmmac_napi_check(priv, chan, dir: DMA_DIR_TX);
6242
6243	if (unlikely(status & tx_hard_error_bump_tc)) {
6244	/* Try to bump up the dma threshold on this failure */
6245	stmmac_bump_dma_threshold(priv, chan);
6246	} else if (unlikely(status == tx_hard_error)) {
6247	stmmac_tx_err(priv, chan);
6248	}
6249
6250	return IRQ_HANDLED;
6251	}
6252
6253	static irqreturn_t stmmac_msi_intr_rx(int irq, void *data)
6254	{
6255	struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data;
6256	struct stmmac_dma_conf *dma_conf;
6257	int chan = rx_q->queue_index;
6258	struct stmmac_priv *priv;
6259
6260	dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]);
6261	priv = container_of(dma_conf, struct stmmac_priv, dma_conf);
6262
6263	/* Check if adapter is up */
6264	if (test_bit(STMMAC_DOWN, &priv->state))
6265	return IRQ_HANDLED;
6266
6267	stmmac_napi_check(priv, chan, dir: DMA_DIR_RX);
6268
6269	return IRQ_HANDLED;
6270	}
6271
6272	/**
6273	* stmmac_ioctl - Entry point for the Ioctl
6274	* @dev: Device pointer.
6275	* @rq: An IOCTL specefic structure, that can contain a pointer to
6276	* a proprietary structure used to pass information to the driver.
6277	* @cmd: IOCTL command
6278	* Description:
6279	* Currently it supports the phy_mii_ioctl(...) and HW time stamping.
6280	*/
6281	static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
6282	{
6283	struct stmmac_priv *priv = netdev_priv (dev);
6284	int ret = -EOPNOTSUPP;
6285
6286	if (!netif_running(dev))
6287	return -EINVAL;
6288
6289	switch (cmd) {
6290	case SIOCGMIIPHY:
6291	case SIOCGMIIREG:
6292	case SIOCSMIIREG:
6293	ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
6294	break;
6295	default:
6296	break;
6297	}
6298
6299	return ret;
6300	}
6301
6302	static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
6303	void *cb_priv)
6304	{
6305	struct stmmac_priv *priv = cb_priv;
6306	int ret = -EOPNOTSUPP;
6307
6308	if (!tc_cls_can_offload_and_chain0(dev: priv->dev, common: type_data))
6309	return ret;
6310
6311	__stmmac_disable_all_queues(priv);
6312
6313	switch (type) {
6314	case TC_SETUP_CLSU32:
6315	ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
6316	break;
6317	case TC_SETUP_CLSFLOWER:
6318	ret = stmmac_tc_setup_cls(priv, priv, type_data);
6319	break;
6320	default:
6321	break;
6322	}
6323
6324	stmmac_enable_all_queues(priv);
6325	return ret;
6326	}
6327
6328	static LIST_HEAD(stmmac_block_cb_list);
6329
6330	static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
6331	void *type_data)
6332	{
6333	struct stmmac_priv *priv = netdev_priv(dev: ndev);
6334
6335	switch (type) {
6336	case TC_QUERY_CAPS:
6337	return stmmac_tc_query_caps(priv, priv, type_data);
6338	case TC_SETUP_QDISC_MQPRIO:
6339	return stmmac_tc_setup_mqprio(priv, priv, type_data);
6340	case TC_SETUP_BLOCK:
6341	return flow_block_cb_setup_simple(f: type_data,
6342	driver_list: &stmmac_block_cb_list,
6343	cb: stmmac_setup_tc_block_cb,
6344	cb_ident: priv, cb_priv: priv, ingress_only: true);
6345	case TC_SETUP_QDISC_CBS:
6346	return stmmac_tc_setup_cbs(priv, priv, type_data);
6347	case TC_SETUP_QDISC_TAPRIO:
6348	return stmmac_tc_setup_taprio(priv, priv, type_data);
6349	case TC_SETUP_QDISC_ETF:
6350	return stmmac_tc_setup_etf(priv, priv, type_data);
6351	default:
6352	return -EOPNOTSUPP;
6353	}
6354	}
6355
6356	static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
6357	struct net_device *sb_dev)
6358	{
6359	int gso = skb_shinfo(skb)->gso_type;
6360
6361	if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
6362	/*
6363	* There is no way to determine the number of TSO/USO
6364	* capable Queues. Let's use always the Queue 0
6365	* because if TSO/USO is supported then at least this
6366	* one will be capable.
6367	*/
6368	return 0;
6369	}
6370
6371	return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
6372	}
6373
6374	static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
6375	{
6376	struct stmmac_priv *priv = netdev_priv(dev: ndev);
6377	int ret = 0;
6378
6379	ret = pm_runtime_resume_and_get(dev: priv->device);
6380	if (ret < 0)
6381	return ret;
6382
6383	ret = eth_mac_addr(dev: ndev, p: addr);
6384	if (ret)
6385	goto set_mac_error;
6386
6387	phylink_rx_clk_stop_block(priv->phylink);
6388	stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
6389	phylink_rx_clk_stop_unblock(priv->phylink);
6390
6391	set_mac_error:
6392	pm_runtime_put(dev: priv->device);
6393
6394	return ret;
6395	}
6396
6397	#ifdef CONFIG_DEBUG_FS
6398	static struct dentry *stmmac_fs_dir;
6399
6400	static void sysfs_display_ring(void *head, int size, int extend_desc,
6401	struct seq_file *seq, dma_addr_t dma_phy_addr)
6402	{
6403	struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
6404	struct dma_desc *p = (struct dma_desc *)head;
6405	unsigned int desc_size;
6406	dma_addr_t dma_addr;
6407	int i;
6408
6409	desc_size = extend_desc ? sizeof(*ep) : sizeof(*p);
6410	for (i = 0; i < size; i++) {
6411	dma_addr = dma_phy_addr + i * desc_size;
6412	seq_printf(m: seq, fmt: "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
6413	i, &dma_addr,
6414	le32_to_cpu(p->des0), le32_to_cpu(p->des1),
6415	le32_to_cpu(p->des2), le32_to_cpu(p->des3));
6416	if (extend_desc)
6417	p = &(++ep)->basic;
6418	else
6419	p++;
6420	}
6421	}
6422
6423	static int stmmac_rings_status_show(struct seq_file *seq, void *v)
6424	{
6425	struct net_device *dev = seq->private;
6426	struct stmmac_priv *priv = netdev_priv(dev);
6427	u32 rx_count = priv->plat->rx_queues_to_use;
6428	u32 tx_count = priv->plat->tx_queues_to_use;
6429	u32 queue;
6430
6431	if ((dev->flags & IFF_UP) == 0)
6432	return 0;
6433
6434	for (queue = 0; queue < rx_count; queue++) {
6435	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
6436
6437	seq_printf(m: seq, fmt: "RX Queue %d:\n", queue);
6438
6439	if (priv->extend_desc) {
6440	seq_printf(m: seq, fmt: "Extended descriptor ring:\n");
6441	sysfs_display_ring(head: (void *)rx_q->dma_erx,
6442	size: priv->dma_conf.dma_rx_size, extend_desc: 1, seq, dma_phy_addr: rx_q->dma_rx_phy);
6443	} else {
6444	seq_printf(m: seq, fmt: "Descriptor ring:\n");
6445	sysfs_display_ring(head: (void *)rx_q->dma_rx,
6446	size: priv->dma_conf.dma_rx_size, extend_desc: 0, seq, dma_phy_addr: rx_q->dma_rx_phy);
6447	}
6448	}
6449
6450	for (queue = 0; queue < tx_count; queue++) {
6451	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
6452
6453	seq_printf(m: seq, fmt: "TX Queue %d:\n", queue);
6454
6455	if (priv->extend_desc) {
6456	seq_printf(m: seq, fmt: "Extended descriptor ring:\n");
6457	sysfs_display_ring(head: (void *)tx_q->dma_etx,
6458	size: priv->dma_conf.dma_tx_size, extend_desc: 1, seq, dma_phy_addr: tx_q->dma_tx_phy);
6459	} else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) {
6460	seq_printf(m: seq, fmt: "Descriptor ring:\n");
6461	sysfs_display_ring(head: (void *)tx_q->dma_tx,
6462	size: priv->dma_conf.dma_tx_size, extend_desc: 0, seq, dma_phy_addr: tx_q->dma_tx_phy);
6463	}
6464	}
6465
6466	return 0;
6467	}
6468	DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
6469
6470	static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
6471	{
6472	static const char * const dwxgmac_timestamp_source[] = {
6473	"None",
6474	"Internal",
6475	"External",
6476	"Both",
6477	};
6478	static const char * const dwxgmac_safety_feature_desc[] = {
6479	"No",
6480	"All Safety Features with ECC and Parity",
6481	"All Safety Features without ECC or Parity",
6482	"All Safety Features with Parity Only",
6483	"ECC Only",
6484	"UNDEFINED",
6485	"UNDEFINED",
6486	"UNDEFINED",
6487	};
6488	struct net_device *dev = seq->private;
6489	struct stmmac_priv *priv = netdev_priv(dev);
6490
6491	if (!priv->hw_cap_support) {
6492	seq_printf(m: seq, fmt: "DMA HW features not supported\n");
6493	return 0;
6494	}
6495
6496	seq_printf(m: seq, fmt: "==============================\n");
6497	seq_printf(m: seq, fmt: "\tDMA HW features\n");
6498	seq_printf(m: seq, fmt: "==============================\n");
6499
6500	seq_printf(m: seq, fmt: "\t10/100 Mbps: %s\n",
6501	(priv->dma_cap.mbps_10_100) ? "Y" : "N");
6502	seq_printf(m: seq, fmt: "\t1000 Mbps: %s\n",
6503	(priv->dma_cap.mbps_1000) ? "Y" : "N");
6504	seq_printf(m: seq, fmt: "\tHalf duplex: %s\n",
6505	(priv->dma_cap.half_duplex) ? "Y" : "N");
6506	if (priv->plat->core_type == DWMAC_CORE_XGMAC) {
6507	seq_printf(m: seq,
6508	fmt: "\tNumber of Additional MAC address registers: %d\n",
6509	priv->dma_cap.multi_addr);
6510	} else {
6511	seq_printf(m: seq, fmt: "\tHash Filter: %s\n",
6512	(priv->dma_cap.hash_filter) ? "Y" : "N");
6513	seq_printf(m: seq, fmt: "\tMultiple MAC address registers: %s\n",
6514	(priv->dma_cap.multi_addr) ? "Y" : "N");
6515	}
6516	seq_printf(m: seq, fmt: "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
6517	(priv->dma_cap.pcs) ? "Y" : "N");
6518	seq_printf(m: seq, fmt: "\tSMA (MDIO) Interface: %s\n",
6519	(priv->dma_cap.sma_mdio) ? "Y" : "N");
6520	seq_printf(m: seq, fmt: "\tPMT Remote wake up: %s\n",
6521	(priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
6522	seq_printf(m: seq, fmt: "\tPMT Magic Frame: %s\n",
6523	(priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
6524	seq_printf(m: seq, fmt: "\tRMON module: %s\n",
6525	(priv->dma_cap.rmon) ? "Y" : "N");
6526	seq_printf(m: seq, fmt: "\tIEEE 1588-2002 Time Stamp: %s\n",
6527	(priv->dma_cap.time_stamp) ? "Y" : "N");
6528	seq_printf(m: seq, fmt: "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
6529	(priv->dma_cap.atime_stamp) ? "Y" : "N");
6530	if (priv->plat->core_type == DWMAC_CORE_XGMAC)
6531	seq_printf(m: seq, fmt: "\tTimestamp System Time Source: %s\n",
6532	dwxgmac_timestamp_source[priv->dma_cap.tssrc]);
6533	seq_printf(m: seq, fmt: "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
6534	(priv->dma_cap.eee) ? "Y" : "N");
6535	seq_printf(m: seq, fmt: "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
6536	seq_printf(m: seq, fmt: "\tChecksum Offload in TX: %s\n",
6537	(priv->dma_cap.tx_coe) ? "Y" : "N");
6538	if (priv->synopsys_id >= DWMAC_CORE_4_00 ||
6539	priv->plat->core_type == DWMAC_CORE_XGMAC) {
6540	seq_printf(m: seq, fmt: "\tIP Checksum Offload in RX: %s\n",
6541	(priv->dma_cap.rx_coe) ? "Y" : "N");
6542	} else {
6543	seq_printf(m: seq, fmt: "\tIP Checksum Offload (type1) in RX: %s\n",
6544	(priv->dma_cap.rx_coe_type1) ? "Y" : "N");
6545	seq_printf(m: seq, fmt: "\tIP Checksum Offload (type2) in RX: %s\n",
6546	(priv->dma_cap.rx_coe_type2) ? "Y" : "N");
6547	seq_printf(m: seq, fmt: "\tRXFIFO > 2048bytes: %s\n",
6548	(priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
6549	}
6550	seq_printf(m: seq, fmt: "\tNumber of Additional RX channel: %d\n",
6551	priv->dma_cap.number_rx_channel);
6552	seq_printf(m: seq, fmt: "\tNumber of Additional TX channel: %d\n",
6553	priv->dma_cap.number_tx_channel);
6554	seq_printf(m: seq, fmt: "\tNumber of Additional RX queues: %d\n",
6555	priv->dma_cap.number_rx_queues);
6556	seq_printf(m: seq, fmt: "\tNumber of Additional TX queues: %d\n",
6557	priv->dma_cap.number_tx_queues);
6558	seq_printf(m: seq, fmt: "\tEnhanced descriptors: %s\n",
6559	(priv->dma_cap.enh_desc) ? "Y" : "N");
6560	seq_printf(m: seq, fmt: "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);
6561	seq_printf(m: seq, fmt: "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);
6562	seq_printf(m: seq, fmt: "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ?
6563	(BIT(priv->dma_cap.hash_tb_sz) << 5) : 0);
6564	seq_printf(m: seq, fmt: "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");
6565	seq_printf(m: seq, fmt: "\tNumber of PPS Outputs: %d\n",
6566	priv->dma_cap.pps_out_num);
6567	seq_printf(m: seq, fmt: "\tSafety Features: %s\n",
6568	dwxgmac_safety_feature_desc[priv->dma_cap.asp]);
6569	seq_printf(m: seq, fmt: "\tFlexible RX Parser: %s\n",
6570	priv->dma_cap.frpsel ? "Y" : "N");
6571	seq_printf(m: seq, fmt: "\tEnhanced Addressing: %d\n",
6572	priv->dma_cap.host_dma_width);
6573	seq_printf(m: seq, fmt: "\tReceive Side Scaling: %s\n",
6574	priv->dma_cap.rssen ? "Y" : "N");
6575	seq_printf(m: seq, fmt: "\tVLAN Hash Filtering: %s\n",
6576	priv->dma_cap.vlhash ? "Y" : "N");
6577	seq_printf(m: seq, fmt: "\tSplit Header: %s\n",
6578	priv->dma_cap.sphen ? "Y" : "N");
6579	seq_printf(m: seq, fmt: "\tVLAN TX Insertion: %s\n",
6580	priv->dma_cap.vlins ? "Y" : "N");
6581	seq_printf(m: seq, fmt: "\tDouble VLAN: %s\n",
6582	priv->dma_cap.dvlan ? "Y" : "N");
6583	seq_printf(m: seq, fmt: "\tNumber of L3/L4 Filters: %d\n",
6584	priv->dma_cap.l3l4fnum);
6585	seq_printf(m: seq, fmt: "\tARP Offloading: %s\n",
6586	priv->dma_cap.arpoffsel ? "Y" : "N");
6587	seq_printf(m: seq, fmt: "\tEnhancements to Scheduled Traffic (EST): %s\n",
6588	priv->dma_cap.estsel ? "Y" : "N");
6589	seq_printf(m: seq, fmt: "\tFrame Preemption (FPE): %s\n",
6590	priv->dma_cap.fpesel ? "Y" : "N");
6591	seq_printf(m: seq, fmt: "\tTime-Based Scheduling (TBS): %s\n",
6592	priv->dma_cap.tbssel ? "Y" : "N");
6593	seq_printf(m: seq, fmt: "\tNumber of DMA Channels Enabled for TBS: %d\n",
6594	priv->dma_cap.tbs_ch_num);
6595	seq_printf(m: seq, fmt: "\tPer-Stream Filtering: %s\n",
6596	priv->dma_cap.sgfsel ? "Y" : "N");
6597	seq_printf(m: seq, fmt: "\tTX Timestamp FIFO Depth: %lu\n",
6598	BIT(priv->dma_cap.ttsfd) >> 1);
6599	seq_printf(m: seq, fmt: "\tNumber of Traffic Classes: %d\n",
6600	priv->dma_cap.numtc);
6601	seq_printf(m: seq, fmt: "\tDCB Feature: %s\n",
6602	priv->dma_cap.dcben ? "Y" : "N");
6603	seq_printf(m: seq, fmt: "\tIEEE 1588 High Word Register: %s\n",
6604	priv->dma_cap.advthword ? "Y" : "N");
6605	seq_printf(m: seq, fmt: "\tPTP Offload: %s\n",
6606	priv->dma_cap.ptoen ? "Y" : "N");
6607	seq_printf(m: seq, fmt: "\tOne-Step Timestamping: %s\n",
6608	priv->dma_cap.osten ? "Y" : "N");
6609	seq_printf(m: seq, fmt: "\tPriority-Based Flow Control: %s\n",
6610	priv->dma_cap.pfcen ? "Y" : "N");
6611	seq_printf(m: seq, fmt: "\tNumber of Flexible RX Parser Instructions: %lu\n",
6612	BIT(priv->dma_cap.frpes) << 6);
6613	seq_printf(m: seq, fmt: "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n",
6614	BIT(priv->dma_cap.frpbs) << 6);
6615	seq_printf(m: seq, fmt: "\tParallel Instruction Processor Engines: %d\n",
6616	priv->dma_cap.frppipe_num);
6617	seq_printf(m: seq, fmt: "\tNumber of Extended VLAN Tag Filters: %lu\n",
6618	priv->dma_cap.nrvf_num ?
6619	(BIT(priv->dma_cap.nrvf_num) << 1) : 0);
6620	seq_printf(m: seq, fmt: "\tWidth of the Time Interval Field in GCL: %d\n",
6621	priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0);
6622	seq_printf(m: seq, fmt: "\tDepth of GCL: %lu\n",
6623	priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0);
6624	seq_printf(m: seq, fmt: "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n",
6625	priv->dma_cap.cbtisel ? "Y" : "N");
6626	seq_printf(m: seq, fmt: "\tNumber of Auxiliary Snapshot Inputs: %d\n",
6627	priv->dma_cap.aux_snapshot_n);
6628	seq_printf(m: seq, fmt: "\tOne-Step Timestamping for PTP over UDP/IP: %s\n",
6629	priv->dma_cap.pou_ost_en ? "Y" : "N");
6630	seq_printf(m: seq, fmt: "\tEnhanced DMA: %s\n",
6631	priv->dma_cap.edma ? "Y" : "N");
6632	seq_printf(m: seq, fmt: "\tDifferent Descriptor Cache: %s\n",
6633	priv->dma_cap.ediffc ? "Y" : "N");
6634	seq_printf(m: seq, fmt: "\tVxLAN/NVGRE: %s\n",
6635	priv->dma_cap.vxn ? "Y" : "N");
6636	seq_printf(m: seq, fmt: "\tDebug Memory Interface: %s\n",
6637	priv->dma_cap.dbgmem ? "Y" : "N");
6638	seq_printf(m: seq, fmt: "\tNumber of Policing Counters: %lu\n",
6639	priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0);
6640	return 0;
6641	}
6642	DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
6643
6644	/* Use network device events to rename debugfs file entries.
6645	*/
6646	static int stmmac_device_event(struct notifier_block *unused,
6647	unsigned long event, void *ptr)
6648	{
6649	struct net_device *dev = netdev_notifier_info_to_dev(info: ptr);
6650	struct stmmac_priv *priv = netdev_priv(dev);
6651
6652	if (dev->netdev_ops != &stmmac_netdev_ops)
6653	goto done;
6654
6655	switch (event) {
6656	case NETDEV_CHANGENAME:
6657	debugfs_change_name(dentry: priv->dbgfs_dir, fmt: "%s", dev->name);
6658	break;
6659	}
6660	done:
6661	return NOTIFY_DONE;
6662	}
6663
6664	static struct notifier_block stmmac_notifier = {
6665	.notifier_call = stmmac_device_event,
6666	};
6667
6668	static void stmmac_init_fs(struct net_device *dev)
6669	{
6670	struct stmmac_priv *priv = netdev_priv(dev);
6671
6672	rtnl_lock();
6673
6674	/* Create per netdev entries */
6675	priv->dbgfs_dir = debugfs_create_dir(name: dev->name, parent: stmmac_fs_dir);
6676
6677	/* Entry to report DMA RX/TX rings */
6678	debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
6679	&stmmac_rings_status_fops);
6680
6681	/* Entry to report the DMA HW features */
6682	debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
6683	&stmmac_dma_cap_fops);
6684
6685	rtnl_unlock();
6686	}
6687
6688	static void stmmac_exit_fs(struct net_device *dev)
6689	{
6690	struct stmmac_priv *priv = netdev_priv(dev);
6691
6692	debugfs_remove_recursive(dentry: priv->dbgfs_dir);
6693	}
6694	#endif /* CONFIG_DEBUG_FS */
6695
6696	static u32 stmmac_vid_crc32_le(__le16 vid_le)
6697	{
6698	unsigned char *data = (unsigned char *)&vid_le;
6699	unsigned char data_byte = 0;
6700	u32 crc = ~0x0;
6701	u32 temp = 0;
6702	int i, bits;
6703
6704	bits = get_bitmask_order(VLAN_VID_MASK);
6705	for (i = 0; i < bits; i++) {
6706	if ((i % 8) == 0)
6707	data_byte = data[i / 8];
6708
6709	temp = ((crc & 1) ^ data_byte) & 1;
6710	crc >>= 1;
6711	data_byte >>= 1;
6712
6713	if (temp)
6714	crc ^= 0xedb88320;
6715	}
6716
6717	return crc;
6718	}
6719
6720	static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
6721	{
6722	u32 crc, hash = 0;
6723	u16 pmatch = 0;
6724	int count = 0;
6725	u16 vid = 0;
6726
6727	for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
6728	__le16 vid_le = cpu_to_le16(vid);
6729	crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
6730	hash |= (1 << crc);
6731	count++;
6732	}
6733
6734	if (!priv->dma_cap.vlhash) {
6735	if (count > 2) /* VID = 0 always passes filter */
6736	return -EOPNOTSUPP;
6737
6738	pmatch = vid;
6739	hash = 0;
6740	}
6741
6742	return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
6743	}
6744
6745	/* FIXME: This may need RXC to be running, but it may be called with BH
6746	* disabled, which means we can't call phylink_rx_clk_stop*().
6747	*/
6748	static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
6749	{
6750	struct stmmac_priv *priv = netdev_priv(dev: ndev);
6751	bool is_double = false;
6752	int ret;
6753
6754	ret = pm_runtime_resume_and_get(dev: priv->device);
6755	if (ret < 0)
6756	return ret;
6757
6758	if (be16_to_cpu(proto) == ETH_P_8021AD)
6759	is_double = true;
6760
6761	set_bit(nr: vid, addr: priv->active_vlans);
6762	ret = stmmac_vlan_update(priv, is_double);
6763	if (ret) {
6764	clear_bit(nr: vid, addr: priv->active_vlans);
6765	goto err_pm_put;
6766	}
6767
6768	if (priv->hw->num_vlan) {
6769	ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
6770	if (ret)
6771	goto err_pm_put;
6772	}
6773	err_pm_put:
6774	pm_runtime_put(dev: priv->device);
6775
6776	return ret;
6777	}
6778
6779	/* FIXME: This may need RXC to be running, but it may be called with BH
6780	* disabled, which means we can't call phylink_rx_clk_stop*().
6781	*/
6782	static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
6783	{
6784	struct stmmac_priv *priv = netdev_priv(dev: ndev);
6785	bool is_double = false;
6786	int ret;
6787
6788	ret = pm_runtime_resume_and_get(dev: priv->device);
6789	if (ret < 0)
6790	return ret;
6791
6792	if (be16_to_cpu(proto) == ETH_P_8021AD)
6793	is_double = true;
6794
6795	clear_bit(nr: vid, addr: priv->active_vlans);
6796
6797	if (priv->hw->num_vlan) {
6798	ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
6799	if (ret)
6800	goto del_vlan_error;
6801	}
6802
6803	ret = stmmac_vlan_update(priv, is_double);
6804
6805	del_vlan_error:
6806	pm_runtime_put(dev: priv->device);
6807
6808	return ret;
6809	}
6810
6811	static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf)
6812	{
6813	struct stmmac_priv *priv = netdev_priv(dev);
6814
6815	switch (bpf->command) {
6816	case XDP_SETUP_PROG:
6817	return stmmac_xdp_set_prog(priv, prog: bpf->prog, extack: bpf->extack);
6818	case XDP_SETUP_XSK_POOL:
6819	return stmmac_xdp_setup_pool(priv, pool: bpf->xsk.pool,
6820	queue: bpf->xsk.queue_id);
6821	default:
6822	return -EOPNOTSUPP;
6823	}
6824	}
6825
6826	static int stmmac_xdp_xmit(struct net_device *dev, int num_frames,
6827	struct xdp_frame **frames, u32 flags)
6828	{
6829	struct stmmac_priv *priv = netdev_priv(dev);
6830	int cpu = smp_processor_id();
6831	struct netdev_queue *nq;
6832	int i, nxmit = 0;
6833	int queue;
6834
6835	if (unlikely(test_bit(STMMAC_DOWN, &priv->state)))
6836	return -ENETDOWN;
6837
6838	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
6839	return -EINVAL;
6840
6841	queue = stmmac_xdp_get_tx_queue(priv, cpu);
6842	nq = netdev_get_tx_queue(dev: priv->dev, index: queue);
6843
6844	__netif_tx_lock(txq: nq, cpu);
6845	/* Avoids TX time-out as we are sharing with slow path */
6846	txq_trans_cond_update(txq: nq);
6847
6848	for (i = 0; i < num_frames; i++) {
6849	int res;
6850
6851	res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf: frames[i], dma_map: true);
6852	if (res == STMMAC_XDP_CONSUMED)
6853	break;
6854
6855	nxmit++;
6856	}
6857
6858	if (flags & XDP_XMIT_FLUSH) {
6859	stmmac_flush_tx_descriptors(priv, queue);
6860	stmmac_tx_timer_arm(priv, queue);
6861	}
6862
6863	__netif_tx_unlock(txq: nq);
6864
6865	return nxmit;
6866	}
6867
6868	void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue)
6869	{
6870	struct stmmac_channel *ch = &priv->channel[queue];
6871	unsigned long flags;
6872
6873	spin_lock_irqsave(&ch->lock, flags);
6874	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
6875	spin_unlock_irqrestore(lock: &ch->lock, flags);
6876
6877	stmmac_stop_rx_dma(priv, chan: queue);
6878	__free_dma_rx_desc_resources(priv, dma_conf: &priv->dma_conf, queue);
6879	}
6880
6881	void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)
6882	{
6883	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
6884	struct stmmac_channel *ch = &priv->channel[queue];
6885	unsigned long flags;
6886	u32 buf_size;
6887	int ret;
6888
6889	ret = __alloc_dma_rx_desc_resources(priv, dma_conf: &priv->dma_conf, queue);
6890	if (ret) {
6891	netdev_err(dev: priv->dev, format: "Failed to alloc RX desc.\n");
6892	return;
6893	}
6894
6895	ret = __init_dma_rx_desc_rings(priv, dma_conf: &priv->dma_conf, queue, GFP_KERNEL);
6896	if (ret) {
6897	__free_dma_rx_desc_resources(priv, dma_conf: &priv->dma_conf, queue);
6898	netdev_err(dev: priv->dev, format: "Failed to init RX desc.\n");
6899	return;
6900	}
6901
6902	stmmac_reset_rx_queue(priv, queue);
6903	stmmac_clear_rx_descriptors(priv, dma_conf: &priv->dma_conf, queue);
6904
6905	stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
6906	rx_q->dma_rx_phy, rx_q->queue_index);
6907
6908	rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num *
6909	sizeof(struct dma_desc));
6910	stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
6911	rx_q->rx_tail_addr, rx_q->queue_index);
6912
6913	if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
6914	buf_size = xsk_pool_get_rx_frame_size(pool: rx_q->xsk_pool);
6915	stmmac_set_dma_bfsize(priv, priv->ioaddr,
6916	buf_size,
6917	rx_q->queue_index);
6918	} else {
6919	stmmac_set_dma_bfsize(priv, priv->ioaddr,
6920	priv->dma_conf.dma_buf_sz,
6921	rx_q->queue_index);
6922	}
6923
6924	stmmac_start_rx_dma(priv, chan: queue);
6925
6926	spin_lock_irqsave(&ch->lock, flags);
6927	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
6928	spin_unlock_irqrestore(lock: &ch->lock, flags);
6929	}
6930
6931	void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue)
6932	{
6933	struct stmmac_channel *ch = &priv->channel[queue];
6934	unsigned long flags;
6935
6936	spin_lock_irqsave(&ch->lock, flags);
6937	stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
6938	spin_unlock_irqrestore(lock: &ch->lock, flags);
6939
6940	stmmac_stop_tx_dma(priv, chan: queue);
6941	__free_dma_tx_desc_resources(priv, dma_conf: &priv->dma_conf, queue);
6942	}
6943
6944	void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)
6945	{
6946	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
6947	struct stmmac_channel *ch = &priv->channel[queue];
6948	unsigned long flags;
6949	int ret;
6950
6951	ret = __alloc_dma_tx_desc_resources(priv, dma_conf: &priv->dma_conf, queue);
6952	if (ret) {
6953	netdev_err(dev: priv->dev, format: "Failed to alloc TX desc.\n");
6954	return;
6955	}
6956
6957	ret = __init_dma_tx_desc_rings(priv, dma_conf: &priv->dma_conf, queue);
6958	if (ret) {
6959	__free_dma_tx_desc_resources(priv, dma_conf: &priv->dma_conf, queue);
6960	netdev_err(dev: priv->dev, format: "Failed to init TX desc.\n");
6961	return;
6962	}
6963
6964	stmmac_reset_tx_queue(priv, queue);
6965	stmmac_clear_tx_descriptors(priv, dma_conf: &priv->dma_conf, queue);
6966
6967	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
6968	tx_q->dma_tx_phy, tx_q->queue_index);
6969
6970	if (tx_q->tbs & STMMAC_TBS_AVAIL)
6971	stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index);
6972
6973	tx_q->tx_tail_addr = tx_q->dma_tx_phy;
6974	stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
6975	tx_q->tx_tail_addr, tx_q->queue_index);
6976
6977	stmmac_start_tx_dma(priv, chan: queue);
6978
6979	spin_lock_irqsave(&ch->lock, flags);
6980	stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
6981	spin_unlock_irqrestore(lock: &ch->lock, flags);
6982	}
6983
6984	void stmmac_xdp_release(struct net_device *dev)
6985	{
6986	struct stmmac_priv *priv = netdev_priv(dev);
6987	u32 chan;
6988
6989	/* Ensure tx function is not running */
6990	netif_tx_disable(dev);
6991
6992	/* Disable NAPI process */
6993	stmmac_disable_all_queues(priv);
6994
6995	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
6996	hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer);
6997
6998	/* Free the IRQ lines */
6999	stmmac_free_irq(dev, irq_err: REQ_IRQ_ERR_ALL, irq_idx: 0);
7000
7001	/* Stop TX/RX DMA channels */
7002	stmmac_stop_all_dma(priv);
7003
7004	/* Release and free the Rx/Tx resources */
7005	free_dma_desc_resources(priv, dma_conf: &priv->dma_conf);
7006
7007	/* Disable the MAC Rx/Tx */
7008	stmmac_mac_set(priv, priv->ioaddr, false);
7009
7010	/* set trans_start so we don't get spurious
7011	* watchdogs during reset
7012	*/
7013	netif_trans_update(dev);
7014	netif_carrier_off(dev);
7015	}
7016
7017	int stmmac_xdp_open(struct net_device *dev)
7018	{
7019	struct stmmac_priv *priv = netdev_priv(dev);
7020	u32 rx_cnt = priv->plat->rx_queues_to_use;
7021	u32 tx_cnt = priv->plat->tx_queues_to_use;
7022	u32 dma_csr_ch = max(rx_cnt, tx_cnt);
7023	struct stmmac_rx_queue *rx_q;
7024	struct stmmac_tx_queue *tx_q;
7025	u32 buf_size;
7026	bool sph_en;
7027	u32 chan;
7028	int ret;
7029
7030	ret = alloc_dma_desc_resources(priv, dma_conf: &priv->dma_conf);
7031	if (ret < 0) {
7032	netdev_err(dev, format: "%s: DMA descriptors allocation failed\n",
7033	__func__);
7034	goto dma_desc_error;
7035	}
7036
7037	ret = init_dma_desc_rings(dev, dma_conf: &priv->dma_conf, GFP_KERNEL);
7038	if (ret < 0) {
7039	netdev_err(dev, format: "%s: DMA descriptors initialization failed\n",
7040	__func__);
7041	goto init_error;
7042	}
7043
7044	stmmac_reset_queues_param(priv);
7045
7046	/* DMA CSR Channel configuration */
7047	for (chan = 0; chan < dma_csr_ch; chan++) {
7048	stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
7049	stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
7050	}
7051
7052	/* Adjust Split header */
7053	sph_en = (priv->hw->rx_csum > 0) && priv->sph_active;
7054
7055	/* DMA RX Channel Configuration */
7056	for (chan = 0; chan < rx_cnt; chan++) {
7057	rx_q = &priv->dma_conf.rx_queue[chan];
7058
7059	stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
7060	rx_q->dma_rx_phy, chan);
7061
7062	rx_q->rx_tail_addr = rx_q->dma_rx_phy +
7063	(rx_q->buf_alloc_num *
7064	sizeof(struct dma_desc));
7065	stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
7066	rx_q->rx_tail_addr, chan);
7067
7068	if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
7069	buf_size = xsk_pool_get_rx_frame_size(pool: rx_q->xsk_pool);
7070	stmmac_set_dma_bfsize(priv, priv->ioaddr,
7071	buf_size,
7072	rx_q->queue_index);
7073	} else {
7074	stmmac_set_dma_bfsize(priv, priv->ioaddr,
7075	priv->dma_conf.dma_buf_sz,
7076	rx_q->queue_index);
7077	}
7078
7079	stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
7080	}
7081
7082	/* DMA TX Channel Configuration */
7083	for (chan = 0; chan < tx_cnt; chan++) {
7084	tx_q = &priv->dma_conf.tx_queue[chan];
7085
7086	stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
7087	tx_q->dma_tx_phy, chan);
7088
7089	tx_q->tx_tail_addr = tx_q->dma_tx_phy;
7090	stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
7091	tx_q->tx_tail_addr, chan);
7092
7093	hrtimer_setup(timer: &tx_q->txtimer, function: stmmac_tx_timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
7094	}
7095
7096	/* Enable the MAC Rx/Tx */
7097	stmmac_mac_set(priv, priv->ioaddr, true);
7098
7099	/* Start Rx & Tx DMA Channels */
7100	stmmac_start_all_dma(priv);
7101
7102	ret = stmmac_request_irq(dev);
7103	if (ret)
7104	goto irq_error;
7105
7106	/* Enable NAPI process*/
7107	stmmac_enable_all_queues(priv);
7108	netif_carrier_on(dev);
7109	netif_tx_start_all_queues(dev);
7110	stmmac_enable_all_dma_irq(priv);
7111
7112	return 0;
7113
7114	irq_error:
7115	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
7116	hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer);
7117
7118	init_error:
7119	free_dma_desc_resources(priv, dma_conf: &priv->dma_conf);
7120	dma_desc_error:
7121	return ret;
7122	}
7123
7124	int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags)
7125	{
7126	struct stmmac_priv *priv = netdev_priv(dev);
7127	struct stmmac_rx_queue *rx_q;
7128	struct stmmac_tx_queue *tx_q;
7129	struct stmmac_channel *ch;
7130
7131	if (test_bit(STMMAC_DOWN, &priv->state) ||
7132	!netif_carrier_ok(dev: priv->dev))
7133	return -ENETDOWN;
7134
7135	if (!stmmac_xdp_is_enabled(priv))
7136	return -EINVAL;
7137
7138	if (queue >= priv->plat->rx_queues_to_use ||
7139	queue >= priv->plat->tx_queues_to_use)
7140	return -EINVAL;
7141
7142	rx_q = &priv->dma_conf.rx_queue[queue];
7143	tx_q = &priv->dma_conf.tx_queue[queue];
7144	ch = &priv->channel[queue];
7145
7146	if (!rx_q->xsk_pool && !tx_q->xsk_pool)
7147	return -EINVAL;
7148
7149	if (!napi_if_scheduled_mark_missed(n: &ch->rxtx_napi)) {
7150	/* EQoS does not have per-DMA channel SW interrupt,
7151	* so we schedule RX Napi straight-away.
7152	*/
7153	if (likely(napi_schedule_prep(&ch->rxtx_napi)))
7154	__napi_schedule(n: &ch->rxtx_napi);
7155	}
7156
7157	return 0;
7158	}
7159
7160	static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
7161	{
7162	struct stmmac_priv *priv = netdev_priv(dev);
7163	u32 tx_cnt = priv->plat->tx_queues_to_use;
7164	u32 rx_cnt = priv->plat->rx_queues_to_use;
7165	unsigned int start;
7166	int q;
7167
7168	for (q = 0; q < tx_cnt; q++) {
7169	struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q];
7170	u64 tx_packets;
7171	u64 tx_bytes;
7172
7173	do {
7174	start = u64_stats_fetch_begin(syncp: &txq_stats->q_syncp);
7175	tx_bytes = u64_stats_read(p: &txq_stats->q.tx_bytes);
7176	} while (u64_stats_fetch_retry(syncp: &txq_stats->q_syncp, start));
7177	do {
7178	start = u64_stats_fetch_begin(syncp: &txq_stats->napi_syncp);
7179	tx_packets = u64_stats_read(p: &txq_stats->napi.tx_packets);
7180	} while (u64_stats_fetch_retry(syncp: &txq_stats->napi_syncp, start));
7181
7182	stats->tx_packets += tx_packets;
7183	stats->tx_bytes += tx_bytes;
7184	}
7185
7186	for (q = 0; q < rx_cnt; q++) {
7187	struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q];
7188	u64 rx_packets;
7189	u64 rx_bytes;
7190
7191	do {
7192	start = u64_stats_fetch_begin(syncp: &rxq_stats->napi_syncp);
7193	rx_packets = u64_stats_read(p: &rxq_stats->napi.rx_packets);
7194	rx_bytes = u64_stats_read(p: &rxq_stats->napi.rx_bytes);
7195	} while (u64_stats_fetch_retry(syncp: &rxq_stats->napi_syncp, start));
7196
7197	stats->rx_packets += rx_packets;
7198	stats->rx_bytes += rx_bytes;
7199	}
7200
7201	stats->rx_dropped = priv->xstats.rx_dropped;
7202	stats->rx_errors = priv->xstats.rx_errors;
7203	stats->tx_dropped = priv->xstats.tx_dropped;
7204	stats->tx_errors = priv->xstats.tx_errors;
7205	stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier;
7206	stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision;
7207	stats->rx_length_errors = priv->xstats.rx_length;
7208	stats->rx_crc_errors = priv->xstats.rx_crc_errors;
7209	stats->rx_over_errors = priv->xstats.rx_overflow_cntr;
7210	stats->rx_missed_errors = priv->xstats.rx_missed_cntr;
7211	}
7212
7213	static const struct net_device_ops stmmac_netdev_ops = {
7214	.ndo_open = stmmac_open,
7215	.ndo_start_xmit = stmmac_xmit,
7216	.ndo_stop = stmmac_release,
7217	.ndo_change_mtu = stmmac_change_mtu,
7218	.ndo_fix_features = stmmac_fix_features,
7219	.ndo_set_features = stmmac_set_features,
7220	.ndo_set_rx_mode = stmmac_set_rx_mode,
7221	.ndo_tx_timeout = stmmac_tx_timeout,
7222	.ndo_eth_ioctl = stmmac_ioctl,
7223	.ndo_get_stats64 = stmmac_get_stats64,
7224	.ndo_setup_tc = stmmac_setup_tc,
7225	.ndo_select_queue = stmmac_select_queue,
7226	.ndo_set_mac_address = stmmac_set_mac_address,
7227	.ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
7228	.ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
7229	.ndo_bpf = stmmac_bpf,
7230	.ndo_xdp_xmit = stmmac_xdp_xmit,
7231	.ndo_xsk_wakeup = stmmac_xsk_wakeup,
7232	.ndo_hwtstamp_get = stmmac_hwtstamp_get,
7233	.ndo_hwtstamp_set = stmmac_hwtstamp_set,
7234	};
7235
7236	static void stmmac_reset_subtask(struct stmmac_priv *priv)
7237	{
7238	if (!test_and_clear_bit(nr: STMMAC_RESET_REQUESTED, addr: &priv->state))
7239	return;
7240	if (test_bit(STMMAC_DOWN, &priv->state))
7241	return;
7242
7243	netdev_err(dev: priv->dev, format: "Reset adapter.\n");
7244
7245	rtnl_lock();
7246	netif_trans_update(dev: priv->dev);
7247	while (test_and_set_bit(nr: STMMAC_RESETING, addr: &priv->state))
7248	usleep_range(min: 1000, max: 2000);
7249
7250	set_bit(nr: STMMAC_DOWN, addr: &priv->state);
7251	dev_close(dev: priv->dev);
7252	dev_open(dev: priv->dev, NULL);
7253	clear_bit(nr: STMMAC_DOWN, addr: &priv->state);
7254	clear_bit(nr: STMMAC_RESETING, addr: &priv->state);
7255	rtnl_unlock();
7256	}
7257
7258	static void stmmac_service_task(struct work_struct *work)
7259	{
7260	struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
7261	service_task);
7262
7263	stmmac_reset_subtask(priv);
7264	clear_bit(nr: STMMAC_SERVICE_SCHED, addr: &priv->state);
7265	}
7266
7267	/**
7268	* stmmac_hw_init - Init the MAC device
7269	* @priv: driver private structure
7270	* Description: this function is to configure the MAC device according to
7271	* some platform parameters or the HW capability register. It prepares the
7272	* driver to use either ring or chain modes and to setup either enhanced or
7273	* normal descriptors.
7274	*/
7275	static int stmmac_hw_init(struct stmmac_priv *priv)
7276	{
7277	int ret;
7278
7279	/* dwmac-sun8i only work in chain mode */
7280	if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I)
7281	chain_mode = 1;
7282	priv->chain_mode = chain_mode;
7283
7284	/* Initialize HW Interface */
7285	ret = stmmac_hwif_init(priv);
7286	if (ret)
7287	return ret;
7288
7289	/* Get the HW capability (new GMAC newer than 3.50a) */
7290	priv->hw_cap_support = stmmac_get_hw_features(priv);
7291	if (priv->hw_cap_support) {
7292	dev_info(priv->device, "DMA HW capability register supported\n");
7293
7294	/* We can override some gmac/dma configuration fields: e.g.
7295	* enh_desc, tx_coe (e.g. that are passed through the
7296	* platform) with the values from the HW capability
7297	* register (if supported).
7298	*/
7299	priv->plat->enh_desc = priv->dma_cap.enh_desc;
7300	priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up &&
7301	!(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL);
7302	if (priv->dma_cap.hash_tb_sz) {
7303	priv->hw->multicast_filter_bins =
7304	(BIT(priv->dma_cap.hash_tb_sz) << 5);
7305	priv->hw->mcast_bits_log2 =
7306	ilog2(priv->hw->multicast_filter_bins);
7307	}
7308
7309	/* TXCOE doesn't work in thresh DMA mode */
7310	if (priv->plat->force_thresh_dma_mode)
7311	priv->plat->tx_coe = 0;
7312	else
7313	priv->plat->tx_coe = priv->dma_cap.tx_coe;
7314
7315	/* In case of GMAC4 rx_coe is from HW cap register. */
7316	priv->plat->rx_coe = priv->dma_cap.rx_coe;
7317
7318	if (priv->dma_cap.rx_coe_type2)
7319	priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
7320	else if (priv->dma_cap.rx_coe_type1)
7321	priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
7322
7323	} else {
7324	dev_info(priv->device, "No HW DMA feature register supported\n");
7325	}
7326
7327	if (priv->plat->rx_coe) {
7328	priv->hw->rx_csum = priv->plat->rx_coe;
7329	dev_info(priv->device, "RX Checksum Offload Engine supported\n");
7330	if (priv->synopsys_id < DWMAC_CORE_4_00)
7331	dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
7332	}
7333	if (priv->plat->tx_coe)
7334	dev_info(priv->device, "TX Checksum insertion supported\n");
7335
7336	if (priv->plat->pmt) {
7337	dev_info(priv->device, "Wake-Up On Lan supported\n");
7338	device_set_wakeup_capable(dev: priv->device, capable: 1);
7339	devm_pm_set_wake_irq(dev: priv->device, irq: priv->wol_irq);
7340	}
7341
7342	if (priv->dma_cap.tsoen)
7343	dev_info(priv->device, "TSO supported\n");
7344
7345	if (priv->dma_cap.number_rx_queues &&
7346	priv->plat->rx_queues_to_use > priv->dma_cap.number_rx_queues) {
7347	dev_warn(priv->device,
7348	"Number of Rx queues (%u) exceeds dma capability\n",
7349	priv->plat->rx_queues_to_use);
7350	priv->plat->rx_queues_to_use = priv->dma_cap.number_rx_queues;
7351	}
7352	if (priv->dma_cap.number_tx_queues &&
7353	priv->plat->tx_queues_to_use > priv->dma_cap.number_tx_queues) {
7354	dev_warn(priv->device,
7355	"Number of Tx queues (%u) exceeds dma capability\n",
7356	priv->plat->tx_queues_to_use);
7357	priv->plat->tx_queues_to_use = priv->dma_cap.number_tx_queues;
7358	}
7359
7360	if (priv->dma_cap.rx_fifo_size &&
7361	priv->plat->rx_fifo_size > priv->dma_cap.rx_fifo_size) {
7362	dev_warn(priv->device,
7363	"Rx FIFO size (%u) exceeds dma capability\n",
7364	priv->plat->rx_fifo_size);
7365	priv->plat->rx_fifo_size = priv->dma_cap.rx_fifo_size;
7366	}
7367	if (priv->dma_cap.tx_fifo_size &&
7368	priv->plat->tx_fifo_size > priv->dma_cap.tx_fifo_size) {
7369	dev_warn(priv->device,
7370	"Tx FIFO size (%u) exceeds dma capability\n",
7371	priv->plat->tx_fifo_size);
7372	priv->plat->tx_fifo_size = priv->dma_cap.tx_fifo_size;
7373	}
7374
7375	priv->hw->vlan_fail_q_en =
7376	(priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN);
7377	priv->hw->vlan_fail_q = priv->plat->vlan_fail_q;
7378
7379	/* Run HW quirks, if any */
7380	if (priv->hwif_quirks) {
7381	ret = priv->hwif_quirks(priv);
7382	if (ret)
7383	return ret;
7384	}
7385
7386	/* Rx Watchdog is available in the COREs newer than the 3.40.
7387	* In some case, for example on bugged HW this feature
7388	* has to be disable and this can be done by passing the
7389	* riwt_off field from the platform.
7390	*/
7391	if ((priv->synopsys_id >= DWMAC_CORE_3_50 ||
7392	priv->plat->core_type == DWMAC_CORE_XGMAC) &&
7393	!priv->plat->riwt_off) {
7394	priv->use_riwt = 1;
7395	dev_info(priv->device,
7396	"Enable RX Mitigation via HW Watchdog Timer\n");
7397	}
7398
7399	/* Unimplemented PCS init (as indicated by stmmac_do_callback()
7400	* perversely returning -EINVAL) is non-fatal.
7401	*/
7402	ret = stmmac_mac_pcs_init(priv);
7403	if (ret != -EINVAL)
7404	return ret;
7405
7406	return 0;
7407	}
7408
7409	static void stmmac_napi_add(struct net_device *dev)
7410	{
7411	struct stmmac_priv *priv = netdev_priv(dev);
7412	u32 queue, maxq;
7413
7414	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
7415
7416	for (queue = 0; queue < maxq; queue++) {
7417	struct stmmac_channel *ch = &priv->channel[queue];
7418
7419	ch->priv_data = priv;
7420	ch->index = queue;
7421	spin_lock_init(&ch->lock);
7422
7423	if (queue < priv->plat->rx_queues_to_use) {
7424	netif_napi_add(dev, napi: &ch->rx_napi, poll: stmmac_napi_poll_rx);
7425	}
7426	if (queue < priv->plat->tx_queues_to_use) {
7427	netif_napi_add_tx(dev, napi: &ch->tx_napi,
7428	poll: stmmac_napi_poll_tx);
7429	}
7430	if (queue < priv->plat->rx_queues_to_use &&
7431	queue < priv->plat->tx_queues_to_use) {
7432	netif_napi_add(dev, napi: &ch->rxtx_napi,
7433	poll: stmmac_napi_poll_rxtx);
7434	}
7435	}
7436	}
7437
7438	static void stmmac_napi_del(struct net_device *dev)
7439	{
7440	struct stmmac_priv *priv = netdev_priv(dev);
7441	u32 queue, maxq;
7442
7443	maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
7444
7445	for (queue = 0; queue < maxq; queue++) {
7446	struct stmmac_channel *ch = &priv->channel[queue];
7447
7448	if (queue < priv->plat->rx_queues_to_use)
7449	netif_napi_del(napi: &ch->rx_napi);
7450	if (queue < priv->plat->tx_queues_to_use)
7451	netif_napi_del(napi: &ch->tx_napi);
7452	if (queue < priv->plat->rx_queues_to_use &&
7453	queue < priv->plat->tx_queues_to_use) {
7454	netif_napi_del(napi: &ch->rxtx_napi);
7455	}
7456	}
7457	}
7458
7459	int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt)
7460	{
7461	struct stmmac_priv *priv = netdev_priv(dev);
7462	int ret = 0, i;
7463
7464	if (netif_running(dev))
7465	stmmac_release(dev);
7466
7467	stmmac_napi_del(dev);
7468
7469	priv->plat->rx_queues_to_use = rx_cnt;
7470	priv->plat->tx_queues_to_use = tx_cnt;
7471	if (!netif_is_rxfh_configured(dev))
7472	for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
7473	priv->rss.table[i] = ethtool_rxfh_indir_default(index: i,
7474	n_rx_rings: rx_cnt);
7475
7476	stmmac_napi_add(dev);
7477
7478	if (netif_running(dev))
7479	ret = stmmac_open(dev);
7480
7481	return ret;
7482	}
7483
7484	int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size)
7485	{
7486	struct stmmac_priv *priv = netdev_priv(dev);
7487	int ret = 0;
7488
7489	if (netif_running(dev))
7490	stmmac_release(dev);
7491
7492	priv->dma_conf.dma_rx_size = rx_size;
7493	priv->dma_conf.dma_tx_size = tx_size;
7494
7495	if (netif_running(dev))
7496	ret = stmmac_open(dev);
7497
7498	return ret;
7499	}
7500
7501	static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp)
7502	{
7503	const struct stmmac_xdp_buff *ctx = (void *)_ctx;
7504	struct dma_desc *desc_contains_ts = ctx->desc;
7505	struct stmmac_priv *priv = ctx->priv;
7506	struct dma_desc *ndesc = ctx->ndesc;
7507	struct dma_desc *desc = ctx->desc;
7508	u64 ns = 0;
7509
7510	if (!priv->hwts_rx_en)
7511	return -ENODATA;
7512
7513	/* For GMAC4, the valid timestamp is from CTX next desc. */
7514	if (dwmac_is_xmac(core_type: priv->plat->core_type))
7515	desc_contains_ts = ndesc;
7516
7517	/* Check if timestamp is available */
7518	if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) {
7519	stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns);
7520	ns -= priv->plat->cdc_error_adj;
7521	*timestamp = ns_to_ktime(ns);
7522	return 0;
7523	}
7524
7525	return -ENODATA;
7526	}
7527
7528	static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = {
7529	.xmo_rx_timestamp = stmmac_xdp_rx_timestamp,
7530	};
7531
7532	static int stmmac_dl_ts_coarse_set(struct devlink *dl, u32 id,
7533	struct devlink_param_gset_ctx *ctx,
7534	struct netlink_ext_ack *extack)
7535	{
7536	struct stmmac_devlink_priv *dl_priv = devlink_priv(devlink: dl);
7537	struct stmmac_priv *priv = dl_priv->stmmac_priv;
7538
7539	priv->tsfupdt_coarse = ctx->val.vbool;
7540
7541	if (priv->tsfupdt_coarse)
7542	priv->systime_flags &= ~PTP_TCR_TSCFUPDT;
7543	else
7544	priv->systime_flags |= PTP_TCR_TSCFUPDT;
7545
7546	/* In Coarse mode, we can use a smaller subsecond increment, let's
7547	* reconfigure the systime, subsecond increment and addend.
7548	*/
7549	stmmac_update_subsecond_increment(priv);
7550
7551	return 0;
7552	}
7553
7554	static int stmmac_dl_ts_coarse_get(struct devlink *dl, u32 id,
7555	struct devlink_param_gset_ctx *ctx,
7556	struct netlink_ext_ack *extack)
7557	{
7558	struct stmmac_devlink_priv *dl_priv = devlink_priv(devlink: dl);
7559	struct stmmac_priv *priv = dl_priv->stmmac_priv;
7560
7561	ctx->val.vbool = priv->tsfupdt_coarse;
7562
7563	return 0;
7564	}
7565
7566	static const struct devlink_param stmmac_devlink_params[] = {
7567	DEVLINK_PARAM_DRIVER(STMMAC_DEVLINK_PARAM_ID_TS_COARSE, "phc_coarse_adj",
7568	DEVLINK_PARAM_TYPE_BOOL,
7569	BIT(DEVLINK_PARAM_CMODE_RUNTIME),
7570	stmmac_dl_ts_coarse_get,
7571	stmmac_dl_ts_coarse_set, NULL),
7572	};
7573
7574	/* None of the generic devlink parameters are implemented */
7575	static const struct devlink_ops stmmac_devlink_ops = {};
7576
7577	static int stmmac_register_devlink(struct stmmac_priv *priv)
7578	{
7579	struct stmmac_devlink_priv *dl_priv;
7580	int ret;
7581
7582	/* For now, what is exposed over devlink is only relevant when
7583	* timestamping is available and we have a valid ptp clock rate
7584	*/
7585	if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp) ||
7586	!priv->plat->clk_ptp_rate)
7587	return 0;
7588
7589	priv->devlink = devlink_alloc(ops: &stmmac_devlink_ops, priv_size: sizeof(*dl_priv),
7590	dev: priv->device);
7591	if (!priv->devlink)
7592	return -ENOMEM;
7593
7594	dl_priv = devlink_priv(devlink: priv->devlink);
7595	dl_priv->stmmac_priv = priv;
7596
7597	ret = devlink_params_register(devlink: priv->devlink, params: stmmac_devlink_params,
7598	ARRAY_SIZE(stmmac_devlink_params));
7599	if (ret)
7600	goto dl_free;
7601
7602	devlink_register(devlink: priv->devlink);
7603	return 0;
7604
7605	dl_free:
7606	devlink_free(devlink: priv->devlink);
7607
7608	return ret;
7609	}
7610
7611	static void stmmac_unregister_devlink(struct stmmac_priv *priv)
7612	{
7613	if (!priv->devlink)
7614	return;
7615
7616	devlink_unregister(devlink: priv->devlink);
7617	devlink_params_unregister(devlink: priv->devlink, params: stmmac_devlink_params,
7618	ARRAY_SIZE(stmmac_devlink_params));
7619	devlink_free(devlink: priv->devlink);
7620	}
7621
7622	struct plat_stmmacenet_data *stmmac_plat_dat_alloc(struct device *dev)
7623	{
7624	struct plat_stmmacenet_data *plat_dat;
7625	int i;
7626
7627	plat_dat = devm_kzalloc(dev, size: sizeof(*plat_dat), GFP_KERNEL);
7628	if (!plat_dat)
7629	return NULL;
7630
7631	/* Set the defaults:
7632	* - phy autodetection
7633	* - determine GMII_Address CR field from CSR clock
7634	* - allow MTU up to JUMBO_LEN
7635	* - hash table size
7636	* - one unicast filter entry
7637	*/
7638	plat_dat->phy_addr = -1;
7639	plat_dat->clk_csr = -1;
7640	plat_dat->maxmtu = JUMBO_LEN;
7641	plat_dat->multicast_filter_bins = HASH_TABLE_SIZE;
7642	plat_dat->unicast_filter_entries = 1;
7643
7644	/* Set the mtl defaults */
7645	plat_dat->tx_queues_to_use = 1;
7646	plat_dat->rx_queues_to_use = 1;
7647
7648	/* Setup the default RX queue channel map */
7649	for (i = 0; i < ARRAY_SIZE(plat_dat->rx_queues_cfg); i++)
7650	plat_dat->rx_queues_cfg[i].chan = i;
7651
7652	return plat_dat;
7653	}
7654	EXPORT_SYMBOL_GPL(stmmac_plat_dat_alloc);
7655
7656	static int __stmmac_dvr_probe(struct device *device,
7657	struct plat_stmmacenet_data *plat_dat,
7658	struct stmmac_resources *res)
7659	{
7660	struct net_device *ndev = NULL;
7661	struct stmmac_priv *priv;
7662	u32 rxq;
7663	int i, ret = 0;
7664
7665	ndev = devm_alloc_etherdev_mqs(dev: device, sizeof_priv: sizeof(struct stmmac_priv),
7666	MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
7667	if (!ndev)
7668	return -ENOMEM;
7669
7670	SET_NETDEV_DEV(ndev, device);
7671
7672	priv = netdev_priv(dev: ndev);
7673	priv->device = device;
7674	priv->dev = ndev;
7675
7676	for (i = 0; i < MTL_MAX_RX_QUEUES; i++)
7677	u64_stats_init(syncp: &priv->xstats.rxq_stats[i].napi_syncp);
7678	for (i = 0; i < MTL_MAX_TX_QUEUES; i++) {
7679	u64_stats_init(syncp: &priv->xstats.txq_stats[i].q_syncp);
7680	u64_stats_init(syncp: &priv->xstats.txq_stats[i].napi_syncp);
7681	}
7682
7683	priv->xstats.pcpu_stats =
7684	devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats);
7685	if (!priv->xstats.pcpu_stats)
7686	return -ENOMEM;
7687
7688	stmmac_set_ethtool_ops(netdev: ndev);
7689	priv->pause_time = pause;
7690	priv->plat = plat_dat;
7691	priv->ioaddr = res->addr;
7692	priv->dev->base_addr = (unsigned long)res->addr;
7693	priv->plat->dma_cfg->multi_msi_en =
7694	(priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN);
7695
7696	priv->dev->irq = res->irq;
7697	priv->wol_irq = res->wol_irq;
7698	priv->lpi_irq = res->lpi_irq;
7699	priv->sfty_irq = res->sfty_irq;
7700	priv->sfty_ce_irq = res->sfty_ce_irq;
7701	priv->sfty_ue_irq = res->sfty_ue_irq;
7702	for (i = 0; i < MTL_MAX_RX_QUEUES; i++)
7703	priv->rx_irq[i] = res->rx_irq[i];
7704	for (i = 0; i < MTL_MAX_TX_QUEUES; i++)
7705	priv->tx_irq[i] = res->tx_irq[i];
7706
7707	if (!is_zero_ether_addr(addr: res->mac))
7708	eth_hw_addr_set(dev: priv->dev, addr: res->mac);
7709
7710	dev_set_drvdata(dev: device, data: priv->dev);
7711
7712	/* Verify driver arguments */
7713	stmmac_verify_args();
7714
7715	priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL);
7716	if (!priv->af_xdp_zc_qps)
7717	return -ENOMEM;
7718
7719	/* Allocate workqueue */
7720	priv->wq = create_singlethread_workqueue("stmmac_wq");
7721	if (!priv->wq) {
7722	dev_err(priv->device, "failed to create workqueue\n");
7723	ret = -ENOMEM;
7724	goto error_wq_init;
7725	}
7726
7727	INIT_WORK(&priv->service_task, stmmac_service_task);
7728
7729	timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
7730
7731	/* Override with kernel parameters if supplied XXX CRS XXX
7732	* this needs to have multiple instances
7733	*/
7734	if ((phyaddr >= 0) && (phyaddr <= 31))
7735	priv->plat->phy_addr = phyaddr;
7736
7737	if (priv->plat->stmmac_rst) {
7738	ret = reset_control_assert(rstc: priv->plat->stmmac_rst);
7739	reset_control_deassert(rstc: priv->plat->stmmac_rst);
7740	/* Some reset controllers have only reset callback instead of
7741	* assert + deassert callbacks pair.
7742	*/
7743	if (ret == -ENOTSUPP)
7744	reset_control_reset(rstc: priv->plat->stmmac_rst);
7745	}
7746
7747	ret = reset_control_deassert(rstc: priv->plat->stmmac_ahb_rst);
7748	if (ret == -ENOTSUPP)
7749	dev_err(priv->device, "unable to bring out of ahb reset: %pe\n",
7750	ERR_PTR(ret));
7751
7752	/* Wait a bit for the reset to take effect */
7753	udelay(usec: 10);
7754
7755	/* Init MAC and get the capabilities */
7756	ret = stmmac_hw_init(priv);
7757	if (ret)
7758	goto error_hw_init;
7759
7760	/* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch.
7761	*/
7762	if (priv->synopsys_id < DWMAC_CORE_5_20)
7763	priv->plat->dma_cfg->dche = false;
7764
7765	stmmac_check_ether_addr(priv);
7766
7767	ndev->netdev_ops = &stmmac_netdev_ops;
7768
7769	ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops;
7770	ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops;
7771
7772	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
7773	NETIF_F_RXCSUM;
7774	ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
7775	NETDEV_XDP_ACT_XSK_ZEROCOPY;
7776
7777	ret = stmmac_tc_init(priv, priv);
7778	if (!ret) {
7779	ndev->hw_features |= NETIF_F_HW_TC;
7780	}
7781
7782	if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) {
7783	ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
7784	if (priv->plat->core_type == DWMAC_CORE_GMAC4)
7785	ndev->hw_features |= NETIF_F_GSO_UDP_L4;
7786	priv->tso = true;
7787	dev_info(priv->device, "TSO feature enabled\n");
7788	}
7789
7790	if (priv->dma_cap.sphen &&
7791	!(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) {
7792	ndev->hw_features |= NETIF_F_GRO;
7793	priv->sph_capable = true;
7794	priv->sph_active = priv->sph_capable;
7795	dev_info(priv->device, "SPH feature enabled\n");
7796	}
7797
7798	/* Ideally our host DMA address width is the same as for the
7799	* device. However, it may differ and then we have to use our
7800	* host DMA width for allocation and the device DMA width for
7801	* register handling.
7802	*/
7803	if (priv->plat->host_dma_width)
7804	priv->dma_cap.host_dma_width = priv->plat->host_dma_width;
7805	else
7806	priv->dma_cap.host_dma_width = priv->dma_cap.addr64;
7807
7808	if (priv->dma_cap.host_dma_width) {
7809	ret = dma_set_mask_and_coherent(dev: device,
7810	DMA_BIT_MASK(priv->dma_cap.host_dma_width));
7811	if (!ret) {
7812	dev_info(priv->device, "Using %d/%d bits DMA host/device width\n",
7813	priv->dma_cap.host_dma_width, priv->dma_cap.addr64);
7814
7815	/*
7816	* If more than 32 bits can be addressed, make sure to
7817	* enable enhanced addressing mode.
7818	*/
7819	if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
7820	priv->plat->dma_cfg->eame = true;
7821	} else {
7822	ret = dma_set_mask_and_coherent(dev: device, DMA_BIT_MASK(32));
7823	if (ret) {
7824	dev_err(priv->device, "Failed to set DMA Mask\n");
7825	goto error_hw_init;
7826	}
7827
7828	priv->dma_cap.host_dma_width = 32;
7829	}
7830	}
7831
7832	ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
7833	ndev->watchdog_timeo = msecs_to_jiffies(m: watchdog);
7834	#ifdef STMMAC_VLAN_TAG_USED
7835	/* Both mac100 and gmac support receive VLAN tag detection */
7836	ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
7837	if (dwmac_is_xmac(core_type: priv->plat->core_type)) {
7838	ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
7839	priv->hw->hw_vlan_en = true;
7840	}
7841	if (priv->dma_cap.vlhash) {
7842	ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
7843	ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
7844	}
7845	if (priv->dma_cap.vlins)
7846	ndev->features |= NETIF_F_HW_VLAN_CTAG_TX;
7847	#endif
7848	priv->msg_enable = netif_msg_init(debug_value: debug, default_msg_enable_bits: default_msg_level);
7849
7850	priv->xstats.threshold = tc;
7851
7852	/* Initialize RSS */
7853	rxq = priv->plat->rx_queues_to_use;
7854	netdev_rss_key_fill(buffer: priv->rss.key, len: sizeof(priv->rss.key));
7855	for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
7856	priv->rss.table[i] = ethtool_rxfh_indir_default(index: i, n_rx_rings: rxq);
7857
7858	if (priv->dma_cap.rssen && priv->plat->rss_en)
7859	ndev->features |= NETIF_F_RXHASH;
7860
7861	ndev->vlan_features |= ndev->features;
7862
7863	/* MTU range: 46 - hw-specific max */
7864	ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
7865
7866	if (priv->plat->core_type == DWMAC_CORE_XGMAC)
7867	ndev->max_mtu = XGMAC_JUMBO_LEN;
7868	else if (priv->plat->enh_desc || priv->synopsys_id >= DWMAC_CORE_4_00)
7869	ndev->max_mtu = JUMBO_LEN;
7870	else
7871	ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
7872
7873	/* Warn if the platform's maxmtu is smaller than the minimum MTU,
7874	* otherwise clamp the maximum MTU above to the platform's maxmtu.
7875	*/
7876	if (priv->plat->maxmtu < ndev->min_mtu)
7877	dev_warn(priv->device,
7878	"%s: warning: maxmtu having invalid value (%d)\n",
7879	__func__, priv->plat->maxmtu);
7880	else if (priv->plat->maxmtu < ndev->max_mtu)
7881	ndev->max_mtu = priv->plat->maxmtu;
7882
7883	ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
7884
7885	/* Setup channels NAPI */
7886	stmmac_napi_add(dev: ndev);
7887
7888	mutex_init(&priv->lock);
7889
7890	stmmac_fpe_init(priv);
7891
7892	stmmac_check_pcs_mode(priv);
7893
7894	pm_runtime_get_noresume(dev: device);
7895	pm_runtime_set_active(dev: device);
7896	if (!pm_runtime_enabled(dev: device))
7897	pm_runtime_enable(dev: device);
7898
7899	ret = stmmac_mdio_register(ndev);
7900	if (ret < 0) {
7901	dev_err_probe(dev: priv->device, err: ret,
7902	fmt: "MDIO bus (id: %d) registration failed\n",
7903	priv->plat->bus_id);
7904	goto error_mdio_register;
7905	}
7906
7907	ret = stmmac_pcs_setup(ndev);
7908	if (ret)
7909	goto error_pcs_setup;
7910
7911	ret = stmmac_phylink_setup(priv);
7912	if (ret) {
7913	netdev_err(dev: ndev, format: "failed to setup phy (%d)\n", ret);
7914	goto error_phy_setup;
7915	}
7916
7917	ret = stmmac_register_devlink(priv);
7918	if (ret)
7919	goto error_devlink_setup;
7920
7921	ret = register_netdev(dev: ndev);
7922	if (ret) {
7923	dev_err(priv->device, "%s: ERROR %i registering the device\n",
7924	__func__, ret);
7925	goto error_netdev_register;
7926	}
7927
7928	#ifdef CONFIG_DEBUG_FS
7929	stmmac_init_fs(dev: ndev);
7930	#endif
7931
7932	if (priv->plat->dump_debug_regs)
7933	priv->plat->dump_debug_regs(priv->plat->bsp_priv);
7934
7935	/* Let pm_runtime_put() disable the clocks.
7936	* If CONFIG_PM is not enabled, the clocks will stay powered.
7937	*/
7938	pm_runtime_put(dev: device);
7939
7940	return ret;
7941
7942	error_netdev_register:
7943	stmmac_unregister_devlink(priv);
7944	error_devlink_setup:
7945	phylink_destroy(priv->phylink);
7946	error_phy_setup:
7947	stmmac_pcs_clean(ndev);
7948	error_pcs_setup:
7949	stmmac_mdio_unregister(ndev);
7950	error_mdio_register:
7951	stmmac_napi_del(dev: ndev);
7952	error_hw_init:
7953	destroy_workqueue(wq: priv->wq);
7954	error_wq_init:
7955	bitmap_free(bitmap: priv->af_xdp_zc_qps);
7956
7957	return ret;
7958	}
7959
7960	/**
7961	* stmmac_dvr_probe
7962	* @dev: device pointer
7963	* @plat_dat: platform data pointer
7964	* @res: stmmac resource pointer
7965	* Description: this is the main probe function used to
7966	* call the alloc_etherdev, allocate the priv structure.
7967	* Return:
7968	* returns 0 on success, otherwise errno.
7969	*/
7970	int stmmac_dvr_probe(struct device *dev, struct plat_stmmacenet_data *plat_dat,
7971	struct stmmac_resources *res)
7972	{
7973	int ret;
7974
7975	if (plat_dat->init) {
7976	ret = plat_dat->init(dev, plat_dat->bsp_priv);
7977	if (ret)
7978	return ret;
7979	}
7980
7981	ret = __stmmac_dvr_probe(device: dev, plat_dat, res);
7982	if (ret && plat_dat->exit)
7983	plat_dat->exit(dev, plat_dat->bsp_priv);
7984
7985	return ret;
7986	}
7987	EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
7988
7989	/**
7990	* stmmac_dvr_remove
7991	* @dev: device pointer
7992	* Description: this function resets the TX/RX processes, disables the MAC RX/TX
7993	* changes the link status, releases the DMA descriptor rings.
7994	*/
7995	void stmmac_dvr_remove(struct device *dev)
7996	{
7997	struct net_device *ndev = dev_get_drvdata(dev);
7998	struct stmmac_priv *priv = netdev_priv(dev: ndev);
7999
8000	netdev_info(dev: priv->dev, format: "%s: removing driver", __func__);
8001
8002	pm_runtime_get_sync(dev);
8003
8004	unregister_netdev(dev: ndev);
8005
8006	#ifdef CONFIG_DEBUG_FS
8007	stmmac_exit_fs(dev: ndev);
8008	#endif
8009	stmmac_unregister_devlink(priv);
8010
8011	phylink_destroy(priv->phylink);
8012	if (priv->plat->stmmac_rst)
8013	reset_control_assert(rstc: priv->plat->stmmac_rst);
8014	reset_control_assert(rstc: priv->plat->stmmac_ahb_rst);
8015
8016	stmmac_pcs_clean(ndev);
8017	stmmac_mdio_unregister(ndev);
8018
8019	destroy_workqueue(wq: priv->wq);
8020	mutex_destroy(lock: &priv->lock);
8021	bitmap_free(bitmap: priv->af_xdp_zc_qps);
8022
8023	pm_runtime_disable(dev);
8024	pm_runtime_put_noidle(dev);
8025
8026	if (priv->plat->exit)
8027	priv->plat->exit(dev, priv->plat->bsp_priv);
8028	}
8029	EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
8030
8031	/**
8032	* stmmac_suspend - suspend callback
8033	* @dev: device pointer
8034	* Description: this is the function to suspend the device and it is called
8035	* by the platform driver to stop the network queue, release the resources,
8036	* program the PMT register (for WoL), clean and release driver resources.
8037	*/
8038	int stmmac_suspend(struct device *dev)
8039	{
8040	struct net_device *ndev = dev_get_drvdata(dev);
8041	struct stmmac_priv *priv = netdev_priv(dev: ndev);
8042	u32 chan;
8043
8044	if (!ndev || !netif_running(dev: ndev))
8045	goto suspend_bsp;
8046
8047	mutex_lock(&priv->lock);
8048
8049	netif_device_detach(dev: ndev);
8050
8051	stmmac_disable_all_queues(priv);
8052
8053	for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
8054	hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer);
8055
8056	if (priv->eee_sw_timer_en) {
8057	priv->tx_path_in_lpi_mode = false;
8058	timer_delete_sync(timer: &priv->eee_ctrl_timer);
8059	}
8060
8061	/* Stop TX/RX DMA */
8062	stmmac_stop_all_dma(priv);
8063
8064	if (priv->plat->serdes_powerdown)
8065	priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
8066
8067	/* Enable Power down mode by programming the PMT regs */
8068	if (priv->wolopts) {
8069	stmmac_pmt(priv, priv->hw, priv->wolopts);
8070	priv->irq_wake = 1;
8071	} else {
8072	stmmac_mac_set(priv, priv->ioaddr, false);
8073	pinctrl_pm_select_sleep_state(dev: priv->device);
8074	}
8075
8076	mutex_unlock(lock: &priv->lock);
8077
8078	rtnl_lock();
8079	phylink_suspend(pl: priv->phylink, mac_wol: !!priv->wolopts);
8080	rtnl_unlock();
8081
8082	if (stmmac_fpe_supported(priv))
8083	ethtool_mmsv_stop(mmsv: &priv->fpe_cfg.mmsv);
8084
8085	suspend_bsp:
8086	if (priv->plat->suspend)
8087	return priv->plat->suspend(dev, priv->plat->bsp_priv);
8088
8089	return 0;
8090	}
8091	EXPORT_SYMBOL_GPL(stmmac_suspend);
8092
8093	static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue)
8094	{
8095	struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue];
8096
8097	rx_q->cur_rx = 0;
8098	rx_q->dirty_rx = 0;
8099	}
8100
8101	static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue)
8102	{
8103	struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue];
8104
8105	tx_q->cur_tx = 0;
8106	tx_q->dirty_tx = 0;
8107	tx_q->mss = 0;
8108
8109	netdev_tx_reset_queue(q: netdev_get_tx_queue(dev: priv->dev, index: queue));
8110	}
8111
8112	/**
8113	* stmmac_reset_queues_param - reset queue parameters
8114	* @priv: device pointer
8115	*/
8116	static void stmmac_reset_queues_param(struct stmmac_priv *priv)
8117	{
8118	u32 rx_cnt = priv->plat->rx_queues_to_use;
8119	u32 tx_cnt = priv->plat->tx_queues_to_use;
8120	u32 queue;
8121
8122	for (queue = 0; queue < rx_cnt; queue++)
8123	stmmac_reset_rx_queue(priv, queue);
8124
8125	for (queue = 0; queue < tx_cnt; queue++)
8126	stmmac_reset_tx_queue(priv, queue);
8127	}
8128
8129	/**
8130	* stmmac_resume - resume callback
8131	* @dev: device pointer
8132	* Description: when resume this function is invoked to setup the DMA and CORE
8133	* in a usable state.
8134	*/
8135	int stmmac_resume(struct device *dev)
8136	{
8137	struct net_device *ndev = dev_get_drvdata(dev);
8138	struct stmmac_priv *priv = netdev_priv(dev: ndev);
8139	int ret;
8140
8141	if (priv->plat->resume) {
8142	ret = priv->plat->resume(dev, priv->plat->bsp_priv);
8143	if (ret)
8144	return ret;
8145	}
8146
8147	if (!netif_running(dev: ndev))
8148	return 0;
8149
8150	/* Power Down bit, into the PM register, is cleared
8151	* automatically as soon as a magic packet or a Wake-up frame
8152	* is received. Anyway, it's better to manually clear
8153	* this bit because it can generate problems while resuming
8154	* from another devices (e.g. serial console).
8155	*/
8156	if (priv->wolopts) {
8157	mutex_lock(&priv->lock);
8158	stmmac_pmt(priv, priv->hw, 0);
8159	mutex_unlock(lock: &priv->lock);
8160	priv->irq_wake = 0;
8161	} else {
8162	pinctrl_pm_select_default_state(dev: priv->device);
8163	/* reset the phy so that it's ready */
8164	if (priv->mii)
8165	stmmac_mdio_reset(mii: priv->mii);
8166	}
8167
8168	if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) &&
8169	priv->plat->serdes_powerup) {
8170	ret = priv->plat->serdes_powerup(ndev,
8171	priv->plat->bsp_priv);
8172
8173	if (ret < 0)
8174	return ret;
8175	}
8176
8177	rtnl_lock();
8178
8179	/* Prepare the PHY to resume, ensuring that its clocks which are
8180	* necessary for the MAC DMA reset to complete are running
8181	*/
8182	phylink_prepare_resume(pl: priv->phylink);
8183
8184	mutex_lock(&priv->lock);
8185
8186	stmmac_reset_queues_param(priv);
8187
8188	stmmac_free_tx_skbufs(priv);
8189	stmmac_clear_descriptors(priv, dma_conf: &priv->dma_conf);
8190
8191	ret = stmmac_hw_setup(dev: ndev);
8192	if (ret < 0) {
8193	netdev_err(dev: priv->dev, format: "%s: Hw setup failed\n", __func__);
8194	mutex_unlock(lock: &priv->lock);
8195	rtnl_unlock();
8196	return ret;
8197	}
8198
8199	stmmac_init_timestamping(priv);
8200
8201	stmmac_init_coalesce(priv);
8202	phylink_rx_clk_stop_block(priv->phylink);
8203	stmmac_set_rx_mode(dev: ndev);
8204
8205	stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);
8206	phylink_rx_clk_stop_unblock(priv->phylink);
8207
8208	stmmac_enable_all_queues(priv);
8209	stmmac_enable_all_dma_irq(priv);
8210
8211	mutex_unlock(lock: &priv->lock);
8212
8213	/* phylink_resume() must be called after the hardware has been
8214	* initialised because it may bring the link up immediately in a
8215	* workqueue thread, which will race with initialisation.
8216	*/
8217	phylink_resume(pl: priv->phylink);
8218	rtnl_unlock();
8219
8220	netif_device_attach(dev: ndev);
8221
8222	return 0;
8223	}
8224	EXPORT_SYMBOL_GPL(stmmac_resume);
8225
8226	/* This is not the same as EXPORT_GPL_SIMPLE_DEV_PM_OPS() when CONFIG_PM=n */
8227	DEFINE_SIMPLE_DEV_PM_OPS(stmmac_simple_pm_ops, stmmac_suspend, stmmac_resume);
8228	EXPORT_SYMBOL_GPL(stmmac_simple_pm_ops);
8229
8230	#ifndef MODULE
8231	static int __init stmmac_cmdline_opt(char *str)
8232	{
8233	char *opt;
8234
8235	if (!str || !*str)
8236	return 1;
8237	while ((opt = strsep(&str, ",")) != NULL) {
8238	if (!strncmp(opt, "debug:", 6)) {
8239	if (kstrtoint(s: opt + 6, base: 0, res: &debug))
8240	goto err;
8241	} else if (!strncmp(opt, "phyaddr:", 8)) {
8242	if (kstrtoint(s: opt + 8, base: 0, res: &phyaddr))
8243	goto err;
8244	} else if (!strncmp(opt, "tc:", 3)) {
8245	if (kstrtoint(s: opt + 3, base: 0, res: &tc))
8246	goto err;
8247	} else if (!strncmp(opt, "watchdog:", 9)) {
8248	if (kstrtoint(s: opt + 9, base: 0, res: &watchdog))
8249	goto err;
8250	} else if (!strncmp(opt, "flow_ctrl:", 10)) {
8251	if (kstrtoint(s: opt + 10, base: 0, res: &flow_ctrl))
8252	goto err;
8253	} else if (!strncmp(opt, "pause:", 6)) {
8254	if (kstrtoint(s: opt + 6, base: 0, res: &pause))
8255	goto err;
8256	} else if (!strncmp(opt, "eee_timer:", 10)) {
8257	if (kstrtoint(s: opt + 10, base: 0, res: &eee_timer))
8258	goto err;
8259	} else if (!strncmp(opt, "chain_mode:", 11)) {
8260	if (kstrtoint(s: opt + 11, base: 0, res: &chain_mode))
8261	goto err;
8262	}
8263	}
8264	return 1;
8265
8266	err:
8267	pr_err("%s: ERROR broken module parameter conversion", __func__);
8268	return 1;
8269	}
8270
8271	__setup("stmmaceth=", stmmac_cmdline_opt);
8272	#endif /* MODULE */
8273
8274	static int __init stmmac_init(void)
8275	{
8276	#ifdef CONFIG_DEBUG_FS
8277	/* Create debugfs main directory if it doesn't exist yet */
8278	if (!stmmac_fs_dir)
8279	stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
8280	register_netdevice_notifier(nb: &stmmac_notifier);
8281	#endif
8282
8283	return 0;
8284	}
8285
8286	static void __exit stmmac_exit(void)
8287	{
8288	#ifdef CONFIG_DEBUG_FS
8289	unregister_netdevice_notifier(nb: &stmmac_notifier);
8290	debugfs_remove_recursive(dentry: stmmac_fs_dir);
8291	#endif
8292	}
8293
8294	module_init(stmmac_init)
8295	module_exit(stmmac_exit)
8296
8297	MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
8298	MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
8299	MODULE_LICENSE("GPL");
8300