1 | // SPDX-License-Identifier: GPL-2.0-only |
---|---|
2 | /******************************************************************************* |
3 | This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. |
4 | ST Ethernet IPs are built around a Synopsys IP Core. |
5 | |
6 | Copyright(C) 2007-2011 STMicroelectronics Ltd |
7 | |
8 | |
9 | Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> |
10 | |
11 | Documentation available at: |
12 | http://www.stlinux.com |
13 | Support available at: |
14 | https://bugzilla.stlinux.com/ |
15 | *******************************************************************************/ |
16 | |
17 | #include <linux/clk.h> |
18 | #include <linux/kernel.h> |
19 | #include <linux/interrupt.h> |
20 | #include <linux/ip.h> |
21 | #include <linux/tcp.h> |
22 | #include <linux/skbuff.h> |
23 | #include <linux/ethtool.h> |
24 | #include <linux/if_ether.h> |
25 | #include <linux/crc32.h> |
26 | #include <linux/mii.h> |
27 | #include <linux/if.h> |
28 | #include <linux/if_vlan.h> |
29 | #include <linux/dma-mapping.h> |
30 | #include <linux/slab.h> |
31 | #include <linux/pm_runtime.h> |
32 | #include <linux/prefetch.h> |
33 | #include <linux/pinctrl/consumer.h> |
34 | #ifdef CONFIG_DEBUG_FS |
35 | #include <linux/debugfs.h> |
36 | #include <linux/seq_file.h> |
37 | #endif /* CONFIG_DEBUG_FS */ |
38 | #include <linux/net_tstamp.h> |
39 | #include <linux/phylink.h> |
40 | #include <linux/udp.h> |
41 | #include <linux/bpf_trace.h> |
42 | #include <net/page_pool/helpers.h> |
43 | #include <net/pkt_cls.h> |
44 | #include <net/xdp_sock_drv.h> |
45 | #include "stmmac_ptp.h" |
46 | #include "stmmac_fpe.h" |
47 | #include "stmmac.h" |
48 | #include "stmmac_xdp.h" |
49 | #include <linux/reset.h> |
50 | #include <linux/of_mdio.h> |
51 | #include "dwmac1000.h" |
52 | #include "dwxgmac2.h" |
53 | #include "hwif.h" |
54 | |
55 | /* As long as the interface is active, we keep the timestamping counter enabled |
56 | * with fine resolution and binary rollover. This avoid non-monotonic behavior |
57 | * (clock jumps) when changing timestamping settings at runtime. |
58 | */ |
59 | #define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | \ |
60 | PTP_TCR_TSCTRLSSR) |
61 | |
62 | #define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) |
63 | #define TSO_MAX_BUFF_SIZE (SZ_16K - 1) |
64 | |
65 | /* Module parameters */ |
66 | #define TX_TIMEO 5000 |
67 | static int watchdog = TX_TIMEO; |
68 | module_param(watchdog, int, 0644); |
69 | MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); |
70 | |
71 | static int debug = -1; |
72 | module_param(debug, int, 0644); |
73 | MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); |
74 | |
75 | static int phyaddr = -1; |
76 | module_param(phyaddr, int, 0444); |
77 | MODULE_PARM_DESC(phyaddr, "Physical device address"); |
78 | |
79 | #define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) |
80 | |
81 | /* Limit to make sure XDP TX and slow path can coexist */ |
82 | #define STMMAC_XSK_TX_BUDGET_MAX 256 |
83 | #define STMMAC_TX_XSK_AVAIL 16 |
84 | #define STMMAC_RX_FILL_BATCH 16 |
85 | |
86 | #define STMMAC_XDP_PASS 0 |
87 | #define STMMAC_XDP_CONSUMED BIT(0) |
88 | #define STMMAC_XDP_TX BIT(1) |
89 | #define STMMAC_XDP_REDIRECT BIT(2) |
90 | |
91 | static int flow_ctrl = 0xdead; |
92 | module_param(flow_ctrl, int, 0644); |
93 | MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off] (obsolete)"); |
94 | |
95 | static int pause = PAUSE_TIME; |
96 | module_param(pause, int, 0644); |
97 | MODULE_PARM_DESC(pause, "Flow Control Pause Time (units of 512 bit times)"); |
98 | |
99 | #define TC_DEFAULT 64 |
100 | static int tc = TC_DEFAULT; |
101 | module_param(tc, int, 0644); |
102 | MODULE_PARM_DESC(tc, "DMA threshold control value"); |
103 | |
104 | /* This is unused */ |
105 | #define DEFAULT_BUFSIZE 1536 |
106 | static int buf_sz = DEFAULT_BUFSIZE; |
107 | module_param(buf_sz, int, 0644); |
108 | MODULE_PARM_DESC(buf_sz, "DMA buffer size"); |
109 | |
110 | static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | |
111 | NETIF_MSG_LINK | NETIF_MSG_IFUP | |
112 | NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); |
113 | |
114 | #define STMMAC_DEFAULT_LPI_TIMER 1000 |
115 | static unsigned int eee_timer = STMMAC_DEFAULT_LPI_TIMER; |
116 | module_param(eee_timer, uint, 0644); |
117 | MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); |
118 | #define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) |
119 | |
120 | /* By default the driver will use the ring mode to manage tx and rx descriptors, |
121 | * but allow user to force to use the chain instead of the ring |
122 | */ |
123 | static unsigned int chain_mode; |
124 | module_param(chain_mode, int, 0444); |
125 | MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); |
126 | |
127 | static irqreturn_t stmmac_interrupt(int irq, void *dev_id); |
128 | /* For MSI interrupts handling */ |
129 | static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); |
130 | static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); |
131 | static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); |
132 | static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); |
133 | static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); |
134 | static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); |
135 | static void stmmac_reset_queues_param(struct stmmac_priv *priv); |
136 | static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); |
137 | static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); |
138 | static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, |
139 | u32 rxmode, u32 chan); |
140 | |
141 | #ifdef CONFIG_DEBUG_FS |
142 | static const struct net_device_ops stmmac_netdev_ops; |
143 | static void stmmac_init_fs(struct net_device *dev); |
144 | static void stmmac_exit_fs(struct net_device *dev); |
145 | #endif |
146 | |
147 | #define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) |
148 | |
149 | int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled) |
150 | { |
151 | int ret = 0; |
152 | |
153 | if (enabled) { |
154 | ret = clk_prepare_enable(clk: priv->plat->stmmac_clk); |
155 | if (ret) |
156 | return ret; |
157 | ret = clk_prepare_enable(clk: priv->plat->pclk); |
158 | if (ret) { |
159 | clk_disable_unprepare(clk: priv->plat->stmmac_clk); |
160 | return ret; |
161 | } |
162 | if (priv->plat->clks_config) { |
163 | ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled); |
164 | if (ret) { |
165 | clk_disable_unprepare(clk: priv->plat->stmmac_clk); |
166 | clk_disable_unprepare(clk: priv->plat->pclk); |
167 | return ret; |
168 | } |
169 | } |
170 | } else { |
171 | clk_disable_unprepare(clk: priv->plat->stmmac_clk); |
172 | clk_disable_unprepare(clk: priv->plat->pclk); |
173 | if (priv->plat->clks_config) |
174 | priv->plat->clks_config(priv->plat->bsp_priv, enabled); |
175 | } |
176 | |
177 | return ret; |
178 | } |
179 | EXPORT_SYMBOL_GPL(stmmac_bus_clks_config); |
180 | |
181 | /** |
182 | * stmmac_set_clk_tx_rate() - set the clock rate for the MAC transmit clock |
183 | * @bsp_priv: BSP private data structure (unused) |
184 | * @clk_tx_i: the transmit clock |
185 | * @interface: the selected interface mode |
186 | * @speed: the speed that the MAC will be operating at |
187 | * |
188 | * Set the transmit clock rate for the MAC, normally 2.5MHz for 10Mbps, |
189 | * 25MHz for 100Mbps and 125MHz for 1Gbps. This is suitable for at least |
190 | * MII, GMII, RGMII and RMII interface modes. Platforms can hook this into |
191 | * the plat_data->set_clk_tx_rate method directly, call it via their own |
192 | * implementation, or implement their own method should they have more |
193 | * complex requirements. It is intended to only be used in this method. |
194 | * |
195 | * plat_data->clk_tx_i must be filled in. |
196 | */ |
197 | int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i, |
198 | phy_interface_t interface, int speed) |
199 | { |
200 | long rate = rgmii_clock(speed); |
201 | |
202 | /* Silently ignore unsupported speeds as rgmii_clock() only |
203 | * supports 10, 100 and 1000Mbps. We do not want to spit |
204 | * errors for 2500 and higher speeds here. |
205 | */ |
206 | if (rate < 0) |
207 | return 0; |
208 | |
209 | return clk_set_rate(clk: clk_tx_i, rate); |
210 | } |
211 | EXPORT_SYMBOL_GPL(stmmac_set_clk_tx_rate); |
212 | |
213 | /** |
214 | * stmmac_verify_args - verify the driver parameters. |
215 | * Description: it checks the driver parameters and set a default in case of |
216 | * errors. |
217 | */ |
218 | static void stmmac_verify_args(void) |
219 | { |
220 | if (unlikely(watchdog < 0)) |
221 | watchdog = TX_TIMEO; |
222 | if (unlikely((pause < 0) || (pause > 0xffff))) |
223 | pause = PAUSE_TIME; |
224 | |
225 | if (flow_ctrl != 0xdead) |
226 | pr_warn("stmmac: module parameter 'flow_ctrl' is obsolete - please remove from your module configuration\n"); |
227 | } |
228 | |
229 | static void __stmmac_disable_all_queues(struct stmmac_priv *priv) |
230 | { |
231 | u32 rx_queues_cnt = priv->plat->rx_queues_to_use; |
232 | u32 tx_queues_cnt = priv->plat->tx_queues_to_use; |
233 | u32 maxq = max(rx_queues_cnt, tx_queues_cnt); |
234 | u32 queue; |
235 | |
236 | for (queue = 0; queue < maxq; queue++) { |
237 | struct stmmac_channel *ch = &priv->channel[queue]; |
238 | |
239 | if (stmmac_xdp_is_enabled(priv) && |
240 | test_bit(queue, priv->af_xdp_zc_qps)) { |
241 | napi_disable(n: &ch->rxtx_napi); |
242 | continue; |
243 | } |
244 | |
245 | if (queue < rx_queues_cnt) |
246 | napi_disable(n: &ch->rx_napi); |
247 | if (queue < tx_queues_cnt) |
248 | napi_disable(n: &ch->tx_napi); |
249 | } |
250 | } |
251 | |
252 | /** |
253 | * stmmac_disable_all_queues - Disable all queues |
254 | * @priv: driver private structure |
255 | */ |
256 | static void stmmac_disable_all_queues(struct stmmac_priv *priv) |
257 | { |
258 | u32 rx_queues_cnt = priv->plat->rx_queues_to_use; |
259 | struct stmmac_rx_queue *rx_q; |
260 | u32 queue; |
261 | |
262 | /* synchronize_rcu() needed for pending XDP buffers to drain */ |
263 | for (queue = 0; queue < rx_queues_cnt; queue++) { |
264 | rx_q = &priv->dma_conf.rx_queue[queue]; |
265 | if (rx_q->xsk_pool) { |
266 | synchronize_rcu(); |
267 | break; |
268 | } |
269 | } |
270 | |
271 | __stmmac_disable_all_queues(priv); |
272 | } |
273 | |
274 | /** |
275 | * stmmac_enable_all_queues - Enable all queues |
276 | * @priv: driver private structure |
277 | */ |
278 | static void stmmac_enable_all_queues(struct stmmac_priv *priv) |
279 | { |
280 | u32 rx_queues_cnt = priv->plat->rx_queues_to_use; |
281 | u32 tx_queues_cnt = priv->plat->tx_queues_to_use; |
282 | u32 maxq = max(rx_queues_cnt, tx_queues_cnt); |
283 | u32 queue; |
284 | |
285 | for (queue = 0; queue < maxq; queue++) { |
286 | struct stmmac_channel *ch = &priv->channel[queue]; |
287 | |
288 | if (stmmac_xdp_is_enabled(priv) && |
289 | test_bit(queue, priv->af_xdp_zc_qps)) { |
290 | napi_enable(n: &ch->rxtx_napi); |
291 | continue; |
292 | } |
293 | |
294 | if (queue < rx_queues_cnt) |
295 | napi_enable(n: &ch->rx_napi); |
296 | if (queue < tx_queues_cnt) |
297 | napi_enable(n: &ch->tx_napi); |
298 | } |
299 | } |
300 | |
301 | static void stmmac_service_event_schedule(struct stmmac_priv *priv) |
302 | { |
303 | if (!test_bit(STMMAC_DOWN, &priv->state) && |
304 | !test_and_set_bit(nr: STMMAC_SERVICE_SCHED, addr: &priv->state)) |
305 | queue_work(wq: priv->wq, work: &priv->service_task); |
306 | } |
307 | |
308 | static void stmmac_global_err(struct stmmac_priv *priv) |
309 | { |
310 | netif_carrier_off(dev: priv->dev); |
311 | set_bit(nr: STMMAC_RESET_REQUESTED, addr: &priv->state); |
312 | stmmac_service_event_schedule(priv); |
313 | } |
314 | |
315 | /** |
316 | * stmmac_clk_csr_set - dynamically set the MDC clock |
317 | * @priv: driver private structure |
318 | * Description: this is to dynamically set the MDC clock according to the csr |
319 | * clock input. |
320 | * Note: |
321 | * If a specific clk_csr value is passed from the platform |
322 | * this means that the CSR Clock Range selection cannot be |
323 | * changed at run-time and it is fixed (as reported in the driver |
324 | * documentation). Viceversa the driver will try to set the MDC |
325 | * clock dynamically according to the actual clock input. |
326 | */ |
327 | static void stmmac_clk_csr_set(struct stmmac_priv *priv) |
328 | { |
329 | unsigned long clk_rate; |
330 | |
331 | clk_rate = clk_get_rate(clk: priv->plat->stmmac_clk); |
332 | |
333 | /* Platform provided default clk_csr would be assumed valid |
334 | * for all other cases except for the below mentioned ones. |
335 | * For values higher than the IEEE 802.3 specified frequency |
336 | * we can not estimate the proper divider as it is not known |
337 | * the frequency of clk_csr_i. So we do not change the default |
338 | * divider. |
339 | */ |
340 | if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) { |
341 | if (clk_rate < CSR_F_35M) |
342 | priv->clk_csr = STMMAC_CSR_20_35M; |
343 | else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M)) |
344 | priv->clk_csr = STMMAC_CSR_35_60M; |
345 | else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M)) |
346 | priv->clk_csr = STMMAC_CSR_60_100M; |
347 | else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M)) |
348 | priv->clk_csr = STMMAC_CSR_100_150M; |
349 | else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M)) |
350 | priv->clk_csr = STMMAC_CSR_150_250M; |
351 | else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M)) |
352 | priv->clk_csr = STMMAC_CSR_250_300M; |
353 | else if ((clk_rate >= CSR_F_300M) && (clk_rate < CSR_F_500M)) |
354 | priv->clk_csr = STMMAC_CSR_300_500M; |
355 | else if ((clk_rate >= CSR_F_500M) && (clk_rate < CSR_F_800M)) |
356 | priv->clk_csr = STMMAC_CSR_500_800M; |
357 | } |
358 | |
359 | if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { |
360 | if (clk_rate > 160000000) |
361 | priv->clk_csr = 0x03; |
362 | else if (clk_rate > 80000000) |
363 | priv->clk_csr = 0x02; |
364 | else if (clk_rate > 40000000) |
365 | priv->clk_csr = 0x01; |
366 | else |
367 | priv->clk_csr = 0; |
368 | } |
369 | |
370 | if (priv->plat->has_xgmac) { |
371 | if (clk_rate > 400000000) |
372 | priv->clk_csr = 0x5; |
373 | else if (clk_rate > 350000000) |
374 | priv->clk_csr = 0x4; |
375 | else if (clk_rate > 300000000) |
376 | priv->clk_csr = 0x3; |
377 | else if (clk_rate > 250000000) |
378 | priv->clk_csr = 0x2; |
379 | else if (clk_rate > 150000000) |
380 | priv->clk_csr = 0x1; |
381 | else |
382 | priv->clk_csr = 0x0; |
383 | } |
384 | } |
385 | |
386 | static void print_pkt(unsigned char *buf, int len) |
387 | { |
388 | pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); |
389 | print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); |
390 | } |
391 | |
392 | static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) |
393 | { |
394 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
395 | u32 avail; |
396 | |
397 | if (tx_q->dirty_tx > tx_q->cur_tx) |
398 | avail = tx_q->dirty_tx - tx_q->cur_tx - 1; |
399 | else |
400 | avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; |
401 | |
402 | return avail; |
403 | } |
404 | |
405 | /** |
406 | * stmmac_rx_dirty - Get RX queue dirty |
407 | * @priv: driver private structure |
408 | * @queue: RX queue index |
409 | */ |
410 | static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) |
411 | { |
412 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; |
413 | u32 dirty; |
414 | |
415 | if (rx_q->dirty_rx <= rx_q->cur_rx) |
416 | dirty = rx_q->cur_rx - rx_q->dirty_rx; |
417 | else |
418 | dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; |
419 | |
420 | return dirty; |
421 | } |
422 | |
423 | static bool stmmac_eee_tx_busy(struct stmmac_priv *priv) |
424 | { |
425 | u32 tx_cnt = priv->plat->tx_queues_to_use; |
426 | u32 queue; |
427 | |
428 | /* check if all TX queues have the work finished */ |
429 | for (queue = 0; queue < tx_cnt; queue++) { |
430 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
431 | |
432 | if (tx_q->dirty_tx != tx_q->cur_tx) |
433 | return true; /* still unfinished work */ |
434 | } |
435 | |
436 | return false; |
437 | } |
438 | |
439 | static void stmmac_restart_sw_lpi_timer(struct stmmac_priv *priv) |
440 | { |
441 | mod_timer(timer: &priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); |
442 | } |
443 | |
444 | /** |
445 | * stmmac_try_to_start_sw_lpi - check and enter in LPI mode |
446 | * @priv: driver private structure |
447 | * Description: this function is to verify and enter in LPI mode in case of |
448 | * EEE. |
449 | */ |
450 | static void stmmac_try_to_start_sw_lpi(struct stmmac_priv *priv) |
451 | { |
452 | if (stmmac_eee_tx_busy(priv)) { |
453 | stmmac_restart_sw_lpi_timer(priv); |
454 | return; |
455 | } |
456 | |
457 | /* Check and enter in LPI mode */ |
458 | if (!priv->tx_path_in_lpi_mode) |
459 | stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_FORCED, |
460 | priv->tx_lpi_clk_stop, 0); |
461 | } |
462 | |
463 | /** |
464 | * stmmac_stop_sw_lpi - stop transmitting LPI |
465 | * @priv: driver private structure |
466 | * Description: When using software-controlled LPI, stop transmitting LPI state. |
467 | */ |
468 | static void stmmac_stop_sw_lpi(struct stmmac_priv *priv) |
469 | { |
470 | timer_delete_sync(timer: &priv->eee_ctrl_timer); |
471 | stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); |
472 | priv->tx_path_in_lpi_mode = false; |
473 | } |
474 | |
475 | /** |
476 | * stmmac_eee_ctrl_timer - EEE TX SW timer. |
477 | * @t: timer_list struct containing private info |
478 | * Description: |
479 | * if there is no data transfer and if we are not in LPI state, |
480 | * then MAC Transmitter can be moved to LPI state. |
481 | */ |
482 | static void stmmac_eee_ctrl_timer(struct timer_list *t) |
483 | { |
484 | struct stmmac_priv *priv = timer_container_of(priv, t, eee_ctrl_timer); |
485 | |
486 | stmmac_try_to_start_sw_lpi(priv); |
487 | } |
488 | |
489 | /* stmmac_get_tx_hwtstamp - get HW TX timestamps |
490 | * @priv: driver private structure |
491 | * @p : descriptor pointer |
492 | * @skb : the socket buffer |
493 | * Description : |
494 | * This function will read timestamp from the descriptor & pass it to stack. |
495 | * and also perform some sanity checks. |
496 | */ |
497 | static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, |
498 | struct dma_desc *p, struct sk_buff *skb) |
499 | { |
500 | struct skb_shared_hwtstamps shhwtstamp; |
501 | bool found = false; |
502 | u64 ns = 0; |
503 | |
504 | if (!priv->hwts_tx_en) |
505 | return; |
506 | |
507 | /* exit if skb doesn't support hw tstamp */ |
508 | if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) |
509 | return; |
510 | |
511 | /* check tx tstamp status */ |
512 | if (stmmac_get_tx_timestamp_status(priv, p)) { |
513 | stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); |
514 | found = true; |
515 | } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { |
516 | found = true; |
517 | } |
518 | |
519 | if (found) { |
520 | ns -= priv->plat->cdc_error_adj; |
521 | |
522 | memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); |
523 | shhwtstamp.hwtstamp = ns_to_ktime(ns); |
524 | |
525 | netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); |
526 | /* pass tstamp to stack */ |
527 | skb_tstamp_tx(orig_skb: skb, hwtstamps: &shhwtstamp); |
528 | } |
529 | } |
530 | |
531 | /* stmmac_get_rx_hwtstamp - get HW RX timestamps |
532 | * @priv: driver private structure |
533 | * @p : descriptor pointer |
534 | * @np : next descriptor pointer |
535 | * @skb : the socket buffer |
536 | * Description : |
537 | * This function will read received packet's timestamp from the descriptor |
538 | * and pass it to stack. It also perform some sanity checks. |
539 | */ |
540 | static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, |
541 | struct dma_desc *np, struct sk_buff *skb) |
542 | { |
543 | struct skb_shared_hwtstamps *shhwtstamp = NULL; |
544 | struct dma_desc *desc = p; |
545 | u64 ns = 0; |
546 | |
547 | if (!priv->hwts_rx_en) |
548 | return; |
549 | /* For GMAC4, the valid timestamp is from CTX next desc. */ |
550 | if (priv->plat->has_gmac4 || priv->plat->has_xgmac) |
551 | desc = np; |
552 | |
553 | /* Check if timestamp is available */ |
554 | if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { |
555 | stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); |
556 | |
557 | ns -= priv->plat->cdc_error_adj; |
558 | |
559 | netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); |
560 | shhwtstamp = skb_hwtstamps(skb); |
561 | memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); |
562 | shhwtstamp->hwtstamp = ns_to_ktime(ns); |
563 | } else { |
564 | netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); |
565 | } |
566 | } |
567 | |
568 | /** |
569 | * stmmac_hwtstamp_set - control hardware timestamping. |
570 | * @dev: device pointer. |
571 | * @config: the timestamping configuration. |
572 | * @extack: netlink extended ack structure for error reporting. |
573 | * Description: |
574 | * This function configures the MAC to enable/disable both outgoing(TX) |
575 | * and incoming(RX) packets time stamping based on user input. |
576 | * Return Value: |
577 | * 0 on success and an appropriate -ve integer on failure. |
578 | */ |
579 | static int stmmac_hwtstamp_set(struct net_device *dev, |
580 | struct kernel_hwtstamp_config *config, |
581 | struct netlink_ext_ack *extack) |
582 | { |
583 | struct stmmac_priv *priv = netdev_priv(dev); |
584 | u32 ptp_v2 = 0; |
585 | u32 tstamp_all = 0; |
586 | u32 ptp_over_ipv4_udp = 0; |
587 | u32 ptp_over_ipv6_udp = 0; |
588 | u32 ptp_over_ethernet = 0; |
589 | u32 snap_type_sel = 0; |
590 | u32 ts_master_en = 0; |
591 | u32 ts_event_en = 0; |
592 | |
593 | if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { |
594 | NL_SET_ERR_MSG_MOD(extack, "No support for HW time stamping"); |
595 | priv->hwts_tx_en = 0; |
596 | priv->hwts_rx_en = 0; |
597 | |
598 | return -EOPNOTSUPP; |
599 | } |
600 | |
601 | if (!netif_running(dev)) { |
602 | NL_SET_ERR_MSG_MOD(extack, |
603 | "Cannot change timestamping configuration while down"); |
604 | return -ENODEV; |
605 | } |
606 | |
607 | netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", |
608 | __func__, config->flags, config->tx_type, config->rx_filter); |
609 | |
610 | if (config->tx_type != HWTSTAMP_TX_OFF && |
611 | config->tx_type != HWTSTAMP_TX_ON) |
612 | return -ERANGE; |
613 | |
614 | if (priv->adv_ts) { |
615 | switch (config->rx_filter) { |
616 | case HWTSTAMP_FILTER_NONE: |
617 | /* time stamp no incoming packet at all */ |
618 | config->rx_filter = HWTSTAMP_FILTER_NONE; |
619 | break; |
620 | |
621 | case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: |
622 | /* PTP v1, UDP, any kind of event packet */ |
623 | config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; |
624 | /* 'xmac' hardware can support Sync, Pdelay_Req and |
625 | * Pdelay_resp by setting bit14 and bits17/16 to 01 |
626 | * This leaves Delay_Req timestamps out. |
627 | * Enable all events *and* general purpose message |
628 | * timestamping |
629 | */ |
630 | snap_type_sel = PTP_TCR_SNAPTYPSEL_1; |
631 | ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; |
632 | ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; |
633 | break; |
634 | |
635 | case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: |
636 | /* PTP v1, UDP, Sync packet */ |
637 | config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; |
638 | /* take time stamp for SYNC messages only */ |
639 | ts_event_en = PTP_TCR_TSEVNTENA; |
640 | |
641 | ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; |
642 | ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; |
643 | break; |
644 | |
645 | case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: |
646 | /* PTP v1, UDP, Delay_req packet */ |
647 | config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; |
648 | /* take time stamp for Delay_Req messages only */ |
649 | ts_master_en = PTP_TCR_TSMSTRENA; |
650 | ts_event_en = PTP_TCR_TSEVNTENA; |
651 | |
652 | ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; |
653 | ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; |
654 | break; |
655 | |
656 | case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: |
657 | /* PTP v2, UDP, any kind of event packet */ |
658 | config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; |
659 | ptp_v2 = PTP_TCR_TSVER2ENA; |
660 | /* take time stamp for all event messages */ |
661 | snap_type_sel = PTP_TCR_SNAPTYPSEL_1; |
662 | |
663 | ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; |
664 | ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; |
665 | break; |
666 | |
667 | case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: |
668 | /* PTP v2, UDP, Sync packet */ |
669 | config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; |
670 | ptp_v2 = PTP_TCR_TSVER2ENA; |
671 | /* take time stamp for SYNC messages only */ |
672 | ts_event_en = PTP_TCR_TSEVNTENA; |
673 | |
674 | ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; |
675 | ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; |
676 | break; |
677 | |
678 | case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: |
679 | /* PTP v2, UDP, Delay_req packet */ |
680 | config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; |
681 | ptp_v2 = PTP_TCR_TSVER2ENA; |
682 | /* take time stamp for Delay_Req messages only */ |
683 | ts_master_en = PTP_TCR_TSMSTRENA; |
684 | ts_event_en = PTP_TCR_TSEVNTENA; |
685 | |
686 | ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; |
687 | ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; |
688 | break; |
689 | |
690 | case HWTSTAMP_FILTER_PTP_V2_EVENT: |
691 | /* PTP v2/802.AS1 any layer, any kind of event packet */ |
692 | config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; |
693 | ptp_v2 = PTP_TCR_TSVER2ENA; |
694 | snap_type_sel = PTP_TCR_SNAPTYPSEL_1; |
695 | if (priv->synopsys_id < DWMAC_CORE_4_10) |
696 | ts_event_en = PTP_TCR_TSEVNTENA; |
697 | ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; |
698 | ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; |
699 | ptp_over_ethernet = PTP_TCR_TSIPENA; |
700 | break; |
701 | |
702 | case HWTSTAMP_FILTER_PTP_V2_SYNC: |
703 | /* PTP v2/802.AS1, any layer, Sync packet */ |
704 | config->rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; |
705 | ptp_v2 = PTP_TCR_TSVER2ENA; |
706 | /* take time stamp for SYNC messages only */ |
707 | ts_event_en = PTP_TCR_TSEVNTENA; |
708 | |
709 | ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; |
710 | ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; |
711 | ptp_over_ethernet = PTP_TCR_TSIPENA; |
712 | break; |
713 | |
714 | case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: |
715 | /* PTP v2/802.AS1, any layer, Delay_req packet */ |
716 | config->rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; |
717 | ptp_v2 = PTP_TCR_TSVER2ENA; |
718 | /* take time stamp for Delay_Req messages only */ |
719 | ts_master_en = PTP_TCR_TSMSTRENA; |
720 | ts_event_en = PTP_TCR_TSEVNTENA; |
721 | |
722 | ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; |
723 | ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; |
724 | ptp_over_ethernet = PTP_TCR_TSIPENA; |
725 | break; |
726 | |
727 | case HWTSTAMP_FILTER_NTP_ALL: |
728 | case HWTSTAMP_FILTER_ALL: |
729 | /* time stamp any incoming packet */ |
730 | config->rx_filter = HWTSTAMP_FILTER_ALL; |
731 | tstamp_all = PTP_TCR_TSENALL; |
732 | break; |
733 | |
734 | default: |
735 | return -ERANGE; |
736 | } |
737 | } else { |
738 | switch (config->rx_filter) { |
739 | case HWTSTAMP_FILTER_NONE: |
740 | config->rx_filter = HWTSTAMP_FILTER_NONE; |
741 | break; |
742 | default: |
743 | /* PTP v1, UDP, any kind of event packet */ |
744 | config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; |
745 | break; |
746 | } |
747 | } |
748 | priv->hwts_rx_en = config->rx_filter != HWTSTAMP_FILTER_NONE; |
749 | priv->hwts_tx_en = config->tx_type == HWTSTAMP_TX_ON; |
750 | |
751 | priv->systime_flags = STMMAC_HWTS_ACTIVE; |
752 | |
753 | if (priv->hwts_tx_en || priv->hwts_rx_en) { |
754 | priv->systime_flags |= tstamp_all | ptp_v2 | |
755 | ptp_over_ethernet | ptp_over_ipv6_udp | |
756 | ptp_over_ipv4_udp | ts_event_en | |
757 | ts_master_en | snap_type_sel; |
758 | } |
759 | |
760 | stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); |
761 | |
762 | priv->tstamp_config = *config; |
763 | |
764 | return 0; |
765 | } |
766 | |
767 | /** |
768 | * stmmac_hwtstamp_get - read hardware timestamping. |
769 | * @dev: device pointer. |
770 | * @config: the timestamping configuration. |
771 | * Description: |
772 | * This function obtain the current hardware timestamping settings |
773 | * as requested. |
774 | */ |
775 | static int stmmac_hwtstamp_get(struct net_device *dev, |
776 | struct kernel_hwtstamp_config *config) |
777 | { |
778 | struct stmmac_priv *priv = netdev_priv(dev); |
779 | |
780 | if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) |
781 | return -EOPNOTSUPP; |
782 | |
783 | *config = priv->tstamp_config; |
784 | |
785 | return 0; |
786 | } |
787 | |
788 | /** |
789 | * stmmac_init_tstamp_counter - init hardware timestamping counter |
790 | * @priv: driver private structure |
791 | * @systime_flags: timestamping flags |
792 | * Description: |
793 | * Initialize hardware counter for packet timestamping. |
794 | * This is valid as long as the interface is open and not suspended. |
795 | * Will be rerun after resuming from suspend, case in which the timestamping |
796 | * flags updated by stmmac_hwtstamp_set() also need to be restored. |
797 | */ |
798 | int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags) |
799 | { |
800 | bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; |
801 | struct timespec64 now; |
802 | u32 sec_inc = 0; |
803 | u64 temp = 0; |
804 | |
805 | if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) |
806 | return -EOPNOTSUPP; |
807 | |
808 | if (!priv->plat->clk_ptp_rate) { |
809 | netdev_err(dev: priv->dev, format: "Invalid PTP clock rate"); |
810 | return -EINVAL; |
811 | } |
812 | |
813 | stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); |
814 | priv->systime_flags = systime_flags; |
815 | |
816 | /* program Sub Second Increment reg */ |
817 | stmmac_config_sub_second_increment(priv, priv->ptpaddr, |
818 | priv->plat->clk_ptp_rate, |
819 | xmac, &sec_inc); |
820 | temp = div_u64(dividend: 1000000000ULL, divisor: sec_inc); |
821 | |
822 | /* Store sub second increment for later use */ |
823 | priv->sub_second_inc = sec_inc; |
824 | |
825 | /* calculate default added value: |
826 | * formula is : |
827 | * addend = (2^32)/freq_div_ratio; |
828 | * where, freq_div_ratio = 1e9ns/sec_inc |
829 | */ |
830 | temp = (u64)(temp << 32); |
831 | priv->default_addend = div_u64(dividend: temp, divisor: priv->plat->clk_ptp_rate); |
832 | stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); |
833 | |
834 | /* initialize system time */ |
835 | ktime_get_real_ts64(tv: &now); |
836 | |
837 | /* lower 32 bits of tv_sec are safe until y2106 */ |
838 | stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); |
839 | |
840 | return 0; |
841 | } |
842 | EXPORT_SYMBOL_GPL(stmmac_init_tstamp_counter); |
843 | |
844 | /** |
845 | * stmmac_init_ptp - init PTP |
846 | * @priv: driver private structure |
847 | * Description: this is to verify if the HW supports the PTPv1 or PTPv2. |
848 | * This is done by looking at the HW cap. register. |
849 | * This function also registers the ptp driver. |
850 | */ |
851 | static int stmmac_init_ptp(struct stmmac_priv *priv) |
852 | { |
853 | bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; |
854 | int ret; |
855 | |
856 | if (priv->plat->ptp_clk_freq_config) |
857 | priv->plat->ptp_clk_freq_config(priv); |
858 | |
859 | ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE); |
860 | if (ret) |
861 | return ret; |
862 | |
863 | priv->adv_ts = 0; |
864 | /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ |
865 | if (xmac && priv->dma_cap.atime_stamp) |
866 | priv->adv_ts = 1; |
867 | /* Dwmac 3.x core with extend_desc can support adv_ts */ |
868 | else if (priv->extend_desc && priv->dma_cap.atime_stamp) |
869 | priv->adv_ts = 1; |
870 | |
871 | if (priv->dma_cap.time_stamp) |
872 | netdev_info(dev: priv->dev, format: "IEEE 1588-2002 Timestamp supported\n"); |
873 | |
874 | if (priv->adv_ts) |
875 | netdev_info(dev: priv->dev, |
876 | format: "IEEE 1588-2008 Advanced Timestamp supported\n"); |
877 | |
878 | priv->hwts_tx_en = 0; |
879 | priv->hwts_rx_en = 0; |
880 | |
881 | if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) |
882 | stmmac_hwtstamp_correct_latency(priv, priv); |
883 | |
884 | return 0; |
885 | } |
886 | |
887 | static void stmmac_release_ptp(struct stmmac_priv *priv) |
888 | { |
889 | clk_disable_unprepare(clk: priv->plat->clk_ptp_ref); |
890 | stmmac_ptp_unregister(priv); |
891 | } |
892 | |
893 | /** |
894 | * stmmac_mac_flow_ctrl - Configure flow control in all queues |
895 | * @priv: driver private structure |
896 | * @duplex: duplex passed to the next function |
897 | * @flow_ctrl: desired flow control modes |
898 | * Description: It is used for configuring the flow control in all queues |
899 | */ |
900 | static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex, |
901 | unsigned int flow_ctrl) |
902 | { |
903 | u32 tx_cnt = priv->plat->tx_queues_to_use; |
904 | |
905 | stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time, |
906 | tx_cnt); |
907 | } |
908 | |
909 | static unsigned long stmmac_mac_get_caps(struct phylink_config *config, |
910 | phy_interface_t interface) |
911 | { |
912 | struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); |
913 | |
914 | /* Refresh the MAC-specific capabilities */ |
915 | stmmac_mac_update_caps(priv); |
916 | |
917 | config->mac_capabilities = priv->hw->link.caps; |
918 | |
919 | if (priv->plat->max_speed) |
920 | phylink_limit_mac_speed(config, max_speed: priv->plat->max_speed); |
921 | |
922 | return config->mac_capabilities; |
923 | } |
924 | |
925 | static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, |
926 | phy_interface_t interface) |
927 | { |
928 | struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); |
929 | struct phylink_pcs *pcs; |
930 | |
931 | if (priv->plat->select_pcs) { |
932 | pcs = priv->plat->select_pcs(priv, interface); |
933 | if (!IS_ERR(ptr: pcs)) |
934 | return pcs; |
935 | } |
936 | |
937 | return NULL; |
938 | } |
939 | |
940 | static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, |
941 | const struct phylink_link_state *state) |
942 | { |
943 | /* Nothing to do, xpcs_config() handles everything */ |
944 | } |
945 | |
946 | static void stmmac_mac_link_down(struct phylink_config *config, |
947 | unsigned int mode, phy_interface_t interface) |
948 | { |
949 | struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); |
950 | |
951 | stmmac_mac_set(priv, priv->ioaddr, false); |
952 | if (priv->dma_cap.eee) |
953 | stmmac_set_eee_pls(priv, priv->hw, false); |
954 | |
955 | if (stmmac_fpe_supported(priv)) |
956 | ethtool_mmsv_link_state_handle(mmsv: &priv->fpe_cfg.mmsv, up: false); |
957 | } |
958 | |
959 | static void stmmac_mac_link_up(struct phylink_config *config, |
960 | struct phy_device *phy, |
961 | unsigned int mode, phy_interface_t interface, |
962 | int speed, int duplex, |
963 | bool tx_pause, bool rx_pause) |
964 | { |
965 | struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); |
966 | unsigned int flow_ctrl; |
967 | u32 old_ctrl, ctrl; |
968 | int ret; |
969 | |
970 | if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && |
971 | priv->plat->serdes_powerup) |
972 | priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); |
973 | |
974 | old_ctrl = readl(addr: priv->ioaddr + MAC_CTRL_REG); |
975 | ctrl = old_ctrl & ~priv->hw->link.speed_mask; |
976 | |
977 | if (interface == PHY_INTERFACE_MODE_USXGMII) { |
978 | switch (speed) { |
979 | case SPEED_10000: |
980 | ctrl |= priv->hw->link.xgmii.speed10000; |
981 | break; |
982 | case SPEED_5000: |
983 | ctrl |= priv->hw->link.xgmii.speed5000; |
984 | break; |
985 | case SPEED_2500: |
986 | ctrl |= priv->hw->link.xgmii.speed2500; |
987 | break; |
988 | default: |
989 | return; |
990 | } |
991 | } else if (interface == PHY_INTERFACE_MODE_XLGMII) { |
992 | switch (speed) { |
993 | case SPEED_100000: |
994 | ctrl |= priv->hw->link.xlgmii.speed100000; |
995 | break; |
996 | case SPEED_50000: |
997 | ctrl |= priv->hw->link.xlgmii.speed50000; |
998 | break; |
999 | case SPEED_40000: |
1000 | ctrl |= priv->hw->link.xlgmii.speed40000; |
1001 | break; |
1002 | case SPEED_25000: |
1003 | ctrl |= priv->hw->link.xlgmii.speed25000; |
1004 | break; |
1005 | case SPEED_10000: |
1006 | ctrl |= priv->hw->link.xgmii.speed10000; |
1007 | break; |
1008 | case SPEED_2500: |
1009 | ctrl |= priv->hw->link.speed2500; |
1010 | break; |
1011 | case SPEED_1000: |
1012 | ctrl |= priv->hw->link.speed1000; |
1013 | break; |
1014 | default: |
1015 | return; |
1016 | } |
1017 | } else { |
1018 | switch (speed) { |
1019 | case SPEED_2500: |
1020 | ctrl |= priv->hw->link.speed2500; |
1021 | break; |
1022 | case SPEED_1000: |
1023 | ctrl |= priv->hw->link.speed1000; |
1024 | break; |
1025 | case SPEED_100: |
1026 | ctrl |= priv->hw->link.speed100; |
1027 | break; |
1028 | case SPEED_10: |
1029 | ctrl |= priv->hw->link.speed10; |
1030 | break; |
1031 | default: |
1032 | return; |
1033 | } |
1034 | } |
1035 | |
1036 | if (priv->plat->fix_mac_speed) |
1037 | priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); |
1038 | |
1039 | if (!duplex) |
1040 | ctrl &= ~priv->hw->link.duplex; |
1041 | else |
1042 | ctrl |= priv->hw->link.duplex; |
1043 | |
1044 | /* Flow Control operation */ |
1045 | if (rx_pause && tx_pause) |
1046 | flow_ctrl = FLOW_AUTO; |
1047 | else if (rx_pause && !tx_pause) |
1048 | flow_ctrl = FLOW_RX; |
1049 | else if (!rx_pause && tx_pause) |
1050 | flow_ctrl = FLOW_TX; |
1051 | else |
1052 | flow_ctrl = FLOW_OFF; |
1053 | |
1054 | stmmac_mac_flow_ctrl(priv, duplex, flow_ctrl); |
1055 | |
1056 | if (ctrl != old_ctrl) |
1057 | writel(val: ctrl, addr: priv->ioaddr + MAC_CTRL_REG); |
1058 | |
1059 | if (priv->plat->set_clk_tx_rate) { |
1060 | ret = priv->plat->set_clk_tx_rate(priv->plat->bsp_priv, |
1061 | priv->plat->clk_tx_i, |
1062 | interface, speed); |
1063 | if (ret < 0) |
1064 | netdev_err(dev: priv->dev, |
1065 | format: "failed to configure transmit clock for %dMbps: %pe\n", |
1066 | speed, ERR_PTR(error: ret)); |
1067 | } |
1068 | |
1069 | stmmac_mac_set(priv, priv->ioaddr, true); |
1070 | if (priv->dma_cap.eee) |
1071 | stmmac_set_eee_pls(priv, priv->hw, true); |
1072 | |
1073 | if (stmmac_fpe_supported(priv)) |
1074 | ethtool_mmsv_link_state_handle(mmsv: &priv->fpe_cfg.mmsv, up: true); |
1075 | |
1076 | if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) |
1077 | stmmac_hwtstamp_correct_latency(priv, priv); |
1078 | } |
1079 | |
1080 | static void stmmac_mac_disable_tx_lpi(struct phylink_config *config) |
1081 | { |
1082 | struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); |
1083 | |
1084 | priv->eee_active = false; |
1085 | |
1086 | mutex_lock(&priv->lock); |
1087 | |
1088 | priv->eee_enabled = false; |
1089 | |
1090 | netdev_dbg(priv->dev, "disable EEE\n"); |
1091 | priv->eee_sw_timer_en = false; |
1092 | timer_delete_sync(timer: &priv->eee_ctrl_timer); |
1093 | stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); |
1094 | priv->tx_path_in_lpi_mode = false; |
1095 | |
1096 | stmmac_set_eee_timer(priv, priv->hw, 0, STMMAC_DEFAULT_TWT_LS); |
1097 | mutex_unlock(lock: &priv->lock); |
1098 | } |
1099 | |
1100 | static int stmmac_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, |
1101 | bool tx_clk_stop) |
1102 | { |
1103 | struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); |
1104 | int ret; |
1105 | |
1106 | priv->tx_lpi_timer = timer; |
1107 | priv->eee_active = true; |
1108 | |
1109 | mutex_lock(&priv->lock); |
1110 | |
1111 | priv->eee_enabled = true; |
1112 | |
1113 | /* Update the transmit clock stop according to PHY capability if |
1114 | * the platform allows |
1115 | */ |
1116 | if (priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP) |
1117 | priv->tx_lpi_clk_stop = tx_clk_stop; |
1118 | |
1119 | stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, |
1120 | STMMAC_DEFAULT_TWT_LS); |
1121 | |
1122 | /* Try to cnfigure the hardware timer. */ |
1123 | ret = stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_TIMER, |
1124 | priv->tx_lpi_clk_stop, priv->tx_lpi_timer); |
1125 | |
1126 | if (ret) { |
1127 | /* Hardware timer mode not supported, or value out of range. |
1128 | * Fall back to using software LPI mode |
1129 | */ |
1130 | priv->eee_sw_timer_en = true; |
1131 | stmmac_restart_sw_lpi_timer(priv); |
1132 | } |
1133 | |
1134 | mutex_unlock(lock: &priv->lock); |
1135 | netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); |
1136 | |
1137 | return 0; |
1138 | } |
1139 | |
1140 | static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode, |
1141 | phy_interface_t interface) |
1142 | { |
1143 | struct net_device *ndev = to_net_dev(config->dev); |
1144 | struct stmmac_priv *priv = netdev_priv(dev: ndev); |
1145 | |
1146 | if (priv->plat->mac_finish) |
1147 | priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, interface); |
1148 | |
1149 | return 0; |
1150 | } |
1151 | |
1152 | static const struct phylink_mac_ops stmmac_phylink_mac_ops = { |
1153 | .mac_get_caps = stmmac_mac_get_caps, |
1154 | .mac_select_pcs = stmmac_mac_select_pcs, |
1155 | .mac_config = stmmac_mac_config, |
1156 | .mac_link_down = stmmac_mac_link_down, |
1157 | .mac_link_up = stmmac_mac_link_up, |
1158 | .mac_disable_tx_lpi = stmmac_mac_disable_tx_lpi, |
1159 | .mac_enable_tx_lpi = stmmac_mac_enable_tx_lpi, |
1160 | .mac_finish = stmmac_mac_finish, |
1161 | }; |
1162 | |
1163 | /** |
1164 | * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported |
1165 | * @priv: driver private structure |
1166 | * Description: this is to verify if the HW supports the PCS. |
1167 | * Physical Coding Sublayer (PCS) interface that can be used when the MAC is |
1168 | * configured for the TBI, RTBI, or SGMII PHY interface. |
1169 | */ |
1170 | static void stmmac_check_pcs_mode(struct stmmac_priv *priv) |
1171 | { |
1172 | int interface = priv->plat->mac_interface; |
1173 | |
1174 | if (priv->dma_cap.pcs) { |
1175 | if ((interface == PHY_INTERFACE_MODE_RGMII) || |
1176 | (interface == PHY_INTERFACE_MODE_RGMII_ID) || |
1177 | (interface == PHY_INTERFACE_MODE_RGMII_RXID) || |
1178 | (interface == PHY_INTERFACE_MODE_RGMII_TXID)) { |
1179 | netdev_dbg(priv->dev, "PCS RGMII support enabled\n"); |
1180 | priv->hw->pcs = STMMAC_PCS_RGMII; |
1181 | } else if (interface == PHY_INTERFACE_MODE_SGMII) { |
1182 | netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); |
1183 | priv->hw->pcs = STMMAC_PCS_SGMII; |
1184 | } |
1185 | } |
1186 | } |
1187 | |
1188 | /** |
1189 | * stmmac_init_phy - PHY initialization |
1190 | * @dev: net device structure |
1191 | * Description: it initializes the driver's PHY state, and attaches the PHY |
1192 | * to the mac driver. |
1193 | * Return value: |
1194 | * 0 on success |
1195 | */ |
1196 | static int stmmac_init_phy(struct net_device *dev) |
1197 | { |
1198 | struct stmmac_priv *priv = netdev_priv(dev); |
1199 | struct fwnode_handle *phy_fwnode; |
1200 | struct fwnode_handle *fwnode; |
1201 | int ret; |
1202 | |
1203 | if (!phylink_expects_phy(pl: priv->phylink)) |
1204 | return 0; |
1205 | |
1206 | fwnode = priv->plat->port_node; |
1207 | if (!fwnode) |
1208 | fwnode = dev_fwnode(priv->device); |
1209 | |
1210 | if (fwnode) |
1211 | phy_fwnode = fwnode_get_phy_node(fwnode); |
1212 | else |
1213 | phy_fwnode = NULL; |
1214 | |
1215 | /* Some DT bindings do not set-up the PHY handle. Let's try to |
1216 | * manually parse it |
1217 | */ |
1218 | if (!phy_fwnode || IS_ERR(ptr: phy_fwnode)) { |
1219 | int addr = priv->plat->phy_addr; |
1220 | struct phy_device *phydev; |
1221 | |
1222 | if (addr < 0) { |
1223 | netdev_err(dev: priv->dev, format: "no phy found\n"); |
1224 | return -ENODEV; |
1225 | } |
1226 | |
1227 | phydev = mdiobus_get_phy(bus: priv->mii, addr); |
1228 | if (!phydev) { |
1229 | netdev_err(dev: priv->dev, format: "no phy at addr %d\n", addr); |
1230 | return -ENODEV; |
1231 | } |
1232 | |
1233 | ret = phylink_connect_phy(priv->phylink, phydev); |
1234 | } else { |
1235 | fwnode_handle_put(fwnode: phy_fwnode); |
1236 | ret = phylink_fwnode_phy_connect(pl: priv->phylink, fwnode, flags: 0); |
1237 | } |
1238 | |
1239 | if (ret == 0) { |
1240 | struct ethtool_keee eee; |
1241 | |
1242 | /* Configure phylib's copy of the LPI timer. Normally, |
1243 | * phylink_config.lpi_timer_default would do this, but there is |
1244 | * a chance that userspace could change the eee_timer setting |
1245 | * via sysfs before the first open. Thus, preserve existing |
1246 | * behaviour. |
1247 | */ |
1248 | if (!phylink_ethtool_get_eee(link: priv->phylink, eee: &eee)) { |
1249 | eee.tx_lpi_timer = priv->tx_lpi_timer; |
1250 | phylink_ethtool_set_eee(link: priv->phylink, eee: &eee); |
1251 | } |
1252 | } |
1253 | |
1254 | if (!priv->plat->pmt) { |
1255 | struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; |
1256 | |
1257 | phylink_ethtool_get_wol(priv->phylink, &wol); |
1258 | device_set_wakeup_capable(dev: priv->device, capable: !!wol.supported); |
1259 | device_set_wakeup_enable(dev: priv->device, enable: !!wol.wolopts); |
1260 | } |
1261 | |
1262 | return ret; |
1263 | } |
1264 | |
1265 | static int stmmac_phy_setup(struct stmmac_priv *priv) |
1266 | { |
1267 | struct stmmac_mdio_bus_data *mdio_bus_data; |
1268 | struct phylink_config *config; |
1269 | struct fwnode_handle *fwnode; |
1270 | struct phylink_pcs *pcs; |
1271 | struct phylink *phylink; |
1272 | |
1273 | config = &priv->phylink_config; |
1274 | |
1275 | config->dev = &priv->dev->dev; |
1276 | config->type = PHYLINK_NETDEV; |
1277 | config->mac_managed_pm = true; |
1278 | |
1279 | /* Stmmac always requires an RX clock for hardware initialization */ |
1280 | config->mac_requires_rxc = true; |
1281 | |
1282 | if (!(priv->plat->flags & STMMAC_FLAG_RX_CLK_RUNS_IN_LPI)) |
1283 | config->eee_rx_clk_stop_enable = true; |
1284 | |
1285 | /* Set the default transmit clock stop bit based on the platform glue */ |
1286 | priv->tx_lpi_clk_stop = priv->plat->flags & |
1287 | STMMAC_FLAG_EN_TX_LPI_CLOCKGATING; |
1288 | |
1289 | mdio_bus_data = priv->plat->mdio_bus_data; |
1290 | if (mdio_bus_data) |
1291 | config->default_an_inband = mdio_bus_data->default_an_inband; |
1292 | |
1293 | /* Get the PHY interface modes (at the PHY end of the link) that |
1294 | * are supported by the platform. |
1295 | */ |
1296 | if (priv->plat->get_interfaces) |
1297 | priv->plat->get_interfaces(priv, priv->plat->bsp_priv, |
1298 | config->supported_interfaces); |
1299 | |
1300 | /* Set the platform/firmware specified interface mode if the |
1301 | * supported interfaces have not already been provided using |
1302 | * phy_interface as a last resort. |
1303 | */ |
1304 | if (phy_interface_empty(intf: config->supported_interfaces)) |
1305 | __set_bit(priv->plat->phy_interface, |
1306 | config->supported_interfaces); |
1307 | |
1308 | /* If we have an xpcs, it defines which PHY interfaces are supported. */ |
1309 | if (priv->hw->xpcs) |
1310 | pcs = xpcs_to_phylink_pcs(xpcs: priv->hw->xpcs); |
1311 | else |
1312 | pcs = priv->hw->phylink_pcs; |
1313 | |
1314 | if (pcs) |
1315 | phy_interface_or(dst: config->supported_interfaces, |
1316 | a: config->supported_interfaces, |
1317 | b: pcs->supported_interfaces); |
1318 | |
1319 | if (priv->dma_cap.eee) { |
1320 | /* Assume all supported interfaces also support LPI */ |
1321 | memcpy(config->lpi_interfaces, config->supported_interfaces, |
1322 | sizeof(config->lpi_interfaces)); |
1323 | |
1324 | /* All full duplex speeds above 100Mbps are supported */ |
1325 | config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD; |
1326 | config->lpi_timer_default = eee_timer * 1000; |
1327 | config->eee_enabled_default = true; |
1328 | } |
1329 | |
1330 | fwnode = priv->plat->port_node; |
1331 | if (!fwnode) |
1332 | fwnode = dev_fwnode(priv->device); |
1333 | |
1334 | phylink = phylink_create(config, fwnode, priv->plat->phy_interface, |
1335 | &stmmac_phylink_mac_ops); |
1336 | if (IS_ERR(ptr: phylink)) |
1337 | return PTR_ERR(ptr: phylink); |
1338 | |
1339 | priv->phylink = phylink; |
1340 | return 0; |
1341 | } |
1342 | |
1343 | static void stmmac_display_rx_rings(struct stmmac_priv *priv, |
1344 | struct stmmac_dma_conf *dma_conf) |
1345 | { |
1346 | u32 rx_cnt = priv->plat->rx_queues_to_use; |
1347 | unsigned int desc_size; |
1348 | void *head_rx; |
1349 | u32 queue; |
1350 | |
1351 | /* Display RX rings */ |
1352 | for (queue = 0; queue < rx_cnt; queue++) { |
1353 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
1354 | |
1355 | pr_info("\tRX Queue %u rings\n", queue); |
1356 | |
1357 | if (priv->extend_desc) { |
1358 | head_rx = (void *)rx_q->dma_erx; |
1359 | desc_size = sizeof(struct dma_extended_desc); |
1360 | } else { |
1361 | head_rx = (void *)rx_q->dma_rx; |
1362 | desc_size = sizeof(struct dma_desc); |
1363 | } |
1364 | |
1365 | /* Display RX ring */ |
1366 | stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, |
1367 | rx_q->dma_rx_phy, desc_size); |
1368 | } |
1369 | } |
1370 | |
1371 | static void stmmac_display_tx_rings(struct stmmac_priv *priv, |
1372 | struct stmmac_dma_conf *dma_conf) |
1373 | { |
1374 | u32 tx_cnt = priv->plat->tx_queues_to_use; |
1375 | unsigned int desc_size; |
1376 | void *head_tx; |
1377 | u32 queue; |
1378 | |
1379 | /* Display TX rings */ |
1380 | for (queue = 0; queue < tx_cnt; queue++) { |
1381 | struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; |
1382 | |
1383 | pr_info("\tTX Queue %d rings\n", queue); |
1384 | |
1385 | if (priv->extend_desc) { |
1386 | head_tx = (void *)tx_q->dma_etx; |
1387 | desc_size = sizeof(struct dma_extended_desc); |
1388 | } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { |
1389 | head_tx = (void *)tx_q->dma_entx; |
1390 | desc_size = sizeof(struct dma_edesc); |
1391 | } else { |
1392 | head_tx = (void *)tx_q->dma_tx; |
1393 | desc_size = sizeof(struct dma_desc); |
1394 | } |
1395 | |
1396 | stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, |
1397 | tx_q->dma_tx_phy, desc_size); |
1398 | } |
1399 | } |
1400 | |
1401 | static void stmmac_display_rings(struct stmmac_priv *priv, |
1402 | struct stmmac_dma_conf *dma_conf) |
1403 | { |
1404 | /* Display RX ring */ |
1405 | stmmac_display_rx_rings(priv, dma_conf); |
1406 | |
1407 | /* Display TX ring */ |
1408 | stmmac_display_tx_rings(priv, dma_conf); |
1409 | } |
1410 | |
1411 | static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) |
1412 | { |
1413 | if (stmmac_xdp_is_enabled(priv)) |
1414 | return XDP_PACKET_HEADROOM; |
1415 | |
1416 | return NET_SKB_PAD; |
1417 | } |
1418 | |
1419 | static int stmmac_set_bfsize(int mtu, int bufsize) |
1420 | { |
1421 | int ret = bufsize; |
1422 | |
1423 | if (mtu >= BUF_SIZE_8KiB) |
1424 | ret = BUF_SIZE_16KiB; |
1425 | else if (mtu >= BUF_SIZE_4KiB) |
1426 | ret = BUF_SIZE_8KiB; |
1427 | else if (mtu >= BUF_SIZE_2KiB) |
1428 | ret = BUF_SIZE_4KiB; |
1429 | else if (mtu > DEFAULT_BUFSIZE) |
1430 | ret = BUF_SIZE_2KiB; |
1431 | else |
1432 | ret = DEFAULT_BUFSIZE; |
1433 | |
1434 | return ret; |
1435 | } |
1436 | |
1437 | /** |
1438 | * stmmac_clear_rx_descriptors - clear RX descriptors |
1439 | * @priv: driver private structure |
1440 | * @dma_conf: structure to take the dma data |
1441 | * @queue: RX queue index |
1442 | * Description: this function is called to clear the RX descriptors |
1443 | * in case of both basic and extended descriptors are used. |
1444 | */ |
1445 | static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, |
1446 | struct stmmac_dma_conf *dma_conf, |
1447 | u32 queue) |
1448 | { |
1449 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
1450 | int i; |
1451 | |
1452 | /* Clear the RX descriptors */ |
1453 | for (i = 0; i < dma_conf->dma_rx_size; i++) |
1454 | if (priv->extend_desc) |
1455 | stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, |
1456 | priv->use_riwt, priv->mode, |
1457 | (i == dma_conf->dma_rx_size - 1), |
1458 | dma_conf->dma_buf_sz); |
1459 | else |
1460 | stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], |
1461 | priv->use_riwt, priv->mode, |
1462 | (i == dma_conf->dma_rx_size - 1), |
1463 | dma_conf->dma_buf_sz); |
1464 | } |
1465 | |
1466 | /** |
1467 | * stmmac_clear_tx_descriptors - clear tx descriptors |
1468 | * @priv: driver private structure |
1469 | * @dma_conf: structure to take the dma data |
1470 | * @queue: TX queue index. |
1471 | * Description: this function is called to clear the TX descriptors |
1472 | * in case of both basic and extended descriptors are used. |
1473 | */ |
1474 | static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, |
1475 | struct stmmac_dma_conf *dma_conf, |
1476 | u32 queue) |
1477 | { |
1478 | struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; |
1479 | int i; |
1480 | |
1481 | /* Clear the TX descriptors */ |
1482 | for (i = 0; i < dma_conf->dma_tx_size; i++) { |
1483 | int last = (i == (dma_conf->dma_tx_size - 1)); |
1484 | struct dma_desc *p; |
1485 | |
1486 | if (priv->extend_desc) |
1487 | p = &tx_q->dma_etx[i].basic; |
1488 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
1489 | p = &tx_q->dma_entx[i].basic; |
1490 | else |
1491 | p = &tx_q->dma_tx[i]; |
1492 | |
1493 | stmmac_init_tx_desc(priv, p, priv->mode, last); |
1494 | } |
1495 | } |
1496 | |
1497 | /** |
1498 | * stmmac_clear_descriptors - clear descriptors |
1499 | * @priv: driver private structure |
1500 | * @dma_conf: structure to take the dma data |
1501 | * Description: this function is called to clear the TX and RX descriptors |
1502 | * in case of both basic and extended descriptors are used. |
1503 | */ |
1504 | static void stmmac_clear_descriptors(struct stmmac_priv *priv, |
1505 | struct stmmac_dma_conf *dma_conf) |
1506 | { |
1507 | u32 rx_queue_cnt = priv->plat->rx_queues_to_use; |
1508 | u32 tx_queue_cnt = priv->plat->tx_queues_to_use; |
1509 | u32 queue; |
1510 | |
1511 | /* Clear the RX descriptors */ |
1512 | for (queue = 0; queue < rx_queue_cnt; queue++) |
1513 | stmmac_clear_rx_descriptors(priv, dma_conf, queue); |
1514 | |
1515 | /* Clear the TX descriptors */ |
1516 | for (queue = 0; queue < tx_queue_cnt; queue++) |
1517 | stmmac_clear_tx_descriptors(priv, dma_conf, queue); |
1518 | } |
1519 | |
1520 | /** |
1521 | * stmmac_init_rx_buffers - init the RX descriptor buffer. |
1522 | * @priv: driver private structure |
1523 | * @dma_conf: structure to take the dma data |
1524 | * @p: descriptor pointer |
1525 | * @i: descriptor index |
1526 | * @flags: gfp flag |
1527 | * @queue: RX queue index |
1528 | * Description: this function is called to allocate a receive buffer, perform |
1529 | * the DMA mapping and init the descriptor. |
1530 | */ |
1531 | static int stmmac_init_rx_buffers(struct stmmac_priv *priv, |
1532 | struct stmmac_dma_conf *dma_conf, |
1533 | struct dma_desc *p, |
1534 | int i, gfp_t flags, u32 queue) |
1535 | { |
1536 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
1537 | struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; |
1538 | gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); |
1539 | |
1540 | if (priv->dma_cap.host_dma_width <= 32) |
1541 | gfp |= GFP_DMA32; |
1542 | |
1543 | if (!buf->page) { |
1544 | buf->page = page_pool_alloc_pages(pool: rx_q->page_pool, gfp); |
1545 | if (!buf->page) |
1546 | return -ENOMEM; |
1547 | buf->page_offset = stmmac_rx_offset(priv); |
1548 | } |
1549 | |
1550 | if (priv->sph && !buf->sec_page) { |
1551 | buf->sec_page = page_pool_alloc_pages(pool: rx_q->page_pool, gfp); |
1552 | if (!buf->sec_page) |
1553 | return -ENOMEM; |
1554 | |
1555 | buf->sec_addr = page_pool_get_dma_addr(page: buf->sec_page); |
1556 | stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); |
1557 | } else { |
1558 | buf->sec_page = NULL; |
1559 | stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); |
1560 | } |
1561 | |
1562 | buf->addr = page_pool_get_dma_addr(page: buf->page) + buf->page_offset; |
1563 | |
1564 | stmmac_set_desc_addr(priv, p, buf->addr); |
1565 | if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) |
1566 | stmmac_init_desc3(priv, p); |
1567 | |
1568 | return 0; |
1569 | } |
1570 | |
1571 | /** |
1572 | * stmmac_free_rx_buffer - free RX dma buffers |
1573 | * @priv: private structure |
1574 | * @rx_q: RX queue |
1575 | * @i: buffer index. |
1576 | */ |
1577 | static void stmmac_free_rx_buffer(struct stmmac_priv *priv, |
1578 | struct stmmac_rx_queue *rx_q, |
1579 | int i) |
1580 | { |
1581 | struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; |
1582 | |
1583 | if (buf->page) |
1584 | page_pool_put_full_page(pool: rx_q->page_pool, page: buf->page, allow_direct: false); |
1585 | buf->page = NULL; |
1586 | |
1587 | if (buf->sec_page) |
1588 | page_pool_put_full_page(pool: rx_q->page_pool, page: buf->sec_page, allow_direct: false); |
1589 | buf->sec_page = NULL; |
1590 | } |
1591 | |
1592 | /** |
1593 | * stmmac_free_tx_buffer - free RX dma buffers |
1594 | * @priv: private structure |
1595 | * @dma_conf: structure to take the dma data |
1596 | * @queue: RX queue index |
1597 | * @i: buffer index. |
1598 | */ |
1599 | static void stmmac_free_tx_buffer(struct stmmac_priv *priv, |
1600 | struct stmmac_dma_conf *dma_conf, |
1601 | u32 queue, int i) |
1602 | { |
1603 | struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; |
1604 | |
1605 | if (tx_q->tx_skbuff_dma[i].buf && |
1606 | tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { |
1607 | if (tx_q->tx_skbuff_dma[i].map_as_page) |
1608 | dma_unmap_page(priv->device, |
1609 | tx_q->tx_skbuff_dma[i].buf, |
1610 | tx_q->tx_skbuff_dma[i].len, |
1611 | DMA_TO_DEVICE); |
1612 | else |
1613 | dma_unmap_single(priv->device, |
1614 | tx_q->tx_skbuff_dma[i].buf, |
1615 | tx_q->tx_skbuff_dma[i].len, |
1616 | DMA_TO_DEVICE); |
1617 | } |
1618 | |
1619 | if (tx_q->xdpf[i] && |
1620 | (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || |
1621 | tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { |
1622 | xdp_return_frame(xdpf: tx_q->xdpf[i]); |
1623 | tx_q->xdpf[i] = NULL; |
1624 | } |
1625 | |
1626 | if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) |
1627 | tx_q->xsk_frames_done++; |
1628 | |
1629 | if (tx_q->tx_skbuff[i] && |
1630 | tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { |
1631 | dev_kfree_skb_any(skb: tx_q->tx_skbuff[i]); |
1632 | tx_q->tx_skbuff[i] = NULL; |
1633 | } |
1634 | |
1635 | tx_q->tx_skbuff_dma[i].buf = 0; |
1636 | tx_q->tx_skbuff_dma[i].map_as_page = false; |
1637 | } |
1638 | |
1639 | /** |
1640 | * dma_free_rx_skbufs - free RX dma buffers |
1641 | * @priv: private structure |
1642 | * @dma_conf: structure to take the dma data |
1643 | * @queue: RX queue index |
1644 | */ |
1645 | static void dma_free_rx_skbufs(struct stmmac_priv *priv, |
1646 | struct stmmac_dma_conf *dma_conf, |
1647 | u32 queue) |
1648 | { |
1649 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
1650 | int i; |
1651 | |
1652 | for (i = 0; i < dma_conf->dma_rx_size; i++) |
1653 | stmmac_free_rx_buffer(priv, rx_q, i); |
1654 | } |
1655 | |
1656 | static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, |
1657 | struct stmmac_dma_conf *dma_conf, |
1658 | u32 queue, gfp_t flags) |
1659 | { |
1660 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
1661 | int i; |
1662 | |
1663 | for (i = 0; i < dma_conf->dma_rx_size; i++) { |
1664 | struct dma_desc *p; |
1665 | int ret; |
1666 | |
1667 | if (priv->extend_desc) |
1668 | p = &((rx_q->dma_erx + i)->basic); |
1669 | else |
1670 | p = rx_q->dma_rx + i; |
1671 | |
1672 | ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, |
1673 | queue); |
1674 | if (ret) |
1675 | return ret; |
1676 | |
1677 | rx_q->buf_alloc_num++; |
1678 | } |
1679 | |
1680 | return 0; |
1681 | } |
1682 | |
1683 | /** |
1684 | * dma_free_rx_xskbufs - free RX dma buffers from XSK pool |
1685 | * @priv: private structure |
1686 | * @dma_conf: structure to take the dma data |
1687 | * @queue: RX queue index |
1688 | */ |
1689 | static void dma_free_rx_xskbufs(struct stmmac_priv *priv, |
1690 | struct stmmac_dma_conf *dma_conf, |
1691 | u32 queue) |
1692 | { |
1693 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
1694 | int i; |
1695 | |
1696 | for (i = 0; i < dma_conf->dma_rx_size; i++) { |
1697 | struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; |
1698 | |
1699 | if (!buf->xdp) |
1700 | continue; |
1701 | |
1702 | xsk_buff_free(xdp: buf->xdp); |
1703 | buf->xdp = NULL; |
1704 | } |
1705 | } |
1706 | |
1707 | static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, |
1708 | struct stmmac_dma_conf *dma_conf, |
1709 | u32 queue) |
1710 | { |
1711 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
1712 | int i; |
1713 | |
1714 | /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) |
1715 | * in struct xdp_buff_xsk to stash driver specific information. Thus, |
1716 | * use this macro to make sure no size violations. |
1717 | */ |
1718 | XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); |
1719 | |
1720 | for (i = 0; i < dma_conf->dma_rx_size; i++) { |
1721 | struct stmmac_rx_buffer *buf; |
1722 | dma_addr_t dma_addr; |
1723 | struct dma_desc *p; |
1724 | |
1725 | if (priv->extend_desc) |
1726 | p = (struct dma_desc *)(rx_q->dma_erx + i); |
1727 | else |
1728 | p = rx_q->dma_rx + i; |
1729 | |
1730 | buf = &rx_q->buf_pool[i]; |
1731 | |
1732 | buf->xdp = xsk_buff_alloc(pool: rx_q->xsk_pool); |
1733 | if (!buf->xdp) |
1734 | return -ENOMEM; |
1735 | |
1736 | dma_addr = xsk_buff_xdp_get_dma(xdp: buf->xdp); |
1737 | stmmac_set_desc_addr(priv, p, dma_addr); |
1738 | rx_q->buf_alloc_num++; |
1739 | } |
1740 | |
1741 | return 0; |
1742 | } |
1743 | |
1744 | static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) |
1745 | { |
1746 | if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) |
1747 | return NULL; |
1748 | |
1749 | return xsk_get_pool_from_qid(dev: priv->dev, queue_id: queue); |
1750 | } |
1751 | |
1752 | /** |
1753 | * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) |
1754 | * @priv: driver private structure |
1755 | * @dma_conf: structure to take the dma data |
1756 | * @queue: RX queue index |
1757 | * @flags: gfp flag. |
1758 | * Description: this function initializes the DMA RX descriptors |
1759 | * and allocates the socket buffers. It supports the chained and ring |
1760 | * modes. |
1761 | */ |
1762 | static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, |
1763 | struct stmmac_dma_conf *dma_conf, |
1764 | u32 queue, gfp_t flags) |
1765 | { |
1766 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
1767 | int ret; |
1768 | |
1769 | netif_dbg(priv, probe, priv->dev, |
1770 | "(%s) dma_rx_phy=0x%08x\n", __func__, |
1771 | (u32)rx_q->dma_rx_phy); |
1772 | |
1773 | stmmac_clear_rx_descriptors(priv, dma_conf, queue); |
1774 | |
1775 | xdp_rxq_info_unreg_mem_model(xdp_rxq: &rx_q->xdp_rxq); |
1776 | |
1777 | rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); |
1778 | |
1779 | if (rx_q->xsk_pool) { |
1780 | WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, |
1781 | MEM_TYPE_XSK_BUFF_POOL, |
1782 | NULL)); |
1783 | netdev_info(dev: priv->dev, |
1784 | format: "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", |
1785 | rx_q->queue_index); |
1786 | xsk_pool_set_rxq_info(pool: rx_q->xsk_pool, rxq: &rx_q->xdp_rxq); |
1787 | } else { |
1788 | WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, |
1789 | MEM_TYPE_PAGE_POOL, |
1790 | rx_q->page_pool)); |
1791 | netdev_info(dev: priv->dev, |
1792 | format: "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", |
1793 | rx_q->queue_index); |
1794 | } |
1795 | |
1796 | if (rx_q->xsk_pool) { |
1797 | /* RX XDP ZC buffer pool may not be populated, e.g. |
1798 | * xdpsock TX-only. |
1799 | */ |
1800 | stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); |
1801 | } else { |
1802 | ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); |
1803 | if (ret < 0) |
1804 | return -ENOMEM; |
1805 | } |
1806 | |
1807 | /* Setup the chained descriptor addresses */ |
1808 | if (priv->mode == STMMAC_CHAIN_MODE) { |
1809 | if (priv->extend_desc) |
1810 | stmmac_mode_init(priv, rx_q->dma_erx, |
1811 | rx_q->dma_rx_phy, |
1812 | dma_conf->dma_rx_size, 1); |
1813 | else |
1814 | stmmac_mode_init(priv, rx_q->dma_rx, |
1815 | rx_q->dma_rx_phy, |
1816 | dma_conf->dma_rx_size, 0); |
1817 | } |
1818 | |
1819 | return 0; |
1820 | } |
1821 | |
1822 | static int init_dma_rx_desc_rings(struct net_device *dev, |
1823 | struct stmmac_dma_conf *dma_conf, |
1824 | gfp_t flags) |
1825 | { |
1826 | struct stmmac_priv *priv = netdev_priv(dev); |
1827 | u32 rx_count = priv->plat->rx_queues_to_use; |
1828 | int queue; |
1829 | int ret; |
1830 | |
1831 | /* RX INITIALIZATION */ |
1832 | netif_dbg(priv, probe, priv->dev, |
1833 | "SKB addresses:\nskb\t\tskb data\tdma data\n"); |
1834 | |
1835 | for (queue = 0; queue < rx_count; queue++) { |
1836 | ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); |
1837 | if (ret) |
1838 | goto err_init_rx_buffers; |
1839 | } |
1840 | |
1841 | return 0; |
1842 | |
1843 | err_init_rx_buffers: |
1844 | while (queue >= 0) { |
1845 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
1846 | |
1847 | if (rx_q->xsk_pool) |
1848 | dma_free_rx_xskbufs(priv, dma_conf, queue); |
1849 | else |
1850 | dma_free_rx_skbufs(priv, dma_conf, queue); |
1851 | |
1852 | rx_q->buf_alloc_num = 0; |
1853 | rx_q->xsk_pool = NULL; |
1854 | |
1855 | queue--; |
1856 | } |
1857 | |
1858 | return ret; |
1859 | } |
1860 | |
1861 | /** |
1862 | * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) |
1863 | * @priv: driver private structure |
1864 | * @dma_conf: structure to take the dma data |
1865 | * @queue: TX queue index |
1866 | * Description: this function initializes the DMA TX descriptors |
1867 | * and allocates the socket buffers. It supports the chained and ring |
1868 | * modes. |
1869 | */ |
1870 | static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, |
1871 | struct stmmac_dma_conf *dma_conf, |
1872 | u32 queue) |
1873 | { |
1874 | struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; |
1875 | int i; |
1876 | |
1877 | netif_dbg(priv, probe, priv->dev, |
1878 | "(%s) dma_tx_phy=0x%08x\n", __func__, |
1879 | (u32)tx_q->dma_tx_phy); |
1880 | |
1881 | /* Setup the chained descriptor addresses */ |
1882 | if (priv->mode == STMMAC_CHAIN_MODE) { |
1883 | if (priv->extend_desc) |
1884 | stmmac_mode_init(priv, tx_q->dma_etx, |
1885 | tx_q->dma_tx_phy, |
1886 | dma_conf->dma_tx_size, 1); |
1887 | else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) |
1888 | stmmac_mode_init(priv, tx_q->dma_tx, |
1889 | tx_q->dma_tx_phy, |
1890 | dma_conf->dma_tx_size, 0); |
1891 | } |
1892 | |
1893 | tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); |
1894 | |
1895 | for (i = 0; i < dma_conf->dma_tx_size; i++) { |
1896 | struct dma_desc *p; |
1897 | |
1898 | if (priv->extend_desc) |
1899 | p = &((tx_q->dma_etx + i)->basic); |
1900 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
1901 | p = &((tx_q->dma_entx + i)->basic); |
1902 | else |
1903 | p = tx_q->dma_tx + i; |
1904 | |
1905 | stmmac_clear_desc(priv, p); |
1906 | |
1907 | tx_q->tx_skbuff_dma[i].buf = 0; |
1908 | tx_q->tx_skbuff_dma[i].map_as_page = false; |
1909 | tx_q->tx_skbuff_dma[i].len = 0; |
1910 | tx_q->tx_skbuff_dma[i].last_segment = false; |
1911 | tx_q->tx_skbuff[i] = NULL; |
1912 | } |
1913 | |
1914 | return 0; |
1915 | } |
1916 | |
1917 | static int init_dma_tx_desc_rings(struct net_device *dev, |
1918 | struct stmmac_dma_conf *dma_conf) |
1919 | { |
1920 | struct stmmac_priv *priv = netdev_priv(dev); |
1921 | u32 tx_queue_cnt; |
1922 | u32 queue; |
1923 | |
1924 | tx_queue_cnt = priv->plat->tx_queues_to_use; |
1925 | |
1926 | for (queue = 0; queue < tx_queue_cnt; queue++) |
1927 | __init_dma_tx_desc_rings(priv, dma_conf, queue); |
1928 | |
1929 | return 0; |
1930 | } |
1931 | |
1932 | /** |
1933 | * init_dma_desc_rings - init the RX/TX descriptor rings |
1934 | * @dev: net device structure |
1935 | * @dma_conf: structure to take the dma data |
1936 | * @flags: gfp flag. |
1937 | * Description: this function initializes the DMA RX/TX descriptors |
1938 | * and allocates the socket buffers. It supports the chained and ring |
1939 | * modes. |
1940 | */ |
1941 | static int init_dma_desc_rings(struct net_device *dev, |
1942 | struct stmmac_dma_conf *dma_conf, |
1943 | gfp_t flags) |
1944 | { |
1945 | struct stmmac_priv *priv = netdev_priv(dev); |
1946 | int ret; |
1947 | |
1948 | ret = init_dma_rx_desc_rings(dev, dma_conf, flags); |
1949 | if (ret) |
1950 | return ret; |
1951 | |
1952 | ret = init_dma_tx_desc_rings(dev, dma_conf); |
1953 | |
1954 | stmmac_clear_descriptors(priv, dma_conf); |
1955 | |
1956 | if (netif_msg_hw(priv)) |
1957 | stmmac_display_rings(priv, dma_conf); |
1958 | |
1959 | return ret; |
1960 | } |
1961 | |
1962 | /** |
1963 | * dma_free_tx_skbufs - free TX dma buffers |
1964 | * @priv: private structure |
1965 | * @dma_conf: structure to take the dma data |
1966 | * @queue: TX queue index |
1967 | */ |
1968 | static void dma_free_tx_skbufs(struct stmmac_priv *priv, |
1969 | struct stmmac_dma_conf *dma_conf, |
1970 | u32 queue) |
1971 | { |
1972 | struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; |
1973 | int i; |
1974 | |
1975 | tx_q->xsk_frames_done = 0; |
1976 | |
1977 | for (i = 0; i < dma_conf->dma_tx_size; i++) |
1978 | stmmac_free_tx_buffer(priv, dma_conf, queue, i); |
1979 | |
1980 | if (tx_q->xsk_pool && tx_q->xsk_frames_done) { |
1981 | xsk_tx_completed(pool: tx_q->xsk_pool, nb_entries: tx_q->xsk_frames_done); |
1982 | tx_q->xsk_frames_done = 0; |
1983 | tx_q->xsk_pool = NULL; |
1984 | } |
1985 | } |
1986 | |
1987 | /** |
1988 | * stmmac_free_tx_skbufs - free TX skb buffers |
1989 | * @priv: private structure |
1990 | */ |
1991 | static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) |
1992 | { |
1993 | u32 tx_queue_cnt = priv->plat->tx_queues_to_use; |
1994 | u32 queue; |
1995 | |
1996 | for (queue = 0; queue < tx_queue_cnt; queue++) |
1997 | dma_free_tx_skbufs(priv, dma_conf: &priv->dma_conf, queue); |
1998 | } |
1999 | |
2000 | /** |
2001 | * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) |
2002 | * @priv: private structure |
2003 | * @dma_conf: structure to take the dma data |
2004 | * @queue: RX queue index |
2005 | */ |
2006 | static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, |
2007 | struct stmmac_dma_conf *dma_conf, |
2008 | u32 queue) |
2009 | { |
2010 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
2011 | |
2012 | /* Release the DMA RX socket buffers */ |
2013 | if (rx_q->xsk_pool) |
2014 | dma_free_rx_xskbufs(priv, dma_conf, queue); |
2015 | else |
2016 | dma_free_rx_skbufs(priv, dma_conf, queue); |
2017 | |
2018 | rx_q->buf_alloc_num = 0; |
2019 | rx_q->xsk_pool = NULL; |
2020 | |
2021 | /* Free DMA regions of consistent memory previously allocated */ |
2022 | if (!priv->extend_desc) |
2023 | dma_free_coherent(dev: priv->device, size: dma_conf->dma_rx_size * |
2024 | sizeof(struct dma_desc), |
2025 | cpu_addr: rx_q->dma_rx, dma_handle: rx_q->dma_rx_phy); |
2026 | else |
2027 | dma_free_coherent(dev: priv->device, size: dma_conf->dma_rx_size * |
2028 | sizeof(struct dma_extended_desc), |
2029 | cpu_addr: rx_q->dma_erx, dma_handle: rx_q->dma_rx_phy); |
2030 | |
2031 | if (xdp_rxq_info_is_reg(xdp_rxq: &rx_q->xdp_rxq)) |
2032 | xdp_rxq_info_unreg(xdp_rxq: &rx_q->xdp_rxq); |
2033 | |
2034 | kfree(objp: rx_q->buf_pool); |
2035 | if (rx_q->page_pool) |
2036 | page_pool_destroy(pool: rx_q->page_pool); |
2037 | } |
2038 | |
2039 | static void free_dma_rx_desc_resources(struct stmmac_priv *priv, |
2040 | struct stmmac_dma_conf *dma_conf) |
2041 | { |
2042 | u32 rx_count = priv->plat->rx_queues_to_use; |
2043 | u32 queue; |
2044 | |
2045 | /* Free RX queue resources */ |
2046 | for (queue = 0; queue < rx_count; queue++) |
2047 | __free_dma_rx_desc_resources(priv, dma_conf, queue); |
2048 | } |
2049 | |
2050 | /** |
2051 | * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) |
2052 | * @priv: private structure |
2053 | * @dma_conf: structure to take the dma data |
2054 | * @queue: TX queue index |
2055 | */ |
2056 | static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, |
2057 | struct stmmac_dma_conf *dma_conf, |
2058 | u32 queue) |
2059 | { |
2060 | struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; |
2061 | size_t size; |
2062 | void *addr; |
2063 | |
2064 | /* Release the DMA TX socket buffers */ |
2065 | dma_free_tx_skbufs(priv, dma_conf, queue); |
2066 | |
2067 | if (priv->extend_desc) { |
2068 | size = sizeof(struct dma_extended_desc); |
2069 | addr = tx_q->dma_etx; |
2070 | } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { |
2071 | size = sizeof(struct dma_edesc); |
2072 | addr = tx_q->dma_entx; |
2073 | } else { |
2074 | size = sizeof(struct dma_desc); |
2075 | addr = tx_q->dma_tx; |
2076 | } |
2077 | |
2078 | size *= dma_conf->dma_tx_size; |
2079 | |
2080 | dma_free_coherent(dev: priv->device, size, cpu_addr: addr, dma_handle: tx_q->dma_tx_phy); |
2081 | |
2082 | kfree(objp: tx_q->tx_skbuff_dma); |
2083 | kfree(objp: tx_q->tx_skbuff); |
2084 | } |
2085 | |
2086 | static void free_dma_tx_desc_resources(struct stmmac_priv *priv, |
2087 | struct stmmac_dma_conf *dma_conf) |
2088 | { |
2089 | u32 tx_count = priv->plat->tx_queues_to_use; |
2090 | u32 queue; |
2091 | |
2092 | /* Free TX queue resources */ |
2093 | for (queue = 0; queue < tx_count; queue++) |
2094 | __free_dma_tx_desc_resources(priv, dma_conf, queue); |
2095 | } |
2096 | |
2097 | /** |
2098 | * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). |
2099 | * @priv: private structure |
2100 | * @dma_conf: structure to take the dma data |
2101 | * @queue: RX queue index |
2102 | * Description: according to which descriptor can be used (extend or basic) |
2103 | * this function allocates the resources for TX and RX paths. In case of |
2104 | * reception, for example, it pre-allocated the RX socket buffer in order to |
2105 | * allow zero-copy mechanism. |
2106 | */ |
2107 | static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, |
2108 | struct stmmac_dma_conf *dma_conf, |
2109 | u32 queue) |
2110 | { |
2111 | struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; |
2112 | struct stmmac_channel *ch = &priv->channel[queue]; |
2113 | bool xdp_prog = stmmac_xdp_is_enabled(priv); |
2114 | struct page_pool_params pp_params = { 0 }; |
2115 | unsigned int dma_buf_sz_pad, num_pages; |
2116 | unsigned int napi_id; |
2117 | int ret; |
2118 | |
2119 | dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz + |
2120 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); |
2121 | num_pages = DIV_ROUND_UP(dma_buf_sz_pad, PAGE_SIZE); |
2122 | |
2123 | rx_q->queue_index = queue; |
2124 | rx_q->priv_data = priv; |
2125 | rx_q->napi_skb_frag_size = num_pages * PAGE_SIZE; |
2126 | |
2127 | pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; |
2128 | pp_params.pool_size = dma_conf->dma_rx_size; |
2129 | pp_params.order = order_base_2(num_pages); |
2130 | pp_params.nid = dev_to_node(dev: priv->device); |
2131 | pp_params.dev = priv->device; |
2132 | pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; |
2133 | pp_params.offset = stmmac_rx_offset(priv); |
2134 | pp_params.max_len = dma_conf->dma_buf_sz; |
2135 | |
2136 | if (priv->sph) { |
2137 | pp_params.offset = 0; |
2138 | pp_params.max_len += stmmac_rx_offset(priv); |
2139 | } |
2140 | |
2141 | rx_q->page_pool = page_pool_create(params: &pp_params); |
2142 | if (IS_ERR(ptr: rx_q->page_pool)) { |
2143 | ret = PTR_ERR(ptr: rx_q->page_pool); |
2144 | rx_q->page_pool = NULL; |
2145 | return ret; |
2146 | } |
2147 | |
2148 | rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, |
2149 | sizeof(*rx_q->buf_pool), |
2150 | GFP_KERNEL); |
2151 | if (!rx_q->buf_pool) |
2152 | return -ENOMEM; |
2153 | |
2154 | if (priv->extend_desc) { |
2155 | rx_q->dma_erx = dma_alloc_coherent(dev: priv->device, |
2156 | size: dma_conf->dma_rx_size * |
2157 | sizeof(struct dma_extended_desc), |
2158 | dma_handle: &rx_q->dma_rx_phy, |
2159 | GFP_KERNEL); |
2160 | if (!rx_q->dma_erx) |
2161 | return -ENOMEM; |
2162 | |
2163 | } else { |
2164 | rx_q->dma_rx = dma_alloc_coherent(dev: priv->device, |
2165 | size: dma_conf->dma_rx_size * |
2166 | sizeof(struct dma_desc), |
2167 | dma_handle: &rx_q->dma_rx_phy, |
2168 | GFP_KERNEL); |
2169 | if (!rx_q->dma_rx) |
2170 | return -ENOMEM; |
2171 | } |
2172 | |
2173 | if (stmmac_xdp_is_enabled(priv) && |
2174 | test_bit(queue, priv->af_xdp_zc_qps)) |
2175 | napi_id = ch->rxtx_napi.napi_id; |
2176 | else |
2177 | napi_id = ch->rx_napi.napi_id; |
2178 | |
2179 | ret = xdp_rxq_info_reg(xdp_rxq: &rx_q->xdp_rxq, dev: priv->dev, |
2180 | queue_index: rx_q->queue_index, |
2181 | napi_id); |
2182 | if (ret) { |
2183 | netdev_err(dev: priv->dev, format: "Failed to register xdp rxq info\n"); |
2184 | return -EINVAL; |
2185 | } |
2186 | |
2187 | return 0; |
2188 | } |
2189 | |
2190 | static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, |
2191 | struct stmmac_dma_conf *dma_conf) |
2192 | { |
2193 | u32 rx_count = priv->plat->rx_queues_to_use; |
2194 | u32 queue; |
2195 | int ret; |
2196 | |
2197 | /* RX queues buffers and DMA */ |
2198 | for (queue = 0; queue < rx_count; queue++) { |
2199 | ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); |
2200 | if (ret) |
2201 | goto err_dma; |
2202 | } |
2203 | |
2204 | return 0; |
2205 | |
2206 | err_dma: |
2207 | free_dma_rx_desc_resources(priv, dma_conf); |
2208 | |
2209 | return ret; |
2210 | } |
2211 | |
2212 | /** |
2213 | * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). |
2214 | * @priv: private structure |
2215 | * @dma_conf: structure to take the dma data |
2216 | * @queue: TX queue index |
2217 | * Description: according to which descriptor can be used (extend or basic) |
2218 | * this function allocates the resources for TX and RX paths. In case of |
2219 | * reception, for example, it pre-allocated the RX socket buffer in order to |
2220 | * allow zero-copy mechanism. |
2221 | */ |
2222 | static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, |
2223 | struct stmmac_dma_conf *dma_conf, |
2224 | u32 queue) |
2225 | { |
2226 | struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; |
2227 | size_t size; |
2228 | void *addr; |
2229 | |
2230 | tx_q->queue_index = queue; |
2231 | tx_q->priv_data = priv; |
2232 | |
2233 | tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, |
2234 | sizeof(*tx_q->tx_skbuff_dma), |
2235 | GFP_KERNEL); |
2236 | if (!tx_q->tx_skbuff_dma) |
2237 | return -ENOMEM; |
2238 | |
2239 | tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, |
2240 | sizeof(struct sk_buff *), |
2241 | GFP_KERNEL); |
2242 | if (!tx_q->tx_skbuff) |
2243 | return -ENOMEM; |
2244 | |
2245 | if (priv->extend_desc) |
2246 | size = sizeof(struct dma_extended_desc); |
2247 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
2248 | size = sizeof(struct dma_edesc); |
2249 | else |
2250 | size = sizeof(struct dma_desc); |
2251 | |
2252 | size *= dma_conf->dma_tx_size; |
2253 | |
2254 | addr = dma_alloc_coherent(dev: priv->device, size, |
2255 | dma_handle: &tx_q->dma_tx_phy, GFP_KERNEL); |
2256 | if (!addr) |
2257 | return -ENOMEM; |
2258 | |
2259 | if (priv->extend_desc) |
2260 | tx_q->dma_etx = addr; |
2261 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
2262 | tx_q->dma_entx = addr; |
2263 | else |
2264 | tx_q->dma_tx = addr; |
2265 | |
2266 | return 0; |
2267 | } |
2268 | |
2269 | static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, |
2270 | struct stmmac_dma_conf *dma_conf) |
2271 | { |
2272 | u32 tx_count = priv->plat->tx_queues_to_use; |
2273 | u32 queue; |
2274 | int ret; |
2275 | |
2276 | /* TX queues buffers and DMA */ |
2277 | for (queue = 0; queue < tx_count; queue++) { |
2278 | ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); |
2279 | if (ret) |
2280 | goto err_dma; |
2281 | } |
2282 | |
2283 | return 0; |
2284 | |
2285 | err_dma: |
2286 | free_dma_tx_desc_resources(priv, dma_conf); |
2287 | return ret; |
2288 | } |
2289 | |
2290 | /** |
2291 | * alloc_dma_desc_resources - alloc TX/RX resources. |
2292 | * @priv: private structure |
2293 | * @dma_conf: structure to take the dma data |
2294 | * Description: according to which descriptor can be used (extend or basic) |
2295 | * this function allocates the resources for TX and RX paths. In case of |
2296 | * reception, for example, it pre-allocated the RX socket buffer in order to |
2297 | * allow zero-copy mechanism. |
2298 | */ |
2299 | static int alloc_dma_desc_resources(struct stmmac_priv *priv, |
2300 | struct stmmac_dma_conf *dma_conf) |
2301 | { |
2302 | /* RX Allocation */ |
2303 | int ret = alloc_dma_rx_desc_resources(priv, dma_conf); |
2304 | |
2305 | if (ret) |
2306 | return ret; |
2307 | |
2308 | ret = alloc_dma_tx_desc_resources(priv, dma_conf); |
2309 | |
2310 | return ret; |
2311 | } |
2312 | |
2313 | /** |
2314 | * free_dma_desc_resources - free dma desc resources |
2315 | * @priv: private structure |
2316 | * @dma_conf: structure to take the dma data |
2317 | */ |
2318 | static void free_dma_desc_resources(struct stmmac_priv *priv, |
2319 | struct stmmac_dma_conf *dma_conf) |
2320 | { |
2321 | /* Release the DMA TX socket buffers */ |
2322 | free_dma_tx_desc_resources(priv, dma_conf); |
2323 | |
2324 | /* Release the DMA RX socket buffers later |
2325 | * to ensure all pending XDP_TX buffers are returned. |
2326 | */ |
2327 | free_dma_rx_desc_resources(priv, dma_conf); |
2328 | } |
2329 | |
2330 | /** |
2331 | * stmmac_mac_enable_rx_queues - Enable MAC rx queues |
2332 | * @priv: driver private structure |
2333 | * Description: It is used for enabling the rx queues in the MAC |
2334 | */ |
2335 | static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) |
2336 | { |
2337 | u32 rx_queues_count = priv->plat->rx_queues_to_use; |
2338 | int queue; |
2339 | u8 mode; |
2340 | |
2341 | for (queue = 0; queue < rx_queues_count; queue++) { |
2342 | mode = priv->plat->rx_queues_cfg[queue].mode_to_use; |
2343 | stmmac_rx_queue_enable(priv, priv->hw, mode, queue); |
2344 | } |
2345 | } |
2346 | |
2347 | /** |
2348 | * stmmac_start_rx_dma - start RX DMA channel |
2349 | * @priv: driver private structure |
2350 | * @chan: RX channel index |
2351 | * Description: |
2352 | * This starts a RX DMA channel |
2353 | */ |
2354 | static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) |
2355 | { |
2356 | netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); |
2357 | stmmac_start_rx(priv, priv->ioaddr, chan); |
2358 | } |
2359 | |
2360 | /** |
2361 | * stmmac_start_tx_dma - start TX DMA channel |
2362 | * @priv: driver private structure |
2363 | * @chan: TX channel index |
2364 | * Description: |
2365 | * This starts a TX DMA channel |
2366 | */ |
2367 | static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) |
2368 | { |
2369 | netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); |
2370 | stmmac_start_tx(priv, priv->ioaddr, chan); |
2371 | } |
2372 | |
2373 | /** |
2374 | * stmmac_stop_rx_dma - stop RX DMA channel |
2375 | * @priv: driver private structure |
2376 | * @chan: RX channel index |
2377 | * Description: |
2378 | * This stops a RX DMA channel |
2379 | */ |
2380 | static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) |
2381 | { |
2382 | netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); |
2383 | stmmac_stop_rx(priv, priv->ioaddr, chan); |
2384 | } |
2385 | |
2386 | /** |
2387 | * stmmac_stop_tx_dma - stop TX DMA channel |
2388 | * @priv: driver private structure |
2389 | * @chan: TX channel index |
2390 | * Description: |
2391 | * This stops a TX DMA channel |
2392 | */ |
2393 | static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) |
2394 | { |
2395 | netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); |
2396 | stmmac_stop_tx(priv, priv->ioaddr, chan); |
2397 | } |
2398 | |
2399 | static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) |
2400 | { |
2401 | u32 rx_channels_count = priv->plat->rx_queues_to_use; |
2402 | u32 tx_channels_count = priv->plat->tx_queues_to_use; |
2403 | u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); |
2404 | u32 chan; |
2405 | |
2406 | for (chan = 0; chan < dma_csr_ch; chan++) { |
2407 | struct stmmac_channel *ch = &priv->channel[chan]; |
2408 | unsigned long flags; |
2409 | |
2410 | spin_lock_irqsave(&ch->lock, flags); |
2411 | stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); |
2412 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
2413 | } |
2414 | } |
2415 | |
2416 | /** |
2417 | * stmmac_start_all_dma - start all RX and TX DMA channels |
2418 | * @priv: driver private structure |
2419 | * Description: |
2420 | * This starts all the RX and TX DMA channels |
2421 | */ |
2422 | static void stmmac_start_all_dma(struct stmmac_priv *priv) |
2423 | { |
2424 | u32 rx_channels_count = priv->plat->rx_queues_to_use; |
2425 | u32 tx_channels_count = priv->plat->tx_queues_to_use; |
2426 | u32 chan = 0; |
2427 | |
2428 | for (chan = 0; chan < rx_channels_count; chan++) |
2429 | stmmac_start_rx_dma(priv, chan); |
2430 | |
2431 | for (chan = 0; chan < tx_channels_count; chan++) |
2432 | stmmac_start_tx_dma(priv, chan); |
2433 | } |
2434 | |
2435 | /** |
2436 | * stmmac_stop_all_dma - stop all RX and TX DMA channels |
2437 | * @priv: driver private structure |
2438 | * Description: |
2439 | * This stops the RX and TX DMA channels |
2440 | */ |
2441 | static void stmmac_stop_all_dma(struct stmmac_priv *priv) |
2442 | { |
2443 | u32 rx_channels_count = priv->plat->rx_queues_to_use; |
2444 | u32 tx_channels_count = priv->plat->tx_queues_to_use; |
2445 | u32 chan = 0; |
2446 | |
2447 | for (chan = 0; chan < rx_channels_count; chan++) |
2448 | stmmac_stop_rx_dma(priv, chan); |
2449 | |
2450 | for (chan = 0; chan < tx_channels_count; chan++) |
2451 | stmmac_stop_tx_dma(priv, chan); |
2452 | } |
2453 | |
2454 | /** |
2455 | * stmmac_dma_operation_mode - HW DMA operation mode |
2456 | * @priv: driver private structure |
2457 | * Description: it is used for configuring the DMA operation mode register in |
2458 | * order to program the tx/rx DMA thresholds or Store-And-Forward mode. |
2459 | */ |
2460 | static void stmmac_dma_operation_mode(struct stmmac_priv *priv) |
2461 | { |
2462 | u32 rx_channels_count = priv->plat->rx_queues_to_use; |
2463 | u32 tx_channels_count = priv->plat->tx_queues_to_use; |
2464 | int rxfifosz = priv->plat->rx_fifo_size; |
2465 | int txfifosz = priv->plat->tx_fifo_size; |
2466 | u32 txmode = 0; |
2467 | u32 rxmode = 0; |
2468 | u32 chan = 0; |
2469 | u8 qmode = 0; |
2470 | |
2471 | if (rxfifosz == 0) |
2472 | rxfifosz = priv->dma_cap.rx_fifo_size; |
2473 | if (txfifosz == 0) |
2474 | txfifosz = priv->dma_cap.tx_fifo_size; |
2475 | |
2476 | /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ |
2477 | if (priv->plat->has_gmac4 || priv->plat->has_xgmac) { |
2478 | rxfifosz /= rx_channels_count; |
2479 | txfifosz /= tx_channels_count; |
2480 | } |
2481 | |
2482 | if (priv->plat->force_thresh_dma_mode) { |
2483 | txmode = tc; |
2484 | rxmode = tc; |
2485 | } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { |
2486 | /* |
2487 | * In case of GMAC, SF mode can be enabled |
2488 | * to perform the TX COE in HW. This depends on: |
2489 | * 1) TX COE if actually supported |
2490 | * 2) There is no bugged Jumbo frame support |
2491 | * that needs to not insert csum in the TDES. |
2492 | */ |
2493 | txmode = SF_DMA_MODE; |
2494 | rxmode = SF_DMA_MODE; |
2495 | priv->xstats.threshold = SF_DMA_MODE; |
2496 | } else { |
2497 | txmode = tc; |
2498 | rxmode = SF_DMA_MODE; |
2499 | } |
2500 | |
2501 | /* configure all channels */ |
2502 | for (chan = 0; chan < rx_channels_count; chan++) { |
2503 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; |
2504 | u32 buf_size; |
2505 | |
2506 | qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; |
2507 | |
2508 | stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, |
2509 | rxfifosz, qmode); |
2510 | |
2511 | if (rx_q->xsk_pool) { |
2512 | buf_size = xsk_pool_get_rx_frame_size(pool: rx_q->xsk_pool); |
2513 | stmmac_set_dma_bfsize(priv, priv->ioaddr, |
2514 | buf_size, |
2515 | chan); |
2516 | } else { |
2517 | stmmac_set_dma_bfsize(priv, priv->ioaddr, |
2518 | priv->dma_conf.dma_buf_sz, |
2519 | chan); |
2520 | } |
2521 | } |
2522 | |
2523 | for (chan = 0; chan < tx_channels_count; chan++) { |
2524 | qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; |
2525 | |
2526 | stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, |
2527 | txfifosz, qmode); |
2528 | } |
2529 | } |
2530 | |
2531 | static void stmmac_xsk_request_timestamp(void *_priv) |
2532 | { |
2533 | struct stmmac_metadata_request *meta_req = _priv; |
2534 | |
2535 | stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc); |
2536 | *meta_req->set_ic = true; |
2537 | } |
2538 | |
2539 | static u64 stmmac_xsk_fill_timestamp(void *_priv) |
2540 | { |
2541 | struct stmmac_xsk_tx_complete *tx_compl = _priv; |
2542 | struct stmmac_priv *priv = tx_compl->priv; |
2543 | struct dma_desc *desc = tx_compl->desc; |
2544 | bool found = false; |
2545 | u64 ns = 0; |
2546 | |
2547 | if (!priv->hwts_tx_en) |
2548 | return 0; |
2549 | |
2550 | /* check tx tstamp status */ |
2551 | if (stmmac_get_tx_timestamp_status(priv, desc)) { |
2552 | stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); |
2553 | found = true; |
2554 | } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { |
2555 | found = true; |
2556 | } |
2557 | |
2558 | if (found) { |
2559 | ns -= priv->plat->cdc_error_adj; |
2560 | return ns_to_ktime(ns); |
2561 | } |
2562 | |
2563 | return 0; |
2564 | } |
2565 | |
2566 | static void stmmac_xsk_request_launch_time(u64 launch_time, void *_priv) |
2567 | { |
2568 | struct timespec64 ts = ns_to_timespec64(nsec: launch_time); |
2569 | struct stmmac_metadata_request *meta_req = _priv; |
2570 | |
2571 | if (meta_req->tbs & STMMAC_TBS_EN) |
2572 | stmmac_set_desc_tbs(meta_req->priv, meta_req->edesc, ts.tv_sec, |
2573 | ts.tv_nsec); |
2574 | } |
2575 | |
2576 | static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = { |
2577 | .tmo_request_timestamp = stmmac_xsk_request_timestamp, |
2578 | .tmo_fill_timestamp = stmmac_xsk_fill_timestamp, |
2579 | .tmo_request_launch_time = stmmac_xsk_request_launch_time, |
2580 | }; |
2581 | |
2582 | static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) |
2583 | { |
2584 | struct netdev_queue *nq = netdev_get_tx_queue(dev: priv->dev, index: queue); |
2585 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
2586 | struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; |
2587 | struct xsk_buff_pool *pool = tx_q->xsk_pool; |
2588 | unsigned int entry = tx_q->cur_tx; |
2589 | struct dma_desc *tx_desc = NULL; |
2590 | struct xdp_desc xdp_desc; |
2591 | bool work_done = true; |
2592 | u32 tx_set_ic_bit = 0; |
2593 | |
2594 | /* Avoids TX time-out as we are sharing with slow path */ |
2595 | txq_trans_cond_update(txq: nq); |
2596 | |
2597 | budget = min(budget, stmmac_tx_avail(priv, queue)); |
2598 | |
2599 | while (budget-- > 0) { |
2600 | struct stmmac_metadata_request meta_req; |
2601 | struct xsk_tx_metadata *meta = NULL; |
2602 | dma_addr_t dma_addr; |
2603 | bool set_ic; |
2604 | |
2605 | /* We are sharing with slow path and stop XSK TX desc submission when |
2606 | * available TX ring is less than threshold. |
2607 | */ |
2608 | if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || |
2609 | !netif_carrier_ok(dev: priv->dev)) { |
2610 | work_done = false; |
2611 | break; |
2612 | } |
2613 | |
2614 | if (!xsk_tx_peek_desc(pool, desc: &xdp_desc)) |
2615 | break; |
2616 | |
2617 | if (priv->est && priv->est->enable && |
2618 | priv->est->max_sdu[queue] && |
2619 | xdp_desc.len > priv->est->max_sdu[queue]) { |
2620 | priv->xstats.max_sdu_txq_drop[queue]++; |
2621 | continue; |
2622 | } |
2623 | |
2624 | if (likely(priv->extend_desc)) |
2625 | tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); |
2626 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
2627 | tx_desc = &tx_q->dma_entx[entry].basic; |
2628 | else |
2629 | tx_desc = tx_q->dma_tx + entry; |
2630 | |
2631 | dma_addr = xsk_buff_raw_get_dma(pool, addr: xdp_desc.addr); |
2632 | meta = xsk_buff_get_metadata(pool, addr: xdp_desc.addr); |
2633 | xsk_buff_raw_dma_sync_for_device(pool, dma: dma_addr, size: xdp_desc.len); |
2634 | |
2635 | tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; |
2636 | |
2637 | /* To return XDP buffer to XSK pool, we simple call |
2638 | * xsk_tx_completed(), so we don't need to fill up |
2639 | * 'buf' and 'xdpf'. |
2640 | */ |
2641 | tx_q->tx_skbuff_dma[entry].buf = 0; |
2642 | tx_q->xdpf[entry] = NULL; |
2643 | |
2644 | tx_q->tx_skbuff_dma[entry].map_as_page = false; |
2645 | tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; |
2646 | tx_q->tx_skbuff_dma[entry].last_segment = true; |
2647 | tx_q->tx_skbuff_dma[entry].is_jumbo = false; |
2648 | |
2649 | stmmac_set_desc_addr(priv, tx_desc, dma_addr); |
2650 | |
2651 | tx_q->tx_count_frames++; |
2652 | |
2653 | if (!priv->tx_coal_frames[queue]) |
2654 | set_ic = false; |
2655 | else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) |
2656 | set_ic = true; |
2657 | else |
2658 | set_ic = false; |
2659 | |
2660 | meta_req.priv = priv; |
2661 | meta_req.tx_desc = tx_desc; |
2662 | meta_req.set_ic = &set_ic; |
2663 | meta_req.tbs = tx_q->tbs; |
2664 | meta_req.edesc = &tx_q->dma_entx[entry]; |
2665 | xsk_tx_metadata_request(meta, ops: &stmmac_xsk_tx_metadata_ops, |
2666 | priv: &meta_req); |
2667 | if (set_ic) { |
2668 | tx_q->tx_count_frames = 0; |
2669 | stmmac_set_tx_ic(priv, tx_desc); |
2670 | tx_set_ic_bit++; |
2671 | } |
2672 | |
2673 | stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, |
2674 | true, priv->mode, true, true, |
2675 | xdp_desc.len); |
2676 | |
2677 | stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); |
2678 | |
2679 | xsk_tx_metadata_to_compl(meta, |
2680 | compl: &tx_q->tx_skbuff_dma[entry].xsk_meta); |
2681 | |
2682 | tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); |
2683 | entry = tx_q->cur_tx; |
2684 | } |
2685 | u64_stats_update_begin(syncp: &txq_stats->napi_syncp); |
2686 | u64_stats_add(p: &txq_stats->napi.tx_set_ic_bit, val: tx_set_ic_bit); |
2687 | u64_stats_update_end(syncp: &txq_stats->napi_syncp); |
2688 | |
2689 | if (tx_desc) { |
2690 | stmmac_flush_tx_descriptors(priv, queue); |
2691 | xsk_tx_release(pool); |
2692 | } |
2693 | |
2694 | /* Return true if all of the 3 conditions are met |
2695 | * a) TX Budget is still available |
2696 | * b) work_done = true when XSK TX desc peek is empty (no more |
2697 | * pending XSK TX for transmission) |
2698 | */ |
2699 | return !!budget && work_done; |
2700 | } |
2701 | |
2702 | static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) |
2703 | { |
2704 | if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { |
2705 | tc += 64; |
2706 | |
2707 | if (priv->plat->force_thresh_dma_mode) |
2708 | stmmac_set_dma_operation_mode(priv, txmode: tc, rxmode: tc, chan); |
2709 | else |
2710 | stmmac_set_dma_operation_mode(priv, txmode: tc, SF_DMA_MODE, |
2711 | chan); |
2712 | |
2713 | priv->xstats.threshold = tc; |
2714 | } |
2715 | } |
2716 | |
2717 | /** |
2718 | * stmmac_tx_clean - to manage the transmission completion |
2719 | * @priv: driver private structure |
2720 | * @budget: napi budget limiting this functions packet handling |
2721 | * @queue: TX queue index |
2722 | * @pending_packets: signal to arm the TX coal timer |
2723 | * Description: it reclaims the transmit resources after transmission completes. |
2724 | * If some packets still needs to be handled, due to TX coalesce, set |
2725 | * pending_packets to true to make NAPI arm the TX coal timer. |
2726 | */ |
2727 | static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, |
2728 | bool *pending_packets) |
2729 | { |
2730 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
2731 | struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; |
2732 | unsigned int bytes_compl = 0, pkts_compl = 0; |
2733 | unsigned int entry, xmits = 0, count = 0; |
2734 | u32 tx_packets = 0, tx_errors = 0; |
2735 | |
2736 | __netif_tx_lock_bh(txq: netdev_get_tx_queue(dev: priv->dev, index: queue)); |
2737 | |
2738 | tx_q->xsk_frames_done = 0; |
2739 | |
2740 | entry = tx_q->dirty_tx; |
2741 | |
2742 | /* Try to clean all TX complete frame in 1 shot */ |
2743 | while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { |
2744 | struct xdp_frame *xdpf; |
2745 | struct sk_buff *skb; |
2746 | struct dma_desc *p; |
2747 | int status; |
2748 | |
2749 | if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || |
2750 | tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { |
2751 | xdpf = tx_q->xdpf[entry]; |
2752 | skb = NULL; |
2753 | } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { |
2754 | xdpf = NULL; |
2755 | skb = tx_q->tx_skbuff[entry]; |
2756 | } else { |
2757 | xdpf = NULL; |
2758 | skb = NULL; |
2759 | } |
2760 | |
2761 | if (priv->extend_desc) |
2762 | p = (struct dma_desc *)(tx_q->dma_etx + entry); |
2763 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
2764 | p = &tx_q->dma_entx[entry].basic; |
2765 | else |
2766 | p = tx_q->dma_tx + entry; |
2767 | |
2768 | status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); |
2769 | /* Check if the descriptor is owned by the DMA */ |
2770 | if (unlikely(status & tx_dma_own)) |
2771 | break; |
2772 | |
2773 | count++; |
2774 | |
2775 | /* Make sure descriptor fields are read after reading |
2776 | * the own bit. |
2777 | */ |
2778 | dma_rmb(); |
2779 | |
2780 | /* Just consider the last segment and ...*/ |
2781 | if (likely(!(status & tx_not_ls))) { |
2782 | /* ... verify the status error condition */ |
2783 | if (unlikely(status & tx_err)) { |
2784 | tx_errors++; |
2785 | if (unlikely(status & tx_err_bump_tc)) |
2786 | stmmac_bump_dma_threshold(priv, chan: queue); |
2787 | } else { |
2788 | tx_packets++; |
2789 | } |
2790 | if (skb) { |
2791 | stmmac_get_tx_hwtstamp(priv, p, skb); |
2792 | } else if (tx_q->xsk_pool && |
2793 | xp_tx_metadata_enabled(pool: tx_q->xsk_pool)) { |
2794 | struct stmmac_xsk_tx_complete tx_compl = { |
2795 | .priv = priv, |
2796 | .desc = p, |
2797 | }; |
2798 | |
2799 | xsk_tx_metadata_complete(compl: &tx_q->tx_skbuff_dma[entry].xsk_meta, |
2800 | ops: &stmmac_xsk_tx_metadata_ops, |
2801 | priv: &tx_compl); |
2802 | } |
2803 | } |
2804 | |
2805 | if (likely(tx_q->tx_skbuff_dma[entry].buf && |
2806 | tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { |
2807 | if (tx_q->tx_skbuff_dma[entry].map_as_page) |
2808 | dma_unmap_page(priv->device, |
2809 | tx_q->tx_skbuff_dma[entry].buf, |
2810 | tx_q->tx_skbuff_dma[entry].len, |
2811 | DMA_TO_DEVICE); |
2812 | else |
2813 | dma_unmap_single(priv->device, |
2814 | tx_q->tx_skbuff_dma[entry].buf, |
2815 | tx_q->tx_skbuff_dma[entry].len, |
2816 | DMA_TO_DEVICE); |
2817 | tx_q->tx_skbuff_dma[entry].buf = 0; |
2818 | tx_q->tx_skbuff_dma[entry].len = 0; |
2819 | tx_q->tx_skbuff_dma[entry].map_as_page = false; |
2820 | } |
2821 | |
2822 | stmmac_clean_desc3(priv, tx_q, p); |
2823 | |
2824 | tx_q->tx_skbuff_dma[entry].last_segment = false; |
2825 | tx_q->tx_skbuff_dma[entry].is_jumbo = false; |
2826 | |
2827 | if (xdpf && |
2828 | tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { |
2829 | xdp_return_frame_rx_napi(xdpf); |
2830 | tx_q->xdpf[entry] = NULL; |
2831 | } |
2832 | |
2833 | if (xdpf && |
2834 | tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { |
2835 | xdp_return_frame(xdpf); |
2836 | tx_q->xdpf[entry] = NULL; |
2837 | } |
2838 | |
2839 | if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) |
2840 | tx_q->xsk_frames_done++; |
2841 | |
2842 | if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { |
2843 | if (likely(skb)) { |
2844 | pkts_compl++; |
2845 | bytes_compl += skb->len; |
2846 | dev_consume_skb_any(skb); |
2847 | tx_q->tx_skbuff[entry] = NULL; |
2848 | } |
2849 | } |
2850 | |
2851 | stmmac_release_tx_desc(priv, p, priv->mode); |
2852 | |
2853 | entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); |
2854 | } |
2855 | tx_q->dirty_tx = entry; |
2856 | |
2857 | netdev_tx_completed_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: queue), |
2858 | pkts: pkts_compl, bytes: bytes_compl); |
2859 | |
2860 | if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, |
2861 | queue))) && |
2862 | stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { |
2863 | |
2864 | netif_dbg(priv, tx_done, priv->dev, |
2865 | "%s: restart transmit\n", __func__); |
2866 | netif_tx_wake_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: queue)); |
2867 | } |
2868 | |
2869 | if (tx_q->xsk_pool) { |
2870 | bool work_done; |
2871 | |
2872 | if (tx_q->xsk_frames_done) |
2873 | xsk_tx_completed(pool: tx_q->xsk_pool, nb_entries: tx_q->xsk_frames_done); |
2874 | |
2875 | if (xsk_uses_need_wakeup(pool: tx_q->xsk_pool)) |
2876 | xsk_set_tx_need_wakeup(pool: tx_q->xsk_pool); |
2877 | |
2878 | /* For XSK TX, we try to send as many as possible. |
2879 | * If XSK work done (XSK TX desc empty and budget still |
2880 | * available), return "budget - 1" to reenable TX IRQ. |
2881 | * Else, return "budget" to make NAPI continue polling. |
2882 | */ |
2883 | work_done = stmmac_xdp_xmit_zc(priv, queue, |
2884 | STMMAC_XSK_TX_BUDGET_MAX); |
2885 | if (work_done) |
2886 | xmits = budget - 1; |
2887 | else |
2888 | xmits = budget; |
2889 | } |
2890 | |
2891 | if (priv->eee_sw_timer_en && !priv->tx_path_in_lpi_mode) |
2892 | stmmac_restart_sw_lpi_timer(priv); |
2893 | |
2894 | /* We still have pending packets, let's call for a new scheduling */ |
2895 | if (tx_q->dirty_tx != tx_q->cur_tx) |
2896 | *pending_packets = true; |
2897 | |
2898 | u64_stats_update_begin(syncp: &txq_stats->napi_syncp); |
2899 | u64_stats_add(p: &txq_stats->napi.tx_packets, val: tx_packets); |
2900 | u64_stats_add(p: &txq_stats->napi.tx_pkt_n, val: tx_packets); |
2901 | u64_stats_inc(p: &txq_stats->napi.tx_clean); |
2902 | u64_stats_update_end(syncp: &txq_stats->napi_syncp); |
2903 | |
2904 | priv->xstats.tx_errors += tx_errors; |
2905 | |
2906 | __netif_tx_unlock_bh(txq: netdev_get_tx_queue(dev: priv->dev, index: queue)); |
2907 | |
2908 | /* Combine decisions from TX clean and XSK TX */ |
2909 | return max(count, xmits); |
2910 | } |
2911 | |
2912 | /** |
2913 | * stmmac_tx_err - to manage the tx error |
2914 | * @priv: driver private structure |
2915 | * @chan: channel index |
2916 | * Description: it cleans the descriptors and restarts the transmission |
2917 | * in case of transmission errors. |
2918 | */ |
2919 | static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) |
2920 | { |
2921 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; |
2922 | |
2923 | netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: chan)); |
2924 | |
2925 | stmmac_stop_tx_dma(priv, chan); |
2926 | dma_free_tx_skbufs(priv, dma_conf: &priv->dma_conf, queue: chan); |
2927 | stmmac_clear_tx_descriptors(priv, dma_conf: &priv->dma_conf, queue: chan); |
2928 | stmmac_reset_tx_queue(priv, queue: chan); |
2929 | stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, |
2930 | tx_q->dma_tx_phy, chan); |
2931 | stmmac_start_tx_dma(priv, chan); |
2932 | |
2933 | priv->xstats.tx_errors++; |
2934 | netif_tx_wake_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: chan)); |
2935 | } |
2936 | |
2937 | /** |
2938 | * stmmac_set_dma_operation_mode - Set DMA operation mode by channel |
2939 | * @priv: driver private structure |
2940 | * @txmode: TX operating mode |
2941 | * @rxmode: RX operating mode |
2942 | * @chan: channel index |
2943 | * Description: it is used for configuring of the DMA operation mode in |
2944 | * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward |
2945 | * mode. |
2946 | */ |
2947 | static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, |
2948 | u32 rxmode, u32 chan) |
2949 | { |
2950 | u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; |
2951 | u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; |
2952 | u32 rx_channels_count = priv->plat->rx_queues_to_use; |
2953 | u32 tx_channels_count = priv->plat->tx_queues_to_use; |
2954 | int rxfifosz = priv->plat->rx_fifo_size; |
2955 | int txfifosz = priv->plat->tx_fifo_size; |
2956 | |
2957 | if (rxfifosz == 0) |
2958 | rxfifosz = priv->dma_cap.rx_fifo_size; |
2959 | if (txfifosz == 0) |
2960 | txfifosz = priv->dma_cap.tx_fifo_size; |
2961 | |
2962 | /* Adjust for real per queue fifo size */ |
2963 | rxfifosz /= rx_channels_count; |
2964 | txfifosz /= tx_channels_count; |
2965 | |
2966 | stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); |
2967 | stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); |
2968 | } |
2969 | |
2970 | static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) |
2971 | { |
2972 | int ret; |
2973 | |
2974 | ret = stmmac_safety_feat_irq_status(priv, priv->dev, |
2975 | priv->ioaddr, priv->dma_cap.asp, &priv->sstats); |
2976 | if (ret && (ret != -EINVAL)) { |
2977 | stmmac_global_err(priv); |
2978 | return true; |
2979 | } |
2980 | |
2981 | return false; |
2982 | } |
2983 | |
2984 | static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) |
2985 | { |
2986 | int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, |
2987 | &priv->xstats, chan, dir); |
2988 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; |
2989 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; |
2990 | struct stmmac_channel *ch = &priv->channel[chan]; |
2991 | struct napi_struct *rx_napi; |
2992 | struct napi_struct *tx_napi; |
2993 | unsigned long flags; |
2994 | |
2995 | rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; |
2996 | tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; |
2997 | |
2998 | if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { |
2999 | if (napi_schedule_prep(n: rx_napi)) { |
3000 | spin_lock_irqsave(&ch->lock, flags); |
3001 | stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); |
3002 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
3003 | __napi_schedule(n: rx_napi); |
3004 | } |
3005 | } |
3006 | |
3007 | if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { |
3008 | if (napi_schedule_prep(n: tx_napi)) { |
3009 | spin_lock_irqsave(&ch->lock, flags); |
3010 | stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); |
3011 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
3012 | __napi_schedule(n: tx_napi); |
3013 | } |
3014 | } |
3015 | |
3016 | return status; |
3017 | } |
3018 | |
3019 | /** |
3020 | * stmmac_dma_interrupt - DMA ISR |
3021 | * @priv: driver private structure |
3022 | * Description: this is the DMA ISR. It is called by the main ISR. |
3023 | * It calls the dwmac dma routine and schedule poll method in case of some |
3024 | * work can be done. |
3025 | */ |
3026 | static void stmmac_dma_interrupt(struct stmmac_priv *priv) |
3027 | { |
3028 | u32 tx_channel_count = priv->plat->tx_queues_to_use; |
3029 | u32 rx_channel_count = priv->plat->rx_queues_to_use; |
3030 | u32 channels_to_check = tx_channel_count > rx_channel_count ? |
3031 | tx_channel_count : rx_channel_count; |
3032 | u32 chan; |
3033 | int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; |
3034 | |
3035 | /* Make sure we never check beyond our status buffer. */ |
3036 | if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) |
3037 | channels_to_check = ARRAY_SIZE(status); |
3038 | |
3039 | for (chan = 0; chan < channels_to_check; chan++) |
3040 | status[chan] = stmmac_napi_check(priv, chan, |
3041 | dir: DMA_DIR_RXTX); |
3042 | |
3043 | for (chan = 0; chan < tx_channel_count; chan++) { |
3044 | if (unlikely(status[chan] & tx_hard_error_bump_tc)) { |
3045 | /* Try to bump up the dma threshold on this failure */ |
3046 | stmmac_bump_dma_threshold(priv, chan); |
3047 | } else if (unlikely(status[chan] == tx_hard_error)) { |
3048 | stmmac_tx_err(priv, chan); |
3049 | } |
3050 | } |
3051 | } |
3052 | |
3053 | /** |
3054 | * stmmac_mmc_setup: setup the Mac Management Counters (MMC) |
3055 | * @priv: driver private structure |
3056 | * Description: this masks the MMC irq, in fact, the counters are managed in SW. |
3057 | */ |
3058 | static void stmmac_mmc_setup(struct stmmac_priv *priv) |
3059 | { |
3060 | unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | |
3061 | MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; |
3062 | |
3063 | stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); |
3064 | |
3065 | if (priv->dma_cap.rmon) { |
3066 | stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); |
3067 | memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); |
3068 | } else |
3069 | netdev_info(dev: priv->dev, format: "No MAC Management Counters available\n"); |
3070 | } |
3071 | |
3072 | /** |
3073 | * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. |
3074 | * @priv: driver private structure |
3075 | * Description: |
3076 | * new GMAC chip generations have a new register to indicate the |
3077 | * presence of the optional feature/functions. |
3078 | * This can be also used to override the value passed through the |
3079 | * platform and necessary for old MAC10/100 and GMAC chips. |
3080 | */ |
3081 | static int stmmac_get_hw_features(struct stmmac_priv *priv) |
3082 | { |
3083 | return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; |
3084 | } |
3085 | |
3086 | /** |
3087 | * stmmac_check_ether_addr - check if the MAC addr is valid |
3088 | * @priv: driver private structure |
3089 | * Description: |
3090 | * it is to verify if the MAC address is valid, in case of failures it |
3091 | * generates a random MAC address |
3092 | */ |
3093 | static void stmmac_check_ether_addr(struct stmmac_priv *priv) |
3094 | { |
3095 | u8 addr[ETH_ALEN]; |
3096 | |
3097 | if (!is_valid_ether_addr(addr: priv->dev->dev_addr)) { |
3098 | stmmac_get_umac_addr(priv, priv->hw, addr, 0); |
3099 | if (is_valid_ether_addr(addr)) |
3100 | eth_hw_addr_set(dev: priv->dev, addr); |
3101 | else |
3102 | eth_hw_addr_random(dev: priv->dev); |
3103 | dev_info(priv->device, "device MAC address %pM\n", |
3104 | priv->dev->dev_addr); |
3105 | } |
3106 | } |
3107 | |
3108 | /** |
3109 | * stmmac_init_dma_engine - DMA init. |
3110 | * @priv: driver private structure |
3111 | * Description: |
3112 | * It inits the DMA invoking the specific MAC/GMAC callback. |
3113 | * Some DMA parameters can be passed from the platform; |
3114 | * in case of these are not passed a default is kept for the MAC or GMAC. |
3115 | */ |
3116 | static int stmmac_init_dma_engine(struct stmmac_priv *priv) |
3117 | { |
3118 | u32 rx_channels_count = priv->plat->rx_queues_to_use; |
3119 | u32 tx_channels_count = priv->plat->tx_queues_to_use; |
3120 | u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); |
3121 | struct stmmac_rx_queue *rx_q; |
3122 | struct stmmac_tx_queue *tx_q; |
3123 | u32 chan = 0; |
3124 | int ret = 0; |
3125 | |
3126 | if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { |
3127 | netdev_err(dev: priv->dev, format: "Invalid DMA configuration\n"); |
3128 | return -EINVAL; |
3129 | } |
3130 | |
3131 | if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) |
3132 | priv->plat->dma_cfg->atds = 1; |
3133 | |
3134 | ret = stmmac_reset(priv, ioaddr: priv->ioaddr); |
3135 | if (ret) { |
3136 | netdev_err(dev: priv->dev, format: "Failed to reset the dma\n"); |
3137 | return ret; |
3138 | } |
3139 | |
3140 | /* DMA Configuration */ |
3141 | stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg); |
3142 | |
3143 | if (priv->plat->axi) |
3144 | stmmac_axi(priv, priv->ioaddr, priv->plat->axi); |
3145 | |
3146 | /* DMA CSR Channel configuration */ |
3147 | for (chan = 0; chan < dma_csr_ch; chan++) { |
3148 | stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); |
3149 | stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); |
3150 | } |
3151 | |
3152 | /* DMA RX Channel Configuration */ |
3153 | for (chan = 0; chan < rx_channels_count; chan++) { |
3154 | rx_q = &priv->dma_conf.rx_queue[chan]; |
3155 | |
3156 | stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, |
3157 | rx_q->dma_rx_phy, chan); |
3158 | |
3159 | rx_q->rx_tail_addr = rx_q->dma_rx_phy + |
3160 | (rx_q->buf_alloc_num * |
3161 | sizeof(struct dma_desc)); |
3162 | stmmac_set_rx_tail_ptr(priv, priv->ioaddr, |
3163 | rx_q->rx_tail_addr, chan); |
3164 | } |
3165 | |
3166 | /* DMA TX Channel Configuration */ |
3167 | for (chan = 0; chan < tx_channels_count; chan++) { |
3168 | tx_q = &priv->dma_conf.tx_queue[chan]; |
3169 | |
3170 | stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, |
3171 | tx_q->dma_tx_phy, chan); |
3172 | |
3173 | tx_q->tx_tail_addr = tx_q->dma_tx_phy; |
3174 | stmmac_set_tx_tail_ptr(priv, priv->ioaddr, |
3175 | tx_q->tx_tail_addr, chan); |
3176 | } |
3177 | |
3178 | return ret; |
3179 | } |
3180 | |
3181 | static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) |
3182 | { |
3183 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
3184 | u32 tx_coal_timer = priv->tx_coal_timer[queue]; |
3185 | struct stmmac_channel *ch; |
3186 | struct napi_struct *napi; |
3187 | |
3188 | if (!tx_coal_timer) |
3189 | return; |
3190 | |
3191 | ch = &priv->channel[tx_q->queue_index]; |
3192 | napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; |
3193 | |
3194 | /* Arm timer only if napi is not already scheduled. |
3195 | * Try to cancel any timer if napi is scheduled, timer will be armed |
3196 | * again in the next scheduled napi. |
3197 | */ |
3198 | if (unlikely(!napi_is_scheduled(napi))) |
3199 | hrtimer_start(timer: &tx_q->txtimer, |
3200 | STMMAC_COAL_TIMER(tx_coal_timer), |
3201 | mode: HRTIMER_MODE_REL); |
3202 | else |
3203 | hrtimer_try_to_cancel(timer: &tx_q->txtimer); |
3204 | } |
3205 | |
3206 | /** |
3207 | * stmmac_tx_timer - mitigation sw timer for tx. |
3208 | * @t: data pointer |
3209 | * Description: |
3210 | * This is the timer handler to directly invoke the stmmac_tx_clean. |
3211 | */ |
3212 | static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) |
3213 | { |
3214 | struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); |
3215 | struct stmmac_priv *priv = tx_q->priv_data; |
3216 | struct stmmac_channel *ch; |
3217 | struct napi_struct *napi; |
3218 | |
3219 | ch = &priv->channel[tx_q->queue_index]; |
3220 | napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; |
3221 | |
3222 | if (likely(napi_schedule_prep(napi))) { |
3223 | unsigned long flags; |
3224 | |
3225 | spin_lock_irqsave(&ch->lock, flags); |
3226 | stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); |
3227 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
3228 | __napi_schedule(n: napi); |
3229 | } |
3230 | |
3231 | return HRTIMER_NORESTART; |
3232 | } |
3233 | |
3234 | /** |
3235 | * stmmac_init_coalesce - init mitigation options. |
3236 | * @priv: driver private structure |
3237 | * Description: |
3238 | * This inits the coalesce parameters: i.e. timer rate, |
3239 | * timer handler and default threshold used for enabling the |
3240 | * interrupt on completion bit. |
3241 | */ |
3242 | static void stmmac_init_coalesce(struct stmmac_priv *priv) |
3243 | { |
3244 | u32 tx_channel_count = priv->plat->tx_queues_to_use; |
3245 | u32 rx_channel_count = priv->plat->rx_queues_to_use; |
3246 | u32 chan; |
3247 | |
3248 | for (chan = 0; chan < tx_channel_count; chan++) { |
3249 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; |
3250 | |
3251 | priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; |
3252 | priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; |
3253 | |
3254 | hrtimer_setup(timer: &tx_q->txtimer, function: stmmac_tx_timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL); |
3255 | } |
3256 | |
3257 | for (chan = 0; chan < rx_channel_count; chan++) |
3258 | priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; |
3259 | } |
3260 | |
3261 | static void stmmac_set_rings_length(struct stmmac_priv *priv) |
3262 | { |
3263 | u32 rx_channels_count = priv->plat->rx_queues_to_use; |
3264 | u32 tx_channels_count = priv->plat->tx_queues_to_use; |
3265 | u32 chan; |
3266 | |
3267 | /* set TX ring length */ |
3268 | for (chan = 0; chan < tx_channels_count; chan++) |
3269 | stmmac_set_tx_ring_len(priv, priv->ioaddr, |
3270 | (priv->dma_conf.dma_tx_size - 1), chan); |
3271 | |
3272 | /* set RX ring length */ |
3273 | for (chan = 0; chan < rx_channels_count; chan++) |
3274 | stmmac_set_rx_ring_len(priv, priv->ioaddr, |
3275 | (priv->dma_conf.dma_rx_size - 1), chan); |
3276 | } |
3277 | |
3278 | /** |
3279 | * stmmac_set_tx_queue_weight - Set TX queue weight |
3280 | * @priv: driver private structure |
3281 | * Description: It is used for setting TX queues weight |
3282 | */ |
3283 | static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) |
3284 | { |
3285 | u32 tx_queues_count = priv->plat->tx_queues_to_use; |
3286 | u32 weight; |
3287 | u32 queue; |
3288 | |
3289 | for (queue = 0; queue < tx_queues_count; queue++) { |
3290 | weight = priv->plat->tx_queues_cfg[queue].weight; |
3291 | stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); |
3292 | } |
3293 | } |
3294 | |
3295 | /** |
3296 | * stmmac_configure_cbs - Configure CBS in TX queue |
3297 | * @priv: driver private structure |
3298 | * Description: It is used for configuring CBS in AVB TX queues |
3299 | */ |
3300 | static void stmmac_configure_cbs(struct stmmac_priv *priv) |
3301 | { |
3302 | u32 tx_queues_count = priv->plat->tx_queues_to_use; |
3303 | u32 mode_to_use; |
3304 | u32 queue; |
3305 | |
3306 | /* queue 0 is reserved for legacy traffic */ |
3307 | for (queue = 1; queue < tx_queues_count; queue++) { |
3308 | mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; |
3309 | if (mode_to_use == MTL_QUEUE_DCB) |
3310 | continue; |
3311 | |
3312 | stmmac_config_cbs(priv, priv->hw, |
3313 | priv->plat->tx_queues_cfg[queue].send_slope, |
3314 | priv->plat->tx_queues_cfg[queue].idle_slope, |
3315 | priv->plat->tx_queues_cfg[queue].high_credit, |
3316 | priv->plat->tx_queues_cfg[queue].low_credit, |
3317 | queue); |
3318 | } |
3319 | } |
3320 | |
3321 | /** |
3322 | * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel |
3323 | * @priv: driver private structure |
3324 | * Description: It is used for mapping RX queues to RX dma channels |
3325 | */ |
3326 | static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) |
3327 | { |
3328 | u32 rx_queues_count = priv->plat->rx_queues_to_use; |
3329 | u32 queue; |
3330 | u32 chan; |
3331 | |
3332 | for (queue = 0; queue < rx_queues_count; queue++) { |
3333 | chan = priv->plat->rx_queues_cfg[queue].chan; |
3334 | stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); |
3335 | } |
3336 | } |
3337 | |
3338 | /** |
3339 | * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority |
3340 | * @priv: driver private structure |
3341 | * Description: It is used for configuring the RX Queue Priority |
3342 | */ |
3343 | static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) |
3344 | { |
3345 | u32 rx_queues_count = priv->plat->rx_queues_to_use; |
3346 | u32 queue; |
3347 | u32 prio; |
3348 | |
3349 | for (queue = 0; queue < rx_queues_count; queue++) { |
3350 | if (!priv->plat->rx_queues_cfg[queue].use_prio) |
3351 | continue; |
3352 | |
3353 | prio = priv->plat->rx_queues_cfg[queue].prio; |
3354 | stmmac_rx_queue_prio(priv, priv->hw, prio, queue); |
3355 | } |
3356 | } |
3357 | |
3358 | /** |
3359 | * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority |
3360 | * @priv: driver private structure |
3361 | * Description: It is used for configuring the TX Queue Priority |
3362 | */ |
3363 | static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) |
3364 | { |
3365 | u32 tx_queues_count = priv->plat->tx_queues_to_use; |
3366 | u32 queue; |
3367 | u32 prio; |
3368 | |
3369 | for (queue = 0; queue < tx_queues_count; queue++) { |
3370 | if (!priv->plat->tx_queues_cfg[queue].use_prio) |
3371 | continue; |
3372 | |
3373 | prio = priv->plat->tx_queues_cfg[queue].prio; |
3374 | stmmac_tx_queue_prio(priv, priv->hw, prio, queue); |
3375 | } |
3376 | } |
3377 | |
3378 | /** |
3379 | * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing |
3380 | * @priv: driver private structure |
3381 | * Description: It is used for configuring the RX queue routing |
3382 | */ |
3383 | static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) |
3384 | { |
3385 | u32 rx_queues_count = priv->plat->rx_queues_to_use; |
3386 | u32 queue; |
3387 | u8 packet; |
3388 | |
3389 | for (queue = 0; queue < rx_queues_count; queue++) { |
3390 | /* no specific packet type routing specified for the queue */ |
3391 | if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) |
3392 | continue; |
3393 | |
3394 | packet = priv->plat->rx_queues_cfg[queue].pkt_route; |
3395 | stmmac_rx_queue_routing(priv, priv->hw, packet, queue); |
3396 | } |
3397 | } |
3398 | |
3399 | static void stmmac_mac_config_rss(struct stmmac_priv *priv) |
3400 | { |
3401 | if (!priv->dma_cap.rssen || !priv->plat->rss_en) { |
3402 | priv->rss.enable = false; |
3403 | return; |
3404 | } |
3405 | |
3406 | if (priv->dev->features & NETIF_F_RXHASH) |
3407 | priv->rss.enable = true; |
3408 | else |
3409 | priv->rss.enable = false; |
3410 | |
3411 | stmmac_rss_configure(priv, priv->hw, &priv->rss, |
3412 | priv->plat->rx_queues_to_use); |
3413 | } |
3414 | |
3415 | /** |
3416 | * stmmac_mtl_configuration - Configure MTL |
3417 | * @priv: driver private structure |
3418 | * Description: It is used for configurring MTL |
3419 | */ |
3420 | static void stmmac_mtl_configuration(struct stmmac_priv *priv) |
3421 | { |
3422 | u32 rx_queues_count = priv->plat->rx_queues_to_use; |
3423 | u32 tx_queues_count = priv->plat->tx_queues_to_use; |
3424 | |
3425 | if (tx_queues_count > 1) |
3426 | stmmac_set_tx_queue_weight(priv); |
3427 | |
3428 | /* Configure MTL RX algorithms */ |
3429 | if (rx_queues_count > 1) |
3430 | stmmac_prog_mtl_rx_algorithms(priv, priv->hw, |
3431 | priv->plat->rx_sched_algorithm); |
3432 | |
3433 | /* Configure MTL TX algorithms */ |
3434 | if (tx_queues_count > 1) |
3435 | stmmac_prog_mtl_tx_algorithms(priv, priv->hw, |
3436 | priv->plat->tx_sched_algorithm); |
3437 | |
3438 | /* Configure CBS in AVB TX queues */ |
3439 | if (tx_queues_count > 1) |
3440 | stmmac_configure_cbs(priv); |
3441 | |
3442 | /* Map RX MTL to DMA channels */ |
3443 | stmmac_rx_queue_dma_chan_map(priv); |
3444 | |
3445 | /* Enable MAC RX Queues */ |
3446 | stmmac_mac_enable_rx_queues(priv); |
3447 | |
3448 | /* Set RX priorities */ |
3449 | if (rx_queues_count > 1) |
3450 | stmmac_mac_config_rx_queues_prio(priv); |
3451 | |
3452 | /* Set TX priorities */ |
3453 | if (tx_queues_count > 1) |
3454 | stmmac_mac_config_tx_queues_prio(priv); |
3455 | |
3456 | /* Set RX routing */ |
3457 | if (rx_queues_count > 1) |
3458 | stmmac_mac_config_rx_queues_routing(priv); |
3459 | |
3460 | /* Receive Side Scaling */ |
3461 | if (rx_queues_count > 1) |
3462 | stmmac_mac_config_rss(priv); |
3463 | } |
3464 | |
3465 | static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) |
3466 | { |
3467 | if (priv->dma_cap.asp) { |
3468 | netdev_info(dev: priv->dev, format: "Enabling Safety Features\n"); |
3469 | stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, |
3470 | priv->plat->safety_feat_cfg); |
3471 | } else { |
3472 | netdev_info(dev: priv->dev, format: "No Safety Features support found\n"); |
3473 | } |
3474 | } |
3475 | |
3476 | /** |
3477 | * stmmac_hw_setup - setup mac in a usable state. |
3478 | * @dev : pointer to the device structure. |
3479 | * @ptp_register: register PTP if set |
3480 | * Description: |
3481 | * this is the main function to setup the HW in a usable state because the |
3482 | * dma engine is reset, the core registers are configured (e.g. AXI, |
3483 | * Checksum features, timers). The DMA is ready to start receiving and |
3484 | * transmitting. |
3485 | * Return value: |
3486 | * 0 on success and an appropriate (-)ve integer as defined in errno.h |
3487 | * file on failure. |
3488 | */ |
3489 | static int stmmac_hw_setup(struct net_device *dev, bool ptp_register) |
3490 | { |
3491 | struct stmmac_priv *priv = netdev_priv(dev); |
3492 | u32 rx_cnt = priv->plat->rx_queues_to_use; |
3493 | u32 tx_cnt = priv->plat->tx_queues_to_use; |
3494 | bool sph_en; |
3495 | u32 chan; |
3496 | int ret; |
3497 | |
3498 | /* Make sure RX clock is enabled */ |
3499 | if (priv->hw->phylink_pcs) |
3500 | phylink_pcs_pre_init(pl: priv->phylink, pcs: priv->hw->phylink_pcs); |
3501 | |
3502 | /* Note that clk_rx_i must be running for reset to complete. This |
3503 | * clock may also be required when setting the MAC address. |
3504 | * |
3505 | * Block the receive clock stop for LPI mode at the PHY in case |
3506 | * the link is established with EEE mode active. |
3507 | */ |
3508 | phylink_rx_clk_stop_block(priv->phylink); |
3509 | |
3510 | /* DMA initialization and SW reset */ |
3511 | ret = stmmac_init_dma_engine(priv); |
3512 | if (ret < 0) { |
3513 | phylink_rx_clk_stop_unblock(priv->phylink); |
3514 | netdev_err(dev: priv->dev, format: "%s: DMA engine initialization failed\n", |
3515 | __func__); |
3516 | return ret; |
3517 | } |
3518 | |
3519 | /* Copy the MAC addr into the HW */ |
3520 | stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); |
3521 | phylink_rx_clk_stop_unblock(priv->phylink); |
3522 | |
3523 | /* PS and related bits will be programmed according to the speed */ |
3524 | if (priv->hw->pcs) { |
3525 | int speed = priv->plat->mac_port_sel_speed; |
3526 | |
3527 | if ((speed == SPEED_10) || (speed == SPEED_100) || |
3528 | (speed == SPEED_1000)) { |
3529 | priv->hw->ps = speed; |
3530 | } else { |
3531 | dev_warn(priv->device, "invalid port speed\n"); |
3532 | priv->hw->ps = 0; |
3533 | } |
3534 | } |
3535 | |
3536 | /* Initialize the MAC Core */ |
3537 | stmmac_core_init(priv, priv->hw, dev); |
3538 | |
3539 | /* Initialize MTL*/ |
3540 | stmmac_mtl_configuration(priv); |
3541 | |
3542 | /* Initialize Safety Features */ |
3543 | stmmac_safety_feat_configuration(priv); |
3544 | |
3545 | ret = stmmac_rx_ipc(priv, priv->hw); |
3546 | if (!ret) { |
3547 | netdev_warn(dev: priv->dev, format: "RX IPC Checksum Offload disabled\n"); |
3548 | priv->plat->rx_coe = STMMAC_RX_COE_NONE; |
3549 | priv->hw->rx_csum = 0; |
3550 | } |
3551 | |
3552 | /* Enable the MAC Rx/Tx */ |
3553 | stmmac_mac_set(priv, priv->ioaddr, true); |
3554 | |
3555 | /* Set the HW DMA mode and the COE */ |
3556 | stmmac_dma_operation_mode(priv); |
3557 | |
3558 | stmmac_mmc_setup(priv); |
3559 | |
3560 | if (ptp_register) { |
3561 | ret = clk_prepare_enable(clk: priv->plat->clk_ptp_ref); |
3562 | if (ret < 0) |
3563 | netdev_warn(dev: priv->dev, |
3564 | format: "failed to enable PTP reference clock: %pe\n", |
3565 | ERR_PTR(error: ret)); |
3566 | } |
3567 | |
3568 | ret = stmmac_init_ptp(priv); |
3569 | if (ret == -EOPNOTSUPP) |
3570 | netdev_info(dev: priv->dev, format: "PTP not supported by HW\n"); |
3571 | else if (ret) |
3572 | netdev_warn(dev: priv->dev, format: "PTP init failed\n"); |
3573 | else if (ptp_register) |
3574 | stmmac_ptp_register(priv); |
3575 | |
3576 | if (priv->use_riwt) { |
3577 | u32 queue; |
3578 | |
3579 | for (queue = 0; queue < rx_cnt; queue++) { |
3580 | if (!priv->rx_riwt[queue]) |
3581 | priv->rx_riwt[queue] = DEF_DMA_RIWT; |
3582 | |
3583 | stmmac_rx_watchdog(priv, priv->ioaddr, |
3584 | priv->rx_riwt[queue], queue); |
3585 | } |
3586 | } |
3587 | |
3588 | if (priv->hw->pcs) |
3589 | stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0); |
3590 | |
3591 | /* set TX and RX rings length */ |
3592 | stmmac_set_rings_length(priv); |
3593 | |
3594 | /* Enable TSO */ |
3595 | if (priv->tso) { |
3596 | for (chan = 0; chan < tx_cnt; chan++) { |
3597 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; |
3598 | |
3599 | /* TSO and TBS cannot co-exist */ |
3600 | if (tx_q->tbs & STMMAC_TBS_AVAIL) |
3601 | continue; |
3602 | |
3603 | stmmac_enable_tso(priv, priv->ioaddr, 1, chan); |
3604 | } |
3605 | } |
3606 | |
3607 | /* Enable Split Header */ |
3608 | sph_en = (priv->hw->rx_csum > 0) && priv->sph; |
3609 | for (chan = 0; chan < rx_cnt; chan++) |
3610 | stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); |
3611 | |
3612 | |
3613 | /* VLAN Tag Insertion */ |
3614 | if (priv->dma_cap.vlins) |
3615 | stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); |
3616 | |
3617 | /* TBS */ |
3618 | for (chan = 0; chan < tx_cnt; chan++) { |
3619 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; |
3620 | int enable = tx_q->tbs & STMMAC_TBS_AVAIL; |
3621 | |
3622 | stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); |
3623 | } |
3624 | |
3625 | /* Configure real RX and TX queues */ |
3626 | netif_set_real_num_rx_queues(dev, rxq: priv->plat->rx_queues_to_use); |
3627 | netif_set_real_num_tx_queues(dev, txq: priv->plat->tx_queues_to_use); |
3628 | |
3629 | /* Start the ball rolling... */ |
3630 | stmmac_start_all_dma(priv); |
3631 | |
3632 | phylink_rx_clk_stop_block(priv->phylink); |
3633 | stmmac_set_hw_vlan_mode(priv, priv->hw); |
3634 | phylink_rx_clk_stop_unblock(priv->phylink); |
3635 | |
3636 | return 0; |
3637 | } |
3638 | |
3639 | static void stmmac_hw_teardown(struct net_device *dev) |
3640 | { |
3641 | struct stmmac_priv *priv = netdev_priv(dev); |
3642 | |
3643 | clk_disable_unprepare(clk: priv->plat->clk_ptp_ref); |
3644 | } |
3645 | |
3646 | static void stmmac_free_irq(struct net_device *dev, |
3647 | enum request_irq_err irq_err, int irq_idx) |
3648 | { |
3649 | struct stmmac_priv *priv = netdev_priv(dev); |
3650 | int j; |
3651 | |
3652 | switch (irq_err) { |
3653 | case REQ_IRQ_ERR_ALL: |
3654 | irq_idx = priv->plat->tx_queues_to_use; |
3655 | fallthrough; |
3656 | case REQ_IRQ_ERR_TX: |
3657 | for (j = irq_idx - 1; j >= 0; j--) { |
3658 | if (priv->tx_irq[j] > 0) { |
3659 | irq_set_affinity_hint(irq: priv->tx_irq[j], NULL); |
3660 | free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); |
3661 | } |
3662 | } |
3663 | irq_idx = priv->plat->rx_queues_to_use; |
3664 | fallthrough; |
3665 | case REQ_IRQ_ERR_RX: |
3666 | for (j = irq_idx - 1; j >= 0; j--) { |
3667 | if (priv->rx_irq[j] > 0) { |
3668 | irq_set_affinity_hint(irq: priv->rx_irq[j], NULL); |
3669 | free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); |
3670 | } |
3671 | } |
3672 | |
3673 | if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) |
3674 | free_irq(priv->sfty_ue_irq, dev); |
3675 | fallthrough; |
3676 | case REQ_IRQ_ERR_SFTY_UE: |
3677 | if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) |
3678 | free_irq(priv->sfty_ce_irq, dev); |
3679 | fallthrough; |
3680 | case REQ_IRQ_ERR_SFTY_CE: |
3681 | if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) |
3682 | free_irq(priv->lpi_irq, dev); |
3683 | fallthrough; |
3684 | case REQ_IRQ_ERR_LPI: |
3685 | if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) |
3686 | free_irq(priv->wol_irq, dev); |
3687 | fallthrough; |
3688 | case REQ_IRQ_ERR_SFTY: |
3689 | if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) |
3690 | free_irq(priv->sfty_irq, dev); |
3691 | fallthrough; |
3692 | case REQ_IRQ_ERR_WOL: |
3693 | free_irq(dev->irq, dev); |
3694 | fallthrough; |
3695 | case REQ_IRQ_ERR_MAC: |
3696 | case REQ_IRQ_ERR_NO: |
3697 | /* If MAC IRQ request error, no more IRQ to free */ |
3698 | break; |
3699 | } |
3700 | } |
3701 | |
3702 | static int stmmac_request_irq_multi_msi(struct net_device *dev) |
3703 | { |
3704 | struct stmmac_priv *priv = netdev_priv(dev); |
3705 | enum request_irq_err irq_err; |
3706 | int irq_idx = 0; |
3707 | char *int_name; |
3708 | int ret; |
3709 | int i; |
3710 | |
3711 | /* For common interrupt */ |
3712 | int_name = priv->int_name_mac; |
3713 | sprintf(buf: int_name, fmt: "%s:%s", dev->name, "mac"); |
3714 | ret = request_irq(irq: dev->irq, handler: stmmac_mac_interrupt, |
3715 | flags: 0, name: int_name, dev); |
3716 | if (unlikely(ret < 0)) { |
3717 | netdev_err(dev: priv->dev, |
3718 | format: "%s: alloc mac MSI %d (error: %d)\n", |
3719 | __func__, dev->irq, ret); |
3720 | irq_err = REQ_IRQ_ERR_MAC; |
3721 | goto irq_error; |
3722 | } |
3723 | |
3724 | /* Request the Wake IRQ in case of another line |
3725 | * is used for WoL |
3726 | */ |
3727 | priv->wol_irq_disabled = true; |
3728 | if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { |
3729 | int_name = priv->int_name_wol; |
3730 | sprintf(buf: int_name, fmt: "%s:%s", dev->name, "wol"); |
3731 | ret = request_irq(irq: priv->wol_irq, |
3732 | handler: stmmac_mac_interrupt, |
3733 | flags: 0, name: int_name, dev); |
3734 | if (unlikely(ret < 0)) { |
3735 | netdev_err(dev: priv->dev, |
3736 | format: "%s: alloc wol MSI %d (error: %d)\n", |
3737 | __func__, priv->wol_irq, ret); |
3738 | irq_err = REQ_IRQ_ERR_WOL; |
3739 | goto irq_error; |
3740 | } |
3741 | } |
3742 | |
3743 | /* Request the LPI IRQ in case of another line |
3744 | * is used for LPI |
3745 | */ |
3746 | if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { |
3747 | int_name = priv->int_name_lpi; |
3748 | sprintf(buf: int_name, fmt: "%s:%s", dev->name, "lpi"); |
3749 | ret = request_irq(irq: priv->lpi_irq, |
3750 | handler: stmmac_mac_interrupt, |
3751 | flags: 0, name: int_name, dev); |
3752 | if (unlikely(ret < 0)) { |
3753 | netdev_err(dev: priv->dev, |
3754 | format: "%s: alloc lpi MSI %d (error: %d)\n", |
3755 | __func__, priv->lpi_irq, ret); |
3756 | irq_err = REQ_IRQ_ERR_LPI; |
3757 | goto irq_error; |
3758 | } |
3759 | } |
3760 | |
3761 | /* Request the common Safety Feature Correctible/Uncorrectible |
3762 | * Error line in case of another line is used |
3763 | */ |
3764 | if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { |
3765 | int_name = priv->int_name_sfty; |
3766 | sprintf(buf: int_name, fmt: "%s:%s", dev->name, "safety"); |
3767 | ret = request_irq(irq: priv->sfty_irq, handler: stmmac_safety_interrupt, |
3768 | flags: 0, name: int_name, dev); |
3769 | if (unlikely(ret < 0)) { |
3770 | netdev_err(dev: priv->dev, |
3771 | format: "%s: alloc sfty MSI %d (error: %d)\n", |
3772 | __func__, priv->sfty_irq, ret); |
3773 | irq_err = REQ_IRQ_ERR_SFTY; |
3774 | goto irq_error; |
3775 | } |
3776 | } |
3777 | |
3778 | /* Request the Safety Feature Correctible Error line in |
3779 | * case of another line is used |
3780 | */ |
3781 | if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { |
3782 | int_name = priv->int_name_sfty_ce; |
3783 | sprintf(buf: int_name, fmt: "%s:%s", dev->name, "safety-ce"); |
3784 | ret = request_irq(irq: priv->sfty_ce_irq, |
3785 | handler: stmmac_safety_interrupt, |
3786 | flags: 0, name: int_name, dev); |
3787 | if (unlikely(ret < 0)) { |
3788 | netdev_err(dev: priv->dev, |
3789 | format: "%s: alloc sfty ce MSI %d (error: %d)\n", |
3790 | __func__, priv->sfty_ce_irq, ret); |
3791 | irq_err = REQ_IRQ_ERR_SFTY_CE; |
3792 | goto irq_error; |
3793 | } |
3794 | } |
3795 | |
3796 | /* Request the Safety Feature Uncorrectible Error line in |
3797 | * case of another line is used |
3798 | */ |
3799 | if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { |
3800 | int_name = priv->int_name_sfty_ue; |
3801 | sprintf(buf: int_name, fmt: "%s:%s", dev->name, "safety-ue"); |
3802 | ret = request_irq(irq: priv->sfty_ue_irq, |
3803 | handler: stmmac_safety_interrupt, |
3804 | flags: 0, name: int_name, dev); |
3805 | if (unlikely(ret < 0)) { |
3806 | netdev_err(dev: priv->dev, |
3807 | format: "%s: alloc sfty ue MSI %d (error: %d)\n", |
3808 | __func__, priv->sfty_ue_irq, ret); |
3809 | irq_err = REQ_IRQ_ERR_SFTY_UE; |
3810 | goto irq_error; |
3811 | } |
3812 | } |
3813 | |
3814 | /* Request Rx MSI irq */ |
3815 | for (i = 0; i < priv->plat->rx_queues_to_use; i++) { |
3816 | if (i >= MTL_MAX_RX_QUEUES) |
3817 | break; |
3818 | if (priv->rx_irq[i] == 0) |
3819 | continue; |
3820 | |
3821 | int_name = priv->int_name_rx_irq[i]; |
3822 | sprintf(buf: int_name, fmt: "%s:%s-%d", dev->name, "rx", i); |
3823 | ret = request_irq(irq: priv->rx_irq[i], |
3824 | handler: stmmac_msi_intr_rx, |
3825 | flags: 0, name: int_name, dev: &priv->dma_conf.rx_queue[i]); |
3826 | if (unlikely(ret < 0)) { |
3827 | netdev_err(dev: priv->dev, |
3828 | format: "%s: alloc rx-%d MSI %d (error: %d)\n", |
3829 | __func__, i, priv->rx_irq[i], ret); |
3830 | irq_err = REQ_IRQ_ERR_RX; |
3831 | irq_idx = i; |
3832 | goto irq_error; |
3833 | } |
3834 | irq_set_affinity_hint(irq: priv->rx_irq[i], |
3835 | cpumask_of(i % num_online_cpus())); |
3836 | } |
3837 | |
3838 | /* Request Tx MSI irq */ |
3839 | for (i = 0; i < priv->plat->tx_queues_to_use; i++) { |
3840 | if (i >= MTL_MAX_TX_QUEUES) |
3841 | break; |
3842 | if (priv->tx_irq[i] == 0) |
3843 | continue; |
3844 | |
3845 | int_name = priv->int_name_tx_irq[i]; |
3846 | sprintf(buf: int_name, fmt: "%s:%s-%d", dev->name, "tx", i); |
3847 | ret = request_irq(irq: priv->tx_irq[i], |
3848 | handler: stmmac_msi_intr_tx, |
3849 | flags: 0, name: int_name, dev: &priv->dma_conf.tx_queue[i]); |
3850 | if (unlikely(ret < 0)) { |
3851 | netdev_err(dev: priv->dev, |
3852 | format: "%s: alloc tx-%d MSI %d (error: %d)\n", |
3853 | __func__, i, priv->tx_irq[i], ret); |
3854 | irq_err = REQ_IRQ_ERR_TX; |
3855 | irq_idx = i; |
3856 | goto irq_error; |
3857 | } |
3858 | irq_set_affinity_hint(irq: priv->tx_irq[i], |
3859 | cpumask_of(i % num_online_cpus())); |
3860 | } |
3861 | |
3862 | return 0; |
3863 | |
3864 | irq_error: |
3865 | stmmac_free_irq(dev, irq_err, irq_idx); |
3866 | return ret; |
3867 | } |
3868 | |
3869 | static int stmmac_request_irq_single(struct net_device *dev) |
3870 | { |
3871 | struct stmmac_priv *priv = netdev_priv(dev); |
3872 | enum request_irq_err irq_err; |
3873 | int ret; |
3874 | |
3875 | ret = request_irq(irq: dev->irq, handler: stmmac_interrupt, |
3876 | IRQF_SHARED, name: dev->name, dev); |
3877 | if (unlikely(ret < 0)) { |
3878 | netdev_err(dev: priv->dev, |
3879 | format: "%s: ERROR: allocating the IRQ %d (error: %d)\n", |
3880 | __func__, dev->irq, ret); |
3881 | irq_err = REQ_IRQ_ERR_MAC; |
3882 | goto irq_error; |
3883 | } |
3884 | |
3885 | /* Request the Wake IRQ in case of another line |
3886 | * is used for WoL |
3887 | */ |
3888 | priv->wol_irq_disabled = true; |
3889 | if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { |
3890 | ret = request_irq(irq: priv->wol_irq, handler: stmmac_interrupt, |
3891 | IRQF_SHARED, name: dev->name, dev); |
3892 | if (unlikely(ret < 0)) { |
3893 | netdev_err(dev: priv->dev, |
3894 | format: "%s: ERROR: allocating the WoL IRQ %d (%d)\n", |
3895 | __func__, priv->wol_irq, ret); |
3896 | irq_err = REQ_IRQ_ERR_WOL; |
3897 | goto irq_error; |
3898 | } |
3899 | } |
3900 | |
3901 | /* Request the IRQ lines */ |
3902 | if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) { |
3903 | ret = request_irq(irq: priv->lpi_irq, handler: stmmac_interrupt, |
3904 | IRQF_SHARED, name: dev->name, dev); |
3905 | if (unlikely(ret < 0)) { |
3906 | netdev_err(dev: priv->dev, |
3907 | format: "%s: ERROR: allocating the LPI IRQ %d (%d)\n", |
3908 | __func__, priv->lpi_irq, ret); |
3909 | irq_err = REQ_IRQ_ERR_LPI; |
3910 | goto irq_error; |
3911 | } |
3912 | } |
3913 | |
3914 | /* Request the common Safety Feature Correctible/Uncorrectible |
3915 | * Error line in case of another line is used |
3916 | */ |
3917 | if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { |
3918 | ret = request_irq(irq: priv->sfty_irq, handler: stmmac_safety_interrupt, |
3919 | IRQF_SHARED, name: dev->name, dev); |
3920 | if (unlikely(ret < 0)) { |
3921 | netdev_err(dev: priv->dev, |
3922 | format: "%s: ERROR: allocating the sfty IRQ %d (%d)\n", |
3923 | __func__, priv->sfty_irq, ret); |
3924 | irq_err = REQ_IRQ_ERR_SFTY; |
3925 | goto irq_error; |
3926 | } |
3927 | } |
3928 | |
3929 | return 0; |
3930 | |
3931 | irq_error: |
3932 | stmmac_free_irq(dev, irq_err, irq_idx: 0); |
3933 | return ret; |
3934 | } |
3935 | |
3936 | static int stmmac_request_irq(struct net_device *dev) |
3937 | { |
3938 | struct stmmac_priv *priv = netdev_priv(dev); |
3939 | int ret; |
3940 | |
3941 | /* Request the IRQ lines */ |
3942 | if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) |
3943 | ret = stmmac_request_irq_multi_msi(dev); |
3944 | else |
3945 | ret = stmmac_request_irq_single(dev); |
3946 | |
3947 | return ret; |
3948 | } |
3949 | |
3950 | /** |
3951 | * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue |
3952 | * @priv: driver private structure |
3953 | * @mtu: MTU to setup the dma queue and buf with |
3954 | * Description: Allocate and generate a dma_conf based on the provided MTU. |
3955 | * Allocate the Tx/Rx DMA queue and init them. |
3956 | * Return value: |
3957 | * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. |
3958 | */ |
3959 | static struct stmmac_dma_conf * |
3960 | stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) |
3961 | { |
3962 | struct stmmac_dma_conf *dma_conf; |
3963 | int chan, bfsize, ret; |
3964 | |
3965 | dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL); |
3966 | if (!dma_conf) { |
3967 | netdev_err(dev: priv->dev, format: "%s: DMA conf allocation failed\n", |
3968 | __func__); |
3969 | return ERR_PTR(error: -ENOMEM); |
3970 | } |
3971 | |
3972 | bfsize = stmmac_set_16kib_bfsize(priv, mtu); |
3973 | if (bfsize < 0) |
3974 | bfsize = 0; |
3975 | |
3976 | if (bfsize < BUF_SIZE_16KiB) |
3977 | bfsize = stmmac_set_bfsize(mtu, bufsize: 0); |
3978 | |
3979 | dma_conf->dma_buf_sz = bfsize; |
3980 | /* Chose the tx/rx size from the already defined one in the |
3981 | * priv struct. (if defined) |
3982 | */ |
3983 | dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; |
3984 | dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; |
3985 | |
3986 | if (!dma_conf->dma_tx_size) |
3987 | dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; |
3988 | if (!dma_conf->dma_rx_size) |
3989 | dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; |
3990 | |
3991 | /* Earlier check for TBS */ |
3992 | for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { |
3993 | struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; |
3994 | int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; |
3995 | |
3996 | /* Setup per-TXQ tbs flag before TX descriptor alloc */ |
3997 | tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; |
3998 | } |
3999 | |
4000 | ret = alloc_dma_desc_resources(priv, dma_conf); |
4001 | if (ret < 0) { |
4002 | netdev_err(dev: priv->dev, format: "%s: DMA descriptors allocation failed\n", |
4003 | __func__); |
4004 | goto alloc_error; |
4005 | } |
4006 | |
4007 | ret = init_dma_desc_rings(dev: priv->dev, dma_conf, GFP_KERNEL); |
4008 | if (ret < 0) { |
4009 | netdev_err(dev: priv->dev, format: "%s: DMA descriptors initialization failed\n", |
4010 | __func__); |
4011 | goto init_error; |
4012 | } |
4013 | |
4014 | return dma_conf; |
4015 | |
4016 | init_error: |
4017 | free_dma_desc_resources(priv, dma_conf); |
4018 | alloc_error: |
4019 | kfree(objp: dma_conf); |
4020 | return ERR_PTR(error: ret); |
4021 | } |
4022 | |
4023 | /** |
4024 | * __stmmac_open - open entry point of the driver |
4025 | * @dev : pointer to the device structure. |
4026 | * @dma_conf : structure to take the dma data |
4027 | * Description: |
4028 | * This function is the open entry point of the driver. |
4029 | * Return value: |
4030 | * 0 on success and an appropriate (-)ve integer as defined in errno.h |
4031 | * file on failure. |
4032 | */ |
4033 | static int __stmmac_open(struct net_device *dev, |
4034 | struct stmmac_dma_conf *dma_conf) |
4035 | { |
4036 | struct stmmac_priv *priv = netdev_priv(dev); |
4037 | int mode = priv->plat->phy_interface; |
4038 | u32 chan; |
4039 | int ret; |
4040 | |
4041 | /* Initialise the tx lpi timer, converting from msec to usec */ |
4042 | if (!priv->tx_lpi_timer) |
4043 | priv->tx_lpi_timer = eee_timer * 1000; |
4044 | |
4045 | ret = pm_runtime_resume_and_get(dev: priv->device); |
4046 | if (ret < 0) |
4047 | return ret; |
4048 | |
4049 | if ((!priv->hw->xpcs || |
4050 | xpcs_get_an_mode(xpcs: priv->hw->xpcs, interface: mode) != DW_AN_C73)) { |
4051 | ret = stmmac_init_phy(dev); |
4052 | if (ret) { |
4053 | netdev_err(dev: priv->dev, |
4054 | format: "%s: Cannot attach to PHY (error: %d)\n", |
4055 | __func__, ret); |
4056 | goto init_phy_error; |
4057 | } |
4058 | } |
4059 | |
4060 | for (int i = 0; i < MTL_MAX_TX_QUEUES; i++) |
4061 | if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) |
4062 | dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; |
4063 | memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); |
4064 | |
4065 | stmmac_reset_queues_param(priv); |
4066 | |
4067 | if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && |
4068 | priv->plat->serdes_powerup) { |
4069 | ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); |
4070 | if (ret < 0) { |
4071 | netdev_err(dev: priv->dev, format: "%s: Serdes powerup failed\n", |
4072 | __func__); |
4073 | goto init_error; |
4074 | } |
4075 | } |
4076 | |
4077 | ret = stmmac_hw_setup(dev, ptp_register: true); |
4078 | if (ret < 0) { |
4079 | netdev_err(dev: priv->dev, format: "%s: Hw setup failed\n", __func__); |
4080 | goto init_error; |
4081 | } |
4082 | |
4083 | stmmac_init_coalesce(priv); |
4084 | |
4085 | phylink_start(priv->phylink); |
4086 | /* We may have called phylink_speed_down before */ |
4087 | phylink_speed_up(pl: priv->phylink); |
4088 | |
4089 | ret = stmmac_request_irq(dev); |
4090 | if (ret) |
4091 | goto irq_error; |
4092 | |
4093 | stmmac_enable_all_queues(priv); |
4094 | netif_tx_start_all_queues(dev: priv->dev); |
4095 | stmmac_enable_all_dma_irq(priv); |
4096 | |
4097 | return 0; |
4098 | |
4099 | irq_error: |
4100 | phylink_stop(priv->phylink); |
4101 | |
4102 | for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) |
4103 | hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer); |
4104 | |
4105 | stmmac_hw_teardown(dev); |
4106 | init_error: |
4107 | phylink_disconnect_phy(priv->phylink); |
4108 | init_phy_error: |
4109 | pm_runtime_put(dev: priv->device); |
4110 | return ret; |
4111 | } |
4112 | |
4113 | static int stmmac_open(struct net_device *dev) |
4114 | { |
4115 | struct stmmac_priv *priv = netdev_priv(dev); |
4116 | struct stmmac_dma_conf *dma_conf; |
4117 | int ret; |
4118 | |
4119 | dma_conf = stmmac_setup_dma_desc(priv, mtu: dev->mtu); |
4120 | if (IS_ERR(ptr: dma_conf)) |
4121 | return PTR_ERR(ptr: dma_conf); |
4122 | |
4123 | ret = __stmmac_open(dev, dma_conf); |
4124 | if (ret) |
4125 | free_dma_desc_resources(priv, dma_conf); |
4126 | |
4127 | kfree(objp: dma_conf); |
4128 | return ret; |
4129 | } |
4130 | |
4131 | /** |
4132 | * stmmac_release - close entry point of the driver |
4133 | * @dev : device pointer. |
4134 | * Description: |
4135 | * This is the stop entry point of the driver. |
4136 | */ |
4137 | static int stmmac_release(struct net_device *dev) |
4138 | { |
4139 | struct stmmac_priv *priv = netdev_priv(dev); |
4140 | u32 chan; |
4141 | |
4142 | if (device_may_wakeup(dev: priv->device)) |
4143 | phylink_speed_down(pl: priv->phylink, sync: false); |
4144 | /* Stop and disconnect the PHY */ |
4145 | phylink_stop(priv->phylink); |
4146 | phylink_disconnect_phy(priv->phylink); |
4147 | |
4148 | stmmac_disable_all_queues(priv); |
4149 | |
4150 | for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) |
4151 | hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer); |
4152 | |
4153 | netif_tx_disable(dev); |
4154 | |
4155 | /* Free the IRQ lines */ |
4156 | stmmac_free_irq(dev, irq_err: REQ_IRQ_ERR_ALL, irq_idx: 0); |
4157 | |
4158 | /* Stop TX/RX DMA and clear the descriptors */ |
4159 | stmmac_stop_all_dma(priv); |
4160 | |
4161 | /* Release and free the Rx/Tx resources */ |
4162 | free_dma_desc_resources(priv, dma_conf: &priv->dma_conf); |
4163 | |
4164 | /* Powerdown Serdes if there is */ |
4165 | if (priv->plat->serdes_powerdown) |
4166 | priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); |
4167 | |
4168 | stmmac_release_ptp(priv); |
4169 | |
4170 | if (stmmac_fpe_supported(priv)) |
4171 | ethtool_mmsv_stop(mmsv: &priv->fpe_cfg.mmsv); |
4172 | |
4173 | pm_runtime_put(dev: priv->device); |
4174 | |
4175 | return 0; |
4176 | } |
4177 | |
4178 | static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, |
4179 | struct stmmac_tx_queue *tx_q) |
4180 | { |
4181 | u16 tag = 0x0, inner_tag = 0x0; |
4182 | u32 inner_type = 0x0; |
4183 | struct dma_desc *p; |
4184 | |
4185 | if (!priv->dma_cap.vlins) |
4186 | return false; |
4187 | if (!skb_vlan_tag_present(skb)) |
4188 | return false; |
4189 | if (skb->vlan_proto == htons(ETH_P_8021AD)) { |
4190 | inner_tag = skb_vlan_tag_get(skb); |
4191 | inner_type = STMMAC_VLAN_INSERT; |
4192 | } |
4193 | |
4194 | tag = skb_vlan_tag_get(skb); |
4195 | |
4196 | if (tx_q->tbs & STMMAC_TBS_AVAIL) |
4197 | p = &tx_q->dma_entx[tx_q->cur_tx].basic; |
4198 | else |
4199 | p = &tx_q->dma_tx[tx_q->cur_tx]; |
4200 | |
4201 | if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type)) |
4202 | return false; |
4203 | |
4204 | stmmac_set_tx_owner(priv, p); |
4205 | tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); |
4206 | return true; |
4207 | } |
4208 | |
4209 | /** |
4210 | * stmmac_tso_allocator - close entry point of the driver |
4211 | * @priv: driver private structure |
4212 | * @des: buffer start address |
4213 | * @total_len: total length to fill in descriptors |
4214 | * @last_segment: condition for the last descriptor |
4215 | * @queue: TX queue index |
4216 | * Description: |
4217 | * This function fills descriptor and request new descriptors according to |
4218 | * buffer length to fill |
4219 | */ |
4220 | static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, |
4221 | int total_len, bool last_segment, u32 queue) |
4222 | { |
4223 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
4224 | struct dma_desc *desc; |
4225 | u32 buff_size; |
4226 | int tmp_len; |
4227 | |
4228 | tmp_len = total_len; |
4229 | |
4230 | while (tmp_len > 0) { |
4231 | dma_addr_t curr_addr; |
4232 | |
4233 | tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, |
4234 | priv->dma_conf.dma_tx_size); |
4235 | WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); |
4236 | |
4237 | if (tx_q->tbs & STMMAC_TBS_AVAIL) |
4238 | desc = &tx_q->dma_entx[tx_q->cur_tx].basic; |
4239 | else |
4240 | desc = &tx_q->dma_tx[tx_q->cur_tx]; |
4241 | |
4242 | curr_addr = des + (total_len - tmp_len); |
4243 | stmmac_set_desc_addr(priv, desc, curr_addr); |
4244 | buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? |
4245 | TSO_MAX_BUFF_SIZE : tmp_len; |
4246 | |
4247 | stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, |
4248 | 0, 1, |
4249 | (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), |
4250 | 0, 0); |
4251 | |
4252 | tmp_len -= TSO_MAX_BUFF_SIZE; |
4253 | } |
4254 | } |
4255 | |
4256 | static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) |
4257 | { |
4258 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
4259 | int desc_size; |
4260 | |
4261 | if (likely(priv->extend_desc)) |
4262 | desc_size = sizeof(struct dma_extended_desc); |
4263 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
4264 | desc_size = sizeof(struct dma_edesc); |
4265 | else |
4266 | desc_size = sizeof(struct dma_desc); |
4267 | |
4268 | /* The own bit must be the latest setting done when prepare the |
4269 | * descriptor and then barrier is needed to make sure that |
4270 | * all is coherent before granting the DMA engine. |
4271 | */ |
4272 | wmb(); |
4273 | |
4274 | tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); |
4275 | stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); |
4276 | } |
4277 | |
4278 | /** |
4279 | * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) |
4280 | * @skb : the socket buffer |
4281 | * @dev : device pointer |
4282 | * Description: this is the transmit function that is called on TSO frames |
4283 | * (support available on GMAC4 and newer chips). |
4284 | * Diagram below show the ring programming in case of TSO frames: |
4285 | * |
4286 | * First Descriptor |
4287 | * -------- |
4288 | * | DES0 |---> buffer1 = L2/L3/L4 header |
4289 | * | DES1 |---> can be used as buffer2 for TCP Payload if the DMA AXI address |
4290 | * | | width is 32-bit, but we never use it. |
4291 | * | | Also can be used as the most-significant 8-bits or 16-bits of |
4292 | * | | buffer1 address pointer if the DMA AXI address width is 40-bit |
4293 | * | | or 48-bit, and we always use it. |
4294 | * | DES2 |---> buffer1 len |
4295 | * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] |
4296 | * -------- |
4297 | * -------- |
4298 | * | DES0 |---> buffer1 = TCP Payload (can continue on next descr...) |
4299 | * | DES1 |---> same as the First Descriptor |
4300 | * | DES2 |---> buffer1 len |
4301 | * | DES3 | |
4302 | * -------- |
4303 | * | |
4304 | * ... |
4305 | * | |
4306 | * -------- |
4307 | * | DES0 |---> buffer1 = Split TCP Payload |
4308 | * | DES1 |---> same as the First Descriptor |
4309 | * | DES2 |---> buffer1 len |
4310 | * | DES3 | |
4311 | * -------- |
4312 | * |
4313 | * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. |
4314 | */ |
4315 | static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) |
4316 | { |
4317 | struct dma_desc *desc, *first, *mss_desc = NULL; |
4318 | struct stmmac_priv *priv = netdev_priv(dev); |
4319 | unsigned int first_entry, tx_packets; |
4320 | struct stmmac_txq_stats *txq_stats; |
4321 | struct stmmac_tx_queue *tx_q; |
4322 | u32 pay_len, mss, queue; |
4323 | int i, first_tx, nfrags; |
4324 | u8 proto_hdr_len, hdr; |
4325 | dma_addr_t des; |
4326 | bool set_ic; |
4327 | |
4328 | /* Always insert VLAN tag to SKB payload for TSO frames. |
4329 | * |
4330 | * Never insert VLAN tag by HW, since segments splited by |
4331 | * TSO engine will be un-tagged by mistake. |
4332 | */ |
4333 | if (skb_vlan_tag_present(skb)) { |
4334 | skb = __vlan_hwaccel_push_inside(skb); |
4335 | if (unlikely(!skb)) { |
4336 | priv->xstats.tx_dropped++; |
4337 | return NETDEV_TX_OK; |
4338 | } |
4339 | } |
4340 | |
4341 | nfrags = skb_shinfo(skb)->nr_frags; |
4342 | queue = skb_get_queue_mapping(skb); |
4343 | |
4344 | tx_q = &priv->dma_conf.tx_queue[queue]; |
4345 | txq_stats = &priv->xstats.txq_stats[queue]; |
4346 | first_tx = tx_q->cur_tx; |
4347 | |
4348 | /* Compute header lengths */ |
4349 | if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { |
4350 | proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); |
4351 | hdr = sizeof(struct udphdr); |
4352 | } else { |
4353 | proto_hdr_len = skb_tcp_all_headers(skb); |
4354 | hdr = tcp_hdrlen(skb); |
4355 | } |
4356 | |
4357 | /* Desc availability based on threshold should be enough safe */ |
4358 | if (unlikely(stmmac_tx_avail(priv, queue) < |
4359 | (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { |
4360 | if (!netif_tx_queue_stopped(dev_queue: netdev_get_tx_queue(dev, index: queue))) { |
4361 | netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, |
4362 | index: queue)); |
4363 | /* This is a hard error, log it. */ |
4364 | netdev_err(dev: priv->dev, |
4365 | format: "%s: Tx Ring full when queue awake\n", |
4366 | __func__); |
4367 | } |
4368 | return NETDEV_TX_BUSY; |
4369 | } |
4370 | |
4371 | pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ |
4372 | |
4373 | mss = skb_shinfo(skb)->gso_size; |
4374 | |
4375 | /* set new MSS value if needed */ |
4376 | if (mss != tx_q->mss) { |
4377 | if (tx_q->tbs & STMMAC_TBS_AVAIL) |
4378 | mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; |
4379 | else |
4380 | mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; |
4381 | |
4382 | stmmac_set_mss(priv, mss_desc, mss); |
4383 | tx_q->mss = mss; |
4384 | tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, |
4385 | priv->dma_conf.dma_tx_size); |
4386 | WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); |
4387 | } |
4388 | |
4389 | if (netif_msg_tx_queued(priv)) { |
4390 | pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", |
4391 | __func__, hdr, proto_hdr_len, pay_len, mss); |
4392 | pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, |
4393 | skb->data_len); |
4394 | } |
4395 | |
4396 | first_entry = tx_q->cur_tx; |
4397 | WARN_ON(tx_q->tx_skbuff[first_entry]); |
4398 | |
4399 | if (tx_q->tbs & STMMAC_TBS_AVAIL) |
4400 | desc = &tx_q->dma_entx[first_entry].basic; |
4401 | else |
4402 | desc = &tx_q->dma_tx[first_entry]; |
4403 | first = desc; |
4404 | |
4405 | /* first descriptor: fill Headers on Buf1 */ |
4406 | des = dma_map_single(priv->device, skb->data, skb_headlen(skb), |
4407 | DMA_TO_DEVICE); |
4408 | if (dma_mapping_error(dev: priv->device, dma_addr: des)) |
4409 | goto dma_map_err; |
4410 | |
4411 | stmmac_set_desc_addr(priv, first, des); |
4412 | stmmac_tso_allocator(priv, des: des + proto_hdr_len, total_len: pay_len, |
4413 | last_segment: (nfrags == 0), queue); |
4414 | |
4415 | /* In case two or more DMA transmit descriptors are allocated for this |
4416 | * non-paged SKB data, the DMA buffer address should be saved to |
4417 | * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, |
4418 | * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee |
4419 | * that stmmac_tx_clean() does not unmap the entire DMA buffer too early |
4420 | * since the tail areas of the DMA buffer can be accessed by DMA engine |
4421 | * sooner or later. |
4422 | * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf |
4423 | * corresponding to the last descriptor, stmmac_tx_clean() will unmap |
4424 | * this DMA buffer right after the DMA engine completely finishes the |
4425 | * full buffer transmission. |
4426 | */ |
4427 | tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; |
4428 | tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb); |
4429 | tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false; |
4430 | tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; |
4431 | |
4432 | /* Prepare fragments */ |
4433 | for (i = 0; i < nfrags; i++) { |
4434 | const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
4435 | |
4436 | des = skb_frag_dma_map(priv->device, frag, 0, |
4437 | skb_frag_size(frag), |
4438 | DMA_TO_DEVICE); |
4439 | if (dma_mapping_error(dev: priv->device, dma_addr: des)) |
4440 | goto dma_map_err; |
4441 | |
4442 | stmmac_tso_allocator(priv, des, total_len: skb_frag_size(frag), |
4443 | last_segment: (i == nfrags - 1), queue); |
4444 | |
4445 | tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; |
4446 | tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); |
4447 | tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; |
4448 | tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; |
4449 | } |
4450 | |
4451 | tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; |
4452 | |
4453 | /* Only the last descriptor gets to point to the skb. */ |
4454 | tx_q->tx_skbuff[tx_q->cur_tx] = skb; |
4455 | tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; |
4456 | |
4457 | /* Manage tx mitigation */ |
4458 | tx_packets = (tx_q->cur_tx + 1) - first_tx; |
4459 | tx_q->tx_count_frames += tx_packets; |
4460 | |
4461 | if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) |
4462 | set_ic = true; |
4463 | else if (!priv->tx_coal_frames[queue]) |
4464 | set_ic = false; |
4465 | else if (tx_packets > priv->tx_coal_frames[queue]) |
4466 | set_ic = true; |
4467 | else if ((tx_q->tx_count_frames % |
4468 | priv->tx_coal_frames[queue]) < tx_packets) |
4469 | set_ic = true; |
4470 | else |
4471 | set_ic = false; |
4472 | |
4473 | if (set_ic) { |
4474 | if (tx_q->tbs & STMMAC_TBS_AVAIL) |
4475 | desc = &tx_q->dma_entx[tx_q->cur_tx].basic; |
4476 | else |
4477 | desc = &tx_q->dma_tx[tx_q->cur_tx]; |
4478 | |
4479 | tx_q->tx_count_frames = 0; |
4480 | stmmac_set_tx_ic(priv, desc); |
4481 | } |
4482 | |
4483 | /* We've used all descriptors we need for this skb, however, |
4484 | * advance cur_tx so that it references a fresh descriptor. |
4485 | * ndo_start_xmit will fill this descriptor the next time it's |
4486 | * called and stmmac_tx_clean may clean up to this descriptor. |
4487 | */ |
4488 | tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); |
4489 | |
4490 | if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { |
4491 | netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", |
4492 | __func__); |
4493 | netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: queue)); |
4494 | } |
4495 | |
4496 | u64_stats_update_begin(syncp: &txq_stats->q_syncp); |
4497 | u64_stats_add(p: &txq_stats->q.tx_bytes, val: skb->len); |
4498 | u64_stats_inc(p: &txq_stats->q.tx_tso_frames); |
4499 | u64_stats_add(p: &txq_stats->q.tx_tso_nfrags, val: nfrags); |
4500 | if (set_ic) |
4501 | u64_stats_inc(p: &txq_stats->q.tx_set_ic_bit); |
4502 | u64_stats_update_end(syncp: &txq_stats->q_syncp); |
4503 | |
4504 | if (priv->sarc_type) |
4505 | stmmac_set_desc_sarc(priv, first, priv->sarc_type); |
4506 | |
4507 | if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && |
4508 | priv->hwts_tx_en)) { |
4509 | /* declare that device is doing timestamping */ |
4510 | skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; |
4511 | stmmac_enable_tx_timestamp(priv, first); |
4512 | } |
4513 | |
4514 | /* Complete the first descriptor before granting the DMA */ |
4515 | stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1, |
4516 | tx_q->tx_skbuff_dma[first_entry].last_segment, |
4517 | hdr / 4, (skb->len - proto_hdr_len)); |
4518 | |
4519 | /* If context desc is used to change MSS */ |
4520 | if (mss_desc) { |
4521 | /* Make sure that first descriptor has been completely |
4522 | * written, including its own bit. This is because MSS is |
4523 | * actually before first descriptor, so we need to make |
4524 | * sure that MSS's own bit is the last thing written. |
4525 | */ |
4526 | dma_wmb(); |
4527 | stmmac_set_tx_owner(priv, mss_desc); |
4528 | } |
4529 | |
4530 | if (netif_msg_pktdata(priv)) { |
4531 | pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", |
4532 | __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, |
4533 | tx_q->cur_tx, first, nfrags); |
4534 | pr_info(">>> frame to be transmitted: "); |
4535 | print_pkt(buf: skb->data, len: skb_headlen(skb)); |
4536 | } |
4537 | |
4538 | netdev_tx_sent_queue(dev_queue: netdev_get_tx_queue(dev, index: queue), bytes: skb->len); |
4539 | skb_tx_timestamp(skb); |
4540 | |
4541 | stmmac_flush_tx_descriptors(priv, queue); |
4542 | stmmac_tx_timer_arm(priv, queue); |
4543 | |
4544 | return NETDEV_TX_OK; |
4545 | |
4546 | dma_map_err: |
4547 | dev_err(priv->device, "Tx dma map failed\n"); |
4548 | dev_kfree_skb(skb); |
4549 | priv->xstats.tx_dropped++; |
4550 | return NETDEV_TX_OK; |
4551 | } |
4552 | |
4553 | /** |
4554 | * stmmac_has_ip_ethertype() - Check if packet has IP ethertype |
4555 | * @skb: socket buffer to check |
4556 | * |
4557 | * Check if a packet has an ethertype that will trigger the IP header checks |
4558 | * and IP/TCP checksum engine of the stmmac core. |
4559 | * |
4560 | * Return: true if the ethertype can trigger the checksum engine, false |
4561 | * otherwise |
4562 | */ |
4563 | static bool stmmac_has_ip_ethertype(struct sk_buff *skb) |
4564 | { |
4565 | int depth = 0; |
4566 | __be16 proto; |
4567 | |
4568 | proto = __vlan_get_protocol(skb, type: eth_header_parse_protocol(skb), |
4569 | depth: &depth); |
4570 | |
4571 | return (depth <= ETH_HLEN) && |
4572 | (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); |
4573 | } |
4574 | |
4575 | /** |
4576 | * stmmac_xmit - Tx entry point of the driver |
4577 | * @skb : the socket buffer |
4578 | * @dev : device pointer |
4579 | * Description : this is the tx entry point of the driver. |
4580 | * It programs the chain or the ring and supports oversized frames |
4581 | * and SG feature. |
4582 | */ |
4583 | static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) |
4584 | { |
4585 | unsigned int first_entry, tx_packets, enh_desc; |
4586 | struct stmmac_priv *priv = netdev_priv(dev); |
4587 | unsigned int nopaged_len = skb_headlen(skb); |
4588 | int i, csum_insertion = 0, is_jumbo = 0; |
4589 | u32 queue = skb_get_queue_mapping(skb); |
4590 | int nfrags = skb_shinfo(skb)->nr_frags; |
4591 | int gso = skb_shinfo(skb)->gso_type; |
4592 | struct stmmac_txq_stats *txq_stats; |
4593 | struct dma_edesc *tbs_desc = NULL; |
4594 | struct dma_desc *desc, *first; |
4595 | struct stmmac_tx_queue *tx_q; |
4596 | bool has_vlan, set_ic; |
4597 | int entry, first_tx; |
4598 | dma_addr_t des; |
4599 | |
4600 | tx_q = &priv->dma_conf.tx_queue[queue]; |
4601 | txq_stats = &priv->xstats.txq_stats[queue]; |
4602 | first_tx = tx_q->cur_tx; |
4603 | |
4604 | if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) |
4605 | stmmac_stop_sw_lpi(priv); |
4606 | |
4607 | /* Manage oversized TCP frames for GMAC4 device */ |
4608 | if (skb_is_gso(skb) && priv->tso) { |
4609 | if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) |
4610 | return stmmac_tso_xmit(skb, dev); |
4611 | if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4)) |
4612 | return stmmac_tso_xmit(skb, dev); |
4613 | } |
4614 | |
4615 | if (priv->est && priv->est->enable && |
4616 | priv->est->max_sdu[queue] && |
4617 | skb->len > priv->est->max_sdu[queue]){ |
4618 | priv->xstats.max_sdu_txq_drop[queue]++; |
4619 | goto max_sdu_err; |
4620 | } |
4621 | |
4622 | if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { |
4623 | if (!netif_tx_queue_stopped(dev_queue: netdev_get_tx_queue(dev, index: queue))) { |
4624 | netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, |
4625 | index: queue)); |
4626 | /* This is a hard error, log it. */ |
4627 | netdev_err(dev: priv->dev, |
4628 | format: "%s: Tx Ring full when queue awake\n", |
4629 | __func__); |
4630 | } |
4631 | return NETDEV_TX_BUSY; |
4632 | } |
4633 | |
4634 | /* Check if VLAN can be inserted by HW */ |
4635 | has_vlan = stmmac_vlan_insert(priv, skb, tx_q); |
4636 | |
4637 | entry = tx_q->cur_tx; |
4638 | first_entry = entry; |
4639 | WARN_ON(tx_q->tx_skbuff[first_entry]); |
4640 | |
4641 | csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); |
4642 | /* DWMAC IPs can be synthesized to support tx coe only for a few tx |
4643 | * queues. In that case, checksum offloading for those queues that don't |
4644 | * support tx coe needs to fallback to software checksum calculation. |
4645 | * |
4646 | * Packets that won't trigger the COE e.g. most DSA-tagged packets will |
4647 | * also have to be checksummed in software. |
4648 | */ |
4649 | if (csum_insertion && |
4650 | (priv->plat->tx_queues_cfg[queue].coe_unsupported || |
4651 | !stmmac_has_ip_ethertype(skb))) { |
4652 | if (unlikely(skb_checksum_help(skb))) |
4653 | goto dma_map_err; |
4654 | csum_insertion = !csum_insertion; |
4655 | } |
4656 | |
4657 | if (likely(priv->extend_desc)) |
4658 | desc = (struct dma_desc *)(tx_q->dma_etx + entry); |
4659 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
4660 | desc = &tx_q->dma_entx[entry].basic; |
4661 | else |
4662 | desc = tx_q->dma_tx + entry; |
4663 | |
4664 | first = desc; |
4665 | |
4666 | if (has_vlan) |
4667 | stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); |
4668 | |
4669 | enh_desc = priv->plat->enh_desc; |
4670 | /* To program the descriptors according to the size of the frame */ |
4671 | if (enh_desc) |
4672 | is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); |
4673 | |
4674 | if (unlikely(is_jumbo)) { |
4675 | entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); |
4676 | if (unlikely(entry < 0) && (entry != -EINVAL)) |
4677 | goto dma_map_err; |
4678 | } |
4679 | |
4680 | for (i = 0; i < nfrags; i++) { |
4681 | const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
4682 | int len = skb_frag_size(frag); |
4683 | bool last_segment = (i == (nfrags - 1)); |
4684 | |
4685 | entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); |
4686 | WARN_ON(tx_q->tx_skbuff[entry]); |
4687 | |
4688 | if (likely(priv->extend_desc)) |
4689 | desc = (struct dma_desc *)(tx_q->dma_etx + entry); |
4690 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
4691 | desc = &tx_q->dma_entx[entry].basic; |
4692 | else |
4693 | desc = tx_q->dma_tx + entry; |
4694 | |
4695 | des = skb_frag_dma_map(priv->device, frag, 0, len, |
4696 | DMA_TO_DEVICE); |
4697 | if (dma_mapping_error(dev: priv->device, dma_addr: des)) |
4698 | goto dma_map_err; /* should reuse desc w/o issues */ |
4699 | |
4700 | tx_q->tx_skbuff_dma[entry].buf = des; |
4701 | |
4702 | stmmac_set_desc_addr(priv, desc, des); |
4703 | |
4704 | tx_q->tx_skbuff_dma[entry].map_as_page = true; |
4705 | tx_q->tx_skbuff_dma[entry].len = len; |
4706 | tx_q->tx_skbuff_dma[entry].last_segment = last_segment; |
4707 | tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; |
4708 | |
4709 | /* Prepare the descriptor and set the own bit too */ |
4710 | stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, |
4711 | priv->mode, 1, last_segment, skb->len); |
4712 | } |
4713 | |
4714 | /* Only the last descriptor gets to point to the skb. */ |
4715 | tx_q->tx_skbuff[entry] = skb; |
4716 | tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; |
4717 | |
4718 | /* According to the coalesce parameter the IC bit for the latest |
4719 | * segment is reset and the timer re-started to clean the tx status. |
4720 | * This approach takes care about the fragments: desc is the first |
4721 | * element in case of no SG. |
4722 | */ |
4723 | tx_packets = (entry + 1) - first_tx; |
4724 | tx_q->tx_count_frames += tx_packets; |
4725 | |
4726 | if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) |
4727 | set_ic = true; |
4728 | else if (!priv->tx_coal_frames[queue]) |
4729 | set_ic = false; |
4730 | else if (tx_packets > priv->tx_coal_frames[queue]) |
4731 | set_ic = true; |
4732 | else if ((tx_q->tx_count_frames % |
4733 | priv->tx_coal_frames[queue]) < tx_packets) |
4734 | set_ic = true; |
4735 | else |
4736 | set_ic = false; |
4737 | |
4738 | if (set_ic) { |
4739 | if (likely(priv->extend_desc)) |
4740 | desc = &tx_q->dma_etx[entry].basic; |
4741 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
4742 | desc = &tx_q->dma_entx[entry].basic; |
4743 | else |
4744 | desc = &tx_q->dma_tx[entry]; |
4745 | |
4746 | tx_q->tx_count_frames = 0; |
4747 | stmmac_set_tx_ic(priv, desc); |
4748 | } |
4749 | |
4750 | /* We've used all descriptors we need for this skb, however, |
4751 | * advance cur_tx so that it references a fresh descriptor. |
4752 | * ndo_start_xmit will fill this descriptor the next time it's |
4753 | * called and stmmac_tx_clean may clean up to this descriptor. |
4754 | */ |
4755 | entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); |
4756 | tx_q->cur_tx = entry; |
4757 | |
4758 | if (netif_msg_pktdata(priv)) { |
4759 | netdev_dbg(priv->dev, |
4760 | "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", |
4761 | __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, |
4762 | entry, first, nfrags); |
4763 | |
4764 | netdev_dbg(priv->dev, ">>> frame to be transmitted: "); |
4765 | print_pkt(buf: skb->data, len: skb->len); |
4766 | } |
4767 | |
4768 | if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { |
4769 | netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", |
4770 | __func__); |
4771 | netif_tx_stop_queue(dev_queue: netdev_get_tx_queue(dev: priv->dev, index: queue)); |
4772 | } |
4773 | |
4774 | u64_stats_update_begin(syncp: &txq_stats->q_syncp); |
4775 | u64_stats_add(p: &txq_stats->q.tx_bytes, val: skb->len); |
4776 | if (set_ic) |
4777 | u64_stats_inc(p: &txq_stats->q.tx_set_ic_bit); |
4778 | u64_stats_update_end(syncp: &txq_stats->q_syncp); |
4779 | |
4780 | if (priv->sarc_type) |
4781 | stmmac_set_desc_sarc(priv, first, priv->sarc_type); |
4782 | |
4783 | /* Ready to fill the first descriptor and set the OWN bit w/o any |
4784 | * problems because all the descriptors are actually ready to be |
4785 | * passed to the DMA engine. |
4786 | */ |
4787 | if (likely(!is_jumbo)) { |
4788 | bool last_segment = (nfrags == 0); |
4789 | |
4790 | des = dma_map_single(priv->device, skb->data, |
4791 | nopaged_len, DMA_TO_DEVICE); |
4792 | if (dma_mapping_error(dev: priv->device, dma_addr: des)) |
4793 | goto dma_map_err; |
4794 | |
4795 | tx_q->tx_skbuff_dma[first_entry].buf = des; |
4796 | tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; |
4797 | tx_q->tx_skbuff_dma[first_entry].map_as_page = false; |
4798 | |
4799 | stmmac_set_desc_addr(priv, first, des); |
4800 | |
4801 | tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; |
4802 | tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; |
4803 | |
4804 | if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && |
4805 | priv->hwts_tx_en)) { |
4806 | /* declare that device is doing timestamping */ |
4807 | skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; |
4808 | stmmac_enable_tx_timestamp(priv, first); |
4809 | } |
4810 | |
4811 | /* Prepare the first descriptor setting the OWN bit too */ |
4812 | stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, |
4813 | csum_insertion, priv->mode, 0, last_segment, |
4814 | skb->len); |
4815 | } |
4816 | |
4817 | if (tx_q->tbs & STMMAC_TBS_EN) { |
4818 | struct timespec64 ts = ns_to_timespec64(nsec: skb->tstamp); |
4819 | |
4820 | tbs_desc = &tx_q->dma_entx[first_entry]; |
4821 | stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); |
4822 | } |
4823 | |
4824 | stmmac_set_tx_owner(priv, first); |
4825 | |
4826 | netdev_tx_sent_queue(dev_queue: netdev_get_tx_queue(dev, index: queue), bytes: skb->len); |
4827 | |
4828 | stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); |
4829 | skb_tx_timestamp(skb); |
4830 | stmmac_flush_tx_descriptors(priv, queue); |
4831 | stmmac_tx_timer_arm(priv, queue); |
4832 | |
4833 | return NETDEV_TX_OK; |
4834 | |
4835 | dma_map_err: |
4836 | netdev_err(dev: priv->dev, format: "Tx DMA map failed\n"); |
4837 | max_sdu_err: |
4838 | dev_kfree_skb(skb); |
4839 | priv->xstats.tx_dropped++; |
4840 | return NETDEV_TX_OK; |
4841 | } |
4842 | |
4843 | static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) |
4844 | { |
4845 | struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); |
4846 | __be16 vlan_proto = veth->h_vlan_proto; |
4847 | u16 vlanid; |
4848 | |
4849 | if ((vlan_proto == htons(ETH_P_8021Q) && |
4850 | dev->features & NETIF_F_HW_VLAN_CTAG_RX) || |
4851 | (vlan_proto == htons(ETH_P_8021AD) && |
4852 | dev->features & NETIF_F_HW_VLAN_STAG_RX)) { |
4853 | /* pop the vlan tag */ |
4854 | vlanid = ntohs(veth->h_vlan_TCI); |
4855 | memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); |
4856 | skb_pull(skb, VLAN_HLEN); |
4857 | __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci: vlanid); |
4858 | } |
4859 | } |
4860 | |
4861 | /** |
4862 | * stmmac_rx_refill - refill used skb preallocated buffers |
4863 | * @priv: driver private structure |
4864 | * @queue: RX queue index |
4865 | * Description : this is to reallocate the skb for the reception process |
4866 | * that is based on zero-copy. |
4867 | */ |
4868 | static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) |
4869 | { |
4870 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; |
4871 | int dirty = stmmac_rx_dirty(priv, queue); |
4872 | unsigned int entry = rx_q->dirty_rx; |
4873 | gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); |
4874 | |
4875 | if (priv->dma_cap.host_dma_width <= 32) |
4876 | gfp |= GFP_DMA32; |
4877 | |
4878 | while (dirty-- > 0) { |
4879 | struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; |
4880 | struct dma_desc *p; |
4881 | bool use_rx_wd; |
4882 | |
4883 | if (priv->extend_desc) |
4884 | p = (struct dma_desc *)(rx_q->dma_erx + entry); |
4885 | else |
4886 | p = rx_q->dma_rx + entry; |
4887 | |
4888 | if (!buf->page) { |
4889 | buf->page = page_pool_alloc_pages(pool: rx_q->page_pool, gfp); |
4890 | if (!buf->page) |
4891 | break; |
4892 | } |
4893 | |
4894 | if (priv->sph && !buf->sec_page) { |
4895 | buf->sec_page = page_pool_alloc_pages(pool: rx_q->page_pool, gfp); |
4896 | if (!buf->sec_page) |
4897 | break; |
4898 | |
4899 | buf->sec_addr = page_pool_get_dma_addr(page: buf->sec_page); |
4900 | } |
4901 | |
4902 | buf->addr = page_pool_get_dma_addr(page: buf->page) + buf->page_offset; |
4903 | |
4904 | stmmac_set_desc_addr(priv, p, buf->addr); |
4905 | if (priv->sph) |
4906 | stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); |
4907 | else |
4908 | stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); |
4909 | stmmac_refill_desc3(priv, rx_q, p); |
4910 | |
4911 | rx_q->rx_count_frames++; |
4912 | rx_q->rx_count_frames += priv->rx_coal_frames[queue]; |
4913 | if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) |
4914 | rx_q->rx_count_frames = 0; |
4915 | |
4916 | use_rx_wd = !priv->rx_coal_frames[queue]; |
4917 | use_rx_wd |= rx_q->rx_count_frames > 0; |
4918 | if (!priv->use_riwt) |
4919 | use_rx_wd = false; |
4920 | |
4921 | dma_wmb(); |
4922 | stmmac_set_rx_owner(priv, p, use_rx_wd); |
4923 | |
4924 | entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); |
4925 | } |
4926 | rx_q->dirty_rx = entry; |
4927 | rx_q->rx_tail_addr = rx_q->dma_rx_phy + |
4928 | (rx_q->dirty_rx * sizeof(struct dma_desc)); |
4929 | stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); |
4930 | } |
4931 | |
4932 | static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, |
4933 | struct dma_desc *p, |
4934 | int status, unsigned int len) |
4935 | { |
4936 | unsigned int plen = 0, hlen = 0; |
4937 | int coe = priv->hw->rx_csum; |
4938 | |
4939 | /* Not first descriptor, buffer is always zero */ |
4940 | if (priv->sph && len) |
4941 | return 0; |
4942 | |
4943 | /* First descriptor, get split header length */ |
4944 | stmmac_get_rx_header_len(priv, p, &hlen); |
4945 | if (priv->sph && hlen) { |
4946 | priv->xstats.rx_split_hdr_pkt_n++; |
4947 | return hlen; |
4948 | } |
4949 | |
4950 | /* First descriptor, not last descriptor and not split header */ |
4951 | if (status & rx_not_ls) |
4952 | return priv->dma_conf.dma_buf_sz; |
4953 | |
4954 | plen = stmmac_get_rx_frame_len(priv, p, coe); |
4955 | |
4956 | /* First descriptor and last descriptor and not split header */ |
4957 | return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); |
4958 | } |
4959 | |
4960 | static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, |
4961 | struct dma_desc *p, |
4962 | int status, unsigned int len) |
4963 | { |
4964 | int coe = priv->hw->rx_csum; |
4965 | unsigned int plen = 0; |
4966 | |
4967 | /* Not split header, buffer is not available */ |
4968 | if (!priv->sph) |
4969 | return 0; |
4970 | |
4971 | /* Not last descriptor */ |
4972 | if (status & rx_not_ls) |
4973 | return priv->dma_conf.dma_buf_sz; |
4974 | |
4975 | plen = stmmac_get_rx_frame_len(priv, p, coe); |
4976 | |
4977 | /* Last descriptor */ |
4978 | return plen - len; |
4979 | } |
4980 | |
4981 | static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, |
4982 | struct xdp_frame *xdpf, bool dma_map) |
4983 | { |
4984 | struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; |
4985 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
4986 | unsigned int entry = tx_q->cur_tx; |
4987 | struct dma_desc *tx_desc; |
4988 | dma_addr_t dma_addr; |
4989 | bool set_ic; |
4990 | |
4991 | if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) |
4992 | return STMMAC_XDP_CONSUMED; |
4993 | |
4994 | if (priv->est && priv->est->enable && |
4995 | priv->est->max_sdu[queue] && |
4996 | xdpf->len > priv->est->max_sdu[queue]) { |
4997 | priv->xstats.max_sdu_txq_drop[queue]++; |
4998 | return STMMAC_XDP_CONSUMED; |
4999 | } |
5000 | |
5001 | if (likely(priv->extend_desc)) |
5002 | tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); |
5003 | else if (tx_q->tbs & STMMAC_TBS_AVAIL) |
5004 | tx_desc = &tx_q->dma_entx[entry].basic; |
5005 | else |
5006 | tx_desc = tx_q->dma_tx + entry; |
5007 | |
5008 | if (dma_map) { |
5009 | dma_addr = dma_map_single(priv->device, xdpf->data, |
5010 | xdpf->len, DMA_TO_DEVICE); |
5011 | if (dma_mapping_error(dev: priv->device, dma_addr)) |
5012 | return STMMAC_XDP_CONSUMED; |
5013 | |
5014 | tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; |
5015 | } else { |
5016 | struct page *page = virt_to_page(xdpf->data); |
5017 | |
5018 | dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + |
5019 | xdpf->headroom; |
5020 | dma_sync_single_for_device(dev: priv->device, addr: dma_addr, |
5021 | size: xdpf->len, dir: DMA_BIDIRECTIONAL); |
5022 | |
5023 | tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; |
5024 | } |
5025 | |
5026 | tx_q->tx_skbuff_dma[entry].buf = dma_addr; |
5027 | tx_q->tx_skbuff_dma[entry].map_as_page = false; |
5028 | tx_q->tx_skbuff_dma[entry].len = xdpf->len; |
5029 | tx_q->tx_skbuff_dma[entry].last_segment = true; |
5030 | tx_q->tx_skbuff_dma[entry].is_jumbo = false; |
5031 | |
5032 | tx_q->xdpf[entry] = xdpf; |
5033 | |
5034 | stmmac_set_desc_addr(priv, tx_desc, dma_addr); |
5035 | |
5036 | stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, |
5037 | true, priv->mode, true, true, |
5038 | xdpf->len); |
5039 | |
5040 | tx_q->tx_count_frames++; |
5041 | |
5042 | if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) |
5043 | set_ic = true; |
5044 | else |
5045 | set_ic = false; |
5046 | |
5047 | if (set_ic) { |
5048 | tx_q->tx_count_frames = 0; |
5049 | stmmac_set_tx_ic(priv, tx_desc); |
5050 | u64_stats_update_begin(syncp: &txq_stats->q_syncp); |
5051 | u64_stats_inc(p: &txq_stats->q.tx_set_ic_bit); |
5052 | u64_stats_update_end(syncp: &txq_stats->q_syncp); |
5053 | } |
5054 | |
5055 | stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); |
5056 | |
5057 | entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); |
5058 | tx_q->cur_tx = entry; |
5059 | |
5060 | return STMMAC_XDP_TX; |
5061 | } |
5062 | |
5063 | static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, |
5064 | int cpu) |
5065 | { |
5066 | int index = cpu; |
5067 | |
5068 | if (unlikely(index < 0)) |
5069 | index = 0; |
5070 | |
5071 | while (index >= priv->plat->tx_queues_to_use) |
5072 | index -= priv->plat->tx_queues_to_use; |
5073 | |
5074 | return index; |
5075 | } |
5076 | |
5077 | static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, |
5078 | struct xdp_buff *xdp) |
5079 | { |
5080 | struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); |
5081 | int cpu = smp_processor_id(); |
5082 | struct netdev_queue *nq; |
5083 | int queue; |
5084 | int res; |
5085 | |
5086 | if (unlikely(!xdpf)) |
5087 | return STMMAC_XDP_CONSUMED; |
5088 | |
5089 | queue = stmmac_xdp_get_tx_queue(priv, cpu); |
5090 | nq = netdev_get_tx_queue(dev: priv->dev, index: queue); |
5091 | |
5092 | __netif_tx_lock(txq: nq, cpu); |
5093 | /* Avoids TX time-out as we are sharing with slow path */ |
5094 | txq_trans_cond_update(txq: nq); |
5095 | |
5096 | res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, dma_map: false); |
5097 | if (res == STMMAC_XDP_TX) |
5098 | stmmac_flush_tx_descriptors(priv, queue); |
5099 | |
5100 | __netif_tx_unlock(txq: nq); |
5101 | |
5102 | return res; |
5103 | } |
5104 | |
5105 | static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, |
5106 | struct bpf_prog *prog, |
5107 | struct xdp_buff *xdp) |
5108 | { |
5109 | u32 act; |
5110 | int res; |
5111 | |
5112 | act = bpf_prog_run_xdp(prog, xdp); |
5113 | switch (act) { |
5114 | case XDP_PASS: |
5115 | res = STMMAC_XDP_PASS; |
5116 | break; |
5117 | case XDP_TX: |
5118 | res = stmmac_xdp_xmit_back(priv, xdp); |
5119 | break; |
5120 | case XDP_REDIRECT: |
5121 | if (xdp_do_redirect(dev: priv->dev, xdp, prog) < 0) |
5122 | res = STMMAC_XDP_CONSUMED; |
5123 | else |
5124 | res = STMMAC_XDP_REDIRECT; |
5125 | break; |
5126 | default: |
5127 | bpf_warn_invalid_xdp_action(dev: priv->dev, prog, act); |
5128 | fallthrough; |
5129 | case XDP_ABORTED: |
5130 | trace_xdp_exception(dev: priv->dev, xdp: prog, act); |
5131 | fallthrough; |
5132 | case XDP_DROP: |
5133 | res = STMMAC_XDP_CONSUMED; |
5134 | break; |
5135 | } |
5136 | |
5137 | return res; |
5138 | } |
5139 | |
5140 | static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, |
5141 | struct xdp_buff *xdp) |
5142 | { |
5143 | struct bpf_prog *prog; |
5144 | int res; |
5145 | |
5146 | prog = READ_ONCE(priv->xdp_prog); |
5147 | if (!prog) { |
5148 | res = STMMAC_XDP_PASS; |
5149 | goto out; |
5150 | } |
5151 | |
5152 | res = __stmmac_xdp_run_prog(priv, prog, xdp); |
5153 | out: |
5154 | return ERR_PTR(error: -res); |
5155 | } |
5156 | |
5157 | static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, |
5158 | int xdp_status) |
5159 | { |
5160 | int cpu = smp_processor_id(); |
5161 | int queue; |
5162 | |
5163 | queue = stmmac_xdp_get_tx_queue(priv, cpu); |
5164 | |
5165 | if (xdp_status & STMMAC_XDP_TX) |
5166 | stmmac_tx_timer_arm(priv, queue); |
5167 | |
5168 | if (xdp_status & STMMAC_XDP_REDIRECT) |
5169 | xdp_do_flush(); |
5170 | } |
5171 | |
5172 | static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, |
5173 | struct xdp_buff *xdp) |
5174 | { |
5175 | unsigned int metasize = xdp->data - xdp->data_meta; |
5176 | unsigned int datasize = xdp->data_end - xdp->data; |
5177 | struct sk_buff *skb; |
5178 | |
5179 | skb = napi_alloc_skb(napi: &ch->rxtx_napi, |
5180 | length: xdp->data_end - xdp->data_hard_start); |
5181 | if (unlikely(!skb)) |
5182 | return NULL; |
5183 | |
5184 | skb_reserve(skb, len: xdp->data - xdp->data_hard_start); |
5185 | memcpy(__skb_put(skb, datasize), xdp->data, datasize); |
5186 | if (metasize) |
5187 | skb_metadata_set(skb, meta_len: metasize); |
5188 | |
5189 | return skb; |
5190 | } |
5191 | |
5192 | static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, |
5193 | struct dma_desc *p, struct dma_desc *np, |
5194 | struct xdp_buff *xdp) |
5195 | { |
5196 | struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; |
5197 | struct stmmac_channel *ch = &priv->channel[queue]; |
5198 | unsigned int len = xdp->data_end - xdp->data; |
5199 | enum pkt_hash_types hash_type; |
5200 | int coe = priv->hw->rx_csum; |
5201 | struct sk_buff *skb; |
5202 | u32 hash; |
5203 | |
5204 | skb = stmmac_construct_skb_zc(ch, xdp); |
5205 | if (!skb) { |
5206 | priv->xstats.rx_dropped++; |
5207 | return; |
5208 | } |
5209 | |
5210 | stmmac_get_rx_hwtstamp(priv, p, np, skb); |
5211 | if (priv->hw->hw_vlan_en) |
5212 | /* MAC level stripping. */ |
5213 | stmmac_rx_hw_vlan(priv, priv->hw, p, skb); |
5214 | else |
5215 | /* Driver level stripping. */ |
5216 | stmmac_rx_vlan(dev: priv->dev, skb); |
5217 | skb->protocol = eth_type_trans(skb, dev: priv->dev); |
5218 | |
5219 | if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) |
5220 | skb_checksum_none_assert(skb); |
5221 | else |
5222 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
5223 | |
5224 | if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) |
5225 | skb_set_hash(skb, hash, type: hash_type); |
5226 | |
5227 | skb_record_rx_queue(skb, rx_queue: queue); |
5228 | napi_gro_receive(napi: &ch->rxtx_napi, skb); |
5229 | |
5230 | u64_stats_update_begin(syncp: &rxq_stats->napi_syncp); |
5231 | u64_stats_inc(p: &rxq_stats->napi.rx_pkt_n); |
5232 | u64_stats_add(p: &rxq_stats->napi.rx_bytes, val: len); |
5233 | u64_stats_update_end(syncp: &rxq_stats->napi_syncp); |
5234 | } |
5235 | |
5236 | static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) |
5237 | { |
5238 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; |
5239 | unsigned int entry = rx_q->dirty_rx; |
5240 | struct dma_desc *rx_desc = NULL; |
5241 | bool ret = true; |
5242 | |
5243 | budget = min(budget, stmmac_rx_dirty(priv, queue)); |
5244 | |
5245 | while (budget-- > 0 && entry != rx_q->cur_rx) { |
5246 | struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; |
5247 | dma_addr_t dma_addr; |
5248 | bool use_rx_wd; |
5249 | |
5250 | if (!buf->xdp) { |
5251 | buf->xdp = xsk_buff_alloc(pool: rx_q->xsk_pool); |
5252 | if (!buf->xdp) { |
5253 | ret = false; |
5254 | break; |
5255 | } |
5256 | } |
5257 | |
5258 | if (priv->extend_desc) |
5259 | rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); |
5260 | else |
5261 | rx_desc = rx_q->dma_rx + entry; |
5262 | |
5263 | dma_addr = xsk_buff_xdp_get_dma(xdp: buf->xdp); |
5264 | stmmac_set_desc_addr(priv, rx_desc, dma_addr); |
5265 | stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); |
5266 | stmmac_refill_desc3(priv, rx_q, rx_desc); |
5267 | |
5268 | rx_q->rx_count_frames++; |
5269 | rx_q->rx_count_frames += priv->rx_coal_frames[queue]; |
5270 | if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) |
5271 | rx_q->rx_count_frames = 0; |
5272 | |
5273 | use_rx_wd = !priv->rx_coal_frames[queue]; |
5274 | use_rx_wd |= rx_q->rx_count_frames > 0; |
5275 | if (!priv->use_riwt) |
5276 | use_rx_wd = false; |
5277 | |
5278 | dma_wmb(); |
5279 | stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); |
5280 | |
5281 | entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); |
5282 | } |
5283 | |
5284 | if (rx_desc) { |
5285 | rx_q->dirty_rx = entry; |
5286 | rx_q->rx_tail_addr = rx_q->dma_rx_phy + |
5287 | (rx_q->dirty_rx * sizeof(struct dma_desc)); |
5288 | stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); |
5289 | } |
5290 | |
5291 | return ret; |
5292 | } |
5293 | |
5294 | static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) |
5295 | { |
5296 | /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used |
5297 | * to represent incoming packet, whereas cb field in the same structure |
5298 | * is used to store driver specific info. Thus, struct stmmac_xdp_buff |
5299 | * is laid on top of xdp and cb fields of struct xdp_buff_xsk. |
5300 | */ |
5301 | return (struct stmmac_xdp_buff *)xdp; |
5302 | } |
5303 | |
5304 | static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) |
5305 | { |
5306 | struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; |
5307 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; |
5308 | unsigned int count = 0, error = 0, len = 0; |
5309 | int dirty = stmmac_rx_dirty(priv, queue); |
5310 | unsigned int next_entry = rx_q->cur_rx; |
5311 | u32 rx_errors = 0, rx_dropped = 0; |
5312 | unsigned int desc_size; |
5313 | struct bpf_prog *prog; |
5314 | bool failure = false; |
5315 | int xdp_status = 0; |
5316 | int status = 0; |
5317 | |
5318 | if (netif_msg_rx_status(priv)) { |
5319 | void *rx_head; |
5320 | |
5321 | netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); |
5322 | if (priv->extend_desc) { |
5323 | rx_head = (void *)rx_q->dma_erx; |
5324 | desc_size = sizeof(struct dma_extended_desc); |
5325 | } else { |
5326 | rx_head = (void *)rx_q->dma_rx; |
5327 | desc_size = sizeof(struct dma_desc); |
5328 | } |
5329 | |
5330 | stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, |
5331 | rx_q->dma_rx_phy, desc_size); |
5332 | } |
5333 | while (count < limit) { |
5334 | struct stmmac_rx_buffer *buf; |
5335 | struct stmmac_xdp_buff *ctx; |
5336 | unsigned int buf1_len = 0; |
5337 | struct dma_desc *np, *p; |
5338 | int entry; |
5339 | int res; |
5340 | |
5341 | if (!count && rx_q->state_saved) { |
5342 | error = rx_q->state.error; |
5343 | len = rx_q->state.len; |
5344 | } else { |
5345 | rx_q->state_saved = false; |
5346 | error = 0; |
5347 | len = 0; |
5348 | } |
5349 | |
5350 | if (count >= limit) |
5351 | break; |
5352 | |
5353 | read_again: |
5354 | buf1_len = 0; |
5355 | entry = next_entry; |
5356 | buf = &rx_q->buf_pool[entry]; |
5357 | |
5358 | if (dirty >= STMMAC_RX_FILL_BATCH) { |
5359 | failure = failure || |
5360 | !stmmac_rx_refill_zc(priv, queue, budget: dirty); |
5361 | dirty = 0; |
5362 | } |
5363 | |
5364 | if (priv->extend_desc) |
5365 | p = (struct dma_desc *)(rx_q->dma_erx + entry); |
5366 | else |
5367 | p = rx_q->dma_rx + entry; |
5368 | |
5369 | /* read the status of the incoming frame */ |
5370 | status = stmmac_rx_status(priv, &priv->xstats, p); |
5371 | /* check if managed by the DMA otherwise go ahead */ |
5372 | if (unlikely(status & dma_own)) |
5373 | break; |
5374 | |
5375 | /* Prefetch the next RX descriptor */ |
5376 | rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, |
5377 | priv->dma_conf.dma_rx_size); |
5378 | next_entry = rx_q->cur_rx; |
5379 | |
5380 | if (priv->extend_desc) |
5381 | np = (struct dma_desc *)(rx_q->dma_erx + next_entry); |
5382 | else |
5383 | np = rx_q->dma_rx + next_entry; |
5384 | |
5385 | prefetch(np); |
5386 | |
5387 | /* Ensure a valid XSK buffer before proceed */ |
5388 | if (!buf->xdp) |
5389 | break; |
5390 | |
5391 | if (priv->extend_desc) |
5392 | stmmac_rx_extended_status(priv, &priv->xstats, |
5393 | rx_q->dma_erx + entry); |
5394 | if (unlikely(status == discard_frame)) { |
5395 | xsk_buff_free(xdp: buf->xdp); |
5396 | buf->xdp = NULL; |
5397 | dirty++; |
5398 | error = 1; |
5399 | if (!priv->hwts_rx_en) |
5400 | rx_errors++; |
5401 | } |
5402 | |
5403 | if (unlikely(error && (status & rx_not_ls))) |
5404 | goto read_again; |
5405 | if (unlikely(error)) { |
5406 | count++; |
5407 | continue; |
5408 | } |
5409 | |
5410 | /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ |
5411 | if (likely(status & rx_not_ls)) { |
5412 | xsk_buff_free(xdp: buf->xdp); |
5413 | buf->xdp = NULL; |
5414 | dirty++; |
5415 | count++; |
5416 | goto read_again; |
5417 | } |
5418 | |
5419 | ctx = xsk_buff_to_stmmac_ctx(xdp: buf->xdp); |
5420 | ctx->priv = priv; |
5421 | ctx->desc = p; |
5422 | ctx->ndesc = np; |
5423 | |
5424 | /* XDP ZC Frame only support primary buffers for now */ |
5425 | buf1_len = stmmac_rx_buf1_len(priv, p, status, len); |
5426 | len += buf1_len; |
5427 | |
5428 | /* ACS is disabled; strip manually. */ |
5429 | if (likely(!(status & rx_not_ls))) { |
5430 | buf1_len -= ETH_FCS_LEN; |
5431 | len -= ETH_FCS_LEN; |
5432 | } |
5433 | |
5434 | /* RX buffer is good and fit into a XSK pool buffer */ |
5435 | buf->xdp->data_end = buf->xdp->data + buf1_len; |
5436 | xsk_buff_dma_sync_for_cpu(xdp: buf->xdp); |
5437 | |
5438 | prog = READ_ONCE(priv->xdp_prog); |
5439 | res = __stmmac_xdp_run_prog(priv, prog, xdp: buf->xdp); |
5440 | |
5441 | switch (res) { |
5442 | case STMMAC_XDP_PASS: |
5443 | stmmac_dispatch_skb_zc(priv, queue, p, np, xdp: buf->xdp); |
5444 | xsk_buff_free(xdp: buf->xdp); |
5445 | break; |
5446 | case STMMAC_XDP_CONSUMED: |
5447 | xsk_buff_free(xdp: buf->xdp); |
5448 | rx_dropped++; |
5449 | break; |
5450 | case STMMAC_XDP_TX: |
5451 | case STMMAC_XDP_REDIRECT: |
5452 | xdp_status |= res; |
5453 | break; |
5454 | } |
5455 | |
5456 | buf->xdp = NULL; |
5457 | dirty++; |
5458 | count++; |
5459 | } |
5460 | |
5461 | if (status & rx_not_ls) { |
5462 | rx_q->state_saved = true; |
5463 | rx_q->state.error = error; |
5464 | rx_q->state.len = len; |
5465 | } |
5466 | |
5467 | stmmac_finalize_xdp_rx(priv, xdp_status); |
5468 | |
5469 | u64_stats_update_begin(syncp: &rxq_stats->napi_syncp); |
5470 | u64_stats_add(p: &rxq_stats->napi.rx_pkt_n, val: count); |
5471 | u64_stats_update_end(syncp: &rxq_stats->napi_syncp); |
5472 | |
5473 | priv->xstats.rx_dropped += rx_dropped; |
5474 | priv->xstats.rx_errors += rx_errors; |
5475 | |
5476 | if (xsk_uses_need_wakeup(pool: rx_q->xsk_pool)) { |
5477 | if (failure || stmmac_rx_dirty(priv, queue) > 0) |
5478 | xsk_set_rx_need_wakeup(pool: rx_q->xsk_pool); |
5479 | else |
5480 | xsk_clear_rx_need_wakeup(pool: rx_q->xsk_pool); |
5481 | |
5482 | return (int)count; |
5483 | } |
5484 | |
5485 | return failure ? limit : (int)count; |
5486 | } |
5487 | |
5488 | /** |
5489 | * stmmac_rx - manage the receive process |
5490 | * @priv: driver private structure |
5491 | * @limit: napi bugget |
5492 | * @queue: RX queue index. |
5493 | * Description : this the function called by the napi poll method. |
5494 | * It gets all the frames inside the ring. |
5495 | */ |
5496 | static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) |
5497 | { |
5498 | u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; |
5499 | struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; |
5500 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; |
5501 | struct stmmac_channel *ch = &priv->channel[queue]; |
5502 | unsigned int count = 0, error = 0, len = 0; |
5503 | int status = 0, coe = priv->hw->rx_csum; |
5504 | unsigned int next_entry = rx_q->cur_rx; |
5505 | enum dma_data_direction dma_dir; |
5506 | unsigned int desc_size; |
5507 | struct sk_buff *skb = NULL; |
5508 | struct stmmac_xdp_buff ctx; |
5509 | int xdp_status = 0; |
5510 | int bufsz; |
5511 | |
5512 | dma_dir = page_pool_get_dma_dir(pool: rx_q->page_pool); |
5513 | bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; |
5514 | limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit); |
5515 | |
5516 | if (netif_msg_rx_status(priv)) { |
5517 | void *rx_head; |
5518 | |
5519 | netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); |
5520 | if (priv->extend_desc) { |
5521 | rx_head = (void *)rx_q->dma_erx; |
5522 | desc_size = sizeof(struct dma_extended_desc); |
5523 | } else { |
5524 | rx_head = (void *)rx_q->dma_rx; |
5525 | desc_size = sizeof(struct dma_desc); |
5526 | } |
5527 | |
5528 | stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, |
5529 | rx_q->dma_rx_phy, desc_size); |
5530 | } |
5531 | while (count < limit) { |
5532 | unsigned int buf1_len = 0, buf2_len = 0; |
5533 | enum pkt_hash_types hash_type; |
5534 | struct stmmac_rx_buffer *buf; |
5535 | struct dma_desc *np, *p; |
5536 | int entry; |
5537 | u32 hash; |
5538 | |
5539 | if (!count && rx_q->state_saved) { |
5540 | skb = rx_q->state.skb; |
5541 | error = rx_q->state.error; |
5542 | len = rx_q->state.len; |
5543 | } else { |
5544 | rx_q->state_saved = false; |
5545 | skb = NULL; |
5546 | error = 0; |
5547 | len = 0; |
5548 | } |
5549 | |
5550 | read_again: |
5551 | if (count >= limit) |
5552 | break; |
5553 | |
5554 | buf1_len = 0; |
5555 | buf2_len = 0; |
5556 | entry = next_entry; |
5557 | buf = &rx_q->buf_pool[entry]; |
5558 | |
5559 | if (priv->extend_desc) |
5560 | p = (struct dma_desc *)(rx_q->dma_erx + entry); |
5561 | else |
5562 | p = rx_q->dma_rx + entry; |
5563 | |
5564 | /* read the status of the incoming frame */ |
5565 | status = stmmac_rx_status(priv, &priv->xstats, p); |
5566 | /* check if managed by the DMA otherwise go ahead */ |
5567 | if (unlikely(status & dma_own)) |
5568 | break; |
5569 | |
5570 | rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, |
5571 | priv->dma_conf.dma_rx_size); |
5572 | next_entry = rx_q->cur_rx; |
5573 | |
5574 | if (priv->extend_desc) |
5575 | np = (struct dma_desc *)(rx_q->dma_erx + next_entry); |
5576 | else |
5577 | np = rx_q->dma_rx + next_entry; |
5578 | |
5579 | prefetch(np); |
5580 | |
5581 | if (priv->extend_desc) |
5582 | stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); |
5583 | if (unlikely(status == discard_frame)) { |
5584 | page_pool_put_page(pool: rx_q->page_pool, page: buf->page, dma_sync_size: 0, allow_direct: true); |
5585 | buf->page = NULL; |
5586 | error = 1; |
5587 | if (!priv->hwts_rx_en) |
5588 | rx_errors++; |
5589 | } |
5590 | |
5591 | if (unlikely(error && (status & rx_not_ls))) |
5592 | goto read_again; |
5593 | if (unlikely(error)) { |
5594 | dev_kfree_skb(skb); |
5595 | skb = NULL; |
5596 | count++; |
5597 | continue; |
5598 | } |
5599 | |
5600 | /* Buffer is good. Go on. */ |
5601 | |
5602 | buf1_len = stmmac_rx_buf1_len(priv, p, status, len); |
5603 | len += buf1_len; |
5604 | buf2_len = stmmac_rx_buf2_len(priv, p, status, len); |
5605 | len += buf2_len; |
5606 | |
5607 | /* ACS is disabled; strip manually. */ |
5608 | if (likely(!(status & rx_not_ls))) { |
5609 | if (buf2_len) { |
5610 | buf2_len -= ETH_FCS_LEN; |
5611 | len -= ETH_FCS_LEN; |
5612 | } else if (buf1_len) { |
5613 | buf1_len -= ETH_FCS_LEN; |
5614 | len -= ETH_FCS_LEN; |
5615 | } |
5616 | } |
5617 | |
5618 | if (!skb) { |
5619 | unsigned int pre_len, sync_len; |
5620 | |
5621 | dma_sync_single_for_cpu(dev: priv->device, addr: buf->addr, |
5622 | size: buf1_len, dir: dma_dir); |
5623 | net_prefetch(page_address(buf->page) + |
5624 | buf->page_offset); |
5625 | |
5626 | xdp_init_buff(xdp: &ctx.xdp, frame_sz: bufsz, rxq: &rx_q->xdp_rxq); |
5627 | xdp_prepare_buff(xdp: &ctx.xdp, page_address(buf->page), |
5628 | headroom: buf->page_offset, data_len: buf1_len, meta_valid: true); |
5629 | |
5630 | pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - |
5631 | buf->page_offset; |
5632 | |
5633 | ctx.priv = priv; |
5634 | ctx.desc = p; |
5635 | ctx.ndesc = np; |
5636 | |
5637 | skb = stmmac_xdp_run_prog(priv, xdp: &ctx.xdp); |
5638 | /* Due xdp_adjust_tail: DMA sync for_device |
5639 | * cover max len CPU touch |
5640 | */ |
5641 | sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - |
5642 | buf->page_offset; |
5643 | sync_len = max(sync_len, pre_len); |
5644 | |
5645 | /* For Not XDP_PASS verdict */ |
5646 | if (IS_ERR(ptr: skb)) { |
5647 | unsigned int xdp_res = -PTR_ERR(ptr: skb); |
5648 | |
5649 | if (xdp_res & STMMAC_XDP_CONSUMED) { |
5650 | page_pool_put_page(pool: rx_q->page_pool, |
5651 | page: virt_to_head_page(x: ctx.xdp.data), |
5652 | dma_sync_size: sync_len, allow_direct: true); |
5653 | buf->page = NULL; |
5654 | rx_dropped++; |
5655 | |
5656 | /* Clear skb as it was set as |
5657 | * status by XDP program. |
5658 | */ |
5659 | skb = NULL; |
5660 | |
5661 | if (unlikely((status & rx_not_ls))) |
5662 | goto read_again; |
5663 | |
5664 | count++; |
5665 | continue; |
5666 | } else if (xdp_res & (STMMAC_XDP_TX | |
5667 | STMMAC_XDP_REDIRECT)) { |
5668 | xdp_status |= xdp_res; |
5669 | buf->page = NULL; |
5670 | skb = NULL; |
5671 | count++; |
5672 | continue; |
5673 | } |
5674 | } |
5675 | } |
5676 | |
5677 | if (!skb) { |
5678 | unsigned int head_pad_len; |
5679 | |
5680 | /* XDP program may expand or reduce tail */ |
5681 | buf1_len = ctx.xdp.data_end - ctx.xdp.data; |
5682 | |
5683 | skb = napi_build_skb(page_address(buf->page), |
5684 | frag_size: rx_q->napi_skb_frag_size); |
5685 | if (!skb) { |
5686 | page_pool_recycle_direct(pool: rx_q->page_pool, |
5687 | page: buf->page); |
5688 | rx_dropped++; |
5689 | count++; |
5690 | goto drain_data; |
5691 | } |
5692 | |
5693 | /* XDP program may adjust header */ |
5694 | head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start; |
5695 | skb_reserve(skb, len: head_pad_len); |
5696 | skb_put(skb, len: buf1_len); |
5697 | skb_mark_for_recycle(skb); |
5698 | buf->page = NULL; |
5699 | } else if (buf1_len) { |
5700 | dma_sync_single_for_cpu(dev: priv->device, addr: buf->addr, |
5701 | size: buf1_len, dir: dma_dir); |
5702 | skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, |
5703 | page: buf->page, off: buf->page_offset, size: buf1_len, |
5704 | truesize: priv->dma_conf.dma_buf_sz); |
5705 | buf->page = NULL; |
5706 | } |
5707 | |
5708 | if (buf2_len) { |
5709 | dma_sync_single_for_cpu(dev: priv->device, addr: buf->sec_addr, |
5710 | size: buf2_len, dir: dma_dir); |
5711 | skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, |
5712 | page: buf->sec_page, off: 0, size: buf2_len, |
5713 | truesize: priv->dma_conf.dma_buf_sz); |
5714 | buf->sec_page = NULL; |
5715 | } |
5716 | |
5717 | drain_data: |
5718 | if (likely(status & rx_not_ls)) |
5719 | goto read_again; |
5720 | if (!skb) |
5721 | continue; |
5722 | |
5723 | /* Got entire packet into SKB. Finish it. */ |
5724 | |
5725 | stmmac_get_rx_hwtstamp(priv, p, np, skb); |
5726 | |
5727 | if (priv->hw->hw_vlan_en) |
5728 | /* MAC level stripping. */ |
5729 | stmmac_rx_hw_vlan(priv, priv->hw, p, skb); |
5730 | else |
5731 | /* Driver level stripping. */ |
5732 | stmmac_rx_vlan(dev: priv->dev, skb); |
5733 | |
5734 | skb->protocol = eth_type_trans(skb, dev: priv->dev); |
5735 | |
5736 | if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) |
5737 | skb_checksum_none_assert(skb); |
5738 | else |
5739 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
5740 | |
5741 | if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) |
5742 | skb_set_hash(skb, hash, type: hash_type); |
5743 | |
5744 | skb_record_rx_queue(skb, rx_queue: queue); |
5745 | napi_gro_receive(napi: &ch->rx_napi, skb); |
5746 | skb = NULL; |
5747 | |
5748 | rx_packets++; |
5749 | rx_bytes += len; |
5750 | count++; |
5751 | } |
5752 | |
5753 | if (status & rx_not_ls || skb) { |
5754 | rx_q->state_saved = true; |
5755 | rx_q->state.skb = skb; |
5756 | rx_q->state.error = error; |
5757 | rx_q->state.len = len; |
5758 | } |
5759 | |
5760 | stmmac_finalize_xdp_rx(priv, xdp_status); |
5761 | |
5762 | stmmac_rx_refill(priv, queue); |
5763 | |
5764 | u64_stats_update_begin(syncp: &rxq_stats->napi_syncp); |
5765 | u64_stats_add(p: &rxq_stats->napi.rx_packets, val: rx_packets); |
5766 | u64_stats_add(p: &rxq_stats->napi.rx_bytes, val: rx_bytes); |
5767 | u64_stats_add(p: &rxq_stats->napi.rx_pkt_n, val: count); |
5768 | u64_stats_update_end(syncp: &rxq_stats->napi_syncp); |
5769 | |
5770 | priv->xstats.rx_dropped += rx_dropped; |
5771 | priv->xstats.rx_errors += rx_errors; |
5772 | |
5773 | return count; |
5774 | } |
5775 | |
5776 | static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) |
5777 | { |
5778 | struct stmmac_channel *ch = |
5779 | container_of(napi, struct stmmac_channel, rx_napi); |
5780 | struct stmmac_priv *priv = ch->priv_data; |
5781 | struct stmmac_rxq_stats *rxq_stats; |
5782 | u32 chan = ch->index; |
5783 | int work_done; |
5784 | |
5785 | rxq_stats = &priv->xstats.rxq_stats[chan]; |
5786 | u64_stats_update_begin(syncp: &rxq_stats->napi_syncp); |
5787 | u64_stats_inc(p: &rxq_stats->napi.poll); |
5788 | u64_stats_update_end(syncp: &rxq_stats->napi_syncp); |
5789 | |
5790 | work_done = stmmac_rx(priv, limit: budget, queue: chan); |
5791 | if (work_done < budget && napi_complete_done(n: napi, work_done)) { |
5792 | unsigned long flags; |
5793 | |
5794 | spin_lock_irqsave(&ch->lock, flags); |
5795 | stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); |
5796 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
5797 | } |
5798 | |
5799 | return work_done; |
5800 | } |
5801 | |
5802 | static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) |
5803 | { |
5804 | struct stmmac_channel *ch = |
5805 | container_of(napi, struct stmmac_channel, tx_napi); |
5806 | struct stmmac_priv *priv = ch->priv_data; |
5807 | struct stmmac_txq_stats *txq_stats; |
5808 | bool pending_packets = false; |
5809 | u32 chan = ch->index; |
5810 | int work_done; |
5811 | |
5812 | txq_stats = &priv->xstats.txq_stats[chan]; |
5813 | u64_stats_update_begin(syncp: &txq_stats->napi_syncp); |
5814 | u64_stats_inc(p: &txq_stats->napi.poll); |
5815 | u64_stats_update_end(syncp: &txq_stats->napi_syncp); |
5816 | |
5817 | work_done = stmmac_tx_clean(priv, budget, queue: chan, pending_packets: &pending_packets); |
5818 | work_done = min(work_done, budget); |
5819 | |
5820 | if (work_done < budget && napi_complete_done(n: napi, work_done)) { |
5821 | unsigned long flags; |
5822 | |
5823 | spin_lock_irqsave(&ch->lock, flags); |
5824 | stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); |
5825 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
5826 | } |
5827 | |
5828 | /* TX still have packet to handle, check if we need to arm tx timer */ |
5829 | if (pending_packets) |
5830 | stmmac_tx_timer_arm(priv, queue: chan); |
5831 | |
5832 | return work_done; |
5833 | } |
5834 | |
5835 | static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) |
5836 | { |
5837 | struct stmmac_channel *ch = |
5838 | container_of(napi, struct stmmac_channel, rxtx_napi); |
5839 | struct stmmac_priv *priv = ch->priv_data; |
5840 | bool tx_pending_packets = false; |
5841 | int rx_done, tx_done, rxtx_done; |
5842 | struct stmmac_rxq_stats *rxq_stats; |
5843 | struct stmmac_txq_stats *txq_stats; |
5844 | u32 chan = ch->index; |
5845 | |
5846 | rxq_stats = &priv->xstats.rxq_stats[chan]; |
5847 | u64_stats_update_begin(syncp: &rxq_stats->napi_syncp); |
5848 | u64_stats_inc(p: &rxq_stats->napi.poll); |
5849 | u64_stats_update_end(syncp: &rxq_stats->napi_syncp); |
5850 | |
5851 | txq_stats = &priv->xstats.txq_stats[chan]; |
5852 | u64_stats_update_begin(syncp: &txq_stats->napi_syncp); |
5853 | u64_stats_inc(p: &txq_stats->napi.poll); |
5854 | u64_stats_update_end(syncp: &txq_stats->napi_syncp); |
5855 | |
5856 | tx_done = stmmac_tx_clean(priv, budget, queue: chan, pending_packets: &tx_pending_packets); |
5857 | tx_done = min(tx_done, budget); |
5858 | |
5859 | rx_done = stmmac_rx_zc(priv, limit: budget, queue: chan); |
5860 | |
5861 | rxtx_done = max(tx_done, rx_done); |
5862 | |
5863 | /* If either TX or RX work is not complete, return budget |
5864 | * and keep pooling |
5865 | */ |
5866 | if (rxtx_done >= budget) |
5867 | return budget; |
5868 | |
5869 | /* all work done, exit the polling mode */ |
5870 | if (napi_complete_done(n: napi, work_done: rxtx_done)) { |
5871 | unsigned long flags; |
5872 | |
5873 | spin_lock_irqsave(&ch->lock, flags); |
5874 | /* Both RX and TX work done are compelte, |
5875 | * so enable both RX & TX IRQs. |
5876 | */ |
5877 | stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); |
5878 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
5879 | } |
5880 | |
5881 | /* TX still have packet to handle, check if we need to arm tx timer */ |
5882 | if (tx_pending_packets) |
5883 | stmmac_tx_timer_arm(priv, queue: chan); |
5884 | |
5885 | return min(rxtx_done, budget - 1); |
5886 | } |
5887 | |
5888 | /** |
5889 | * stmmac_tx_timeout |
5890 | * @dev : Pointer to net device structure |
5891 | * @txqueue: the index of the hanging transmit queue |
5892 | * Description: this function is called when a packet transmission fails to |
5893 | * complete within a reasonable time. The driver will mark the error in the |
5894 | * netdev structure and arrange for the device to be reset to a sane state |
5895 | * in order to transmit a new packet. |
5896 | */ |
5897 | static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) |
5898 | { |
5899 | struct stmmac_priv *priv = netdev_priv(dev); |
5900 | |
5901 | stmmac_global_err(priv); |
5902 | } |
5903 | |
5904 | /** |
5905 | * stmmac_set_rx_mode - entry point for multicast addressing |
5906 | * @dev : pointer to the device structure |
5907 | * Description: |
5908 | * This function is a driver entry point which gets called by the kernel |
5909 | * whenever multicast addresses must be enabled/disabled. |
5910 | * Return value: |
5911 | * void. |
5912 | * |
5913 | * FIXME: This may need RXC to be running, but it may be called with BH |
5914 | * disabled, which means we can't call phylink_rx_clk_stop*(). |
5915 | */ |
5916 | static void stmmac_set_rx_mode(struct net_device *dev) |
5917 | { |
5918 | struct stmmac_priv *priv = netdev_priv(dev); |
5919 | |
5920 | stmmac_set_filter(priv, priv->hw, dev); |
5921 | } |
5922 | |
5923 | /** |
5924 | * stmmac_change_mtu - entry point to change MTU size for the device. |
5925 | * @dev : device pointer. |
5926 | * @new_mtu : the new MTU size for the device. |
5927 | * Description: the Maximum Transfer Unit (MTU) is used by the network layer |
5928 | * to drive packet transmission. Ethernet has an MTU of 1500 octets |
5929 | * (ETH_DATA_LEN). This value can be changed with ifconfig. |
5930 | * Return value: |
5931 | * 0 on success and an appropriate (-)ve integer as defined in errno.h |
5932 | * file on failure. |
5933 | */ |
5934 | static int stmmac_change_mtu(struct net_device *dev, int new_mtu) |
5935 | { |
5936 | struct stmmac_priv *priv = netdev_priv(dev); |
5937 | int txfifosz = priv->plat->tx_fifo_size; |
5938 | struct stmmac_dma_conf *dma_conf; |
5939 | const int mtu = new_mtu; |
5940 | int ret; |
5941 | |
5942 | if (txfifosz == 0) |
5943 | txfifosz = priv->dma_cap.tx_fifo_size; |
5944 | |
5945 | txfifosz /= priv->plat->tx_queues_to_use; |
5946 | |
5947 | if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { |
5948 | netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); |
5949 | return -EINVAL; |
5950 | } |
5951 | |
5952 | new_mtu = STMMAC_ALIGN(new_mtu); |
5953 | |
5954 | /* If condition true, FIFO is too small or MTU too large */ |
5955 | if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) |
5956 | return -EINVAL; |
5957 | |
5958 | if (netif_running(dev)) { |
5959 | netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); |
5960 | /* Try to allocate the new DMA conf with the new mtu */ |
5961 | dma_conf = stmmac_setup_dma_desc(priv, mtu); |
5962 | if (IS_ERR(ptr: dma_conf)) { |
5963 | netdev_err(dev: priv->dev, format: "failed allocating new dma conf for new MTU %d\n", |
5964 | mtu); |
5965 | return PTR_ERR(ptr: dma_conf); |
5966 | } |
5967 | |
5968 | stmmac_release(dev); |
5969 | |
5970 | ret = __stmmac_open(dev, dma_conf); |
5971 | if (ret) { |
5972 | free_dma_desc_resources(priv, dma_conf); |
5973 | kfree(objp: dma_conf); |
5974 | netdev_err(dev: priv->dev, format: "failed reopening the interface after MTU change\n"); |
5975 | return ret; |
5976 | } |
5977 | |
5978 | kfree(objp: dma_conf); |
5979 | |
5980 | stmmac_set_rx_mode(dev); |
5981 | } |
5982 | |
5983 | WRITE_ONCE(dev->mtu, mtu); |
5984 | netdev_update_features(dev); |
5985 | |
5986 | return 0; |
5987 | } |
5988 | |
5989 | static netdev_features_t stmmac_fix_features(struct net_device *dev, |
5990 | netdev_features_t features) |
5991 | { |
5992 | struct stmmac_priv *priv = netdev_priv(dev); |
5993 | |
5994 | if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) |
5995 | features &= ~NETIF_F_RXCSUM; |
5996 | |
5997 | if (!priv->plat->tx_coe) |
5998 | features &= ~NETIF_F_CSUM_MASK; |
5999 | |
6000 | /* Some GMAC devices have a bugged Jumbo frame support that |
6001 | * needs to have the Tx COE disabled for oversized frames |
6002 | * (due to limited buffer sizes). In this case we disable |
6003 | * the TX csum insertion in the TDES and not use SF. |
6004 | */ |
6005 | if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) |
6006 | features &= ~NETIF_F_CSUM_MASK; |
6007 | |
6008 | /* Disable tso if asked by ethtool */ |
6009 | if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { |
6010 | if (features & NETIF_F_TSO) |
6011 | priv->tso = true; |
6012 | else |
6013 | priv->tso = false; |
6014 | } |
6015 | |
6016 | return features; |
6017 | } |
6018 | |
6019 | static int stmmac_set_features(struct net_device *netdev, |
6020 | netdev_features_t features) |
6021 | { |
6022 | struct stmmac_priv *priv = netdev_priv(dev: netdev); |
6023 | |
6024 | /* Keep the COE Type in case of csum is supporting */ |
6025 | if (features & NETIF_F_RXCSUM) |
6026 | priv->hw->rx_csum = priv->plat->rx_coe; |
6027 | else |
6028 | priv->hw->rx_csum = 0; |
6029 | /* No check needed because rx_coe has been set before and it will be |
6030 | * fixed in case of issue. |
6031 | */ |
6032 | stmmac_rx_ipc(priv, priv->hw); |
6033 | |
6034 | if (priv->sph_cap) { |
6035 | bool sph_en = (priv->hw->rx_csum > 0) && priv->sph; |
6036 | u32 chan; |
6037 | |
6038 | for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) |
6039 | stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); |
6040 | } |
6041 | |
6042 | if (features & NETIF_F_HW_VLAN_CTAG_RX) |
6043 | priv->hw->hw_vlan_en = true; |
6044 | else |
6045 | priv->hw->hw_vlan_en = false; |
6046 | |
6047 | phylink_rx_clk_stop_block(priv->phylink); |
6048 | stmmac_set_hw_vlan_mode(priv, priv->hw); |
6049 | phylink_rx_clk_stop_unblock(priv->phylink); |
6050 | |
6051 | return 0; |
6052 | } |
6053 | |
6054 | static void stmmac_common_interrupt(struct stmmac_priv *priv) |
6055 | { |
6056 | u32 rx_cnt = priv->plat->rx_queues_to_use; |
6057 | u32 tx_cnt = priv->plat->tx_queues_to_use; |
6058 | u32 queues_count; |
6059 | u32 queue; |
6060 | bool xmac; |
6061 | |
6062 | xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac; |
6063 | queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; |
6064 | |
6065 | if (priv->irq_wake) |
6066 | pm_wakeup_event(dev: priv->device, msec: 0); |
6067 | |
6068 | if (priv->dma_cap.estsel) |
6069 | stmmac_est_irq_status(priv, priv, priv->dev, |
6070 | &priv->xstats, tx_cnt); |
6071 | |
6072 | if (stmmac_fpe_supported(priv)) |
6073 | stmmac_fpe_irq_status(priv); |
6074 | |
6075 | /* To handle GMAC own interrupts */ |
6076 | if ((priv->plat->has_gmac) || xmac) { |
6077 | int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); |
6078 | |
6079 | if (unlikely(status)) { |
6080 | /* For LPI we need to save the tx status */ |
6081 | if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) |
6082 | priv->tx_path_in_lpi_mode = true; |
6083 | if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) |
6084 | priv->tx_path_in_lpi_mode = false; |
6085 | } |
6086 | |
6087 | for (queue = 0; queue < queues_count; queue++) |
6088 | stmmac_host_mtl_irq_status(priv, priv->hw, queue); |
6089 | |
6090 | /* PCS link status */ |
6091 | if (priv->hw->pcs && |
6092 | !(priv->plat->flags & STMMAC_FLAG_HAS_INTEGRATED_PCS)) { |
6093 | if (priv->xstats.pcs_link) |
6094 | netif_carrier_on(dev: priv->dev); |
6095 | else |
6096 | netif_carrier_off(dev: priv->dev); |
6097 | } |
6098 | |
6099 | stmmac_timestamp_interrupt(priv, priv); |
6100 | } |
6101 | } |
6102 | |
6103 | /** |
6104 | * stmmac_interrupt - main ISR |
6105 | * @irq: interrupt number. |
6106 | * @dev_id: to pass the net device pointer. |
6107 | * Description: this is the main driver interrupt service routine. |
6108 | * It can call: |
6109 | * o DMA service routine (to manage incoming frame reception and transmission |
6110 | * status) |
6111 | * o Core interrupts to manage: remote wake-up, management counter, LPI |
6112 | * interrupts. |
6113 | */ |
6114 | static irqreturn_t stmmac_interrupt(int irq, void *dev_id) |
6115 | { |
6116 | struct net_device *dev = (struct net_device *)dev_id; |
6117 | struct stmmac_priv *priv = netdev_priv(dev); |
6118 | |
6119 | /* Check if adapter is up */ |
6120 | if (test_bit(STMMAC_DOWN, &priv->state)) |
6121 | return IRQ_HANDLED; |
6122 | |
6123 | /* Check ASP error if it isn't delivered via an individual IRQ */ |
6124 | if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv)) |
6125 | return IRQ_HANDLED; |
6126 | |
6127 | /* To handle Common interrupts */ |
6128 | stmmac_common_interrupt(priv); |
6129 | |
6130 | /* To handle DMA interrupts */ |
6131 | stmmac_dma_interrupt(priv); |
6132 | |
6133 | return IRQ_HANDLED; |
6134 | } |
6135 | |
6136 | static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) |
6137 | { |
6138 | struct net_device *dev = (struct net_device *)dev_id; |
6139 | struct stmmac_priv *priv = netdev_priv(dev); |
6140 | |
6141 | /* Check if adapter is up */ |
6142 | if (test_bit(STMMAC_DOWN, &priv->state)) |
6143 | return IRQ_HANDLED; |
6144 | |
6145 | /* To handle Common interrupts */ |
6146 | stmmac_common_interrupt(priv); |
6147 | |
6148 | return IRQ_HANDLED; |
6149 | } |
6150 | |
6151 | static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) |
6152 | { |
6153 | struct net_device *dev = (struct net_device *)dev_id; |
6154 | struct stmmac_priv *priv = netdev_priv(dev); |
6155 | |
6156 | /* Check if adapter is up */ |
6157 | if (test_bit(STMMAC_DOWN, &priv->state)) |
6158 | return IRQ_HANDLED; |
6159 | |
6160 | /* Check if a fatal error happened */ |
6161 | stmmac_safety_feat_interrupt(priv); |
6162 | |
6163 | return IRQ_HANDLED; |
6164 | } |
6165 | |
6166 | static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) |
6167 | { |
6168 | struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; |
6169 | struct stmmac_dma_conf *dma_conf; |
6170 | int chan = tx_q->queue_index; |
6171 | struct stmmac_priv *priv; |
6172 | int status; |
6173 | |
6174 | dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); |
6175 | priv = container_of(dma_conf, struct stmmac_priv, dma_conf); |
6176 | |
6177 | /* Check if adapter is up */ |
6178 | if (test_bit(STMMAC_DOWN, &priv->state)) |
6179 | return IRQ_HANDLED; |
6180 | |
6181 | status = stmmac_napi_check(priv, chan, dir: DMA_DIR_TX); |
6182 | |
6183 | if (unlikely(status & tx_hard_error_bump_tc)) { |
6184 | /* Try to bump up the dma threshold on this failure */ |
6185 | stmmac_bump_dma_threshold(priv, chan); |
6186 | } else if (unlikely(status == tx_hard_error)) { |
6187 | stmmac_tx_err(priv, chan); |
6188 | } |
6189 | |
6190 | return IRQ_HANDLED; |
6191 | } |
6192 | |
6193 | static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) |
6194 | { |
6195 | struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; |
6196 | struct stmmac_dma_conf *dma_conf; |
6197 | int chan = rx_q->queue_index; |
6198 | struct stmmac_priv *priv; |
6199 | |
6200 | dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); |
6201 | priv = container_of(dma_conf, struct stmmac_priv, dma_conf); |
6202 | |
6203 | /* Check if adapter is up */ |
6204 | if (test_bit(STMMAC_DOWN, &priv->state)) |
6205 | return IRQ_HANDLED; |
6206 | |
6207 | stmmac_napi_check(priv, chan, dir: DMA_DIR_RX); |
6208 | |
6209 | return IRQ_HANDLED; |
6210 | } |
6211 | |
6212 | /** |
6213 | * stmmac_ioctl - Entry point for the Ioctl |
6214 | * @dev: Device pointer. |
6215 | * @rq: An IOCTL specefic structure, that can contain a pointer to |
6216 | * a proprietary structure used to pass information to the driver. |
6217 | * @cmd: IOCTL command |
6218 | * Description: |
6219 | * Currently it supports the phy_mii_ioctl(...) and HW time stamping. |
6220 | */ |
6221 | static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) |
6222 | { |
6223 | struct stmmac_priv *priv = netdev_priv (dev); |
6224 | int ret = -EOPNOTSUPP; |
6225 | |
6226 | if (!netif_running(dev)) |
6227 | return -EINVAL; |
6228 | |
6229 | switch (cmd) { |
6230 | case SIOCGMIIPHY: |
6231 | case SIOCGMIIREG: |
6232 | case SIOCSMIIREG: |
6233 | ret = phylink_mii_ioctl(priv->phylink, rq, cmd); |
6234 | break; |
6235 | default: |
6236 | break; |
6237 | } |
6238 | |
6239 | return ret; |
6240 | } |
6241 | |
6242 | static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, |
6243 | void *cb_priv) |
6244 | { |
6245 | struct stmmac_priv *priv = cb_priv; |
6246 | int ret = -EOPNOTSUPP; |
6247 | |
6248 | if (!tc_cls_can_offload_and_chain0(dev: priv->dev, common: type_data)) |
6249 | return ret; |
6250 | |
6251 | __stmmac_disable_all_queues(priv); |
6252 | |
6253 | switch (type) { |
6254 | case TC_SETUP_CLSU32: |
6255 | ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); |
6256 | break; |
6257 | case TC_SETUP_CLSFLOWER: |
6258 | ret = stmmac_tc_setup_cls(priv, priv, type_data); |
6259 | break; |
6260 | default: |
6261 | break; |
6262 | } |
6263 | |
6264 | stmmac_enable_all_queues(priv); |
6265 | return ret; |
6266 | } |
6267 | |
6268 | static LIST_HEAD(stmmac_block_cb_list); |
6269 | |
6270 | static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, |
6271 | void *type_data) |
6272 | { |
6273 | struct stmmac_priv *priv = netdev_priv(dev: ndev); |
6274 | |
6275 | switch (type) { |
6276 | case TC_QUERY_CAPS: |
6277 | return stmmac_tc_query_caps(priv, priv, type_data); |
6278 | case TC_SETUP_QDISC_MQPRIO: |
6279 | return stmmac_tc_setup_mqprio(priv, priv, type_data); |
6280 | case TC_SETUP_BLOCK: |
6281 | return flow_block_cb_setup_simple(f: type_data, |
6282 | driver_list: &stmmac_block_cb_list, |
6283 | cb: stmmac_setup_tc_block_cb, |
6284 | cb_ident: priv, cb_priv: priv, ingress_only: true); |
6285 | case TC_SETUP_QDISC_CBS: |
6286 | return stmmac_tc_setup_cbs(priv, priv, type_data); |
6287 | case TC_SETUP_QDISC_TAPRIO: |
6288 | return stmmac_tc_setup_taprio(priv, priv, type_data); |
6289 | case TC_SETUP_QDISC_ETF: |
6290 | return stmmac_tc_setup_etf(priv, priv, type_data); |
6291 | default: |
6292 | return -EOPNOTSUPP; |
6293 | } |
6294 | } |
6295 | |
6296 | static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, |
6297 | struct net_device *sb_dev) |
6298 | { |
6299 | int gso = skb_shinfo(skb)->gso_type; |
6300 | |
6301 | if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { |
6302 | /* |
6303 | * There is no way to determine the number of TSO/USO |
6304 | * capable Queues. Let's use always the Queue 0 |
6305 | * because if TSO/USO is supported then at least this |
6306 | * one will be capable. |
6307 | */ |
6308 | return 0; |
6309 | } |
6310 | |
6311 | return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; |
6312 | } |
6313 | |
6314 | static int stmmac_set_mac_address(struct net_device *ndev, void *addr) |
6315 | { |
6316 | struct stmmac_priv *priv = netdev_priv(dev: ndev); |
6317 | int ret = 0; |
6318 | |
6319 | ret = pm_runtime_resume_and_get(dev: priv->device); |
6320 | if (ret < 0) |
6321 | return ret; |
6322 | |
6323 | ret = eth_mac_addr(dev: ndev, p: addr); |
6324 | if (ret) |
6325 | goto set_mac_error; |
6326 | |
6327 | phylink_rx_clk_stop_block(priv->phylink); |
6328 | stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); |
6329 | phylink_rx_clk_stop_unblock(priv->phylink); |
6330 | |
6331 | set_mac_error: |
6332 | pm_runtime_put(dev: priv->device); |
6333 | |
6334 | return ret; |
6335 | } |
6336 | |
6337 | #ifdef CONFIG_DEBUG_FS |
6338 | static struct dentry *stmmac_fs_dir; |
6339 | |
6340 | static void sysfs_display_ring(void *head, int size, int extend_desc, |
6341 | struct seq_file *seq, dma_addr_t dma_phy_addr) |
6342 | { |
6343 | struct dma_extended_desc *ep = (struct dma_extended_desc *)head; |
6344 | struct dma_desc *p = (struct dma_desc *)head; |
6345 | unsigned int desc_size; |
6346 | dma_addr_t dma_addr; |
6347 | int i; |
6348 | |
6349 | desc_size = extend_desc ? sizeof(*ep) : sizeof(*p); |
6350 | for (i = 0; i < size; i++) { |
6351 | dma_addr = dma_phy_addr + i * desc_size; |
6352 | seq_printf(m: seq, fmt: "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", |
6353 | i, &dma_addr, |
6354 | le32_to_cpu(p->des0), le32_to_cpu(p->des1), |
6355 | le32_to_cpu(p->des2), le32_to_cpu(p->des3)); |
6356 | if (extend_desc) |
6357 | p = &(++ep)->basic; |
6358 | else |
6359 | p++; |
6360 | } |
6361 | } |
6362 | |
6363 | static int stmmac_rings_status_show(struct seq_file *seq, void *v) |
6364 | { |
6365 | struct net_device *dev = seq->private; |
6366 | struct stmmac_priv *priv = netdev_priv(dev); |
6367 | u32 rx_count = priv->plat->rx_queues_to_use; |
6368 | u32 tx_count = priv->plat->tx_queues_to_use; |
6369 | u32 queue; |
6370 | |
6371 | if ((dev->flags & IFF_UP) == 0) |
6372 | return 0; |
6373 | |
6374 | for (queue = 0; queue < rx_count; queue++) { |
6375 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; |
6376 | |
6377 | seq_printf(m: seq, fmt: "RX Queue %d:\n", queue); |
6378 | |
6379 | if (priv->extend_desc) { |
6380 | seq_printf(m: seq, fmt: "Extended descriptor ring:\n"); |
6381 | sysfs_display_ring(head: (void *)rx_q->dma_erx, |
6382 | size: priv->dma_conf.dma_rx_size, extend_desc: 1, seq, dma_phy_addr: rx_q->dma_rx_phy); |
6383 | } else { |
6384 | seq_printf(m: seq, fmt: "Descriptor ring:\n"); |
6385 | sysfs_display_ring(head: (void *)rx_q->dma_rx, |
6386 | size: priv->dma_conf.dma_rx_size, extend_desc: 0, seq, dma_phy_addr: rx_q->dma_rx_phy); |
6387 | } |
6388 | } |
6389 | |
6390 | for (queue = 0; queue < tx_count; queue++) { |
6391 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
6392 | |
6393 | seq_printf(m: seq, fmt: "TX Queue %d:\n", queue); |
6394 | |
6395 | if (priv->extend_desc) { |
6396 | seq_printf(m: seq, fmt: "Extended descriptor ring:\n"); |
6397 | sysfs_display_ring(head: (void *)tx_q->dma_etx, |
6398 | size: priv->dma_conf.dma_tx_size, extend_desc: 1, seq, dma_phy_addr: tx_q->dma_tx_phy); |
6399 | } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { |
6400 | seq_printf(m: seq, fmt: "Descriptor ring:\n"); |
6401 | sysfs_display_ring(head: (void *)tx_q->dma_tx, |
6402 | size: priv->dma_conf.dma_tx_size, extend_desc: 0, seq, dma_phy_addr: tx_q->dma_tx_phy); |
6403 | } |
6404 | } |
6405 | |
6406 | return 0; |
6407 | } |
6408 | DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); |
6409 | |
6410 | static int stmmac_dma_cap_show(struct seq_file *seq, void *v) |
6411 | { |
6412 | static const char * const dwxgmac_timestamp_source[] = { |
6413 | "None", |
6414 | "Internal", |
6415 | "External", |
6416 | "Both", |
6417 | }; |
6418 | static const char * const dwxgmac_safety_feature_desc[] = { |
6419 | "No", |
6420 | "All Safety Features with ECC and Parity", |
6421 | "All Safety Features without ECC or Parity", |
6422 | "All Safety Features with Parity Only", |
6423 | "ECC Only", |
6424 | "UNDEFINED", |
6425 | "UNDEFINED", |
6426 | "UNDEFINED", |
6427 | }; |
6428 | struct net_device *dev = seq->private; |
6429 | struct stmmac_priv *priv = netdev_priv(dev); |
6430 | |
6431 | if (!priv->hw_cap_support) { |
6432 | seq_printf(m: seq, fmt: "DMA HW features not supported\n"); |
6433 | return 0; |
6434 | } |
6435 | |
6436 | seq_printf(m: seq, fmt: "==============================\n"); |
6437 | seq_printf(m: seq, fmt: "\tDMA HW features\n"); |
6438 | seq_printf(m: seq, fmt: "==============================\n"); |
6439 | |
6440 | seq_printf(m: seq, fmt: "\t10/100 Mbps: %s\n", |
6441 | (priv->dma_cap.mbps_10_100) ? "Y": "N"); |
6442 | seq_printf(m: seq, fmt: "\t1000 Mbps: %s\n", |
6443 | (priv->dma_cap.mbps_1000) ? "Y": "N"); |
6444 | seq_printf(m: seq, fmt: "\tHalf duplex: %s\n", |
6445 | (priv->dma_cap.half_duplex) ? "Y": "N"); |
6446 | if (priv->plat->has_xgmac) { |
6447 | seq_printf(m: seq, |
6448 | fmt: "\tNumber of Additional MAC address registers: %d\n", |
6449 | priv->dma_cap.multi_addr); |
6450 | } else { |
6451 | seq_printf(m: seq, fmt: "\tHash Filter: %s\n", |
6452 | (priv->dma_cap.hash_filter) ? "Y": "N"); |
6453 | seq_printf(m: seq, fmt: "\tMultiple MAC address registers: %s\n", |
6454 | (priv->dma_cap.multi_addr) ? "Y": "N"); |
6455 | } |
6456 | seq_printf(m: seq, fmt: "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", |
6457 | (priv->dma_cap.pcs) ? "Y": "N"); |
6458 | seq_printf(m: seq, fmt: "\tSMA (MDIO) Interface: %s\n", |
6459 | (priv->dma_cap.sma_mdio) ? "Y": "N"); |
6460 | seq_printf(m: seq, fmt: "\tPMT Remote wake up: %s\n", |
6461 | (priv->dma_cap.pmt_remote_wake_up) ? "Y": "N"); |
6462 | seq_printf(m: seq, fmt: "\tPMT Magic Frame: %s\n", |
6463 | (priv->dma_cap.pmt_magic_frame) ? "Y": "N"); |
6464 | seq_printf(m: seq, fmt: "\tRMON module: %s\n", |
6465 | (priv->dma_cap.rmon) ? "Y": "N"); |
6466 | seq_printf(m: seq, fmt: "\tIEEE 1588-2002 Time Stamp: %s\n", |
6467 | (priv->dma_cap.time_stamp) ? "Y": "N"); |
6468 | seq_printf(m: seq, fmt: "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", |
6469 | (priv->dma_cap.atime_stamp) ? "Y": "N"); |
6470 | if (priv->plat->has_xgmac) |
6471 | seq_printf(m: seq, fmt: "\tTimestamp System Time Source: %s\n", |
6472 | dwxgmac_timestamp_source[priv->dma_cap.tssrc]); |
6473 | seq_printf(m: seq, fmt: "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", |
6474 | (priv->dma_cap.eee) ? "Y": "N"); |
6475 | seq_printf(m: seq, fmt: "\tAV features: %s\n", (priv->dma_cap.av) ? "Y": "N"); |
6476 | seq_printf(m: seq, fmt: "\tChecksum Offload in TX: %s\n", |
6477 | (priv->dma_cap.tx_coe) ? "Y": "N"); |
6478 | if (priv->synopsys_id >= DWMAC_CORE_4_00 || |
6479 | priv->plat->has_xgmac) { |
6480 | seq_printf(m: seq, fmt: "\tIP Checksum Offload in RX: %s\n", |
6481 | (priv->dma_cap.rx_coe) ? "Y": "N"); |
6482 | } else { |
6483 | seq_printf(m: seq, fmt: "\tIP Checksum Offload (type1) in RX: %s\n", |
6484 | (priv->dma_cap.rx_coe_type1) ? "Y": "N"); |
6485 | seq_printf(m: seq, fmt: "\tIP Checksum Offload (type2) in RX: %s\n", |
6486 | (priv->dma_cap.rx_coe_type2) ? "Y": "N"); |
6487 | seq_printf(m: seq, fmt: "\tRXFIFO > 2048bytes: %s\n", |
6488 | (priv->dma_cap.rxfifo_over_2048) ? "Y": "N"); |
6489 | } |
6490 | seq_printf(m: seq, fmt: "\tNumber of Additional RX channel: %d\n", |
6491 | priv->dma_cap.number_rx_channel); |
6492 | seq_printf(m: seq, fmt: "\tNumber of Additional TX channel: %d\n", |
6493 | priv->dma_cap.number_tx_channel); |
6494 | seq_printf(m: seq, fmt: "\tNumber of Additional RX queues: %d\n", |
6495 | priv->dma_cap.number_rx_queues); |
6496 | seq_printf(m: seq, fmt: "\tNumber of Additional TX queues: %d\n", |
6497 | priv->dma_cap.number_tx_queues); |
6498 | seq_printf(m: seq, fmt: "\tEnhanced descriptors: %s\n", |
6499 | (priv->dma_cap.enh_desc) ? "Y": "N"); |
6500 | seq_printf(m: seq, fmt: "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); |
6501 | seq_printf(m: seq, fmt: "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); |
6502 | seq_printf(m: seq, fmt: "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? |
6503 | (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); |
6504 | seq_printf(m: seq, fmt: "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y": "N"); |
6505 | seq_printf(m: seq, fmt: "\tNumber of PPS Outputs: %d\n", |
6506 | priv->dma_cap.pps_out_num); |
6507 | seq_printf(m: seq, fmt: "\tSafety Features: %s\n", |
6508 | dwxgmac_safety_feature_desc[priv->dma_cap.asp]); |
6509 | seq_printf(m: seq, fmt: "\tFlexible RX Parser: %s\n", |
6510 | priv->dma_cap.frpsel ? "Y": "N"); |
6511 | seq_printf(m: seq, fmt: "\tEnhanced Addressing: %d\n", |
6512 | priv->dma_cap.host_dma_width); |
6513 | seq_printf(m: seq, fmt: "\tReceive Side Scaling: %s\n", |
6514 | priv->dma_cap.rssen ? "Y": "N"); |
6515 | seq_printf(m: seq, fmt: "\tVLAN Hash Filtering: %s\n", |
6516 | priv->dma_cap.vlhash ? "Y": "N"); |
6517 | seq_printf(m: seq, fmt: "\tSplit Header: %s\n", |
6518 | priv->dma_cap.sphen ? "Y": "N"); |
6519 | seq_printf(m: seq, fmt: "\tVLAN TX Insertion: %s\n", |
6520 | priv->dma_cap.vlins ? "Y": "N"); |
6521 | seq_printf(m: seq, fmt: "\tDouble VLAN: %s\n", |
6522 | priv->dma_cap.dvlan ? "Y": "N"); |
6523 | seq_printf(m: seq, fmt: "\tNumber of L3/L4 Filters: %d\n", |
6524 | priv->dma_cap.l3l4fnum); |
6525 | seq_printf(m: seq, fmt: "\tARP Offloading: %s\n", |
6526 | priv->dma_cap.arpoffsel ? "Y": "N"); |
6527 | seq_printf(m: seq, fmt: "\tEnhancements to Scheduled Traffic (EST): %s\n", |
6528 | priv->dma_cap.estsel ? "Y": "N"); |
6529 | seq_printf(m: seq, fmt: "\tFrame Preemption (FPE): %s\n", |
6530 | priv->dma_cap.fpesel ? "Y": "N"); |
6531 | seq_printf(m: seq, fmt: "\tTime-Based Scheduling (TBS): %s\n", |
6532 | priv->dma_cap.tbssel ? "Y": "N"); |
6533 | seq_printf(m: seq, fmt: "\tNumber of DMA Channels Enabled for TBS: %d\n", |
6534 | priv->dma_cap.tbs_ch_num); |
6535 | seq_printf(m: seq, fmt: "\tPer-Stream Filtering: %s\n", |
6536 | priv->dma_cap.sgfsel ? "Y": "N"); |
6537 | seq_printf(m: seq, fmt: "\tTX Timestamp FIFO Depth: %lu\n", |
6538 | BIT(priv->dma_cap.ttsfd) >> 1); |
6539 | seq_printf(m: seq, fmt: "\tNumber of Traffic Classes: %d\n", |
6540 | priv->dma_cap.numtc); |
6541 | seq_printf(m: seq, fmt: "\tDCB Feature: %s\n", |
6542 | priv->dma_cap.dcben ? "Y": "N"); |
6543 | seq_printf(m: seq, fmt: "\tIEEE 1588 High Word Register: %s\n", |
6544 | priv->dma_cap.advthword ? "Y": "N"); |
6545 | seq_printf(m: seq, fmt: "\tPTP Offload: %s\n", |
6546 | priv->dma_cap.ptoen ? "Y": "N"); |
6547 | seq_printf(m: seq, fmt: "\tOne-Step Timestamping: %s\n", |
6548 | priv->dma_cap.osten ? "Y": "N"); |
6549 | seq_printf(m: seq, fmt: "\tPriority-Based Flow Control: %s\n", |
6550 | priv->dma_cap.pfcen ? "Y": "N"); |
6551 | seq_printf(m: seq, fmt: "\tNumber of Flexible RX Parser Instructions: %lu\n", |
6552 | BIT(priv->dma_cap.frpes) << 6); |
6553 | seq_printf(m: seq, fmt: "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", |
6554 | BIT(priv->dma_cap.frpbs) << 6); |
6555 | seq_printf(m: seq, fmt: "\tParallel Instruction Processor Engines: %d\n", |
6556 | priv->dma_cap.frppipe_num); |
6557 | seq_printf(m: seq, fmt: "\tNumber of Extended VLAN Tag Filters: %lu\n", |
6558 | priv->dma_cap.nrvf_num ? |
6559 | (BIT(priv->dma_cap.nrvf_num) << 1) : 0); |
6560 | seq_printf(m: seq, fmt: "\tWidth of the Time Interval Field in GCL: %d\n", |
6561 | priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); |
6562 | seq_printf(m: seq, fmt: "\tDepth of GCL: %lu\n", |
6563 | priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); |
6564 | seq_printf(m: seq, fmt: "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", |
6565 | priv->dma_cap.cbtisel ? "Y": "N"); |
6566 | seq_printf(m: seq, fmt: "\tNumber of Auxiliary Snapshot Inputs: %d\n", |
6567 | priv->dma_cap.aux_snapshot_n); |
6568 | seq_printf(m: seq, fmt: "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", |
6569 | priv->dma_cap.pou_ost_en ? "Y": "N"); |
6570 | seq_printf(m: seq, fmt: "\tEnhanced DMA: %s\n", |
6571 | priv->dma_cap.edma ? "Y": "N"); |
6572 | seq_printf(m: seq, fmt: "\tDifferent Descriptor Cache: %s\n", |
6573 | priv->dma_cap.ediffc ? "Y": "N"); |
6574 | seq_printf(m: seq, fmt: "\tVxLAN/NVGRE: %s\n", |
6575 | priv->dma_cap.vxn ? "Y": "N"); |
6576 | seq_printf(m: seq, fmt: "\tDebug Memory Interface: %s\n", |
6577 | priv->dma_cap.dbgmem ? "Y": "N"); |
6578 | seq_printf(m: seq, fmt: "\tNumber of Policing Counters: %lu\n", |
6579 | priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); |
6580 | return 0; |
6581 | } |
6582 | DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); |
6583 | |
6584 | /* Use network device events to rename debugfs file entries. |
6585 | */ |
6586 | static int stmmac_device_event(struct notifier_block *unused, |
6587 | unsigned long event, void *ptr) |
6588 | { |
6589 | struct net_device *dev = netdev_notifier_info_to_dev(info: ptr); |
6590 | struct stmmac_priv *priv = netdev_priv(dev); |
6591 | |
6592 | if (dev->netdev_ops != &stmmac_netdev_ops) |
6593 | goto done; |
6594 | |
6595 | switch (event) { |
6596 | case NETDEV_CHANGENAME: |
6597 | debugfs_change_name(dentry: priv->dbgfs_dir, fmt: "%s", dev->name); |
6598 | break; |
6599 | } |
6600 | done: |
6601 | return NOTIFY_DONE; |
6602 | } |
6603 | |
6604 | static struct notifier_block stmmac_notifier = { |
6605 | .notifier_call = stmmac_device_event, |
6606 | }; |
6607 | |
6608 | static void stmmac_init_fs(struct net_device *dev) |
6609 | { |
6610 | struct stmmac_priv *priv = netdev_priv(dev); |
6611 | |
6612 | rtnl_lock(); |
6613 | |
6614 | /* Create per netdev entries */ |
6615 | priv->dbgfs_dir = debugfs_create_dir(name: dev->name, parent: stmmac_fs_dir); |
6616 | |
6617 | /* Entry to report DMA RX/TX rings */ |
6618 | debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, |
6619 | &stmmac_rings_status_fops); |
6620 | |
6621 | /* Entry to report the DMA HW features */ |
6622 | debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, |
6623 | &stmmac_dma_cap_fops); |
6624 | |
6625 | rtnl_unlock(); |
6626 | } |
6627 | |
6628 | static void stmmac_exit_fs(struct net_device *dev) |
6629 | { |
6630 | struct stmmac_priv *priv = netdev_priv(dev); |
6631 | |
6632 | debugfs_remove_recursive(dentry: priv->dbgfs_dir); |
6633 | } |
6634 | #endif /* CONFIG_DEBUG_FS */ |
6635 | |
6636 | static u32 stmmac_vid_crc32_le(__le16 vid_le) |
6637 | { |
6638 | unsigned char *data = (unsigned char *)&vid_le; |
6639 | unsigned char data_byte = 0; |
6640 | u32 crc = ~0x0; |
6641 | u32 temp = 0; |
6642 | int i, bits; |
6643 | |
6644 | bits = get_bitmask_order(VLAN_VID_MASK); |
6645 | for (i = 0; i < bits; i++) { |
6646 | if ((i % 8) == 0) |
6647 | data_byte = data[i / 8]; |
6648 | |
6649 | temp = ((crc & 1) ^ data_byte) & 1; |
6650 | crc >>= 1; |
6651 | data_byte >>= 1; |
6652 | |
6653 | if (temp) |
6654 | crc ^= 0xedb88320; |
6655 | } |
6656 | |
6657 | return crc; |
6658 | } |
6659 | |
6660 | static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) |
6661 | { |
6662 | u32 crc, hash = 0; |
6663 | u16 pmatch = 0; |
6664 | int count = 0; |
6665 | u16 vid = 0; |
6666 | |
6667 | for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { |
6668 | __le16 vid_le = cpu_to_le16(vid); |
6669 | crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; |
6670 | hash |= (1 << crc); |
6671 | count++; |
6672 | } |
6673 | |
6674 | if (!priv->dma_cap.vlhash) { |
6675 | if (count > 2) /* VID = 0 always passes filter */ |
6676 | return -EOPNOTSUPP; |
6677 | |
6678 | pmatch = vid; |
6679 | hash = 0; |
6680 | } |
6681 | |
6682 | return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); |
6683 | } |
6684 | |
6685 | /* FIXME: This may need RXC to be running, but it may be called with BH |
6686 | * disabled, which means we can't call phylink_rx_clk_stop*(). |
6687 | */ |
6688 | static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) |
6689 | { |
6690 | struct stmmac_priv *priv = netdev_priv(dev: ndev); |
6691 | bool is_double = false; |
6692 | int ret; |
6693 | |
6694 | ret = pm_runtime_resume_and_get(dev: priv->device); |
6695 | if (ret < 0) |
6696 | return ret; |
6697 | |
6698 | if (be16_to_cpu(proto) == ETH_P_8021AD) |
6699 | is_double = true; |
6700 | |
6701 | set_bit(nr: vid, addr: priv->active_vlans); |
6702 | ret = stmmac_vlan_update(priv, is_double); |
6703 | if (ret) { |
6704 | clear_bit(nr: vid, addr: priv->active_vlans); |
6705 | goto err_pm_put; |
6706 | } |
6707 | |
6708 | if (priv->hw->num_vlan) { |
6709 | ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); |
6710 | if (ret) |
6711 | goto err_pm_put; |
6712 | } |
6713 | err_pm_put: |
6714 | pm_runtime_put(dev: priv->device); |
6715 | |
6716 | return ret; |
6717 | } |
6718 | |
6719 | /* FIXME: This may need RXC to be running, but it may be called with BH |
6720 | * disabled, which means we can't call phylink_rx_clk_stop*(). |
6721 | */ |
6722 | static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) |
6723 | { |
6724 | struct stmmac_priv *priv = netdev_priv(dev: ndev); |
6725 | bool is_double = false; |
6726 | int ret; |
6727 | |
6728 | ret = pm_runtime_resume_and_get(dev: priv->device); |
6729 | if (ret < 0) |
6730 | return ret; |
6731 | |
6732 | if (be16_to_cpu(proto) == ETH_P_8021AD) |
6733 | is_double = true; |
6734 | |
6735 | clear_bit(nr: vid, addr: priv->active_vlans); |
6736 | |
6737 | if (priv->hw->num_vlan) { |
6738 | ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); |
6739 | if (ret) |
6740 | goto del_vlan_error; |
6741 | } |
6742 | |
6743 | ret = stmmac_vlan_update(priv, is_double); |
6744 | |
6745 | del_vlan_error: |
6746 | pm_runtime_put(dev: priv->device); |
6747 | |
6748 | return ret; |
6749 | } |
6750 | |
6751 | static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) |
6752 | { |
6753 | struct stmmac_priv *priv = netdev_priv(dev); |
6754 | |
6755 | switch (bpf->command) { |
6756 | case XDP_SETUP_PROG: |
6757 | return stmmac_xdp_set_prog(priv, prog: bpf->prog, extack: bpf->extack); |
6758 | case XDP_SETUP_XSK_POOL: |
6759 | return stmmac_xdp_setup_pool(priv, pool: bpf->xsk.pool, |
6760 | queue: bpf->xsk.queue_id); |
6761 | default: |
6762 | return -EOPNOTSUPP; |
6763 | } |
6764 | } |
6765 | |
6766 | static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, |
6767 | struct xdp_frame **frames, u32 flags) |
6768 | { |
6769 | struct stmmac_priv *priv = netdev_priv(dev); |
6770 | int cpu = smp_processor_id(); |
6771 | struct netdev_queue *nq; |
6772 | int i, nxmit = 0; |
6773 | int queue; |
6774 | |
6775 | if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) |
6776 | return -ENETDOWN; |
6777 | |
6778 | if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) |
6779 | return -EINVAL; |
6780 | |
6781 | queue = stmmac_xdp_get_tx_queue(priv, cpu); |
6782 | nq = netdev_get_tx_queue(dev: priv->dev, index: queue); |
6783 | |
6784 | __netif_tx_lock(txq: nq, cpu); |
6785 | /* Avoids TX time-out as we are sharing with slow path */ |
6786 | txq_trans_cond_update(txq: nq); |
6787 | |
6788 | for (i = 0; i < num_frames; i++) { |
6789 | int res; |
6790 | |
6791 | res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf: frames[i], dma_map: true); |
6792 | if (res == STMMAC_XDP_CONSUMED) |
6793 | break; |
6794 | |
6795 | nxmit++; |
6796 | } |
6797 | |
6798 | if (flags & XDP_XMIT_FLUSH) { |
6799 | stmmac_flush_tx_descriptors(priv, queue); |
6800 | stmmac_tx_timer_arm(priv, queue); |
6801 | } |
6802 | |
6803 | __netif_tx_unlock(txq: nq); |
6804 | |
6805 | return nxmit; |
6806 | } |
6807 | |
6808 | void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) |
6809 | { |
6810 | struct stmmac_channel *ch = &priv->channel[queue]; |
6811 | unsigned long flags; |
6812 | |
6813 | spin_lock_irqsave(&ch->lock, flags); |
6814 | stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); |
6815 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
6816 | |
6817 | stmmac_stop_rx_dma(priv, chan: queue); |
6818 | __free_dma_rx_desc_resources(priv, dma_conf: &priv->dma_conf, queue); |
6819 | } |
6820 | |
6821 | void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) |
6822 | { |
6823 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; |
6824 | struct stmmac_channel *ch = &priv->channel[queue]; |
6825 | unsigned long flags; |
6826 | u32 buf_size; |
6827 | int ret; |
6828 | |
6829 | ret = __alloc_dma_rx_desc_resources(priv, dma_conf: &priv->dma_conf, queue); |
6830 | if (ret) { |
6831 | netdev_err(dev: priv->dev, format: "Failed to alloc RX desc.\n"); |
6832 | return; |
6833 | } |
6834 | |
6835 | ret = __init_dma_rx_desc_rings(priv, dma_conf: &priv->dma_conf, queue, GFP_KERNEL); |
6836 | if (ret) { |
6837 | __free_dma_rx_desc_resources(priv, dma_conf: &priv->dma_conf, queue); |
6838 | netdev_err(dev: priv->dev, format: "Failed to init RX desc.\n"); |
6839 | return; |
6840 | } |
6841 | |
6842 | stmmac_reset_rx_queue(priv, queue); |
6843 | stmmac_clear_rx_descriptors(priv, dma_conf: &priv->dma_conf, queue); |
6844 | |
6845 | stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, |
6846 | rx_q->dma_rx_phy, rx_q->queue_index); |
6847 | |
6848 | rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * |
6849 | sizeof(struct dma_desc)); |
6850 | stmmac_set_rx_tail_ptr(priv, priv->ioaddr, |
6851 | rx_q->rx_tail_addr, rx_q->queue_index); |
6852 | |
6853 | if (rx_q->xsk_pool && rx_q->buf_alloc_num) { |
6854 | buf_size = xsk_pool_get_rx_frame_size(pool: rx_q->xsk_pool); |
6855 | stmmac_set_dma_bfsize(priv, priv->ioaddr, |
6856 | buf_size, |
6857 | rx_q->queue_index); |
6858 | } else { |
6859 | stmmac_set_dma_bfsize(priv, priv->ioaddr, |
6860 | priv->dma_conf.dma_buf_sz, |
6861 | rx_q->queue_index); |
6862 | } |
6863 | |
6864 | stmmac_start_rx_dma(priv, chan: queue); |
6865 | |
6866 | spin_lock_irqsave(&ch->lock, flags); |
6867 | stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); |
6868 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
6869 | } |
6870 | |
6871 | void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) |
6872 | { |
6873 | struct stmmac_channel *ch = &priv->channel[queue]; |
6874 | unsigned long flags; |
6875 | |
6876 | spin_lock_irqsave(&ch->lock, flags); |
6877 | stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); |
6878 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
6879 | |
6880 | stmmac_stop_tx_dma(priv, chan: queue); |
6881 | __free_dma_tx_desc_resources(priv, dma_conf: &priv->dma_conf, queue); |
6882 | } |
6883 | |
6884 | void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) |
6885 | { |
6886 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
6887 | struct stmmac_channel *ch = &priv->channel[queue]; |
6888 | unsigned long flags; |
6889 | int ret; |
6890 | |
6891 | ret = __alloc_dma_tx_desc_resources(priv, dma_conf: &priv->dma_conf, queue); |
6892 | if (ret) { |
6893 | netdev_err(dev: priv->dev, format: "Failed to alloc TX desc.\n"); |
6894 | return; |
6895 | } |
6896 | |
6897 | ret = __init_dma_tx_desc_rings(priv, dma_conf: &priv->dma_conf, queue); |
6898 | if (ret) { |
6899 | __free_dma_tx_desc_resources(priv, dma_conf: &priv->dma_conf, queue); |
6900 | netdev_err(dev: priv->dev, format: "Failed to init TX desc.\n"); |
6901 | return; |
6902 | } |
6903 | |
6904 | stmmac_reset_tx_queue(priv, queue); |
6905 | stmmac_clear_tx_descriptors(priv, dma_conf: &priv->dma_conf, queue); |
6906 | |
6907 | stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, |
6908 | tx_q->dma_tx_phy, tx_q->queue_index); |
6909 | |
6910 | if (tx_q->tbs & STMMAC_TBS_AVAIL) |
6911 | stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); |
6912 | |
6913 | tx_q->tx_tail_addr = tx_q->dma_tx_phy; |
6914 | stmmac_set_tx_tail_ptr(priv, priv->ioaddr, |
6915 | tx_q->tx_tail_addr, tx_q->queue_index); |
6916 | |
6917 | stmmac_start_tx_dma(priv, chan: queue); |
6918 | |
6919 | spin_lock_irqsave(&ch->lock, flags); |
6920 | stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); |
6921 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
6922 | } |
6923 | |
6924 | void stmmac_xdp_release(struct net_device *dev) |
6925 | { |
6926 | struct stmmac_priv *priv = netdev_priv(dev); |
6927 | u32 chan; |
6928 | |
6929 | /* Ensure tx function is not running */ |
6930 | netif_tx_disable(dev); |
6931 | |
6932 | /* Disable NAPI process */ |
6933 | stmmac_disable_all_queues(priv); |
6934 | |
6935 | for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) |
6936 | hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer); |
6937 | |
6938 | /* Free the IRQ lines */ |
6939 | stmmac_free_irq(dev, irq_err: REQ_IRQ_ERR_ALL, irq_idx: 0); |
6940 | |
6941 | /* Stop TX/RX DMA channels */ |
6942 | stmmac_stop_all_dma(priv); |
6943 | |
6944 | /* Release and free the Rx/Tx resources */ |
6945 | free_dma_desc_resources(priv, dma_conf: &priv->dma_conf); |
6946 | |
6947 | /* Disable the MAC Rx/Tx */ |
6948 | stmmac_mac_set(priv, priv->ioaddr, false); |
6949 | |
6950 | /* set trans_start so we don't get spurious |
6951 | * watchdogs during reset |
6952 | */ |
6953 | netif_trans_update(dev); |
6954 | netif_carrier_off(dev); |
6955 | } |
6956 | |
6957 | int stmmac_xdp_open(struct net_device *dev) |
6958 | { |
6959 | struct stmmac_priv *priv = netdev_priv(dev); |
6960 | u32 rx_cnt = priv->plat->rx_queues_to_use; |
6961 | u32 tx_cnt = priv->plat->tx_queues_to_use; |
6962 | u32 dma_csr_ch = max(rx_cnt, tx_cnt); |
6963 | struct stmmac_rx_queue *rx_q; |
6964 | struct stmmac_tx_queue *tx_q; |
6965 | u32 buf_size; |
6966 | bool sph_en; |
6967 | u32 chan; |
6968 | int ret; |
6969 | |
6970 | ret = alloc_dma_desc_resources(priv, dma_conf: &priv->dma_conf); |
6971 | if (ret < 0) { |
6972 | netdev_err(dev, format: "%s: DMA descriptors allocation failed\n", |
6973 | __func__); |
6974 | goto dma_desc_error; |
6975 | } |
6976 | |
6977 | ret = init_dma_desc_rings(dev, dma_conf: &priv->dma_conf, GFP_KERNEL); |
6978 | if (ret < 0) { |
6979 | netdev_err(dev, format: "%s: DMA descriptors initialization failed\n", |
6980 | __func__); |
6981 | goto init_error; |
6982 | } |
6983 | |
6984 | stmmac_reset_queues_param(priv); |
6985 | |
6986 | /* DMA CSR Channel configuration */ |
6987 | for (chan = 0; chan < dma_csr_ch; chan++) { |
6988 | stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); |
6989 | stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); |
6990 | } |
6991 | |
6992 | /* Adjust Split header */ |
6993 | sph_en = (priv->hw->rx_csum > 0) && priv->sph; |
6994 | |
6995 | /* DMA RX Channel Configuration */ |
6996 | for (chan = 0; chan < rx_cnt; chan++) { |
6997 | rx_q = &priv->dma_conf.rx_queue[chan]; |
6998 | |
6999 | stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, |
7000 | rx_q->dma_rx_phy, chan); |
7001 | |
7002 | rx_q->rx_tail_addr = rx_q->dma_rx_phy + |
7003 | (rx_q->buf_alloc_num * |
7004 | sizeof(struct dma_desc)); |
7005 | stmmac_set_rx_tail_ptr(priv, priv->ioaddr, |
7006 | rx_q->rx_tail_addr, chan); |
7007 | |
7008 | if (rx_q->xsk_pool && rx_q->buf_alloc_num) { |
7009 | buf_size = xsk_pool_get_rx_frame_size(pool: rx_q->xsk_pool); |
7010 | stmmac_set_dma_bfsize(priv, priv->ioaddr, |
7011 | buf_size, |
7012 | rx_q->queue_index); |
7013 | } else { |
7014 | stmmac_set_dma_bfsize(priv, priv->ioaddr, |
7015 | priv->dma_conf.dma_buf_sz, |
7016 | rx_q->queue_index); |
7017 | } |
7018 | |
7019 | stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); |
7020 | } |
7021 | |
7022 | /* DMA TX Channel Configuration */ |
7023 | for (chan = 0; chan < tx_cnt; chan++) { |
7024 | tx_q = &priv->dma_conf.tx_queue[chan]; |
7025 | |
7026 | stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, |
7027 | tx_q->dma_tx_phy, chan); |
7028 | |
7029 | tx_q->tx_tail_addr = tx_q->dma_tx_phy; |
7030 | stmmac_set_tx_tail_ptr(priv, priv->ioaddr, |
7031 | tx_q->tx_tail_addr, chan); |
7032 | |
7033 | hrtimer_setup(timer: &tx_q->txtimer, function: stmmac_tx_timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL); |
7034 | } |
7035 | |
7036 | /* Enable the MAC Rx/Tx */ |
7037 | stmmac_mac_set(priv, priv->ioaddr, true); |
7038 | |
7039 | /* Start Rx & Tx DMA Channels */ |
7040 | stmmac_start_all_dma(priv); |
7041 | |
7042 | ret = stmmac_request_irq(dev); |
7043 | if (ret) |
7044 | goto irq_error; |
7045 | |
7046 | /* Enable NAPI process*/ |
7047 | stmmac_enable_all_queues(priv); |
7048 | netif_carrier_on(dev); |
7049 | netif_tx_start_all_queues(dev); |
7050 | stmmac_enable_all_dma_irq(priv); |
7051 | |
7052 | return 0; |
7053 | |
7054 | irq_error: |
7055 | for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) |
7056 | hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer); |
7057 | |
7058 | stmmac_hw_teardown(dev); |
7059 | init_error: |
7060 | free_dma_desc_resources(priv, dma_conf: &priv->dma_conf); |
7061 | dma_desc_error: |
7062 | return ret; |
7063 | } |
7064 | |
7065 | int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) |
7066 | { |
7067 | struct stmmac_priv *priv = netdev_priv(dev); |
7068 | struct stmmac_rx_queue *rx_q; |
7069 | struct stmmac_tx_queue *tx_q; |
7070 | struct stmmac_channel *ch; |
7071 | |
7072 | if (test_bit(STMMAC_DOWN, &priv->state) || |
7073 | !netif_carrier_ok(dev: priv->dev)) |
7074 | return -ENETDOWN; |
7075 | |
7076 | if (!stmmac_xdp_is_enabled(priv)) |
7077 | return -EINVAL; |
7078 | |
7079 | if (queue >= priv->plat->rx_queues_to_use || |
7080 | queue >= priv->plat->tx_queues_to_use) |
7081 | return -EINVAL; |
7082 | |
7083 | rx_q = &priv->dma_conf.rx_queue[queue]; |
7084 | tx_q = &priv->dma_conf.tx_queue[queue]; |
7085 | ch = &priv->channel[queue]; |
7086 | |
7087 | if (!rx_q->xsk_pool && !tx_q->xsk_pool) |
7088 | return -EINVAL; |
7089 | |
7090 | if (!napi_if_scheduled_mark_missed(n: &ch->rxtx_napi)) { |
7091 | /* EQoS does not have per-DMA channel SW interrupt, |
7092 | * so we schedule RX Napi straight-away. |
7093 | */ |
7094 | if (likely(napi_schedule_prep(&ch->rxtx_napi))) |
7095 | __napi_schedule(n: &ch->rxtx_napi); |
7096 | } |
7097 | |
7098 | return 0; |
7099 | } |
7100 | |
7101 | static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) |
7102 | { |
7103 | struct stmmac_priv *priv = netdev_priv(dev); |
7104 | u32 tx_cnt = priv->plat->tx_queues_to_use; |
7105 | u32 rx_cnt = priv->plat->rx_queues_to_use; |
7106 | unsigned int start; |
7107 | int q; |
7108 | |
7109 | for (q = 0; q < tx_cnt; q++) { |
7110 | struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; |
7111 | u64 tx_packets; |
7112 | u64 tx_bytes; |
7113 | |
7114 | do { |
7115 | start = u64_stats_fetch_begin(syncp: &txq_stats->q_syncp); |
7116 | tx_bytes = u64_stats_read(p: &txq_stats->q.tx_bytes); |
7117 | } while (u64_stats_fetch_retry(syncp: &txq_stats->q_syncp, start)); |
7118 | do { |
7119 | start = u64_stats_fetch_begin(syncp: &txq_stats->napi_syncp); |
7120 | tx_packets = u64_stats_read(p: &txq_stats->napi.tx_packets); |
7121 | } while (u64_stats_fetch_retry(syncp: &txq_stats->napi_syncp, start)); |
7122 | |
7123 | stats->tx_packets += tx_packets; |
7124 | stats->tx_bytes += tx_bytes; |
7125 | } |
7126 | |
7127 | for (q = 0; q < rx_cnt; q++) { |
7128 | struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; |
7129 | u64 rx_packets; |
7130 | u64 rx_bytes; |
7131 | |
7132 | do { |
7133 | start = u64_stats_fetch_begin(syncp: &rxq_stats->napi_syncp); |
7134 | rx_packets = u64_stats_read(p: &rxq_stats->napi.rx_packets); |
7135 | rx_bytes = u64_stats_read(p: &rxq_stats->napi.rx_bytes); |
7136 | } while (u64_stats_fetch_retry(syncp: &rxq_stats->napi_syncp, start)); |
7137 | |
7138 | stats->rx_packets += rx_packets; |
7139 | stats->rx_bytes += rx_bytes; |
7140 | } |
7141 | |
7142 | stats->rx_dropped = priv->xstats.rx_dropped; |
7143 | stats->rx_errors = priv->xstats.rx_errors; |
7144 | stats->tx_dropped = priv->xstats.tx_dropped; |
7145 | stats->tx_errors = priv->xstats.tx_errors; |
7146 | stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; |
7147 | stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; |
7148 | stats->rx_length_errors = priv->xstats.rx_length; |
7149 | stats->rx_crc_errors = priv->xstats.rx_crc_errors; |
7150 | stats->rx_over_errors = priv->xstats.rx_overflow_cntr; |
7151 | stats->rx_missed_errors = priv->xstats.rx_missed_cntr; |
7152 | } |
7153 | |
7154 | static const struct net_device_ops stmmac_netdev_ops = { |
7155 | .ndo_open = stmmac_open, |
7156 | .ndo_start_xmit = stmmac_xmit, |
7157 | .ndo_stop = stmmac_release, |
7158 | .ndo_change_mtu = stmmac_change_mtu, |
7159 | .ndo_fix_features = stmmac_fix_features, |
7160 | .ndo_set_features = stmmac_set_features, |
7161 | .ndo_set_rx_mode = stmmac_set_rx_mode, |
7162 | .ndo_tx_timeout = stmmac_tx_timeout, |
7163 | .ndo_eth_ioctl = stmmac_ioctl, |
7164 | .ndo_get_stats64 = stmmac_get_stats64, |
7165 | .ndo_setup_tc = stmmac_setup_tc, |
7166 | .ndo_select_queue = stmmac_select_queue, |
7167 | .ndo_set_mac_address = stmmac_set_mac_address, |
7168 | .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, |
7169 | .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, |
7170 | .ndo_bpf = stmmac_bpf, |
7171 | .ndo_xdp_xmit = stmmac_xdp_xmit, |
7172 | .ndo_xsk_wakeup = stmmac_xsk_wakeup, |
7173 | .ndo_hwtstamp_get = stmmac_hwtstamp_get, |
7174 | .ndo_hwtstamp_set = stmmac_hwtstamp_set, |
7175 | }; |
7176 | |
7177 | static void stmmac_reset_subtask(struct stmmac_priv *priv) |
7178 | { |
7179 | if (!test_and_clear_bit(nr: STMMAC_RESET_REQUESTED, addr: &priv->state)) |
7180 | return; |
7181 | if (test_bit(STMMAC_DOWN, &priv->state)) |
7182 | return; |
7183 | |
7184 | netdev_err(dev: priv->dev, format: "Reset adapter.\n"); |
7185 | |
7186 | rtnl_lock(); |
7187 | netif_trans_update(dev: priv->dev); |
7188 | while (test_and_set_bit(nr: STMMAC_RESETING, addr: &priv->state)) |
7189 | usleep_range(min: 1000, max: 2000); |
7190 | |
7191 | set_bit(nr: STMMAC_DOWN, addr: &priv->state); |
7192 | dev_close(dev: priv->dev); |
7193 | dev_open(dev: priv->dev, NULL); |
7194 | clear_bit(nr: STMMAC_DOWN, addr: &priv->state); |
7195 | clear_bit(nr: STMMAC_RESETING, addr: &priv->state); |
7196 | rtnl_unlock(); |
7197 | } |
7198 | |
7199 | static void stmmac_service_task(struct work_struct *work) |
7200 | { |
7201 | struct stmmac_priv *priv = container_of(work, struct stmmac_priv, |
7202 | service_task); |
7203 | |
7204 | stmmac_reset_subtask(priv); |
7205 | clear_bit(nr: STMMAC_SERVICE_SCHED, addr: &priv->state); |
7206 | } |
7207 | |
7208 | /** |
7209 | * stmmac_hw_init - Init the MAC device |
7210 | * @priv: driver private structure |
7211 | * Description: this function is to configure the MAC device according to |
7212 | * some platform parameters or the HW capability register. It prepares the |
7213 | * driver to use either ring or chain modes and to setup either enhanced or |
7214 | * normal descriptors. |
7215 | */ |
7216 | static int stmmac_hw_init(struct stmmac_priv *priv) |
7217 | { |
7218 | int ret; |
7219 | |
7220 | /* dwmac-sun8i only work in chain mode */ |
7221 | if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) |
7222 | chain_mode = 1; |
7223 | priv->chain_mode = chain_mode; |
7224 | |
7225 | /* Initialize HW Interface */ |
7226 | ret = stmmac_hwif_init(priv); |
7227 | if (ret) |
7228 | return ret; |
7229 | |
7230 | /* Get the HW capability (new GMAC newer than 3.50a) */ |
7231 | priv->hw_cap_support = stmmac_get_hw_features(priv); |
7232 | if (priv->hw_cap_support) { |
7233 | dev_info(priv->device, "DMA HW capability register supported\n"); |
7234 | |
7235 | /* We can override some gmac/dma configuration fields: e.g. |
7236 | * enh_desc, tx_coe (e.g. that are passed through the |
7237 | * platform) with the values from the HW capability |
7238 | * register (if supported). |
7239 | */ |
7240 | priv->plat->enh_desc = priv->dma_cap.enh_desc; |
7241 | priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && |
7242 | !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); |
7243 | priv->hw->pmt = priv->plat->pmt; |
7244 | if (priv->dma_cap.hash_tb_sz) { |
7245 | priv->hw->multicast_filter_bins = |
7246 | (BIT(priv->dma_cap.hash_tb_sz) << 5); |
7247 | priv->hw->mcast_bits_log2 = |
7248 | ilog2(priv->hw->multicast_filter_bins); |
7249 | } |
7250 | |
7251 | /* TXCOE doesn't work in thresh DMA mode */ |
7252 | if (priv->plat->force_thresh_dma_mode) |
7253 | priv->plat->tx_coe = 0; |
7254 | else |
7255 | priv->plat->tx_coe = priv->dma_cap.tx_coe; |
7256 | |
7257 | /* In case of GMAC4 rx_coe is from HW cap register. */ |
7258 | priv->plat->rx_coe = priv->dma_cap.rx_coe; |
7259 | |
7260 | if (priv->dma_cap.rx_coe_type2) |
7261 | priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; |
7262 | else if (priv->dma_cap.rx_coe_type1) |
7263 | priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; |
7264 | |
7265 | } else { |
7266 | dev_info(priv->device, "No HW DMA feature register supported\n"); |
7267 | } |
7268 | |
7269 | if (priv->plat->rx_coe) { |
7270 | priv->hw->rx_csum = priv->plat->rx_coe; |
7271 | dev_info(priv->device, "RX Checksum Offload Engine supported\n"); |
7272 | if (priv->synopsys_id < DWMAC_CORE_4_00) |
7273 | dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); |
7274 | } |
7275 | if (priv->plat->tx_coe) |
7276 | dev_info(priv->device, "TX Checksum insertion supported\n"); |
7277 | |
7278 | if (priv->plat->pmt) { |
7279 | dev_info(priv->device, "Wake-Up On Lan supported\n"); |
7280 | device_set_wakeup_capable(dev: priv->device, capable: 1); |
7281 | } |
7282 | |
7283 | if (priv->dma_cap.tsoen) |
7284 | dev_info(priv->device, "TSO supported\n"); |
7285 | |
7286 | if (priv->dma_cap.number_rx_queues && |
7287 | priv->plat->rx_queues_to_use > priv->dma_cap.number_rx_queues) { |
7288 | dev_warn(priv->device, |
7289 | "Number of Rx queues (%u) exceeds dma capability\n", |
7290 | priv->plat->rx_queues_to_use); |
7291 | priv->plat->rx_queues_to_use = priv->dma_cap.number_rx_queues; |
7292 | } |
7293 | if (priv->dma_cap.number_tx_queues && |
7294 | priv->plat->tx_queues_to_use > priv->dma_cap.number_tx_queues) { |
7295 | dev_warn(priv->device, |
7296 | "Number of Tx queues (%u) exceeds dma capability\n", |
7297 | priv->plat->tx_queues_to_use); |
7298 | priv->plat->tx_queues_to_use = priv->dma_cap.number_tx_queues; |
7299 | } |
7300 | |
7301 | if (priv->dma_cap.rx_fifo_size && |
7302 | priv->plat->rx_fifo_size > priv->dma_cap.rx_fifo_size) { |
7303 | dev_warn(priv->device, |
7304 | "Rx FIFO size (%u) exceeds dma capability\n", |
7305 | priv->plat->rx_fifo_size); |
7306 | priv->plat->rx_fifo_size = priv->dma_cap.rx_fifo_size; |
7307 | } |
7308 | if (priv->dma_cap.tx_fifo_size && |
7309 | priv->plat->tx_fifo_size > priv->dma_cap.tx_fifo_size) { |
7310 | dev_warn(priv->device, |
7311 | "Tx FIFO size (%u) exceeds dma capability\n", |
7312 | priv->plat->tx_fifo_size); |
7313 | priv->plat->tx_fifo_size = priv->dma_cap.tx_fifo_size; |
7314 | } |
7315 | |
7316 | priv->hw->vlan_fail_q_en = |
7317 | (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); |
7318 | priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; |
7319 | |
7320 | /* Run HW quirks, if any */ |
7321 | if (priv->hwif_quirks) { |
7322 | ret = priv->hwif_quirks(priv); |
7323 | if (ret) |
7324 | return ret; |
7325 | } |
7326 | |
7327 | /* Rx Watchdog is available in the COREs newer than the 3.40. |
7328 | * In some case, for example on bugged HW this feature |
7329 | * has to be disable and this can be done by passing the |
7330 | * riwt_off field from the platform. |
7331 | */ |
7332 | if (((priv->synopsys_id >= DWMAC_CORE_3_50) || |
7333 | (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) { |
7334 | priv->use_riwt = 1; |
7335 | dev_info(priv->device, |
7336 | "Enable RX Mitigation via HW Watchdog Timer\n"); |
7337 | } |
7338 | |
7339 | return 0; |
7340 | } |
7341 | |
7342 | static void stmmac_napi_add(struct net_device *dev) |
7343 | { |
7344 | struct stmmac_priv *priv = netdev_priv(dev); |
7345 | u32 queue, maxq; |
7346 | |
7347 | maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); |
7348 | |
7349 | for (queue = 0; queue < maxq; queue++) { |
7350 | struct stmmac_channel *ch = &priv->channel[queue]; |
7351 | |
7352 | ch->priv_data = priv; |
7353 | ch->index = queue; |
7354 | spin_lock_init(&ch->lock); |
7355 | |
7356 | if (queue < priv->plat->rx_queues_to_use) { |
7357 | netif_napi_add(dev, napi: &ch->rx_napi, poll: stmmac_napi_poll_rx); |
7358 | } |
7359 | if (queue < priv->plat->tx_queues_to_use) { |
7360 | netif_napi_add_tx(dev, napi: &ch->tx_napi, |
7361 | poll: stmmac_napi_poll_tx); |
7362 | } |
7363 | if (queue < priv->plat->rx_queues_to_use && |
7364 | queue < priv->plat->tx_queues_to_use) { |
7365 | netif_napi_add(dev, napi: &ch->rxtx_napi, |
7366 | poll: stmmac_napi_poll_rxtx); |
7367 | } |
7368 | } |
7369 | } |
7370 | |
7371 | static void stmmac_napi_del(struct net_device *dev) |
7372 | { |
7373 | struct stmmac_priv *priv = netdev_priv(dev); |
7374 | u32 queue, maxq; |
7375 | |
7376 | maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); |
7377 | |
7378 | for (queue = 0; queue < maxq; queue++) { |
7379 | struct stmmac_channel *ch = &priv->channel[queue]; |
7380 | |
7381 | if (queue < priv->plat->rx_queues_to_use) |
7382 | netif_napi_del(napi: &ch->rx_napi); |
7383 | if (queue < priv->plat->tx_queues_to_use) |
7384 | netif_napi_del(napi: &ch->tx_napi); |
7385 | if (queue < priv->plat->rx_queues_to_use && |
7386 | queue < priv->plat->tx_queues_to_use) { |
7387 | netif_napi_del(napi: &ch->rxtx_napi); |
7388 | } |
7389 | } |
7390 | } |
7391 | |
7392 | int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) |
7393 | { |
7394 | struct stmmac_priv *priv = netdev_priv(dev); |
7395 | int ret = 0, i; |
7396 | |
7397 | if (netif_running(dev)) |
7398 | stmmac_release(dev); |
7399 | |
7400 | stmmac_napi_del(dev); |
7401 | |
7402 | priv->plat->rx_queues_to_use = rx_cnt; |
7403 | priv->plat->tx_queues_to_use = tx_cnt; |
7404 | if (!netif_is_rxfh_configured(dev)) |
7405 | for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) |
7406 | priv->rss.table[i] = ethtool_rxfh_indir_default(index: i, |
7407 | n_rx_rings: rx_cnt); |
7408 | |
7409 | stmmac_napi_add(dev); |
7410 | |
7411 | if (netif_running(dev)) |
7412 | ret = stmmac_open(dev); |
7413 | |
7414 | return ret; |
7415 | } |
7416 | |
7417 | int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) |
7418 | { |
7419 | struct stmmac_priv *priv = netdev_priv(dev); |
7420 | int ret = 0; |
7421 | |
7422 | if (netif_running(dev)) |
7423 | stmmac_release(dev); |
7424 | |
7425 | priv->dma_conf.dma_rx_size = rx_size; |
7426 | priv->dma_conf.dma_tx_size = tx_size; |
7427 | |
7428 | if (netif_running(dev)) |
7429 | ret = stmmac_open(dev); |
7430 | |
7431 | return ret; |
7432 | } |
7433 | |
7434 | static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) |
7435 | { |
7436 | const struct stmmac_xdp_buff *ctx = (void *)_ctx; |
7437 | struct dma_desc *desc_contains_ts = ctx->desc; |
7438 | struct stmmac_priv *priv = ctx->priv; |
7439 | struct dma_desc *ndesc = ctx->ndesc; |
7440 | struct dma_desc *desc = ctx->desc; |
7441 | u64 ns = 0; |
7442 | |
7443 | if (!priv->hwts_rx_en) |
7444 | return -ENODATA; |
7445 | |
7446 | /* For GMAC4, the valid timestamp is from CTX next desc. */ |
7447 | if (priv->plat->has_gmac4 || priv->plat->has_xgmac) |
7448 | desc_contains_ts = ndesc; |
7449 | |
7450 | /* Check if timestamp is available */ |
7451 | if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { |
7452 | stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); |
7453 | ns -= priv->plat->cdc_error_adj; |
7454 | *timestamp = ns_to_ktime(ns); |
7455 | return 0; |
7456 | } |
7457 | |
7458 | return -ENODATA; |
7459 | } |
7460 | |
7461 | static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { |
7462 | .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, |
7463 | }; |
7464 | |
7465 | /** |
7466 | * stmmac_dvr_probe |
7467 | * @device: device pointer |
7468 | * @plat_dat: platform data pointer |
7469 | * @res: stmmac resource pointer |
7470 | * Description: this is the main probe function used to |
7471 | * call the alloc_etherdev, allocate the priv structure. |
7472 | * Return: |
7473 | * returns 0 on success, otherwise errno. |
7474 | */ |
7475 | int stmmac_dvr_probe(struct device *device, |
7476 | struct plat_stmmacenet_data *plat_dat, |
7477 | struct stmmac_resources *res) |
7478 | { |
7479 | struct net_device *ndev = NULL; |
7480 | struct stmmac_priv *priv; |
7481 | u32 rxq; |
7482 | int i, ret = 0; |
7483 | |
7484 | ndev = devm_alloc_etherdev_mqs(dev: device, sizeof_priv: sizeof(struct stmmac_priv), |
7485 | MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); |
7486 | if (!ndev) |
7487 | return -ENOMEM; |
7488 | |
7489 | SET_NETDEV_DEV(ndev, device); |
7490 | |
7491 | priv = netdev_priv(dev: ndev); |
7492 | priv->device = device; |
7493 | priv->dev = ndev; |
7494 | |
7495 | for (i = 0; i < MTL_MAX_RX_QUEUES; i++) |
7496 | u64_stats_init(syncp: &priv->xstats.rxq_stats[i].napi_syncp); |
7497 | for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { |
7498 | u64_stats_init(syncp: &priv->xstats.txq_stats[i].q_syncp); |
7499 | u64_stats_init(syncp: &priv->xstats.txq_stats[i].napi_syncp); |
7500 | } |
7501 | |
7502 | priv->xstats.pcpu_stats = |
7503 | devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats); |
7504 | if (!priv->xstats.pcpu_stats) |
7505 | return -ENOMEM; |
7506 | |
7507 | stmmac_set_ethtool_ops(netdev: ndev); |
7508 | priv->pause_time = pause; |
7509 | priv->plat = plat_dat; |
7510 | priv->ioaddr = res->addr; |
7511 | priv->dev->base_addr = (unsigned long)res->addr; |
7512 | priv->plat->dma_cfg->multi_msi_en = |
7513 | (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); |
7514 | |
7515 | priv->dev->irq = res->irq; |
7516 | priv->wol_irq = res->wol_irq; |
7517 | priv->lpi_irq = res->lpi_irq; |
7518 | priv->sfty_irq = res->sfty_irq; |
7519 | priv->sfty_ce_irq = res->sfty_ce_irq; |
7520 | priv->sfty_ue_irq = res->sfty_ue_irq; |
7521 | for (i = 0; i < MTL_MAX_RX_QUEUES; i++) |
7522 | priv->rx_irq[i] = res->rx_irq[i]; |
7523 | for (i = 0; i < MTL_MAX_TX_QUEUES; i++) |
7524 | priv->tx_irq[i] = res->tx_irq[i]; |
7525 | |
7526 | if (!is_zero_ether_addr(addr: res->mac)) |
7527 | eth_hw_addr_set(dev: priv->dev, addr: res->mac); |
7528 | |
7529 | dev_set_drvdata(dev: device, data: priv->dev); |
7530 | |
7531 | /* Verify driver arguments */ |
7532 | stmmac_verify_args(); |
7533 | |
7534 | priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); |
7535 | if (!priv->af_xdp_zc_qps) |
7536 | return -ENOMEM; |
7537 | |
7538 | /* Allocate workqueue */ |
7539 | priv->wq = create_singlethread_workqueue("stmmac_wq"); |
7540 | if (!priv->wq) { |
7541 | dev_err(priv->device, "failed to create workqueue\n"); |
7542 | ret = -ENOMEM; |
7543 | goto error_wq_init; |
7544 | } |
7545 | |
7546 | INIT_WORK(&priv->service_task, stmmac_service_task); |
7547 | |
7548 | timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); |
7549 | |
7550 | /* Override with kernel parameters if supplied XXX CRS XXX |
7551 | * this needs to have multiple instances |
7552 | */ |
7553 | if ((phyaddr >= 0) && (phyaddr <= 31)) |
7554 | priv->plat->phy_addr = phyaddr; |
7555 | |
7556 | if (priv->plat->stmmac_rst) { |
7557 | ret = reset_control_assert(rstc: priv->plat->stmmac_rst); |
7558 | reset_control_deassert(rstc: priv->plat->stmmac_rst); |
7559 | /* Some reset controllers have only reset callback instead of |
7560 | * assert + deassert callbacks pair. |
7561 | */ |
7562 | if (ret == -ENOTSUPP) |
7563 | reset_control_reset(rstc: priv->plat->stmmac_rst); |
7564 | } |
7565 | |
7566 | ret = reset_control_deassert(rstc: priv->plat->stmmac_ahb_rst); |
7567 | if (ret == -ENOTSUPP) |
7568 | dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", |
7569 | ERR_PTR(ret)); |
7570 | |
7571 | /* Wait a bit for the reset to take effect */ |
7572 | udelay(usec: 10); |
7573 | |
7574 | /* Init MAC and get the capabilities */ |
7575 | ret = stmmac_hw_init(priv); |
7576 | if (ret) |
7577 | goto error_hw_init; |
7578 | |
7579 | /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. |
7580 | */ |
7581 | if (priv->synopsys_id < DWMAC_CORE_5_20) |
7582 | priv->plat->dma_cfg->dche = false; |
7583 | |
7584 | stmmac_check_ether_addr(priv); |
7585 | |
7586 | ndev->netdev_ops = &stmmac_netdev_ops; |
7587 | |
7588 | ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; |
7589 | ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops; |
7590 | |
7591 | ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | |
7592 | NETIF_F_RXCSUM; |
7593 | ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | |
7594 | NETDEV_XDP_ACT_XSK_ZEROCOPY; |
7595 | |
7596 | ret = stmmac_tc_init(priv, priv); |
7597 | if (!ret) { |
7598 | ndev->hw_features |= NETIF_F_HW_TC; |
7599 | } |
7600 | |
7601 | if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { |
7602 | ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; |
7603 | if (priv->plat->has_gmac4) |
7604 | ndev->hw_features |= NETIF_F_GSO_UDP_L4; |
7605 | priv->tso = true; |
7606 | dev_info(priv->device, "TSO feature enabled\n"); |
7607 | } |
7608 | |
7609 | if (priv->dma_cap.sphen && |
7610 | !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { |
7611 | ndev->hw_features |= NETIF_F_GRO; |
7612 | priv->sph_cap = true; |
7613 | priv->sph = priv->sph_cap; |
7614 | dev_info(priv->device, "SPH feature enabled\n"); |
7615 | } |
7616 | |
7617 | /* Ideally our host DMA address width is the same as for the |
7618 | * device. However, it may differ and then we have to use our |
7619 | * host DMA width for allocation and the device DMA width for |
7620 | * register handling. |
7621 | */ |
7622 | if (priv->plat->host_dma_width) |
7623 | priv->dma_cap.host_dma_width = priv->plat->host_dma_width; |
7624 | else |
7625 | priv->dma_cap.host_dma_width = priv->dma_cap.addr64; |
7626 | |
7627 | if (priv->dma_cap.host_dma_width) { |
7628 | ret = dma_set_mask_and_coherent(dev: device, |
7629 | DMA_BIT_MASK(priv->dma_cap.host_dma_width)); |
7630 | if (!ret) { |
7631 | dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", |
7632 | priv->dma_cap.host_dma_width, priv->dma_cap.addr64); |
7633 | |
7634 | /* |
7635 | * If more than 32 bits can be addressed, make sure to |
7636 | * enable enhanced addressing mode. |
7637 | */ |
7638 | if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) |
7639 | priv->plat->dma_cfg->eame = true; |
7640 | } else { |
7641 | ret = dma_set_mask_and_coherent(dev: device, DMA_BIT_MASK(32)); |
7642 | if (ret) { |
7643 | dev_err(priv->device, "Failed to set DMA Mask\n"); |
7644 | goto error_hw_init; |
7645 | } |
7646 | |
7647 | priv->dma_cap.host_dma_width = 32; |
7648 | } |
7649 | } |
7650 | |
7651 | ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; |
7652 | ndev->watchdog_timeo = msecs_to_jiffies(m: watchdog); |
7653 | #ifdef STMMAC_VLAN_TAG_USED |
7654 | /* Both mac100 and gmac support receive VLAN tag detection */ |
7655 | ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; |
7656 | if (priv->plat->has_gmac4 || priv->plat->has_xgmac) { |
7657 | ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; |
7658 | priv->hw->hw_vlan_en = true; |
7659 | } |
7660 | if (priv->dma_cap.vlhash) { |
7661 | ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; |
7662 | ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; |
7663 | } |
7664 | if (priv->dma_cap.vlins) { |
7665 | ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; |
7666 | if (priv->dma_cap.dvlan) |
7667 | ndev->features |= NETIF_F_HW_VLAN_STAG_TX; |
7668 | } |
7669 | #endif |
7670 | priv->msg_enable = netif_msg_init(debug_value: debug, default_msg_enable_bits: default_msg_level); |
7671 | |
7672 | priv->xstats.threshold = tc; |
7673 | |
7674 | /* Initialize RSS */ |
7675 | rxq = priv->plat->rx_queues_to_use; |
7676 | netdev_rss_key_fill(buffer: priv->rss.key, len: sizeof(priv->rss.key)); |
7677 | for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) |
7678 | priv->rss.table[i] = ethtool_rxfh_indir_default(index: i, n_rx_rings: rxq); |
7679 | |
7680 | if (priv->dma_cap.rssen && priv->plat->rss_en) |
7681 | ndev->features |= NETIF_F_RXHASH; |
7682 | |
7683 | ndev->vlan_features |= ndev->features; |
7684 | |
7685 | /* MTU range: 46 - hw-specific max */ |
7686 | ndev->min_mtu = ETH_ZLEN - ETH_HLEN; |
7687 | if (priv->plat->has_xgmac) |
7688 | ndev->max_mtu = XGMAC_JUMBO_LEN; |
7689 | else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00)) |
7690 | ndev->max_mtu = JUMBO_LEN; |
7691 | else |
7692 | ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); |
7693 | /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu |
7694 | * as well as plat->maxmtu < ndev->min_mtu which is a invalid range. |
7695 | */ |
7696 | if ((priv->plat->maxmtu < ndev->max_mtu) && |
7697 | (priv->plat->maxmtu >= ndev->min_mtu)) |
7698 | ndev->max_mtu = priv->plat->maxmtu; |
7699 | else if (priv->plat->maxmtu < ndev->min_mtu) |
7700 | dev_warn(priv->device, |
7701 | "%s: warning: maxmtu having invalid value (%d)\n", |
7702 | __func__, priv->plat->maxmtu); |
7703 | |
7704 | ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; |
7705 | |
7706 | /* Setup channels NAPI */ |
7707 | stmmac_napi_add(dev: ndev); |
7708 | |
7709 | mutex_init(&priv->lock); |
7710 | |
7711 | stmmac_fpe_init(priv); |
7712 | |
7713 | /* If a specific clk_csr value is passed from the platform |
7714 | * this means that the CSR Clock Range selection cannot be |
7715 | * changed at run-time and it is fixed. Viceversa the driver'll try to |
7716 | * set the MDC clock dynamically according to the csr actual |
7717 | * clock input. |
7718 | */ |
7719 | if (priv->plat->clk_csr >= 0) |
7720 | priv->clk_csr = priv->plat->clk_csr; |
7721 | else |
7722 | stmmac_clk_csr_set(priv); |
7723 | |
7724 | stmmac_check_pcs_mode(priv); |
7725 | |
7726 | pm_runtime_get_noresume(dev: device); |
7727 | pm_runtime_set_active(dev: device); |
7728 | if (!pm_runtime_enabled(dev: device)) |
7729 | pm_runtime_enable(dev: device); |
7730 | |
7731 | ret = stmmac_mdio_register(ndev); |
7732 | if (ret < 0) { |
7733 | dev_err_probe(dev: priv->device, err: ret, |
7734 | fmt: "MDIO bus (id: %d) registration failed\n", |
7735 | priv->plat->bus_id); |
7736 | goto error_mdio_register; |
7737 | } |
7738 | |
7739 | ret = stmmac_pcs_setup(ndev); |
7740 | if (ret) |
7741 | goto error_pcs_setup; |
7742 | |
7743 | ret = stmmac_phy_setup(priv); |
7744 | if (ret) { |
7745 | netdev_err(dev: ndev, format: "failed to setup phy (%d)\n", ret); |
7746 | goto error_phy_setup; |
7747 | } |
7748 | |
7749 | ret = register_netdev(dev: ndev); |
7750 | if (ret) { |
7751 | dev_err(priv->device, "%s: ERROR %i registering the device\n", |
7752 | __func__, ret); |
7753 | goto error_netdev_register; |
7754 | } |
7755 | |
7756 | #ifdef CONFIG_DEBUG_FS |
7757 | stmmac_init_fs(dev: ndev); |
7758 | #endif |
7759 | |
7760 | if (priv->plat->dump_debug_regs) |
7761 | priv->plat->dump_debug_regs(priv->plat->bsp_priv); |
7762 | |
7763 | /* Let pm_runtime_put() disable the clocks. |
7764 | * If CONFIG_PM is not enabled, the clocks will stay powered. |
7765 | */ |
7766 | pm_runtime_put(dev: device); |
7767 | |
7768 | return ret; |
7769 | |
7770 | error_netdev_register: |
7771 | phylink_destroy(priv->phylink); |
7772 | error_phy_setup: |
7773 | stmmac_pcs_clean(ndev); |
7774 | error_pcs_setup: |
7775 | stmmac_mdio_unregister(ndev); |
7776 | error_mdio_register: |
7777 | stmmac_napi_del(dev: ndev); |
7778 | error_hw_init: |
7779 | destroy_workqueue(wq: priv->wq); |
7780 | error_wq_init: |
7781 | bitmap_free(bitmap: priv->af_xdp_zc_qps); |
7782 | |
7783 | return ret; |
7784 | } |
7785 | EXPORT_SYMBOL_GPL(stmmac_dvr_probe); |
7786 | |
7787 | /** |
7788 | * stmmac_dvr_remove |
7789 | * @dev: device pointer |
7790 | * Description: this function resets the TX/RX processes, disables the MAC RX/TX |
7791 | * changes the link status, releases the DMA descriptor rings. |
7792 | */ |
7793 | void stmmac_dvr_remove(struct device *dev) |
7794 | { |
7795 | struct net_device *ndev = dev_get_drvdata(dev); |
7796 | struct stmmac_priv *priv = netdev_priv(dev: ndev); |
7797 | |
7798 | netdev_info(dev: priv->dev, format: "%s: removing driver", __func__); |
7799 | |
7800 | pm_runtime_get_sync(dev); |
7801 | |
7802 | unregister_netdev(dev: ndev); |
7803 | |
7804 | #ifdef CONFIG_DEBUG_FS |
7805 | stmmac_exit_fs(dev: ndev); |
7806 | #endif |
7807 | phylink_destroy(priv->phylink); |
7808 | if (priv->plat->stmmac_rst) |
7809 | reset_control_assert(rstc: priv->plat->stmmac_rst); |
7810 | reset_control_assert(rstc: priv->plat->stmmac_ahb_rst); |
7811 | |
7812 | stmmac_pcs_clean(ndev); |
7813 | stmmac_mdio_unregister(ndev); |
7814 | |
7815 | destroy_workqueue(wq: priv->wq); |
7816 | mutex_destroy(lock: &priv->lock); |
7817 | bitmap_free(bitmap: priv->af_xdp_zc_qps); |
7818 | |
7819 | pm_runtime_disable(dev); |
7820 | pm_runtime_put_noidle(dev); |
7821 | } |
7822 | EXPORT_SYMBOL_GPL(stmmac_dvr_remove); |
7823 | |
7824 | /** |
7825 | * stmmac_suspend - suspend callback |
7826 | * @dev: device pointer |
7827 | * Description: this is the function to suspend the device and it is called |
7828 | * by the platform driver to stop the network queue, release the resources, |
7829 | * program the PMT register (for WoL), clean and release driver resources. |
7830 | */ |
7831 | int stmmac_suspend(struct device *dev) |
7832 | { |
7833 | struct net_device *ndev = dev_get_drvdata(dev); |
7834 | struct stmmac_priv *priv = netdev_priv(dev: ndev); |
7835 | u32 chan; |
7836 | |
7837 | if (!ndev || !netif_running(dev: ndev)) |
7838 | return 0; |
7839 | |
7840 | mutex_lock(&priv->lock); |
7841 | |
7842 | netif_device_detach(dev: ndev); |
7843 | |
7844 | stmmac_disable_all_queues(priv); |
7845 | |
7846 | for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) |
7847 | hrtimer_cancel(timer: &priv->dma_conf.tx_queue[chan].txtimer); |
7848 | |
7849 | if (priv->eee_sw_timer_en) { |
7850 | priv->tx_path_in_lpi_mode = false; |
7851 | timer_delete_sync(timer: &priv->eee_ctrl_timer); |
7852 | } |
7853 | |
7854 | /* Stop TX/RX DMA */ |
7855 | stmmac_stop_all_dma(priv); |
7856 | |
7857 | if (priv->plat->serdes_powerdown) |
7858 | priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); |
7859 | |
7860 | /* Enable Power down mode by programming the PMT regs */ |
7861 | if (device_may_wakeup(dev: priv->device) && priv->plat->pmt) { |
7862 | stmmac_pmt(priv, priv->hw, priv->wolopts); |
7863 | priv->irq_wake = 1; |
7864 | } else { |
7865 | stmmac_mac_set(priv, priv->ioaddr, false); |
7866 | pinctrl_pm_select_sleep_state(dev: priv->device); |
7867 | } |
7868 | |
7869 | mutex_unlock(lock: &priv->lock); |
7870 | |
7871 | rtnl_lock(); |
7872 | if (device_may_wakeup(dev: priv->device) && !priv->plat->pmt) |
7873 | phylink_speed_down(pl: priv->phylink, sync: false); |
7874 | |
7875 | phylink_suspend(pl: priv->phylink, |
7876 | mac_wol: device_may_wakeup(dev: priv->device) && priv->plat->pmt); |
7877 | rtnl_unlock(); |
7878 | |
7879 | if (stmmac_fpe_supported(priv)) |
7880 | ethtool_mmsv_stop(mmsv: &priv->fpe_cfg.mmsv); |
7881 | |
7882 | return 0; |
7883 | } |
7884 | EXPORT_SYMBOL_GPL(stmmac_suspend); |
7885 | |
7886 | static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) |
7887 | { |
7888 | struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; |
7889 | |
7890 | rx_q->cur_rx = 0; |
7891 | rx_q->dirty_rx = 0; |
7892 | } |
7893 | |
7894 | static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) |
7895 | { |
7896 | struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; |
7897 | |
7898 | tx_q->cur_tx = 0; |
7899 | tx_q->dirty_tx = 0; |
7900 | tx_q->mss = 0; |
7901 | |
7902 | netdev_tx_reset_queue(q: netdev_get_tx_queue(dev: priv->dev, index: queue)); |
7903 | } |
7904 | |
7905 | /** |
7906 | * stmmac_reset_queues_param - reset queue parameters |
7907 | * @priv: device pointer |
7908 | */ |
7909 | static void stmmac_reset_queues_param(struct stmmac_priv *priv) |
7910 | { |
7911 | u32 rx_cnt = priv->plat->rx_queues_to_use; |
7912 | u32 tx_cnt = priv->plat->tx_queues_to_use; |
7913 | u32 queue; |
7914 | |
7915 | for (queue = 0; queue < rx_cnt; queue++) |
7916 | stmmac_reset_rx_queue(priv, queue); |
7917 | |
7918 | for (queue = 0; queue < tx_cnt; queue++) |
7919 | stmmac_reset_tx_queue(priv, queue); |
7920 | } |
7921 | |
7922 | /** |
7923 | * stmmac_resume - resume callback |
7924 | * @dev: device pointer |
7925 | * Description: when resume this function is invoked to setup the DMA and CORE |
7926 | * in a usable state. |
7927 | */ |
7928 | int stmmac_resume(struct device *dev) |
7929 | { |
7930 | struct net_device *ndev = dev_get_drvdata(dev); |
7931 | struct stmmac_priv *priv = netdev_priv(dev: ndev); |
7932 | int ret; |
7933 | |
7934 | if (!netif_running(dev: ndev)) |
7935 | return 0; |
7936 | |
7937 | /* Power Down bit, into the PM register, is cleared |
7938 | * automatically as soon as a magic packet or a Wake-up frame |
7939 | * is received. Anyway, it's better to manually clear |
7940 | * this bit because it can generate problems while resuming |
7941 | * from another devices (e.g. serial console). |
7942 | */ |
7943 | if (device_may_wakeup(dev: priv->device) && priv->plat->pmt) { |
7944 | mutex_lock(&priv->lock); |
7945 | stmmac_pmt(priv, priv->hw, 0); |
7946 | mutex_unlock(lock: &priv->lock); |
7947 | priv->irq_wake = 0; |
7948 | } else { |
7949 | pinctrl_pm_select_default_state(dev: priv->device); |
7950 | /* reset the phy so that it's ready */ |
7951 | if (priv->mii) |
7952 | stmmac_mdio_reset(mii: priv->mii); |
7953 | } |
7954 | |
7955 | if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && |
7956 | priv->plat->serdes_powerup) { |
7957 | ret = priv->plat->serdes_powerup(ndev, |
7958 | priv->plat->bsp_priv); |
7959 | |
7960 | if (ret < 0) |
7961 | return ret; |
7962 | } |
7963 | |
7964 | rtnl_lock(); |
7965 | |
7966 | /* Prepare the PHY to resume, ensuring that its clocks which are |
7967 | * necessary for the MAC DMA reset to complete are running |
7968 | */ |
7969 | phylink_prepare_resume(pl: priv->phylink); |
7970 | |
7971 | mutex_lock(&priv->lock); |
7972 | |
7973 | stmmac_reset_queues_param(priv); |
7974 | |
7975 | stmmac_free_tx_skbufs(priv); |
7976 | stmmac_clear_descriptors(priv, dma_conf: &priv->dma_conf); |
7977 | |
7978 | stmmac_hw_setup(dev: ndev, ptp_register: false); |
7979 | stmmac_init_coalesce(priv); |
7980 | phylink_rx_clk_stop_block(priv->phylink); |
7981 | stmmac_set_rx_mode(dev: ndev); |
7982 | |
7983 | stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw); |
7984 | phylink_rx_clk_stop_unblock(priv->phylink); |
7985 | |
7986 | stmmac_enable_all_queues(priv); |
7987 | stmmac_enable_all_dma_irq(priv); |
7988 | |
7989 | mutex_unlock(lock: &priv->lock); |
7990 | |
7991 | /* phylink_resume() must be called after the hardware has been |
7992 | * initialised because it may bring the link up immediately in a |
7993 | * workqueue thread, which will race with initialisation. |
7994 | */ |
7995 | phylink_resume(pl: priv->phylink); |
7996 | if (device_may_wakeup(dev: priv->device) && !priv->plat->pmt) |
7997 | phylink_speed_up(pl: priv->phylink); |
7998 | |
7999 | rtnl_unlock(); |
8000 | |
8001 | netif_device_attach(dev: ndev); |
8002 | |
8003 | return 0; |
8004 | } |
8005 | EXPORT_SYMBOL_GPL(stmmac_resume); |
8006 | |
8007 | #ifndef MODULE |
8008 | static int __init stmmac_cmdline_opt(char *str) |
8009 | { |
8010 | char *opt; |
8011 | |
8012 | if (!str || !*str) |
8013 | return 1; |
8014 | while ((opt = strsep(&str, ",")) != NULL) { |
8015 | if (!strncmp(opt, "debug:", 6)) { |
8016 | if (kstrtoint(s: opt + 6, base: 0, res: &debug)) |
8017 | goto err; |
8018 | } else if (!strncmp(opt, "phyaddr:", 8)) { |
8019 | if (kstrtoint(s: opt + 8, base: 0, res: &phyaddr)) |
8020 | goto err; |
8021 | } else if (!strncmp(opt, "tc:", 3)) { |
8022 | if (kstrtoint(s: opt + 3, base: 0, res: &tc)) |
8023 | goto err; |
8024 | } else if (!strncmp(opt, "watchdog:", 9)) { |
8025 | if (kstrtoint(s: opt + 9, base: 0, res: &watchdog)) |
8026 | goto err; |
8027 | } else if (!strncmp(opt, "flow_ctrl:", 10)) { |
8028 | if (kstrtoint(s: opt + 10, base: 0, res: &flow_ctrl)) |
8029 | goto err; |
8030 | } else if (!strncmp(opt, "pause:", 6)) { |
8031 | if (kstrtoint(s: opt + 6, base: 0, res: &pause)) |
8032 | goto err; |
8033 | } else if (!strncmp(opt, "eee_timer:", 10)) { |
8034 | if (kstrtoint(s: opt + 10, base: 0, res: &eee_timer)) |
8035 | goto err; |
8036 | } else if (!strncmp(opt, "chain_mode:", 11)) { |
8037 | if (kstrtoint(s: opt + 11, base: 0, res: &chain_mode)) |
8038 | goto err; |
8039 | } |
8040 | } |
8041 | return 1; |
8042 | |
8043 | err: |
8044 | pr_err("%s: ERROR broken module parameter conversion", __func__); |
8045 | return 1; |
8046 | } |
8047 | |
8048 | __setup("stmmaceth=", stmmac_cmdline_opt); |
8049 | #endif /* MODULE */ |
8050 | |
8051 | static int __init stmmac_init(void) |
8052 | { |
8053 | #ifdef CONFIG_DEBUG_FS |
8054 | /* Create debugfs main directory if it doesn't exist yet */ |
8055 | if (!stmmac_fs_dir) |
8056 | stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); |
8057 | register_netdevice_notifier(nb: &stmmac_notifier); |
8058 | #endif |
8059 | |
8060 | return 0; |
8061 | } |
8062 | |
8063 | static void __exit stmmac_exit(void) |
8064 | { |
8065 | #ifdef CONFIG_DEBUG_FS |
8066 | unregister_netdevice_notifier(nb: &stmmac_notifier); |
8067 | debugfs_remove_recursive(dentry: stmmac_fs_dir); |
8068 | #endif |
8069 | } |
8070 | |
8071 | module_init(stmmac_init) |
8072 | module_exit(stmmac_exit) |
8073 | |
8074 | MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); |
8075 | MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); |
8076 | MODULE_LICENSE("GPL"); |
8077 |
Definitions
- watchdog
- debug
- phyaddr
- flow_ctrl
- pause
- tc
- buf_sz
- default_msg_level
- eee_timer
- chain_mode
- stmmac_netdev_ops
- stmmac_bus_clks_config
- stmmac_set_clk_tx_rate
- stmmac_verify_args
- __stmmac_disable_all_queues
- stmmac_disable_all_queues
- stmmac_enable_all_queues
- stmmac_service_event_schedule
- stmmac_global_err
- stmmac_clk_csr_set
- print_pkt
- stmmac_tx_avail
- stmmac_rx_dirty
- stmmac_eee_tx_busy
- stmmac_restart_sw_lpi_timer
- stmmac_try_to_start_sw_lpi
- stmmac_stop_sw_lpi
- stmmac_eee_ctrl_timer
- stmmac_get_tx_hwtstamp
- stmmac_get_rx_hwtstamp
- stmmac_hwtstamp_set
- stmmac_hwtstamp_get
- stmmac_init_tstamp_counter
- stmmac_init_ptp
- stmmac_release_ptp
- stmmac_mac_flow_ctrl
- stmmac_mac_get_caps
- stmmac_mac_select_pcs
- stmmac_mac_config
- stmmac_mac_link_down
- stmmac_mac_link_up
- stmmac_mac_disable_tx_lpi
- stmmac_mac_enable_tx_lpi
- stmmac_mac_finish
- stmmac_phylink_mac_ops
- stmmac_check_pcs_mode
- stmmac_init_phy
- stmmac_phy_setup
- stmmac_display_rx_rings
- stmmac_display_tx_rings
- stmmac_display_rings
- stmmac_rx_offset
- stmmac_set_bfsize
- stmmac_clear_rx_descriptors
- stmmac_clear_tx_descriptors
- stmmac_clear_descriptors
- stmmac_init_rx_buffers
- stmmac_free_rx_buffer
- stmmac_free_tx_buffer
- dma_free_rx_skbufs
- stmmac_alloc_rx_buffers
- dma_free_rx_xskbufs
- stmmac_alloc_rx_buffers_zc
- stmmac_get_xsk_pool
- __init_dma_rx_desc_rings
- init_dma_rx_desc_rings
- __init_dma_tx_desc_rings
- init_dma_tx_desc_rings
- init_dma_desc_rings
- dma_free_tx_skbufs
- stmmac_free_tx_skbufs
- __free_dma_rx_desc_resources
- free_dma_rx_desc_resources
- __free_dma_tx_desc_resources
- free_dma_tx_desc_resources
- __alloc_dma_rx_desc_resources
- alloc_dma_rx_desc_resources
- __alloc_dma_tx_desc_resources
- alloc_dma_tx_desc_resources
- alloc_dma_desc_resources
- free_dma_desc_resources
- stmmac_mac_enable_rx_queues
- stmmac_start_rx_dma
- stmmac_start_tx_dma
- stmmac_stop_rx_dma
- stmmac_stop_tx_dma
- stmmac_enable_all_dma_irq
- stmmac_start_all_dma
- stmmac_stop_all_dma
- stmmac_dma_operation_mode
- stmmac_xsk_request_timestamp
- stmmac_xsk_fill_timestamp
- stmmac_xsk_request_launch_time
- stmmac_xsk_tx_metadata_ops
- stmmac_xdp_xmit_zc
- stmmac_bump_dma_threshold
- stmmac_tx_clean
- stmmac_tx_err
- stmmac_set_dma_operation_mode
- stmmac_safety_feat_interrupt
- stmmac_napi_check
- stmmac_dma_interrupt
- stmmac_mmc_setup
- stmmac_get_hw_features
- stmmac_check_ether_addr
- stmmac_init_dma_engine
- stmmac_tx_timer_arm
- stmmac_tx_timer
- stmmac_init_coalesce
- stmmac_set_rings_length
- stmmac_set_tx_queue_weight
- stmmac_configure_cbs
- stmmac_rx_queue_dma_chan_map
- stmmac_mac_config_rx_queues_prio
- stmmac_mac_config_tx_queues_prio
- stmmac_mac_config_rx_queues_routing
- stmmac_mac_config_rss
- stmmac_mtl_configuration
- stmmac_safety_feat_configuration
- stmmac_hw_setup
- stmmac_hw_teardown
- stmmac_free_irq
- stmmac_request_irq_multi_msi
- stmmac_request_irq_single
- stmmac_request_irq
- stmmac_setup_dma_desc
- __stmmac_open
- stmmac_open
- stmmac_release
- stmmac_vlan_insert
- stmmac_tso_allocator
- stmmac_flush_tx_descriptors
- stmmac_tso_xmit
- stmmac_has_ip_ethertype
- stmmac_xmit
- stmmac_rx_vlan
- stmmac_rx_refill
- stmmac_rx_buf1_len
- stmmac_rx_buf2_len
- stmmac_xdp_xmit_xdpf
- stmmac_xdp_get_tx_queue
- stmmac_xdp_xmit_back
- __stmmac_xdp_run_prog
- stmmac_xdp_run_prog
- stmmac_finalize_xdp_rx
- stmmac_construct_skb_zc
- stmmac_dispatch_skb_zc
- stmmac_rx_refill_zc
- xsk_buff_to_stmmac_ctx
- stmmac_rx_zc
- stmmac_rx
- stmmac_napi_poll_rx
- stmmac_napi_poll_tx
- stmmac_napi_poll_rxtx
- stmmac_tx_timeout
- stmmac_set_rx_mode
- stmmac_change_mtu
- stmmac_fix_features
- stmmac_set_features
- stmmac_common_interrupt
- stmmac_interrupt
- stmmac_mac_interrupt
- stmmac_safety_interrupt
- stmmac_msi_intr_tx
- stmmac_msi_intr_rx
- stmmac_ioctl
- stmmac_setup_tc_block_cb
- stmmac_block_cb_list
- stmmac_setup_tc
- stmmac_select_queue
- stmmac_set_mac_address
- stmmac_fs_dir
- sysfs_display_ring
- stmmac_rings_status_show
- stmmac_dma_cap_show
- stmmac_device_event
- stmmac_notifier
- stmmac_init_fs
- stmmac_exit_fs
- stmmac_vid_crc32_le
- stmmac_vlan_update
- stmmac_vlan_rx_add_vid
- stmmac_vlan_rx_kill_vid
- stmmac_bpf
- stmmac_xdp_xmit
- stmmac_disable_rx_queue
- stmmac_enable_rx_queue
- stmmac_disable_tx_queue
- stmmac_enable_tx_queue
- stmmac_xdp_release
- stmmac_xdp_open
- stmmac_xsk_wakeup
- stmmac_get_stats64
- stmmac_netdev_ops
- stmmac_reset_subtask
- stmmac_service_task
- stmmac_hw_init
- stmmac_napi_add
- stmmac_napi_del
- stmmac_reinit_queues
- stmmac_reinit_ringparam
- stmmac_xdp_rx_timestamp
- stmmac_xdp_metadata_ops
- stmmac_dvr_probe
- stmmac_dvr_remove
- stmmac_suspend
- stmmac_reset_rx_queue
- stmmac_reset_tx_queue
- stmmac_reset_queues_param
- stmmac_resume
- stmmac_cmdline_opt
- stmmac_init
Improve your Profiling and Debugging skills
Find out more