1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2018 Intel Corporation */ |
3 | |
4 | #include <linux/module.h> |
5 | #include <linux/types.h> |
6 | #include <linux/if_vlan.h> |
7 | #include <linux/tcp.h> |
8 | #include <linux/udp.h> |
9 | #include <linux/ip.h> |
10 | #include <linux/pm_runtime.h> |
11 | #include <net/pkt_sched.h> |
12 | #include <linux/bpf_trace.h> |
13 | #include <net/xdp_sock_drv.h> |
14 | #include <linux/pci.h> |
15 | |
16 | #include <net/ipv6.h> |
17 | |
18 | #include "igc.h" |
19 | #include "igc_hw.h" |
20 | #include "igc_tsn.h" |
21 | #include "igc_xdp.h" |
22 | |
23 | #define DRV_SUMMARY "Intel(R) 2.5G Ethernet Linux Driver" |
24 | |
25 | #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) |
26 | |
27 | #define IGC_XDP_PASS 0 |
28 | #define IGC_XDP_CONSUMED BIT(0) |
29 | #define IGC_XDP_TX BIT(1) |
30 | #define IGC_XDP_REDIRECT BIT(2) |
31 | |
32 | static int debug = -1; |
33 | |
34 | MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>" ); |
35 | MODULE_DESCRIPTION(DRV_SUMMARY); |
36 | MODULE_LICENSE("GPL v2" ); |
37 | module_param(debug, int, 0); |
38 | MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)" ); |
39 | |
40 | char igc_driver_name[] = "igc" ; |
41 | static const char igc_driver_string[] = DRV_SUMMARY; |
42 | static const char igc_copyright[] = |
43 | "Copyright(c) 2018 Intel Corporation." ; |
44 | |
45 | static const struct igc_info *igc_info_tbl[] = { |
46 | [board_base] = &igc_base_info, |
47 | }; |
48 | |
49 | static const struct pci_device_id igc_pci_tbl[] = { |
50 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base }, |
51 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base }, |
52 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, |
53 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, |
54 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, |
55 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base }, |
56 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base }, |
57 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base }, |
58 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), board_base }, |
59 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base }, |
60 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base }, |
61 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base }, |
62 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base }, |
63 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base }, |
64 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base }, |
65 | { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, |
66 | /* required last entry */ |
67 | {0, } |
68 | }; |
69 | |
70 | MODULE_DEVICE_TABLE(pci, igc_pci_tbl); |
71 | |
72 | enum latency_range { |
73 | lowest_latency = 0, |
74 | low_latency = 1, |
75 | bulk_latency = 2, |
76 | latency_invalid = 255 |
77 | }; |
78 | |
79 | void igc_reset(struct igc_adapter *adapter) |
80 | { |
81 | struct net_device *dev = adapter->netdev; |
82 | struct igc_hw *hw = &adapter->hw; |
83 | struct igc_fc_info *fc = &hw->fc; |
84 | u32 pba, hwm; |
85 | |
86 | /* Repartition PBA for greater than 9k MTU if required */ |
87 | pba = IGC_PBA_34K; |
88 | |
89 | /* flow control settings |
90 | * The high water mark must be low enough to fit one full frame |
91 | * after transmitting the pause frame. As such we must have enough |
92 | * space to allow for us to complete our current transmit and then |
93 | * receive the frame that is in progress from the link partner. |
94 | * Set it to: |
95 | * - the full Rx FIFO size minus one full Tx plus one full Rx frame |
96 | */ |
97 | hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); |
98 | |
99 | fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ |
100 | fc->low_water = fc->high_water - 16; |
101 | fc->pause_time = 0xFFFF; |
102 | fc->send_xon = 1; |
103 | fc->current_mode = fc->requested_mode; |
104 | |
105 | hw->mac.ops.reset_hw(hw); |
106 | |
107 | if (hw->mac.ops.init_hw(hw)) |
108 | netdev_err(dev, format: "Error on hardware initialization\n" ); |
109 | |
110 | /* Re-establish EEE setting */ |
111 | igc_set_eee_i225(hw, adv2p5G: true, adv1G: true, adv100M: true); |
112 | |
113 | if (!netif_running(dev: adapter->netdev)) |
114 | igc_power_down_phy_copper_base(hw: &adapter->hw); |
115 | |
116 | /* Enable HW to recognize an 802.1Q VLAN Ethernet packet */ |
117 | wr32(IGC_VET, ETH_P_8021Q); |
118 | |
119 | /* Re-enable PTP, where applicable. */ |
120 | igc_ptp_reset(adapter); |
121 | |
122 | /* Re-enable TSN offloading, where applicable. */ |
123 | igc_tsn_reset(adapter); |
124 | |
125 | igc_get_phy_info(hw); |
126 | } |
127 | |
128 | /** |
129 | * igc_power_up_link - Power up the phy link |
130 | * @adapter: address of board private structure |
131 | */ |
132 | static void igc_power_up_link(struct igc_adapter *adapter) |
133 | { |
134 | igc_reset_phy(hw: &adapter->hw); |
135 | |
136 | igc_power_up_phy_copper(hw: &adapter->hw); |
137 | |
138 | igc_setup_link(hw: &adapter->hw); |
139 | } |
140 | |
141 | /** |
142 | * igc_release_hw_control - release control of the h/w to f/w |
143 | * @adapter: address of board private structure |
144 | * |
145 | * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit. |
146 | * For ASF and Pass Through versions of f/w this means that the |
147 | * driver is no longer loaded. |
148 | */ |
149 | static void igc_release_hw_control(struct igc_adapter *adapter) |
150 | { |
151 | struct igc_hw *hw = &adapter->hw; |
152 | u32 ctrl_ext; |
153 | |
154 | if (!pci_device_is_present(pdev: adapter->pdev)) |
155 | return; |
156 | |
157 | /* Let firmware take over control of h/w */ |
158 | ctrl_ext = rd32(IGC_CTRL_EXT); |
159 | wr32(IGC_CTRL_EXT, |
160 | ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); |
161 | } |
162 | |
163 | /** |
164 | * igc_get_hw_control - get control of the h/w from f/w |
165 | * @adapter: address of board private structure |
166 | * |
167 | * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit. |
168 | * For ASF and Pass Through versions of f/w this means that |
169 | * the driver is loaded. |
170 | */ |
171 | static void igc_get_hw_control(struct igc_adapter *adapter) |
172 | { |
173 | struct igc_hw *hw = &adapter->hw; |
174 | u32 ctrl_ext; |
175 | |
176 | /* Let firmware know the driver has taken over */ |
177 | ctrl_ext = rd32(IGC_CTRL_EXT); |
178 | wr32(IGC_CTRL_EXT, |
179 | ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); |
180 | } |
181 | |
182 | static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf) |
183 | { |
184 | dma_unmap_single(dev, dma_unmap_addr(buf, dma), |
185 | dma_unmap_len(buf, len), DMA_TO_DEVICE); |
186 | |
187 | dma_unmap_len_set(buf, len, 0); |
188 | } |
189 | |
190 | /** |
191 | * igc_clean_tx_ring - Free Tx Buffers |
192 | * @tx_ring: ring to be cleaned |
193 | */ |
194 | static void igc_clean_tx_ring(struct igc_ring *tx_ring) |
195 | { |
196 | u16 i = tx_ring->next_to_clean; |
197 | struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; |
198 | u32 xsk_frames = 0; |
199 | |
200 | while (i != tx_ring->next_to_use) { |
201 | union igc_adv_tx_desc *eop_desc, *tx_desc; |
202 | |
203 | switch (tx_buffer->type) { |
204 | case IGC_TX_BUFFER_TYPE_XSK: |
205 | xsk_frames++; |
206 | break; |
207 | case IGC_TX_BUFFER_TYPE_XDP: |
208 | xdp_return_frame(xdpf: tx_buffer->xdpf); |
209 | igc_unmap_tx_buffer(dev: tx_ring->dev, buf: tx_buffer); |
210 | break; |
211 | case IGC_TX_BUFFER_TYPE_SKB: |
212 | dev_kfree_skb_any(skb: tx_buffer->skb); |
213 | igc_unmap_tx_buffer(dev: tx_ring->dev, buf: tx_buffer); |
214 | break; |
215 | default: |
216 | netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n" ); |
217 | break; |
218 | } |
219 | |
220 | /* check for eop_desc to determine the end of the packet */ |
221 | eop_desc = tx_buffer->next_to_watch; |
222 | tx_desc = IGC_TX_DESC(tx_ring, i); |
223 | |
224 | /* unmap remaining buffers */ |
225 | while (tx_desc != eop_desc) { |
226 | tx_buffer++; |
227 | tx_desc++; |
228 | i++; |
229 | if (unlikely(i == tx_ring->count)) { |
230 | i = 0; |
231 | tx_buffer = tx_ring->tx_buffer_info; |
232 | tx_desc = IGC_TX_DESC(tx_ring, 0); |
233 | } |
234 | |
235 | /* unmap any remaining paged data */ |
236 | if (dma_unmap_len(tx_buffer, len)) |
237 | igc_unmap_tx_buffer(dev: tx_ring->dev, buf: tx_buffer); |
238 | } |
239 | |
240 | tx_buffer->next_to_watch = NULL; |
241 | |
242 | /* move us one more past the eop_desc for start of next pkt */ |
243 | tx_buffer++; |
244 | i++; |
245 | if (unlikely(i == tx_ring->count)) { |
246 | i = 0; |
247 | tx_buffer = tx_ring->tx_buffer_info; |
248 | } |
249 | } |
250 | |
251 | if (tx_ring->xsk_pool && xsk_frames) |
252 | xsk_tx_completed(pool: tx_ring->xsk_pool, nb_entries: xsk_frames); |
253 | |
254 | /* reset BQL for queue */ |
255 | netdev_tx_reset_queue(q: txring_txq(tx_ring)); |
256 | |
257 | /* Zero out the buffer ring */ |
258 | memset(tx_ring->tx_buffer_info, 0, |
259 | sizeof(*tx_ring->tx_buffer_info) * tx_ring->count); |
260 | |
261 | /* Zero out the descriptor ring */ |
262 | memset(tx_ring->desc, 0, tx_ring->size); |
263 | |
264 | /* reset next_to_use and next_to_clean */ |
265 | tx_ring->next_to_use = 0; |
266 | tx_ring->next_to_clean = 0; |
267 | } |
268 | |
269 | /** |
270 | * igc_free_tx_resources - Free Tx Resources per Queue |
271 | * @tx_ring: Tx descriptor ring for a specific queue |
272 | * |
273 | * Free all transmit software resources |
274 | */ |
275 | void igc_free_tx_resources(struct igc_ring *tx_ring) |
276 | { |
277 | igc_disable_tx_ring(ring: tx_ring); |
278 | |
279 | vfree(addr: tx_ring->tx_buffer_info); |
280 | tx_ring->tx_buffer_info = NULL; |
281 | |
282 | /* if not set, then don't free */ |
283 | if (!tx_ring->desc) |
284 | return; |
285 | |
286 | dma_free_coherent(dev: tx_ring->dev, size: tx_ring->size, |
287 | cpu_addr: tx_ring->desc, dma_handle: tx_ring->dma); |
288 | |
289 | tx_ring->desc = NULL; |
290 | } |
291 | |
292 | /** |
293 | * igc_free_all_tx_resources - Free Tx Resources for All Queues |
294 | * @adapter: board private structure |
295 | * |
296 | * Free all transmit software resources |
297 | */ |
298 | static void igc_free_all_tx_resources(struct igc_adapter *adapter) |
299 | { |
300 | int i; |
301 | |
302 | for (i = 0; i < adapter->num_tx_queues; i++) |
303 | igc_free_tx_resources(tx_ring: adapter->tx_ring[i]); |
304 | } |
305 | |
306 | /** |
307 | * igc_clean_all_tx_rings - Free Tx Buffers for all queues |
308 | * @adapter: board private structure |
309 | */ |
310 | static void igc_clean_all_tx_rings(struct igc_adapter *adapter) |
311 | { |
312 | int i; |
313 | |
314 | for (i = 0; i < adapter->num_tx_queues; i++) |
315 | if (adapter->tx_ring[i]) |
316 | igc_clean_tx_ring(tx_ring: adapter->tx_ring[i]); |
317 | } |
318 | |
319 | static void igc_disable_tx_ring_hw(struct igc_ring *ring) |
320 | { |
321 | struct igc_hw *hw = &ring->q_vector->adapter->hw; |
322 | u8 idx = ring->reg_idx; |
323 | u32 txdctl; |
324 | |
325 | txdctl = rd32(IGC_TXDCTL(idx)); |
326 | txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE; |
327 | txdctl |= IGC_TXDCTL_SWFLUSH; |
328 | wr32(IGC_TXDCTL(idx), txdctl); |
329 | } |
330 | |
331 | /** |
332 | * igc_disable_all_tx_rings_hw - Disable all transmit queue operation |
333 | * @adapter: board private structure |
334 | */ |
335 | static void igc_disable_all_tx_rings_hw(struct igc_adapter *adapter) |
336 | { |
337 | int i; |
338 | |
339 | for (i = 0; i < adapter->num_tx_queues; i++) { |
340 | struct igc_ring *tx_ring = adapter->tx_ring[i]; |
341 | |
342 | igc_disable_tx_ring_hw(ring: tx_ring); |
343 | } |
344 | } |
345 | |
346 | /** |
347 | * igc_setup_tx_resources - allocate Tx resources (Descriptors) |
348 | * @tx_ring: tx descriptor ring (for a specific queue) to setup |
349 | * |
350 | * Return 0 on success, negative on failure |
351 | */ |
352 | int igc_setup_tx_resources(struct igc_ring *tx_ring) |
353 | { |
354 | struct net_device *ndev = tx_ring->netdev; |
355 | struct device *dev = tx_ring->dev; |
356 | int size = 0; |
357 | |
358 | size = sizeof(struct igc_tx_buffer) * tx_ring->count; |
359 | tx_ring->tx_buffer_info = vzalloc(size); |
360 | if (!tx_ring->tx_buffer_info) |
361 | goto err; |
362 | |
363 | /* round up to nearest 4K */ |
364 | tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc); |
365 | tx_ring->size = ALIGN(tx_ring->size, 4096); |
366 | |
367 | tx_ring->desc = dma_alloc_coherent(dev, size: tx_ring->size, |
368 | dma_handle: &tx_ring->dma, GFP_KERNEL); |
369 | |
370 | if (!tx_ring->desc) |
371 | goto err; |
372 | |
373 | tx_ring->next_to_use = 0; |
374 | tx_ring->next_to_clean = 0; |
375 | |
376 | return 0; |
377 | |
378 | err: |
379 | vfree(addr: tx_ring->tx_buffer_info); |
380 | netdev_err(dev: ndev, format: "Unable to allocate memory for Tx descriptor ring\n" ); |
381 | return -ENOMEM; |
382 | } |
383 | |
384 | /** |
385 | * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues |
386 | * @adapter: board private structure |
387 | * |
388 | * Return 0 on success, negative on failure |
389 | */ |
390 | static int igc_setup_all_tx_resources(struct igc_adapter *adapter) |
391 | { |
392 | struct net_device *dev = adapter->netdev; |
393 | int i, err = 0; |
394 | |
395 | for (i = 0; i < adapter->num_tx_queues; i++) { |
396 | err = igc_setup_tx_resources(tx_ring: adapter->tx_ring[i]); |
397 | if (err) { |
398 | netdev_err(dev, format: "Error on Tx queue %u setup\n" , i); |
399 | for (i--; i >= 0; i--) |
400 | igc_free_tx_resources(tx_ring: adapter->tx_ring[i]); |
401 | break; |
402 | } |
403 | } |
404 | |
405 | return err; |
406 | } |
407 | |
408 | static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring) |
409 | { |
410 | u16 i = rx_ring->next_to_clean; |
411 | |
412 | dev_kfree_skb(rx_ring->skb); |
413 | rx_ring->skb = NULL; |
414 | |
415 | /* Free all the Rx ring sk_buffs */ |
416 | while (i != rx_ring->next_to_alloc) { |
417 | struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; |
418 | |
419 | /* Invalidate cache lines that may have been written to by |
420 | * device so that we avoid corrupting memory. |
421 | */ |
422 | dma_sync_single_range_for_cpu(dev: rx_ring->dev, |
423 | addr: buffer_info->dma, |
424 | offset: buffer_info->page_offset, |
425 | size: igc_rx_bufsz(ring: rx_ring), |
426 | dir: DMA_FROM_DEVICE); |
427 | |
428 | /* free resources associated with mapping */ |
429 | dma_unmap_page_attrs(dev: rx_ring->dev, |
430 | addr: buffer_info->dma, |
431 | igc_rx_pg_size(rx_ring), |
432 | dir: DMA_FROM_DEVICE, |
433 | IGC_RX_DMA_ATTR); |
434 | __page_frag_cache_drain(page: buffer_info->page, |
435 | count: buffer_info->pagecnt_bias); |
436 | |
437 | i++; |
438 | if (i == rx_ring->count) |
439 | i = 0; |
440 | } |
441 | } |
442 | |
443 | static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring) |
444 | { |
445 | struct igc_rx_buffer *bi; |
446 | u16 i; |
447 | |
448 | for (i = 0; i < ring->count; i++) { |
449 | bi = &ring->rx_buffer_info[i]; |
450 | if (!bi->xdp) |
451 | continue; |
452 | |
453 | xsk_buff_free(xdp: bi->xdp); |
454 | bi->xdp = NULL; |
455 | } |
456 | } |
457 | |
458 | /** |
459 | * igc_clean_rx_ring - Free Rx Buffers per Queue |
460 | * @ring: ring to free buffers from |
461 | */ |
462 | static void igc_clean_rx_ring(struct igc_ring *ring) |
463 | { |
464 | if (ring->xsk_pool) |
465 | igc_clean_rx_ring_xsk_pool(ring); |
466 | else |
467 | igc_clean_rx_ring_page_shared(rx_ring: ring); |
468 | |
469 | clear_ring_uses_large_buffer(ring); |
470 | |
471 | ring->next_to_alloc = 0; |
472 | ring->next_to_clean = 0; |
473 | ring->next_to_use = 0; |
474 | } |
475 | |
476 | /** |
477 | * igc_clean_all_rx_rings - Free Rx Buffers for all queues |
478 | * @adapter: board private structure |
479 | */ |
480 | static void igc_clean_all_rx_rings(struct igc_adapter *adapter) |
481 | { |
482 | int i; |
483 | |
484 | for (i = 0; i < adapter->num_rx_queues; i++) |
485 | if (adapter->rx_ring[i]) |
486 | igc_clean_rx_ring(ring: adapter->rx_ring[i]); |
487 | } |
488 | |
489 | /** |
490 | * igc_free_rx_resources - Free Rx Resources |
491 | * @rx_ring: ring to clean the resources from |
492 | * |
493 | * Free all receive software resources |
494 | */ |
495 | void igc_free_rx_resources(struct igc_ring *rx_ring) |
496 | { |
497 | igc_clean_rx_ring(ring: rx_ring); |
498 | |
499 | xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq); |
500 | |
501 | vfree(addr: rx_ring->rx_buffer_info); |
502 | rx_ring->rx_buffer_info = NULL; |
503 | |
504 | /* if not set, then don't free */ |
505 | if (!rx_ring->desc) |
506 | return; |
507 | |
508 | dma_free_coherent(dev: rx_ring->dev, size: rx_ring->size, |
509 | cpu_addr: rx_ring->desc, dma_handle: rx_ring->dma); |
510 | |
511 | rx_ring->desc = NULL; |
512 | } |
513 | |
514 | /** |
515 | * igc_free_all_rx_resources - Free Rx Resources for All Queues |
516 | * @adapter: board private structure |
517 | * |
518 | * Free all receive software resources |
519 | */ |
520 | static void igc_free_all_rx_resources(struct igc_adapter *adapter) |
521 | { |
522 | int i; |
523 | |
524 | for (i = 0; i < adapter->num_rx_queues; i++) |
525 | igc_free_rx_resources(rx_ring: adapter->rx_ring[i]); |
526 | } |
527 | |
528 | /** |
529 | * igc_setup_rx_resources - allocate Rx resources (Descriptors) |
530 | * @rx_ring: rx descriptor ring (for a specific queue) to setup |
531 | * |
532 | * Returns 0 on success, negative on failure |
533 | */ |
534 | int igc_setup_rx_resources(struct igc_ring *rx_ring) |
535 | { |
536 | struct net_device *ndev = rx_ring->netdev; |
537 | struct device *dev = rx_ring->dev; |
538 | u8 index = rx_ring->queue_index; |
539 | int size, desc_len, res; |
540 | |
541 | /* XDP RX-queue info */ |
542 | if (xdp_rxq_info_is_reg(xdp_rxq: &rx_ring->xdp_rxq)) |
543 | xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq); |
544 | res = xdp_rxq_info_reg(xdp_rxq: &rx_ring->xdp_rxq, dev: ndev, queue_index: index, |
545 | napi_id: rx_ring->q_vector->napi.napi_id); |
546 | if (res < 0) { |
547 | netdev_err(dev: ndev, format: "Failed to register xdp_rxq index %u\n" , |
548 | index); |
549 | return res; |
550 | } |
551 | |
552 | size = sizeof(struct igc_rx_buffer) * rx_ring->count; |
553 | rx_ring->rx_buffer_info = vzalloc(size); |
554 | if (!rx_ring->rx_buffer_info) |
555 | goto err; |
556 | |
557 | desc_len = sizeof(union igc_adv_rx_desc); |
558 | |
559 | /* Round up to nearest 4K */ |
560 | rx_ring->size = rx_ring->count * desc_len; |
561 | rx_ring->size = ALIGN(rx_ring->size, 4096); |
562 | |
563 | rx_ring->desc = dma_alloc_coherent(dev, size: rx_ring->size, |
564 | dma_handle: &rx_ring->dma, GFP_KERNEL); |
565 | |
566 | if (!rx_ring->desc) |
567 | goto err; |
568 | |
569 | rx_ring->next_to_alloc = 0; |
570 | rx_ring->next_to_clean = 0; |
571 | rx_ring->next_to_use = 0; |
572 | |
573 | return 0; |
574 | |
575 | err: |
576 | xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq); |
577 | vfree(addr: rx_ring->rx_buffer_info); |
578 | rx_ring->rx_buffer_info = NULL; |
579 | netdev_err(dev: ndev, format: "Unable to allocate memory for Rx descriptor ring\n" ); |
580 | return -ENOMEM; |
581 | } |
582 | |
583 | /** |
584 | * igc_setup_all_rx_resources - wrapper to allocate Rx resources |
585 | * (Descriptors) for all queues |
586 | * @adapter: board private structure |
587 | * |
588 | * Return 0 on success, negative on failure |
589 | */ |
590 | static int igc_setup_all_rx_resources(struct igc_adapter *adapter) |
591 | { |
592 | struct net_device *dev = adapter->netdev; |
593 | int i, err = 0; |
594 | |
595 | for (i = 0; i < adapter->num_rx_queues; i++) { |
596 | err = igc_setup_rx_resources(rx_ring: adapter->rx_ring[i]); |
597 | if (err) { |
598 | netdev_err(dev, format: "Error on Rx queue %u setup\n" , i); |
599 | for (i--; i >= 0; i--) |
600 | igc_free_rx_resources(rx_ring: adapter->rx_ring[i]); |
601 | break; |
602 | } |
603 | } |
604 | |
605 | return err; |
606 | } |
607 | |
608 | static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter, |
609 | struct igc_ring *ring) |
610 | { |
611 | if (!igc_xdp_is_enabled(adapter) || |
612 | !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags)) |
613 | return NULL; |
614 | |
615 | return xsk_get_pool_from_qid(dev: ring->netdev, queue_id: ring->queue_index); |
616 | } |
617 | |
618 | /** |
619 | * igc_configure_rx_ring - Configure a receive ring after Reset |
620 | * @adapter: board private structure |
621 | * @ring: receive ring to be configured |
622 | * |
623 | * Configure the Rx unit of the MAC after a reset. |
624 | */ |
625 | static void igc_configure_rx_ring(struct igc_adapter *adapter, |
626 | struct igc_ring *ring) |
627 | { |
628 | struct igc_hw *hw = &adapter->hw; |
629 | union igc_adv_rx_desc *rx_desc; |
630 | int reg_idx = ring->reg_idx; |
631 | u32 srrctl = 0, rxdctl = 0; |
632 | u64 rdba = ring->dma; |
633 | u32 buf_size; |
634 | |
635 | xdp_rxq_info_unreg_mem_model(xdp_rxq: &ring->xdp_rxq); |
636 | ring->xsk_pool = igc_get_xsk_pool(adapter, ring); |
637 | if (ring->xsk_pool) { |
638 | WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, |
639 | MEM_TYPE_XSK_BUFF_POOL, |
640 | NULL)); |
641 | xsk_pool_set_rxq_info(pool: ring->xsk_pool, rxq: &ring->xdp_rxq); |
642 | } else { |
643 | WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, |
644 | MEM_TYPE_PAGE_SHARED, |
645 | NULL)); |
646 | } |
647 | |
648 | if (igc_xdp_is_enabled(adapter)) |
649 | set_ring_uses_large_buffer(ring); |
650 | |
651 | /* disable the queue */ |
652 | wr32(IGC_RXDCTL(reg_idx), 0); |
653 | |
654 | /* Set DMA base address registers */ |
655 | wr32(IGC_RDBAL(reg_idx), |
656 | rdba & 0x00000000ffffffffULL); |
657 | wr32(IGC_RDBAH(reg_idx), rdba >> 32); |
658 | wr32(IGC_RDLEN(reg_idx), |
659 | ring->count * sizeof(union igc_adv_rx_desc)); |
660 | |
661 | /* initialize head and tail */ |
662 | ring->tail = adapter->io_addr + IGC_RDT(reg_idx); |
663 | wr32(IGC_RDH(reg_idx), 0); |
664 | writel(val: 0, addr: ring->tail); |
665 | |
666 | /* reset next-to- use/clean to place SW in sync with hardware */ |
667 | ring->next_to_clean = 0; |
668 | ring->next_to_use = 0; |
669 | |
670 | if (ring->xsk_pool) |
671 | buf_size = xsk_pool_get_rx_frame_size(pool: ring->xsk_pool); |
672 | else if (ring_uses_large_buffer(ring)) |
673 | buf_size = IGC_RXBUFFER_3072; |
674 | else |
675 | buf_size = IGC_RXBUFFER_2048; |
676 | |
677 | srrctl = rd32(IGC_SRRCTL(reg_idx)); |
678 | srrctl &= ~(IGC_SRRCTL_BSIZEPKT_MASK | IGC_SRRCTL_BSIZEHDR_MASK | |
679 | IGC_SRRCTL_DESCTYPE_MASK); |
680 | srrctl |= IGC_SRRCTL_BSIZEHDR(IGC_RX_HDR_LEN); |
681 | srrctl |= IGC_SRRCTL_BSIZEPKT(buf_size); |
682 | srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; |
683 | |
684 | wr32(IGC_SRRCTL(reg_idx), srrctl); |
685 | |
686 | rxdctl |= IGC_RX_PTHRESH; |
687 | rxdctl |= IGC_RX_HTHRESH << 8; |
688 | rxdctl |= IGC_RX_WTHRESH << 16; |
689 | |
690 | /* initialize rx_buffer_info */ |
691 | memset(ring->rx_buffer_info, 0, |
692 | sizeof(struct igc_rx_buffer) * ring->count); |
693 | |
694 | /* initialize Rx descriptor 0 */ |
695 | rx_desc = IGC_RX_DESC(ring, 0); |
696 | rx_desc->wb.upper.length = 0; |
697 | |
698 | /* enable receive descriptor fetching */ |
699 | rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; |
700 | |
701 | wr32(IGC_RXDCTL(reg_idx), rxdctl); |
702 | } |
703 | |
704 | /** |
705 | * igc_configure_rx - Configure receive Unit after Reset |
706 | * @adapter: board private structure |
707 | * |
708 | * Configure the Rx unit of the MAC after a reset. |
709 | */ |
710 | static void igc_configure_rx(struct igc_adapter *adapter) |
711 | { |
712 | int i; |
713 | |
714 | /* Setup the HW Rx Head and Tail Descriptor Pointers and |
715 | * the Base and Length of the Rx Descriptor Ring |
716 | */ |
717 | for (i = 0; i < adapter->num_rx_queues; i++) |
718 | igc_configure_rx_ring(adapter, ring: adapter->rx_ring[i]); |
719 | } |
720 | |
721 | /** |
722 | * igc_configure_tx_ring - Configure transmit ring after Reset |
723 | * @adapter: board private structure |
724 | * @ring: tx ring to configure |
725 | * |
726 | * Configure a transmit ring after a reset. |
727 | */ |
728 | static void igc_configure_tx_ring(struct igc_adapter *adapter, |
729 | struct igc_ring *ring) |
730 | { |
731 | struct igc_hw *hw = &adapter->hw; |
732 | int reg_idx = ring->reg_idx; |
733 | u64 tdba = ring->dma; |
734 | u32 txdctl = 0; |
735 | |
736 | ring->xsk_pool = igc_get_xsk_pool(adapter, ring); |
737 | |
738 | /* disable the queue */ |
739 | wr32(IGC_TXDCTL(reg_idx), 0); |
740 | wrfl(); |
741 | |
742 | wr32(IGC_TDLEN(reg_idx), |
743 | ring->count * sizeof(union igc_adv_tx_desc)); |
744 | wr32(IGC_TDBAL(reg_idx), |
745 | tdba & 0x00000000ffffffffULL); |
746 | wr32(IGC_TDBAH(reg_idx), tdba >> 32); |
747 | |
748 | ring->tail = adapter->io_addr + IGC_TDT(reg_idx); |
749 | wr32(IGC_TDH(reg_idx), 0); |
750 | writel(val: 0, addr: ring->tail); |
751 | |
752 | txdctl |= IGC_TX_PTHRESH; |
753 | txdctl |= IGC_TX_HTHRESH << 8; |
754 | txdctl |= IGC_TX_WTHRESH << 16; |
755 | |
756 | txdctl |= IGC_TXDCTL_QUEUE_ENABLE; |
757 | wr32(IGC_TXDCTL(reg_idx), txdctl); |
758 | } |
759 | |
760 | /** |
761 | * igc_configure_tx - Configure transmit Unit after Reset |
762 | * @adapter: board private structure |
763 | * |
764 | * Configure the Tx unit of the MAC after a reset. |
765 | */ |
766 | static void igc_configure_tx(struct igc_adapter *adapter) |
767 | { |
768 | int i; |
769 | |
770 | for (i = 0; i < adapter->num_tx_queues; i++) |
771 | igc_configure_tx_ring(adapter, ring: adapter->tx_ring[i]); |
772 | } |
773 | |
774 | /** |
775 | * igc_setup_mrqc - configure the multiple receive queue control registers |
776 | * @adapter: Board private structure |
777 | */ |
778 | static void igc_setup_mrqc(struct igc_adapter *adapter) |
779 | { |
780 | struct igc_hw *hw = &adapter->hw; |
781 | u32 j, num_rx_queues; |
782 | u32 mrqc, rxcsum; |
783 | u32 [10]; |
784 | |
785 | netdev_rss_key_fill(buffer: rss_key, len: sizeof(rss_key)); |
786 | for (j = 0; j < 10; j++) |
787 | wr32(IGC_RSSRK(j), rss_key[j]); |
788 | |
789 | num_rx_queues = adapter->rss_queues; |
790 | |
791 | if (adapter->rss_indir_tbl_init != num_rx_queues) { |
792 | for (j = 0; j < IGC_RETA_SIZE; j++) |
793 | adapter->rss_indir_tbl[j] = |
794 | (j * num_rx_queues) / IGC_RETA_SIZE; |
795 | adapter->rss_indir_tbl_init = num_rx_queues; |
796 | } |
797 | igc_write_rss_indir_tbl(adapter); |
798 | |
799 | /* Disable raw packet checksumming so that RSS hash is placed in |
800 | * descriptor on writeback. No need to enable TCP/UDP/IP checksum |
801 | * offloads as they are enabled by default |
802 | */ |
803 | rxcsum = rd32(IGC_RXCSUM); |
804 | rxcsum |= IGC_RXCSUM_PCSD; |
805 | |
806 | /* Enable Receive Checksum Offload for SCTP */ |
807 | rxcsum |= IGC_RXCSUM_CRCOFL; |
808 | |
809 | /* Don't need to set TUOFL or IPOFL, they default to 1 */ |
810 | wr32(IGC_RXCSUM, rxcsum); |
811 | |
812 | /* Generate RSS hash based on packet types, TCP/UDP |
813 | * port numbers and/or IPv4/v6 src and dst addresses |
814 | */ |
815 | mrqc = IGC_MRQC_RSS_FIELD_IPV4 | |
816 | IGC_MRQC_RSS_FIELD_IPV4_TCP | |
817 | IGC_MRQC_RSS_FIELD_IPV6 | |
818 | IGC_MRQC_RSS_FIELD_IPV6_TCP | |
819 | IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; |
820 | |
821 | if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP) |
822 | mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; |
823 | if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP) |
824 | mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; |
825 | |
826 | mrqc |= IGC_MRQC_ENABLE_RSS_MQ; |
827 | |
828 | wr32(IGC_MRQC, mrqc); |
829 | } |
830 | |
831 | /** |
832 | * igc_setup_rctl - configure the receive control registers |
833 | * @adapter: Board private structure |
834 | */ |
835 | static void igc_setup_rctl(struct igc_adapter *adapter) |
836 | { |
837 | struct igc_hw *hw = &adapter->hw; |
838 | u32 rctl; |
839 | |
840 | rctl = rd32(IGC_RCTL); |
841 | |
842 | rctl &= ~(3 << IGC_RCTL_MO_SHIFT); |
843 | rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC); |
844 | |
845 | rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF | |
846 | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); |
847 | |
848 | /* enable stripping of CRC. Newer features require |
849 | * that the HW strips the CRC. |
850 | */ |
851 | rctl |= IGC_RCTL_SECRC; |
852 | |
853 | /* disable store bad packets and clear size bits. */ |
854 | rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256); |
855 | |
856 | /* enable LPE to allow for reception of jumbo frames */ |
857 | rctl |= IGC_RCTL_LPE; |
858 | |
859 | /* disable queue 0 to prevent tail write w/o re-config */ |
860 | wr32(IGC_RXDCTL(0), 0); |
861 | |
862 | /* This is useful for sniffing bad packets. */ |
863 | if (adapter->netdev->features & NETIF_F_RXALL) { |
864 | /* UPE and MPE will be handled by normal PROMISC logic |
865 | * in set_rx_mode |
866 | */ |
867 | rctl |= (IGC_RCTL_SBP | /* Receive bad packets */ |
868 | IGC_RCTL_BAM | /* RX All Bcast Pkts */ |
869 | IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ |
870 | |
871 | rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */ |
872 | IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */ |
873 | } |
874 | |
875 | wr32(IGC_RCTL, rctl); |
876 | } |
877 | |
878 | /** |
879 | * igc_setup_tctl - configure the transmit control registers |
880 | * @adapter: Board private structure |
881 | */ |
882 | static void igc_setup_tctl(struct igc_adapter *adapter) |
883 | { |
884 | struct igc_hw *hw = &adapter->hw; |
885 | u32 tctl; |
886 | |
887 | /* disable queue 0 which icould be enabled by default */ |
888 | wr32(IGC_TXDCTL(0), 0); |
889 | |
890 | /* Program the Transmit Control Register */ |
891 | tctl = rd32(IGC_TCTL); |
892 | tctl &= ~IGC_TCTL_CT; |
893 | tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC | |
894 | (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT); |
895 | |
896 | /* Enable transmits */ |
897 | tctl |= IGC_TCTL_EN; |
898 | |
899 | wr32(IGC_TCTL, tctl); |
900 | } |
901 | |
902 | /** |
903 | * igc_set_mac_filter_hw() - Set MAC address filter in hardware |
904 | * @adapter: Pointer to adapter where the filter should be set |
905 | * @index: Filter index |
906 | * @type: MAC address filter type (source or destination) |
907 | * @addr: MAC address |
908 | * @queue: If non-negative, queue assignment feature is enabled and frames |
909 | * matching the filter are enqueued onto 'queue'. Otherwise, queue |
910 | * assignment is disabled. |
911 | */ |
912 | static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index, |
913 | enum igc_mac_filter_type type, |
914 | const u8 *addr, int queue) |
915 | { |
916 | struct net_device *dev = adapter->netdev; |
917 | struct igc_hw *hw = &adapter->hw; |
918 | u32 ral, rah; |
919 | |
920 | if (WARN_ON(index >= hw->mac.rar_entry_count)) |
921 | return; |
922 | |
923 | ral = le32_to_cpup(p: (__le32 *)(addr)); |
924 | rah = le16_to_cpup(p: (__le16 *)(addr + 4)); |
925 | |
926 | if (type == IGC_MAC_FILTER_TYPE_SRC) { |
927 | rah &= ~IGC_RAH_ASEL_MASK; |
928 | rah |= IGC_RAH_ASEL_SRC_ADDR; |
929 | } |
930 | |
931 | if (queue >= 0) { |
932 | rah &= ~IGC_RAH_QSEL_MASK; |
933 | rah |= (queue << IGC_RAH_QSEL_SHIFT); |
934 | rah |= IGC_RAH_QSEL_ENABLE; |
935 | } |
936 | |
937 | rah |= IGC_RAH_AV; |
938 | |
939 | wr32(IGC_RAL(index), ral); |
940 | wr32(IGC_RAH(index), rah); |
941 | |
942 | netdev_dbg(dev, "MAC address filter set in HW: index %d" , index); |
943 | } |
944 | |
945 | /** |
946 | * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware |
947 | * @adapter: Pointer to adapter where the filter should be cleared |
948 | * @index: Filter index |
949 | */ |
950 | static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index) |
951 | { |
952 | struct net_device *dev = adapter->netdev; |
953 | struct igc_hw *hw = &adapter->hw; |
954 | |
955 | if (WARN_ON(index >= hw->mac.rar_entry_count)) |
956 | return; |
957 | |
958 | wr32(IGC_RAL(index), 0); |
959 | wr32(IGC_RAH(index), 0); |
960 | |
961 | netdev_dbg(dev, "MAC address filter cleared in HW: index %d" , index); |
962 | } |
963 | |
964 | /* Set default MAC address for the PF in the first RAR entry */ |
965 | static void igc_set_default_mac_filter(struct igc_adapter *adapter) |
966 | { |
967 | struct net_device *dev = adapter->netdev; |
968 | u8 *addr = adapter->hw.mac.addr; |
969 | |
970 | netdev_dbg(dev, "Set default MAC address filter: address %pM" , addr); |
971 | |
972 | igc_set_mac_filter_hw(adapter, index: 0, type: IGC_MAC_FILTER_TYPE_DST, addr, queue: -1); |
973 | } |
974 | |
975 | /** |
976 | * igc_set_mac - Change the Ethernet Address of the NIC |
977 | * @netdev: network interface device structure |
978 | * @p: pointer to an address structure |
979 | * |
980 | * Returns 0 on success, negative on failure |
981 | */ |
982 | static int igc_set_mac(struct net_device *netdev, void *p) |
983 | { |
984 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
985 | struct igc_hw *hw = &adapter->hw; |
986 | struct sockaddr *addr = p; |
987 | |
988 | if (!is_valid_ether_addr(addr: addr->sa_data)) |
989 | return -EADDRNOTAVAIL; |
990 | |
991 | eth_hw_addr_set(dev: netdev, addr: addr->sa_data); |
992 | memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); |
993 | |
994 | /* set the correct pool for the new PF MAC address in entry 0 */ |
995 | igc_set_default_mac_filter(adapter); |
996 | |
997 | return 0; |
998 | } |
999 | |
1000 | /** |
1001 | * igc_write_mc_addr_list - write multicast addresses to MTA |
1002 | * @netdev: network interface device structure |
1003 | * |
1004 | * Writes multicast address list to the MTA hash table. |
1005 | * Returns: -ENOMEM on failure |
1006 | * 0 on no addresses written |
1007 | * X on writing X addresses to MTA |
1008 | **/ |
1009 | static int igc_write_mc_addr_list(struct net_device *netdev) |
1010 | { |
1011 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
1012 | struct igc_hw *hw = &adapter->hw; |
1013 | struct netdev_hw_addr *ha; |
1014 | u8 *mta_list; |
1015 | int i; |
1016 | |
1017 | if (netdev_mc_empty(netdev)) { |
1018 | /* nothing to program, so clear mc list */ |
1019 | igc_update_mc_addr_list(hw, NULL, mc_addr_count: 0); |
1020 | return 0; |
1021 | } |
1022 | |
1023 | mta_list = kcalloc(netdev_mc_count(netdev), size: 6, GFP_ATOMIC); |
1024 | if (!mta_list) |
1025 | return -ENOMEM; |
1026 | |
1027 | /* The shared function expects a packed array of only addresses. */ |
1028 | i = 0; |
1029 | netdev_for_each_mc_addr(ha, netdev) |
1030 | memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); |
1031 | |
1032 | igc_update_mc_addr_list(hw, mc_addr_list: mta_list, mc_addr_count: i); |
1033 | kfree(objp: mta_list); |
1034 | |
1035 | return netdev_mc_count(netdev); |
1036 | } |
1037 | |
1038 | static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime, |
1039 | bool *first_flag, bool *insert_empty) |
1040 | { |
1041 | struct igc_adapter *adapter = netdev_priv(dev: ring->netdev); |
1042 | ktime_t cycle_time = adapter->cycle_time; |
1043 | ktime_t base_time = adapter->base_time; |
1044 | ktime_t now = ktime_get_clocktai(); |
1045 | ktime_t baset_est, end_of_cycle; |
1046 | s32 launchtime; |
1047 | s64 n; |
1048 | |
1049 | n = div64_s64(ktime_sub_ns(now, base_time), divisor: cycle_time); |
1050 | |
1051 | baset_est = ktime_add_ns(base_time, cycle_time * (n)); |
1052 | end_of_cycle = ktime_add_ns(baset_est, cycle_time); |
1053 | |
1054 | if (ktime_compare(cmp1: txtime, cmp2: end_of_cycle) >= 0) { |
1055 | if (baset_est != ring->last_ff_cycle) { |
1056 | *first_flag = true; |
1057 | ring->last_ff_cycle = baset_est; |
1058 | |
1059 | if (ktime_compare(cmp1: end_of_cycle, cmp2: ring->last_tx_cycle) > 0) |
1060 | *insert_empty = true; |
1061 | } |
1062 | } |
1063 | |
1064 | /* Introducing a window at end of cycle on which packets |
1065 | * potentially not honor launchtime. Window of 5us chosen |
1066 | * considering software update the tail pointer and packets |
1067 | * are dma'ed to packet buffer. |
1068 | */ |
1069 | if ((ktime_sub_ns(end_of_cycle, now) < 5 * NSEC_PER_USEC)) |
1070 | netdev_warn(dev: ring->netdev, format: "Packet with txtime=%llu may not be honoured\n" , |
1071 | txtime); |
1072 | |
1073 | ring->last_tx_cycle = end_of_cycle; |
1074 | |
1075 | launchtime = ktime_sub_ns(txtime, baset_est); |
1076 | if (launchtime > 0) |
1077 | div_s64_rem(dividend: launchtime, divisor: cycle_time, remainder: &launchtime); |
1078 | else |
1079 | launchtime = 0; |
1080 | |
1081 | return cpu_to_le32(launchtime); |
1082 | } |
1083 | |
1084 | static int igc_init_empty_frame(struct igc_ring *ring, |
1085 | struct igc_tx_buffer *buffer, |
1086 | struct sk_buff *skb) |
1087 | { |
1088 | unsigned int size; |
1089 | dma_addr_t dma; |
1090 | |
1091 | size = skb_headlen(skb); |
1092 | |
1093 | dma = dma_map_single(ring->dev, skb->data, size, DMA_TO_DEVICE); |
1094 | if (dma_mapping_error(dev: ring->dev, dma_addr: dma)) { |
1095 | netdev_err_once(ring->netdev, "Failed to map DMA for TX\n" ); |
1096 | return -ENOMEM; |
1097 | } |
1098 | |
1099 | buffer->skb = skb; |
1100 | buffer->protocol = 0; |
1101 | buffer->bytecount = skb->len; |
1102 | buffer->gso_segs = 1; |
1103 | buffer->time_stamp = jiffies; |
1104 | dma_unmap_len_set(buffer, len, skb->len); |
1105 | dma_unmap_addr_set(buffer, dma, dma); |
1106 | |
1107 | return 0; |
1108 | } |
1109 | |
1110 | static int igc_init_tx_empty_descriptor(struct igc_ring *ring, |
1111 | struct sk_buff *skb, |
1112 | struct igc_tx_buffer *first) |
1113 | { |
1114 | union igc_adv_tx_desc *desc; |
1115 | u32 cmd_type, olinfo_status; |
1116 | int err; |
1117 | |
1118 | if (!igc_desc_unused(ring)) |
1119 | return -EBUSY; |
1120 | |
1121 | err = igc_init_empty_frame(ring, buffer: first, skb); |
1122 | if (err) |
1123 | return err; |
1124 | |
1125 | cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | |
1126 | IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD | |
1127 | first->bytecount; |
1128 | olinfo_status = first->bytecount << IGC_ADVTXD_PAYLEN_SHIFT; |
1129 | |
1130 | desc = IGC_TX_DESC(ring, ring->next_to_use); |
1131 | desc->read.cmd_type_len = cpu_to_le32(cmd_type); |
1132 | desc->read.olinfo_status = cpu_to_le32(olinfo_status); |
1133 | desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(first, dma)); |
1134 | |
1135 | netdev_tx_sent_queue(dev_queue: txring_txq(tx_ring: ring), bytes: skb->len); |
1136 | |
1137 | first->next_to_watch = desc; |
1138 | |
1139 | ring->next_to_use++; |
1140 | if (ring->next_to_use == ring->count) |
1141 | ring->next_to_use = 0; |
1142 | |
1143 | return 0; |
1144 | } |
1145 | |
1146 | #define IGC_EMPTY_FRAME_SIZE 60 |
1147 | |
1148 | static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, |
1149 | __le32 launch_time, bool first_flag, |
1150 | u32 vlan_macip_lens, u32 type_tucmd, |
1151 | u32 mss_l4len_idx) |
1152 | { |
1153 | struct igc_adv_tx_context_desc *context_desc; |
1154 | u16 i = tx_ring->next_to_use; |
1155 | |
1156 | context_desc = IGC_TX_CTXTDESC(tx_ring, i); |
1157 | |
1158 | i++; |
1159 | tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; |
1160 | |
1161 | /* set bits to identify this as an advanced context descriptor */ |
1162 | type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; |
1163 | |
1164 | /* For i225, context index must be unique per ring. */ |
1165 | if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) |
1166 | mss_l4len_idx |= tx_ring->reg_idx << 4; |
1167 | |
1168 | if (first_flag) |
1169 | mss_l4len_idx |= IGC_ADVTXD_TSN_CNTX_FIRST; |
1170 | |
1171 | context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); |
1172 | context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); |
1173 | context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); |
1174 | context_desc->launch_time = launch_time; |
1175 | } |
1176 | |
1177 | static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first, |
1178 | __le32 launch_time, bool first_flag) |
1179 | { |
1180 | struct sk_buff *skb = first->skb; |
1181 | u32 vlan_macip_lens = 0; |
1182 | u32 type_tucmd = 0; |
1183 | |
1184 | if (skb->ip_summed != CHECKSUM_PARTIAL) { |
1185 | csum_failed: |
1186 | if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) && |
1187 | !tx_ring->launchtime_enable) |
1188 | return; |
1189 | goto no_csum; |
1190 | } |
1191 | |
1192 | switch (skb->csum_offset) { |
1193 | case offsetof(struct tcphdr, check): |
1194 | type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; |
1195 | fallthrough; |
1196 | case offsetof(struct udphdr, check): |
1197 | break; |
1198 | case offsetof(struct sctphdr, checksum): |
1199 | /* validate that this is actually an SCTP request */ |
1200 | if (skb_csum_is_sctp(skb)) { |
1201 | type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP; |
1202 | break; |
1203 | } |
1204 | fallthrough; |
1205 | default: |
1206 | skb_checksum_help(skb); |
1207 | goto csum_failed; |
1208 | } |
1209 | |
1210 | /* update TX checksum flag */ |
1211 | first->tx_flags |= IGC_TX_FLAGS_CSUM; |
1212 | vlan_macip_lens = skb_checksum_start_offset(skb) - |
1213 | skb_network_offset(skb); |
1214 | no_csum: |
1215 | vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT; |
1216 | vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; |
1217 | |
1218 | igc_tx_ctxtdesc(tx_ring, launch_time, first_flag, |
1219 | vlan_macip_lens, type_tucmd, mss_l4len_idx: 0); |
1220 | } |
1221 | |
1222 | static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) |
1223 | { |
1224 | struct net_device *netdev = tx_ring->netdev; |
1225 | |
1226 | netif_stop_subqueue(dev: netdev, queue_index: tx_ring->queue_index); |
1227 | |
1228 | /* memory barriier comment */ |
1229 | smp_mb(); |
1230 | |
1231 | /* We need to check again in a case another CPU has just |
1232 | * made room available. |
1233 | */ |
1234 | if (igc_desc_unused(ring: tx_ring) < size) |
1235 | return -EBUSY; |
1236 | |
1237 | /* A reprieve! */ |
1238 | netif_wake_subqueue(dev: netdev, queue_index: tx_ring->queue_index); |
1239 | |
1240 | u64_stats_update_begin(syncp: &tx_ring->tx_syncp2); |
1241 | tx_ring->tx_stats.restart_queue2++; |
1242 | u64_stats_update_end(syncp: &tx_ring->tx_syncp2); |
1243 | |
1244 | return 0; |
1245 | } |
1246 | |
1247 | static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) |
1248 | { |
1249 | if (igc_desc_unused(ring: tx_ring) >= size) |
1250 | return 0; |
1251 | return __igc_maybe_stop_tx(tx_ring, size); |
1252 | } |
1253 | |
1254 | #define IGC_SET_FLAG(_input, _flag, _result) \ |
1255 | (((_flag) <= (_result)) ? \ |
1256 | ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ |
1257 | ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) |
1258 | |
1259 | static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) |
1260 | { |
1261 | /* set type for advanced descriptor with frame checksum insertion */ |
1262 | u32 cmd_type = IGC_ADVTXD_DTYP_DATA | |
1263 | IGC_ADVTXD_DCMD_DEXT | |
1264 | IGC_ADVTXD_DCMD_IFCS; |
1265 | |
1266 | /* set HW vlan bit if vlan is present */ |
1267 | cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN, |
1268 | IGC_ADVTXD_DCMD_VLE); |
1269 | |
1270 | /* set segmentation bits for TSO */ |
1271 | cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, |
1272 | (IGC_ADVTXD_DCMD_TSE)); |
1273 | |
1274 | /* set timestamp bit if present, will select the register set |
1275 | * based on the _TSTAMP(_X) bit. |
1276 | */ |
1277 | cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, |
1278 | (IGC_ADVTXD_MAC_TSTAMP)); |
1279 | |
1280 | cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_1, |
1281 | (IGC_ADVTXD_TSTAMP_REG_1)); |
1282 | |
1283 | cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_2, |
1284 | (IGC_ADVTXD_TSTAMP_REG_2)); |
1285 | |
1286 | cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_3, |
1287 | (IGC_ADVTXD_TSTAMP_REG_3)); |
1288 | |
1289 | /* insert frame checksum */ |
1290 | cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS); |
1291 | |
1292 | return cmd_type; |
1293 | } |
1294 | |
1295 | static void igc_tx_olinfo_status(struct igc_ring *tx_ring, |
1296 | union igc_adv_tx_desc *tx_desc, |
1297 | u32 tx_flags, unsigned int paylen) |
1298 | { |
1299 | u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT; |
1300 | |
1301 | /* insert L4 checksum */ |
1302 | olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) * |
1303 | ((IGC_TXD_POPTS_TXSM << 8) / |
1304 | IGC_TX_FLAGS_CSUM); |
1305 | |
1306 | /* insert IPv4 checksum */ |
1307 | olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) * |
1308 | (((IGC_TXD_POPTS_IXSM << 8)) / |
1309 | IGC_TX_FLAGS_IPV4); |
1310 | |
1311 | tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); |
1312 | } |
1313 | |
1314 | static int igc_tx_map(struct igc_ring *tx_ring, |
1315 | struct igc_tx_buffer *first, |
1316 | const u8 hdr_len) |
1317 | { |
1318 | struct sk_buff *skb = first->skb; |
1319 | struct igc_tx_buffer *tx_buffer; |
1320 | union igc_adv_tx_desc *tx_desc; |
1321 | u32 tx_flags = first->tx_flags; |
1322 | skb_frag_t *frag; |
1323 | u16 i = tx_ring->next_to_use; |
1324 | unsigned int data_len, size; |
1325 | dma_addr_t dma; |
1326 | u32 cmd_type; |
1327 | |
1328 | cmd_type = igc_tx_cmd_type(skb, tx_flags); |
1329 | tx_desc = IGC_TX_DESC(tx_ring, i); |
1330 | |
1331 | igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen: skb->len - hdr_len); |
1332 | |
1333 | size = skb_headlen(skb); |
1334 | data_len = skb->data_len; |
1335 | |
1336 | dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); |
1337 | |
1338 | tx_buffer = first; |
1339 | |
1340 | for (frag = &skb_shinfo(skb)->frags[0];; frag++) { |
1341 | if (dma_mapping_error(dev: tx_ring->dev, dma_addr: dma)) |
1342 | goto dma_error; |
1343 | |
1344 | /* record length, and DMA address */ |
1345 | dma_unmap_len_set(tx_buffer, len, size); |
1346 | dma_unmap_addr_set(tx_buffer, dma, dma); |
1347 | |
1348 | tx_desc->read.buffer_addr = cpu_to_le64(dma); |
1349 | |
1350 | while (unlikely(size > IGC_MAX_DATA_PER_TXD)) { |
1351 | tx_desc->read.cmd_type_len = |
1352 | cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD); |
1353 | |
1354 | i++; |
1355 | tx_desc++; |
1356 | if (i == tx_ring->count) { |
1357 | tx_desc = IGC_TX_DESC(tx_ring, 0); |
1358 | i = 0; |
1359 | } |
1360 | tx_desc->read.olinfo_status = 0; |
1361 | |
1362 | dma += IGC_MAX_DATA_PER_TXD; |
1363 | size -= IGC_MAX_DATA_PER_TXD; |
1364 | |
1365 | tx_desc->read.buffer_addr = cpu_to_le64(dma); |
1366 | } |
1367 | |
1368 | if (likely(!data_len)) |
1369 | break; |
1370 | |
1371 | tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); |
1372 | |
1373 | i++; |
1374 | tx_desc++; |
1375 | if (i == tx_ring->count) { |
1376 | tx_desc = IGC_TX_DESC(tx_ring, 0); |
1377 | i = 0; |
1378 | } |
1379 | tx_desc->read.olinfo_status = 0; |
1380 | |
1381 | size = skb_frag_size(frag); |
1382 | data_len -= size; |
1383 | |
1384 | dma = skb_frag_dma_map(dev: tx_ring->dev, frag, offset: 0, |
1385 | size, dir: DMA_TO_DEVICE); |
1386 | |
1387 | tx_buffer = &tx_ring->tx_buffer_info[i]; |
1388 | } |
1389 | |
1390 | /* write last descriptor with RS and EOP bits */ |
1391 | cmd_type |= size | IGC_TXD_DCMD; |
1392 | tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); |
1393 | |
1394 | netdev_tx_sent_queue(dev_queue: txring_txq(tx_ring), bytes: first->bytecount); |
1395 | |
1396 | /* set the timestamp */ |
1397 | first->time_stamp = jiffies; |
1398 | |
1399 | skb_tx_timestamp(skb); |
1400 | |
1401 | /* Force memory writes to complete before letting h/w know there |
1402 | * are new descriptors to fetch. (Only applicable for weak-ordered |
1403 | * memory model archs, such as IA-64). |
1404 | * |
1405 | * We also need this memory barrier to make certain all of the |
1406 | * status bits have been updated before next_to_watch is written. |
1407 | */ |
1408 | wmb(); |
1409 | |
1410 | /* set next_to_watch value indicating a packet is present */ |
1411 | first->next_to_watch = tx_desc; |
1412 | |
1413 | i++; |
1414 | if (i == tx_ring->count) |
1415 | i = 0; |
1416 | |
1417 | tx_ring->next_to_use = i; |
1418 | |
1419 | /* Make sure there is space in the ring for the next send. */ |
1420 | igc_maybe_stop_tx(tx_ring, DESC_NEEDED); |
1421 | |
1422 | if (netif_xmit_stopped(dev_queue: txring_txq(tx_ring)) || !netdev_xmit_more()) { |
1423 | writel(val: i, addr: tx_ring->tail); |
1424 | } |
1425 | |
1426 | return 0; |
1427 | dma_error: |
1428 | netdev_err(dev: tx_ring->netdev, format: "TX DMA map failed\n" ); |
1429 | tx_buffer = &tx_ring->tx_buffer_info[i]; |
1430 | |
1431 | /* clear dma mappings for failed tx_buffer_info map */ |
1432 | while (tx_buffer != first) { |
1433 | if (dma_unmap_len(tx_buffer, len)) |
1434 | igc_unmap_tx_buffer(dev: tx_ring->dev, buf: tx_buffer); |
1435 | |
1436 | if (i-- == 0) |
1437 | i += tx_ring->count; |
1438 | tx_buffer = &tx_ring->tx_buffer_info[i]; |
1439 | } |
1440 | |
1441 | if (dma_unmap_len(tx_buffer, len)) |
1442 | igc_unmap_tx_buffer(dev: tx_ring->dev, buf: tx_buffer); |
1443 | |
1444 | dev_kfree_skb_any(skb: tx_buffer->skb); |
1445 | tx_buffer->skb = NULL; |
1446 | |
1447 | tx_ring->next_to_use = i; |
1448 | |
1449 | return -1; |
1450 | } |
1451 | |
1452 | static int igc_tso(struct igc_ring *tx_ring, |
1453 | struct igc_tx_buffer *first, |
1454 | __le32 launch_time, bool first_flag, |
1455 | u8 *hdr_len) |
1456 | { |
1457 | u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; |
1458 | struct sk_buff *skb = first->skb; |
1459 | union { |
1460 | struct iphdr *v4; |
1461 | struct ipv6hdr *v6; |
1462 | unsigned char *hdr; |
1463 | } ip; |
1464 | union { |
1465 | struct tcphdr *tcp; |
1466 | struct udphdr *udp; |
1467 | unsigned char *hdr; |
1468 | } l4; |
1469 | u32 paylen, l4_offset; |
1470 | int err; |
1471 | |
1472 | if (skb->ip_summed != CHECKSUM_PARTIAL) |
1473 | return 0; |
1474 | |
1475 | if (!skb_is_gso(skb)) |
1476 | return 0; |
1477 | |
1478 | err = skb_cow_head(skb, headroom: 0); |
1479 | if (err < 0) |
1480 | return err; |
1481 | |
1482 | ip.hdr = skb_network_header(skb); |
1483 | l4.hdr = skb_checksum_start(skb); |
1484 | |
1485 | /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ |
1486 | type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; |
1487 | |
1488 | /* initialize outer IP header fields */ |
1489 | if (ip.v4->version == 4) { |
1490 | unsigned char *csum_start = skb_checksum_start(skb); |
1491 | unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); |
1492 | |
1493 | /* IP header will have to cancel out any data that |
1494 | * is not a part of the outer IP header |
1495 | */ |
1496 | ip.v4->check = csum_fold(sum: csum_partial(buff: trans_start, |
1497 | len: csum_start - trans_start, |
1498 | sum: 0)); |
1499 | type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; |
1500 | |
1501 | ip.v4->tot_len = 0; |
1502 | first->tx_flags |= IGC_TX_FLAGS_TSO | |
1503 | IGC_TX_FLAGS_CSUM | |
1504 | IGC_TX_FLAGS_IPV4; |
1505 | } else { |
1506 | ip.v6->payload_len = 0; |
1507 | first->tx_flags |= IGC_TX_FLAGS_TSO | |
1508 | IGC_TX_FLAGS_CSUM; |
1509 | } |
1510 | |
1511 | /* determine offset of inner transport header */ |
1512 | l4_offset = l4.hdr - skb->data; |
1513 | |
1514 | /* remove payload length from inner checksum */ |
1515 | paylen = skb->len - l4_offset; |
1516 | if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { |
1517 | /* compute length of segmentation header */ |
1518 | *hdr_len = (l4.tcp->doff * 4) + l4_offset; |
1519 | csum_replace_by_diff(sum: &l4.tcp->check, |
1520 | diff: (__force __wsum)htonl(paylen)); |
1521 | } else { |
1522 | /* compute length of segmentation header */ |
1523 | *hdr_len = sizeof(*l4.udp) + l4_offset; |
1524 | csum_replace_by_diff(sum: &l4.udp->check, |
1525 | diff: (__force __wsum)htonl(paylen)); |
1526 | } |
1527 | |
1528 | /* update gso size and bytecount with header size */ |
1529 | first->gso_segs = skb_shinfo(skb)->gso_segs; |
1530 | first->bytecount += (first->gso_segs - 1) * *hdr_len; |
1531 | |
1532 | /* MSS L4LEN IDX */ |
1533 | mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; |
1534 | mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; |
1535 | |
1536 | /* VLAN MACLEN IPLEN */ |
1537 | vlan_macip_lens = l4.hdr - ip.hdr; |
1538 | vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; |
1539 | vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; |
1540 | |
1541 | igc_tx_ctxtdesc(tx_ring, launch_time, first_flag, |
1542 | vlan_macip_lens, type_tucmd, mss_l4len_idx); |
1543 | |
1544 | return 1; |
1545 | } |
1546 | |
1547 | static bool igc_request_tx_tstamp(struct igc_adapter *adapter, struct sk_buff *skb, u32 *flags) |
1548 | { |
1549 | int i; |
1550 | |
1551 | for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) { |
1552 | struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i]; |
1553 | |
1554 | if (tstamp->skb) |
1555 | continue; |
1556 | |
1557 | tstamp->skb = skb_get(skb); |
1558 | tstamp->start = jiffies; |
1559 | *flags = tstamp->flags; |
1560 | |
1561 | return true; |
1562 | } |
1563 | |
1564 | return false; |
1565 | } |
1566 | |
1567 | static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, |
1568 | struct igc_ring *tx_ring) |
1569 | { |
1570 | struct igc_adapter *adapter = netdev_priv(dev: tx_ring->netdev); |
1571 | bool first_flag = false, insert_empty = false; |
1572 | u16 count = TXD_USE_COUNT(skb_headlen(skb)); |
1573 | __be16 protocol = vlan_get_protocol(skb); |
1574 | struct igc_tx_buffer *first; |
1575 | __le32 launch_time = 0; |
1576 | u32 tx_flags = 0; |
1577 | unsigned short f; |
1578 | ktime_t txtime; |
1579 | u8 hdr_len = 0; |
1580 | int tso = 0; |
1581 | |
1582 | /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, |
1583 | * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, |
1584 | * + 2 desc gap to keep tail from touching head, |
1585 | * + 1 desc for context descriptor, |
1586 | * otherwise try next time |
1587 | */ |
1588 | for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) |
1589 | count += TXD_USE_COUNT(skb_frag_size( |
1590 | &skb_shinfo(skb)->frags[f])); |
1591 | |
1592 | if (igc_maybe_stop_tx(tx_ring, size: count + 5)) { |
1593 | /* this is a hard error */ |
1594 | return NETDEV_TX_BUSY; |
1595 | } |
1596 | |
1597 | if (!tx_ring->launchtime_enable) |
1598 | goto done; |
1599 | |
1600 | txtime = skb->tstamp; |
1601 | skb->tstamp = ktime_set(secs: 0, nsecs: 0); |
1602 | launch_time = igc_tx_launchtime(ring: tx_ring, txtime, first_flag: &first_flag, insert_empty: &insert_empty); |
1603 | |
1604 | if (insert_empty) { |
1605 | struct igc_tx_buffer *empty_info; |
1606 | struct sk_buff *empty; |
1607 | void *data; |
1608 | |
1609 | empty_info = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; |
1610 | empty = alloc_skb(IGC_EMPTY_FRAME_SIZE, GFP_ATOMIC); |
1611 | if (!empty) |
1612 | goto done; |
1613 | |
1614 | data = skb_put(skb: empty, IGC_EMPTY_FRAME_SIZE); |
1615 | memset(data, 0, IGC_EMPTY_FRAME_SIZE); |
1616 | |
1617 | igc_tx_ctxtdesc(tx_ring, launch_time: 0, first_flag: false, vlan_macip_lens: 0, type_tucmd: 0, mss_l4len_idx: 0); |
1618 | |
1619 | if (igc_init_tx_empty_descriptor(ring: tx_ring, |
1620 | skb: empty, |
1621 | first: empty_info) < 0) |
1622 | dev_kfree_skb_any(skb: empty); |
1623 | } |
1624 | |
1625 | done: |
1626 | /* record the location of the first descriptor for this packet */ |
1627 | first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; |
1628 | first->type = IGC_TX_BUFFER_TYPE_SKB; |
1629 | first->skb = skb; |
1630 | first->bytecount = skb->len; |
1631 | first->gso_segs = 1; |
1632 | |
1633 | if (adapter->qbv_transition || tx_ring->oper_gate_closed) |
1634 | goto out_drop; |
1635 | |
1636 | if (tx_ring->max_sdu > 0 && first->bytecount > tx_ring->max_sdu) { |
1637 | adapter->stats.txdrop++; |
1638 | goto out_drop; |
1639 | } |
1640 | |
1641 | if (unlikely(test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags) && |
1642 | skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { |
1643 | /* FIXME: add support for retrieving timestamps from |
1644 | * the other timer registers before skipping the |
1645 | * timestamping request. |
1646 | */ |
1647 | unsigned long flags; |
1648 | u32 tstamp_flags; |
1649 | |
1650 | spin_lock_irqsave(&adapter->ptp_tx_lock, flags); |
1651 | if (igc_request_tx_tstamp(adapter, skb, flags: &tstamp_flags)) { |
1652 | skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; |
1653 | tx_flags |= IGC_TX_FLAGS_TSTAMP | tstamp_flags; |
1654 | } else { |
1655 | adapter->tx_hwtstamp_skipped++; |
1656 | } |
1657 | |
1658 | spin_unlock_irqrestore(lock: &adapter->ptp_tx_lock, flags); |
1659 | } |
1660 | |
1661 | if (skb_vlan_tag_present(skb)) { |
1662 | tx_flags |= IGC_TX_FLAGS_VLAN; |
1663 | tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT); |
1664 | } |
1665 | |
1666 | /* record initial flags and protocol */ |
1667 | first->tx_flags = tx_flags; |
1668 | first->protocol = protocol; |
1669 | |
1670 | tso = igc_tso(tx_ring, first, launch_time, first_flag, hdr_len: &hdr_len); |
1671 | if (tso < 0) |
1672 | goto out_drop; |
1673 | else if (!tso) |
1674 | igc_tx_csum(tx_ring, first, launch_time, first_flag); |
1675 | |
1676 | igc_tx_map(tx_ring, first, hdr_len); |
1677 | |
1678 | return NETDEV_TX_OK; |
1679 | |
1680 | out_drop: |
1681 | dev_kfree_skb_any(skb: first->skb); |
1682 | first->skb = NULL; |
1683 | |
1684 | return NETDEV_TX_OK; |
1685 | } |
1686 | |
1687 | static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter, |
1688 | struct sk_buff *skb) |
1689 | { |
1690 | unsigned int r_idx = skb->queue_mapping; |
1691 | |
1692 | if (r_idx >= adapter->num_tx_queues) |
1693 | r_idx = r_idx % adapter->num_tx_queues; |
1694 | |
1695 | return adapter->tx_ring[r_idx]; |
1696 | } |
1697 | |
1698 | static netdev_tx_t igc_xmit_frame(struct sk_buff *skb, |
1699 | struct net_device *netdev) |
1700 | { |
1701 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
1702 | |
1703 | /* The minimum packet size with TCTL.PSP set is 17 so pad the skb |
1704 | * in order to meet this minimum size requirement. |
1705 | */ |
1706 | if (skb->len < 17) { |
1707 | if (skb_padto(skb, len: 17)) |
1708 | return NETDEV_TX_OK; |
1709 | skb->len = 17; |
1710 | } |
1711 | |
1712 | return igc_xmit_frame_ring(skb, tx_ring: igc_tx_queue_mapping(adapter, skb)); |
1713 | } |
1714 | |
1715 | static void igc_rx_checksum(struct igc_ring *ring, |
1716 | union igc_adv_rx_desc *rx_desc, |
1717 | struct sk_buff *skb) |
1718 | { |
1719 | skb_checksum_none_assert(skb); |
1720 | |
1721 | /* Ignore Checksum bit is set */ |
1722 | if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM)) |
1723 | return; |
1724 | |
1725 | /* Rx checksum disabled via ethtool */ |
1726 | if (!(ring->netdev->features & NETIF_F_RXCSUM)) |
1727 | return; |
1728 | |
1729 | /* TCP/UDP checksum error bit is set */ |
1730 | if (igc_test_staterr(rx_desc, |
1731 | IGC_RXDEXT_STATERR_L4E | |
1732 | IGC_RXDEXT_STATERR_IPE)) { |
1733 | /* work around errata with sctp packets where the TCPE aka |
1734 | * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) |
1735 | * packets (aka let the stack check the crc32c) |
1736 | */ |
1737 | if (!(skb->len == 60 && |
1738 | test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { |
1739 | u64_stats_update_begin(syncp: &ring->rx_syncp); |
1740 | ring->rx_stats.csum_err++; |
1741 | u64_stats_update_end(syncp: &ring->rx_syncp); |
1742 | } |
1743 | /* let the stack verify checksum errors */ |
1744 | return; |
1745 | } |
1746 | /* It must be a TCP or UDP packet with a valid checksum */ |
1747 | if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS | |
1748 | IGC_RXD_STAT_UDPCS)) |
1749 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
1750 | |
1751 | netdev_dbg(ring->netdev, "cksum success: bits %08X\n" , |
1752 | le32_to_cpu(rx_desc->wb.upper.status_error)); |
1753 | } |
1754 | |
1755 | /* Mapping HW RSS Type to enum pkt_hash_types */ |
1756 | static const enum pkt_hash_types [IGC_RSS_TYPE_MAX_TABLE] = { |
1757 | [IGC_RSS_TYPE_NO_HASH] = PKT_HASH_TYPE_L2, |
1758 | [IGC_RSS_TYPE_HASH_TCP_IPV4] = PKT_HASH_TYPE_L4, |
1759 | [IGC_RSS_TYPE_HASH_IPV4] = PKT_HASH_TYPE_L3, |
1760 | [IGC_RSS_TYPE_HASH_TCP_IPV6] = PKT_HASH_TYPE_L4, |
1761 | [IGC_RSS_TYPE_HASH_IPV6_EX] = PKT_HASH_TYPE_L3, |
1762 | [IGC_RSS_TYPE_HASH_IPV6] = PKT_HASH_TYPE_L3, |
1763 | [IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = PKT_HASH_TYPE_L4, |
1764 | [IGC_RSS_TYPE_HASH_UDP_IPV4] = PKT_HASH_TYPE_L4, |
1765 | [IGC_RSS_TYPE_HASH_UDP_IPV6] = PKT_HASH_TYPE_L4, |
1766 | [IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = PKT_HASH_TYPE_L4, |
1767 | [10] = PKT_HASH_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW */ |
1768 | [11] = PKT_HASH_TYPE_NONE, /* keep array sized for SW bit-mask */ |
1769 | [12] = PKT_HASH_TYPE_NONE, /* to handle future HW revisons */ |
1770 | [13] = PKT_HASH_TYPE_NONE, |
1771 | [14] = PKT_HASH_TYPE_NONE, |
1772 | [15] = PKT_HASH_TYPE_NONE, |
1773 | }; |
1774 | |
1775 | static inline void igc_rx_hash(struct igc_ring *ring, |
1776 | union igc_adv_rx_desc *rx_desc, |
1777 | struct sk_buff *skb) |
1778 | { |
1779 | if (ring->netdev->features & NETIF_F_RXHASH) { |
1780 | u32 = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss); |
1781 | u32 = igc_rss_type(rx_desc); |
1782 | |
1783 | skb_set_hash(skb, hash: rss_hash, type: igc_rss_type_table[rss_type]); |
1784 | } |
1785 | } |
1786 | |
1787 | static void igc_rx_vlan(struct igc_ring *rx_ring, |
1788 | union igc_adv_rx_desc *rx_desc, |
1789 | struct sk_buff *skb) |
1790 | { |
1791 | struct net_device *dev = rx_ring->netdev; |
1792 | u16 vid; |
1793 | |
1794 | if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && |
1795 | igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) { |
1796 | if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) && |
1797 | test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags)) |
1798 | vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan); |
1799 | else |
1800 | vid = le16_to_cpu(rx_desc->wb.upper.vlan); |
1801 | |
1802 | __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci: vid); |
1803 | } |
1804 | } |
1805 | |
1806 | /** |
1807 | * igc_process_skb_fields - Populate skb header fields from Rx descriptor |
1808 | * @rx_ring: rx descriptor ring packet is being transacted on |
1809 | * @rx_desc: pointer to the EOP Rx descriptor |
1810 | * @skb: pointer to current skb being populated |
1811 | * |
1812 | * This function checks the ring, descriptor, and packet information in order |
1813 | * to populate the hash, checksum, VLAN, protocol, and other fields within the |
1814 | * skb. |
1815 | */ |
1816 | static void igc_process_skb_fields(struct igc_ring *rx_ring, |
1817 | union igc_adv_rx_desc *rx_desc, |
1818 | struct sk_buff *skb) |
1819 | { |
1820 | igc_rx_hash(ring: rx_ring, rx_desc, skb); |
1821 | |
1822 | igc_rx_checksum(ring: rx_ring, rx_desc, skb); |
1823 | |
1824 | igc_rx_vlan(rx_ring, rx_desc, skb); |
1825 | |
1826 | skb_record_rx_queue(skb, rx_queue: rx_ring->queue_index); |
1827 | |
1828 | skb->protocol = eth_type_trans(skb, dev: rx_ring->netdev); |
1829 | } |
1830 | |
1831 | static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features) |
1832 | { |
1833 | bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX); |
1834 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
1835 | struct igc_hw *hw = &adapter->hw; |
1836 | u32 ctrl; |
1837 | |
1838 | ctrl = rd32(IGC_CTRL); |
1839 | |
1840 | if (enable) { |
1841 | /* enable VLAN tag insert/strip */ |
1842 | ctrl |= IGC_CTRL_VME; |
1843 | } else { |
1844 | /* disable VLAN tag insert/strip */ |
1845 | ctrl &= ~IGC_CTRL_VME; |
1846 | } |
1847 | wr32(IGC_CTRL, ctrl); |
1848 | } |
1849 | |
1850 | static void igc_restore_vlan(struct igc_adapter *adapter) |
1851 | { |
1852 | igc_vlan_mode(netdev: adapter->netdev, features: adapter->netdev->features); |
1853 | } |
1854 | |
1855 | static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring, |
1856 | const unsigned int size, |
1857 | int *rx_buffer_pgcnt) |
1858 | { |
1859 | struct igc_rx_buffer *rx_buffer; |
1860 | |
1861 | rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; |
1862 | *rx_buffer_pgcnt = |
1863 | #if (PAGE_SIZE < 8192) |
1864 | page_count(page: rx_buffer->page); |
1865 | #else |
1866 | 0; |
1867 | #endif |
1868 | prefetchw(x: rx_buffer->page); |
1869 | |
1870 | /* we are reusing so sync this buffer for CPU use */ |
1871 | dma_sync_single_range_for_cpu(dev: rx_ring->dev, |
1872 | addr: rx_buffer->dma, |
1873 | offset: rx_buffer->page_offset, |
1874 | size, |
1875 | dir: DMA_FROM_DEVICE); |
1876 | |
1877 | rx_buffer->pagecnt_bias--; |
1878 | |
1879 | return rx_buffer; |
1880 | } |
1881 | |
1882 | static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer, |
1883 | unsigned int truesize) |
1884 | { |
1885 | #if (PAGE_SIZE < 8192) |
1886 | buffer->page_offset ^= truesize; |
1887 | #else |
1888 | buffer->page_offset += truesize; |
1889 | #endif |
1890 | } |
1891 | |
1892 | static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring, |
1893 | unsigned int size) |
1894 | { |
1895 | unsigned int truesize; |
1896 | |
1897 | #if (PAGE_SIZE < 8192) |
1898 | truesize = igc_rx_pg_size(ring) / 2; |
1899 | #else |
1900 | truesize = ring_uses_build_skb(ring) ? |
1901 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + |
1902 | SKB_DATA_ALIGN(IGC_SKB_PAD + size) : |
1903 | SKB_DATA_ALIGN(size); |
1904 | #endif |
1905 | return truesize; |
1906 | } |
1907 | |
1908 | /** |
1909 | * igc_add_rx_frag - Add contents of Rx buffer to sk_buff |
1910 | * @rx_ring: rx descriptor ring to transact packets on |
1911 | * @rx_buffer: buffer containing page to add |
1912 | * @skb: sk_buff to place the data into |
1913 | * @size: size of buffer to be added |
1914 | * |
1915 | * This function will add the data contained in rx_buffer->page to the skb. |
1916 | */ |
1917 | static void igc_add_rx_frag(struct igc_ring *rx_ring, |
1918 | struct igc_rx_buffer *rx_buffer, |
1919 | struct sk_buff *skb, |
1920 | unsigned int size) |
1921 | { |
1922 | unsigned int truesize; |
1923 | |
1924 | #if (PAGE_SIZE < 8192) |
1925 | truesize = igc_rx_pg_size(rx_ring) / 2; |
1926 | #else |
1927 | truesize = ring_uses_build_skb(rx_ring) ? |
1928 | SKB_DATA_ALIGN(IGC_SKB_PAD + size) : |
1929 | SKB_DATA_ALIGN(size); |
1930 | #endif |
1931 | skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page: rx_buffer->page, |
1932 | off: rx_buffer->page_offset, size, truesize); |
1933 | |
1934 | igc_rx_buffer_flip(buffer: rx_buffer, truesize); |
1935 | } |
1936 | |
1937 | static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, |
1938 | struct igc_rx_buffer *rx_buffer, |
1939 | struct xdp_buff *xdp) |
1940 | { |
1941 | unsigned int size = xdp->data_end - xdp->data; |
1942 | unsigned int truesize = igc_get_rx_frame_truesize(ring: rx_ring, size); |
1943 | unsigned int metasize = xdp->data - xdp->data_meta; |
1944 | struct sk_buff *skb; |
1945 | |
1946 | /* prefetch first cache line of first page */ |
1947 | net_prefetch(p: xdp->data_meta); |
1948 | |
1949 | /* build an skb around the page buffer */ |
1950 | skb = napi_build_skb(data: xdp->data_hard_start, frag_size: truesize); |
1951 | if (unlikely(!skb)) |
1952 | return NULL; |
1953 | |
1954 | /* update pointers within the skb to store the data */ |
1955 | skb_reserve(skb, len: xdp->data - xdp->data_hard_start); |
1956 | __skb_put(skb, len: size); |
1957 | if (metasize) |
1958 | skb_metadata_set(skb, meta_len: metasize); |
1959 | |
1960 | igc_rx_buffer_flip(buffer: rx_buffer, truesize); |
1961 | return skb; |
1962 | } |
1963 | |
1964 | static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, |
1965 | struct igc_rx_buffer *rx_buffer, |
1966 | struct xdp_buff *xdp, |
1967 | ktime_t timestamp) |
1968 | { |
1969 | unsigned int metasize = xdp->data - xdp->data_meta; |
1970 | unsigned int size = xdp->data_end - xdp->data; |
1971 | unsigned int truesize = igc_get_rx_frame_truesize(ring: rx_ring, size); |
1972 | void *va = xdp->data; |
1973 | unsigned int headlen; |
1974 | struct sk_buff *skb; |
1975 | |
1976 | /* prefetch first cache line of first page */ |
1977 | net_prefetch(p: xdp->data_meta); |
1978 | |
1979 | /* allocate a skb to store the frags */ |
1980 | skb = napi_alloc_skb(napi: &rx_ring->q_vector->napi, |
1981 | IGC_RX_HDR_LEN + metasize); |
1982 | if (unlikely(!skb)) |
1983 | return NULL; |
1984 | |
1985 | if (timestamp) |
1986 | skb_hwtstamps(skb)->hwtstamp = timestamp; |
1987 | |
1988 | /* Determine available headroom for copy */ |
1989 | headlen = size; |
1990 | if (headlen > IGC_RX_HDR_LEN) |
1991 | headlen = eth_get_headlen(dev: skb->dev, data: va, IGC_RX_HDR_LEN); |
1992 | |
1993 | /* align pull length to size of long to optimize memcpy performance */ |
1994 | memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta, |
1995 | ALIGN(headlen + metasize, sizeof(long))); |
1996 | |
1997 | if (metasize) { |
1998 | skb_metadata_set(skb, meta_len: metasize); |
1999 | __skb_pull(skb, len: metasize); |
2000 | } |
2001 | |
2002 | /* update all of the pointers */ |
2003 | size -= headlen; |
2004 | if (size) { |
2005 | skb_add_rx_frag(skb, i: 0, page: rx_buffer->page, |
2006 | off: (va + headlen) - page_address(rx_buffer->page), |
2007 | size, truesize); |
2008 | igc_rx_buffer_flip(buffer: rx_buffer, truesize); |
2009 | } else { |
2010 | rx_buffer->pagecnt_bias++; |
2011 | } |
2012 | |
2013 | return skb; |
2014 | } |
2015 | |
2016 | /** |
2017 | * igc_reuse_rx_page - page flip buffer and store it back on the ring |
2018 | * @rx_ring: rx descriptor ring to store buffers on |
2019 | * @old_buff: donor buffer to have page reused |
2020 | * |
2021 | * Synchronizes page for reuse by the adapter |
2022 | */ |
2023 | static void igc_reuse_rx_page(struct igc_ring *rx_ring, |
2024 | struct igc_rx_buffer *old_buff) |
2025 | { |
2026 | u16 nta = rx_ring->next_to_alloc; |
2027 | struct igc_rx_buffer *new_buff; |
2028 | |
2029 | new_buff = &rx_ring->rx_buffer_info[nta]; |
2030 | |
2031 | /* update, and store next to alloc */ |
2032 | nta++; |
2033 | rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; |
2034 | |
2035 | /* Transfer page from old buffer to new buffer. |
2036 | * Move each member individually to avoid possible store |
2037 | * forwarding stalls. |
2038 | */ |
2039 | new_buff->dma = old_buff->dma; |
2040 | new_buff->page = old_buff->page; |
2041 | new_buff->page_offset = old_buff->page_offset; |
2042 | new_buff->pagecnt_bias = old_buff->pagecnt_bias; |
2043 | } |
2044 | |
2045 | static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer, |
2046 | int rx_buffer_pgcnt) |
2047 | { |
2048 | unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; |
2049 | struct page *page = rx_buffer->page; |
2050 | |
2051 | /* avoid re-using remote and pfmemalloc pages */ |
2052 | if (!dev_page_is_reusable(page)) |
2053 | return false; |
2054 | |
2055 | #if (PAGE_SIZE < 8192) |
2056 | /* if we are only owner of page we can reuse it */ |
2057 | if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1)) |
2058 | return false; |
2059 | #else |
2060 | #define IGC_LAST_OFFSET \ |
2061 | (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048) |
2062 | |
2063 | if (rx_buffer->page_offset > IGC_LAST_OFFSET) |
2064 | return false; |
2065 | #endif |
2066 | |
2067 | /* If we have drained the page fragment pool we need to update |
2068 | * the pagecnt_bias and page count so that we fully restock the |
2069 | * number of references the driver holds. |
2070 | */ |
2071 | if (unlikely(pagecnt_bias == 1)) { |
2072 | page_ref_add(page, USHRT_MAX - 1); |
2073 | rx_buffer->pagecnt_bias = USHRT_MAX; |
2074 | } |
2075 | |
2076 | return true; |
2077 | } |
2078 | |
2079 | /** |
2080 | * igc_is_non_eop - process handling of non-EOP buffers |
2081 | * @rx_ring: Rx ring being processed |
2082 | * @rx_desc: Rx descriptor for current buffer |
2083 | * |
2084 | * This function updates next to clean. If the buffer is an EOP buffer |
2085 | * this function exits returning false, otherwise it will place the |
2086 | * sk_buff in the next buffer to be chained and return true indicating |
2087 | * that this is in fact a non-EOP buffer. |
2088 | */ |
2089 | static bool igc_is_non_eop(struct igc_ring *rx_ring, |
2090 | union igc_adv_rx_desc *rx_desc) |
2091 | { |
2092 | u32 ntc = rx_ring->next_to_clean + 1; |
2093 | |
2094 | /* fetch, update, and store next to clean */ |
2095 | ntc = (ntc < rx_ring->count) ? ntc : 0; |
2096 | rx_ring->next_to_clean = ntc; |
2097 | |
2098 | prefetch(IGC_RX_DESC(rx_ring, ntc)); |
2099 | |
2100 | if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP))) |
2101 | return false; |
2102 | |
2103 | return true; |
2104 | } |
2105 | |
2106 | /** |
2107 | * igc_cleanup_headers - Correct corrupted or empty headers |
2108 | * @rx_ring: rx descriptor ring packet is being transacted on |
2109 | * @rx_desc: pointer to the EOP Rx descriptor |
2110 | * @skb: pointer to current skb being fixed |
2111 | * |
2112 | * Address the case where we are pulling data in on pages only |
2113 | * and as such no data is present in the skb header. |
2114 | * |
2115 | * In addition if skb is not at least 60 bytes we need to pad it so that |
2116 | * it is large enough to qualify as a valid Ethernet frame. |
2117 | * |
2118 | * Returns true if an error was encountered and skb was freed. |
2119 | */ |
2120 | static bool (struct igc_ring *rx_ring, |
2121 | union igc_adv_rx_desc *rx_desc, |
2122 | struct sk_buff *skb) |
2123 | { |
2124 | /* XDP packets use error pointer so abort at this point */ |
2125 | if (IS_ERR(ptr: skb)) |
2126 | return true; |
2127 | |
2128 | if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) { |
2129 | struct net_device *netdev = rx_ring->netdev; |
2130 | |
2131 | if (!(netdev->features & NETIF_F_RXALL)) { |
2132 | dev_kfree_skb_any(skb); |
2133 | return true; |
2134 | } |
2135 | } |
2136 | |
2137 | /* if eth_skb_pad returns an error the skb was freed */ |
2138 | if (eth_skb_pad(skb)) |
2139 | return true; |
2140 | |
2141 | return false; |
2142 | } |
2143 | |
2144 | static void igc_put_rx_buffer(struct igc_ring *rx_ring, |
2145 | struct igc_rx_buffer *rx_buffer, |
2146 | int rx_buffer_pgcnt) |
2147 | { |
2148 | if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) { |
2149 | /* hand second half of page back to the ring */ |
2150 | igc_reuse_rx_page(rx_ring, old_buff: rx_buffer); |
2151 | } else { |
2152 | /* We are not reusing the buffer so unmap it and free |
2153 | * any references we are holding to it |
2154 | */ |
2155 | dma_unmap_page_attrs(dev: rx_ring->dev, addr: rx_buffer->dma, |
2156 | igc_rx_pg_size(rx_ring), dir: DMA_FROM_DEVICE, |
2157 | IGC_RX_DMA_ATTR); |
2158 | __page_frag_cache_drain(page: rx_buffer->page, |
2159 | count: rx_buffer->pagecnt_bias); |
2160 | } |
2161 | |
2162 | /* clear contents of rx_buffer */ |
2163 | rx_buffer->page = NULL; |
2164 | } |
2165 | |
2166 | static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) |
2167 | { |
2168 | struct igc_adapter *adapter = rx_ring->q_vector->adapter; |
2169 | |
2170 | if (ring_uses_build_skb(rx_ring)) |
2171 | return IGC_SKB_PAD; |
2172 | if (igc_xdp_is_enabled(adapter)) |
2173 | return XDP_PACKET_HEADROOM; |
2174 | |
2175 | return 0; |
2176 | } |
2177 | |
2178 | static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, |
2179 | struct igc_rx_buffer *bi) |
2180 | { |
2181 | struct page *page = bi->page; |
2182 | dma_addr_t dma; |
2183 | |
2184 | /* since we are recycling buffers we should seldom need to alloc */ |
2185 | if (likely(page)) |
2186 | return true; |
2187 | |
2188 | /* alloc new page for storage */ |
2189 | page = dev_alloc_pages(order: igc_rx_pg_order(ring: rx_ring)); |
2190 | if (unlikely(!page)) { |
2191 | rx_ring->rx_stats.alloc_failed++; |
2192 | return false; |
2193 | } |
2194 | |
2195 | /* map page for use */ |
2196 | dma = dma_map_page_attrs(dev: rx_ring->dev, page, offset: 0, |
2197 | igc_rx_pg_size(rx_ring), |
2198 | dir: DMA_FROM_DEVICE, |
2199 | IGC_RX_DMA_ATTR); |
2200 | |
2201 | /* if mapping failed free memory back to system since |
2202 | * there isn't much point in holding memory we can't use |
2203 | */ |
2204 | if (dma_mapping_error(dev: rx_ring->dev, dma_addr: dma)) { |
2205 | __free_page(page); |
2206 | |
2207 | rx_ring->rx_stats.alloc_failed++; |
2208 | return false; |
2209 | } |
2210 | |
2211 | bi->dma = dma; |
2212 | bi->page = page; |
2213 | bi->page_offset = igc_rx_offset(rx_ring); |
2214 | page_ref_add(page, USHRT_MAX - 1); |
2215 | bi->pagecnt_bias = USHRT_MAX; |
2216 | |
2217 | return true; |
2218 | } |
2219 | |
2220 | /** |
2221 | * igc_alloc_rx_buffers - Replace used receive buffers; packet split |
2222 | * @rx_ring: rx descriptor ring |
2223 | * @cleaned_count: number of buffers to clean |
2224 | */ |
2225 | static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) |
2226 | { |
2227 | union igc_adv_rx_desc *rx_desc; |
2228 | u16 i = rx_ring->next_to_use; |
2229 | struct igc_rx_buffer *bi; |
2230 | u16 bufsz; |
2231 | |
2232 | /* nothing to do */ |
2233 | if (!cleaned_count) |
2234 | return; |
2235 | |
2236 | rx_desc = IGC_RX_DESC(rx_ring, i); |
2237 | bi = &rx_ring->rx_buffer_info[i]; |
2238 | i -= rx_ring->count; |
2239 | |
2240 | bufsz = igc_rx_bufsz(ring: rx_ring); |
2241 | |
2242 | do { |
2243 | if (!igc_alloc_mapped_page(rx_ring, bi)) |
2244 | break; |
2245 | |
2246 | /* sync the buffer for use by the device */ |
2247 | dma_sync_single_range_for_device(dev: rx_ring->dev, addr: bi->dma, |
2248 | offset: bi->page_offset, size: bufsz, |
2249 | dir: DMA_FROM_DEVICE); |
2250 | |
2251 | /* Refresh the desc even if buffer_addrs didn't change |
2252 | * because each write-back erases this info. |
2253 | */ |
2254 | rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); |
2255 | |
2256 | rx_desc++; |
2257 | bi++; |
2258 | i++; |
2259 | if (unlikely(!i)) { |
2260 | rx_desc = IGC_RX_DESC(rx_ring, 0); |
2261 | bi = rx_ring->rx_buffer_info; |
2262 | i -= rx_ring->count; |
2263 | } |
2264 | |
2265 | /* clear the length for the next_to_use descriptor */ |
2266 | rx_desc->wb.upper.length = 0; |
2267 | |
2268 | cleaned_count--; |
2269 | } while (cleaned_count); |
2270 | |
2271 | i += rx_ring->count; |
2272 | |
2273 | if (rx_ring->next_to_use != i) { |
2274 | /* record the next descriptor to use */ |
2275 | rx_ring->next_to_use = i; |
2276 | |
2277 | /* update next to alloc since we have filled the ring */ |
2278 | rx_ring->next_to_alloc = i; |
2279 | |
2280 | /* Force memory writes to complete before letting h/w |
2281 | * know there are new descriptors to fetch. (Only |
2282 | * applicable for weak-ordered memory model archs, |
2283 | * such as IA-64). |
2284 | */ |
2285 | wmb(); |
2286 | writel(val: i, addr: rx_ring->tail); |
2287 | } |
2288 | } |
2289 | |
2290 | static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count) |
2291 | { |
2292 | union igc_adv_rx_desc *desc; |
2293 | u16 i = ring->next_to_use; |
2294 | struct igc_rx_buffer *bi; |
2295 | dma_addr_t dma; |
2296 | bool ok = true; |
2297 | |
2298 | if (!count) |
2299 | return ok; |
2300 | |
2301 | XSK_CHECK_PRIV_TYPE(struct igc_xdp_buff); |
2302 | |
2303 | desc = IGC_RX_DESC(ring, i); |
2304 | bi = &ring->rx_buffer_info[i]; |
2305 | i -= ring->count; |
2306 | |
2307 | do { |
2308 | bi->xdp = xsk_buff_alloc(pool: ring->xsk_pool); |
2309 | if (!bi->xdp) { |
2310 | ok = false; |
2311 | break; |
2312 | } |
2313 | |
2314 | dma = xsk_buff_xdp_get_dma(xdp: bi->xdp); |
2315 | desc->read.pkt_addr = cpu_to_le64(dma); |
2316 | |
2317 | desc++; |
2318 | bi++; |
2319 | i++; |
2320 | if (unlikely(!i)) { |
2321 | desc = IGC_RX_DESC(ring, 0); |
2322 | bi = ring->rx_buffer_info; |
2323 | i -= ring->count; |
2324 | } |
2325 | |
2326 | /* Clear the length for the next_to_use descriptor. */ |
2327 | desc->wb.upper.length = 0; |
2328 | |
2329 | count--; |
2330 | } while (count); |
2331 | |
2332 | i += ring->count; |
2333 | |
2334 | if (ring->next_to_use != i) { |
2335 | ring->next_to_use = i; |
2336 | |
2337 | /* Force memory writes to complete before letting h/w |
2338 | * know there are new descriptors to fetch. (Only |
2339 | * applicable for weak-ordered memory model archs, |
2340 | * such as IA-64). |
2341 | */ |
2342 | wmb(); |
2343 | writel(val: i, addr: ring->tail); |
2344 | } |
2345 | |
2346 | return ok; |
2347 | } |
2348 | |
2349 | /* This function requires __netif_tx_lock is held by the caller. */ |
2350 | static int igc_xdp_init_tx_descriptor(struct igc_ring *ring, |
2351 | struct xdp_frame *xdpf) |
2352 | { |
2353 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(frame: xdpf); |
2354 | u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; |
2355 | u16 count, index = ring->next_to_use; |
2356 | struct igc_tx_buffer *head = &ring->tx_buffer_info[index]; |
2357 | struct igc_tx_buffer *buffer = head; |
2358 | union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index); |
2359 | u32 olinfo_status, len = xdpf->len, cmd_type; |
2360 | void *data = xdpf->data; |
2361 | u16 i; |
2362 | |
2363 | count = TXD_USE_COUNT(len); |
2364 | for (i = 0; i < nr_frags; i++) |
2365 | count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i])); |
2366 | |
2367 | if (igc_maybe_stop_tx(tx_ring: ring, size: count + 3)) { |
2368 | /* this is a hard error */ |
2369 | return -EBUSY; |
2370 | } |
2371 | |
2372 | i = 0; |
2373 | head->bytecount = xdp_get_frame_len(xdpf); |
2374 | head->type = IGC_TX_BUFFER_TYPE_XDP; |
2375 | head->gso_segs = 1; |
2376 | head->xdpf = xdpf; |
2377 | |
2378 | olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT; |
2379 | desc->read.olinfo_status = cpu_to_le32(olinfo_status); |
2380 | |
2381 | for (;;) { |
2382 | dma_addr_t dma; |
2383 | |
2384 | dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE); |
2385 | if (dma_mapping_error(dev: ring->dev, dma_addr: dma)) { |
2386 | netdev_err_once(ring->netdev, |
2387 | "Failed to map DMA for TX\n" ); |
2388 | goto unmap; |
2389 | } |
2390 | |
2391 | dma_unmap_len_set(buffer, len, len); |
2392 | dma_unmap_addr_set(buffer, dma, dma); |
2393 | |
2394 | cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | |
2395 | IGC_ADVTXD_DCMD_IFCS | len; |
2396 | |
2397 | desc->read.cmd_type_len = cpu_to_le32(cmd_type); |
2398 | desc->read.buffer_addr = cpu_to_le64(dma); |
2399 | |
2400 | buffer->protocol = 0; |
2401 | |
2402 | if (++index == ring->count) |
2403 | index = 0; |
2404 | |
2405 | if (i == nr_frags) |
2406 | break; |
2407 | |
2408 | buffer = &ring->tx_buffer_info[index]; |
2409 | desc = IGC_TX_DESC(ring, index); |
2410 | desc->read.olinfo_status = 0; |
2411 | |
2412 | data = skb_frag_address(frag: &sinfo->frags[i]); |
2413 | len = skb_frag_size(frag: &sinfo->frags[i]); |
2414 | i++; |
2415 | } |
2416 | desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD); |
2417 | |
2418 | netdev_tx_sent_queue(dev_queue: txring_txq(tx_ring: ring), bytes: head->bytecount); |
2419 | /* set the timestamp */ |
2420 | head->time_stamp = jiffies; |
2421 | /* set next_to_watch value indicating a packet is present */ |
2422 | head->next_to_watch = desc; |
2423 | ring->next_to_use = index; |
2424 | |
2425 | return 0; |
2426 | |
2427 | unmap: |
2428 | for (;;) { |
2429 | buffer = &ring->tx_buffer_info[index]; |
2430 | if (dma_unmap_len(buffer, len)) |
2431 | dma_unmap_page(ring->dev, |
2432 | dma_unmap_addr(buffer, dma), |
2433 | dma_unmap_len(buffer, len), |
2434 | DMA_TO_DEVICE); |
2435 | dma_unmap_len_set(buffer, len, 0); |
2436 | if (buffer == head) |
2437 | break; |
2438 | |
2439 | if (!index) |
2440 | index += ring->count; |
2441 | index--; |
2442 | } |
2443 | |
2444 | return -ENOMEM; |
2445 | } |
2446 | |
2447 | static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter, |
2448 | int cpu) |
2449 | { |
2450 | int index = cpu; |
2451 | |
2452 | if (unlikely(index < 0)) |
2453 | index = 0; |
2454 | |
2455 | while (index >= adapter->num_tx_queues) |
2456 | index -= adapter->num_tx_queues; |
2457 | |
2458 | return adapter->tx_ring[index]; |
2459 | } |
2460 | |
2461 | static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp) |
2462 | { |
2463 | struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); |
2464 | int cpu = smp_processor_id(); |
2465 | struct netdev_queue *nq; |
2466 | struct igc_ring *ring; |
2467 | int res; |
2468 | |
2469 | if (unlikely(!xdpf)) |
2470 | return -EFAULT; |
2471 | |
2472 | ring = igc_xdp_get_tx_ring(adapter, cpu); |
2473 | nq = txring_txq(tx_ring: ring); |
2474 | |
2475 | __netif_tx_lock(txq: nq, cpu); |
2476 | /* Avoid transmit queue timeout since we share it with the slow path */ |
2477 | txq_trans_cond_update(txq: nq); |
2478 | res = igc_xdp_init_tx_descriptor(ring, xdpf); |
2479 | __netif_tx_unlock(txq: nq); |
2480 | return res; |
2481 | } |
2482 | |
2483 | /* This function assumes rcu_read_lock() is held by the caller. */ |
2484 | static int __igc_xdp_run_prog(struct igc_adapter *adapter, |
2485 | struct bpf_prog *prog, |
2486 | struct xdp_buff *xdp) |
2487 | { |
2488 | u32 act = bpf_prog_run_xdp(prog, xdp); |
2489 | |
2490 | switch (act) { |
2491 | case XDP_PASS: |
2492 | return IGC_XDP_PASS; |
2493 | case XDP_TX: |
2494 | if (igc_xdp_xmit_back(adapter, xdp) < 0) |
2495 | goto out_failure; |
2496 | return IGC_XDP_TX; |
2497 | case XDP_REDIRECT: |
2498 | if (xdp_do_redirect(dev: adapter->netdev, xdp, prog) < 0) |
2499 | goto out_failure; |
2500 | return IGC_XDP_REDIRECT; |
2501 | break; |
2502 | default: |
2503 | bpf_warn_invalid_xdp_action(dev: adapter->netdev, prog, act); |
2504 | fallthrough; |
2505 | case XDP_ABORTED: |
2506 | out_failure: |
2507 | trace_xdp_exception(dev: adapter->netdev, xdp: prog, act); |
2508 | fallthrough; |
2509 | case XDP_DROP: |
2510 | return IGC_XDP_CONSUMED; |
2511 | } |
2512 | } |
2513 | |
2514 | static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter, |
2515 | struct xdp_buff *xdp) |
2516 | { |
2517 | struct bpf_prog *prog; |
2518 | int res; |
2519 | |
2520 | prog = READ_ONCE(adapter->xdp_prog); |
2521 | if (!prog) { |
2522 | res = IGC_XDP_PASS; |
2523 | goto out; |
2524 | } |
2525 | |
2526 | res = __igc_xdp_run_prog(adapter, prog, xdp); |
2527 | |
2528 | out: |
2529 | return ERR_PTR(error: -res); |
2530 | } |
2531 | |
2532 | /* This function assumes __netif_tx_lock is held by the caller. */ |
2533 | static void igc_flush_tx_descriptors(struct igc_ring *ring) |
2534 | { |
2535 | /* Once tail pointer is updated, hardware can fetch the descriptors |
2536 | * any time so we issue a write membar here to ensure all memory |
2537 | * writes are complete before the tail pointer is updated. |
2538 | */ |
2539 | wmb(); |
2540 | writel(val: ring->next_to_use, addr: ring->tail); |
2541 | } |
2542 | |
2543 | static void igc_finalize_xdp(struct igc_adapter *adapter, int status) |
2544 | { |
2545 | int cpu = smp_processor_id(); |
2546 | struct netdev_queue *nq; |
2547 | struct igc_ring *ring; |
2548 | |
2549 | if (status & IGC_XDP_TX) { |
2550 | ring = igc_xdp_get_tx_ring(adapter, cpu); |
2551 | nq = txring_txq(tx_ring: ring); |
2552 | |
2553 | __netif_tx_lock(txq: nq, cpu); |
2554 | igc_flush_tx_descriptors(ring); |
2555 | __netif_tx_unlock(txq: nq); |
2556 | } |
2557 | |
2558 | if (status & IGC_XDP_REDIRECT) |
2559 | xdp_do_flush(); |
2560 | } |
2561 | |
2562 | static void igc_update_rx_stats(struct igc_q_vector *q_vector, |
2563 | unsigned int packets, unsigned int bytes) |
2564 | { |
2565 | struct igc_ring *ring = q_vector->rx.ring; |
2566 | |
2567 | u64_stats_update_begin(syncp: &ring->rx_syncp); |
2568 | ring->rx_stats.packets += packets; |
2569 | ring->rx_stats.bytes += bytes; |
2570 | u64_stats_update_end(syncp: &ring->rx_syncp); |
2571 | |
2572 | q_vector->rx.total_packets += packets; |
2573 | q_vector->rx.total_bytes += bytes; |
2574 | } |
2575 | |
2576 | static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) |
2577 | { |
2578 | unsigned int total_bytes = 0, total_packets = 0; |
2579 | struct igc_adapter *adapter = q_vector->adapter; |
2580 | struct igc_ring *rx_ring = q_vector->rx.ring; |
2581 | struct sk_buff *skb = rx_ring->skb; |
2582 | u16 cleaned_count = igc_desc_unused(ring: rx_ring); |
2583 | int xdp_status = 0, rx_buffer_pgcnt; |
2584 | |
2585 | while (likely(total_packets < budget)) { |
2586 | union igc_adv_rx_desc *rx_desc; |
2587 | struct igc_rx_buffer *rx_buffer; |
2588 | unsigned int size, truesize; |
2589 | struct igc_xdp_buff ctx; |
2590 | ktime_t timestamp = 0; |
2591 | int pkt_offset = 0; |
2592 | void *pktbuf; |
2593 | |
2594 | /* return some buffers to hardware, one at a time is too slow */ |
2595 | if (cleaned_count >= IGC_RX_BUFFER_WRITE) { |
2596 | igc_alloc_rx_buffers(rx_ring, cleaned_count); |
2597 | cleaned_count = 0; |
2598 | } |
2599 | |
2600 | rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean); |
2601 | size = le16_to_cpu(rx_desc->wb.upper.length); |
2602 | if (!size) |
2603 | break; |
2604 | |
2605 | /* This memory barrier is needed to keep us from reading |
2606 | * any other fields out of the rx_desc until we know the |
2607 | * descriptor has been written back |
2608 | */ |
2609 | dma_rmb(); |
2610 | |
2611 | rx_buffer = igc_get_rx_buffer(rx_ring, size, rx_buffer_pgcnt: &rx_buffer_pgcnt); |
2612 | truesize = igc_get_rx_frame_truesize(ring: rx_ring, size); |
2613 | |
2614 | pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset; |
2615 | |
2616 | if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) { |
2617 | timestamp = igc_ptp_rx_pktstamp(adapter: q_vector->adapter, |
2618 | buf: pktbuf); |
2619 | ctx.rx_ts = timestamp; |
2620 | pkt_offset = IGC_TS_HDR_LEN; |
2621 | size -= IGC_TS_HDR_LEN; |
2622 | } |
2623 | |
2624 | if (!skb) { |
2625 | xdp_init_buff(xdp: &ctx.xdp, frame_sz: truesize, rxq: &rx_ring->xdp_rxq); |
2626 | xdp_prepare_buff(xdp: &ctx.xdp, hard_start: pktbuf - igc_rx_offset(rx_ring), |
2627 | headroom: igc_rx_offset(rx_ring) + pkt_offset, |
2628 | data_len: size, meta_valid: true); |
2629 | xdp_buff_clear_frags_flag(xdp: &ctx.xdp); |
2630 | ctx.rx_desc = rx_desc; |
2631 | |
2632 | skb = igc_xdp_run_prog(adapter, xdp: &ctx.xdp); |
2633 | } |
2634 | |
2635 | if (IS_ERR(ptr: skb)) { |
2636 | unsigned int xdp_res = -PTR_ERR(ptr: skb); |
2637 | |
2638 | switch (xdp_res) { |
2639 | case IGC_XDP_CONSUMED: |
2640 | rx_buffer->pagecnt_bias++; |
2641 | break; |
2642 | case IGC_XDP_TX: |
2643 | case IGC_XDP_REDIRECT: |
2644 | igc_rx_buffer_flip(buffer: rx_buffer, truesize); |
2645 | xdp_status |= xdp_res; |
2646 | break; |
2647 | } |
2648 | |
2649 | total_packets++; |
2650 | total_bytes += size; |
2651 | } else if (skb) |
2652 | igc_add_rx_frag(rx_ring, rx_buffer, skb, size); |
2653 | else if (ring_uses_build_skb(rx_ring)) |
2654 | skb = igc_build_skb(rx_ring, rx_buffer, xdp: &ctx.xdp); |
2655 | else |
2656 | skb = igc_construct_skb(rx_ring, rx_buffer, xdp: &ctx.xdp, |
2657 | timestamp); |
2658 | |
2659 | /* exit if we failed to retrieve a buffer */ |
2660 | if (!skb) { |
2661 | rx_ring->rx_stats.alloc_failed++; |
2662 | rx_buffer->pagecnt_bias++; |
2663 | break; |
2664 | } |
2665 | |
2666 | igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt); |
2667 | cleaned_count++; |
2668 | |
2669 | /* fetch next buffer in frame if non-eop */ |
2670 | if (igc_is_non_eop(rx_ring, rx_desc)) |
2671 | continue; |
2672 | |
2673 | /* verify the packet layout is correct */ |
2674 | if (igc_cleanup_headers(rx_ring, rx_desc, skb)) { |
2675 | skb = NULL; |
2676 | continue; |
2677 | } |
2678 | |
2679 | /* probably a little skewed due to removing CRC */ |
2680 | total_bytes += skb->len; |
2681 | |
2682 | /* populate checksum, VLAN, and protocol */ |
2683 | igc_process_skb_fields(rx_ring, rx_desc, skb); |
2684 | |
2685 | napi_gro_receive(napi: &q_vector->napi, skb); |
2686 | |
2687 | /* reset skb pointer */ |
2688 | skb = NULL; |
2689 | |
2690 | /* update budget accounting */ |
2691 | total_packets++; |
2692 | } |
2693 | |
2694 | if (xdp_status) |
2695 | igc_finalize_xdp(adapter, status: xdp_status); |
2696 | |
2697 | /* place incomplete frames back on ring for completion */ |
2698 | rx_ring->skb = skb; |
2699 | |
2700 | igc_update_rx_stats(q_vector, packets: total_packets, bytes: total_bytes); |
2701 | |
2702 | if (cleaned_count) |
2703 | igc_alloc_rx_buffers(rx_ring, cleaned_count); |
2704 | |
2705 | return total_packets; |
2706 | } |
2707 | |
2708 | static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring, |
2709 | struct xdp_buff *xdp) |
2710 | { |
2711 | unsigned int totalsize = xdp->data_end - xdp->data_meta; |
2712 | unsigned int metasize = xdp->data - xdp->data_meta; |
2713 | struct sk_buff *skb; |
2714 | |
2715 | net_prefetch(p: xdp->data_meta); |
2716 | |
2717 | skb = __napi_alloc_skb(napi: &ring->q_vector->napi, length: totalsize, |
2718 | GFP_ATOMIC | __GFP_NOWARN); |
2719 | if (unlikely(!skb)) |
2720 | return NULL; |
2721 | |
2722 | memcpy(__skb_put(skb, totalsize), xdp->data_meta, |
2723 | ALIGN(totalsize, sizeof(long))); |
2724 | |
2725 | if (metasize) { |
2726 | skb_metadata_set(skb, meta_len: metasize); |
2727 | __skb_pull(skb, len: metasize); |
2728 | } |
2729 | |
2730 | return skb; |
2731 | } |
2732 | |
2733 | static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector, |
2734 | union igc_adv_rx_desc *desc, |
2735 | struct xdp_buff *xdp, |
2736 | ktime_t timestamp) |
2737 | { |
2738 | struct igc_ring *ring = q_vector->rx.ring; |
2739 | struct sk_buff *skb; |
2740 | |
2741 | skb = igc_construct_skb_zc(ring, xdp); |
2742 | if (!skb) { |
2743 | ring->rx_stats.alloc_failed++; |
2744 | return; |
2745 | } |
2746 | |
2747 | if (timestamp) |
2748 | skb_hwtstamps(skb)->hwtstamp = timestamp; |
2749 | |
2750 | if (igc_cleanup_headers(rx_ring: ring, rx_desc: desc, skb)) |
2751 | return; |
2752 | |
2753 | igc_process_skb_fields(rx_ring: ring, rx_desc: desc, skb); |
2754 | napi_gro_receive(napi: &q_vector->napi, skb); |
2755 | } |
2756 | |
2757 | static struct igc_xdp_buff *xsk_buff_to_igc_ctx(struct xdp_buff *xdp) |
2758 | { |
2759 | /* xdp_buff pointer used by ZC code path is alloc as xdp_buff_xsk. The |
2760 | * igc_xdp_buff shares its layout with xdp_buff_xsk and private |
2761 | * igc_xdp_buff fields fall into xdp_buff_xsk->cb |
2762 | */ |
2763 | return (struct igc_xdp_buff *)xdp; |
2764 | } |
2765 | |
2766 | static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget) |
2767 | { |
2768 | struct igc_adapter *adapter = q_vector->adapter; |
2769 | struct igc_ring *ring = q_vector->rx.ring; |
2770 | u16 cleaned_count = igc_desc_unused(ring); |
2771 | int total_bytes = 0, total_packets = 0; |
2772 | u16 ntc = ring->next_to_clean; |
2773 | struct bpf_prog *prog; |
2774 | bool failure = false; |
2775 | int xdp_status = 0; |
2776 | |
2777 | rcu_read_lock(); |
2778 | |
2779 | prog = READ_ONCE(adapter->xdp_prog); |
2780 | |
2781 | while (likely(total_packets < budget)) { |
2782 | union igc_adv_rx_desc *desc; |
2783 | struct igc_rx_buffer *bi; |
2784 | struct igc_xdp_buff *ctx; |
2785 | ktime_t timestamp = 0; |
2786 | unsigned int size; |
2787 | int res; |
2788 | |
2789 | desc = IGC_RX_DESC(ring, ntc); |
2790 | size = le16_to_cpu(desc->wb.upper.length); |
2791 | if (!size) |
2792 | break; |
2793 | |
2794 | /* This memory barrier is needed to keep us from reading |
2795 | * any other fields out of the rx_desc until we know the |
2796 | * descriptor has been written back |
2797 | */ |
2798 | dma_rmb(); |
2799 | |
2800 | bi = &ring->rx_buffer_info[ntc]; |
2801 | |
2802 | ctx = xsk_buff_to_igc_ctx(xdp: bi->xdp); |
2803 | ctx->rx_desc = desc; |
2804 | |
2805 | if (igc_test_staterr(rx_desc: desc, IGC_RXDADV_STAT_TSIP)) { |
2806 | timestamp = igc_ptp_rx_pktstamp(adapter: q_vector->adapter, |
2807 | buf: bi->xdp->data); |
2808 | ctx->rx_ts = timestamp; |
2809 | |
2810 | bi->xdp->data += IGC_TS_HDR_LEN; |
2811 | |
2812 | /* HW timestamp has been copied into local variable. Metadata |
2813 | * length when XDP program is called should be 0. |
2814 | */ |
2815 | bi->xdp->data_meta += IGC_TS_HDR_LEN; |
2816 | size -= IGC_TS_HDR_LEN; |
2817 | } |
2818 | |
2819 | bi->xdp->data_end = bi->xdp->data + size; |
2820 | xsk_buff_dma_sync_for_cpu(xdp: bi->xdp, pool: ring->xsk_pool); |
2821 | |
2822 | res = __igc_xdp_run_prog(adapter, prog, xdp: bi->xdp); |
2823 | switch (res) { |
2824 | case IGC_XDP_PASS: |
2825 | igc_dispatch_skb_zc(q_vector, desc, xdp: bi->xdp, timestamp); |
2826 | fallthrough; |
2827 | case IGC_XDP_CONSUMED: |
2828 | xsk_buff_free(xdp: bi->xdp); |
2829 | break; |
2830 | case IGC_XDP_TX: |
2831 | case IGC_XDP_REDIRECT: |
2832 | xdp_status |= res; |
2833 | break; |
2834 | } |
2835 | |
2836 | bi->xdp = NULL; |
2837 | total_bytes += size; |
2838 | total_packets++; |
2839 | cleaned_count++; |
2840 | ntc++; |
2841 | if (ntc == ring->count) |
2842 | ntc = 0; |
2843 | } |
2844 | |
2845 | ring->next_to_clean = ntc; |
2846 | rcu_read_unlock(); |
2847 | |
2848 | if (cleaned_count >= IGC_RX_BUFFER_WRITE) |
2849 | failure = !igc_alloc_rx_buffers_zc(ring, count: cleaned_count); |
2850 | |
2851 | if (xdp_status) |
2852 | igc_finalize_xdp(adapter, status: xdp_status); |
2853 | |
2854 | igc_update_rx_stats(q_vector, packets: total_packets, bytes: total_bytes); |
2855 | |
2856 | if (xsk_uses_need_wakeup(pool: ring->xsk_pool)) { |
2857 | if (failure || ring->next_to_clean == ring->next_to_use) |
2858 | xsk_set_rx_need_wakeup(pool: ring->xsk_pool); |
2859 | else |
2860 | xsk_clear_rx_need_wakeup(pool: ring->xsk_pool); |
2861 | return total_packets; |
2862 | } |
2863 | |
2864 | return failure ? budget : total_packets; |
2865 | } |
2866 | |
2867 | static void igc_update_tx_stats(struct igc_q_vector *q_vector, |
2868 | unsigned int packets, unsigned int bytes) |
2869 | { |
2870 | struct igc_ring *ring = q_vector->tx.ring; |
2871 | |
2872 | u64_stats_update_begin(syncp: &ring->tx_syncp); |
2873 | ring->tx_stats.bytes += bytes; |
2874 | ring->tx_stats.packets += packets; |
2875 | u64_stats_update_end(syncp: &ring->tx_syncp); |
2876 | |
2877 | q_vector->tx.total_bytes += bytes; |
2878 | q_vector->tx.total_packets += packets; |
2879 | } |
2880 | |
2881 | static void igc_xdp_xmit_zc(struct igc_ring *ring) |
2882 | { |
2883 | struct xsk_buff_pool *pool = ring->xsk_pool; |
2884 | struct netdev_queue *nq = txring_txq(tx_ring: ring); |
2885 | union igc_adv_tx_desc *tx_desc = NULL; |
2886 | int cpu = smp_processor_id(); |
2887 | struct xdp_desc xdp_desc; |
2888 | u16 budget, ntu; |
2889 | |
2890 | if (!netif_carrier_ok(dev: ring->netdev)) |
2891 | return; |
2892 | |
2893 | __netif_tx_lock(txq: nq, cpu); |
2894 | |
2895 | /* Avoid transmit queue timeout since we share it with the slow path */ |
2896 | txq_trans_cond_update(txq: nq); |
2897 | |
2898 | ntu = ring->next_to_use; |
2899 | budget = igc_desc_unused(ring); |
2900 | |
2901 | while (xsk_tx_peek_desc(pool, desc: &xdp_desc) && budget--) { |
2902 | u32 cmd_type, olinfo_status; |
2903 | struct igc_tx_buffer *bi; |
2904 | dma_addr_t dma; |
2905 | |
2906 | cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | |
2907 | IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD | |
2908 | xdp_desc.len; |
2909 | olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT; |
2910 | |
2911 | dma = xsk_buff_raw_get_dma(pool, addr: xdp_desc.addr); |
2912 | xsk_buff_raw_dma_sync_for_device(pool, dma, size: xdp_desc.len); |
2913 | |
2914 | tx_desc = IGC_TX_DESC(ring, ntu); |
2915 | tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); |
2916 | tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); |
2917 | tx_desc->read.buffer_addr = cpu_to_le64(dma); |
2918 | |
2919 | bi = &ring->tx_buffer_info[ntu]; |
2920 | bi->type = IGC_TX_BUFFER_TYPE_XSK; |
2921 | bi->protocol = 0; |
2922 | bi->bytecount = xdp_desc.len; |
2923 | bi->gso_segs = 1; |
2924 | bi->time_stamp = jiffies; |
2925 | bi->next_to_watch = tx_desc; |
2926 | |
2927 | netdev_tx_sent_queue(dev_queue: txring_txq(tx_ring: ring), bytes: xdp_desc.len); |
2928 | |
2929 | ntu++; |
2930 | if (ntu == ring->count) |
2931 | ntu = 0; |
2932 | } |
2933 | |
2934 | ring->next_to_use = ntu; |
2935 | if (tx_desc) { |
2936 | igc_flush_tx_descriptors(ring); |
2937 | xsk_tx_release(pool); |
2938 | } |
2939 | |
2940 | __netif_tx_unlock(txq: nq); |
2941 | } |
2942 | |
2943 | /** |
2944 | * igc_clean_tx_irq - Reclaim resources after transmit completes |
2945 | * @q_vector: pointer to q_vector containing needed info |
2946 | * @napi_budget: Used to determine if we are in netpoll |
2947 | * |
2948 | * returns true if ring is completely cleaned |
2949 | */ |
2950 | static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) |
2951 | { |
2952 | struct igc_adapter *adapter = q_vector->adapter; |
2953 | unsigned int total_bytes = 0, total_packets = 0; |
2954 | unsigned int budget = q_vector->tx.work_limit; |
2955 | struct igc_ring *tx_ring = q_vector->tx.ring; |
2956 | unsigned int i = tx_ring->next_to_clean; |
2957 | struct igc_tx_buffer *tx_buffer; |
2958 | union igc_adv_tx_desc *tx_desc; |
2959 | u32 xsk_frames = 0; |
2960 | |
2961 | if (test_bit(__IGC_DOWN, &adapter->state)) |
2962 | return true; |
2963 | |
2964 | tx_buffer = &tx_ring->tx_buffer_info[i]; |
2965 | tx_desc = IGC_TX_DESC(tx_ring, i); |
2966 | i -= tx_ring->count; |
2967 | |
2968 | do { |
2969 | union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; |
2970 | |
2971 | /* if next_to_watch is not set then there is no work pending */ |
2972 | if (!eop_desc) |
2973 | break; |
2974 | |
2975 | /* prevent any other reads prior to eop_desc */ |
2976 | smp_rmb(); |
2977 | |
2978 | /* if DD is not set pending work has not been completed */ |
2979 | if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD))) |
2980 | break; |
2981 | |
2982 | /* clear next_to_watch to prevent false hangs */ |
2983 | tx_buffer->next_to_watch = NULL; |
2984 | |
2985 | /* update the statistics for this packet */ |
2986 | total_bytes += tx_buffer->bytecount; |
2987 | total_packets += tx_buffer->gso_segs; |
2988 | |
2989 | switch (tx_buffer->type) { |
2990 | case IGC_TX_BUFFER_TYPE_XSK: |
2991 | xsk_frames++; |
2992 | break; |
2993 | case IGC_TX_BUFFER_TYPE_XDP: |
2994 | xdp_return_frame(xdpf: tx_buffer->xdpf); |
2995 | igc_unmap_tx_buffer(dev: tx_ring->dev, buf: tx_buffer); |
2996 | break; |
2997 | case IGC_TX_BUFFER_TYPE_SKB: |
2998 | napi_consume_skb(skb: tx_buffer->skb, budget: napi_budget); |
2999 | igc_unmap_tx_buffer(dev: tx_ring->dev, buf: tx_buffer); |
3000 | break; |
3001 | default: |
3002 | netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n" ); |
3003 | break; |
3004 | } |
3005 | |
3006 | /* clear last DMA location and unmap remaining buffers */ |
3007 | while (tx_desc != eop_desc) { |
3008 | tx_buffer++; |
3009 | tx_desc++; |
3010 | i++; |
3011 | if (unlikely(!i)) { |
3012 | i -= tx_ring->count; |
3013 | tx_buffer = tx_ring->tx_buffer_info; |
3014 | tx_desc = IGC_TX_DESC(tx_ring, 0); |
3015 | } |
3016 | |
3017 | /* unmap any remaining paged data */ |
3018 | if (dma_unmap_len(tx_buffer, len)) |
3019 | igc_unmap_tx_buffer(dev: tx_ring->dev, buf: tx_buffer); |
3020 | } |
3021 | |
3022 | /* move us one more past the eop_desc for start of next pkt */ |
3023 | tx_buffer++; |
3024 | tx_desc++; |
3025 | i++; |
3026 | if (unlikely(!i)) { |
3027 | i -= tx_ring->count; |
3028 | tx_buffer = tx_ring->tx_buffer_info; |
3029 | tx_desc = IGC_TX_DESC(tx_ring, 0); |
3030 | } |
3031 | |
3032 | /* issue prefetch for next Tx descriptor */ |
3033 | prefetch(tx_desc); |
3034 | |
3035 | /* update budget accounting */ |
3036 | budget--; |
3037 | } while (likely(budget)); |
3038 | |
3039 | netdev_tx_completed_queue(dev_queue: txring_txq(tx_ring), |
3040 | pkts: total_packets, bytes: total_bytes); |
3041 | |
3042 | i += tx_ring->count; |
3043 | tx_ring->next_to_clean = i; |
3044 | |
3045 | igc_update_tx_stats(q_vector, packets: total_packets, bytes: total_bytes); |
3046 | |
3047 | if (tx_ring->xsk_pool) { |
3048 | if (xsk_frames) |
3049 | xsk_tx_completed(pool: tx_ring->xsk_pool, nb_entries: xsk_frames); |
3050 | if (xsk_uses_need_wakeup(pool: tx_ring->xsk_pool)) |
3051 | xsk_set_tx_need_wakeup(pool: tx_ring->xsk_pool); |
3052 | igc_xdp_xmit_zc(ring: tx_ring); |
3053 | } |
3054 | |
3055 | if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { |
3056 | struct igc_hw *hw = &adapter->hw; |
3057 | |
3058 | /* Detect a transmit hang in hardware, this serializes the |
3059 | * check with the clearing of time_stamp and movement of i |
3060 | */ |
3061 | clear_bit(nr: IGC_RING_FLAG_TX_DETECT_HANG, addr: &tx_ring->flags); |
3062 | if (tx_buffer->next_to_watch && |
3063 | time_after(jiffies, tx_buffer->time_stamp + |
3064 | (adapter->tx_timeout_factor * HZ)) && |
3065 | !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF) && |
3066 | (rd32(IGC_TDH(tx_ring->reg_idx)) != readl(addr: tx_ring->tail)) && |
3067 | !tx_ring->oper_gate_closed) { |
3068 | /* detected Tx unit hang */ |
3069 | netdev_err(dev: tx_ring->netdev, |
3070 | format: "Detected Tx Unit Hang\n" |
3071 | " Tx Queue <%d>\n" |
3072 | " TDH <%x>\n" |
3073 | " TDT <%x>\n" |
3074 | " next_to_use <%x>\n" |
3075 | " next_to_clean <%x>\n" |
3076 | "buffer_info[next_to_clean]\n" |
3077 | " time_stamp <%lx>\n" |
3078 | " next_to_watch <%p>\n" |
3079 | " jiffies <%lx>\n" |
3080 | " desc.status <%x>\n" , |
3081 | tx_ring->queue_index, |
3082 | rd32(IGC_TDH(tx_ring->reg_idx)), |
3083 | readl(addr: tx_ring->tail), |
3084 | tx_ring->next_to_use, |
3085 | tx_ring->next_to_clean, |
3086 | tx_buffer->time_stamp, |
3087 | tx_buffer->next_to_watch, |
3088 | jiffies, |
3089 | tx_buffer->next_to_watch->wb.status); |
3090 | netif_stop_subqueue(dev: tx_ring->netdev, |
3091 | queue_index: tx_ring->queue_index); |
3092 | |
3093 | /* we are about to reset, no point in enabling stuff */ |
3094 | return true; |
3095 | } |
3096 | } |
3097 | |
3098 | #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) |
3099 | if (unlikely(total_packets && |
3100 | netif_carrier_ok(tx_ring->netdev) && |
3101 | igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { |
3102 | /* Make sure that anybody stopping the queue after this |
3103 | * sees the new next_to_clean. |
3104 | */ |
3105 | smp_mb(); |
3106 | if (__netif_subqueue_stopped(dev: tx_ring->netdev, |
3107 | queue_index: tx_ring->queue_index) && |
3108 | !(test_bit(__IGC_DOWN, &adapter->state))) { |
3109 | netif_wake_subqueue(dev: tx_ring->netdev, |
3110 | queue_index: tx_ring->queue_index); |
3111 | |
3112 | u64_stats_update_begin(syncp: &tx_ring->tx_syncp); |
3113 | tx_ring->tx_stats.restart_queue++; |
3114 | u64_stats_update_end(syncp: &tx_ring->tx_syncp); |
3115 | } |
3116 | } |
3117 | |
3118 | return !!budget; |
3119 | } |
3120 | |
3121 | static int igc_find_mac_filter(struct igc_adapter *adapter, |
3122 | enum igc_mac_filter_type type, const u8 *addr) |
3123 | { |
3124 | struct igc_hw *hw = &adapter->hw; |
3125 | int max_entries = hw->mac.rar_entry_count; |
3126 | u32 ral, rah; |
3127 | int i; |
3128 | |
3129 | for (i = 0; i < max_entries; i++) { |
3130 | ral = rd32(IGC_RAL(i)); |
3131 | rah = rd32(IGC_RAH(i)); |
3132 | |
3133 | if (!(rah & IGC_RAH_AV)) |
3134 | continue; |
3135 | if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type) |
3136 | continue; |
3137 | if ((rah & IGC_RAH_RAH_MASK) != |
3138 | le16_to_cpup(p: (__le16 *)(addr + 4))) |
3139 | continue; |
3140 | if (ral != le32_to_cpup(p: (__le32 *)(addr))) |
3141 | continue; |
3142 | |
3143 | return i; |
3144 | } |
3145 | |
3146 | return -1; |
3147 | } |
3148 | |
3149 | static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter) |
3150 | { |
3151 | struct igc_hw *hw = &adapter->hw; |
3152 | int max_entries = hw->mac.rar_entry_count; |
3153 | u32 rah; |
3154 | int i; |
3155 | |
3156 | for (i = 0; i < max_entries; i++) { |
3157 | rah = rd32(IGC_RAH(i)); |
3158 | |
3159 | if (!(rah & IGC_RAH_AV)) |
3160 | return i; |
3161 | } |
3162 | |
3163 | return -1; |
3164 | } |
3165 | |
3166 | /** |
3167 | * igc_add_mac_filter() - Add MAC address filter |
3168 | * @adapter: Pointer to adapter where the filter should be added |
3169 | * @type: MAC address filter type (source or destination) |
3170 | * @addr: MAC address |
3171 | * @queue: If non-negative, queue assignment feature is enabled and frames |
3172 | * matching the filter are enqueued onto 'queue'. Otherwise, queue |
3173 | * assignment is disabled. |
3174 | * |
3175 | * Return: 0 in case of success, negative errno code otherwise. |
3176 | */ |
3177 | static int igc_add_mac_filter(struct igc_adapter *adapter, |
3178 | enum igc_mac_filter_type type, const u8 *addr, |
3179 | int queue) |
3180 | { |
3181 | struct net_device *dev = adapter->netdev; |
3182 | int index; |
3183 | |
3184 | index = igc_find_mac_filter(adapter, type, addr); |
3185 | if (index >= 0) |
3186 | goto update_filter; |
3187 | |
3188 | index = igc_get_avail_mac_filter_slot(adapter); |
3189 | if (index < 0) |
3190 | return -ENOSPC; |
3191 | |
3192 | netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n" , |
3193 | index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src" , |
3194 | addr, queue); |
3195 | |
3196 | update_filter: |
3197 | igc_set_mac_filter_hw(adapter, index, type, addr, queue); |
3198 | return 0; |
3199 | } |
3200 | |
3201 | /** |
3202 | * igc_del_mac_filter() - Delete MAC address filter |
3203 | * @adapter: Pointer to adapter where the filter should be deleted from |
3204 | * @type: MAC address filter type (source or destination) |
3205 | * @addr: MAC address |
3206 | */ |
3207 | static void igc_del_mac_filter(struct igc_adapter *adapter, |
3208 | enum igc_mac_filter_type type, const u8 *addr) |
3209 | { |
3210 | struct net_device *dev = adapter->netdev; |
3211 | int index; |
3212 | |
3213 | index = igc_find_mac_filter(adapter, type, addr); |
3214 | if (index < 0) |
3215 | return; |
3216 | |
3217 | if (index == 0) { |
3218 | /* If this is the default filter, we don't actually delete it. |
3219 | * We just reset to its default value i.e. disable queue |
3220 | * assignment. |
3221 | */ |
3222 | netdev_dbg(dev, "Disable default MAC filter queue assignment" ); |
3223 | |
3224 | igc_set_mac_filter_hw(adapter, index: 0, type, addr, queue: -1); |
3225 | } else { |
3226 | netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n" , |
3227 | index, |
3228 | type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src" , |
3229 | addr); |
3230 | |
3231 | igc_clear_mac_filter_hw(adapter, index); |
3232 | } |
3233 | } |
3234 | |
3235 | /** |
3236 | * igc_add_vlan_prio_filter() - Add VLAN priority filter |
3237 | * @adapter: Pointer to adapter where the filter should be added |
3238 | * @prio: VLAN priority value |
3239 | * @queue: Queue number which matching frames are assigned to |
3240 | * |
3241 | * Return: 0 in case of success, negative errno code otherwise. |
3242 | */ |
3243 | static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio, |
3244 | int queue) |
3245 | { |
3246 | struct net_device *dev = adapter->netdev; |
3247 | struct igc_hw *hw = &adapter->hw; |
3248 | u32 vlanpqf; |
3249 | |
3250 | vlanpqf = rd32(IGC_VLANPQF); |
3251 | |
3252 | if (vlanpqf & IGC_VLANPQF_VALID(prio)) { |
3253 | netdev_dbg(dev, "VLAN priority filter already in use\n" ); |
3254 | return -EEXIST; |
3255 | } |
3256 | |
3257 | vlanpqf |= IGC_VLANPQF_QSEL(prio, queue); |
3258 | vlanpqf |= IGC_VLANPQF_VALID(prio); |
3259 | |
3260 | wr32(IGC_VLANPQF, vlanpqf); |
3261 | |
3262 | netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n" , |
3263 | prio, queue); |
3264 | return 0; |
3265 | } |
3266 | |
3267 | /** |
3268 | * igc_del_vlan_prio_filter() - Delete VLAN priority filter |
3269 | * @adapter: Pointer to adapter where the filter should be deleted from |
3270 | * @prio: VLAN priority value |
3271 | */ |
3272 | static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio) |
3273 | { |
3274 | struct igc_hw *hw = &adapter->hw; |
3275 | u32 vlanpqf; |
3276 | |
3277 | vlanpqf = rd32(IGC_VLANPQF); |
3278 | |
3279 | vlanpqf &= ~IGC_VLANPQF_VALID(prio); |
3280 | vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK); |
3281 | |
3282 | wr32(IGC_VLANPQF, vlanpqf); |
3283 | |
3284 | netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n" , |
3285 | prio); |
3286 | } |
3287 | |
3288 | static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter) |
3289 | { |
3290 | struct igc_hw *hw = &adapter->hw; |
3291 | int i; |
3292 | |
3293 | for (i = 0; i < MAX_ETYPE_FILTER; i++) { |
3294 | u32 etqf = rd32(IGC_ETQF(i)); |
3295 | |
3296 | if (!(etqf & IGC_ETQF_FILTER_ENABLE)) |
3297 | return i; |
3298 | } |
3299 | |
3300 | return -1; |
3301 | } |
3302 | |
3303 | /** |
3304 | * igc_add_etype_filter() - Add ethertype filter |
3305 | * @adapter: Pointer to adapter where the filter should be added |
3306 | * @etype: Ethertype value |
3307 | * @queue: If non-negative, queue assignment feature is enabled and frames |
3308 | * matching the filter are enqueued onto 'queue'. Otherwise, queue |
3309 | * assignment is disabled. |
3310 | * |
3311 | * Return: 0 in case of success, negative errno code otherwise. |
3312 | */ |
3313 | static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype, |
3314 | int queue) |
3315 | { |
3316 | struct igc_hw *hw = &adapter->hw; |
3317 | int index; |
3318 | u32 etqf; |
3319 | |
3320 | index = igc_get_avail_etype_filter_slot(adapter); |
3321 | if (index < 0) |
3322 | return -ENOSPC; |
3323 | |
3324 | etqf = rd32(IGC_ETQF(index)); |
3325 | |
3326 | etqf &= ~IGC_ETQF_ETYPE_MASK; |
3327 | etqf |= etype; |
3328 | |
3329 | if (queue >= 0) { |
3330 | etqf &= ~IGC_ETQF_QUEUE_MASK; |
3331 | etqf |= (queue << IGC_ETQF_QUEUE_SHIFT); |
3332 | etqf |= IGC_ETQF_QUEUE_ENABLE; |
3333 | } |
3334 | |
3335 | etqf |= IGC_ETQF_FILTER_ENABLE; |
3336 | |
3337 | wr32(IGC_ETQF(index), etqf); |
3338 | |
3339 | netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n" , |
3340 | etype, queue); |
3341 | return 0; |
3342 | } |
3343 | |
3344 | static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype) |
3345 | { |
3346 | struct igc_hw *hw = &adapter->hw; |
3347 | int i; |
3348 | |
3349 | for (i = 0; i < MAX_ETYPE_FILTER; i++) { |
3350 | u32 etqf = rd32(IGC_ETQF(i)); |
3351 | |
3352 | if ((etqf & IGC_ETQF_ETYPE_MASK) == etype) |
3353 | return i; |
3354 | } |
3355 | |
3356 | return -1; |
3357 | } |
3358 | |
3359 | /** |
3360 | * igc_del_etype_filter() - Delete ethertype filter |
3361 | * @adapter: Pointer to adapter where the filter should be deleted from |
3362 | * @etype: Ethertype value |
3363 | */ |
3364 | static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype) |
3365 | { |
3366 | struct igc_hw *hw = &adapter->hw; |
3367 | int index; |
3368 | |
3369 | index = igc_find_etype_filter(adapter, etype); |
3370 | if (index < 0) |
3371 | return; |
3372 | |
3373 | wr32(IGC_ETQF(index), 0); |
3374 | |
3375 | netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n" , |
3376 | etype); |
3377 | } |
3378 | |
3379 | static int igc_flex_filter_select(struct igc_adapter *adapter, |
3380 | struct igc_flex_filter *input, |
3381 | u32 *fhft) |
3382 | { |
3383 | struct igc_hw *hw = &adapter->hw; |
3384 | u8 fhft_index; |
3385 | u32 fhftsl; |
3386 | |
3387 | if (input->index >= MAX_FLEX_FILTER) { |
3388 | dev_err(&adapter->pdev->dev, "Wrong Flex Filter index selected!\n" ); |
3389 | return -EINVAL; |
3390 | } |
3391 | |
3392 | /* Indirect table select register */ |
3393 | fhftsl = rd32(IGC_FHFTSL); |
3394 | fhftsl &= ~IGC_FHFTSL_FTSL_MASK; |
3395 | switch (input->index) { |
3396 | case 0 ... 7: |
3397 | fhftsl |= 0x00; |
3398 | break; |
3399 | case 8 ... 15: |
3400 | fhftsl |= 0x01; |
3401 | break; |
3402 | case 16 ... 23: |
3403 | fhftsl |= 0x02; |
3404 | break; |
3405 | case 24 ... 31: |
3406 | fhftsl |= 0x03; |
3407 | break; |
3408 | } |
3409 | wr32(IGC_FHFTSL, fhftsl); |
3410 | |
3411 | /* Normalize index down to host table register */ |
3412 | fhft_index = input->index % 8; |
3413 | |
3414 | *fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) : |
3415 | IGC_FHFT_EXT(fhft_index - 4); |
3416 | |
3417 | return 0; |
3418 | } |
3419 | |
3420 | static int igc_write_flex_filter_ll(struct igc_adapter *adapter, |
3421 | struct igc_flex_filter *input) |
3422 | { |
3423 | struct device *dev = &adapter->pdev->dev; |
3424 | struct igc_hw *hw = &adapter->hw; |
3425 | u8 *data = input->data; |
3426 | u8 *mask = input->mask; |
3427 | u32 queuing; |
3428 | u32 fhft; |
3429 | u32 wufc; |
3430 | int ret; |
3431 | int i; |
3432 | |
3433 | /* Length has to be aligned to 8. Otherwise the filter will fail. Bail |
3434 | * out early to avoid surprises later. |
3435 | */ |
3436 | if (input->length % 8 != 0) { |
3437 | dev_err(dev, "The length of a flex filter has to be 8 byte aligned!\n" ); |
3438 | return -EINVAL; |
3439 | } |
3440 | |
3441 | /* Select corresponding flex filter register and get base for host table. */ |
3442 | ret = igc_flex_filter_select(adapter, input, fhft: &fhft); |
3443 | if (ret) |
3444 | return ret; |
3445 | |
3446 | /* When adding a filter globally disable flex filter feature. That is |
3447 | * recommended within the datasheet. |
3448 | */ |
3449 | wufc = rd32(IGC_WUFC); |
3450 | wufc &= ~IGC_WUFC_FLEX_HQ; |
3451 | wr32(IGC_WUFC, wufc); |
3452 | |
3453 | /* Configure filter */ |
3454 | queuing = input->length & IGC_FHFT_LENGTH_MASK; |
3455 | queuing |= (input->rx_queue << IGC_FHFT_QUEUE_SHIFT) & IGC_FHFT_QUEUE_MASK; |
3456 | queuing |= (input->prio << IGC_FHFT_PRIO_SHIFT) & IGC_FHFT_PRIO_MASK; |
3457 | |
3458 | if (input->immediate_irq) |
3459 | queuing |= IGC_FHFT_IMM_INT; |
3460 | |
3461 | if (input->drop) |
3462 | queuing |= IGC_FHFT_DROP; |
3463 | |
3464 | wr32(fhft + 0xFC, queuing); |
3465 | |
3466 | /* Write data (128 byte) and mask (128 bit) */ |
3467 | for (i = 0; i < 16; ++i) { |
3468 | const size_t data_idx = i * 8; |
3469 | const size_t row_idx = i * 16; |
3470 | u32 dw0 = |
3471 | (data[data_idx + 0] << 0) | |
3472 | (data[data_idx + 1] << 8) | |
3473 | (data[data_idx + 2] << 16) | |
3474 | (data[data_idx + 3] << 24); |
3475 | u32 dw1 = |
3476 | (data[data_idx + 4] << 0) | |
3477 | (data[data_idx + 5] << 8) | |
3478 | (data[data_idx + 6] << 16) | |
3479 | (data[data_idx + 7] << 24); |
3480 | u32 tmp; |
3481 | |
3482 | /* Write row: dw0, dw1 and mask */ |
3483 | wr32(fhft + row_idx, dw0); |
3484 | wr32(fhft + row_idx + 4, dw1); |
3485 | |
3486 | /* mask is only valid for MASK(7, 0) */ |
3487 | tmp = rd32(fhft + row_idx + 8); |
3488 | tmp &= ~GENMASK(7, 0); |
3489 | tmp |= mask[i]; |
3490 | wr32(fhft + row_idx + 8, tmp); |
3491 | } |
3492 | |
3493 | /* Enable filter. */ |
3494 | wufc |= IGC_WUFC_FLEX_HQ; |
3495 | if (input->index > 8) { |
3496 | /* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */ |
3497 | u32 wufc_ext = rd32(IGC_WUFC_EXT); |
3498 | |
3499 | wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8)); |
3500 | |
3501 | wr32(IGC_WUFC_EXT, wufc_ext); |
3502 | } else { |
3503 | wufc |= (IGC_WUFC_FLX0 << input->index); |
3504 | } |
3505 | wr32(IGC_WUFC, wufc); |
3506 | |
3507 | dev_dbg(&adapter->pdev->dev, "Added flex filter %u to HW.\n" , |
3508 | input->index); |
3509 | |
3510 | return 0; |
3511 | } |
3512 | |
3513 | static void igc_flex_filter_add_field(struct igc_flex_filter *flex, |
3514 | const void *src, unsigned int offset, |
3515 | size_t len, const void *mask) |
3516 | { |
3517 | int i; |
3518 | |
3519 | /* data */ |
3520 | memcpy(&flex->data[offset], src, len); |
3521 | |
3522 | /* mask */ |
3523 | for (i = 0; i < len; ++i) { |
3524 | const unsigned int idx = i + offset; |
3525 | const u8 *ptr = mask; |
3526 | |
3527 | if (mask) { |
3528 | if (ptr[i] & 0xff) |
3529 | flex->mask[idx / 8] |= BIT(idx % 8); |
3530 | |
3531 | continue; |
3532 | } |
3533 | |
3534 | flex->mask[idx / 8] |= BIT(idx % 8); |
3535 | } |
3536 | } |
3537 | |
3538 | static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter) |
3539 | { |
3540 | struct igc_hw *hw = &adapter->hw; |
3541 | u32 wufc, wufc_ext; |
3542 | int i; |
3543 | |
3544 | wufc = rd32(IGC_WUFC); |
3545 | wufc_ext = rd32(IGC_WUFC_EXT); |
3546 | |
3547 | for (i = 0; i < MAX_FLEX_FILTER; i++) { |
3548 | if (i < 8) { |
3549 | if (!(wufc & (IGC_WUFC_FLX0 << i))) |
3550 | return i; |
3551 | } else { |
3552 | if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8)))) |
3553 | return i; |
3554 | } |
3555 | } |
3556 | |
3557 | return -ENOSPC; |
3558 | } |
3559 | |
3560 | static bool igc_flex_filter_in_use(struct igc_adapter *adapter) |
3561 | { |
3562 | struct igc_hw *hw = &adapter->hw; |
3563 | u32 wufc, wufc_ext; |
3564 | |
3565 | wufc = rd32(IGC_WUFC); |
3566 | wufc_ext = rd32(IGC_WUFC_EXT); |
3567 | |
3568 | if (wufc & IGC_WUFC_FILTER_MASK) |
3569 | return true; |
3570 | |
3571 | if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK) |
3572 | return true; |
3573 | |
3574 | return false; |
3575 | } |
3576 | |
3577 | static int igc_add_flex_filter(struct igc_adapter *adapter, |
3578 | struct igc_nfc_rule *rule) |
3579 | { |
3580 | struct igc_flex_filter flex = { }; |
3581 | struct igc_nfc_filter *filter = &rule->filter; |
3582 | unsigned int eth_offset, user_offset; |
3583 | int ret, index; |
3584 | bool vlan; |
3585 | |
3586 | index = igc_find_avail_flex_filter_slot(adapter); |
3587 | if (index < 0) |
3588 | return -ENOSPC; |
3589 | |
3590 | /* Construct the flex filter: |
3591 | * -> dest_mac [6] |
3592 | * -> src_mac [6] |
3593 | * -> tpid [2] |
3594 | * -> vlan tci [2] |
3595 | * -> ether type [2] |
3596 | * -> user data [8] |
3597 | * -> = 26 bytes => 32 length |
3598 | */ |
3599 | flex.index = index; |
3600 | flex.length = 32; |
3601 | flex.rx_queue = rule->action; |
3602 | |
3603 | vlan = rule->filter.vlan_tci || rule->filter.vlan_etype; |
3604 | eth_offset = vlan ? 16 : 12; |
3605 | user_offset = vlan ? 18 : 14; |
3606 | |
3607 | /* Add destination MAC */ |
3608 | if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) |
3609 | igc_flex_filter_add_field(flex: &flex, src: &filter->dst_addr, offset: 0, |
3610 | ETH_ALEN, NULL); |
3611 | |
3612 | /* Add source MAC */ |
3613 | if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) |
3614 | igc_flex_filter_add_field(flex: &flex, src: &filter->src_addr, offset: 6, |
3615 | ETH_ALEN, NULL); |
3616 | |
3617 | /* Add VLAN etype */ |
3618 | if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE) |
3619 | igc_flex_filter_add_field(flex: &flex, src: &filter->vlan_etype, offset: 12, |
3620 | len: sizeof(filter->vlan_etype), |
3621 | NULL); |
3622 | |
3623 | /* Add VLAN TCI */ |
3624 | if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) |
3625 | igc_flex_filter_add_field(flex: &flex, src: &filter->vlan_tci, offset: 14, |
3626 | len: sizeof(filter->vlan_tci), NULL); |
3627 | |
3628 | /* Add Ether type */ |
3629 | if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { |
3630 | __be16 etype = cpu_to_be16(filter->etype); |
3631 | |
3632 | igc_flex_filter_add_field(flex: &flex, src: &etype, offset: eth_offset, |
3633 | len: sizeof(etype), NULL); |
3634 | } |
3635 | |
3636 | /* Add user data */ |
3637 | if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA) |
3638 | igc_flex_filter_add_field(flex: &flex, src: &filter->user_data, |
3639 | offset: user_offset, |
3640 | len: sizeof(filter->user_data), |
3641 | mask: filter->user_mask); |
3642 | |
3643 | /* Add it down to the hardware and enable it. */ |
3644 | ret = igc_write_flex_filter_ll(adapter, input: &flex); |
3645 | if (ret) |
3646 | return ret; |
3647 | |
3648 | filter->flex_index = index; |
3649 | |
3650 | return 0; |
3651 | } |
3652 | |
3653 | static void igc_del_flex_filter(struct igc_adapter *adapter, |
3654 | u16 reg_index) |
3655 | { |
3656 | struct igc_hw *hw = &adapter->hw; |
3657 | u32 wufc; |
3658 | |
3659 | /* Just disable the filter. The filter table itself is kept |
3660 | * intact. Another flex_filter_add() should override the "old" data |
3661 | * then. |
3662 | */ |
3663 | if (reg_index > 8) { |
3664 | u32 wufc_ext = rd32(IGC_WUFC_EXT); |
3665 | |
3666 | wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8)); |
3667 | wr32(IGC_WUFC_EXT, wufc_ext); |
3668 | } else { |
3669 | wufc = rd32(IGC_WUFC); |
3670 | |
3671 | wufc &= ~(IGC_WUFC_FLX0 << reg_index); |
3672 | wr32(IGC_WUFC, wufc); |
3673 | } |
3674 | |
3675 | if (igc_flex_filter_in_use(adapter)) |
3676 | return; |
3677 | |
3678 | /* No filters are in use, we may disable flex filters */ |
3679 | wufc = rd32(IGC_WUFC); |
3680 | wufc &= ~IGC_WUFC_FLEX_HQ; |
3681 | wr32(IGC_WUFC, wufc); |
3682 | } |
3683 | |
3684 | static int igc_enable_nfc_rule(struct igc_adapter *adapter, |
3685 | struct igc_nfc_rule *rule) |
3686 | { |
3687 | int err; |
3688 | |
3689 | if (rule->flex) { |
3690 | return igc_add_flex_filter(adapter, rule); |
3691 | } |
3692 | |
3693 | if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) { |
3694 | err = igc_add_etype_filter(adapter, etype: rule->filter.etype, |
3695 | queue: rule->action); |
3696 | if (err) |
3697 | return err; |
3698 | } |
3699 | |
3700 | if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) { |
3701 | err = igc_add_mac_filter(adapter, type: IGC_MAC_FILTER_TYPE_SRC, |
3702 | addr: rule->filter.src_addr, queue: rule->action); |
3703 | if (err) |
3704 | return err; |
3705 | } |
3706 | |
3707 | if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) { |
3708 | err = igc_add_mac_filter(adapter, type: IGC_MAC_FILTER_TYPE_DST, |
3709 | addr: rule->filter.dst_addr, queue: rule->action); |
3710 | if (err) |
3711 | return err; |
3712 | } |
3713 | |
3714 | if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { |
3715 | int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> |
3716 | VLAN_PRIO_SHIFT; |
3717 | |
3718 | err = igc_add_vlan_prio_filter(adapter, prio, queue: rule->action); |
3719 | if (err) |
3720 | return err; |
3721 | } |
3722 | |
3723 | return 0; |
3724 | } |
3725 | |
3726 | static void igc_disable_nfc_rule(struct igc_adapter *adapter, |
3727 | const struct igc_nfc_rule *rule) |
3728 | { |
3729 | if (rule->flex) { |
3730 | igc_del_flex_filter(adapter, reg_index: rule->filter.flex_index); |
3731 | return; |
3732 | } |
3733 | |
3734 | if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) |
3735 | igc_del_etype_filter(adapter, etype: rule->filter.etype); |
3736 | |
3737 | if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) { |
3738 | int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >> |
3739 | VLAN_PRIO_SHIFT; |
3740 | |
3741 | igc_del_vlan_prio_filter(adapter, prio); |
3742 | } |
3743 | |
3744 | if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) |
3745 | igc_del_mac_filter(adapter, type: IGC_MAC_FILTER_TYPE_SRC, |
3746 | addr: rule->filter.src_addr); |
3747 | |
3748 | if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) |
3749 | igc_del_mac_filter(adapter, type: IGC_MAC_FILTER_TYPE_DST, |
3750 | addr: rule->filter.dst_addr); |
3751 | } |
3752 | |
3753 | /** |
3754 | * igc_get_nfc_rule() - Get NFC rule |
3755 | * @adapter: Pointer to adapter |
3756 | * @location: Rule location |
3757 | * |
3758 | * Context: Expects adapter->nfc_rule_lock to be held by caller. |
3759 | * |
3760 | * Return: Pointer to NFC rule at @location. If not found, NULL. |
3761 | */ |
3762 | struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter, |
3763 | u32 location) |
3764 | { |
3765 | struct igc_nfc_rule *rule; |
3766 | |
3767 | list_for_each_entry(rule, &adapter->nfc_rule_list, list) { |
3768 | if (rule->location == location) |
3769 | return rule; |
3770 | if (rule->location > location) |
3771 | break; |
3772 | } |
3773 | |
3774 | return NULL; |
3775 | } |
3776 | |
3777 | /** |
3778 | * igc_del_nfc_rule() - Delete NFC rule |
3779 | * @adapter: Pointer to adapter |
3780 | * @rule: Pointer to rule to be deleted |
3781 | * |
3782 | * Disable NFC rule in hardware and delete it from adapter. |
3783 | * |
3784 | * Context: Expects adapter->nfc_rule_lock to be held by caller. |
3785 | */ |
3786 | void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) |
3787 | { |
3788 | igc_disable_nfc_rule(adapter, rule); |
3789 | |
3790 | list_del(entry: &rule->list); |
3791 | adapter->nfc_rule_count--; |
3792 | |
3793 | kfree(objp: rule); |
3794 | } |
3795 | |
3796 | static void igc_flush_nfc_rules(struct igc_adapter *adapter) |
3797 | { |
3798 | struct igc_nfc_rule *rule, *tmp; |
3799 | |
3800 | mutex_lock(&adapter->nfc_rule_lock); |
3801 | |
3802 | list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list) |
3803 | igc_del_nfc_rule(adapter, rule); |
3804 | |
3805 | mutex_unlock(lock: &adapter->nfc_rule_lock); |
3806 | } |
3807 | |
3808 | /** |
3809 | * igc_add_nfc_rule() - Add NFC rule |
3810 | * @adapter: Pointer to adapter |
3811 | * @rule: Pointer to rule to be added |
3812 | * |
3813 | * Enable NFC rule in hardware and add it to adapter. |
3814 | * |
3815 | * Context: Expects adapter->nfc_rule_lock to be held by caller. |
3816 | * |
3817 | * Return: 0 on success, negative errno on failure. |
3818 | */ |
3819 | int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule) |
3820 | { |
3821 | struct igc_nfc_rule *pred, *cur; |
3822 | int err; |
3823 | |
3824 | err = igc_enable_nfc_rule(adapter, rule); |
3825 | if (err) |
3826 | return err; |
3827 | |
3828 | pred = NULL; |
3829 | list_for_each_entry(cur, &adapter->nfc_rule_list, list) { |
3830 | if (cur->location >= rule->location) |
3831 | break; |
3832 | pred = cur; |
3833 | } |
3834 | |
3835 | list_add(new: &rule->list, head: pred ? &pred->list : &adapter->nfc_rule_list); |
3836 | adapter->nfc_rule_count++; |
3837 | return 0; |
3838 | } |
3839 | |
3840 | static void igc_restore_nfc_rules(struct igc_adapter *adapter) |
3841 | { |
3842 | struct igc_nfc_rule *rule; |
3843 | |
3844 | mutex_lock(&adapter->nfc_rule_lock); |
3845 | |
3846 | list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list) |
3847 | igc_enable_nfc_rule(adapter, rule); |
3848 | |
3849 | mutex_unlock(lock: &adapter->nfc_rule_lock); |
3850 | } |
3851 | |
3852 | static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) |
3853 | { |
3854 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
3855 | |
3856 | return igc_add_mac_filter(adapter, type: IGC_MAC_FILTER_TYPE_DST, addr, queue: -1); |
3857 | } |
3858 | |
3859 | static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) |
3860 | { |
3861 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
3862 | |
3863 | igc_del_mac_filter(adapter, type: IGC_MAC_FILTER_TYPE_DST, addr); |
3864 | return 0; |
3865 | } |
3866 | |
3867 | /** |
3868 | * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set |
3869 | * @netdev: network interface device structure |
3870 | * |
3871 | * The set_rx_mode entry point is called whenever the unicast or multicast |
3872 | * address lists or the network interface flags are updated. This routine is |
3873 | * responsible for configuring the hardware for proper unicast, multicast, |
3874 | * promiscuous mode, and all-multi behavior. |
3875 | */ |
3876 | static void igc_set_rx_mode(struct net_device *netdev) |
3877 | { |
3878 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
3879 | struct igc_hw *hw = &adapter->hw; |
3880 | u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; |
3881 | int count; |
3882 | |
3883 | /* Check for Promiscuous and All Multicast modes */ |
3884 | if (netdev->flags & IFF_PROMISC) { |
3885 | rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; |
3886 | } else { |
3887 | if (netdev->flags & IFF_ALLMULTI) { |
3888 | rctl |= IGC_RCTL_MPE; |
3889 | } else { |
3890 | /* Write addresses to the MTA, if the attempt fails |
3891 | * then we should just turn on promiscuous mode so |
3892 | * that we can at least receive multicast traffic |
3893 | */ |
3894 | count = igc_write_mc_addr_list(netdev); |
3895 | if (count < 0) |
3896 | rctl |= IGC_RCTL_MPE; |
3897 | } |
3898 | } |
3899 | |
3900 | /* Write addresses to available RAR registers, if there is not |
3901 | * sufficient space to store all the addresses then enable |
3902 | * unicast promiscuous mode |
3903 | */ |
3904 | if (__dev_uc_sync(dev: netdev, sync: igc_uc_sync, unsync: igc_uc_unsync)) |
3905 | rctl |= IGC_RCTL_UPE; |
3906 | |
3907 | /* update state of unicast and multicast */ |
3908 | rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); |
3909 | wr32(IGC_RCTL, rctl); |
3910 | |
3911 | #if (PAGE_SIZE < 8192) |
3912 | if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) |
3913 | rlpml = IGC_MAX_FRAME_BUILD_SKB; |
3914 | #endif |
3915 | wr32(IGC_RLPML, rlpml); |
3916 | } |
3917 | |
3918 | /** |
3919 | * igc_configure - configure the hardware for RX and TX |
3920 | * @adapter: private board structure |
3921 | */ |
3922 | static void igc_configure(struct igc_adapter *adapter) |
3923 | { |
3924 | struct net_device *netdev = adapter->netdev; |
3925 | int i = 0; |
3926 | |
3927 | igc_get_hw_control(adapter); |
3928 | igc_set_rx_mode(netdev); |
3929 | |
3930 | igc_restore_vlan(adapter); |
3931 | |
3932 | igc_setup_tctl(adapter); |
3933 | igc_setup_mrqc(adapter); |
3934 | igc_setup_rctl(adapter); |
3935 | |
3936 | igc_set_default_mac_filter(adapter); |
3937 | igc_restore_nfc_rules(adapter); |
3938 | |
3939 | igc_configure_tx(adapter); |
3940 | igc_configure_rx(adapter); |
3941 | |
3942 | igc_rx_fifo_flush_base(hw: &adapter->hw); |
3943 | |
3944 | /* call igc_desc_unused which always leaves |
3945 | * at least 1 descriptor unused to make sure |
3946 | * next_to_use != next_to_clean |
3947 | */ |
3948 | for (i = 0; i < adapter->num_rx_queues; i++) { |
3949 | struct igc_ring *ring = adapter->rx_ring[i]; |
3950 | |
3951 | if (ring->xsk_pool) |
3952 | igc_alloc_rx_buffers_zc(ring, count: igc_desc_unused(ring)); |
3953 | else |
3954 | igc_alloc_rx_buffers(rx_ring: ring, cleaned_count: igc_desc_unused(ring)); |
3955 | } |
3956 | } |
3957 | |
3958 | /** |
3959 | * igc_write_ivar - configure ivar for given MSI-X vector |
3960 | * @hw: pointer to the HW structure |
3961 | * @msix_vector: vector number we are allocating to a given ring |
3962 | * @index: row index of IVAR register to write within IVAR table |
3963 | * @offset: column offset of in IVAR, should be multiple of 8 |
3964 | * |
3965 | * The IVAR table consists of 2 columns, |
3966 | * each containing an cause allocation for an Rx and Tx ring, and a |
3967 | * variable number of rows depending on the number of queues supported. |
3968 | */ |
3969 | static void igc_write_ivar(struct igc_hw *hw, int msix_vector, |
3970 | int index, int offset) |
3971 | { |
3972 | u32 ivar = array_rd32(IGC_IVAR0, index); |
3973 | |
3974 | /* clear any bits that are currently set */ |
3975 | ivar &= ~((u32)0xFF << offset); |
3976 | |
3977 | /* write vector and valid bit */ |
3978 | ivar |= (msix_vector | IGC_IVAR_VALID) << offset; |
3979 | |
3980 | array_wr32(IGC_IVAR0, index, ivar); |
3981 | } |
3982 | |
3983 | static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) |
3984 | { |
3985 | struct igc_adapter *adapter = q_vector->adapter; |
3986 | struct igc_hw *hw = &adapter->hw; |
3987 | int rx_queue = IGC_N0_QUEUE; |
3988 | int tx_queue = IGC_N0_QUEUE; |
3989 | |
3990 | if (q_vector->rx.ring) |
3991 | rx_queue = q_vector->rx.ring->reg_idx; |
3992 | if (q_vector->tx.ring) |
3993 | tx_queue = q_vector->tx.ring->reg_idx; |
3994 | |
3995 | switch (hw->mac.type) { |
3996 | case igc_i225: |
3997 | if (rx_queue > IGC_N0_QUEUE) |
3998 | igc_write_ivar(hw, msix_vector, |
3999 | index: rx_queue >> 1, |
4000 | offset: (rx_queue & 0x1) << 4); |
4001 | if (tx_queue > IGC_N0_QUEUE) |
4002 | igc_write_ivar(hw, msix_vector, |
4003 | index: tx_queue >> 1, |
4004 | offset: ((tx_queue & 0x1) << 4) + 8); |
4005 | q_vector->eims_value = BIT(msix_vector); |
4006 | break; |
4007 | default: |
4008 | WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n" ); |
4009 | break; |
4010 | } |
4011 | |
4012 | /* add q_vector eims value to global eims_enable_mask */ |
4013 | adapter->eims_enable_mask |= q_vector->eims_value; |
4014 | |
4015 | /* configure q_vector to set itr on first interrupt */ |
4016 | q_vector->set_itr = 1; |
4017 | } |
4018 | |
4019 | /** |
4020 | * igc_configure_msix - Configure MSI-X hardware |
4021 | * @adapter: Pointer to adapter structure |
4022 | * |
4023 | * igc_configure_msix sets up the hardware to properly |
4024 | * generate MSI-X interrupts. |
4025 | */ |
4026 | static void igc_configure_msix(struct igc_adapter *adapter) |
4027 | { |
4028 | struct igc_hw *hw = &adapter->hw; |
4029 | int i, vector = 0; |
4030 | u32 tmp; |
4031 | |
4032 | adapter->eims_enable_mask = 0; |
4033 | |
4034 | /* set vector for other causes, i.e. link changes */ |
4035 | switch (hw->mac.type) { |
4036 | case igc_i225: |
4037 | /* Turn on MSI-X capability first, or our settings |
4038 | * won't stick. And it will take days to debug. |
4039 | */ |
4040 | wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | |
4041 | IGC_GPIE_PBA | IGC_GPIE_EIAME | |
4042 | IGC_GPIE_NSICR); |
4043 | |
4044 | /* enable msix_other interrupt */ |
4045 | adapter->eims_other = BIT(vector); |
4046 | tmp = (vector++ | IGC_IVAR_VALID) << 8; |
4047 | |
4048 | wr32(IGC_IVAR_MISC, tmp); |
4049 | break; |
4050 | default: |
4051 | /* do nothing, since nothing else supports MSI-X */ |
4052 | break; |
4053 | } /* switch (hw->mac.type) */ |
4054 | |
4055 | adapter->eims_enable_mask |= adapter->eims_other; |
4056 | |
4057 | for (i = 0; i < adapter->num_q_vectors; i++) |
4058 | igc_assign_vector(q_vector: adapter->q_vector[i], msix_vector: vector++); |
4059 | |
4060 | wrfl(); |
4061 | } |
4062 | |
4063 | /** |
4064 | * igc_irq_enable - Enable default interrupt generation settings |
4065 | * @adapter: board private structure |
4066 | */ |
4067 | static void igc_irq_enable(struct igc_adapter *adapter) |
4068 | { |
4069 | struct igc_hw *hw = &adapter->hw; |
4070 | |
4071 | if (adapter->msix_entries) { |
4072 | u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; |
4073 | u32 regval = rd32(IGC_EIAC); |
4074 | |
4075 | wr32(IGC_EIAC, regval | adapter->eims_enable_mask); |
4076 | regval = rd32(IGC_EIAM); |
4077 | wr32(IGC_EIAM, regval | adapter->eims_enable_mask); |
4078 | wr32(IGC_EIMS, adapter->eims_enable_mask); |
4079 | wr32(IGC_IMS, ims); |
4080 | } else { |
4081 | wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); |
4082 | wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); |
4083 | } |
4084 | } |
4085 | |
4086 | /** |
4087 | * igc_irq_disable - Mask off interrupt generation on the NIC |
4088 | * @adapter: board private structure |
4089 | */ |
4090 | static void igc_irq_disable(struct igc_adapter *adapter) |
4091 | { |
4092 | struct igc_hw *hw = &adapter->hw; |
4093 | |
4094 | if (adapter->msix_entries) { |
4095 | u32 regval = rd32(IGC_EIAM); |
4096 | |
4097 | wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); |
4098 | wr32(IGC_EIMC, adapter->eims_enable_mask); |
4099 | regval = rd32(IGC_EIAC); |
4100 | wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); |
4101 | } |
4102 | |
4103 | wr32(IGC_IAM, 0); |
4104 | wr32(IGC_IMC, ~0); |
4105 | wrfl(); |
4106 | |
4107 | if (adapter->msix_entries) { |
4108 | int vector = 0, i; |
4109 | |
4110 | synchronize_irq(irq: adapter->msix_entries[vector++].vector); |
4111 | |
4112 | for (i = 0; i < adapter->num_q_vectors; i++) |
4113 | synchronize_irq(irq: adapter->msix_entries[vector++].vector); |
4114 | } else { |
4115 | synchronize_irq(irq: adapter->pdev->irq); |
4116 | } |
4117 | } |
4118 | |
4119 | void igc_set_flag_queue_pairs(struct igc_adapter *adapter, |
4120 | const u32 ) |
4121 | { |
4122 | /* Determine if we need to pair queues. */ |
4123 | /* If rss_queues > half of max_rss_queues, pair the queues in |
4124 | * order to conserve interrupts due to limited supply. |
4125 | */ |
4126 | if (adapter->rss_queues > (max_rss_queues / 2)) |
4127 | adapter->flags |= IGC_FLAG_QUEUE_PAIRS; |
4128 | else |
4129 | adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; |
4130 | } |
4131 | |
4132 | unsigned int (struct igc_adapter *adapter) |
4133 | { |
4134 | return IGC_MAX_RX_QUEUES; |
4135 | } |
4136 | |
4137 | static void igc_init_queue_configuration(struct igc_adapter *adapter) |
4138 | { |
4139 | u32 ; |
4140 | |
4141 | max_rss_queues = igc_get_max_rss_queues(adapter); |
4142 | adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); |
4143 | |
4144 | igc_set_flag_queue_pairs(adapter, max_rss_queues); |
4145 | } |
4146 | |
4147 | /** |
4148 | * igc_reset_q_vector - Reset config for interrupt vector |
4149 | * @adapter: board private structure to initialize |
4150 | * @v_idx: Index of vector to be reset |
4151 | * |
4152 | * If NAPI is enabled it will delete any references to the |
4153 | * NAPI struct. This is preparation for igc_free_q_vector. |
4154 | */ |
4155 | static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) |
4156 | { |
4157 | struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; |
4158 | |
4159 | /* if we're coming from igc_set_interrupt_capability, the vectors are |
4160 | * not yet allocated |
4161 | */ |
4162 | if (!q_vector) |
4163 | return; |
4164 | |
4165 | if (q_vector->tx.ring) |
4166 | adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; |
4167 | |
4168 | if (q_vector->rx.ring) |
4169 | adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; |
4170 | |
4171 | netif_napi_del(napi: &q_vector->napi); |
4172 | } |
4173 | |
4174 | /** |
4175 | * igc_free_q_vector - Free memory allocated for specific interrupt vector |
4176 | * @adapter: board private structure to initialize |
4177 | * @v_idx: Index of vector to be freed |
4178 | * |
4179 | * This function frees the memory allocated to the q_vector. |
4180 | */ |
4181 | static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) |
4182 | { |
4183 | struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; |
4184 | |
4185 | adapter->q_vector[v_idx] = NULL; |
4186 | |
4187 | /* igc_get_stats64() might access the rings on this vector, |
4188 | * we must wait a grace period before freeing it. |
4189 | */ |
4190 | if (q_vector) |
4191 | kfree_rcu(q_vector, rcu); |
4192 | } |
4193 | |
4194 | /** |
4195 | * igc_free_q_vectors - Free memory allocated for interrupt vectors |
4196 | * @adapter: board private structure to initialize |
4197 | * |
4198 | * This function frees the memory allocated to the q_vectors. In addition if |
4199 | * NAPI is enabled it will delete any references to the NAPI struct prior |
4200 | * to freeing the q_vector. |
4201 | */ |
4202 | static void igc_free_q_vectors(struct igc_adapter *adapter) |
4203 | { |
4204 | int v_idx = adapter->num_q_vectors; |
4205 | |
4206 | adapter->num_tx_queues = 0; |
4207 | adapter->num_rx_queues = 0; |
4208 | adapter->num_q_vectors = 0; |
4209 | |
4210 | while (v_idx--) { |
4211 | igc_reset_q_vector(adapter, v_idx); |
4212 | igc_free_q_vector(adapter, v_idx); |
4213 | } |
4214 | } |
4215 | |
4216 | /** |
4217 | * igc_update_itr - update the dynamic ITR value based on statistics |
4218 | * @q_vector: pointer to q_vector |
4219 | * @ring_container: ring info to update the itr for |
4220 | * |
4221 | * Stores a new ITR value based on packets and byte |
4222 | * counts during the last interrupt. The advantage of per interrupt |
4223 | * computation is faster updates and more accurate ITR for the current |
4224 | * traffic pattern. Constants in this function were computed |
4225 | * based on theoretical maximum wire speed and thresholds were set based |
4226 | * on testing data as well as attempting to minimize response time |
4227 | * while increasing bulk throughput. |
4228 | * NOTE: These calculations are only valid when operating in a single- |
4229 | * queue environment. |
4230 | */ |
4231 | static void igc_update_itr(struct igc_q_vector *q_vector, |
4232 | struct igc_ring_container *ring_container) |
4233 | { |
4234 | unsigned int packets = ring_container->total_packets; |
4235 | unsigned int bytes = ring_container->total_bytes; |
4236 | u8 itrval = ring_container->itr; |
4237 | |
4238 | /* no packets, exit with status unchanged */ |
4239 | if (packets == 0) |
4240 | return; |
4241 | |
4242 | switch (itrval) { |
4243 | case lowest_latency: |
4244 | /* handle TSO and jumbo frames */ |
4245 | if (bytes / packets > 8000) |
4246 | itrval = bulk_latency; |
4247 | else if ((packets < 5) && (bytes > 512)) |
4248 | itrval = low_latency; |
4249 | break; |
4250 | case low_latency: /* 50 usec aka 20000 ints/s */ |
4251 | if (bytes > 10000) { |
4252 | /* this if handles the TSO accounting */ |
4253 | if (bytes / packets > 8000) |
4254 | itrval = bulk_latency; |
4255 | else if ((packets < 10) || ((bytes / packets) > 1200)) |
4256 | itrval = bulk_latency; |
4257 | else if ((packets > 35)) |
4258 | itrval = lowest_latency; |
4259 | } else if (bytes / packets > 2000) { |
4260 | itrval = bulk_latency; |
4261 | } else if (packets <= 2 && bytes < 512) { |
4262 | itrval = lowest_latency; |
4263 | } |
4264 | break; |
4265 | case bulk_latency: /* 250 usec aka 4000 ints/s */ |
4266 | if (bytes > 25000) { |
4267 | if (packets > 35) |
4268 | itrval = low_latency; |
4269 | } else if (bytes < 1500) { |
4270 | itrval = low_latency; |
4271 | } |
4272 | break; |
4273 | } |
4274 | |
4275 | /* clear work counters since we have the values we need */ |
4276 | ring_container->total_bytes = 0; |
4277 | ring_container->total_packets = 0; |
4278 | |
4279 | /* write updated itr to ring container */ |
4280 | ring_container->itr = itrval; |
4281 | } |
4282 | |
4283 | static void igc_set_itr(struct igc_q_vector *q_vector) |
4284 | { |
4285 | struct igc_adapter *adapter = q_vector->adapter; |
4286 | u32 new_itr = q_vector->itr_val; |
4287 | u8 current_itr = 0; |
4288 | |
4289 | /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ |
4290 | switch (adapter->link_speed) { |
4291 | case SPEED_10: |
4292 | case SPEED_100: |
4293 | current_itr = 0; |
4294 | new_itr = IGC_4K_ITR; |
4295 | goto set_itr_now; |
4296 | default: |
4297 | break; |
4298 | } |
4299 | |
4300 | igc_update_itr(q_vector, ring_container: &q_vector->tx); |
4301 | igc_update_itr(q_vector, ring_container: &q_vector->rx); |
4302 | |
4303 | current_itr = max(q_vector->rx.itr, q_vector->tx.itr); |
4304 | |
4305 | /* conservative mode (itr 3) eliminates the lowest_latency setting */ |
4306 | if (current_itr == lowest_latency && |
4307 | ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || |
4308 | (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) |
4309 | current_itr = low_latency; |
4310 | |
4311 | switch (current_itr) { |
4312 | /* counts and packets in update_itr are dependent on these numbers */ |
4313 | case lowest_latency: |
4314 | new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ |
4315 | break; |
4316 | case low_latency: |
4317 | new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ |
4318 | break; |
4319 | case bulk_latency: |
4320 | new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ |
4321 | break; |
4322 | default: |
4323 | break; |
4324 | } |
4325 | |
4326 | set_itr_now: |
4327 | if (new_itr != q_vector->itr_val) { |
4328 | /* this attempts to bias the interrupt rate towards Bulk |
4329 | * by adding intermediate steps when interrupt rate is |
4330 | * increasing |
4331 | */ |
4332 | new_itr = new_itr > q_vector->itr_val ? |
4333 | max((new_itr * q_vector->itr_val) / |
4334 | (new_itr + (q_vector->itr_val >> 2)), |
4335 | new_itr) : new_itr; |
4336 | /* Don't write the value here; it resets the adapter's |
4337 | * internal timer, and causes us to delay far longer than |
4338 | * we should between interrupts. Instead, we write the ITR |
4339 | * value at the beginning of the next interrupt so the timing |
4340 | * ends up being correct. |
4341 | */ |
4342 | q_vector->itr_val = new_itr; |
4343 | q_vector->set_itr = 1; |
4344 | } |
4345 | } |
4346 | |
4347 | static void igc_reset_interrupt_capability(struct igc_adapter *adapter) |
4348 | { |
4349 | int v_idx = adapter->num_q_vectors; |
4350 | |
4351 | if (adapter->msix_entries) { |
4352 | pci_disable_msix(dev: adapter->pdev); |
4353 | kfree(objp: adapter->msix_entries); |
4354 | adapter->msix_entries = NULL; |
4355 | } else if (adapter->flags & IGC_FLAG_HAS_MSI) { |
4356 | pci_disable_msi(dev: adapter->pdev); |
4357 | } |
4358 | |
4359 | while (v_idx--) |
4360 | igc_reset_q_vector(adapter, v_idx); |
4361 | } |
4362 | |
4363 | /** |
4364 | * igc_set_interrupt_capability - set MSI or MSI-X if supported |
4365 | * @adapter: Pointer to adapter structure |
4366 | * @msix: boolean value for MSI-X capability |
4367 | * |
4368 | * Attempt to configure interrupts using the best available |
4369 | * capabilities of the hardware and kernel. |
4370 | */ |
4371 | static void igc_set_interrupt_capability(struct igc_adapter *adapter, |
4372 | bool msix) |
4373 | { |
4374 | int numvecs, i; |
4375 | int err; |
4376 | |
4377 | if (!msix) |
4378 | goto msi_only; |
4379 | adapter->flags |= IGC_FLAG_HAS_MSIX; |
4380 | |
4381 | /* Number of supported queues. */ |
4382 | adapter->num_rx_queues = adapter->rss_queues; |
4383 | |
4384 | adapter->num_tx_queues = adapter->rss_queues; |
4385 | |
4386 | /* start with one vector for every Rx queue */ |
4387 | numvecs = adapter->num_rx_queues; |
4388 | |
4389 | /* if Tx handler is separate add 1 for every Tx queue */ |
4390 | if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) |
4391 | numvecs += adapter->num_tx_queues; |
4392 | |
4393 | /* store the number of vectors reserved for queues */ |
4394 | adapter->num_q_vectors = numvecs; |
4395 | |
4396 | /* add 1 vector for link status interrupts */ |
4397 | numvecs++; |
4398 | |
4399 | adapter->msix_entries = kcalloc(n: numvecs, size: sizeof(struct msix_entry), |
4400 | GFP_KERNEL); |
4401 | |
4402 | if (!adapter->msix_entries) |
4403 | return; |
4404 | |
4405 | /* populate entry values */ |
4406 | for (i = 0; i < numvecs; i++) |
4407 | adapter->msix_entries[i].entry = i; |
4408 | |
4409 | err = pci_enable_msix_range(dev: adapter->pdev, |
4410 | entries: adapter->msix_entries, |
4411 | minvec: numvecs, |
4412 | maxvec: numvecs); |
4413 | if (err > 0) |
4414 | return; |
4415 | |
4416 | kfree(objp: adapter->msix_entries); |
4417 | adapter->msix_entries = NULL; |
4418 | |
4419 | igc_reset_interrupt_capability(adapter); |
4420 | |
4421 | msi_only: |
4422 | adapter->flags &= ~IGC_FLAG_HAS_MSIX; |
4423 | |
4424 | adapter->rss_queues = 1; |
4425 | adapter->flags |= IGC_FLAG_QUEUE_PAIRS; |
4426 | adapter->num_rx_queues = 1; |
4427 | adapter->num_tx_queues = 1; |
4428 | adapter->num_q_vectors = 1; |
4429 | if (!pci_enable_msi(dev: adapter->pdev)) |
4430 | adapter->flags |= IGC_FLAG_HAS_MSI; |
4431 | } |
4432 | |
4433 | /** |
4434 | * igc_update_ring_itr - update the dynamic ITR value based on packet size |
4435 | * @q_vector: pointer to q_vector |
4436 | * |
4437 | * Stores a new ITR value based on strictly on packet size. This |
4438 | * algorithm is less sophisticated than that used in igc_update_itr, |
4439 | * due to the difficulty of synchronizing statistics across multiple |
4440 | * receive rings. The divisors and thresholds used by this function |
4441 | * were determined based on theoretical maximum wire speed and testing |
4442 | * data, in order to minimize response time while increasing bulk |
4443 | * throughput. |
4444 | * NOTE: This function is called only when operating in a multiqueue |
4445 | * receive environment. |
4446 | */ |
4447 | static void igc_update_ring_itr(struct igc_q_vector *q_vector) |
4448 | { |
4449 | struct igc_adapter *adapter = q_vector->adapter; |
4450 | int new_val = q_vector->itr_val; |
4451 | int avg_wire_size = 0; |
4452 | unsigned int packets; |
4453 | |
4454 | /* For non-gigabit speeds, just fix the interrupt rate at 4000 |
4455 | * ints/sec - ITR timer value of 120 ticks. |
4456 | */ |
4457 | switch (adapter->link_speed) { |
4458 | case SPEED_10: |
4459 | case SPEED_100: |
4460 | new_val = IGC_4K_ITR; |
4461 | goto set_itr_val; |
4462 | default: |
4463 | break; |
4464 | } |
4465 | |
4466 | packets = q_vector->rx.total_packets; |
4467 | if (packets) |
4468 | avg_wire_size = q_vector->rx.total_bytes / packets; |
4469 | |
4470 | packets = q_vector->tx.total_packets; |
4471 | if (packets) |
4472 | avg_wire_size = max_t(u32, avg_wire_size, |
4473 | q_vector->tx.total_bytes / packets); |
4474 | |
4475 | /* if avg_wire_size isn't set no work was done */ |
4476 | if (!avg_wire_size) |
4477 | goto clear_counts; |
4478 | |
4479 | /* Add 24 bytes to size to account for CRC, preamble, and gap */ |
4480 | avg_wire_size += 24; |
4481 | |
4482 | /* Don't starve jumbo frames */ |
4483 | avg_wire_size = min(avg_wire_size, 3000); |
4484 | |
4485 | /* Give a little boost to mid-size frames */ |
4486 | if (avg_wire_size > 300 && avg_wire_size < 1200) |
4487 | new_val = avg_wire_size / 3; |
4488 | else |
4489 | new_val = avg_wire_size / 2; |
4490 | |
4491 | /* conservative mode (itr 3) eliminates the lowest_latency setting */ |
4492 | if (new_val < IGC_20K_ITR && |
4493 | ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || |
4494 | (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) |
4495 | new_val = IGC_20K_ITR; |
4496 | |
4497 | set_itr_val: |
4498 | if (new_val != q_vector->itr_val) { |
4499 | q_vector->itr_val = new_val; |
4500 | q_vector->set_itr = 1; |
4501 | } |
4502 | clear_counts: |
4503 | q_vector->rx.total_bytes = 0; |
4504 | q_vector->rx.total_packets = 0; |
4505 | q_vector->tx.total_bytes = 0; |
4506 | q_vector->tx.total_packets = 0; |
4507 | } |
4508 | |
4509 | static void igc_ring_irq_enable(struct igc_q_vector *q_vector) |
4510 | { |
4511 | struct igc_adapter *adapter = q_vector->adapter; |
4512 | struct igc_hw *hw = &adapter->hw; |
4513 | |
4514 | if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || |
4515 | (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { |
4516 | if (adapter->num_q_vectors == 1) |
4517 | igc_set_itr(q_vector); |
4518 | else |
4519 | igc_update_ring_itr(q_vector); |
4520 | } |
4521 | |
4522 | if (!test_bit(__IGC_DOWN, &adapter->state)) { |
4523 | if (adapter->msix_entries) |
4524 | wr32(IGC_EIMS, q_vector->eims_value); |
4525 | else |
4526 | igc_irq_enable(adapter); |
4527 | } |
4528 | } |
4529 | |
4530 | static void igc_add_ring(struct igc_ring *ring, |
4531 | struct igc_ring_container *head) |
4532 | { |
4533 | head->ring = ring; |
4534 | head->count++; |
4535 | } |
4536 | |
4537 | /** |
4538 | * igc_cache_ring_register - Descriptor ring to register mapping |
4539 | * @adapter: board private structure to initialize |
4540 | * |
4541 | * Once we know the feature-set enabled for the device, we'll cache |
4542 | * the register offset the descriptor ring is assigned to. |
4543 | */ |
4544 | static void igc_cache_ring_register(struct igc_adapter *adapter) |
4545 | { |
4546 | int i = 0, j = 0; |
4547 | |
4548 | switch (adapter->hw.mac.type) { |
4549 | case igc_i225: |
4550 | default: |
4551 | for (; i < adapter->num_rx_queues; i++) |
4552 | adapter->rx_ring[i]->reg_idx = i; |
4553 | for (; j < adapter->num_tx_queues; j++) |
4554 | adapter->tx_ring[j]->reg_idx = j; |
4555 | break; |
4556 | } |
4557 | } |
4558 | |
4559 | /** |
4560 | * igc_poll - NAPI Rx polling callback |
4561 | * @napi: napi polling structure |
4562 | * @budget: count of how many packets we should handle |
4563 | */ |
4564 | static int igc_poll(struct napi_struct *napi, int budget) |
4565 | { |
4566 | struct igc_q_vector *q_vector = container_of(napi, |
4567 | struct igc_q_vector, |
4568 | napi); |
4569 | struct igc_ring *rx_ring = q_vector->rx.ring; |
4570 | bool clean_complete = true; |
4571 | int work_done = 0; |
4572 | |
4573 | if (q_vector->tx.ring) |
4574 | clean_complete = igc_clean_tx_irq(q_vector, napi_budget: budget); |
4575 | |
4576 | if (rx_ring) { |
4577 | int cleaned = rx_ring->xsk_pool ? |
4578 | igc_clean_rx_irq_zc(q_vector, budget) : |
4579 | igc_clean_rx_irq(q_vector, budget); |
4580 | |
4581 | work_done += cleaned; |
4582 | if (cleaned >= budget) |
4583 | clean_complete = false; |
4584 | } |
4585 | |
4586 | /* If all work not completed, return budget and keep polling */ |
4587 | if (!clean_complete) |
4588 | return budget; |
4589 | |
4590 | /* Exit the polling mode, but don't re-enable interrupts if stack might |
4591 | * poll us due to busy-polling |
4592 | */ |
4593 | if (likely(napi_complete_done(napi, work_done))) |
4594 | igc_ring_irq_enable(q_vector); |
4595 | |
4596 | return min(work_done, budget - 1); |
4597 | } |
4598 | |
4599 | /** |
4600 | * igc_alloc_q_vector - Allocate memory for a single interrupt vector |
4601 | * @adapter: board private structure to initialize |
4602 | * @v_count: q_vectors allocated on adapter, used for ring interleaving |
4603 | * @v_idx: index of vector in adapter struct |
4604 | * @txr_count: total number of Tx rings to allocate |
4605 | * @txr_idx: index of first Tx ring to allocate |
4606 | * @rxr_count: total number of Rx rings to allocate |
4607 | * @rxr_idx: index of first Rx ring to allocate |
4608 | * |
4609 | * We allocate one q_vector. If allocation fails we return -ENOMEM. |
4610 | */ |
4611 | static int igc_alloc_q_vector(struct igc_adapter *adapter, |
4612 | unsigned int v_count, unsigned int v_idx, |
4613 | unsigned int txr_count, unsigned int txr_idx, |
4614 | unsigned int rxr_count, unsigned int rxr_idx) |
4615 | { |
4616 | struct igc_q_vector *q_vector; |
4617 | struct igc_ring *ring; |
4618 | int ring_count; |
4619 | |
4620 | /* igc only supports 1 Tx and/or 1 Rx queue per vector */ |
4621 | if (txr_count > 1 || rxr_count > 1) |
4622 | return -ENOMEM; |
4623 | |
4624 | ring_count = txr_count + rxr_count; |
4625 | |
4626 | /* allocate q_vector and rings */ |
4627 | q_vector = adapter->q_vector[v_idx]; |
4628 | if (!q_vector) |
4629 | q_vector = kzalloc(struct_size(q_vector, ring, ring_count), |
4630 | GFP_KERNEL); |
4631 | else |
4632 | memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); |
4633 | if (!q_vector) |
4634 | return -ENOMEM; |
4635 | |
4636 | /* initialize NAPI */ |
4637 | netif_napi_add(dev: adapter->netdev, napi: &q_vector->napi, poll: igc_poll); |
4638 | |
4639 | /* tie q_vector and adapter together */ |
4640 | adapter->q_vector[v_idx] = q_vector; |
4641 | q_vector->adapter = adapter; |
4642 | |
4643 | /* initialize work limits */ |
4644 | q_vector->tx.work_limit = adapter->tx_work_limit; |
4645 | |
4646 | /* initialize ITR configuration */ |
4647 | q_vector->itr_register = adapter->io_addr + IGC_EITR(0); |
4648 | q_vector->itr_val = IGC_START_ITR; |
4649 | |
4650 | /* initialize pointer to rings */ |
4651 | ring = q_vector->ring; |
4652 | |
4653 | /* initialize ITR */ |
4654 | if (rxr_count) { |
4655 | /* rx or rx/tx vector */ |
4656 | if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) |
4657 | q_vector->itr_val = adapter->rx_itr_setting; |
4658 | } else { |
4659 | /* tx only vector */ |
4660 | if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) |
4661 | q_vector->itr_val = adapter->tx_itr_setting; |
4662 | } |
4663 | |
4664 | if (txr_count) { |
4665 | /* assign generic ring traits */ |
4666 | ring->dev = &adapter->pdev->dev; |
4667 | ring->netdev = adapter->netdev; |
4668 | |
4669 | /* configure backlink on ring */ |
4670 | ring->q_vector = q_vector; |
4671 | |
4672 | /* update q_vector Tx values */ |
4673 | igc_add_ring(ring, head: &q_vector->tx); |
4674 | |
4675 | /* apply Tx specific ring traits */ |
4676 | ring->count = adapter->tx_ring_count; |
4677 | ring->queue_index = txr_idx; |
4678 | |
4679 | /* assign ring to adapter */ |
4680 | adapter->tx_ring[txr_idx] = ring; |
4681 | |
4682 | /* push pointer to next ring */ |
4683 | ring++; |
4684 | } |
4685 | |
4686 | if (rxr_count) { |
4687 | /* assign generic ring traits */ |
4688 | ring->dev = &adapter->pdev->dev; |
4689 | ring->netdev = adapter->netdev; |
4690 | |
4691 | /* configure backlink on ring */ |
4692 | ring->q_vector = q_vector; |
4693 | |
4694 | /* update q_vector Rx values */ |
4695 | igc_add_ring(ring, head: &q_vector->rx); |
4696 | |
4697 | /* apply Rx specific ring traits */ |
4698 | ring->count = adapter->rx_ring_count; |
4699 | ring->queue_index = rxr_idx; |
4700 | |
4701 | /* assign ring to adapter */ |
4702 | adapter->rx_ring[rxr_idx] = ring; |
4703 | } |
4704 | |
4705 | return 0; |
4706 | } |
4707 | |
4708 | /** |
4709 | * igc_alloc_q_vectors - Allocate memory for interrupt vectors |
4710 | * @adapter: board private structure to initialize |
4711 | * |
4712 | * We allocate one q_vector per queue interrupt. If allocation fails we |
4713 | * return -ENOMEM. |
4714 | */ |
4715 | static int igc_alloc_q_vectors(struct igc_adapter *adapter) |
4716 | { |
4717 | int rxr_remaining = adapter->num_rx_queues; |
4718 | int txr_remaining = adapter->num_tx_queues; |
4719 | int rxr_idx = 0, txr_idx = 0, v_idx = 0; |
4720 | int q_vectors = adapter->num_q_vectors; |
4721 | int err; |
4722 | |
4723 | if (q_vectors >= (rxr_remaining + txr_remaining)) { |
4724 | for (; rxr_remaining; v_idx++) { |
4725 | err = igc_alloc_q_vector(adapter, v_count: q_vectors, v_idx, |
4726 | txr_count: 0, txr_idx: 0, rxr_count: 1, rxr_idx); |
4727 | |
4728 | if (err) |
4729 | goto err_out; |
4730 | |
4731 | /* update counts and index */ |
4732 | rxr_remaining--; |
4733 | rxr_idx++; |
4734 | } |
4735 | } |
4736 | |
4737 | for (; v_idx < q_vectors; v_idx++) { |
4738 | int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); |
4739 | int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); |
4740 | |
4741 | err = igc_alloc_q_vector(adapter, v_count: q_vectors, v_idx, |
4742 | txr_count: tqpv, txr_idx, rxr_count: rqpv, rxr_idx); |
4743 | |
4744 | if (err) |
4745 | goto err_out; |
4746 | |
4747 | /* update counts and index */ |
4748 | rxr_remaining -= rqpv; |
4749 | txr_remaining -= tqpv; |
4750 | rxr_idx++; |
4751 | txr_idx++; |
4752 | } |
4753 | |
4754 | return 0; |
4755 | |
4756 | err_out: |
4757 | adapter->num_tx_queues = 0; |
4758 | adapter->num_rx_queues = 0; |
4759 | adapter->num_q_vectors = 0; |
4760 | |
4761 | while (v_idx--) |
4762 | igc_free_q_vector(adapter, v_idx); |
4763 | |
4764 | return -ENOMEM; |
4765 | } |
4766 | |
4767 | /** |
4768 | * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors |
4769 | * @adapter: Pointer to adapter structure |
4770 | * @msix: boolean for MSI-X capability |
4771 | * |
4772 | * This function initializes the interrupts and allocates all of the queues. |
4773 | */ |
4774 | static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) |
4775 | { |
4776 | struct net_device *dev = adapter->netdev; |
4777 | int err = 0; |
4778 | |
4779 | igc_set_interrupt_capability(adapter, msix); |
4780 | |
4781 | err = igc_alloc_q_vectors(adapter); |
4782 | if (err) { |
4783 | netdev_err(dev, format: "Unable to allocate memory for vectors\n" ); |
4784 | goto err_alloc_q_vectors; |
4785 | } |
4786 | |
4787 | igc_cache_ring_register(adapter); |
4788 | |
4789 | return 0; |
4790 | |
4791 | err_alloc_q_vectors: |
4792 | igc_reset_interrupt_capability(adapter); |
4793 | return err; |
4794 | } |
4795 | |
4796 | /** |
4797 | * igc_sw_init - Initialize general software structures (struct igc_adapter) |
4798 | * @adapter: board private structure to initialize |
4799 | * |
4800 | * igc_sw_init initializes the Adapter private data structure. |
4801 | * Fields are initialized based on PCI device information and |
4802 | * OS network device settings (MTU size). |
4803 | */ |
4804 | static int igc_sw_init(struct igc_adapter *adapter) |
4805 | { |
4806 | struct net_device *netdev = adapter->netdev; |
4807 | struct pci_dev *pdev = adapter->pdev; |
4808 | struct igc_hw *hw = &adapter->hw; |
4809 | |
4810 | pci_read_config_word(dev: pdev, PCI_COMMAND, val: &hw->bus.pci_cmd_word); |
4811 | |
4812 | /* set default ring sizes */ |
4813 | adapter->tx_ring_count = IGC_DEFAULT_TXD; |
4814 | adapter->rx_ring_count = IGC_DEFAULT_RXD; |
4815 | |
4816 | /* set default ITR values */ |
4817 | adapter->rx_itr_setting = IGC_DEFAULT_ITR; |
4818 | adapter->tx_itr_setting = IGC_DEFAULT_ITR; |
4819 | |
4820 | /* set default work limits */ |
4821 | adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; |
4822 | |
4823 | /* adjust max frame to be at least the size of a standard frame */ |
4824 | adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + |
4825 | VLAN_HLEN; |
4826 | adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; |
4827 | |
4828 | mutex_init(&adapter->nfc_rule_lock); |
4829 | INIT_LIST_HEAD(list: &adapter->nfc_rule_list); |
4830 | adapter->nfc_rule_count = 0; |
4831 | |
4832 | spin_lock_init(&adapter->stats64_lock); |
4833 | spin_lock_init(&adapter->qbv_tx_lock); |
4834 | /* Assume MSI-X interrupts, will be checked during IRQ allocation */ |
4835 | adapter->flags |= IGC_FLAG_HAS_MSIX; |
4836 | |
4837 | igc_init_queue_configuration(adapter); |
4838 | |
4839 | /* This call may decrease the number of queues */ |
4840 | if (igc_init_interrupt_scheme(adapter, msix: true)) { |
4841 | netdev_err(dev: netdev, format: "Unable to allocate memory for queues\n" ); |
4842 | return -ENOMEM; |
4843 | } |
4844 | |
4845 | /* Explicitly disable IRQ since the NIC can be in any state. */ |
4846 | igc_irq_disable(adapter); |
4847 | |
4848 | set_bit(nr: __IGC_DOWN, addr: &adapter->state); |
4849 | |
4850 | return 0; |
4851 | } |
4852 | |
4853 | /** |
4854 | * igc_up - Open the interface and prepare it to handle traffic |
4855 | * @adapter: board private structure |
4856 | */ |
4857 | void igc_up(struct igc_adapter *adapter) |
4858 | { |
4859 | struct igc_hw *hw = &adapter->hw; |
4860 | int i = 0; |
4861 | |
4862 | /* hardware has been reset, we need to reload some things */ |
4863 | igc_configure(adapter); |
4864 | |
4865 | clear_bit(nr: __IGC_DOWN, addr: &adapter->state); |
4866 | |
4867 | for (i = 0; i < adapter->num_q_vectors; i++) |
4868 | napi_enable(n: &adapter->q_vector[i]->napi); |
4869 | |
4870 | if (adapter->msix_entries) |
4871 | igc_configure_msix(adapter); |
4872 | else |
4873 | igc_assign_vector(q_vector: adapter->q_vector[0], msix_vector: 0); |
4874 | |
4875 | /* Clear any pending interrupts. */ |
4876 | rd32(IGC_ICR); |
4877 | igc_irq_enable(adapter); |
4878 | |
4879 | netif_tx_start_all_queues(dev: adapter->netdev); |
4880 | |
4881 | /* start the watchdog. */ |
4882 | hw->mac.get_link_status = true; |
4883 | schedule_work(work: &adapter->watchdog_task); |
4884 | } |
4885 | |
4886 | /** |
4887 | * igc_update_stats - Update the board statistics counters |
4888 | * @adapter: board private structure |
4889 | */ |
4890 | void igc_update_stats(struct igc_adapter *adapter) |
4891 | { |
4892 | struct rtnl_link_stats64 *net_stats = &adapter->stats64; |
4893 | struct pci_dev *pdev = adapter->pdev; |
4894 | struct igc_hw *hw = &adapter->hw; |
4895 | u64 _bytes, _packets; |
4896 | u64 bytes, packets; |
4897 | unsigned int start; |
4898 | u32 mpc; |
4899 | int i; |
4900 | |
4901 | /* Prevent stats update while adapter is being reset, or if the pci |
4902 | * connection is down. |
4903 | */ |
4904 | if (adapter->link_speed == 0) |
4905 | return; |
4906 | if (pci_channel_offline(pdev)) |
4907 | return; |
4908 | |
4909 | packets = 0; |
4910 | bytes = 0; |
4911 | |
4912 | rcu_read_lock(); |
4913 | for (i = 0; i < adapter->num_rx_queues; i++) { |
4914 | struct igc_ring *ring = adapter->rx_ring[i]; |
4915 | u32 rqdpc = rd32(IGC_RQDPC(i)); |
4916 | |
4917 | if (hw->mac.type >= igc_i225) |
4918 | wr32(IGC_RQDPC(i), 0); |
4919 | |
4920 | if (rqdpc) { |
4921 | ring->rx_stats.drops += rqdpc; |
4922 | net_stats->rx_fifo_errors += rqdpc; |
4923 | } |
4924 | |
4925 | do { |
4926 | start = u64_stats_fetch_begin(syncp: &ring->rx_syncp); |
4927 | _bytes = ring->rx_stats.bytes; |
4928 | _packets = ring->rx_stats.packets; |
4929 | } while (u64_stats_fetch_retry(syncp: &ring->rx_syncp, start)); |
4930 | bytes += _bytes; |
4931 | packets += _packets; |
4932 | } |
4933 | |
4934 | net_stats->rx_bytes = bytes; |
4935 | net_stats->rx_packets = packets; |
4936 | |
4937 | packets = 0; |
4938 | bytes = 0; |
4939 | for (i = 0; i < adapter->num_tx_queues; i++) { |
4940 | struct igc_ring *ring = adapter->tx_ring[i]; |
4941 | |
4942 | do { |
4943 | start = u64_stats_fetch_begin(syncp: &ring->tx_syncp); |
4944 | _bytes = ring->tx_stats.bytes; |
4945 | _packets = ring->tx_stats.packets; |
4946 | } while (u64_stats_fetch_retry(syncp: &ring->tx_syncp, start)); |
4947 | bytes += _bytes; |
4948 | packets += _packets; |
4949 | } |
4950 | net_stats->tx_bytes = bytes; |
4951 | net_stats->tx_packets = packets; |
4952 | rcu_read_unlock(); |
4953 | |
4954 | /* read stats registers */ |
4955 | adapter->stats.crcerrs += rd32(IGC_CRCERRS); |
4956 | adapter->stats.gprc += rd32(IGC_GPRC); |
4957 | adapter->stats.gorc += rd32(IGC_GORCL); |
4958 | rd32(IGC_GORCH); /* clear GORCL */ |
4959 | adapter->stats.bprc += rd32(IGC_BPRC); |
4960 | adapter->stats.mprc += rd32(IGC_MPRC); |
4961 | adapter->stats.roc += rd32(IGC_ROC); |
4962 | |
4963 | adapter->stats.prc64 += rd32(IGC_PRC64); |
4964 | adapter->stats.prc127 += rd32(IGC_PRC127); |
4965 | adapter->stats.prc255 += rd32(IGC_PRC255); |
4966 | adapter->stats.prc511 += rd32(IGC_PRC511); |
4967 | adapter->stats.prc1023 += rd32(IGC_PRC1023); |
4968 | adapter->stats.prc1522 += rd32(IGC_PRC1522); |
4969 | adapter->stats.tlpic += rd32(IGC_TLPIC); |
4970 | adapter->stats.rlpic += rd32(IGC_RLPIC); |
4971 | adapter->stats.hgptc += rd32(IGC_HGPTC); |
4972 | |
4973 | mpc = rd32(IGC_MPC); |
4974 | adapter->stats.mpc += mpc; |
4975 | net_stats->rx_fifo_errors += mpc; |
4976 | adapter->stats.scc += rd32(IGC_SCC); |
4977 | adapter->stats.ecol += rd32(IGC_ECOL); |
4978 | adapter->stats.mcc += rd32(IGC_MCC); |
4979 | adapter->stats.latecol += rd32(IGC_LATECOL); |
4980 | adapter->stats.dc += rd32(IGC_DC); |
4981 | adapter->stats.rlec += rd32(IGC_RLEC); |
4982 | adapter->stats.xonrxc += rd32(IGC_XONRXC); |
4983 | adapter->stats.xontxc += rd32(IGC_XONTXC); |
4984 | adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); |
4985 | adapter->stats.xofftxc += rd32(IGC_XOFFTXC); |
4986 | adapter->stats.fcruc += rd32(IGC_FCRUC); |
4987 | adapter->stats.gptc += rd32(IGC_GPTC); |
4988 | adapter->stats.gotc += rd32(IGC_GOTCL); |
4989 | rd32(IGC_GOTCH); /* clear GOTCL */ |
4990 | adapter->stats.rnbc += rd32(IGC_RNBC); |
4991 | adapter->stats.ruc += rd32(IGC_RUC); |
4992 | adapter->stats.rfc += rd32(IGC_RFC); |
4993 | adapter->stats.rjc += rd32(IGC_RJC); |
4994 | adapter->stats.tor += rd32(IGC_TORH); |
4995 | adapter->stats.tot += rd32(IGC_TOTH); |
4996 | adapter->stats.tpr += rd32(IGC_TPR); |
4997 | |
4998 | adapter->stats.ptc64 += rd32(IGC_PTC64); |
4999 | adapter->stats.ptc127 += rd32(IGC_PTC127); |
5000 | adapter->stats.ptc255 += rd32(IGC_PTC255); |
5001 | adapter->stats.ptc511 += rd32(IGC_PTC511); |
5002 | adapter->stats.ptc1023 += rd32(IGC_PTC1023); |
5003 | adapter->stats.ptc1522 += rd32(IGC_PTC1522); |
5004 | |
5005 | adapter->stats.mptc += rd32(IGC_MPTC); |
5006 | adapter->stats.bptc += rd32(IGC_BPTC); |
5007 | |
5008 | adapter->stats.tpt += rd32(IGC_TPT); |
5009 | adapter->stats.colc += rd32(IGC_COLC); |
5010 | adapter->stats.colc += rd32(IGC_RERC); |
5011 | |
5012 | adapter->stats.algnerrc += rd32(IGC_ALGNERRC); |
5013 | |
5014 | adapter->stats.tsctc += rd32(IGC_TSCTC); |
5015 | |
5016 | adapter->stats.iac += rd32(IGC_IAC); |
5017 | |
5018 | /* Fill out the OS statistics structure */ |
5019 | net_stats->multicast = adapter->stats.mprc; |
5020 | net_stats->collisions = adapter->stats.colc; |
5021 | |
5022 | /* Rx Errors */ |
5023 | |
5024 | /* RLEC on some newer hardware can be incorrect so build |
5025 | * our own version based on RUC and ROC |
5026 | */ |
5027 | net_stats->rx_errors = adapter->stats.rxerrc + |
5028 | adapter->stats.crcerrs + adapter->stats.algnerrc + |
5029 | adapter->stats.ruc + adapter->stats.roc + |
5030 | adapter->stats.cexterr; |
5031 | net_stats->rx_length_errors = adapter->stats.ruc + |
5032 | adapter->stats.roc; |
5033 | net_stats->rx_crc_errors = adapter->stats.crcerrs; |
5034 | net_stats->rx_frame_errors = adapter->stats.algnerrc; |
5035 | net_stats->rx_missed_errors = adapter->stats.mpc; |
5036 | |
5037 | /* Tx Errors */ |
5038 | net_stats->tx_errors = adapter->stats.ecol + |
5039 | adapter->stats.latecol; |
5040 | net_stats->tx_aborted_errors = adapter->stats.ecol; |
5041 | net_stats->tx_window_errors = adapter->stats.latecol; |
5042 | net_stats->tx_carrier_errors = adapter->stats.tncrs; |
5043 | |
5044 | /* Tx Dropped */ |
5045 | net_stats->tx_dropped = adapter->stats.txdrop; |
5046 | |
5047 | /* Management Stats */ |
5048 | adapter->stats.mgptc += rd32(IGC_MGTPTC); |
5049 | adapter->stats.mgprc += rd32(IGC_MGTPRC); |
5050 | adapter->stats.mgpdc += rd32(IGC_MGTPDC); |
5051 | } |
5052 | |
5053 | /** |
5054 | * igc_down - Close the interface |
5055 | * @adapter: board private structure |
5056 | */ |
5057 | void igc_down(struct igc_adapter *adapter) |
5058 | { |
5059 | struct net_device *netdev = adapter->netdev; |
5060 | struct igc_hw *hw = &adapter->hw; |
5061 | u32 tctl, rctl; |
5062 | int i = 0; |
5063 | |
5064 | set_bit(nr: __IGC_DOWN, addr: &adapter->state); |
5065 | |
5066 | igc_ptp_suspend(adapter); |
5067 | |
5068 | if (pci_device_is_present(pdev: adapter->pdev)) { |
5069 | /* disable receives in the hardware */ |
5070 | rctl = rd32(IGC_RCTL); |
5071 | wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); |
5072 | /* flush and sleep below */ |
5073 | } |
5074 | /* set trans_start so we don't get spurious watchdogs during reset */ |
5075 | netif_trans_update(dev: netdev); |
5076 | |
5077 | netif_carrier_off(dev: netdev); |
5078 | netif_tx_stop_all_queues(dev: netdev); |
5079 | |
5080 | if (pci_device_is_present(pdev: adapter->pdev)) { |
5081 | /* disable transmits in the hardware */ |
5082 | tctl = rd32(IGC_TCTL); |
5083 | tctl &= ~IGC_TCTL_EN; |
5084 | wr32(IGC_TCTL, tctl); |
5085 | /* flush both disables and wait for them to finish */ |
5086 | wrfl(); |
5087 | usleep_range(min: 10000, max: 20000); |
5088 | |
5089 | igc_irq_disable(adapter); |
5090 | } |
5091 | |
5092 | adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; |
5093 | |
5094 | for (i = 0; i < adapter->num_q_vectors; i++) { |
5095 | if (adapter->q_vector[i]) { |
5096 | napi_synchronize(n: &adapter->q_vector[i]->napi); |
5097 | napi_disable(n: &adapter->q_vector[i]->napi); |
5098 | } |
5099 | } |
5100 | |
5101 | del_timer_sync(timer: &adapter->watchdog_timer); |
5102 | del_timer_sync(timer: &adapter->phy_info_timer); |
5103 | |
5104 | /* record the stats before reset*/ |
5105 | spin_lock(lock: &adapter->stats64_lock); |
5106 | igc_update_stats(adapter); |
5107 | spin_unlock(lock: &adapter->stats64_lock); |
5108 | |
5109 | adapter->link_speed = 0; |
5110 | adapter->link_duplex = 0; |
5111 | |
5112 | if (!pci_channel_offline(pdev: adapter->pdev)) |
5113 | igc_reset(adapter); |
5114 | |
5115 | /* clear VLAN promisc flag so VFTA will be updated if necessary */ |
5116 | adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; |
5117 | |
5118 | igc_disable_all_tx_rings_hw(adapter); |
5119 | igc_clean_all_tx_rings(adapter); |
5120 | igc_clean_all_rx_rings(adapter); |
5121 | } |
5122 | |
5123 | void igc_reinit_locked(struct igc_adapter *adapter) |
5124 | { |
5125 | while (test_and_set_bit(nr: __IGC_RESETTING, addr: &adapter->state)) |
5126 | usleep_range(min: 1000, max: 2000); |
5127 | igc_down(adapter); |
5128 | igc_up(adapter); |
5129 | clear_bit(nr: __IGC_RESETTING, addr: &adapter->state); |
5130 | } |
5131 | |
5132 | static void igc_reset_task(struct work_struct *work) |
5133 | { |
5134 | struct igc_adapter *adapter; |
5135 | |
5136 | adapter = container_of(work, struct igc_adapter, reset_task); |
5137 | |
5138 | rtnl_lock(); |
5139 | /* If we're already down or resetting, just bail */ |
5140 | if (test_bit(__IGC_DOWN, &adapter->state) || |
5141 | test_bit(__IGC_RESETTING, &adapter->state)) { |
5142 | rtnl_unlock(); |
5143 | return; |
5144 | } |
5145 | |
5146 | igc_rings_dump(adapter); |
5147 | igc_regs_dump(adapter); |
5148 | netdev_err(dev: adapter->netdev, format: "Reset adapter\n" ); |
5149 | igc_reinit_locked(adapter); |
5150 | rtnl_unlock(); |
5151 | } |
5152 | |
5153 | /** |
5154 | * igc_change_mtu - Change the Maximum Transfer Unit |
5155 | * @netdev: network interface device structure |
5156 | * @new_mtu: new value for maximum frame size |
5157 | * |
5158 | * Returns 0 on success, negative on failure |
5159 | */ |
5160 | static int igc_change_mtu(struct net_device *netdev, int new_mtu) |
5161 | { |
5162 | int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; |
5163 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
5164 | |
5165 | if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) { |
5166 | netdev_dbg(netdev, "Jumbo frames not supported with XDP" ); |
5167 | return -EINVAL; |
5168 | } |
5169 | |
5170 | /* adjust max frame to be at least the size of a standard frame */ |
5171 | if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) |
5172 | max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; |
5173 | |
5174 | while (test_and_set_bit(nr: __IGC_RESETTING, addr: &adapter->state)) |
5175 | usleep_range(min: 1000, max: 2000); |
5176 | |
5177 | /* igc_down has a dependency on max_frame_size */ |
5178 | adapter->max_frame_size = max_frame; |
5179 | |
5180 | if (netif_running(dev: netdev)) |
5181 | igc_down(adapter); |
5182 | |
5183 | netdev_dbg(netdev, "changing MTU from %d to %d\n" , netdev->mtu, new_mtu); |
5184 | netdev->mtu = new_mtu; |
5185 | |
5186 | if (netif_running(dev: netdev)) |
5187 | igc_up(adapter); |
5188 | else |
5189 | igc_reset(adapter); |
5190 | |
5191 | clear_bit(nr: __IGC_RESETTING, addr: &adapter->state); |
5192 | |
5193 | return 0; |
5194 | } |
5195 | |
5196 | /** |
5197 | * igc_tx_timeout - Respond to a Tx Hang |
5198 | * @netdev: network interface device structure |
5199 | * @txqueue: queue number that timed out |
5200 | **/ |
5201 | static void igc_tx_timeout(struct net_device *netdev, |
5202 | unsigned int __always_unused txqueue) |
5203 | { |
5204 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
5205 | struct igc_hw *hw = &adapter->hw; |
5206 | |
5207 | /* Do the reset outside of interrupt context */ |
5208 | adapter->tx_timeout_count++; |
5209 | schedule_work(work: &adapter->reset_task); |
5210 | wr32(IGC_EICS, |
5211 | (adapter->eims_enable_mask & ~adapter->eims_other)); |
5212 | } |
5213 | |
5214 | /** |
5215 | * igc_get_stats64 - Get System Network Statistics |
5216 | * @netdev: network interface device structure |
5217 | * @stats: rtnl_link_stats64 pointer |
5218 | * |
5219 | * Returns the address of the device statistics structure. |
5220 | * The statistics are updated here and also from the timer callback. |
5221 | */ |
5222 | static void igc_get_stats64(struct net_device *netdev, |
5223 | struct rtnl_link_stats64 *stats) |
5224 | { |
5225 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
5226 | |
5227 | spin_lock(lock: &adapter->stats64_lock); |
5228 | if (!test_bit(__IGC_RESETTING, &adapter->state)) |
5229 | igc_update_stats(adapter); |
5230 | memcpy(stats, &adapter->stats64, sizeof(*stats)); |
5231 | spin_unlock(lock: &adapter->stats64_lock); |
5232 | } |
5233 | |
5234 | static netdev_features_t igc_fix_features(struct net_device *netdev, |
5235 | netdev_features_t features) |
5236 | { |
5237 | /* Since there is no support for separate Rx/Tx vlan accel |
5238 | * enable/disable make sure Tx flag is always in same state as Rx. |
5239 | */ |
5240 | if (features & NETIF_F_HW_VLAN_CTAG_RX) |
5241 | features |= NETIF_F_HW_VLAN_CTAG_TX; |
5242 | else |
5243 | features &= ~NETIF_F_HW_VLAN_CTAG_TX; |
5244 | |
5245 | return features; |
5246 | } |
5247 | |
5248 | static int igc_set_features(struct net_device *netdev, |
5249 | netdev_features_t features) |
5250 | { |
5251 | netdev_features_t changed = netdev->features ^ features; |
5252 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
5253 | |
5254 | if (changed & NETIF_F_HW_VLAN_CTAG_RX) |
5255 | igc_vlan_mode(netdev, features); |
5256 | |
5257 | /* Add VLAN support */ |
5258 | if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) |
5259 | return 0; |
5260 | |
5261 | if (!(features & NETIF_F_NTUPLE)) |
5262 | igc_flush_nfc_rules(adapter); |
5263 | |
5264 | netdev->features = features; |
5265 | |
5266 | if (netif_running(dev: netdev)) |
5267 | igc_reinit_locked(adapter); |
5268 | else |
5269 | igc_reset(adapter); |
5270 | |
5271 | return 1; |
5272 | } |
5273 | |
5274 | static netdev_features_t |
5275 | igc_features_check(struct sk_buff *skb, struct net_device *dev, |
5276 | netdev_features_t features) |
5277 | { |
5278 | unsigned int network_hdr_len, mac_hdr_len; |
5279 | |
5280 | /* Make certain the headers can be described by a context descriptor */ |
5281 | mac_hdr_len = skb_network_header(skb) - skb->data; |
5282 | if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) |
5283 | return features & ~(NETIF_F_HW_CSUM | |
5284 | NETIF_F_SCTP_CRC | |
5285 | NETIF_F_HW_VLAN_CTAG_TX | |
5286 | NETIF_F_TSO | |
5287 | NETIF_F_TSO6); |
5288 | |
5289 | network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); |
5290 | if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) |
5291 | return features & ~(NETIF_F_HW_CSUM | |
5292 | NETIF_F_SCTP_CRC | |
5293 | NETIF_F_TSO | |
5294 | NETIF_F_TSO6); |
5295 | |
5296 | /* We can only support IPv4 TSO in tunnels if we can mangle the |
5297 | * inner IP ID field, so strip TSO if MANGLEID is not supported. |
5298 | */ |
5299 | if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) |
5300 | features &= ~NETIF_F_TSO; |
5301 | |
5302 | return features; |
5303 | } |
5304 | |
5305 | static void igc_tsync_interrupt(struct igc_adapter *adapter) |
5306 | { |
5307 | u32 ack, tsauxc, sec, nsec, tsicr; |
5308 | struct igc_hw *hw = &adapter->hw; |
5309 | struct ptp_clock_event event; |
5310 | struct timespec64 ts; |
5311 | |
5312 | tsicr = rd32(IGC_TSICR); |
5313 | ack = 0; |
5314 | |
5315 | if (tsicr & IGC_TSICR_SYS_WRAP) { |
5316 | event.type = PTP_CLOCK_PPS; |
5317 | if (adapter->ptp_caps.pps) |
5318 | ptp_clock_event(ptp: adapter->ptp_clock, event: &event); |
5319 | ack |= IGC_TSICR_SYS_WRAP; |
5320 | } |
5321 | |
5322 | if (tsicr & IGC_TSICR_TXTS) { |
5323 | /* retrieve hardware timestamp */ |
5324 | igc_ptp_tx_tstamp_event(adapter); |
5325 | ack |= IGC_TSICR_TXTS; |
5326 | } |
5327 | |
5328 | if (tsicr & IGC_TSICR_TT0) { |
5329 | spin_lock(lock: &adapter->tmreg_lock); |
5330 | ts = timespec64_add(lhs: adapter->perout[0].start, |
5331 | rhs: adapter->perout[0].period); |
5332 | wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0); |
5333 | wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec); |
5334 | tsauxc = rd32(IGC_TSAUXC); |
5335 | tsauxc |= IGC_TSAUXC_EN_TT0; |
5336 | wr32(IGC_TSAUXC, tsauxc); |
5337 | adapter->perout[0].start = ts; |
5338 | spin_unlock(lock: &adapter->tmreg_lock); |
5339 | ack |= IGC_TSICR_TT0; |
5340 | } |
5341 | |
5342 | if (tsicr & IGC_TSICR_TT1) { |
5343 | spin_lock(lock: &adapter->tmreg_lock); |
5344 | ts = timespec64_add(lhs: adapter->perout[1].start, |
5345 | rhs: adapter->perout[1].period); |
5346 | wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0); |
5347 | wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec); |
5348 | tsauxc = rd32(IGC_TSAUXC); |
5349 | tsauxc |= IGC_TSAUXC_EN_TT1; |
5350 | wr32(IGC_TSAUXC, tsauxc); |
5351 | adapter->perout[1].start = ts; |
5352 | spin_unlock(lock: &adapter->tmreg_lock); |
5353 | ack |= IGC_TSICR_TT1; |
5354 | } |
5355 | |
5356 | if (tsicr & IGC_TSICR_AUTT0) { |
5357 | nsec = rd32(IGC_AUXSTMPL0); |
5358 | sec = rd32(IGC_AUXSTMPH0); |
5359 | event.type = PTP_CLOCK_EXTTS; |
5360 | event.index = 0; |
5361 | event.timestamp = sec * NSEC_PER_SEC + nsec; |
5362 | ptp_clock_event(ptp: adapter->ptp_clock, event: &event); |
5363 | ack |= IGC_TSICR_AUTT0; |
5364 | } |
5365 | |
5366 | if (tsicr & IGC_TSICR_AUTT1) { |
5367 | nsec = rd32(IGC_AUXSTMPL1); |
5368 | sec = rd32(IGC_AUXSTMPH1); |
5369 | event.type = PTP_CLOCK_EXTTS; |
5370 | event.index = 1; |
5371 | event.timestamp = sec * NSEC_PER_SEC + nsec; |
5372 | ptp_clock_event(ptp: adapter->ptp_clock, event: &event); |
5373 | ack |= IGC_TSICR_AUTT1; |
5374 | } |
5375 | |
5376 | /* acknowledge the interrupts */ |
5377 | wr32(IGC_TSICR, ack); |
5378 | } |
5379 | |
5380 | /** |
5381 | * igc_msix_other - msix other interrupt handler |
5382 | * @irq: interrupt number |
5383 | * @data: pointer to a q_vector |
5384 | */ |
5385 | static irqreturn_t igc_msix_other(int irq, void *data) |
5386 | { |
5387 | struct igc_adapter *adapter = data; |
5388 | struct igc_hw *hw = &adapter->hw; |
5389 | u32 icr = rd32(IGC_ICR); |
5390 | |
5391 | /* reading ICR causes bit 31 of EICR to be cleared */ |
5392 | if (icr & IGC_ICR_DRSTA) |
5393 | schedule_work(work: &adapter->reset_task); |
5394 | |
5395 | if (icr & IGC_ICR_DOUTSYNC) { |
5396 | /* HW is reporting DMA is out of sync */ |
5397 | adapter->stats.doosync++; |
5398 | } |
5399 | |
5400 | if (icr & IGC_ICR_LSC) { |
5401 | hw->mac.get_link_status = true; |
5402 | /* guard against interrupt when we're going down */ |
5403 | if (!test_bit(__IGC_DOWN, &adapter->state)) |
5404 | mod_timer(timer: &adapter->watchdog_timer, expires: jiffies + 1); |
5405 | } |
5406 | |
5407 | if (icr & IGC_ICR_TS) |
5408 | igc_tsync_interrupt(adapter); |
5409 | |
5410 | wr32(IGC_EIMS, adapter->eims_other); |
5411 | |
5412 | return IRQ_HANDLED; |
5413 | } |
5414 | |
5415 | static void igc_write_itr(struct igc_q_vector *q_vector) |
5416 | { |
5417 | u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; |
5418 | |
5419 | if (!q_vector->set_itr) |
5420 | return; |
5421 | |
5422 | if (!itr_val) |
5423 | itr_val = IGC_ITR_VAL_MASK; |
5424 | |
5425 | itr_val |= IGC_EITR_CNT_IGNR; |
5426 | |
5427 | writel(val: itr_val, addr: q_vector->itr_register); |
5428 | q_vector->set_itr = 0; |
5429 | } |
5430 | |
5431 | static irqreturn_t igc_msix_ring(int irq, void *data) |
5432 | { |
5433 | struct igc_q_vector *q_vector = data; |
5434 | |
5435 | /* Write the ITR value calculated from the previous interrupt. */ |
5436 | igc_write_itr(q_vector); |
5437 | |
5438 | napi_schedule(n: &q_vector->napi); |
5439 | |
5440 | return IRQ_HANDLED; |
5441 | } |
5442 | |
5443 | /** |
5444 | * igc_request_msix - Initialize MSI-X interrupts |
5445 | * @adapter: Pointer to adapter structure |
5446 | * |
5447 | * igc_request_msix allocates MSI-X vectors and requests interrupts from the |
5448 | * kernel. |
5449 | */ |
5450 | static int igc_request_msix(struct igc_adapter *adapter) |
5451 | { |
5452 | unsigned int num_q_vectors = adapter->num_q_vectors; |
5453 | int i = 0, err = 0, vector = 0, free_vector = 0; |
5454 | struct net_device *netdev = adapter->netdev; |
5455 | |
5456 | err = request_irq(irq: adapter->msix_entries[vector].vector, |
5457 | handler: &igc_msix_other, flags: 0, name: netdev->name, dev: adapter); |
5458 | if (err) |
5459 | goto err_out; |
5460 | |
5461 | if (num_q_vectors > MAX_Q_VECTORS) { |
5462 | num_q_vectors = MAX_Q_VECTORS; |
5463 | dev_warn(&adapter->pdev->dev, |
5464 | "The number of queue vectors (%d) is higher than max allowed (%d)\n" , |
5465 | adapter->num_q_vectors, MAX_Q_VECTORS); |
5466 | } |
5467 | for (i = 0; i < num_q_vectors; i++) { |
5468 | struct igc_q_vector *q_vector = adapter->q_vector[i]; |
5469 | |
5470 | vector++; |
5471 | |
5472 | q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); |
5473 | |
5474 | if (q_vector->rx.ring && q_vector->tx.ring) |
5475 | sprintf(buf: q_vector->name, fmt: "%s-TxRx-%u" , netdev->name, |
5476 | q_vector->rx.ring->queue_index); |
5477 | else if (q_vector->tx.ring) |
5478 | sprintf(buf: q_vector->name, fmt: "%s-tx-%u" , netdev->name, |
5479 | q_vector->tx.ring->queue_index); |
5480 | else if (q_vector->rx.ring) |
5481 | sprintf(buf: q_vector->name, fmt: "%s-rx-%u" , netdev->name, |
5482 | q_vector->rx.ring->queue_index); |
5483 | else |
5484 | sprintf(buf: q_vector->name, fmt: "%s-unused" , netdev->name); |
5485 | |
5486 | err = request_irq(irq: adapter->msix_entries[vector].vector, |
5487 | handler: igc_msix_ring, flags: 0, name: q_vector->name, |
5488 | dev: q_vector); |
5489 | if (err) |
5490 | goto err_free; |
5491 | } |
5492 | |
5493 | igc_configure_msix(adapter); |
5494 | return 0; |
5495 | |
5496 | err_free: |
5497 | /* free already assigned IRQs */ |
5498 | free_irq(adapter->msix_entries[free_vector++].vector, adapter); |
5499 | |
5500 | vector--; |
5501 | for (i = 0; i < vector; i++) { |
5502 | free_irq(adapter->msix_entries[free_vector++].vector, |
5503 | adapter->q_vector[i]); |
5504 | } |
5505 | err_out: |
5506 | return err; |
5507 | } |
5508 | |
5509 | /** |
5510 | * igc_clear_interrupt_scheme - reset the device to a state of no interrupts |
5511 | * @adapter: Pointer to adapter structure |
5512 | * |
5513 | * This function resets the device so that it has 0 rx queues, tx queues, and |
5514 | * MSI-X interrupts allocated. |
5515 | */ |
5516 | static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) |
5517 | { |
5518 | igc_free_q_vectors(adapter); |
5519 | igc_reset_interrupt_capability(adapter); |
5520 | } |
5521 | |
5522 | /* Need to wait a few seconds after link up to get diagnostic information from |
5523 | * the phy |
5524 | */ |
5525 | static void igc_update_phy_info(struct timer_list *t) |
5526 | { |
5527 | struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); |
5528 | |
5529 | igc_get_phy_info(hw: &adapter->hw); |
5530 | } |
5531 | |
5532 | /** |
5533 | * igc_has_link - check shared code for link and determine up/down |
5534 | * @adapter: pointer to driver private info |
5535 | */ |
5536 | bool igc_has_link(struct igc_adapter *adapter) |
5537 | { |
5538 | struct igc_hw *hw = &adapter->hw; |
5539 | bool link_active = false; |
5540 | |
5541 | /* get_link_status is set on LSC (link status) interrupt or |
5542 | * rx sequence error interrupt. get_link_status will stay |
5543 | * false until the igc_check_for_link establishes link |
5544 | * for copper adapters ONLY |
5545 | */ |
5546 | if (!hw->mac.get_link_status) |
5547 | return true; |
5548 | hw->mac.ops.check_for_link(hw); |
5549 | link_active = !hw->mac.get_link_status; |
5550 | |
5551 | if (hw->mac.type == igc_i225) { |
5552 | if (!netif_carrier_ok(dev: adapter->netdev)) { |
5553 | adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; |
5554 | } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { |
5555 | adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; |
5556 | adapter->link_check_timeout = jiffies; |
5557 | } |
5558 | } |
5559 | |
5560 | return link_active; |
5561 | } |
5562 | |
5563 | /** |
5564 | * igc_watchdog - Timer Call-back |
5565 | * @t: timer for the watchdog |
5566 | */ |
5567 | static void igc_watchdog(struct timer_list *t) |
5568 | { |
5569 | struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); |
5570 | /* Do the rest outside of interrupt context */ |
5571 | schedule_work(work: &adapter->watchdog_task); |
5572 | } |
5573 | |
5574 | static void igc_watchdog_task(struct work_struct *work) |
5575 | { |
5576 | struct igc_adapter *adapter = container_of(work, |
5577 | struct igc_adapter, |
5578 | watchdog_task); |
5579 | struct net_device *netdev = adapter->netdev; |
5580 | struct igc_hw *hw = &adapter->hw; |
5581 | struct igc_phy_info *phy = &hw->phy; |
5582 | u16 phy_data, retry_count = 20; |
5583 | u32 link; |
5584 | int i; |
5585 | |
5586 | link = igc_has_link(adapter); |
5587 | |
5588 | if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { |
5589 | if (time_after(jiffies, (adapter->link_check_timeout + HZ))) |
5590 | adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; |
5591 | else |
5592 | link = false; |
5593 | } |
5594 | |
5595 | if (link) { |
5596 | /* Cancel scheduled suspend requests. */ |
5597 | pm_runtime_resume(dev: netdev->dev.parent); |
5598 | |
5599 | if (!netif_carrier_ok(dev: netdev)) { |
5600 | u32 ctrl; |
5601 | |
5602 | hw->mac.ops.get_speed_and_duplex(hw, |
5603 | &adapter->link_speed, |
5604 | &adapter->link_duplex); |
5605 | |
5606 | ctrl = rd32(IGC_CTRL); |
5607 | /* Link status message must follow this format */ |
5608 | netdev_info(dev: netdev, |
5609 | format: "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n" , |
5610 | adapter->link_speed, |
5611 | adapter->link_duplex == FULL_DUPLEX ? |
5612 | "Full" : "Half" , |
5613 | (ctrl & IGC_CTRL_TFCE) && |
5614 | (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : |
5615 | (ctrl & IGC_CTRL_RFCE) ? "RX" : |
5616 | (ctrl & IGC_CTRL_TFCE) ? "TX" : "None" ); |
5617 | |
5618 | /* disable EEE if enabled */ |
5619 | if ((adapter->flags & IGC_FLAG_EEE) && |
5620 | adapter->link_duplex == HALF_DUPLEX) { |
5621 | netdev_info(dev: netdev, |
5622 | format: "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n" ); |
5623 | adapter->hw.dev_spec._base.eee_enable = false; |
5624 | adapter->flags &= ~IGC_FLAG_EEE; |
5625 | } |
5626 | |
5627 | /* check if SmartSpeed worked */ |
5628 | igc_check_downshift(hw); |
5629 | if (phy->speed_downgraded) |
5630 | netdev_warn(dev: netdev, format: "Link Speed was downgraded by SmartSpeed\n" ); |
5631 | |
5632 | /* adjust timeout factor according to speed/duplex */ |
5633 | adapter->tx_timeout_factor = 1; |
5634 | switch (adapter->link_speed) { |
5635 | case SPEED_10: |
5636 | adapter->tx_timeout_factor = 14; |
5637 | break; |
5638 | case SPEED_100: |
5639 | case SPEED_1000: |
5640 | case SPEED_2500: |
5641 | adapter->tx_timeout_factor = 1; |
5642 | break; |
5643 | } |
5644 | |
5645 | /* Once the launch time has been set on the wire, there |
5646 | * is a delay before the link speed can be determined |
5647 | * based on link-up activity. Write into the register |
5648 | * as soon as we know the correct link speed. |
5649 | */ |
5650 | igc_tsn_adjust_txtime_offset(adapter); |
5651 | |
5652 | if (adapter->link_speed != SPEED_1000) |
5653 | goto no_wait; |
5654 | |
5655 | /* wait for Remote receiver status OK */ |
5656 | retry_read_status: |
5657 | if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, |
5658 | data: &phy_data)) { |
5659 | if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && |
5660 | retry_count) { |
5661 | msleep(msecs: 100); |
5662 | retry_count--; |
5663 | goto retry_read_status; |
5664 | } else if (!retry_count) { |
5665 | netdev_err(dev: netdev, format: "exceed max 2 second\n" ); |
5666 | } |
5667 | } else { |
5668 | netdev_err(dev: netdev, format: "read 1000Base-T Status Reg\n" ); |
5669 | } |
5670 | no_wait: |
5671 | netif_carrier_on(dev: netdev); |
5672 | |
5673 | /* link state has changed, schedule phy info update */ |
5674 | if (!test_bit(__IGC_DOWN, &adapter->state)) |
5675 | mod_timer(timer: &adapter->phy_info_timer, |
5676 | expires: round_jiffies(j: jiffies + 2 * HZ)); |
5677 | } |
5678 | } else { |
5679 | if (netif_carrier_ok(dev: netdev)) { |
5680 | adapter->link_speed = 0; |
5681 | adapter->link_duplex = 0; |
5682 | |
5683 | /* Links status message must follow this format */ |
5684 | netdev_info(dev: netdev, format: "NIC Link is Down\n" ); |
5685 | netif_carrier_off(dev: netdev); |
5686 | |
5687 | /* link state has changed, schedule phy info update */ |
5688 | if (!test_bit(__IGC_DOWN, &adapter->state)) |
5689 | mod_timer(timer: &adapter->phy_info_timer, |
5690 | expires: round_jiffies(j: jiffies + 2 * HZ)); |
5691 | |
5692 | pm_schedule_suspend(dev: netdev->dev.parent, |
5693 | MSEC_PER_SEC * 5); |
5694 | } |
5695 | } |
5696 | |
5697 | spin_lock(lock: &adapter->stats64_lock); |
5698 | igc_update_stats(adapter); |
5699 | spin_unlock(lock: &adapter->stats64_lock); |
5700 | |
5701 | for (i = 0; i < adapter->num_tx_queues; i++) { |
5702 | struct igc_ring *tx_ring = adapter->tx_ring[i]; |
5703 | |
5704 | if (!netif_carrier_ok(dev: netdev)) { |
5705 | /* We've lost link, so the controller stops DMA, |
5706 | * but we've got queued Tx work that's never going |
5707 | * to get done, so reset controller to flush Tx. |
5708 | * (Do the reset outside of interrupt context). |
5709 | */ |
5710 | if (igc_desc_unused(ring: tx_ring) + 1 < tx_ring->count) { |
5711 | adapter->tx_timeout_count++; |
5712 | schedule_work(work: &adapter->reset_task); |
5713 | /* return immediately since reset is imminent */ |
5714 | return; |
5715 | } |
5716 | } |
5717 | |
5718 | /* Force detection of hung controller every watchdog period */ |
5719 | set_bit(nr: IGC_RING_FLAG_TX_DETECT_HANG, addr: &tx_ring->flags); |
5720 | } |
5721 | |
5722 | /* Cause software interrupt to ensure Rx ring is cleaned */ |
5723 | if (adapter->flags & IGC_FLAG_HAS_MSIX) { |
5724 | u32 eics = 0; |
5725 | |
5726 | for (i = 0; i < adapter->num_q_vectors; i++) |
5727 | eics |= adapter->q_vector[i]->eims_value; |
5728 | wr32(IGC_EICS, eics); |
5729 | } else { |
5730 | wr32(IGC_ICS, IGC_ICS_RXDMT0); |
5731 | } |
5732 | |
5733 | igc_ptp_tx_hang(adapter); |
5734 | |
5735 | /* Reset the timer */ |
5736 | if (!test_bit(__IGC_DOWN, &adapter->state)) { |
5737 | if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) |
5738 | mod_timer(timer: &adapter->watchdog_timer, |
5739 | expires: round_jiffies(j: jiffies + HZ)); |
5740 | else |
5741 | mod_timer(timer: &adapter->watchdog_timer, |
5742 | expires: round_jiffies(j: jiffies + 2 * HZ)); |
5743 | } |
5744 | } |
5745 | |
5746 | /** |
5747 | * igc_intr_msi - Interrupt Handler |
5748 | * @irq: interrupt number |
5749 | * @data: pointer to a network interface device structure |
5750 | */ |
5751 | static irqreturn_t igc_intr_msi(int irq, void *data) |
5752 | { |
5753 | struct igc_adapter *adapter = data; |
5754 | struct igc_q_vector *q_vector = adapter->q_vector[0]; |
5755 | struct igc_hw *hw = &adapter->hw; |
5756 | /* read ICR disables interrupts using IAM */ |
5757 | u32 icr = rd32(IGC_ICR); |
5758 | |
5759 | igc_write_itr(q_vector); |
5760 | |
5761 | if (icr & IGC_ICR_DRSTA) |
5762 | schedule_work(work: &adapter->reset_task); |
5763 | |
5764 | if (icr & IGC_ICR_DOUTSYNC) { |
5765 | /* HW is reporting DMA is out of sync */ |
5766 | adapter->stats.doosync++; |
5767 | } |
5768 | |
5769 | if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { |
5770 | hw->mac.get_link_status = true; |
5771 | if (!test_bit(__IGC_DOWN, &adapter->state)) |
5772 | mod_timer(timer: &adapter->watchdog_timer, expires: jiffies + 1); |
5773 | } |
5774 | |
5775 | if (icr & IGC_ICR_TS) |
5776 | igc_tsync_interrupt(adapter); |
5777 | |
5778 | napi_schedule(n: &q_vector->napi); |
5779 | |
5780 | return IRQ_HANDLED; |
5781 | } |
5782 | |
5783 | /** |
5784 | * igc_intr - Legacy Interrupt Handler |
5785 | * @irq: interrupt number |
5786 | * @data: pointer to a network interface device structure |
5787 | */ |
5788 | static irqreturn_t igc_intr(int irq, void *data) |
5789 | { |
5790 | struct igc_adapter *adapter = data; |
5791 | struct igc_q_vector *q_vector = adapter->q_vector[0]; |
5792 | struct igc_hw *hw = &adapter->hw; |
5793 | /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No |
5794 | * need for the IMC write |
5795 | */ |
5796 | u32 icr = rd32(IGC_ICR); |
5797 | |
5798 | /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is |
5799 | * not set, then the adapter didn't send an interrupt |
5800 | */ |
5801 | if (!(icr & IGC_ICR_INT_ASSERTED)) |
5802 | return IRQ_NONE; |
5803 | |
5804 | igc_write_itr(q_vector); |
5805 | |
5806 | if (icr & IGC_ICR_DRSTA) |
5807 | schedule_work(work: &adapter->reset_task); |
5808 | |
5809 | if (icr & IGC_ICR_DOUTSYNC) { |
5810 | /* HW is reporting DMA is out of sync */ |
5811 | adapter->stats.doosync++; |
5812 | } |
5813 | |
5814 | if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { |
5815 | hw->mac.get_link_status = true; |
5816 | /* guard against interrupt when we're going down */ |
5817 | if (!test_bit(__IGC_DOWN, &adapter->state)) |
5818 | mod_timer(timer: &adapter->watchdog_timer, expires: jiffies + 1); |
5819 | } |
5820 | |
5821 | if (icr & IGC_ICR_TS) |
5822 | igc_tsync_interrupt(adapter); |
5823 | |
5824 | napi_schedule(n: &q_vector->napi); |
5825 | |
5826 | return IRQ_HANDLED; |
5827 | } |
5828 | |
5829 | static void igc_free_irq(struct igc_adapter *adapter) |
5830 | { |
5831 | if (adapter->msix_entries) { |
5832 | int vector = 0, i; |
5833 | |
5834 | free_irq(adapter->msix_entries[vector++].vector, adapter); |
5835 | |
5836 | for (i = 0; i < adapter->num_q_vectors; i++) |
5837 | free_irq(adapter->msix_entries[vector++].vector, |
5838 | adapter->q_vector[i]); |
5839 | } else { |
5840 | free_irq(adapter->pdev->irq, adapter); |
5841 | } |
5842 | } |
5843 | |
5844 | /** |
5845 | * igc_request_irq - initialize interrupts |
5846 | * @adapter: Pointer to adapter structure |
5847 | * |
5848 | * Attempts to configure interrupts using the best available |
5849 | * capabilities of the hardware and kernel. |
5850 | */ |
5851 | static int igc_request_irq(struct igc_adapter *adapter) |
5852 | { |
5853 | struct net_device *netdev = adapter->netdev; |
5854 | struct pci_dev *pdev = adapter->pdev; |
5855 | int err = 0; |
5856 | |
5857 | if (adapter->flags & IGC_FLAG_HAS_MSIX) { |
5858 | err = igc_request_msix(adapter); |
5859 | if (!err) |
5860 | goto request_done; |
5861 | /* fall back to MSI */ |
5862 | igc_free_all_tx_resources(adapter); |
5863 | igc_free_all_rx_resources(adapter); |
5864 | |
5865 | igc_clear_interrupt_scheme(adapter); |
5866 | err = igc_init_interrupt_scheme(adapter, msix: false); |
5867 | if (err) |
5868 | goto request_done; |
5869 | igc_setup_all_tx_resources(adapter); |
5870 | igc_setup_all_rx_resources(adapter); |
5871 | igc_configure(adapter); |
5872 | } |
5873 | |
5874 | igc_assign_vector(q_vector: adapter->q_vector[0], msix_vector: 0); |
5875 | |
5876 | if (adapter->flags & IGC_FLAG_HAS_MSI) { |
5877 | err = request_irq(irq: pdev->irq, handler: &igc_intr_msi, flags: 0, |
5878 | name: netdev->name, dev: adapter); |
5879 | if (!err) |
5880 | goto request_done; |
5881 | |
5882 | /* fall back to legacy interrupts */ |
5883 | igc_reset_interrupt_capability(adapter); |
5884 | adapter->flags &= ~IGC_FLAG_HAS_MSI; |
5885 | } |
5886 | |
5887 | err = request_irq(irq: pdev->irq, handler: &igc_intr, IRQF_SHARED, |
5888 | name: netdev->name, dev: adapter); |
5889 | |
5890 | if (err) |
5891 | netdev_err(dev: netdev, format: "Error %d getting interrupt\n" , err); |
5892 | |
5893 | request_done: |
5894 | return err; |
5895 | } |
5896 | |
5897 | /** |
5898 | * __igc_open - Called when a network interface is made active |
5899 | * @netdev: network interface device structure |
5900 | * @resuming: boolean indicating if the device is resuming |
5901 | * |
5902 | * Returns 0 on success, negative value on failure |
5903 | * |
5904 | * The open entry point is called when a network interface is made |
5905 | * active by the system (IFF_UP). At this point all resources needed |
5906 | * for transmit and receive operations are allocated, the interrupt |
5907 | * handler is registered with the OS, the watchdog timer is started, |
5908 | * and the stack is notified that the interface is ready. |
5909 | */ |
5910 | static int __igc_open(struct net_device *netdev, bool resuming) |
5911 | { |
5912 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
5913 | struct pci_dev *pdev = adapter->pdev; |
5914 | struct igc_hw *hw = &adapter->hw; |
5915 | int err = 0; |
5916 | int i = 0; |
5917 | |
5918 | /* disallow open during test */ |
5919 | |
5920 | if (test_bit(__IGC_TESTING, &adapter->state)) { |
5921 | WARN_ON(resuming); |
5922 | return -EBUSY; |
5923 | } |
5924 | |
5925 | if (!resuming) |
5926 | pm_runtime_get_sync(dev: &pdev->dev); |
5927 | |
5928 | netif_carrier_off(dev: netdev); |
5929 | |
5930 | /* allocate transmit descriptors */ |
5931 | err = igc_setup_all_tx_resources(adapter); |
5932 | if (err) |
5933 | goto err_setup_tx; |
5934 | |
5935 | /* allocate receive descriptors */ |
5936 | err = igc_setup_all_rx_resources(adapter); |
5937 | if (err) |
5938 | goto err_setup_rx; |
5939 | |
5940 | igc_power_up_link(adapter); |
5941 | |
5942 | igc_configure(adapter); |
5943 | |
5944 | err = igc_request_irq(adapter); |
5945 | if (err) |
5946 | goto err_req_irq; |
5947 | |
5948 | /* Notify the stack of the actual queue counts. */ |
5949 | err = netif_set_real_num_tx_queues(dev: netdev, txq: adapter->num_tx_queues); |
5950 | if (err) |
5951 | goto err_set_queues; |
5952 | |
5953 | err = netif_set_real_num_rx_queues(dev: netdev, rxq: adapter->num_rx_queues); |
5954 | if (err) |
5955 | goto err_set_queues; |
5956 | |
5957 | clear_bit(nr: __IGC_DOWN, addr: &adapter->state); |
5958 | |
5959 | for (i = 0; i < adapter->num_q_vectors; i++) |
5960 | napi_enable(n: &adapter->q_vector[i]->napi); |
5961 | |
5962 | /* Clear any pending interrupts. */ |
5963 | rd32(IGC_ICR); |
5964 | igc_irq_enable(adapter); |
5965 | |
5966 | if (!resuming) |
5967 | pm_runtime_put(dev: &pdev->dev); |
5968 | |
5969 | netif_tx_start_all_queues(dev: netdev); |
5970 | |
5971 | /* start the watchdog. */ |
5972 | hw->mac.get_link_status = true; |
5973 | schedule_work(work: &adapter->watchdog_task); |
5974 | |
5975 | return IGC_SUCCESS; |
5976 | |
5977 | err_set_queues: |
5978 | igc_free_irq(adapter); |
5979 | err_req_irq: |
5980 | igc_release_hw_control(adapter); |
5981 | igc_power_down_phy_copper_base(hw: &adapter->hw); |
5982 | igc_free_all_rx_resources(adapter); |
5983 | err_setup_rx: |
5984 | igc_free_all_tx_resources(adapter); |
5985 | err_setup_tx: |
5986 | igc_reset(adapter); |
5987 | if (!resuming) |
5988 | pm_runtime_put(dev: &pdev->dev); |
5989 | |
5990 | return err; |
5991 | } |
5992 | |
5993 | int igc_open(struct net_device *netdev) |
5994 | { |
5995 | return __igc_open(netdev, resuming: false); |
5996 | } |
5997 | |
5998 | /** |
5999 | * __igc_close - Disables a network interface |
6000 | * @netdev: network interface device structure |
6001 | * @suspending: boolean indicating the device is suspending |
6002 | * |
6003 | * Returns 0, this is not allowed to fail |
6004 | * |
6005 | * The close entry point is called when an interface is de-activated |
6006 | * by the OS. The hardware is still under the driver's control, but |
6007 | * needs to be disabled. A global MAC reset is issued to stop the |
6008 | * hardware, and all transmit and receive resources are freed. |
6009 | */ |
6010 | static int __igc_close(struct net_device *netdev, bool suspending) |
6011 | { |
6012 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
6013 | struct pci_dev *pdev = adapter->pdev; |
6014 | |
6015 | WARN_ON(test_bit(__IGC_RESETTING, &adapter->state)); |
6016 | |
6017 | if (!suspending) |
6018 | pm_runtime_get_sync(dev: &pdev->dev); |
6019 | |
6020 | igc_down(adapter); |
6021 | |
6022 | igc_release_hw_control(adapter); |
6023 | |
6024 | igc_free_irq(adapter); |
6025 | |
6026 | igc_free_all_tx_resources(adapter); |
6027 | igc_free_all_rx_resources(adapter); |
6028 | |
6029 | if (!suspending) |
6030 | pm_runtime_put_sync(dev: &pdev->dev); |
6031 | |
6032 | return 0; |
6033 | } |
6034 | |
6035 | int igc_close(struct net_device *netdev) |
6036 | { |
6037 | if (netif_device_present(dev: netdev) || netdev->dismantle) |
6038 | return __igc_close(netdev, suspending: false); |
6039 | return 0; |
6040 | } |
6041 | |
6042 | /** |
6043 | * igc_ioctl - Access the hwtstamp interface |
6044 | * @netdev: network interface device structure |
6045 | * @ifr: interface request data |
6046 | * @cmd: ioctl command |
6047 | **/ |
6048 | static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) |
6049 | { |
6050 | switch (cmd) { |
6051 | case SIOCGHWTSTAMP: |
6052 | return igc_ptp_get_ts_config(netdev, ifr); |
6053 | case SIOCSHWTSTAMP: |
6054 | return igc_ptp_set_ts_config(netdev, ifr); |
6055 | default: |
6056 | return -EOPNOTSUPP; |
6057 | } |
6058 | } |
6059 | |
6060 | static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue, |
6061 | bool enable) |
6062 | { |
6063 | struct igc_ring *ring; |
6064 | |
6065 | if (queue < 0 || queue >= adapter->num_tx_queues) |
6066 | return -EINVAL; |
6067 | |
6068 | ring = adapter->tx_ring[queue]; |
6069 | ring->launchtime_enable = enable; |
6070 | |
6071 | return 0; |
6072 | } |
6073 | |
6074 | static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now) |
6075 | { |
6076 | struct timespec64 b; |
6077 | |
6078 | b = ktime_to_timespec64(base_time); |
6079 | |
6080 | return timespec64_compare(lhs: now, rhs: &b) > 0; |
6081 | } |
6082 | |
6083 | static bool validate_schedule(struct igc_adapter *adapter, |
6084 | const struct tc_taprio_qopt_offload *qopt) |
6085 | { |
6086 | int queue_uses[IGC_MAX_TX_QUEUES] = { }; |
6087 | struct igc_hw *hw = &adapter->hw; |
6088 | struct timespec64 now; |
6089 | size_t n; |
6090 | |
6091 | if (qopt->cycle_time_extension) |
6092 | return false; |
6093 | |
6094 | igc_ptp_read(adapter, ts: &now); |
6095 | |
6096 | /* If we program the controller's BASET registers with a time |
6097 | * in the future, it will hold all the packets until that |
6098 | * time, causing a lot of TX Hangs, so to avoid that, we |
6099 | * reject schedules that would start in the future. |
6100 | * Note: Limitation above is no longer in i226. |
6101 | */ |
6102 | if (!is_base_time_past(base_time: qopt->base_time, now: &now) && |
6103 | igc_is_device_id_i225(hw)) |
6104 | return false; |
6105 | |
6106 | for (n = 0; n < qopt->num_entries; n++) { |
6107 | const struct tc_taprio_sched_entry *e, *prev; |
6108 | int i; |
6109 | |
6110 | prev = n ? &qopt->entries[n - 1] : NULL; |
6111 | e = &qopt->entries[n]; |
6112 | |
6113 | /* i225 only supports "global" frame preemption |
6114 | * settings. |
6115 | */ |
6116 | if (e->command != TC_TAPRIO_CMD_SET_GATES) |
6117 | return false; |
6118 | |
6119 | for (i = 0; i < adapter->num_tx_queues; i++) |
6120 | if (e->gate_mask & BIT(i)) { |
6121 | queue_uses[i]++; |
6122 | |
6123 | /* There are limitations: A single queue cannot |
6124 | * be opened and closed multiple times per cycle |
6125 | * unless the gate stays open. Check for it. |
6126 | */ |
6127 | if (queue_uses[i] > 1 && |
6128 | !(prev->gate_mask & BIT(i))) |
6129 | return false; |
6130 | } |
6131 | } |
6132 | |
6133 | return true; |
6134 | } |
6135 | |
6136 | static int igc_tsn_enable_launchtime(struct igc_adapter *adapter, |
6137 | struct tc_etf_qopt_offload *qopt) |
6138 | { |
6139 | struct igc_hw *hw = &adapter->hw; |
6140 | int err; |
6141 | |
6142 | if (hw->mac.type != igc_i225) |
6143 | return -EOPNOTSUPP; |
6144 | |
6145 | err = igc_save_launchtime_params(adapter, queue: qopt->queue, enable: qopt->enable); |
6146 | if (err) |
6147 | return err; |
6148 | |
6149 | return igc_tsn_offload_apply(adapter); |
6150 | } |
6151 | |
6152 | static int igc_qbv_clear_schedule(struct igc_adapter *adapter) |
6153 | { |
6154 | unsigned long flags; |
6155 | int i; |
6156 | |
6157 | adapter->base_time = 0; |
6158 | adapter->cycle_time = NSEC_PER_SEC; |
6159 | adapter->taprio_offload_enable = false; |
6160 | adapter->qbv_config_change_errors = 0; |
6161 | adapter->qbv_count = 0; |
6162 | |
6163 | for (i = 0; i < adapter->num_tx_queues; i++) { |
6164 | struct igc_ring *ring = adapter->tx_ring[i]; |
6165 | |
6166 | ring->start_time = 0; |
6167 | ring->end_time = NSEC_PER_SEC; |
6168 | ring->max_sdu = 0; |
6169 | } |
6170 | |
6171 | spin_lock_irqsave(&adapter->qbv_tx_lock, flags); |
6172 | |
6173 | adapter->qbv_transition = false; |
6174 | |
6175 | for (i = 0; i < adapter->num_tx_queues; i++) { |
6176 | struct igc_ring *ring = adapter->tx_ring[i]; |
6177 | |
6178 | ring->oper_gate_closed = false; |
6179 | ring->admin_gate_closed = false; |
6180 | } |
6181 | |
6182 | spin_unlock_irqrestore(lock: &adapter->qbv_tx_lock, flags); |
6183 | |
6184 | return 0; |
6185 | } |
6186 | |
6187 | static int igc_tsn_clear_schedule(struct igc_adapter *adapter) |
6188 | { |
6189 | igc_qbv_clear_schedule(adapter); |
6190 | |
6191 | return 0; |
6192 | } |
6193 | |
6194 | static void igc_taprio_stats(struct net_device *dev, |
6195 | struct tc_taprio_qopt_stats *stats) |
6196 | { |
6197 | /* When Strict_End is enabled, the tx_overruns counter |
6198 | * will always be zero. |
6199 | */ |
6200 | stats->tx_overruns = 0; |
6201 | } |
6202 | |
6203 | static void igc_taprio_queue_stats(struct net_device *dev, |
6204 | struct tc_taprio_qopt_queue_stats *queue_stats) |
6205 | { |
6206 | struct tc_taprio_qopt_stats *stats = &queue_stats->stats; |
6207 | |
6208 | /* When Strict_End is enabled, the tx_overruns counter |
6209 | * will always be zero. |
6210 | */ |
6211 | stats->tx_overruns = 0; |
6212 | } |
6213 | |
6214 | static int igc_save_qbv_schedule(struct igc_adapter *adapter, |
6215 | struct tc_taprio_qopt_offload *qopt) |
6216 | { |
6217 | bool queue_configured[IGC_MAX_TX_QUEUES] = { }; |
6218 | struct igc_hw *hw = &adapter->hw; |
6219 | u32 start_time = 0, end_time = 0; |
6220 | struct timespec64 now; |
6221 | unsigned long flags; |
6222 | size_t n; |
6223 | int i; |
6224 | |
6225 | switch (qopt->cmd) { |
6226 | case TAPRIO_CMD_REPLACE: |
6227 | break; |
6228 | case TAPRIO_CMD_DESTROY: |
6229 | return igc_tsn_clear_schedule(adapter); |
6230 | case TAPRIO_CMD_STATS: |
6231 | igc_taprio_stats(dev: adapter->netdev, stats: &qopt->stats); |
6232 | return 0; |
6233 | case TAPRIO_CMD_QUEUE_STATS: |
6234 | igc_taprio_queue_stats(dev: adapter->netdev, queue_stats: &qopt->queue_stats); |
6235 | return 0; |
6236 | default: |
6237 | return -EOPNOTSUPP; |
6238 | } |
6239 | |
6240 | if (qopt->base_time < 0) |
6241 | return -ERANGE; |
6242 | |
6243 | if (igc_is_device_id_i225(hw) && adapter->taprio_offload_enable) |
6244 | return -EALREADY; |
6245 | |
6246 | if (!validate_schedule(adapter, qopt)) |
6247 | return -EINVAL; |
6248 | |
6249 | adapter->cycle_time = qopt->cycle_time; |
6250 | adapter->base_time = qopt->base_time; |
6251 | adapter->taprio_offload_enable = true; |
6252 | |
6253 | igc_ptp_read(adapter, ts: &now); |
6254 | |
6255 | for (n = 0; n < qopt->num_entries; n++) { |
6256 | struct tc_taprio_sched_entry *e = &qopt->entries[n]; |
6257 | |
6258 | end_time += e->interval; |
6259 | |
6260 | /* If any of the conditions below are true, we need to manually |
6261 | * control the end time of the cycle. |
6262 | * 1. Qbv users can specify a cycle time that is not equal |
6263 | * to the total GCL intervals. Hence, recalculation is |
6264 | * necessary here to exclude the time interval that |
6265 | * exceeds the cycle time. |
6266 | * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2, |
6267 | * once the end of the list is reached, it will switch |
6268 | * to the END_OF_CYCLE state and leave the gates in the |
6269 | * same state until the next cycle is started. |
6270 | */ |
6271 | if (end_time > adapter->cycle_time || |
6272 | n + 1 == qopt->num_entries) |
6273 | end_time = adapter->cycle_time; |
6274 | |
6275 | for (i = 0; i < adapter->num_tx_queues; i++) { |
6276 | struct igc_ring *ring = adapter->tx_ring[i]; |
6277 | |
6278 | if (!(e->gate_mask & BIT(i))) |
6279 | continue; |
6280 | |
6281 | /* Check whether a queue stays open for more than one |
6282 | * entry. If so, keep the start and advance the end |
6283 | * time. |
6284 | */ |
6285 | if (!queue_configured[i]) |
6286 | ring->start_time = start_time; |
6287 | ring->end_time = end_time; |
6288 | |
6289 | if (ring->start_time >= adapter->cycle_time) |
6290 | queue_configured[i] = false; |
6291 | else |
6292 | queue_configured[i] = true; |
6293 | } |
6294 | |
6295 | start_time += e->interval; |
6296 | } |
6297 | |
6298 | spin_lock_irqsave(&adapter->qbv_tx_lock, flags); |
6299 | |
6300 | /* Check whether a queue gets configured. |
6301 | * If not, set the start and end time to be end time. |
6302 | */ |
6303 | for (i = 0; i < adapter->num_tx_queues; i++) { |
6304 | struct igc_ring *ring = adapter->tx_ring[i]; |
6305 | |
6306 | if (!is_base_time_past(base_time: qopt->base_time, now: &now)) { |
6307 | ring->admin_gate_closed = false; |
6308 | } else { |
6309 | ring->oper_gate_closed = false; |
6310 | ring->admin_gate_closed = false; |
6311 | } |
6312 | |
6313 | if (!queue_configured[i]) { |
6314 | if (!is_base_time_past(base_time: qopt->base_time, now: &now)) |
6315 | ring->admin_gate_closed = true; |
6316 | else |
6317 | ring->oper_gate_closed = true; |
6318 | |
6319 | ring->start_time = end_time; |
6320 | ring->end_time = end_time; |
6321 | } |
6322 | } |
6323 | |
6324 | spin_unlock_irqrestore(lock: &adapter->qbv_tx_lock, flags); |
6325 | |
6326 | for (i = 0; i < adapter->num_tx_queues; i++) { |
6327 | struct igc_ring *ring = adapter->tx_ring[i]; |
6328 | struct net_device *dev = adapter->netdev; |
6329 | |
6330 | if (qopt->max_sdu[i]) |
6331 | ring->max_sdu = qopt->max_sdu[i] + dev->hard_header_len - ETH_TLEN; |
6332 | else |
6333 | ring->max_sdu = 0; |
6334 | } |
6335 | |
6336 | return 0; |
6337 | } |
6338 | |
6339 | static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter, |
6340 | struct tc_taprio_qopt_offload *qopt) |
6341 | { |
6342 | struct igc_hw *hw = &adapter->hw; |
6343 | int err; |
6344 | |
6345 | if (hw->mac.type != igc_i225) |
6346 | return -EOPNOTSUPP; |
6347 | |
6348 | err = igc_save_qbv_schedule(adapter, qopt); |
6349 | if (err) |
6350 | return err; |
6351 | |
6352 | return igc_tsn_offload_apply(adapter); |
6353 | } |
6354 | |
6355 | static int igc_save_cbs_params(struct igc_adapter *adapter, int queue, |
6356 | bool enable, int idleslope, int sendslope, |
6357 | int hicredit, int locredit) |
6358 | { |
6359 | bool cbs_status[IGC_MAX_SR_QUEUES] = { false }; |
6360 | struct net_device *netdev = adapter->netdev; |
6361 | struct igc_ring *ring; |
6362 | int i; |
6363 | |
6364 | /* i225 has two sets of credit-based shaper logic. |
6365 | * Supporting it only on the top two priority queues |
6366 | */ |
6367 | if (queue < 0 || queue > 1) |
6368 | return -EINVAL; |
6369 | |
6370 | ring = adapter->tx_ring[queue]; |
6371 | |
6372 | for (i = 0; i < IGC_MAX_SR_QUEUES; i++) |
6373 | if (adapter->tx_ring[i]) |
6374 | cbs_status[i] = adapter->tx_ring[i]->cbs_enable; |
6375 | |
6376 | /* CBS should be enabled on the highest priority queue first in order |
6377 | * for the CBS algorithm to operate as intended. |
6378 | */ |
6379 | if (enable) { |
6380 | if (queue == 1 && !cbs_status[0]) { |
6381 | netdev_err(dev: netdev, |
6382 | format: "Enabling CBS on queue1 before queue0\n" ); |
6383 | return -EINVAL; |
6384 | } |
6385 | } else { |
6386 | if (queue == 0 && cbs_status[1]) { |
6387 | netdev_err(dev: netdev, |
6388 | format: "Disabling CBS on queue0 before queue1\n" ); |
6389 | return -EINVAL; |
6390 | } |
6391 | } |
6392 | |
6393 | ring->cbs_enable = enable; |
6394 | ring->idleslope = idleslope; |
6395 | ring->sendslope = sendslope; |
6396 | ring->hicredit = hicredit; |
6397 | ring->locredit = locredit; |
6398 | |
6399 | return 0; |
6400 | } |
6401 | |
6402 | static int igc_tsn_enable_cbs(struct igc_adapter *adapter, |
6403 | struct tc_cbs_qopt_offload *qopt) |
6404 | { |
6405 | struct igc_hw *hw = &adapter->hw; |
6406 | int err; |
6407 | |
6408 | if (hw->mac.type != igc_i225) |
6409 | return -EOPNOTSUPP; |
6410 | |
6411 | if (qopt->queue < 0 || qopt->queue > 1) |
6412 | return -EINVAL; |
6413 | |
6414 | err = igc_save_cbs_params(adapter, queue: qopt->queue, enable: qopt->enable, |
6415 | idleslope: qopt->idleslope, sendslope: qopt->sendslope, |
6416 | hicredit: qopt->hicredit, locredit: qopt->locredit); |
6417 | if (err) |
6418 | return err; |
6419 | |
6420 | return igc_tsn_offload_apply(adapter); |
6421 | } |
6422 | |
6423 | static int igc_tc_query_caps(struct igc_adapter *adapter, |
6424 | struct tc_query_caps_base *base) |
6425 | { |
6426 | struct igc_hw *hw = &adapter->hw; |
6427 | |
6428 | switch (base->type) { |
6429 | case TC_SETUP_QDISC_TAPRIO: { |
6430 | struct tc_taprio_caps *caps = base->caps; |
6431 | |
6432 | caps->broken_mqprio = true; |
6433 | |
6434 | if (hw->mac.type == igc_i225) { |
6435 | caps->supports_queue_max_sdu = true; |
6436 | caps->gate_mask_per_txq = true; |
6437 | } |
6438 | |
6439 | return 0; |
6440 | } |
6441 | default: |
6442 | return -EOPNOTSUPP; |
6443 | } |
6444 | } |
6445 | |
6446 | static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, |
6447 | void *type_data) |
6448 | { |
6449 | struct igc_adapter *adapter = netdev_priv(dev); |
6450 | |
6451 | adapter->tc_setup_type = type; |
6452 | |
6453 | switch (type) { |
6454 | case TC_QUERY_CAPS: |
6455 | return igc_tc_query_caps(adapter, base: type_data); |
6456 | case TC_SETUP_QDISC_TAPRIO: |
6457 | return igc_tsn_enable_qbv_scheduling(adapter, qopt: type_data); |
6458 | |
6459 | case TC_SETUP_QDISC_ETF: |
6460 | return igc_tsn_enable_launchtime(adapter, qopt: type_data); |
6461 | |
6462 | case TC_SETUP_QDISC_CBS: |
6463 | return igc_tsn_enable_cbs(adapter, qopt: type_data); |
6464 | |
6465 | default: |
6466 | return -EOPNOTSUPP; |
6467 | } |
6468 | } |
6469 | |
6470 | static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf) |
6471 | { |
6472 | struct igc_adapter *adapter = netdev_priv(dev); |
6473 | |
6474 | switch (bpf->command) { |
6475 | case XDP_SETUP_PROG: |
6476 | return igc_xdp_set_prog(adapter, prog: bpf->prog, extack: bpf->extack); |
6477 | case XDP_SETUP_XSK_POOL: |
6478 | return igc_xdp_setup_pool(adapter, pool: bpf->xsk.pool, |
6479 | queue_id: bpf->xsk.queue_id); |
6480 | default: |
6481 | return -EOPNOTSUPP; |
6482 | } |
6483 | } |
6484 | |
6485 | static int igc_xdp_xmit(struct net_device *dev, int num_frames, |
6486 | struct xdp_frame **frames, u32 flags) |
6487 | { |
6488 | struct igc_adapter *adapter = netdev_priv(dev); |
6489 | int cpu = smp_processor_id(); |
6490 | struct netdev_queue *nq; |
6491 | struct igc_ring *ring; |
6492 | int i, drops; |
6493 | |
6494 | if (unlikely(!netif_carrier_ok(dev))) |
6495 | return -ENETDOWN; |
6496 | |
6497 | if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) |
6498 | return -EINVAL; |
6499 | |
6500 | ring = igc_xdp_get_tx_ring(adapter, cpu); |
6501 | nq = txring_txq(tx_ring: ring); |
6502 | |
6503 | __netif_tx_lock(txq: nq, cpu); |
6504 | |
6505 | /* Avoid transmit queue timeout since we share it with the slow path */ |
6506 | txq_trans_cond_update(txq: nq); |
6507 | |
6508 | drops = 0; |
6509 | for (i = 0; i < num_frames; i++) { |
6510 | int err; |
6511 | struct xdp_frame *xdpf = frames[i]; |
6512 | |
6513 | err = igc_xdp_init_tx_descriptor(ring, xdpf); |
6514 | if (err) { |
6515 | xdp_return_frame_rx_napi(xdpf); |
6516 | drops++; |
6517 | } |
6518 | } |
6519 | |
6520 | if (flags & XDP_XMIT_FLUSH) |
6521 | igc_flush_tx_descriptors(ring); |
6522 | |
6523 | __netif_tx_unlock(txq: nq); |
6524 | |
6525 | return num_frames - drops; |
6526 | } |
6527 | |
6528 | static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter, |
6529 | struct igc_q_vector *q_vector) |
6530 | { |
6531 | struct igc_hw *hw = &adapter->hw; |
6532 | u32 eics = 0; |
6533 | |
6534 | eics |= q_vector->eims_value; |
6535 | wr32(IGC_EICS, eics); |
6536 | } |
6537 | |
6538 | int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) |
6539 | { |
6540 | struct igc_adapter *adapter = netdev_priv(dev); |
6541 | struct igc_q_vector *q_vector; |
6542 | struct igc_ring *ring; |
6543 | |
6544 | if (test_bit(__IGC_DOWN, &adapter->state)) |
6545 | return -ENETDOWN; |
6546 | |
6547 | if (!igc_xdp_is_enabled(adapter)) |
6548 | return -ENXIO; |
6549 | |
6550 | if (queue_id >= adapter->num_rx_queues) |
6551 | return -EINVAL; |
6552 | |
6553 | ring = adapter->rx_ring[queue_id]; |
6554 | |
6555 | if (!ring->xsk_pool) |
6556 | return -ENXIO; |
6557 | |
6558 | q_vector = adapter->q_vector[queue_id]; |
6559 | if (!napi_if_scheduled_mark_missed(n: &q_vector->napi)) |
6560 | igc_trigger_rxtxq_interrupt(adapter, q_vector); |
6561 | |
6562 | return 0; |
6563 | } |
6564 | |
6565 | static const struct net_device_ops igc_netdev_ops = { |
6566 | .ndo_open = igc_open, |
6567 | .ndo_stop = igc_close, |
6568 | .ndo_start_xmit = igc_xmit_frame, |
6569 | .ndo_set_rx_mode = igc_set_rx_mode, |
6570 | .ndo_set_mac_address = igc_set_mac, |
6571 | .ndo_change_mtu = igc_change_mtu, |
6572 | .ndo_tx_timeout = igc_tx_timeout, |
6573 | .ndo_get_stats64 = igc_get_stats64, |
6574 | .ndo_fix_features = igc_fix_features, |
6575 | .ndo_set_features = igc_set_features, |
6576 | .ndo_features_check = igc_features_check, |
6577 | .ndo_eth_ioctl = igc_ioctl, |
6578 | .ndo_setup_tc = igc_setup_tc, |
6579 | .ndo_bpf = igc_bpf, |
6580 | .ndo_xdp_xmit = igc_xdp_xmit, |
6581 | .ndo_xsk_wakeup = igc_xsk_wakeup, |
6582 | }; |
6583 | |
6584 | /* PCIe configuration access */ |
6585 | void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) |
6586 | { |
6587 | struct igc_adapter *adapter = hw->back; |
6588 | |
6589 | pci_read_config_word(dev: adapter->pdev, where: reg, val: value); |
6590 | } |
6591 | |
6592 | void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value) |
6593 | { |
6594 | struct igc_adapter *adapter = hw->back; |
6595 | |
6596 | pci_write_config_word(dev: adapter->pdev, where: reg, val: *value); |
6597 | } |
6598 | |
6599 | s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) |
6600 | { |
6601 | struct igc_adapter *adapter = hw->back; |
6602 | |
6603 | if (!pci_is_pcie(dev: adapter->pdev)) |
6604 | return -IGC_ERR_CONFIG; |
6605 | |
6606 | pcie_capability_read_word(dev: adapter->pdev, pos: reg, val: value); |
6607 | |
6608 | return IGC_SUCCESS; |
6609 | } |
6610 | |
6611 | s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value) |
6612 | { |
6613 | struct igc_adapter *adapter = hw->back; |
6614 | |
6615 | if (!pci_is_pcie(dev: adapter->pdev)) |
6616 | return -IGC_ERR_CONFIG; |
6617 | |
6618 | pcie_capability_write_word(dev: adapter->pdev, pos: reg, val: *value); |
6619 | |
6620 | return IGC_SUCCESS; |
6621 | } |
6622 | |
6623 | u32 igc_rd32(struct igc_hw *hw, u32 reg) |
6624 | { |
6625 | struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw); |
6626 | u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); |
6627 | u32 value = 0; |
6628 | |
6629 | if (IGC_REMOVED(hw_addr)) |
6630 | return ~value; |
6631 | |
6632 | value = readl(addr: &hw_addr[reg]); |
6633 | |
6634 | /* reads should not return all F's */ |
6635 | if (!(~value) && (!reg || !(~readl(addr: hw_addr)))) { |
6636 | struct net_device *netdev = igc->netdev; |
6637 | |
6638 | hw->hw_addr = NULL; |
6639 | netif_device_detach(dev: netdev); |
6640 | netdev_err(dev: netdev, format: "PCIe link lost, device now detached\n" ); |
6641 | WARN(pci_device_is_present(igc->pdev), |
6642 | "igc: Failed to read reg 0x%x!\n" , reg); |
6643 | } |
6644 | |
6645 | return value; |
6646 | } |
6647 | |
6648 | /* Mapping HW RSS Type to enum xdp_rss_hash_type */ |
6649 | static enum xdp_rss_hash_type [IGC_RSS_TYPE_MAX_TABLE] = { |
6650 | [IGC_RSS_TYPE_NO_HASH] = XDP_RSS_TYPE_L2, |
6651 | [IGC_RSS_TYPE_HASH_TCP_IPV4] = XDP_RSS_TYPE_L4_IPV4_TCP, |
6652 | [IGC_RSS_TYPE_HASH_IPV4] = XDP_RSS_TYPE_L3_IPV4, |
6653 | [IGC_RSS_TYPE_HASH_TCP_IPV6] = XDP_RSS_TYPE_L4_IPV6_TCP, |
6654 | [IGC_RSS_TYPE_HASH_IPV6_EX] = XDP_RSS_TYPE_L3_IPV6_EX, |
6655 | [IGC_RSS_TYPE_HASH_IPV6] = XDP_RSS_TYPE_L3_IPV6, |
6656 | [IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX, |
6657 | [IGC_RSS_TYPE_HASH_UDP_IPV4] = XDP_RSS_TYPE_L4_IPV4_UDP, |
6658 | [IGC_RSS_TYPE_HASH_UDP_IPV6] = XDP_RSS_TYPE_L4_IPV6_UDP, |
6659 | [IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX, |
6660 | [10] = XDP_RSS_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW */ |
6661 | [11] = XDP_RSS_TYPE_NONE, /* keep array sized for SW bit-mask */ |
6662 | [12] = XDP_RSS_TYPE_NONE, /* to handle future HW revisons */ |
6663 | [13] = XDP_RSS_TYPE_NONE, |
6664 | [14] = XDP_RSS_TYPE_NONE, |
6665 | [15] = XDP_RSS_TYPE_NONE, |
6666 | }; |
6667 | |
6668 | static int igc_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash, |
6669 | enum xdp_rss_hash_type *) |
6670 | { |
6671 | const struct igc_xdp_buff *ctx = (void *)_ctx; |
6672 | |
6673 | if (!(ctx->xdp.rxq->dev->features & NETIF_F_RXHASH)) |
6674 | return -ENODATA; |
6675 | |
6676 | *hash = le32_to_cpu(ctx->rx_desc->wb.lower.hi_dword.rss); |
6677 | *rss_type = igc_xdp_rss_type[igc_rss_type(rx_desc: ctx->rx_desc)]; |
6678 | |
6679 | return 0; |
6680 | } |
6681 | |
6682 | static int igc_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) |
6683 | { |
6684 | const struct igc_xdp_buff *ctx = (void *)_ctx; |
6685 | |
6686 | if (igc_test_staterr(rx_desc: ctx->rx_desc, IGC_RXDADV_STAT_TSIP)) { |
6687 | *timestamp = ctx->rx_ts; |
6688 | |
6689 | return 0; |
6690 | } |
6691 | |
6692 | return -ENODATA; |
6693 | } |
6694 | |
6695 | static const struct xdp_metadata_ops igc_xdp_metadata_ops = { |
6696 | .xmo_rx_hash = igc_xdp_rx_hash, |
6697 | .xmo_rx_timestamp = igc_xdp_rx_timestamp, |
6698 | }; |
6699 | |
6700 | static enum hrtimer_restart igc_qbv_scheduling_timer(struct hrtimer *timer) |
6701 | { |
6702 | struct igc_adapter *adapter = container_of(timer, struct igc_adapter, |
6703 | hrtimer); |
6704 | unsigned long flags; |
6705 | unsigned int i; |
6706 | |
6707 | spin_lock_irqsave(&adapter->qbv_tx_lock, flags); |
6708 | |
6709 | adapter->qbv_transition = true; |
6710 | for (i = 0; i < adapter->num_tx_queues; i++) { |
6711 | struct igc_ring *tx_ring = adapter->tx_ring[i]; |
6712 | |
6713 | if (tx_ring->admin_gate_closed) { |
6714 | tx_ring->admin_gate_closed = false; |
6715 | tx_ring->oper_gate_closed = true; |
6716 | } else { |
6717 | tx_ring->oper_gate_closed = false; |
6718 | } |
6719 | } |
6720 | adapter->qbv_transition = false; |
6721 | |
6722 | spin_unlock_irqrestore(lock: &adapter->qbv_tx_lock, flags); |
6723 | |
6724 | return HRTIMER_NORESTART; |
6725 | } |
6726 | |
6727 | /** |
6728 | * igc_probe - Device Initialization Routine |
6729 | * @pdev: PCI device information struct |
6730 | * @ent: entry in igc_pci_tbl |
6731 | * |
6732 | * Returns 0 on success, negative on failure |
6733 | * |
6734 | * igc_probe initializes an adapter identified by a pci_dev structure. |
6735 | * The OS initialization, configuring the adapter private structure, |
6736 | * and a hardware reset occur. |
6737 | */ |
6738 | static int igc_probe(struct pci_dev *pdev, |
6739 | const struct pci_device_id *ent) |
6740 | { |
6741 | struct igc_adapter *adapter; |
6742 | struct net_device *netdev; |
6743 | struct igc_hw *hw; |
6744 | const struct igc_info *ei = igc_info_tbl[ent->driver_data]; |
6745 | int err; |
6746 | |
6747 | err = pci_enable_device_mem(dev: pdev); |
6748 | if (err) |
6749 | return err; |
6750 | |
6751 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64)); |
6752 | if (err) { |
6753 | dev_err(&pdev->dev, |
6754 | "No usable DMA configuration, aborting\n" ); |
6755 | goto err_dma; |
6756 | } |
6757 | |
6758 | err = pci_request_mem_regions(pdev, name: igc_driver_name); |
6759 | if (err) |
6760 | goto err_pci_reg; |
6761 | |
6762 | err = pci_enable_ptm(dev: pdev, NULL); |
6763 | if (err < 0) |
6764 | dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n" ); |
6765 | |
6766 | pci_set_master(dev: pdev); |
6767 | |
6768 | err = -ENOMEM; |
6769 | netdev = alloc_etherdev_mq(sizeof(struct igc_adapter), |
6770 | IGC_MAX_TX_QUEUES); |
6771 | |
6772 | if (!netdev) |
6773 | goto err_alloc_etherdev; |
6774 | |
6775 | SET_NETDEV_DEV(netdev, &pdev->dev); |
6776 | |
6777 | pci_set_drvdata(pdev, data: netdev); |
6778 | adapter = netdev_priv(dev: netdev); |
6779 | adapter->netdev = netdev; |
6780 | adapter->pdev = pdev; |
6781 | hw = &adapter->hw; |
6782 | hw->back = adapter; |
6783 | adapter->port_num = hw->bus.func; |
6784 | adapter->msg_enable = netif_msg_init(debug_value: debug, DEFAULT_MSG_ENABLE); |
6785 | |
6786 | err = pci_save_state(dev: pdev); |
6787 | if (err) |
6788 | goto err_ioremap; |
6789 | |
6790 | err = -EIO; |
6791 | adapter->io_addr = ioremap(pci_resource_start(pdev, 0), |
6792 | pci_resource_len(pdev, 0)); |
6793 | if (!adapter->io_addr) |
6794 | goto err_ioremap; |
6795 | |
6796 | /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */ |
6797 | hw->hw_addr = adapter->io_addr; |
6798 | |
6799 | netdev->netdev_ops = &igc_netdev_ops; |
6800 | netdev->xdp_metadata_ops = &igc_xdp_metadata_ops; |
6801 | igc_ethtool_set_ops(netdev); |
6802 | netdev->watchdog_timeo = 5 * HZ; |
6803 | |
6804 | netdev->mem_start = pci_resource_start(pdev, 0); |
6805 | netdev->mem_end = pci_resource_end(pdev, 0); |
6806 | |
6807 | /* PCI config space info */ |
6808 | hw->vendor_id = pdev->vendor; |
6809 | hw->device_id = pdev->device; |
6810 | hw->revision_id = pdev->revision; |
6811 | hw->subsystem_vendor_id = pdev->subsystem_vendor; |
6812 | hw->subsystem_device_id = pdev->subsystem_device; |
6813 | |
6814 | /* Copy the default MAC and PHY function pointers */ |
6815 | memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); |
6816 | memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); |
6817 | |
6818 | /* Initialize skew-specific constants */ |
6819 | err = ei->get_invariants(hw); |
6820 | if (err) |
6821 | goto err_sw_init; |
6822 | |
6823 | /* Add supported features to the features list*/ |
6824 | netdev->features |= NETIF_F_SG; |
6825 | netdev->features |= NETIF_F_TSO; |
6826 | netdev->features |= NETIF_F_TSO6; |
6827 | netdev->features |= NETIF_F_TSO_ECN; |
6828 | netdev->features |= NETIF_F_RXHASH; |
6829 | netdev->features |= NETIF_F_RXCSUM; |
6830 | netdev->features |= NETIF_F_HW_CSUM; |
6831 | netdev->features |= NETIF_F_SCTP_CRC; |
6832 | netdev->features |= NETIF_F_HW_TC; |
6833 | |
6834 | #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ |
6835 | NETIF_F_GSO_GRE_CSUM | \ |
6836 | NETIF_F_GSO_IPXIP4 | \ |
6837 | NETIF_F_GSO_IPXIP6 | \ |
6838 | NETIF_F_GSO_UDP_TUNNEL | \ |
6839 | NETIF_F_GSO_UDP_TUNNEL_CSUM) |
6840 | |
6841 | netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES; |
6842 | netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES; |
6843 | |
6844 | /* setup the private structure */ |
6845 | err = igc_sw_init(adapter); |
6846 | if (err) |
6847 | goto err_sw_init; |
6848 | |
6849 | /* copy netdev features into list of user selectable features */ |
6850 | netdev->hw_features |= NETIF_F_NTUPLE; |
6851 | netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; |
6852 | netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; |
6853 | netdev->hw_features |= netdev->features; |
6854 | |
6855 | netdev->features |= NETIF_F_HIGHDMA; |
6856 | |
6857 | netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; |
6858 | netdev->mpls_features |= NETIF_F_HW_CSUM; |
6859 | netdev->hw_enc_features |= netdev->vlan_features; |
6860 | |
6861 | netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | |
6862 | NETDEV_XDP_ACT_XSK_ZEROCOPY; |
6863 | |
6864 | /* MTU range: 68 - 9216 */ |
6865 | netdev->min_mtu = ETH_MIN_MTU; |
6866 | netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; |
6867 | |
6868 | /* before reading the NVM, reset the controller to put the device in a |
6869 | * known good starting state |
6870 | */ |
6871 | hw->mac.ops.reset_hw(hw); |
6872 | |
6873 | if (igc_get_flash_presence_i225(hw)) { |
6874 | if (hw->nvm.ops.validate(hw) < 0) { |
6875 | dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n" ); |
6876 | err = -EIO; |
6877 | goto err_eeprom; |
6878 | } |
6879 | } |
6880 | |
6881 | if (eth_platform_get_mac_address(dev: &pdev->dev, mac_addr: hw->mac.addr)) { |
6882 | /* copy the MAC address out of the NVM */ |
6883 | if (hw->mac.ops.read_mac_addr(hw)) |
6884 | dev_err(&pdev->dev, "NVM Read Error\n" ); |
6885 | } |
6886 | |
6887 | eth_hw_addr_set(dev: netdev, addr: hw->mac.addr); |
6888 | |
6889 | if (!is_valid_ether_addr(addr: netdev->dev_addr)) { |
6890 | dev_err(&pdev->dev, "Invalid MAC Address\n" ); |
6891 | err = -EIO; |
6892 | goto err_eeprom; |
6893 | } |
6894 | |
6895 | /* configure RXPBSIZE and TXPBSIZE */ |
6896 | wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT); |
6897 | wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT); |
6898 | |
6899 | timer_setup(&adapter->watchdog_timer, igc_watchdog, 0); |
6900 | timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0); |
6901 | |
6902 | INIT_WORK(&adapter->reset_task, igc_reset_task); |
6903 | INIT_WORK(&adapter->watchdog_task, igc_watchdog_task); |
6904 | |
6905 | hrtimer_init(timer: &adapter->hrtimer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL); |
6906 | adapter->hrtimer.function = &igc_qbv_scheduling_timer; |
6907 | |
6908 | /* Initialize link properties that are user-changeable */ |
6909 | adapter->fc_autoneg = true; |
6910 | hw->mac.autoneg = true; |
6911 | hw->phy.autoneg_advertised = 0xaf; |
6912 | |
6913 | hw->fc.requested_mode = igc_fc_default; |
6914 | hw->fc.current_mode = igc_fc_default; |
6915 | |
6916 | /* By default, support wake on port A */ |
6917 | adapter->flags |= IGC_FLAG_WOL_SUPPORTED; |
6918 | |
6919 | /* initialize the wol settings based on the eeprom settings */ |
6920 | if (adapter->flags & IGC_FLAG_WOL_SUPPORTED) |
6921 | adapter->wol |= IGC_WUFC_MAG; |
6922 | |
6923 | device_set_wakeup_enable(dev: &adapter->pdev->dev, |
6924 | enable: adapter->flags & IGC_FLAG_WOL_SUPPORTED); |
6925 | |
6926 | igc_ptp_init(adapter); |
6927 | |
6928 | igc_tsn_clear_schedule(adapter); |
6929 | |
6930 | /* reset the hardware with the new settings */ |
6931 | igc_reset(adapter); |
6932 | |
6933 | /* let the f/w know that the h/w is now under the control of the |
6934 | * driver. |
6935 | */ |
6936 | igc_get_hw_control(adapter); |
6937 | |
6938 | strscpy(p: netdev->name, q: "eth%d" , size: sizeof(netdev->name)); |
6939 | err = register_netdev(dev: netdev); |
6940 | if (err) |
6941 | goto err_register; |
6942 | |
6943 | /* carrier off reporting is important to ethtool even BEFORE open */ |
6944 | netif_carrier_off(dev: netdev); |
6945 | |
6946 | /* Check if Media Autosense is enabled */ |
6947 | adapter->ei = *ei; |
6948 | |
6949 | /* print pcie link status and MAC address */ |
6950 | pcie_print_link_status(dev: pdev); |
6951 | netdev_info(dev: netdev, format: "MAC: %pM\n" , netdev->dev_addr); |
6952 | |
6953 | dev_pm_set_driver_flags(dev: &pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); |
6954 | /* Disable EEE for internal PHY devices */ |
6955 | hw->dev_spec._base.eee_enable = false; |
6956 | adapter->flags &= ~IGC_FLAG_EEE; |
6957 | igc_set_eee_i225(hw, adv2p5G: false, adv1G: false, adv100M: false); |
6958 | |
6959 | pm_runtime_put_noidle(dev: &pdev->dev); |
6960 | |
6961 | return 0; |
6962 | |
6963 | err_register: |
6964 | igc_release_hw_control(adapter); |
6965 | err_eeprom: |
6966 | if (!igc_check_reset_block(hw)) |
6967 | igc_reset_phy(hw); |
6968 | err_sw_init: |
6969 | igc_clear_interrupt_scheme(adapter); |
6970 | iounmap(addr: adapter->io_addr); |
6971 | err_ioremap: |
6972 | free_netdev(dev: netdev); |
6973 | err_alloc_etherdev: |
6974 | pci_release_mem_regions(pdev); |
6975 | err_pci_reg: |
6976 | err_dma: |
6977 | pci_disable_device(dev: pdev); |
6978 | return err; |
6979 | } |
6980 | |
6981 | /** |
6982 | * igc_remove - Device Removal Routine |
6983 | * @pdev: PCI device information struct |
6984 | * |
6985 | * igc_remove is called by the PCI subsystem to alert the driver |
6986 | * that it should release a PCI device. This could be caused by a |
6987 | * Hot-Plug event, or because the driver is going to be removed from |
6988 | * memory. |
6989 | */ |
6990 | static void igc_remove(struct pci_dev *pdev) |
6991 | { |
6992 | struct net_device *netdev = pci_get_drvdata(pdev); |
6993 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
6994 | |
6995 | pm_runtime_get_noresume(dev: &pdev->dev); |
6996 | |
6997 | igc_flush_nfc_rules(adapter); |
6998 | |
6999 | igc_ptp_stop(adapter); |
7000 | |
7001 | pci_disable_ptm(dev: pdev); |
7002 | pci_clear_master(dev: pdev); |
7003 | |
7004 | set_bit(nr: __IGC_DOWN, addr: &adapter->state); |
7005 | |
7006 | del_timer_sync(timer: &adapter->watchdog_timer); |
7007 | del_timer_sync(timer: &adapter->phy_info_timer); |
7008 | |
7009 | cancel_work_sync(work: &adapter->reset_task); |
7010 | cancel_work_sync(work: &adapter->watchdog_task); |
7011 | hrtimer_cancel(timer: &adapter->hrtimer); |
7012 | |
7013 | /* Release control of h/w to f/w. If f/w is AMT enabled, this |
7014 | * would have already happened in close and is redundant. |
7015 | */ |
7016 | igc_release_hw_control(adapter); |
7017 | unregister_netdev(dev: netdev); |
7018 | |
7019 | igc_clear_interrupt_scheme(adapter); |
7020 | pci_iounmap(dev: pdev, adapter->io_addr); |
7021 | pci_release_mem_regions(pdev); |
7022 | |
7023 | free_netdev(dev: netdev); |
7024 | |
7025 | pci_disable_device(dev: pdev); |
7026 | } |
7027 | |
7028 | static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, |
7029 | bool runtime) |
7030 | { |
7031 | struct net_device *netdev = pci_get_drvdata(pdev); |
7032 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
7033 | u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; |
7034 | struct igc_hw *hw = &adapter->hw; |
7035 | u32 ctrl, rctl, status; |
7036 | bool wake; |
7037 | |
7038 | rtnl_lock(); |
7039 | netif_device_detach(dev: netdev); |
7040 | |
7041 | if (netif_running(dev: netdev)) |
7042 | __igc_close(netdev, suspending: true); |
7043 | |
7044 | igc_ptp_suspend(adapter); |
7045 | |
7046 | igc_clear_interrupt_scheme(adapter); |
7047 | rtnl_unlock(); |
7048 | |
7049 | status = rd32(IGC_STATUS); |
7050 | if (status & IGC_STATUS_LU) |
7051 | wufc &= ~IGC_WUFC_LNKC; |
7052 | |
7053 | if (wufc) { |
7054 | igc_setup_rctl(adapter); |
7055 | igc_set_rx_mode(netdev); |
7056 | |
7057 | /* turn on all-multi mode if wake on multicast is enabled */ |
7058 | if (wufc & IGC_WUFC_MC) { |
7059 | rctl = rd32(IGC_RCTL); |
7060 | rctl |= IGC_RCTL_MPE; |
7061 | wr32(IGC_RCTL, rctl); |
7062 | } |
7063 | |
7064 | ctrl = rd32(IGC_CTRL); |
7065 | ctrl |= IGC_CTRL_ADVD3WUC; |
7066 | wr32(IGC_CTRL, ctrl); |
7067 | |
7068 | /* Allow time for pending master requests to run */ |
7069 | igc_disable_pcie_master(hw); |
7070 | |
7071 | wr32(IGC_WUC, IGC_WUC_PME_EN); |
7072 | wr32(IGC_WUFC, wufc); |
7073 | } else { |
7074 | wr32(IGC_WUC, 0); |
7075 | wr32(IGC_WUFC, 0); |
7076 | } |
7077 | |
7078 | wake = wufc || adapter->en_mng_pt; |
7079 | if (!wake) |
7080 | igc_power_down_phy_copper_base(hw: &adapter->hw); |
7081 | else |
7082 | igc_power_up_link(adapter); |
7083 | |
7084 | if (enable_wake) |
7085 | *enable_wake = wake; |
7086 | |
7087 | /* Release control of h/w to f/w. If f/w is AMT enabled, this |
7088 | * would have already happened in close and is redundant. |
7089 | */ |
7090 | igc_release_hw_control(adapter); |
7091 | |
7092 | pci_disable_device(dev: pdev); |
7093 | |
7094 | return 0; |
7095 | } |
7096 | |
7097 | #ifdef CONFIG_PM |
7098 | static int __maybe_unused igc_runtime_suspend(struct device *dev) |
7099 | { |
7100 | return __igc_shutdown(to_pci_dev(dev), NULL, runtime: 1); |
7101 | } |
7102 | |
7103 | static void igc_deliver_wake_packet(struct net_device *netdev) |
7104 | { |
7105 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
7106 | struct igc_hw *hw = &adapter->hw; |
7107 | struct sk_buff *skb; |
7108 | u32 wupl; |
7109 | |
7110 | wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; |
7111 | |
7112 | /* WUPM stores only the first 128 bytes of the wake packet. |
7113 | * Read the packet only if we have the whole thing. |
7114 | */ |
7115 | if (wupl == 0 || wupl > IGC_WUPM_BYTES) |
7116 | return; |
7117 | |
7118 | skb = netdev_alloc_skb_ip_align(dev: netdev, IGC_WUPM_BYTES); |
7119 | if (!skb) |
7120 | return; |
7121 | |
7122 | skb_put(skb, len: wupl); |
7123 | |
7124 | /* Ensure reads are 32-bit aligned */ |
7125 | wupl = roundup(wupl, 4); |
7126 | |
7127 | memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); |
7128 | |
7129 | skb->protocol = eth_type_trans(skb, dev: netdev); |
7130 | netif_rx(skb); |
7131 | } |
7132 | |
7133 | static int __maybe_unused igc_resume(struct device *dev) |
7134 | { |
7135 | struct pci_dev *pdev = to_pci_dev(dev); |
7136 | struct net_device *netdev = pci_get_drvdata(pdev); |
7137 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
7138 | struct igc_hw *hw = &adapter->hw; |
7139 | u32 err, val; |
7140 | |
7141 | pci_set_power_state(dev: pdev, PCI_D0); |
7142 | pci_restore_state(dev: pdev); |
7143 | pci_save_state(dev: pdev); |
7144 | |
7145 | if (!pci_device_is_present(pdev)) |
7146 | return -ENODEV; |
7147 | err = pci_enable_device_mem(dev: pdev); |
7148 | if (err) { |
7149 | netdev_err(dev: netdev, format: "Cannot enable PCI device from suspend\n" ); |
7150 | return err; |
7151 | } |
7152 | pci_set_master(dev: pdev); |
7153 | |
7154 | pci_enable_wake(dev: pdev, PCI_D3hot, enable: 0); |
7155 | pci_enable_wake(dev: pdev, PCI_D3cold, enable: 0); |
7156 | |
7157 | if (igc_init_interrupt_scheme(adapter, msix: true)) { |
7158 | netdev_err(dev: netdev, format: "Unable to allocate memory for queues\n" ); |
7159 | return -ENOMEM; |
7160 | } |
7161 | |
7162 | igc_reset(adapter); |
7163 | |
7164 | /* let the f/w know that the h/w is now under the control of the |
7165 | * driver. |
7166 | */ |
7167 | igc_get_hw_control(adapter); |
7168 | |
7169 | val = rd32(IGC_WUS); |
7170 | if (val & WAKE_PKT_WUS) |
7171 | igc_deliver_wake_packet(netdev); |
7172 | |
7173 | wr32(IGC_WUS, ~0); |
7174 | |
7175 | rtnl_lock(); |
7176 | if (!err && netif_running(dev: netdev)) |
7177 | err = __igc_open(netdev, resuming: true); |
7178 | |
7179 | if (!err) |
7180 | netif_device_attach(dev: netdev); |
7181 | rtnl_unlock(); |
7182 | |
7183 | return err; |
7184 | } |
7185 | |
7186 | static int __maybe_unused igc_runtime_resume(struct device *dev) |
7187 | { |
7188 | return igc_resume(dev); |
7189 | } |
7190 | |
7191 | static int __maybe_unused igc_suspend(struct device *dev) |
7192 | { |
7193 | return __igc_shutdown(to_pci_dev(dev), NULL, runtime: 0); |
7194 | } |
7195 | |
7196 | static int __maybe_unused igc_runtime_idle(struct device *dev) |
7197 | { |
7198 | struct net_device *netdev = dev_get_drvdata(dev); |
7199 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
7200 | |
7201 | if (!igc_has_link(adapter)) |
7202 | pm_schedule_suspend(dev, MSEC_PER_SEC * 5); |
7203 | |
7204 | return -EBUSY; |
7205 | } |
7206 | #endif /* CONFIG_PM */ |
7207 | |
7208 | static void igc_shutdown(struct pci_dev *pdev) |
7209 | { |
7210 | bool wake; |
7211 | |
7212 | __igc_shutdown(pdev, enable_wake: &wake, runtime: 0); |
7213 | |
7214 | if (system_state == SYSTEM_POWER_OFF) { |
7215 | pci_wake_from_d3(dev: pdev, enable: wake); |
7216 | pci_set_power_state(dev: pdev, PCI_D3hot); |
7217 | } |
7218 | } |
7219 | |
7220 | /** |
7221 | * igc_io_error_detected - called when PCI error is detected |
7222 | * @pdev: Pointer to PCI device |
7223 | * @state: The current PCI connection state |
7224 | * |
7225 | * This function is called after a PCI bus error affecting |
7226 | * this device has been detected. |
7227 | **/ |
7228 | static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev, |
7229 | pci_channel_state_t state) |
7230 | { |
7231 | struct net_device *netdev = pci_get_drvdata(pdev); |
7232 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
7233 | |
7234 | netif_device_detach(dev: netdev); |
7235 | |
7236 | if (state == pci_channel_io_perm_failure) |
7237 | return PCI_ERS_RESULT_DISCONNECT; |
7238 | |
7239 | if (netif_running(dev: netdev)) |
7240 | igc_down(adapter); |
7241 | pci_disable_device(dev: pdev); |
7242 | |
7243 | /* Request a slot reset. */ |
7244 | return PCI_ERS_RESULT_NEED_RESET; |
7245 | } |
7246 | |
7247 | /** |
7248 | * igc_io_slot_reset - called after the PCI bus has been reset. |
7249 | * @pdev: Pointer to PCI device |
7250 | * |
7251 | * Restart the card from scratch, as if from a cold-boot. Implementation |
7252 | * resembles the first-half of the igc_resume routine. |
7253 | **/ |
7254 | static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev) |
7255 | { |
7256 | struct net_device *netdev = pci_get_drvdata(pdev); |
7257 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
7258 | struct igc_hw *hw = &adapter->hw; |
7259 | pci_ers_result_t result; |
7260 | |
7261 | if (pci_enable_device_mem(dev: pdev)) { |
7262 | netdev_err(dev: netdev, format: "Could not re-enable PCI device after reset\n" ); |
7263 | result = PCI_ERS_RESULT_DISCONNECT; |
7264 | } else { |
7265 | pci_set_master(dev: pdev); |
7266 | pci_restore_state(dev: pdev); |
7267 | pci_save_state(dev: pdev); |
7268 | |
7269 | pci_enable_wake(dev: pdev, PCI_D3hot, enable: 0); |
7270 | pci_enable_wake(dev: pdev, PCI_D3cold, enable: 0); |
7271 | |
7272 | /* In case of PCI error, adapter loses its HW address |
7273 | * so we should re-assign it here. |
7274 | */ |
7275 | hw->hw_addr = adapter->io_addr; |
7276 | |
7277 | igc_reset(adapter); |
7278 | wr32(IGC_WUS, ~0); |
7279 | result = PCI_ERS_RESULT_RECOVERED; |
7280 | } |
7281 | |
7282 | return result; |
7283 | } |
7284 | |
7285 | /** |
7286 | * igc_io_resume - called when traffic can start to flow again. |
7287 | * @pdev: Pointer to PCI device |
7288 | * |
7289 | * This callback is called when the error recovery driver tells us that |
7290 | * its OK to resume normal operation. Implementation resembles the |
7291 | * second-half of the igc_resume routine. |
7292 | */ |
7293 | static void igc_io_resume(struct pci_dev *pdev) |
7294 | { |
7295 | struct net_device *netdev = pci_get_drvdata(pdev); |
7296 | struct igc_adapter *adapter = netdev_priv(dev: netdev); |
7297 | |
7298 | rtnl_lock(); |
7299 | if (netif_running(dev: netdev)) { |
7300 | if (igc_open(netdev)) { |
7301 | netdev_err(dev: netdev, format: "igc_open failed after reset\n" ); |
7302 | return; |
7303 | } |
7304 | } |
7305 | |
7306 | netif_device_attach(dev: netdev); |
7307 | |
7308 | /* let the f/w know that the h/w is now under the control of the |
7309 | * driver. |
7310 | */ |
7311 | igc_get_hw_control(adapter); |
7312 | rtnl_unlock(); |
7313 | } |
7314 | |
7315 | static const struct pci_error_handlers igc_err_handler = { |
7316 | .error_detected = igc_io_error_detected, |
7317 | .slot_reset = igc_io_slot_reset, |
7318 | .resume = igc_io_resume, |
7319 | }; |
7320 | |
7321 | #ifdef CONFIG_PM |
7322 | static const struct dev_pm_ops igc_pm_ops = { |
7323 | SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) |
7324 | SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, |
7325 | igc_runtime_idle) |
7326 | }; |
7327 | #endif |
7328 | |
7329 | static struct pci_driver igc_driver = { |
7330 | .name = igc_driver_name, |
7331 | .id_table = igc_pci_tbl, |
7332 | .probe = igc_probe, |
7333 | .remove = igc_remove, |
7334 | #ifdef CONFIG_PM |
7335 | .driver.pm = &igc_pm_ops, |
7336 | #endif |
7337 | .shutdown = igc_shutdown, |
7338 | .err_handler = &igc_err_handler, |
7339 | }; |
7340 | |
7341 | /** |
7342 | * igc_reinit_queues - return error |
7343 | * @adapter: pointer to adapter structure |
7344 | */ |
7345 | int igc_reinit_queues(struct igc_adapter *adapter) |
7346 | { |
7347 | struct net_device *netdev = adapter->netdev; |
7348 | int err = 0; |
7349 | |
7350 | if (netif_running(dev: netdev)) |
7351 | igc_close(netdev); |
7352 | |
7353 | igc_reset_interrupt_capability(adapter); |
7354 | |
7355 | if (igc_init_interrupt_scheme(adapter, msix: true)) { |
7356 | netdev_err(dev: netdev, format: "Unable to allocate memory for queues\n" ); |
7357 | return -ENOMEM; |
7358 | } |
7359 | |
7360 | if (netif_running(dev: netdev)) |
7361 | err = igc_open(netdev); |
7362 | |
7363 | return err; |
7364 | } |
7365 | |
7366 | /** |
7367 | * igc_get_hw_dev - return device |
7368 | * @hw: pointer to hardware structure |
7369 | * |
7370 | * used by hardware layer to print debugging information |
7371 | */ |
7372 | struct net_device *igc_get_hw_dev(struct igc_hw *hw) |
7373 | { |
7374 | struct igc_adapter *adapter = hw->back; |
7375 | |
7376 | return adapter->netdev; |
7377 | } |
7378 | |
7379 | static void igc_disable_rx_ring_hw(struct igc_ring *ring) |
7380 | { |
7381 | struct igc_hw *hw = &ring->q_vector->adapter->hw; |
7382 | u8 idx = ring->reg_idx; |
7383 | u32 rxdctl; |
7384 | |
7385 | rxdctl = rd32(IGC_RXDCTL(idx)); |
7386 | rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE; |
7387 | rxdctl |= IGC_RXDCTL_SWFLUSH; |
7388 | wr32(IGC_RXDCTL(idx), rxdctl); |
7389 | } |
7390 | |
7391 | void igc_disable_rx_ring(struct igc_ring *ring) |
7392 | { |
7393 | igc_disable_rx_ring_hw(ring); |
7394 | igc_clean_rx_ring(ring); |
7395 | } |
7396 | |
7397 | void igc_enable_rx_ring(struct igc_ring *ring) |
7398 | { |
7399 | struct igc_adapter *adapter = ring->q_vector->adapter; |
7400 | |
7401 | igc_configure_rx_ring(adapter, ring); |
7402 | |
7403 | if (ring->xsk_pool) |
7404 | igc_alloc_rx_buffers_zc(ring, count: igc_desc_unused(ring)); |
7405 | else |
7406 | igc_alloc_rx_buffers(rx_ring: ring, cleaned_count: igc_desc_unused(ring)); |
7407 | } |
7408 | |
7409 | void igc_disable_tx_ring(struct igc_ring *ring) |
7410 | { |
7411 | igc_disable_tx_ring_hw(ring); |
7412 | igc_clean_tx_ring(tx_ring: ring); |
7413 | } |
7414 | |
7415 | void igc_enable_tx_ring(struct igc_ring *ring) |
7416 | { |
7417 | struct igc_adapter *adapter = ring->q_vector->adapter; |
7418 | |
7419 | igc_configure_tx_ring(adapter, ring); |
7420 | } |
7421 | |
7422 | /** |
7423 | * igc_init_module - Driver Registration Routine |
7424 | * |
7425 | * igc_init_module is the first routine called when the driver is |
7426 | * loaded. All it does is register with the PCI subsystem. |
7427 | */ |
7428 | static int __init igc_init_module(void) |
7429 | { |
7430 | int ret; |
7431 | |
7432 | pr_info("%s\n" , igc_driver_string); |
7433 | pr_info("%s\n" , igc_copyright); |
7434 | |
7435 | ret = pci_register_driver(&igc_driver); |
7436 | return ret; |
7437 | } |
7438 | |
7439 | module_init(igc_init_module); |
7440 | |
7441 | /** |
7442 | * igc_exit_module - Driver Exit Cleanup Routine |
7443 | * |
7444 | * igc_exit_module is called just before the driver is removed |
7445 | * from memory. |
7446 | */ |
7447 | static void __exit igc_exit_module(void) |
7448 | { |
7449 | pci_unregister_driver(dev: &igc_driver); |
7450 | } |
7451 | |
7452 | module_exit(igc_exit_module); |
7453 | /* igc_main.c */ |
7454 | |