1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2007 - 2018 Intel Corporation. */ |
3 | |
4 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
5 | |
6 | #include <linux/module.h> |
7 | #include <linux/types.h> |
8 | #include <linux/init.h> |
9 | #include <linux/bitops.h> |
10 | #include <linux/vmalloc.h> |
11 | #include <linux/pagemap.h> |
12 | #include <linux/netdevice.h> |
13 | #include <linux/ipv6.h> |
14 | #include <linux/slab.h> |
15 | #include <net/checksum.h> |
16 | #include <net/ip6_checksum.h> |
17 | #include <net/pkt_sched.h> |
18 | #include <net/pkt_cls.h> |
19 | #include <linux/net_tstamp.h> |
20 | #include <linux/mii.h> |
21 | #include <linux/ethtool.h> |
22 | #include <linux/if.h> |
23 | #include <linux/if_vlan.h> |
24 | #include <linux/pci.h> |
25 | #include <linux/delay.h> |
26 | #include <linux/interrupt.h> |
27 | #include <linux/ip.h> |
28 | #include <linux/tcp.h> |
29 | #include <linux/sctp.h> |
30 | #include <linux/if_ether.h> |
31 | #include <linux/prefetch.h> |
32 | #include <linux/bpf.h> |
33 | #include <linux/bpf_trace.h> |
34 | #include <linux/pm_runtime.h> |
35 | #include <linux/etherdevice.h> |
36 | #ifdef CONFIG_IGB_DCA |
37 | #include <linux/dca.h> |
38 | #endif |
39 | #include <linux/i2c.h> |
40 | #include "igb.h" |
41 | |
42 | enum queue_mode { |
43 | QUEUE_MODE_STRICT_PRIORITY, |
44 | QUEUE_MODE_STREAM_RESERVATION, |
45 | }; |
46 | |
47 | enum tx_queue_prio { |
48 | TX_QUEUE_PRIO_HIGH, |
49 | TX_QUEUE_PRIO_LOW, |
50 | }; |
51 | |
52 | char igb_driver_name[] = "igb" ; |
53 | static const char igb_driver_string[] = |
54 | "Intel(R) Gigabit Ethernet Network Driver" ; |
55 | static const char igb_copyright[] = |
56 | "Copyright (c) 2007-2014 Intel Corporation." ; |
57 | |
58 | static const struct e1000_info *igb_info_tbl[] = { |
59 | [board_82575] = &e1000_82575_info, |
60 | }; |
61 | |
62 | static const struct pci_device_id igb_pci_tbl[] = { |
63 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) }, |
64 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) }, |
65 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) }, |
66 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 }, |
67 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 }, |
68 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 }, |
69 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 }, |
70 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 }, |
71 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 }, |
72 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 }, |
73 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 }, |
74 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 }, |
75 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 }, |
76 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 }, |
77 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 }, |
78 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 }, |
79 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 }, |
80 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 }, |
81 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 }, |
82 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 }, |
83 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 }, |
84 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 }, |
85 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 }, |
86 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 }, |
87 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 }, |
88 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 }, |
89 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 }, |
90 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 }, |
91 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 }, |
92 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 }, |
93 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 }, |
94 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 }, |
95 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 }, |
96 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 }, |
97 | { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 }, |
98 | /* required last entry */ |
99 | {0, } |
100 | }; |
101 | |
102 | MODULE_DEVICE_TABLE(pci, igb_pci_tbl); |
103 | |
104 | static int igb_setup_all_tx_resources(struct igb_adapter *); |
105 | static int igb_setup_all_rx_resources(struct igb_adapter *); |
106 | static void igb_free_all_tx_resources(struct igb_adapter *); |
107 | static void igb_free_all_rx_resources(struct igb_adapter *); |
108 | static void igb_setup_mrqc(struct igb_adapter *); |
109 | static int igb_probe(struct pci_dev *, const struct pci_device_id *); |
110 | static void igb_remove(struct pci_dev *pdev); |
111 | static void igb_init_queue_configuration(struct igb_adapter *adapter); |
112 | static int igb_sw_init(struct igb_adapter *); |
113 | int igb_open(struct net_device *); |
114 | int igb_close(struct net_device *); |
115 | static void igb_configure(struct igb_adapter *); |
116 | static void igb_configure_tx(struct igb_adapter *); |
117 | static void igb_configure_rx(struct igb_adapter *); |
118 | static void igb_clean_all_tx_rings(struct igb_adapter *); |
119 | static void igb_clean_all_rx_rings(struct igb_adapter *); |
120 | static void igb_clean_tx_ring(struct igb_ring *); |
121 | static void igb_clean_rx_ring(struct igb_ring *); |
122 | static void igb_set_rx_mode(struct net_device *); |
123 | static void igb_update_phy_info(struct timer_list *); |
124 | static void igb_watchdog(struct timer_list *); |
125 | static void igb_watchdog_task(struct work_struct *); |
126 | static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *); |
127 | static void igb_get_stats64(struct net_device *dev, |
128 | struct rtnl_link_stats64 *stats); |
129 | static int igb_change_mtu(struct net_device *, int); |
130 | static int igb_set_mac(struct net_device *, void *); |
131 | static void igb_set_uta(struct igb_adapter *adapter, bool set); |
132 | static irqreturn_t igb_intr(int irq, void *); |
133 | static irqreturn_t igb_intr_msi(int irq, void *); |
134 | static irqreturn_t igb_msix_other(int irq, void *); |
135 | static irqreturn_t igb_msix_ring(int irq, void *); |
136 | #ifdef CONFIG_IGB_DCA |
137 | static void igb_update_dca(struct igb_q_vector *); |
138 | static void igb_setup_dca(struct igb_adapter *); |
139 | #endif /* CONFIG_IGB_DCA */ |
140 | static int igb_poll(struct napi_struct *, int); |
141 | static bool igb_clean_tx_irq(struct igb_q_vector *, int); |
142 | static int igb_clean_rx_irq(struct igb_q_vector *, int); |
143 | static int igb_ioctl(struct net_device *, struct ifreq *, int cmd); |
144 | static void igb_tx_timeout(struct net_device *, unsigned int txqueue); |
145 | static void igb_reset_task(struct work_struct *); |
146 | static void igb_vlan_mode(struct net_device *netdev, |
147 | netdev_features_t features); |
148 | static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16); |
149 | static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16); |
150 | static void igb_restore_vlan(struct igb_adapter *); |
151 | static void igb_rar_set_index(struct igb_adapter *, u32); |
152 | static void igb_ping_all_vfs(struct igb_adapter *); |
153 | static void igb_msg_task(struct igb_adapter *); |
154 | static void igb_vmm_control(struct igb_adapter *); |
155 | static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *); |
156 | static void igb_flush_mac_table(struct igb_adapter *); |
157 | static int igb_available_rars(struct igb_adapter *, u8); |
158 | static void igb_set_default_mac_filter(struct igb_adapter *); |
159 | static int igb_uc_sync(struct net_device *, const unsigned char *); |
160 | static int igb_uc_unsync(struct net_device *, const unsigned char *); |
161 | static void igb_restore_vf_multicasts(struct igb_adapter *adapter); |
162 | static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac); |
163 | static int igb_ndo_set_vf_vlan(struct net_device *netdev, |
164 | int vf, u16 vlan, u8 qos, __be16 vlan_proto); |
165 | static int igb_ndo_set_vf_bw(struct net_device *, int, int, int); |
166 | static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, |
167 | bool setting); |
168 | static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, |
169 | bool setting); |
170 | static int igb_ndo_get_vf_config(struct net_device *netdev, int vf, |
171 | struct ifla_vf_info *ivi); |
172 | static void igb_check_vf_rate_limit(struct igb_adapter *); |
173 | static void igb_nfc_filter_exit(struct igb_adapter *adapter); |
174 | static void igb_nfc_filter_restore(struct igb_adapter *adapter); |
175 | |
176 | #ifdef CONFIG_PCI_IOV |
177 | static int igb_vf_configure(struct igb_adapter *adapter, int vf); |
178 | static int igb_disable_sriov(struct pci_dev *dev, bool reinit); |
179 | #endif |
180 | |
181 | static int igb_suspend(struct device *); |
182 | static int igb_resume(struct device *); |
183 | static int igb_runtime_suspend(struct device *dev); |
184 | static int igb_runtime_resume(struct device *dev); |
185 | static int igb_runtime_idle(struct device *dev); |
186 | #ifdef CONFIG_PM |
187 | static const struct dev_pm_ops igb_pm_ops = { |
188 | SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume) |
189 | SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume, |
190 | igb_runtime_idle) |
191 | }; |
192 | #endif |
193 | static void igb_shutdown(struct pci_dev *); |
194 | static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs); |
195 | #ifdef CONFIG_IGB_DCA |
196 | static int igb_notify_dca(struct notifier_block *, unsigned long, void *); |
197 | static struct notifier_block dca_notifier = { |
198 | .notifier_call = igb_notify_dca, |
199 | .next = NULL, |
200 | .priority = 0 |
201 | }; |
202 | #endif |
203 | #ifdef CONFIG_PCI_IOV |
204 | static unsigned int max_vfs; |
205 | module_param(max_vfs, uint, 0); |
206 | MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function" ); |
207 | #endif /* CONFIG_PCI_IOV */ |
208 | |
209 | static pci_ers_result_t igb_io_error_detected(struct pci_dev *, |
210 | pci_channel_state_t); |
211 | static pci_ers_result_t igb_io_slot_reset(struct pci_dev *); |
212 | static void igb_io_resume(struct pci_dev *); |
213 | |
214 | static const struct pci_error_handlers igb_err_handler = { |
215 | .error_detected = igb_io_error_detected, |
216 | .slot_reset = igb_io_slot_reset, |
217 | .resume = igb_io_resume, |
218 | }; |
219 | |
220 | static void igb_init_dmac(struct igb_adapter *adapter, u32 pba); |
221 | |
222 | static struct pci_driver igb_driver = { |
223 | .name = igb_driver_name, |
224 | .id_table = igb_pci_tbl, |
225 | .probe = igb_probe, |
226 | .remove = igb_remove, |
227 | #ifdef CONFIG_PM |
228 | .driver.pm = &igb_pm_ops, |
229 | #endif |
230 | .shutdown = igb_shutdown, |
231 | .sriov_configure = igb_pci_sriov_configure, |
232 | .err_handler = &igb_err_handler |
233 | }; |
234 | |
235 | MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>" ); |
236 | MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver" ); |
237 | MODULE_LICENSE("GPL v2" ); |
238 | |
239 | #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) |
240 | static int debug = -1; |
241 | module_param(debug, int, 0); |
242 | MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)" ); |
243 | |
244 | struct igb_reg_info { |
245 | u32 ofs; |
246 | char *name; |
247 | }; |
248 | |
249 | static const struct igb_reg_info igb_reg_info_tbl[] = { |
250 | |
251 | /* General Registers */ |
252 | {E1000_CTRL, "CTRL" }, |
253 | {E1000_STATUS, "STATUS" }, |
254 | {E1000_CTRL_EXT, "CTRL_EXT" }, |
255 | |
256 | /* Interrupt Registers */ |
257 | {E1000_ICR, "ICR" }, |
258 | |
259 | /* RX Registers */ |
260 | {E1000_RCTL, "RCTL" }, |
261 | {E1000_RDLEN(0), "RDLEN" }, |
262 | {E1000_RDH(0), "RDH" }, |
263 | {E1000_RDT(0), "RDT" }, |
264 | {E1000_RXDCTL(0), "RXDCTL" }, |
265 | {E1000_RDBAL(0), "RDBAL" }, |
266 | {E1000_RDBAH(0), "RDBAH" }, |
267 | |
268 | /* TX Registers */ |
269 | {E1000_TCTL, "TCTL" }, |
270 | {E1000_TDBAL(0), "TDBAL" }, |
271 | {E1000_TDBAH(0), "TDBAH" }, |
272 | {E1000_TDLEN(0), "TDLEN" }, |
273 | {E1000_TDH(0), "TDH" }, |
274 | {E1000_TDT(0), "TDT" }, |
275 | {E1000_TXDCTL(0), "TXDCTL" }, |
276 | {E1000_TDFH, "TDFH" }, |
277 | {E1000_TDFT, "TDFT" }, |
278 | {E1000_TDFHS, "TDFHS" }, |
279 | {E1000_TDFPC, "TDFPC" }, |
280 | |
281 | /* List Terminator */ |
282 | {} |
283 | }; |
284 | |
285 | /* igb_regdump - register printout routine */ |
286 | static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo) |
287 | { |
288 | int n = 0; |
289 | char rname[16]; |
290 | u32 regs[8]; |
291 | |
292 | switch (reginfo->ofs) { |
293 | case E1000_RDLEN(0): |
294 | for (n = 0; n < 4; n++) |
295 | regs[n] = rd32(E1000_RDLEN(n)); |
296 | break; |
297 | case E1000_RDH(0): |
298 | for (n = 0; n < 4; n++) |
299 | regs[n] = rd32(E1000_RDH(n)); |
300 | break; |
301 | case E1000_RDT(0): |
302 | for (n = 0; n < 4; n++) |
303 | regs[n] = rd32(E1000_RDT(n)); |
304 | break; |
305 | case E1000_RXDCTL(0): |
306 | for (n = 0; n < 4; n++) |
307 | regs[n] = rd32(E1000_RXDCTL(n)); |
308 | break; |
309 | case E1000_RDBAL(0): |
310 | for (n = 0; n < 4; n++) |
311 | regs[n] = rd32(E1000_RDBAL(n)); |
312 | break; |
313 | case E1000_RDBAH(0): |
314 | for (n = 0; n < 4; n++) |
315 | regs[n] = rd32(E1000_RDBAH(n)); |
316 | break; |
317 | case E1000_TDBAL(0): |
318 | for (n = 0; n < 4; n++) |
319 | regs[n] = rd32(E1000_TDBAL(n)); |
320 | break; |
321 | case E1000_TDBAH(0): |
322 | for (n = 0; n < 4; n++) |
323 | regs[n] = rd32(E1000_TDBAH(n)); |
324 | break; |
325 | case E1000_TDLEN(0): |
326 | for (n = 0; n < 4; n++) |
327 | regs[n] = rd32(E1000_TDLEN(n)); |
328 | break; |
329 | case E1000_TDH(0): |
330 | for (n = 0; n < 4; n++) |
331 | regs[n] = rd32(E1000_TDH(n)); |
332 | break; |
333 | case E1000_TDT(0): |
334 | for (n = 0; n < 4; n++) |
335 | regs[n] = rd32(E1000_TDT(n)); |
336 | break; |
337 | case E1000_TXDCTL(0): |
338 | for (n = 0; n < 4; n++) |
339 | regs[n] = rd32(E1000_TXDCTL(n)); |
340 | break; |
341 | default: |
342 | pr_info("%-15s %08x\n" , reginfo->name, rd32(reginfo->ofs)); |
343 | return; |
344 | } |
345 | |
346 | snprintf(buf: rname, size: 16, fmt: "%s%s" , reginfo->name, "[0-3]" ); |
347 | pr_info("%-15s %08x %08x %08x %08x\n" , rname, regs[0], regs[1], |
348 | regs[2], regs[3]); |
349 | } |
350 | |
351 | /* igb_dump - Print registers, Tx-rings and Rx-rings */ |
352 | static void igb_dump(struct igb_adapter *adapter) |
353 | { |
354 | struct net_device *netdev = adapter->netdev; |
355 | struct e1000_hw *hw = &adapter->hw; |
356 | struct igb_reg_info *reginfo; |
357 | struct igb_ring *tx_ring; |
358 | union e1000_adv_tx_desc *tx_desc; |
359 | struct my_u0 { __le64 a; __le64 b; } *u0; |
360 | struct igb_ring *rx_ring; |
361 | union e1000_adv_rx_desc *rx_desc; |
362 | u32 staterr; |
363 | u16 i, n; |
364 | |
365 | if (!netif_msg_hw(adapter)) |
366 | return; |
367 | |
368 | /* Print netdevice Info */ |
369 | if (netdev) { |
370 | dev_info(&adapter->pdev->dev, "Net device Info\n" ); |
371 | pr_info("Device Name state trans_start\n" ); |
372 | pr_info("%-15s %016lX %016lX\n" , netdev->name, |
373 | netdev->state, dev_trans_start(netdev)); |
374 | } |
375 | |
376 | /* Print Registers */ |
377 | dev_info(&adapter->pdev->dev, "Register Dump\n" ); |
378 | pr_info(" Register Name Value\n" ); |
379 | for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl; |
380 | reginfo->name; reginfo++) { |
381 | igb_regdump(hw, reginfo); |
382 | } |
383 | |
384 | /* Print TX Ring Summary */ |
385 | if (!netdev || !netif_running(dev: netdev)) |
386 | goto exit; |
387 | |
388 | dev_info(&adapter->pdev->dev, "TX Rings Summary\n" ); |
389 | pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n" ); |
390 | for (n = 0; n < adapter->num_tx_queues; n++) { |
391 | struct igb_tx_buffer *buffer_info; |
392 | tx_ring = adapter->tx_ring[n]; |
393 | buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean]; |
394 | pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n" , |
395 | n, tx_ring->next_to_use, tx_ring->next_to_clean, |
396 | (u64)dma_unmap_addr(buffer_info, dma), |
397 | dma_unmap_len(buffer_info, len), |
398 | buffer_info->next_to_watch, |
399 | (u64)buffer_info->time_stamp); |
400 | } |
401 | |
402 | /* Print TX Rings */ |
403 | if (!netif_msg_tx_done(adapter)) |
404 | goto rx_ring_summary; |
405 | |
406 | dev_info(&adapter->pdev->dev, "TX Rings Dump\n" ); |
407 | |
408 | /* Transmit Descriptor Formats |
409 | * |
410 | * Advanced Transmit Descriptor |
411 | * +--------------------------------------------------------------+ |
412 | * 0 | Buffer Address [63:0] | |
413 | * +--------------------------------------------------------------+ |
414 | * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN | |
415 | * +--------------------------------------------------------------+ |
416 | * 63 46 45 40 39 38 36 35 32 31 24 15 0 |
417 | */ |
418 | |
419 | for (n = 0; n < adapter->num_tx_queues; n++) { |
420 | tx_ring = adapter->tx_ring[n]; |
421 | pr_info("------------------------------------\n" ); |
422 | pr_info("TX QUEUE INDEX = %d\n" , tx_ring->queue_index); |
423 | pr_info("------------------------------------\n" ); |
424 | pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n" ); |
425 | |
426 | for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) { |
427 | const char *next_desc; |
428 | struct igb_tx_buffer *buffer_info; |
429 | tx_desc = IGB_TX_DESC(tx_ring, i); |
430 | buffer_info = &tx_ring->tx_buffer_info[i]; |
431 | u0 = (struct my_u0 *)tx_desc; |
432 | if (i == tx_ring->next_to_use && |
433 | i == tx_ring->next_to_clean) |
434 | next_desc = " NTC/U" ; |
435 | else if (i == tx_ring->next_to_use) |
436 | next_desc = " NTU" ; |
437 | else if (i == tx_ring->next_to_clean) |
438 | next_desc = " NTC" ; |
439 | else |
440 | next_desc = "" ; |
441 | |
442 | pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n" , |
443 | i, le64_to_cpu(u0->a), |
444 | le64_to_cpu(u0->b), |
445 | (u64)dma_unmap_addr(buffer_info, dma), |
446 | dma_unmap_len(buffer_info, len), |
447 | buffer_info->next_to_watch, |
448 | (u64)buffer_info->time_stamp, |
449 | buffer_info->skb, next_desc); |
450 | |
451 | if (netif_msg_pktdata(adapter) && buffer_info->skb) |
452 | print_hex_dump(KERN_INFO, prefix_str: "" , |
453 | prefix_type: DUMP_PREFIX_ADDRESS, |
454 | rowsize: 16, groupsize: 1, buf: buffer_info->skb->data, |
455 | dma_unmap_len(buffer_info, len), |
456 | ascii: true); |
457 | } |
458 | } |
459 | |
460 | /* Print RX Rings Summary */ |
461 | rx_ring_summary: |
462 | dev_info(&adapter->pdev->dev, "RX Rings Summary\n" ); |
463 | pr_info("Queue [NTU] [NTC]\n" ); |
464 | for (n = 0; n < adapter->num_rx_queues; n++) { |
465 | rx_ring = adapter->rx_ring[n]; |
466 | pr_info(" %5d %5X %5X\n" , |
467 | n, rx_ring->next_to_use, rx_ring->next_to_clean); |
468 | } |
469 | |
470 | /* Print RX Rings */ |
471 | if (!netif_msg_rx_status(adapter)) |
472 | goto exit; |
473 | |
474 | dev_info(&adapter->pdev->dev, "RX Rings Dump\n" ); |
475 | |
476 | /* Advanced Receive Descriptor (Read) Format |
477 | * 63 1 0 |
478 | * +-----------------------------------------------------+ |
479 | * 0 | Packet Buffer Address [63:1] |A0/NSE| |
480 | * +----------------------------------------------+------+ |
481 | * 8 | Header Buffer Address [63:1] | DD | |
482 | * +-----------------------------------------------------+ |
483 | * |
484 | * |
485 | * Advanced Receive Descriptor (Write-Back) Format |
486 | * |
487 | * 63 48 47 32 31 30 21 20 17 16 4 3 0 |
488 | * +------------------------------------------------------+ |
489 | * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS | |
490 | * | Checksum Ident | | | | Type | Type | |
491 | * +------------------------------------------------------+ |
492 | * 8 | VLAN Tag | Length | Extended Error | Extended Status | |
493 | * +------------------------------------------------------+ |
494 | * 63 48 47 32 31 20 19 0 |
495 | */ |
496 | |
497 | for (n = 0; n < adapter->num_rx_queues; n++) { |
498 | rx_ring = adapter->rx_ring[n]; |
499 | pr_info("------------------------------------\n" ); |
500 | pr_info("RX QUEUE INDEX = %d\n" , rx_ring->queue_index); |
501 | pr_info("------------------------------------\n" ); |
502 | pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n" ); |
503 | pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n" ); |
504 | |
505 | for (i = 0; i < rx_ring->count; i++) { |
506 | const char *next_desc; |
507 | struct igb_rx_buffer *buffer_info; |
508 | buffer_info = &rx_ring->rx_buffer_info[i]; |
509 | rx_desc = IGB_RX_DESC(rx_ring, i); |
510 | u0 = (struct my_u0 *)rx_desc; |
511 | staterr = le32_to_cpu(rx_desc->wb.upper.status_error); |
512 | |
513 | if (i == rx_ring->next_to_use) |
514 | next_desc = " NTU" ; |
515 | else if (i == rx_ring->next_to_clean) |
516 | next_desc = " NTC" ; |
517 | else |
518 | next_desc = "" ; |
519 | |
520 | if (staterr & E1000_RXD_STAT_DD) { |
521 | /* Descriptor Done */ |
522 | pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n" , |
523 | "RWB" , i, |
524 | le64_to_cpu(u0->a), |
525 | le64_to_cpu(u0->b), |
526 | next_desc); |
527 | } else { |
528 | pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n" , |
529 | "R " , i, |
530 | le64_to_cpu(u0->a), |
531 | le64_to_cpu(u0->b), |
532 | (u64)buffer_info->dma, |
533 | next_desc); |
534 | |
535 | if (netif_msg_pktdata(adapter) && |
536 | buffer_info->dma && buffer_info->page) { |
537 | print_hex_dump(KERN_INFO, prefix_str: "" , |
538 | prefix_type: DUMP_PREFIX_ADDRESS, |
539 | rowsize: 16, groupsize: 1, |
540 | page_address(buffer_info->page) + |
541 | buffer_info->page_offset, |
542 | len: igb_rx_bufsz(ring: rx_ring), ascii: true); |
543 | } |
544 | } |
545 | } |
546 | } |
547 | |
548 | exit: |
549 | return; |
550 | } |
551 | |
552 | /** |
553 | * igb_get_i2c_data - Reads the I2C SDA data bit |
554 | * @data: opaque pointer to adapter struct |
555 | * |
556 | * Returns the I2C data bit value |
557 | **/ |
558 | static int igb_get_i2c_data(void *data) |
559 | { |
560 | struct igb_adapter *adapter = (struct igb_adapter *)data; |
561 | struct e1000_hw *hw = &adapter->hw; |
562 | s32 i2cctl = rd32(E1000_I2CPARAMS); |
563 | |
564 | return !!(i2cctl & E1000_I2C_DATA_IN); |
565 | } |
566 | |
567 | /** |
568 | * igb_set_i2c_data - Sets the I2C data bit |
569 | * @data: pointer to hardware structure |
570 | * @state: I2C data value (0 or 1) to set |
571 | * |
572 | * Sets the I2C data bit |
573 | **/ |
574 | static void igb_set_i2c_data(void *data, int state) |
575 | { |
576 | struct igb_adapter *adapter = (struct igb_adapter *)data; |
577 | struct e1000_hw *hw = &adapter->hw; |
578 | s32 i2cctl = rd32(E1000_I2CPARAMS); |
579 | |
580 | if (state) { |
581 | i2cctl |= E1000_I2C_DATA_OUT | E1000_I2C_DATA_OE_N; |
582 | } else { |
583 | i2cctl &= ~E1000_I2C_DATA_OE_N; |
584 | i2cctl &= ~E1000_I2C_DATA_OUT; |
585 | } |
586 | |
587 | wr32(E1000_I2CPARAMS, i2cctl); |
588 | wrfl(); |
589 | } |
590 | |
591 | /** |
592 | * igb_set_i2c_clk - Sets the I2C SCL clock |
593 | * @data: pointer to hardware structure |
594 | * @state: state to set clock |
595 | * |
596 | * Sets the I2C clock line to state |
597 | **/ |
598 | static void igb_set_i2c_clk(void *data, int state) |
599 | { |
600 | struct igb_adapter *adapter = (struct igb_adapter *)data; |
601 | struct e1000_hw *hw = &adapter->hw; |
602 | s32 i2cctl = rd32(E1000_I2CPARAMS); |
603 | |
604 | if (state) { |
605 | i2cctl |= E1000_I2C_CLK_OUT | E1000_I2C_CLK_OE_N; |
606 | } else { |
607 | i2cctl &= ~E1000_I2C_CLK_OUT; |
608 | i2cctl &= ~E1000_I2C_CLK_OE_N; |
609 | } |
610 | wr32(E1000_I2CPARAMS, i2cctl); |
611 | wrfl(); |
612 | } |
613 | |
614 | /** |
615 | * igb_get_i2c_clk - Gets the I2C SCL clock state |
616 | * @data: pointer to hardware structure |
617 | * |
618 | * Gets the I2C clock state |
619 | **/ |
620 | static int igb_get_i2c_clk(void *data) |
621 | { |
622 | struct igb_adapter *adapter = (struct igb_adapter *)data; |
623 | struct e1000_hw *hw = &adapter->hw; |
624 | s32 i2cctl = rd32(E1000_I2CPARAMS); |
625 | |
626 | return !!(i2cctl & E1000_I2C_CLK_IN); |
627 | } |
628 | |
629 | static const struct i2c_algo_bit_data igb_i2c_algo = { |
630 | .setsda = igb_set_i2c_data, |
631 | .setscl = igb_set_i2c_clk, |
632 | .getsda = igb_get_i2c_data, |
633 | .getscl = igb_get_i2c_clk, |
634 | .udelay = 5, |
635 | .timeout = 20, |
636 | }; |
637 | |
638 | /** |
639 | * igb_get_hw_dev - return device |
640 | * @hw: pointer to hardware structure |
641 | * |
642 | * used by hardware layer to print debugging information |
643 | **/ |
644 | struct net_device *igb_get_hw_dev(struct e1000_hw *hw) |
645 | { |
646 | struct igb_adapter *adapter = hw->back; |
647 | return adapter->netdev; |
648 | } |
649 | |
650 | /** |
651 | * igb_init_module - Driver Registration Routine |
652 | * |
653 | * igb_init_module is the first routine called when the driver is |
654 | * loaded. All it does is register with the PCI subsystem. |
655 | **/ |
656 | static int __init igb_init_module(void) |
657 | { |
658 | int ret; |
659 | |
660 | pr_info("%s\n" , igb_driver_string); |
661 | pr_info("%s\n" , igb_copyright); |
662 | |
663 | #ifdef CONFIG_IGB_DCA |
664 | dca_register_notify(nb: &dca_notifier); |
665 | #endif |
666 | ret = pci_register_driver(&igb_driver); |
667 | return ret; |
668 | } |
669 | |
670 | module_init(igb_init_module); |
671 | |
672 | /** |
673 | * igb_exit_module - Driver Exit Cleanup Routine |
674 | * |
675 | * igb_exit_module is called just before the driver is removed |
676 | * from memory. |
677 | **/ |
678 | static void __exit igb_exit_module(void) |
679 | { |
680 | #ifdef CONFIG_IGB_DCA |
681 | dca_unregister_notify(nb: &dca_notifier); |
682 | #endif |
683 | pci_unregister_driver(dev: &igb_driver); |
684 | } |
685 | |
686 | module_exit(igb_exit_module); |
687 | |
688 | #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1)) |
689 | /** |
690 | * igb_cache_ring_register - Descriptor ring to register mapping |
691 | * @adapter: board private structure to initialize |
692 | * |
693 | * Once we know the feature-set enabled for the device, we'll cache |
694 | * the register offset the descriptor ring is assigned to. |
695 | **/ |
696 | static void igb_cache_ring_register(struct igb_adapter *adapter) |
697 | { |
698 | int i = 0, j = 0; |
699 | u32 rbase_offset = adapter->vfs_allocated_count; |
700 | |
701 | switch (adapter->hw.mac.type) { |
702 | case e1000_82576: |
703 | /* The queues are allocated for virtualization such that VF 0 |
704 | * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc. |
705 | * In order to avoid collision we start at the first free queue |
706 | * and continue consuming queues in the same sequence |
707 | */ |
708 | if (adapter->vfs_allocated_count) { |
709 | for (; i < adapter->rss_queues; i++) |
710 | adapter->rx_ring[i]->reg_idx = rbase_offset + |
711 | Q_IDX_82576(i); |
712 | } |
713 | fallthrough; |
714 | case e1000_82575: |
715 | case e1000_82580: |
716 | case e1000_i350: |
717 | case e1000_i354: |
718 | case e1000_i210: |
719 | case e1000_i211: |
720 | default: |
721 | for (; i < adapter->num_rx_queues; i++) |
722 | adapter->rx_ring[i]->reg_idx = rbase_offset + i; |
723 | for (; j < adapter->num_tx_queues; j++) |
724 | adapter->tx_ring[j]->reg_idx = rbase_offset + j; |
725 | break; |
726 | } |
727 | } |
728 | |
729 | u32 igb_rd32(struct e1000_hw *hw, u32 reg) |
730 | { |
731 | struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw); |
732 | u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr); |
733 | u32 value = 0; |
734 | |
735 | if (E1000_REMOVED(hw_addr)) |
736 | return ~value; |
737 | |
738 | value = readl(addr: &hw_addr[reg]); |
739 | |
740 | /* reads should not return all F's */ |
741 | if (!(~value) && (!reg || !(~readl(addr: hw_addr)))) { |
742 | struct net_device *netdev = igb->netdev; |
743 | hw->hw_addr = NULL; |
744 | netdev_err(dev: netdev, format: "PCIe link lost\n" ); |
745 | WARN(pci_device_is_present(igb->pdev), |
746 | "igb: Failed to read reg 0x%x!\n" , reg); |
747 | } |
748 | |
749 | return value; |
750 | } |
751 | |
752 | /** |
753 | * igb_write_ivar - configure ivar for given MSI-X vector |
754 | * @hw: pointer to the HW structure |
755 | * @msix_vector: vector number we are allocating to a given ring |
756 | * @index: row index of IVAR register to write within IVAR table |
757 | * @offset: column offset of in IVAR, should be multiple of 8 |
758 | * |
759 | * This function is intended to handle the writing of the IVAR register |
760 | * for adapters 82576 and newer. The IVAR table consists of 2 columns, |
761 | * each containing an cause allocation for an Rx and Tx ring, and a |
762 | * variable number of rows depending on the number of queues supported. |
763 | **/ |
764 | static void igb_write_ivar(struct e1000_hw *hw, int msix_vector, |
765 | int index, int offset) |
766 | { |
767 | u32 ivar = array_rd32(E1000_IVAR0, index); |
768 | |
769 | /* clear any bits that are currently set */ |
770 | ivar &= ~((u32)0xFF << offset); |
771 | |
772 | /* write vector and valid bit */ |
773 | ivar |= (msix_vector | E1000_IVAR_VALID) << offset; |
774 | |
775 | array_wr32(E1000_IVAR0, index, ivar); |
776 | } |
777 | |
778 | #define IGB_N0_QUEUE -1 |
779 | static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector) |
780 | { |
781 | struct igb_adapter *adapter = q_vector->adapter; |
782 | struct e1000_hw *hw = &adapter->hw; |
783 | int rx_queue = IGB_N0_QUEUE; |
784 | int tx_queue = IGB_N0_QUEUE; |
785 | u32 msixbm = 0; |
786 | |
787 | if (q_vector->rx.ring) |
788 | rx_queue = q_vector->rx.ring->reg_idx; |
789 | if (q_vector->tx.ring) |
790 | tx_queue = q_vector->tx.ring->reg_idx; |
791 | |
792 | switch (hw->mac.type) { |
793 | case e1000_82575: |
794 | /* The 82575 assigns vectors using a bitmask, which matches the |
795 | * bitmask for the EICR/EIMS/EIMC registers. To assign one |
796 | * or more queues to a vector, we write the appropriate bits |
797 | * into the MSIXBM register for that vector. |
798 | */ |
799 | if (rx_queue > IGB_N0_QUEUE) |
800 | msixbm = E1000_EICR_RX_QUEUE0 << rx_queue; |
801 | if (tx_queue > IGB_N0_QUEUE) |
802 | msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue; |
803 | if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0) |
804 | msixbm |= E1000_EIMS_OTHER; |
805 | array_wr32(E1000_MSIXBM(0), msix_vector, msixbm); |
806 | q_vector->eims_value = msixbm; |
807 | break; |
808 | case e1000_82576: |
809 | /* 82576 uses a table that essentially consists of 2 columns |
810 | * with 8 rows. The ordering is column-major so we use the |
811 | * lower 3 bits as the row index, and the 4th bit as the |
812 | * column offset. |
813 | */ |
814 | if (rx_queue > IGB_N0_QUEUE) |
815 | igb_write_ivar(hw, msix_vector, |
816 | index: rx_queue & 0x7, |
817 | offset: (rx_queue & 0x8) << 1); |
818 | if (tx_queue > IGB_N0_QUEUE) |
819 | igb_write_ivar(hw, msix_vector, |
820 | index: tx_queue & 0x7, |
821 | offset: ((tx_queue & 0x8) << 1) + 8); |
822 | q_vector->eims_value = BIT(msix_vector); |
823 | break; |
824 | case e1000_82580: |
825 | case e1000_i350: |
826 | case e1000_i354: |
827 | case e1000_i210: |
828 | case e1000_i211: |
829 | /* On 82580 and newer adapters the scheme is similar to 82576 |
830 | * however instead of ordering column-major we have things |
831 | * ordered row-major. So we traverse the table by using |
832 | * bit 0 as the column offset, and the remaining bits as the |
833 | * row index. |
834 | */ |
835 | if (rx_queue > IGB_N0_QUEUE) |
836 | igb_write_ivar(hw, msix_vector, |
837 | index: rx_queue >> 1, |
838 | offset: (rx_queue & 0x1) << 4); |
839 | if (tx_queue > IGB_N0_QUEUE) |
840 | igb_write_ivar(hw, msix_vector, |
841 | index: tx_queue >> 1, |
842 | offset: ((tx_queue & 0x1) << 4) + 8); |
843 | q_vector->eims_value = BIT(msix_vector); |
844 | break; |
845 | default: |
846 | BUG(); |
847 | break; |
848 | } |
849 | |
850 | /* add q_vector eims value to global eims_enable_mask */ |
851 | adapter->eims_enable_mask |= q_vector->eims_value; |
852 | |
853 | /* configure q_vector to set itr on first interrupt */ |
854 | q_vector->set_itr = 1; |
855 | } |
856 | |
857 | /** |
858 | * igb_configure_msix - Configure MSI-X hardware |
859 | * @adapter: board private structure to initialize |
860 | * |
861 | * igb_configure_msix sets up the hardware to properly |
862 | * generate MSI-X interrupts. |
863 | **/ |
864 | static void igb_configure_msix(struct igb_adapter *adapter) |
865 | { |
866 | u32 tmp; |
867 | int i, vector = 0; |
868 | struct e1000_hw *hw = &adapter->hw; |
869 | |
870 | adapter->eims_enable_mask = 0; |
871 | |
872 | /* set vector for other causes, i.e. link changes */ |
873 | switch (hw->mac.type) { |
874 | case e1000_82575: |
875 | tmp = rd32(E1000_CTRL_EXT); |
876 | /* enable MSI-X PBA support*/ |
877 | tmp |= E1000_CTRL_EXT_PBA_CLR; |
878 | |
879 | /* Auto-Mask interrupts upon ICR read. */ |
880 | tmp |= E1000_CTRL_EXT_EIAME; |
881 | tmp |= E1000_CTRL_EXT_IRCA; |
882 | |
883 | wr32(E1000_CTRL_EXT, tmp); |
884 | |
885 | /* enable msix_other interrupt */ |
886 | array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER); |
887 | adapter->eims_other = E1000_EIMS_OTHER; |
888 | |
889 | break; |
890 | |
891 | case e1000_82576: |
892 | case e1000_82580: |
893 | case e1000_i350: |
894 | case e1000_i354: |
895 | case e1000_i210: |
896 | case e1000_i211: |
897 | /* Turn on MSI-X capability first, or our settings |
898 | * won't stick. And it will take days to debug. |
899 | */ |
900 | wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE | |
901 | E1000_GPIE_PBA | E1000_GPIE_EIAME | |
902 | E1000_GPIE_NSICR); |
903 | |
904 | /* enable msix_other interrupt */ |
905 | adapter->eims_other = BIT(vector); |
906 | tmp = (vector++ | E1000_IVAR_VALID) << 8; |
907 | |
908 | wr32(E1000_IVAR_MISC, tmp); |
909 | break; |
910 | default: |
911 | /* do nothing, since nothing else supports MSI-X */ |
912 | break; |
913 | } /* switch (hw->mac.type) */ |
914 | |
915 | adapter->eims_enable_mask |= adapter->eims_other; |
916 | |
917 | for (i = 0; i < adapter->num_q_vectors; i++) |
918 | igb_assign_vector(q_vector: adapter->q_vector[i], msix_vector: vector++); |
919 | |
920 | wrfl(); |
921 | } |
922 | |
923 | /** |
924 | * igb_request_msix - Initialize MSI-X interrupts |
925 | * @adapter: board private structure to initialize |
926 | * |
927 | * igb_request_msix allocates MSI-X vectors and requests interrupts from the |
928 | * kernel. |
929 | **/ |
930 | static int igb_request_msix(struct igb_adapter *adapter) |
931 | { |
932 | unsigned int num_q_vectors = adapter->num_q_vectors; |
933 | struct net_device *netdev = adapter->netdev; |
934 | int i, err = 0, vector = 0, free_vector = 0; |
935 | |
936 | err = request_irq(irq: adapter->msix_entries[vector].vector, |
937 | handler: igb_msix_other, flags: 0, name: netdev->name, dev: adapter); |
938 | if (err) |
939 | goto err_out; |
940 | |
941 | if (num_q_vectors > MAX_Q_VECTORS) { |
942 | num_q_vectors = MAX_Q_VECTORS; |
943 | dev_warn(&adapter->pdev->dev, |
944 | "The number of queue vectors (%d) is higher than max allowed (%d)\n" , |
945 | adapter->num_q_vectors, MAX_Q_VECTORS); |
946 | } |
947 | for (i = 0; i < num_q_vectors; i++) { |
948 | struct igb_q_vector *q_vector = adapter->q_vector[i]; |
949 | |
950 | vector++; |
951 | |
952 | q_vector->itr_register = adapter->io_addr + E1000_EITR(vector); |
953 | |
954 | if (q_vector->rx.ring && q_vector->tx.ring) |
955 | sprintf(buf: q_vector->name, fmt: "%s-TxRx-%u" , netdev->name, |
956 | q_vector->rx.ring->queue_index); |
957 | else if (q_vector->tx.ring) |
958 | sprintf(buf: q_vector->name, fmt: "%s-tx-%u" , netdev->name, |
959 | q_vector->tx.ring->queue_index); |
960 | else if (q_vector->rx.ring) |
961 | sprintf(buf: q_vector->name, fmt: "%s-rx-%u" , netdev->name, |
962 | q_vector->rx.ring->queue_index); |
963 | else |
964 | sprintf(buf: q_vector->name, fmt: "%s-unused" , netdev->name); |
965 | |
966 | err = request_irq(irq: adapter->msix_entries[vector].vector, |
967 | handler: igb_msix_ring, flags: 0, name: q_vector->name, |
968 | dev: q_vector); |
969 | if (err) |
970 | goto err_free; |
971 | } |
972 | |
973 | igb_configure_msix(adapter); |
974 | return 0; |
975 | |
976 | err_free: |
977 | /* free already assigned IRQs */ |
978 | free_irq(adapter->msix_entries[free_vector++].vector, adapter); |
979 | |
980 | vector--; |
981 | for (i = 0; i < vector; i++) { |
982 | free_irq(adapter->msix_entries[free_vector++].vector, |
983 | adapter->q_vector[i]); |
984 | } |
985 | err_out: |
986 | return err; |
987 | } |
988 | |
989 | /** |
990 | * igb_free_q_vector - Free memory allocated for specific interrupt vector |
991 | * @adapter: board private structure to initialize |
992 | * @v_idx: Index of vector to be freed |
993 | * |
994 | * This function frees the memory allocated to the q_vector. |
995 | **/ |
996 | static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx) |
997 | { |
998 | struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; |
999 | |
1000 | adapter->q_vector[v_idx] = NULL; |
1001 | |
1002 | /* igb_get_stats64() might access the rings on this vector, |
1003 | * we must wait a grace period before freeing it. |
1004 | */ |
1005 | if (q_vector) |
1006 | kfree_rcu(q_vector, rcu); |
1007 | } |
1008 | |
1009 | /** |
1010 | * igb_reset_q_vector - Reset config for interrupt vector |
1011 | * @adapter: board private structure to initialize |
1012 | * @v_idx: Index of vector to be reset |
1013 | * |
1014 | * If NAPI is enabled it will delete any references to the |
1015 | * NAPI struct. This is preparation for igb_free_q_vector. |
1016 | **/ |
1017 | static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx) |
1018 | { |
1019 | struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; |
1020 | |
1021 | /* Coming from igb_set_interrupt_capability, the vectors are not yet |
1022 | * allocated. So, q_vector is NULL so we should stop here. |
1023 | */ |
1024 | if (!q_vector) |
1025 | return; |
1026 | |
1027 | if (q_vector->tx.ring) |
1028 | adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; |
1029 | |
1030 | if (q_vector->rx.ring) |
1031 | adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; |
1032 | |
1033 | netif_napi_del(napi: &q_vector->napi); |
1034 | |
1035 | } |
1036 | |
1037 | static void igb_reset_interrupt_capability(struct igb_adapter *adapter) |
1038 | { |
1039 | int v_idx = adapter->num_q_vectors; |
1040 | |
1041 | if (adapter->flags & IGB_FLAG_HAS_MSIX) |
1042 | pci_disable_msix(dev: adapter->pdev); |
1043 | else if (adapter->flags & IGB_FLAG_HAS_MSI) |
1044 | pci_disable_msi(dev: adapter->pdev); |
1045 | |
1046 | while (v_idx--) |
1047 | igb_reset_q_vector(adapter, v_idx); |
1048 | } |
1049 | |
1050 | /** |
1051 | * igb_free_q_vectors - Free memory allocated for interrupt vectors |
1052 | * @adapter: board private structure to initialize |
1053 | * |
1054 | * This function frees the memory allocated to the q_vectors. In addition if |
1055 | * NAPI is enabled it will delete any references to the NAPI struct prior |
1056 | * to freeing the q_vector. |
1057 | **/ |
1058 | static void igb_free_q_vectors(struct igb_adapter *adapter) |
1059 | { |
1060 | int v_idx = adapter->num_q_vectors; |
1061 | |
1062 | adapter->num_tx_queues = 0; |
1063 | adapter->num_rx_queues = 0; |
1064 | adapter->num_q_vectors = 0; |
1065 | |
1066 | while (v_idx--) { |
1067 | igb_reset_q_vector(adapter, v_idx); |
1068 | igb_free_q_vector(adapter, v_idx); |
1069 | } |
1070 | } |
1071 | |
1072 | /** |
1073 | * igb_clear_interrupt_scheme - reset the device to a state of no interrupts |
1074 | * @adapter: board private structure to initialize |
1075 | * |
1076 | * This function resets the device so that it has 0 Rx queues, Tx queues, and |
1077 | * MSI-X interrupts allocated. |
1078 | */ |
1079 | static void igb_clear_interrupt_scheme(struct igb_adapter *adapter) |
1080 | { |
1081 | igb_free_q_vectors(adapter); |
1082 | igb_reset_interrupt_capability(adapter); |
1083 | } |
1084 | |
1085 | /** |
1086 | * igb_set_interrupt_capability - set MSI or MSI-X if supported |
1087 | * @adapter: board private structure to initialize |
1088 | * @msix: boolean value of MSIX capability |
1089 | * |
1090 | * Attempt to configure interrupts using the best available |
1091 | * capabilities of the hardware and kernel. |
1092 | **/ |
1093 | static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix) |
1094 | { |
1095 | int err; |
1096 | int numvecs, i; |
1097 | |
1098 | if (!msix) |
1099 | goto msi_only; |
1100 | adapter->flags |= IGB_FLAG_HAS_MSIX; |
1101 | |
1102 | /* Number of supported queues. */ |
1103 | adapter->num_rx_queues = adapter->rss_queues; |
1104 | if (adapter->vfs_allocated_count) |
1105 | adapter->num_tx_queues = 1; |
1106 | else |
1107 | adapter->num_tx_queues = adapter->rss_queues; |
1108 | |
1109 | /* start with one vector for every Rx queue */ |
1110 | numvecs = adapter->num_rx_queues; |
1111 | |
1112 | /* if Tx handler is separate add 1 for every Tx queue */ |
1113 | if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS)) |
1114 | numvecs += adapter->num_tx_queues; |
1115 | |
1116 | /* store the number of vectors reserved for queues */ |
1117 | adapter->num_q_vectors = numvecs; |
1118 | |
1119 | /* add 1 vector for link status interrupts */ |
1120 | numvecs++; |
1121 | for (i = 0; i < numvecs; i++) |
1122 | adapter->msix_entries[i].entry = i; |
1123 | |
1124 | err = pci_enable_msix_range(dev: adapter->pdev, |
1125 | entries: adapter->msix_entries, |
1126 | minvec: numvecs, |
1127 | maxvec: numvecs); |
1128 | if (err > 0) |
1129 | return; |
1130 | |
1131 | igb_reset_interrupt_capability(adapter); |
1132 | |
1133 | /* If we can't do MSI-X, try MSI */ |
1134 | msi_only: |
1135 | adapter->flags &= ~IGB_FLAG_HAS_MSIX; |
1136 | #ifdef CONFIG_PCI_IOV |
1137 | /* disable SR-IOV for non MSI-X configurations */ |
1138 | if (adapter->vf_data) { |
1139 | struct e1000_hw *hw = &adapter->hw; |
1140 | /* disable iov and allow time for transactions to clear */ |
1141 | pci_disable_sriov(dev: adapter->pdev); |
1142 | msleep(msecs: 500); |
1143 | |
1144 | kfree(objp: adapter->vf_mac_list); |
1145 | adapter->vf_mac_list = NULL; |
1146 | kfree(objp: adapter->vf_data); |
1147 | adapter->vf_data = NULL; |
1148 | wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ); |
1149 | wrfl(); |
1150 | msleep(msecs: 100); |
1151 | dev_info(&adapter->pdev->dev, "IOV Disabled\n" ); |
1152 | } |
1153 | #endif |
1154 | adapter->vfs_allocated_count = 0; |
1155 | adapter->rss_queues = 1; |
1156 | adapter->flags |= IGB_FLAG_QUEUE_PAIRS; |
1157 | adapter->num_rx_queues = 1; |
1158 | adapter->num_tx_queues = 1; |
1159 | adapter->num_q_vectors = 1; |
1160 | if (!pci_enable_msi(dev: adapter->pdev)) |
1161 | adapter->flags |= IGB_FLAG_HAS_MSI; |
1162 | } |
1163 | |
1164 | static void igb_add_ring(struct igb_ring *ring, |
1165 | struct igb_ring_container *head) |
1166 | { |
1167 | head->ring = ring; |
1168 | head->count++; |
1169 | } |
1170 | |
1171 | /** |
1172 | * igb_alloc_q_vector - Allocate memory for a single interrupt vector |
1173 | * @adapter: board private structure to initialize |
1174 | * @v_count: q_vectors allocated on adapter, used for ring interleaving |
1175 | * @v_idx: index of vector in adapter struct |
1176 | * @txr_count: total number of Tx rings to allocate |
1177 | * @txr_idx: index of first Tx ring to allocate |
1178 | * @rxr_count: total number of Rx rings to allocate |
1179 | * @rxr_idx: index of first Rx ring to allocate |
1180 | * |
1181 | * We allocate one q_vector. If allocation fails we return -ENOMEM. |
1182 | **/ |
1183 | static int igb_alloc_q_vector(struct igb_adapter *adapter, |
1184 | int v_count, int v_idx, |
1185 | int txr_count, int txr_idx, |
1186 | int rxr_count, int rxr_idx) |
1187 | { |
1188 | struct igb_q_vector *q_vector; |
1189 | struct igb_ring *ring; |
1190 | int ring_count; |
1191 | size_t size; |
1192 | |
1193 | /* igb only supports 1 Tx and/or 1 Rx queue per vector */ |
1194 | if (txr_count > 1 || rxr_count > 1) |
1195 | return -ENOMEM; |
1196 | |
1197 | ring_count = txr_count + rxr_count; |
1198 | size = kmalloc_size_roundup(struct_size(q_vector, ring, ring_count)); |
1199 | |
1200 | /* allocate q_vector and rings */ |
1201 | q_vector = adapter->q_vector[v_idx]; |
1202 | if (!q_vector) { |
1203 | q_vector = kzalloc(size, GFP_KERNEL); |
1204 | } else if (size > ksize(objp: q_vector)) { |
1205 | struct igb_q_vector *new_q_vector; |
1206 | |
1207 | new_q_vector = kzalloc(size, GFP_KERNEL); |
1208 | if (new_q_vector) |
1209 | kfree_rcu(q_vector, rcu); |
1210 | q_vector = new_q_vector; |
1211 | } else { |
1212 | memset(q_vector, 0, size); |
1213 | } |
1214 | if (!q_vector) |
1215 | return -ENOMEM; |
1216 | |
1217 | /* initialize NAPI */ |
1218 | netif_napi_add(dev: adapter->netdev, napi: &q_vector->napi, poll: igb_poll); |
1219 | |
1220 | /* tie q_vector and adapter together */ |
1221 | adapter->q_vector[v_idx] = q_vector; |
1222 | q_vector->adapter = adapter; |
1223 | |
1224 | /* initialize work limits */ |
1225 | q_vector->tx.work_limit = adapter->tx_work_limit; |
1226 | |
1227 | /* initialize ITR configuration */ |
1228 | q_vector->itr_register = adapter->io_addr + E1000_EITR(0); |
1229 | q_vector->itr_val = IGB_START_ITR; |
1230 | |
1231 | /* initialize pointer to rings */ |
1232 | ring = q_vector->ring; |
1233 | |
1234 | /* intialize ITR */ |
1235 | if (rxr_count) { |
1236 | /* rx or rx/tx vector */ |
1237 | if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) |
1238 | q_vector->itr_val = adapter->rx_itr_setting; |
1239 | } else { |
1240 | /* tx only vector */ |
1241 | if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) |
1242 | q_vector->itr_val = adapter->tx_itr_setting; |
1243 | } |
1244 | |
1245 | if (txr_count) { |
1246 | /* assign generic ring traits */ |
1247 | ring->dev = &adapter->pdev->dev; |
1248 | ring->netdev = adapter->netdev; |
1249 | |
1250 | /* configure backlink on ring */ |
1251 | ring->q_vector = q_vector; |
1252 | |
1253 | /* update q_vector Tx values */ |
1254 | igb_add_ring(ring, head: &q_vector->tx); |
1255 | |
1256 | /* For 82575, context index must be unique per ring. */ |
1257 | if (adapter->hw.mac.type == e1000_82575) |
1258 | set_bit(nr: IGB_RING_FLAG_TX_CTX_IDX, addr: &ring->flags); |
1259 | |
1260 | /* apply Tx specific ring traits */ |
1261 | ring->count = adapter->tx_ring_count; |
1262 | ring->queue_index = txr_idx; |
1263 | |
1264 | ring->cbs_enable = false; |
1265 | ring->idleslope = 0; |
1266 | ring->sendslope = 0; |
1267 | ring->hicredit = 0; |
1268 | ring->locredit = 0; |
1269 | |
1270 | u64_stats_init(syncp: &ring->tx_syncp); |
1271 | u64_stats_init(syncp: &ring->tx_syncp2); |
1272 | |
1273 | /* assign ring to adapter */ |
1274 | adapter->tx_ring[txr_idx] = ring; |
1275 | |
1276 | /* push pointer to next ring */ |
1277 | ring++; |
1278 | } |
1279 | |
1280 | if (rxr_count) { |
1281 | /* assign generic ring traits */ |
1282 | ring->dev = &adapter->pdev->dev; |
1283 | ring->netdev = adapter->netdev; |
1284 | |
1285 | /* configure backlink on ring */ |
1286 | ring->q_vector = q_vector; |
1287 | |
1288 | /* update q_vector Rx values */ |
1289 | igb_add_ring(ring, head: &q_vector->rx); |
1290 | |
1291 | /* set flag indicating ring supports SCTP checksum offload */ |
1292 | if (adapter->hw.mac.type >= e1000_82576) |
1293 | set_bit(nr: IGB_RING_FLAG_RX_SCTP_CSUM, addr: &ring->flags); |
1294 | |
1295 | /* On i350, i354, i210, and i211, loopback VLAN packets |
1296 | * have the tag byte-swapped. |
1297 | */ |
1298 | if (adapter->hw.mac.type >= e1000_i350) |
1299 | set_bit(nr: IGB_RING_FLAG_RX_LB_VLAN_BSWAP, addr: &ring->flags); |
1300 | |
1301 | /* apply Rx specific ring traits */ |
1302 | ring->count = adapter->rx_ring_count; |
1303 | ring->queue_index = rxr_idx; |
1304 | |
1305 | u64_stats_init(syncp: &ring->rx_syncp); |
1306 | |
1307 | /* assign ring to adapter */ |
1308 | adapter->rx_ring[rxr_idx] = ring; |
1309 | } |
1310 | |
1311 | return 0; |
1312 | } |
1313 | |
1314 | |
1315 | /** |
1316 | * igb_alloc_q_vectors - Allocate memory for interrupt vectors |
1317 | * @adapter: board private structure to initialize |
1318 | * |
1319 | * We allocate one q_vector per queue interrupt. If allocation fails we |
1320 | * return -ENOMEM. |
1321 | **/ |
1322 | static int igb_alloc_q_vectors(struct igb_adapter *adapter) |
1323 | { |
1324 | int q_vectors = adapter->num_q_vectors; |
1325 | int rxr_remaining = adapter->num_rx_queues; |
1326 | int txr_remaining = adapter->num_tx_queues; |
1327 | int rxr_idx = 0, txr_idx = 0, v_idx = 0; |
1328 | int err; |
1329 | |
1330 | if (q_vectors >= (rxr_remaining + txr_remaining)) { |
1331 | for (; rxr_remaining; v_idx++) { |
1332 | err = igb_alloc_q_vector(adapter, v_count: q_vectors, v_idx, |
1333 | txr_count: 0, txr_idx: 0, rxr_count: 1, rxr_idx); |
1334 | |
1335 | if (err) |
1336 | goto err_out; |
1337 | |
1338 | /* update counts and index */ |
1339 | rxr_remaining--; |
1340 | rxr_idx++; |
1341 | } |
1342 | } |
1343 | |
1344 | for (; v_idx < q_vectors; v_idx++) { |
1345 | int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); |
1346 | int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); |
1347 | |
1348 | err = igb_alloc_q_vector(adapter, v_count: q_vectors, v_idx, |
1349 | txr_count: tqpv, txr_idx, rxr_count: rqpv, rxr_idx); |
1350 | |
1351 | if (err) |
1352 | goto err_out; |
1353 | |
1354 | /* update counts and index */ |
1355 | rxr_remaining -= rqpv; |
1356 | txr_remaining -= tqpv; |
1357 | rxr_idx++; |
1358 | txr_idx++; |
1359 | } |
1360 | |
1361 | return 0; |
1362 | |
1363 | err_out: |
1364 | adapter->num_tx_queues = 0; |
1365 | adapter->num_rx_queues = 0; |
1366 | adapter->num_q_vectors = 0; |
1367 | |
1368 | while (v_idx--) |
1369 | igb_free_q_vector(adapter, v_idx); |
1370 | |
1371 | return -ENOMEM; |
1372 | } |
1373 | |
1374 | /** |
1375 | * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors |
1376 | * @adapter: board private structure to initialize |
1377 | * @msix: boolean value of MSIX capability |
1378 | * |
1379 | * This function initializes the interrupts and allocates all of the queues. |
1380 | **/ |
1381 | static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix) |
1382 | { |
1383 | struct pci_dev *pdev = adapter->pdev; |
1384 | int err; |
1385 | |
1386 | igb_set_interrupt_capability(adapter, msix); |
1387 | |
1388 | err = igb_alloc_q_vectors(adapter); |
1389 | if (err) { |
1390 | dev_err(&pdev->dev, "Unable to allocate memory for vectors\n" ); |
1391 | goto err_alloc_q_vectors; |
1392 | } |
1393 | |
1394 | igb_cache_ring_register(adapter); |
1395 | |
1396 | return 0; |
1397 | |
1398 | err_alloc_q_vectors: |
1399 | igb_reset_interrupt_capability(adapter); |
1400 | return err; |
1401 | } |
1402 | |
1403 | /** |
1404 | * igb_request_irq - initialize interrupts |
1405 | * @adapter: board private structure to initialize |
1406 | * |
1407 | * Attempts to configure interrupts using the best available |
1408 | * capabilities of the hardware and kernel. |
1409 | **/ |
1410 | static int igb_request_irq(struct igb_adapter *adapter) |
1411 | { |
1412 | struct net_device *netdev = adapter->netdev; |
1413 | struct pci_dev *pdev = adapter->pdev; |
1414 | int err = 0; |
1415 | |
1416 | if (adapter->flags & IGB_FLAG_HAS_MSIX) { |
1417 | err = igb_request_msix(adapter); |
1418 | if (!err) |
1419 | goto request_done; |
1420 | /* fall back to MSI */ |
1421 | igb_free_all_tx_resources(adapter); |
1422 | igb_free_all_rx_resources(adapter); |
1423 | |
1424 | igb_clear_interrupt_scheme(adapter); |
1425 | err = igb_init_interrupt_scheme(adapter, msix: false); |
1426 | if (err) |
1427 | goto request_done; |
1428 | |
1429 | igb_setup_all_tx_resources(adapter); |
1430 | igb_setup_all_rx_resources(adapter); |
1431 | igb_configure(adapter); |
1432 | } |
1433 | |
1434 | igb_assign_vector(q_vector: adapter->q_vector[0], msix_vector: 0); |
1435 | |
1436 | if (adapter->flags & IGB_FLAG_HAS_MSI) { |
1437 | err = request_irq(irq: pdev->irq, handler: igb_intr_msi, flags: 0, |
1438 | name: netdev->name, dev: adapter); |
1439 | if (!err) |
1440 | goto request_done; |
1441 | |
1442 | /* fall back to legacy interrupts */ |
1443 | igb_reset_interrupt_capability(adapter); |
1444 | adapter->flags &= ~IGB_FLAG_HAS_MSI; |
1445 | } |
1446 | |
1447 | err = request_irq(irq: pdev->irq, handler: igb_intr, IRQF_SHARED, |
1448 | name: netdev->name, dev: adapter); |
1449 | |
1450 | if (err) |
1451 | dev_err(&pdev->dev, "Error %d getting interrupt\n" , |
1452 | err); |
1453 | |
1454 | request_done: |
1455 | return err; |
1456 | } |
1457 | |
1458 | static void igb_free_irq(struct igb_adapter *adapter) |
1459 | { |
1460 | if (adapter->flags & IGB_FLAG_HAS_MSIX) { |
1461 | int vector = 0, i; |
1462 | |
1463 | free_irq(adapter->msix_entries[vector++].vector, adapter); |
1464 | |
1465 | for (i = 0; i < adapter->num_q_vectors; i++) |
1466 | free_irq(adapter->msix_entries[vector++].vector, |
1467 | adapter->q_vector[i]); |
1468 | } else { |
1469 | free_irq(adapter->pdev->irq, adapter); |
1470 | } |
1471 | } |
1472 | |
1473 | /** |
1474 | * igb_irq_disable - Mask off interrupt generation on the NIC |
1475 | * @adapter: board private structure |
1476 | **/ |
1477 | static void igb_irq_disable(struct igb_adapter *adapter) |
1478 | { |
1479 | struct e1000_hw *hw = &adapter->hw; |
1480 | |
1481 | /* we need to be careful when disabling interrupts. The VFs are also |
1482 | * mapped into these registers and so clearing the bits can cause |
1483 | * issues on the VF drivers so we only need to clear what we set |
1484 | */ |
1485 | if (adapter->flags & IGB_FLAG_HAS_MSIX) { |
1486 | u32 regval = rd32(E1000_EIAM); |
1487 | |
1488 | wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask); |
1489 | wr32(E1000_EIMC, adapter->eims_enable_mask); |
1490 | regval = rd32(E1000_EIAC); |
1491 | wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask); |
1492 | } |
1493 | |
1494 | wr32(E1000_IAM, 0); |
1495 | wr32(E1000_IMC, ~0); |
1496 | wrfl(); |
1497 | if (adapter->flags & IGB_FLAG_HAS_MSIX) { |
1498 | int i; |
1499 | |
1500 | for (i = 0; i < adapter->num_q_vectors; i++) |
1501 | synchronize_irq(irq: adapter->msix_entries[i].vector); |
1502 | } else { |
1503 | synchronize_irq(irq: adapter->pdev->irq); |
1504 | } |
1505 | } |
1506 | |
1507 | /** |
1508 | * igb_irq_enable - Enable default interrupt generation settings |
1509 | * @adapter: board private structure |
1510 | **/ |
1511 | static void igb_irq_enable(struct igb_adapter *adapter) |
1512 | { |
1513 | struct e1000_hw *hw = &adapter->hw; |
1514 | |
1515 | if (adapter->flags & IGB_FLAG_HAS_MSIX) { |
1516 | u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA; |
1517 | u32 regval = rd32(E1000_EIAC); |
1518 | |
1519 | wr32(E1000_EIAC, regval | adapter->eims_enable_mask); |
1520 | regval = rd32(E1000_EIAM); |
1521 | wr32(E1000_EIAM, regval | adapter->eims_enable_mask); |
1522 | wr32(E1000_EIMS, adapter->eims_enable_mask); |
1523 | if (adapter->vfs_allocated_count) { |
1524 | wr32(E1000_MBVFIMR, 0xFF); |
1525 | ims |= E1000_IMS_VMMB; |
1526 | } |
1527 | wr32(E1000_IMS, ims); |
1528 | } else { |
1529 | wr32(E1000_IMS, IMS_ENABLE_MASK | |
1530 | E1000_IMS_DRSTA); |
1531 | wr32(E1000_IAM, IMS_ENABLE_MASK | |
1532 | E1000_IMS_DRSTA); |
1533 | } |
1534 | } |
1535 | |
1536 | static void igb_update_mng_vlan(struct igb_adapter *adapter) |
1537 | { |
1538 | struct e1000_hw *hw = &adapter->hw; |
1539 | u16 pf_id = adapter->vfs_allocated_count; |
1540 | u16 vid = adapter->hw.mng_cookie.vlan_id; |
1541 | u16 old_vid = adapter->mng_vlan_id; |
1542 | |
1543 | if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { |
1544 | /* add VID to filter table */ |
1545 | igb_vfta_set(hw, vid, vind: pf_id, vlan_on: true, vlvf_bypass: true); |
1546 | adapter->mng_vlan_id = vid; |
1547 | } else { |
1548 | adapter->mng_vlan_id = IGB_MNG_VLAN_NONE; |
1549 | } |
1550 | |
1551 | if ((old_vid != (u16)IGB_MNG_VLAN_NONE) && |
1552 | (vid != old_vid) && |
1553 | !test_bit(old_vid, adapter->active_vlans)) { |
1554 | /* remove VID from filter table */ |
1555 | igb_vfta_set(hw, vid, vind: pf_id, vlan_on: false, vlvf_bypass: true); |
1556 | } |
1557 | } |
1558 | |
1559 | /** |
1560 | * igb_release_hw_control - release control of the h/w to f/w |
1561 | * @adapter: address of board private structure |
1562 | * |
1563 | * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. |
1564 | * For ASF and Pass Through versions of f/w this means that the |
1565 | * driver is no longer loaded. |
1566 | **/ |
1567 | static void igb_release_hw_control(struct igb_adapter *adapter) |
1568 | { |
1569 | struct e1000_hw *hw = &adapter->hw; |
1570 | u32 ctrl_ext; |
1571 | |
1572 | /* Let firmware take over control of h/w */ |
1573 | ctrl_ext = rd32(E1000_CTRL_EXT); |
1574 | wr32(E1000_CTRL_EXT, |
1575 | ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); |
1576 | } |
1577 | |
1578 | /** |
1579 | * igb_get_hw_control - get control of the h/w from f/w |
1580 | * @adapter: address of board private structure |
1581 | * |
1582 | * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. |
1583 | * For ASF and Pass Through versions of f/w this means that |
1584 | * the driver is loaded. |
1585 | **/ |
1586 | static void igb_get_hw_control(struct igb_adapter *adapter) |
1587 | { |
1588 | struct e1000_hw *hw = &adapter->hw; |
1589 | u32 ctrl_ext; |
1590 | |
1591 | /* Let firmware know the driver has taken over */ |
1592 | ctrl_ext = rd32(E1000_CTRL_EXT); |
1593 | wr32(E1000_CTRL_EXT, |
1594 | ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); |
1595 | } |
1596 | |
1597 | static void enable_fqtss(struct igb_adapter *adapter, bool enable) |
1598 | { |
1599 | struct net_device *netdev = adapter->netdev; |
1600 | struct e1000_hw *hw = &adapter->hw; |
1601 | |
1602 | WARN_ON(hw->mac.type != e1000_i210); |
1603 | |
1604 | if (enable) |
1605 | adapter->flags |= IGB_FLAG_FQTSS; |
1606 | else |
1607 | adapter->flags &= ~IGB_FLAG_FQTSS; |
1608 | |
1609 | if (netif_running(dev: netdev)) |
1610 | schedule_work(work: &adapter->reset_task); |
1611 | } |
1612 | |
1613 | static bool is_fqtss_enabled(struct igb_adapter *adapter) |
1614 | { |
1615 | return (adapter->flags & IGB_FLAG_FQTSS) ? true : false; |
1616 | } |
1617 | |
1618 | static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue, |
1619 | enum tx_queue_prio prio) |
1620 | { |
1621 | u32 val; |
1622 | |
1623 | WARN_ON(hw->mac.type != e1000_i210); |
1624 | WARN_ON(queue < 0 || queue > 4); |
1625 | |
1626 | val = rd32(E1000_I210_TXDCTL(queue)); |
1627 | |
1628 | if (prio == TX_QUEUE_PRIO_HIGH) |
1629 | val |= E1000_TXDCTL_PRIORITY; |
1630 | else |
1631 | val &= ~E1000_TXDCTL_PRIORITY; |
1632 | |
1633 | wr32(E1000_I210_TXDCTL(queue), val); |
1634 | } |
1635 | |
1636 | static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode) |
1637 | { |
1638 | u32 val; |
1639 | |
1640 | WARN_ON(hw->mac.type != e1000_i210); |
1641 | WARN_ON(queue < 0 || queue > 1); |
1642 | |
1643 | val = rd32(E1000_I210_TQAVCC(queue)); |
1644 | |
1645 | if (mode == QUEUE_MODE_STREAM_RESERVATION) |
1646 | val |= E1000_TQAVCC_QUEUEMODE; |
1647 | else |
1648 | val &= ~E1000_TQAVCC_QUEUEMODE; |
1649 | |
1650 | wr32(E1000_I210_TQAVCC(queue), val); |
1651 | } |
1652 | |
1653 | static bool is_any_cbs_enabled(struct igb_adapter *adapter) |
1654 | { |
1655 | int i; |
1656 | |
1657 | for (i = 0; i < adapter->num_tx_queues; i++) { |
1658 | if (adapter->tx_ring[i]->cbs_enable) |
1659 | return true; |
1660 | } |
1661 | |
1662 | return false; |
1663 | } |
1664 | |
1665 | static bool is_any_txtime_enabled(struct igb_adapter *adapter) |
1666 | { |
1667 | int i; |
1668 | |
1669 | for (i = 0; i < adapter->num_tx_queues; i++) { |
1670 | if (adapter->tx_ring[i]->launchtime_enable) |
1671 | return true; |
1672 | } |
1673 | |
1674 | return false; |
1675 | } |
1676 | |
1677 | /** |
1678 | * igb_config_tx_modes - Configure "Qav Tx mode" features on igb |
1679 | * @adapter: pointer to adapter struct |
1680 | * @queue: queue number |
1681 | * |
1682 | * Configure CBS and Launchtime for a given hardware queue. |
1683 | * Parameters are retrieved from the correct Tx ring, so |
1684 | * igb_save_cbs_params() and igb_save_txtime_params() should be used |
1685 | * for setting those correctly prior to this function being called. |
1686 | **/ |
1687 | static void igb_config_tx_modes(struct igb_adapter *adapter, int queue) |
1688 | { |
1689 | struct net_device *netdev = adapter->netdev; |
1690 | struct e1000_hw *hw = &adapter->hw; |
1691 | struct igb_ring *ring; |
1692 | u32 tqavcc, tqavctrl; |
1693 | u16 value; |
1694 | |
1695 | WARN_ON(hw->mac.type != e1000_i210); |
1696 | WARN_ON(queue < 0 || queue > 1); |
1697 | ring = adapter->tx_ring[queue]; |
1698 | |
1699 | /* If any of the Qav features is enabled, configure queues as SR and |
1700 | * with HIGH PRIO. If none is, then configure them with LOW PRIO and |
1701 | * as SP. |
1702 | */ |
1703 | if (ring->cbs_enable || ring->launchtime_enable) { |
1704 | set_tx_desc_fetch_prio(hw, queue, prio: TX_QUEUE_PRIO_HIGH); |
1705 | set_queue_mode(hw, queue, mode: QUEUE_MODE_STREAM_RESERVATION); |
1706 | } else { |
1707 | set_tx_desc_fetch_prio(hw, queue, prio: TX_QUEUE_PRIO_LOW); |
1708 | set_queue_mode(hw, queue, mode: QUEUE_MODE_STRICT_PRIORITY); |
1709 | } |
1710 | |
1711 | /* If CBS is enabled, set DataTranARB and config its parameters. */ |
1712 | if (ring->cbs_enable || queue == 0) { |
1713 | /* i210 does not allow the queue 0 to be in the Strict |
1714 | * Priority mode while the Qav mode is enabled, so, |
1715 | * instead of disabling strict priority mode, we give |
1716 | * queue 0 the maximum of credits possible. |
1717 | * |
1718 | * See section 8.12.19 of the i210 datasheet, "Note: |
1719 | * Queue0 QueueMode must be set to 1b when |
1720 | * TransmitMode is set to Qav." |
1721 | */ |
1722 | if (queue == 0 && !ring->cbs_enable) { |
1723 | /* max "linkspeed" idleslope in kbps */ |
1724 | ring->idleslope = 1000000; |
1725 | ring->hicredit = ETH_FRAME_LEN; |
1726 | } |
1727 | |
1728 | /* Always set data transfer arbitration to credit-based |
1729 | * shaper algorithm on TQAVCTRL if CBS is enabled for any of |
1730 | * the queues. |
1731 | */ |
1732 | tqavctrl = rd32(E1000_I210_TQAVCTRL); |
1733 | tqavctrl |= E1000_TQAVCTRL_DATATRANARB; |
1734 | wr32(E1000_I210_TQAVCTRL, tqavctrl); |
1735 | |
1736 | /* According to i210 datasheet section 7.2.7.7, we should set |
1737 | * the 'idleSlope' field from TQAVCC register following the |
1738 | * equation: |
1739 | * |
1740 | * For 100 Mbps link speed: |
1741 | * |
1742 | * value = BW * 0x7735 * 0.2 (E1) |
1743 | * |
1744 | * For 1000Mbps link speed: |
1745 | * |
1746 | * value = BW * 0x7735 * 2 (E2) |
1747 | * |
1748 | * E1 and E2 can be merged into one equation as shown below. |
1749 | * Note that 'link-speed' is in Mbps. |
1750 | * |
1751 | * value = BW * 0x7735 * 2 * link-speed |
1752 | * -------------- (E3) |
1753 | * 1000 |
1754 | * |
1755 | * 'BW' is the percentage bandwidth out of full link speed |
1756 | * which can be found with the following equation. Note that |
1757 | * idleSlope here is the parameter from this function which |
1758 | * is in kbps. |
1759 | * |
1760 | * BW = idleSlope |
1761 | * ----------------- (E4) |
1762 | * link-speed * 1000 |
1763 | * |
1764 | * That said, we can come up with a generic equation to |
1765 | * calculate the value we should set it TQAVCC register by |
1766 | * replacing 'BW' in E3 by E4. The resulting equation is: |
1767 | * |
1768 | * value = idleSlope * 0x7735 * 2 * link-speed |
1769 | * ----------------- -------------- (E5) |
1770 | * link-speed * 1000 1000 |
1771 | * |
1772 | * 'link-speed' is present in both sides of the fraction so |
1773 | * it is canceled out. The final equation is the following: |
1774 | * |
1775 | * value = idleSlope * 61034 |
1776 | * ----------------- (E6) |
1777 | * 1000000 |
1778 | * |
1779 | * NOTE: For i210, given the above, we can see that idleslope |
1780 | * is represented in 16.38431 kbps units by the value at |
1781 | * the TQAVCC register (1Gbps / 61034), which reduces |
1782 | * the granularity for idleslope increments. |
1783 | * For instance, if you want to configure a 2576kbps |
1784 | * idleslope, the value to be written on the register |
1785 | * would have to be 157.23. If rounded down, you end |
1786 | * up with less bandwidth available than originally |
1787 | * required (~2572 kbps). If rounded up, you end up |
1788 | * with a higher bandwidth (~2589 kbps). Below the |
1789 | * approach we take is to always round up the |
1790 | * calculated value, so the resulting bandwidth might |
1791 | * be slightly higher for some configurations. |
1792 | */ |
1793 | value = DIV_ROUND_UP_ULL(ring->idleslope * 61034ULL, 1000000); |
1794 | |
1795 | tqavcc = rd32(E1000_I210_TQAVCC(queue)); |
1796 | tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK; |
1797 | tqavcc |= value; |
1798 | wr32(E1000_I210_TQAVCC(queue), tqavcc); |
1799 | |
1800 | wr32(E1000_I210_TQAVHC(queue), |
1801 | 0x80000000 + ring->hicredit * 0x7735); |
1802 | } else { |
1803 | |
1804 | /* Set idleSlope to zero. */ |
1805 | tqavcc = rd32(E1000_I210_TQAVCC(queue)); |
1806 | tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK; |
1807 | wr32(E1000_I210_TQAVCC(queue), tqavcc); |
1808 | |
1809 | /* Set hiCredit to zero. */ |
1810 | wr32(E1000_I210_TQAVHC(queue), 0); |
1811 | |
1812 | /* If CBS is not enabled for any queues anymore, then return to |
1813 | * the default state of Data Transmission Arbitration on |
1814 | * TQAVCTRL. |
1815 | */ |
1816 | if (!is_any_cbs_enabled(adapter)) { |
1817 | tqavctrl = rd32(E1000_I210_TQAVCTRL); |
1818 | tqavctrl &= ~E1000_TQAVCTRL_DATATRANARB; |
1819 | wr32(E1000_I210_TQAVCTRL, tqavctrl); |
1820 | } |
1821 | } |
1822 | |
1823 | /* If LaunchTime is enabled, set DataTranTIM. */ |
1824 | if (ring->launchtime_enable) { |
1825 | /* Always set DataTranTIM on TQAVCTRL if LaunchTime is enabled |
1826 | * for any of the SR queues, and configure fetchtime delta. |
1827 | * XXX NOTE: |
1828 | * - LaunchTime will be enabled for all SR queues. |
1829 | * - A fixed offset can be added relative to the launch |
1830 | * time of all packets if configured at reg LAUNCH_OS0. |
1831 | * We are keeping it as 0 for now (default value). |
1832 | */ |
1833 | tqavctrl = rd32(E1000_I210_TQAVCTRL); |
1834 | tqavctrl |= E1000_TQAVCTRL_DATATRANTIM | |
1835 | E1000_TQAVCTRL_FETCHTIME_DELTA; |
1836 | wr32(E1000_I210_TQAVCTRL, tqavctrl); |
1837 | } else { |
1838 | /* If Launchtime is not enabled for any SR queues anymore, |
1839 | * then clear DataTranTIM on TQAVCTRL and clear fetchtime delta, |
1840 | * effectively disabling Launchtime. |
1841 | */ |
1842 | if (!is_any_txtime_enabled(adapter)) { |
1843 | tqavctrl = rd32(E1000_I210_TQAVCTRL); |
1844 | tqavctrl &= ~E1000_TQAVCTRL_DATATRANTIM; |
1845 | tqavctrl &= ~E1000_TQAVCTRL_FETCHTIME_DELTA; |
1846 | wr32(E1000_I210_TQAVCTRL, tqavctrl); |
1847 | } |
1848 | } |
1849 | |
1850 | /* XXX: In i210 controller the sendSlope and loCredit parameters from |
1851 | * CBS are not configurable by software so we don't do any 'controller |
1852 | * configuration' in respect to these parameters. |
1853 | */ |
1854 | |
1855 | netdev_dbg(netdev, "Qav Tx mode: cbs %s, launchtime %s, queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n" , |
1856 | ring->cbs_enable ? "enabled" : "disabled" , |
1857 | ring->launchtime_enable ? "enabled" : "disabled" , |
1858 | queue, |
1859 | ring->idleslope, ring->sendslope, |
1860 | ring->hicredit, ring->locredit); |
1861 | } |
1862 | |
1863 | static int igb_save_txtime_params(struct igb_adapter *adapter, int queue, |
1864 | bool enable) |
1865 | { |
1866 | struct igb_ring *ring; |
1867 | |
1868 | if (queue < 0 || queue > adapter->num_tx_queues) |
1869 | return -EINVAL; |
1870 | |
1871 | ring = adapter->tx_ring[queue]; |
1872 | ring->launchtime_enable = enable; |
1873 | |
1874 | return 0; |
1875 | } |
1876 | |
1877 | static int igb_save_cbs_params(struct igb_adapter *adapter, int queue, |
1878 | bool enable, int idleslope, int sendslope, |
1879 | int hicredit, int locredit) |
1880 | { |
1881 | struct igb_ring *ring; |
1882 | |
1883 | if (queue < 0 || queue > adapter->num_tx_queues) |
1884 | return -EINVAL; |
1885 | |
1886 | ring = adapter->tx_ring[queue]; |
1887 | |
1888 | ring->cbs_enable = enable; |
1889 | ring->idleslope = idleslope; |
1890 | ring->sendslope = sendslope; |
1891 | ring->hicredit = hicredit; |
1892 | ring->locredit = locredit; |
1893 | |
1894 | return 0; |
1895 | } |
1896 | |
1897 | /** |
1898 | * igb_setup_tx_mode - Switch to/from Qav Tx mode when applicable |
1899 | * @adapter: pointer to adapter struct |
1900 | * |
1901 | * Configure TQAVCTRL register switching the controller's Tx mode |
1902 | * if FQTSS mode is enabled or disabled. Additionally, will issue |
1903 | * a call to igb_config_tx_modes() per queue so any previously saved |
1904 | * Tx parameters are applied. |
1905 | **/ |
1906 | static void igb_setup_tx_mode(struct igb_adapter *adapter) |
1907 | { |
1908 | struct net_device *netdev = adapter->netdev; |
1909 | struct e1000_hw *hw = &adapter->hw; |
1910 | u32 val; |
1911 | |
1912 | /* Only i210 controller supports changing the transmission mode. */ |
1913 | if (hw->mac.type != e1000_i210) |
1914 | return; |
1915 | |
1916 | if (is_fqtss_enabled(adapter)) { |
1917 | int i, max_queue; |
1918 | |
1919 | /* Configure TQAVCTRL register: set transmit mode to 'Qav', |
1920 | * set data fetch arbitration to 'round robin', set SP_WAIT_SR |
1921 | * so SP queues wait for SR ones. |
1922 | */ |
1923 | val = rd32(E1000_I210_TQAVCTRL); |
1924 | val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_SP_WAIT_SR; |
1925 | val &= ~E1000_TQAVCTRL_DATAFETCHARB; |
1926 | wr32(E1000_I210_TQAVCTRL, val); |
1927 | |
1928 | /* Configure Tx and Rx packet buffers sizes as described in |
1929 | * i210 datasheet section 7.2.7.7. |
1930 | */ |
1931 | val = rd32(E1000_TXPBS); |
1932 | val &= ~I210_TXPBSIZE_MASK; |
1933 | val |= I210_TXPBSIZE_PB0_6KB | I210_TXPBSIZE_PB1_6KB | |
1934 | I210_TXPBSIZE_PB2_6KB | I210_TXPBSIZE_PB3_6KB; |
1935 | wr32(E1000_TXPBS, val); |
1936 | |
1937 | val = rd32(E1000_RXPBS); |
1938 | val &= ~I210_RXPBSIZE_MASK; |
1939 | val |= I210_RXPBSIZE_PB_30KB; |
1940 | wr32(E1000_RXPBS, val); |
1941 | |
1942 | /* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ |
1943 | * register should not exceed the buffer size programmed in |
1944 | * TXPBS. The smallest buffer size programmed in TXPBS is 4kB |
1945 | * so according to the datasheet we should set MAX_TPKT_SIZE to |
1946 | * 4kB / 64. |
1947 | * |
1948 | * However, when we do so, no frame from queue 2 and 3 are |
1949 | * transmitted. It seems the MAX_TPKT_SIZE should not be great |
1950 | * or _equal_ to the buffer size programmed in TXPBS. For this |
1951 | * reason, we set MAX_ TPKT_SIZE to (4kB - 1) / 64. |
1952 | */ |
1953 | val = (4096 - 1) / 64; |
1954 | wr32(E1000_I210_DTXMXPKTSZ, val); |
1955 | |
1956 | /* Since FQTSS mode is enabled, apply any CBS configuration |
1957 | * previously set. If no previous CBS configuration has been |
1958 | * done, then the initial configuration is applied, which means |
1959 | * CBS is disabled. |
1960 | */ |
1961 | max_queue = (adapter->num_tx_queues < I210_SR_QUEUES_NUM) ? |
1962 | adapter->num_tx_queues : I210_SR_QUEUES_NUM; |
1963 | |
1964 | for (i = 0; i < max_queue; i++) { |
1965 | igb_config_tx_modes(adapter, queue: i); |
1966 | } |
1967 | } else { |
1968 | wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT); |
1969 | wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT); |
1970 | wr32(E1000_I210_DTXMXPKTSZ, I210_DTXMXPKTSZ_DEFAULT); |
1971 | |
1972 | val = rd32(E1000_I210_TQAVCTRL); |
1973 | /* According to Section 8.12.21, the other flags we've set when |
1974 | * enabling FQTSS are not relevant when disabling FQTSS so we |
1975 | * don't set they here. |
1976 | */ |
1977 | val &= ~E1000_TQAVCTRL_XMIT_MODE; |
1978 | wr32(E1000_I210_TQAVCTRL, val); |
1979 | } |
1980 | |
1981 | netdev_dbg(netdev, "FQTSS %s\n" , (is_fqtss_enabled(adapter)) ? |
1982 | "enabled" : "disabled" ); |
1983 | } |
1984 | |
1985 | /** |
1986 | * igb_configure - configure the hardware for RX and TX |
1987 | * @adapter: private board structure |
1988 | **/ |
1989 | static void igb_configure(struct igb_adapter *adapter) |
1990 | { |
1991 | struct net_device *netdev = adapter->netdev; |
1992 | int i; |
1993 | |
1994 | igb_get_hw_control(adapter); |
1995 | igb_set_rx_mode(netdev); |
1996 | igb_setup_tx_mode(adapter); |
1997 | |
1998 | igb_restore_vlan(adapter); |
1999 | |
2000 | igb_setup_tctl(adapter); |
2001 | igb_setup_mrqc(adapter); |
2002 | igb_setup_rctl(adapter); |
2003 | |
2004 | igb_nfc_filter_restore(adapter); |
2005 | igb_configure_tx(adapter); |
2006 | igb_configure_rx(adapter); |
2007 | |
2008 | igb_rx_fifo_flush_82575(hw: &adapter->hw); |
2009 | |
2010 | /* call igb_desc_unused which always leaves |
2011 | * at least 1 descriptor unused to make sure |
2012 | * next_to_use != next_to_clean |
2013 | */ |
2014 | for (i = 0; i < adapter->num_rx_queues; i++) { |
2015 | struct igb_ring *ring = adapter->rx_ring[i]; |
2016 | igb_alloc_rx_buffers(ring, igb_desc_unused(ring)); |
2017 | } |
2018 | } |
2019 | |
2020 | /** |
2021 | * igb_power_up_link - Power up the phy/serdes link |
2022 | * @adapter: address of board private structure |
2023 | **/ |
2024 | void igb_power_up_link(struct igb_adapter *adapter) |
2025 | { |
2026 | igb_reset_phy(hw: &adapter->hw); |
2027 | |
2028 | if (adapter->hw.phy.media_type == e1000_media_type_copper) |
2029 | igb_power_up_phy_copper(hw: &adapter->hw); |
2030 | else |
2031 | igb_power_up_serdes_link_82575(hw: &adapter->hw); |
2032 | |
2033 | igb_setup_link(hw: &adapter->hw); |
2034 | } |
2035 | |
2036 | /** |
2037 | * igb_power_down_link - Power down the phy/serdes link |
2038 | * @adapter: address of board private structure |
2039 | */ |
2040 | static void igb_power_down_link(struct igb_adapter *adapter) |
2041 | { |
2042 | if (adapter->hw.phy.media_type == e1000_media_type_copper) |
2043 | igb_power_down_phy_copper_82575(hw: &adapter->hw); |
2044 | else |
2045 | igb_shutdown_serdes_link_82575(hw: &adapter->hw); |
2046 | } |
2047 | |
2048 | /** |
2049 | * igb_check_swap_media - Detect and switch function for Media Auto Sense |
2050 | * @adapter: address of the board private structure |
2051 | **/ |
2052 | static void igb_check_swap_media(struct igb_adapter *adapter) |
2053 | { |
2054 | struct e1000_hw *hw = &adapter->hw; |
2055 | u32 ctrl_ext, connsw; |
2056 | bool swap_now = false; |
2057 | |
2058 | ctrl_ext = rd32(E1000_CTRL_EXT); |
2059 | connsw = rd32(E1000_CONNSW); |
2060 | |
2061 | /* need to live swap if current media is copper and we have fiber/serdes |
2062 | * to go to. |
2063 | */ |
2064 | |
2065 | if ((hw->phy.media_type == e1000_media_type_copper) && |
2066 | (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) { |
2067 | swap_now = true; |
2068 | } else if ((hw->phy.media_type != e1000_media_type_copper) && |
2069 | !(connsw & E1000_CONNSW_SERDESD)) { |
2070 | /* copper signal takes time to appear */ |
2071 | if (adapter->copper_tries < 4) { |
2072 | adapter->copper_tries++; |
2073 | connsw |= E1000_CONNSW_AUTOSENSE_CONF; |
2074 | wr32(E1000_CONNSW, connsw); |
2075 | return; |
2076 | } else { |
2077 | adapter->copper_tries = 0; |
2078 | if ((connsw & E1000_CONNSW_PHYSD) && |
2079 | (!(connsw & E1000_CONNSW_PHY_PDN))) { |
2080 | swap_now = true; |
2081 | connsw &= ~E1000_CONNSW_AUTOSENSE_CONF; |
2082 | wr32(E1000_CONNSW, connsw); |
2083 | } |
2084 | } |
2085 | } |
2086 | |
2087 | if (!swap_now) |
2088 | return; |
2089 | |
2090 | switch (hw->phy.media_type) { |
2091 | case e1000_media_type_copper: |
2092 | netdev_info(dev: adapter->netdev, |
2093 | format: "MAS: changing media to fiber/serdes\n" ); |
2094 | ctrl_ext |= |
2095 | E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; |
2096 | adapter->flags |= IGB_FLAG_MEDIA_RESET; |
2097 | adapter->copper_tries = 0; |
2098 | break; |
2099 | case e1000_media_type_internal_serdes: |
2100 | case e1000_media_type_fiber: |
2101 | netdev_info(dev: adapter->netdev, |
2102 | format: "MAS: changing media to copper\n" ); |
2103 | ctrl_ext &= |
2104 | ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; |
2105 | adapter->flags |= IGB_FLAG_MEDIA_RESET; |
2106 | break; |
2107 | default: |
2108 | /* shouldn't get here during regular operation */ |
2109 | netdev_err(dev: adapter->netdev, |
2110 | format: "AMS: Invalid media type found, returning\n" ); |
2111 | break; |
2112 | } |
2113 | wr32(E1000_CTRL_EXT, ctrl_ext); |
2114 | } |
2115 | |
2116 | /** |
2117 | * igb_up - Open the interface and prepare it to handle traffic |
2118 | * @adapter: board private structure |
2119 | **/ |
2120 | int igb_up(struct igb_adapter *adapter) |
2121 | { |
2122 | struct e1000_hw *hw = &adapter->hw; |
2123 | int i; |
2124 | |
2125 | /* hardware has been reset, we need to reload some things */ |
2126 | igb_configure(adapter); |
2127 | |
2128 | clear_bit(nr: __IGB_DOWN, addr: &adapter->state); |
2129 | |
2130 | for (i = 0; i < adapter->num_q_vectors; i++) |
2131 | napi_enable(n: &(adapter->q_vector[i]->napi)); |
2132 | |
2133 | if (adapter->flags & IGB_FLAG_HAS_MSIX) |
2134 | igb_configure_msix(adapter); |
2135 | else |
2136 | igb_assign_vector(q_vector: adapter->q_vector[0], msix_vector: 0); |
2137 | |
2138 | /* Clear any pending interrupts. */ |
2139 | rd32(E1000_TSICR); |
2140 | rd32(E1000_ICR); |
2141 | igb_irq_enable(adapter); |
2142 | |
2143 | /* notify VFs that reset has been completed */ |
2144 | if (adapter->vfs_allocated_count) { |
2145 | u32 reg_data = rd32(E1000_CTRL_EXT); |
2146 | |
2147 | reg_data |= E1000_CTRL_EXT_PFRSTD; |
2148 | wr32(E1000_CTRL_EXT, reg_data); |
2149 | } |
2150 | |
2151 | netif_tx_start_all_queues(dev: adapter->netdev); |
2152 | |
2153 | /* start the watchdog. */ |
2154 | hw->mac.get_link_status = 1; |
2155 | schedule_work(work: &adapter->watchdog_task); |
2156 | |
2157 | if ((adapter->flags & IGB_FLAG_EEE) && |
2158 | (!hw->dev_spec._82575.eee_disable)) |
2159 | adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T; |
2160 | |
2161 | return 0; |
2162 | } |
2163 | |
2164 | void igb_down(struct igb_adapter *adapter) |
2165 | { |
2166 | struct net_device *netdev = adapter->netdev; |
2167 | struct e1000_hw *hw = &adapter->hw; |
2168 | u32 tctl, rctl; |
2169 | int i; |
2170 | |
2171 | /* signal that we're down so the interrupt handler does not |
2172 | * reschedule our watchdog timer |
2173 | */ |
2174 | set_bit(nr: __IGB_DOWN, addr: &adapter->state); |
2175 | |
2176 | /* disable receives in the hardware */ |
2177 | rctl = rd32(E1000_RCTL); |
2178 | wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN); |
2179 | /* flush and sleep below */ |
2180 | |
2181 | igb_nfc_filter_exit(adapter); |
2182 | |
2183 | netif_carrier_off(dev: netdev); |
2184 | netif_tx_stop_all_queues(dev: netdev); |
2185 | |
2186 | /* disable transmits in the hardware */ |
2187 | tctl = rd32(E1000_TCTL); |
2188 | tctl &= ~E1000_TCTL_EN; |
2189 | wr32(E1000_TCTL, tctl); |
2190 | /* flush both disables and wait for them to finish */ |
2191 | wrfl(); |
2192 | usleep_range(min: 10000, max: 11000); |
2193 | |
2194 | igb_irq_disable(adapter); |
2195 | |
2196 | adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE; |
2197 | |
2198 | for (i = 0; i < adapter->num_q_vectors; i++) { |
2199 | if (adapter->q_vector[i]) { |
2200 | napi_synchronize(n: &adapter->q_vector[i]->napi); |
2201 | napi_disable(n: &adapter->q_vector[i]->napi); |
2202 | } |
2203 | } |
2204 | |
2205 | del_timer_sync(timer: &adapter->watchdog_timer); |
2206 | del_timer_sync(timer: &adapter->phy_info_timer); |
2207 | |
2208 | /* record the stats before reset*/ |
2209 | spin_lock(lock: &adapter->stats64_lock); |
2210 | igb_update_stats(adapter); |
2211 | spin_unlock(lock: &adapter->stats64_lock); |
2212 | |
2213 | adapter->link_speed = 0; |
2214 | adapter->link_duplex = 0; |
2215 | |
2216 | if (!pci_channel_offline(pdev: adapter->pdev)) |
2217 | igb_reset(adapter); |
2218 | |
2219 | /* clear VLAN promisc flag so VFTA will be updated if necessary */ |
2220 | adapter->flags &= ~IGB_FLAG_VLAN_PROMISC; |
2221 | |
2222 | igb_clean_all_tx_rings(adapter); |
2223 | igb_clean_all_rx_rings(adapter); |
2224 | #ifdef CONFIG_IGB_DCA |
2225 | |
2226 | /* since we reset the hardware DCA settings were cleared */ |
2227 | igb_setup_dca(adapter); |
2228 | #endif |
2229 | } |
2230 | |
2231 | void igb_reinit_locked(struct igb_adapter *adapter) |
2232 | { |
2233 | while (test_and_set_bit(nr: __IGB_RESETTING, addr: &adapter->state)) |
2234 | usleep_range(min: 1000, max: 2000); |
2235 | igb_down(adapter); |
2236 | igb_up(adapter); |
2237 | clear_bit(nr: __IGB_RESETTING, addr: &adapter->state); |
2238 | } |
2239 | |
2240 | /** igb_enable_mas - Media Autosense re-enable after swap |
2241 | * |
2242 | * @adapter: adapter struct |
2243 | **/ |
2244 | static void igb_enable_mas(struct igb_adapter *adapter) |
2245 | { |
2246 | struct e1000_hw *hw = &adapter->hw; |
2247 | u32 connsw = rd32(E1000_CONNSW); |
2248 | |
2249 | /* configure for SerDes media detect */ |
2250 | if ((hw->phy.media_type == e1000_media_type_copper) && |
2251 | (!(connsw & E1000_CONNSW_SERDESD))) { |
2252 | connsw |= E1000_CONNSW_ENRGSRC; |
2253 | connsw |= E1000_CONNSW_AUTOSENSE_EN; |
2254 | wr32(E1000_CONNSW, connsw); |
2255 | wrfl(); |
2256 | } |
2257 | } |
2258 | |
2259 | #ifdef CONFIG_IGB_HWMON |
2260 | /** |
2261 | * igb_set_i2c_bb - Init I2C interface |
2262 | * @hw: pointer to hardware structure |
2263 | **/ |
2264 | static void igb_set_i2c_bb(struct e1000_hw *hw) |
2265 | { |
2266 | u32 ctrl_ext; |
2267 | s32 i2cctl; |
2268 | |
2269 | ctrl_ext = rd32(E1000_CTRL_EXT); |
2270 | ctrl_ext |= E1000_CTRL_I2C_ENA; |
2271 | wr32(E1000_CTRL_EXT, ctrl_ext); |
2272 | wrfl(); |
2273 | |
2274 | i2cctl = rd32(E1000_I2CPARAMS); |
2275 | i2cctl |= E1000_I2CBB_EN |
2276 | | E1000_I2C_CLK_OE_N |
2277 | | E1000_I2C_DATA_OE_N; |
2278 | wr32(E1000_I2CPARAMS, i2cctl); |
2279 | wrfl(); |
2280 | } |
2281 | #endif |
2282 | |
2283 | void igb_reset(struct igb_adapter *adapter) |
2284 | { |
2285 | struct pci_dev *pdev = adapter->pdev; |
2286 | struct e1000_hw *hw = &adapter->hw; |
2287 | struct e1000_mac_info *mac = &hw->mac; |
2288 | struct e1000_fc_info *fc = &hw->fc; |
2289 | u32 pba, hwm; |
2290 | |
2291 | /* Repartition Pba for greater than 9k mtu |
2292 | * To take effect CTRL.RST is required. |
2293 | */ |
2294 | switch (mac->type) { |
2295 | case e1000_i350: |
2296 | case e1000_i354: |
2297 | case e1000_82580: |
2298 | pba = rd32(E1000_RXPBS); |
2299 | pba = igb_rxpbs_adjust_82580(data: pba); |
2300 | break; |
2301 | case e1000_82576: |
2302 | pba = rd32(E1000_RXPBS); |
2303 | pba &= E1000_RXPBS_SIZE_MASK_82576; |
2304 | break; |
2305 | case e1000_82575: |
2306 | case e1000_i210: |
2307 | case e1000_i211: |
2308 | default: |
2309 | pba = E1000_PBA_34K; |
2310 | break; |
2311 | } |
2312 | |
2313 | if (mac->type == e1000_82575) { |
2314 | u32 min_rx_space, min_tx_space, needed_tx_space; |
2315 | |
2316 | /* write Rx PBA so that hardware can report correct Tx PBA */ |
2317 | wr32(E1000_PBA, pba); |
2318 | |
2319 | /* To maintain wire speed transmits, the Tx FIFO should be |
2320 | * large enough to accommodate two full transmit packets, |
2321 | * rounded up to the next 1KB and expressed in KB. Likewise, |
2322 | * the Rx FIFO should be large enough to accommodate at least |
2323 | * one full receive packet and is similarly rounded up and |
2324 | * expressed in KB. |
2325 | */ |
2326 | min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024); |
2327 | |
2328 | /* The Tx FIFO also stores 16 bytes of information about the Tx |
2329 | * but don't include Ethernet FCS because hardware appends it. |
2330 | * We only need to round down to the nearest 512 byte block |
2331 | * count since the value we care about is 2 frames, not 1. |
2332 | */ |
2333 | min_tx_space = adapter->max_frame_size; |
2334 | min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN; |
2335 | min_tx_space = DIV_ROUND_UP(min_tx_space, 512); |
2336 | |
2337 | /* upper 16 bits has Tx packet buffer allocation size in KB */ |
2338 | needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16); |
2339 | |
2340 | /* If current Tx allocation is less than the min Tx FIFO size, |
2341 | * and the min Tx FIFO size is less than the current Rx FIFO |
2342 | * allocation, take space away from current Rx allocation. |
2343 | */ |
2344 | if (needed_tx_space < pba) { |
2345 | pba -= needed_tx_space; |
2346 | |
2347 | /* if short on Rx space, Rx wins and must trump Tx |
2348 | * adjustment |
2349 | */ |
2350 | if (pba < min_rx_space) |
2351 | pba = min_rx_space; |
2352 | } |
2353 | |
2354 | /* adjust PBA for jumbo frames */ |
2355 | wr32(E1000_PBA, pba); |
2356 | } |
2357 | |
2358 | /* flow control settings |
2359 | * The high water mark must be low enough to fit one full frame |
2360 | * after transmitting the pause frame. As such we must have enough |
2361 | * space to allow for us to complete our current transmit and then |
2362 | * receive the frame that is in progress from the link partner. |
2363 | * Set it to: |
2364 | * - the full Rx FIFO size minus one full Tx plus one full Rx frame |
2365 | */ |
2366 | hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE); |
2367 | |
2368 | fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */ |
2369 | fc->low_water = fc->high_water - 16; |
2370 | fc->pause_time = 0xFFFF; |
2371 | fc->send_xon = 1; |
2372 | fc->current_mode = fc->requested_mode; |
2373 | |
2374 | /* disable receive for all VFs and wait one second */ |
2375 | if (adapter->vfs_allocated_count) { |
2376 | int i; |
2377 | |
2378 | for (i = 0 ; i < adapter->vfs_allocated_count; i++) |
2379 | adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC; |
2380 | |
2381 | /* ping all the active vfs to let them know we are going down */ |
2382 | igb_ping_all_vfs(adapter); |
2383 | |
2384 | /* disable transmits and receives */ |
2385 | wr32(E1000_VFRE, 0); |
2386 | wr32(E1000_VFTE, 0); |
2387 | } |
2388 | |
2389 | /* Allow time for pending master requests to run */ |
2390 | hw->mac.ops.reset_hw(hw); |
2391 | wr32(E1000_WUC, 0); |
2392 | |
2393 | if (adapter->flags & IGB_FLAG_MEDIA_RESET) { |
2394 | /* need to resetup here after media swap */ |
2395 | adapter->ei.get_invariants(hw); |
2396 | adapter->flags &= ~IGB_FLAG_MEDIA_RESET; |
2397 | } |
2398 | if ((mac->type == e1000_82575 || mac->type == e1000_i350) && |
2399 | (adapter->flags & IGB_FLAG_MAS_ENABLE)) { |
2400 | igb_enable_mas(adapter); |
2401 | } |
2402 | if (hw->mac.ops.init_hw(hw)) |
2403 | dev_err(&pdev->dev, "Hardware Error\n" ); |
2404 | |
2405 | /* RAR registers were cleared during init_hw, clear mac table */ |
2406 | igb_flush_mac_table(adapter); |
2407 | __dev_uc_unsync(dev: adapter->netdev, NULL); |
2408 | |
2409 | /* Recover default RAR entry */ |
2410 | igb_set_default_mac_filter(adapter); |
2411 | |
2412 | /* Flow control settings reset on hardware reset, so guarantee flow |
2413 | * control is off when forcing speed. |
2414 | */ |
2415 | if (!hw->mac.autoneg) |
2416 | igb_force_mac_fc(hw); |
2417 | |
2418 | igb_init_dmac(adapter, pba); |
2419 | #ifdef CONFIG_IGB_HWMON |
2420 | /* Re-initialize the thermal sensor on i350 devices. */ |
2421 | if (!test_bit(__IGB_DOWN, &adapter->state)) { |
2422 | if (mac->type == e1000_i350 && hw->bus.func == 0) { |
2423 | /* If present, re-initialize the external thermal sensor |
2424 | * interface. |
2425 | */ |
2426 | if (adapter->ets) |
2427 | igb_set_i2c_bb(hw); |
2428 | mac->ops.init_thermal_sensor_thresh(hw); |
2429 | } |
2430 | } |
2431 | #endif |
2432 | /* Re-establish EEE setting */ |
2433 | if (hw->phy.media_type == e1000_media_type_copper) { |
2434 | switch (mac->type) { |
2435 | case e1000_i350: |
2436 | case e1000_i210: |
2437 | case e1000_i211: |
2438 | igb_set_eee_i350(hw, adv1G: true, adv100M: true); |
2439 | break; |
2440 | case e1000_i354: |
2441 | igb_set_eee_i354(hw, adv1G: true, adv100M: true); |
2442 | break; |
2443 | default: |
2444 | break; |
2445 | } |
2446 | } |
2447 | if (!netif_running(dev: adapter->netdev)) |
2448 | igb_power_down_link(adapter); |
2449 | |
2450 | igb_update_mng_vlan(adapter); |
2451 | |
2452 | /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */ |
2453 | wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE); |
2454 | |
2455 | /* Re-enable PTP, where applicable. */ |
2456 | if (adapter->ptp_flags & IGB_PTP_ENABLED) |
2457 | igb_ptp_reset(adapter); |
2458 | |
2459 | igb_get_phy_info(hw); |
2460 | } |
2461 | |
2462 | static netdev_features_t igb_fix_features(struct net_device *netdev, |
2463 | netdev_features_t features) |
2464 | { |
2465 | /* Since there is no support for separate Rx/Tx vlan accel |
2466 | * enable/disable make sure Tx flag is always in same state as Rx. |
2467 | */ |
2468 | if (features & NETIF_F_HW_VLAN_CTAG_RX) |
2469 | features |= NETIF_F_HW_VLAN_CTAG_TX; |
2470 | else |
2471 | features &= ~NETIF_F_HW_VLAN_CTAG_TX; |
2472 | |
2473 | return features; |
2474 | } |
2475 | |
2476 | static int igb_set_features(struct net_device *netdev, |
2477 | netdev_features_t features) |
2478 | { |
2479 | netdev_features_t changed = netdev->features ^ features; |
2480 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
2481 | |
2482 | if (changed & NETIF_F_HW_VLAN_CTAG_RX) |
2483 | igb_vlan_mode(netdev, features); |
2484 | |
2485 | if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) |
2486 | return 0; |
2487 | |
2488 | if (!(features & NETIF_F_NTUPLE)) { |
2489 | struct hlist_node *node2; |
2490 | struct igb_nfc_filter *rule; |
2491 | |
2492 | spin_lock(lock: &adapter->nfc_lock); |
2493 | hlist_for_each_entry_safe(rule, node2, |
2494 | &adapter->nfc_filter_list, nfc_node) { |
2495 | igb_erase_filter(adapter, input: rule); |
2496 | hlist_del(n: &rule->nfc_node); |
2497 | kfree(objp: rule); |
2498 | } |
2499 | spin_unlock(lock: &adapter->nfc_lock); |
2500 | adapter->nfc_filter_count = 0; |
2501 | } |
2502 | |
2503 | netdev->features = features; |
2504 | |
2505 | if (netif_running(dev: netdev)) |
2506 | igb_reinit_locked(adapter); |
2507 | else |
2508 | igb_reset(adapter); |
2509 | |
2510 | return 1; |
2511 | } |
2512 | |
2513 | static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], |
2514 | struct net_device *dev, |
2515 | const unsigned char *addr, u16 vid, |
2516 | u16 flags, |
2517 | struct netlink_ext_ack *extack) |
2518 | { |
2519 | /* guarantee we can provide a unique filter for the unicast address */ |
2520 | if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) { |
2521 | struct igb_adapter *adapter = netdev_priv(dev); |
2522 | int vfn = adapter->vfs_allocated_count; |
2523 | |
2524 | if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn)) |
2525 | return -ENOMEM; |
2526 | } |
2527 | |
2528 | return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags); |
2529 | } |
2530 | |
2531 | #define IGB_MAX_MAC_HDR_LEN 127 |
2532 | #define IGB_MAX_NETWORK_HDR_LEN 511 |
2533 | |
2534 | static netdev_features_t |
2535 | igb_features_check(struct sk_buff *skb, struct net_device *dev, |
2536 | netdev_features_t features) |
2537 | { |
2538 | unsigned int network_hdr_len, mac_hdr_len; |
2539 | |
2540 | /* Make certain the headers can be described by a context descriptor */ |
2541 | mac_hdr_len = skb_network_header(skb) - skb->data; |
2542 | if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN)) |
2543 | return features & ~(NETIF_F_HW_CSUM | |
2544 | NETIF_F_SCTP_CRC | |
2545 | NETIF_F_GSO_UDP_L4 | |
2546 | NETIF_F_HW_VLAN_CTAG_TX | |
2547 | NETIF_F_TSO | |
2548 | NETIF_F_TSO6); |
2549 | |
2550 | network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); |
2551 | if (unlikely(network_hdr_len > IGB_MAX_NETWORK_HDR_LEN)) |
2552 | return features & ~(NETIF_F_HW_CSUM | |
2553 | NETIF_F_SCTP_CRC | |
2554 | NETIF_F_GSO_UDP_L4 | |
2555 | NETIF_F_TSO | |
2556 | NETIF_F_TSO6); |
2557 | |
2558 | /* We can only support IPV4 TSO in tunnels if we can mangle the |
2559 | * inner IP ID field, so strip TSO if MANGLEID is not supported. |
2560 | */ |
2561 | if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) |
2562 | features &= ~NETIF_F_TSO; |
2563 | |
2564 | return features; |
2565 | } |
2566 | |
2567 | static void igb_offload_apply(struct igb_adapter *adapter, s32 queue) |
2568 | { |
2569 | if (!is_fqtss_enabled(adapter)) { |
2570 | enable_fqtss(adapter, enable: true); |
2571 | return; |
2572 | } |
2573 | |
2574 | igb_config_tx_modes(adapter, queue); |
2575 | |
2576 | if (!is_any_cbs_enabled(adapter) && !is_any_txtime_enabled(adapter)) |
2577 | enable_fqtss(adapter, enable: false); |
2578 | } |
2579 | |
2580 | static int igb_offload_cbs(struct igb_adapter *adapter, |
2581 | struct tc_cbs_qopt_offload *qopt) |
2582 | { |
2583 | struct e1000_hw *hw = &adapter->hw; |
2584 | int err; |
2585 | |
2586 | /* CBS offloading is only supported by i210 controller. */ |
2587 | if (hw->mac.type != e1000_i210) |
2588 | return -EOPNOTSUPP; |
2589 | |
2590 | /* CBS offloading is only supported by queue 0 and queue 1. */ |
2591 | if (qopt->queue < 0 || qopt->queue > 1) |
2592 | return -EINVAL; |
2593 | |
2594 | err = igb_save_cbs_params(adapter, queue: qopt->queue, enable: qopt->enable, |
2595 | idleslope: qopt->idleslope, sendslope: qopt->sendslope, |
2596 | hicredit: qopt->hicredit, locredit: qopt->locredit); |
2597 | if (err) |
2598 | return err; |
2599 | |
2600 | igb_offload_apply(adapter, queue: qopt->queue); |
2601 | |
2602 | return 0; |
2603 | } |
2604 | |
2605 | #define ETHER_TYPE_FULL_MASK ((__force __be16)~0) |
2606 | #define VLAN_PRIO_FULL_MASK (0x07) |
2607 | |
2608 | static int igb_parse_cls_flower(struct igb_adapter *adapter, |
2609 | struct flow_cls_offload *f, |
2610 | int traffic_class, |
2611 | struct igb_nfc_filter *input) |
2612 | { |
2613 | struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f); |
2614 | struct flow_dissector *dissector = rule->match.dissector; |
2615 | struct netlink_ext_ack *extack = f->common.extack; |
2616 | |
2617 | if (dissector->used_keys & |
2618 | ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | |
2619 | BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | |
2620 | BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | |
2621 | BIT_ULL(FLOW_DISSECTOR_KEY_VLAN))) { |
2622 | NL_SET_ERR_MSG_MOD(extack, |
2623 | "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported" ); |
2624 | return -EOPNOTSUPP; |
2625 | } |
2626 | |
2627 | if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ETH_ADDRS)) { |
2628 | struct flow_match_eth_addrs match; |
2629 | |
2630 | flow_rule_match_eth_addrs(rule, out: &match); |
2631 | if (!is_zero_ether_addr(addr: match.mask->dst)) { |
2632 | if (!is_broadcast_ether_addr(addr: match.mask->dst)) { |
2633 | NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address" ); |
2634 | return -EINVAL; |
2635 | } |
2636 | |
2637 | input->filter.match_flags |= |
2638 | IGB_FILTER_FLAG_DST_MAC_ADDR; |
2639 | ether_addr_copy(dst: input->filter.dst_addr, src: match.key->dst); |
2640 | } |
2641 | |
2642 | if (!is_zero_ether_addr(addr: match.mask->src)) { |
2643 | if (!is_broadcast_ether_addr(addr: match.mask->src)) { |
2644 | NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address" ); |
2645 | return -EINVAL; |
2646 | } |
2647 | |
2648 | input->filter.match_flags |= |
2649 | IGB_FILTER_FLAG_SRC_MAC_ADDR; |
2650 | ether_addr_copy(dst: input->filter.src_addr, src: match.key->src); |
2651 | } |
2652 | } |
2653 | |
2654 | if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_BASIC)) { |
2655 | struct flow_match_basic match; |
2656 | |
2657 | flow_rule_match_basic(rule, out: &match); |
2658 | if (match.mask->n_proto) { |
2659 | if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) { |
2660 | NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter" ); |
2661 | return -EINVAL; |
2662 | } |
2663 | |
2664 | input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE; |
2665 | input->filter.etype = match.key->n_proto; |
2666 | } |
2667 | } |
2668 | |
2669 | if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_VLAN)) { |
2670 | struct flow_match_vlan match; |
2671 | |
2672 | flow_rule_match_vlan(rule, out: &match); |
2673 | if (match.mask->vlan_priority) { |
2674 | if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) { |
2675 | NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority" ); |
2676 | return -EINVAL; |
2677 | } |
2678 | |
2679 | input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI; |
2680 | input->filter.vlan_tci = |
2681 | (__force __be16)match.key->vlan_priority; |
2682 | } |
2683 | } |
2684 | |
2685 | input->action = traffic_class; |
2686 | input->cookie = f->cookie; |
2687 | |
2688 | return 0; |
2689 | } |
2690 | |
2691 | static int igb_configure_clsflower(struct igb_adapter *adapter, |
2692 | struct flow_cls_offload *cls_flower) |
2693 | { |
2694 | struct netlink_ext_ack *extack = cls_flower->common.extack; |
2695 | struct igb_nfc_filter *filter, *f; |
2696 | int err, tc; |
2697 | |
2698 | tc = tc_classid_to_hwtc(dev: adapter->netdev, classid: cls_flower->classid); |
2699 | if (tc < 0) { |
2700 | NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class" ); |
2701 | return -EINVAL; |
2702 | } |
2703 | |
2704 | filter = kzalloc(size: sizeof(*filter), GFP_KERNEL); |
2705 | if (!filter) |
2706 | return -ENOMEM; |
2707 | |
2708 | err = igb_parse_cls_flower(adapter, f: cls_flower, traffic_class: tc, input: filter); |
2709 | if (err < 0) |
2710 | goto err_parse; |
2711 | |
2712 | spin_lock(lock: &adapter->nfc_lock); |
2713 | |
2714 | hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) { |
2715 | if (!memcmp(p: &f->filter, q: &filter->filter, size: sizeof(f->filter))) { |
2716 | err = -EEXIST; |
2717 | NL_SET_ERR_MSG_MOD(extack, |
2718 | "This filter is already set in ethtool" ); |
2719 | goto err_locked; |
2720 | } |
2721 | } |
2722 | |
2723 | hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) { |
2724 | if (!memcmp(p: &f->filter, q: &filter->filter, size: sizeof(f->filter))) { |
2725 | err = -EEXIST; |
2726 | NL_SET_ERR_MSG_MOD(extack, |
2727 | "This filter is already set in cls_flower" ); |
2728 | goto err_locked; |
2729 | } |
2730 | } |
2731 | |
2732 | err = igb_add_filter(adapter, input: filter); |
2733 | if (err < 0) { |
2734 | NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter" ); |
2735 | goto err_locked; |
2736 | } |
2737 | |
2738 | hlist_add_head(n: &filter->nfc_node, h: &adapter->cls_flower_list); |
2739 | |
2740 | spin_unlock(lock: &adapter->nfc_lock); |
2741 | |
2742 | return 0; |
2743 | |
2744 | err_locked: |
2745 | spin_unlock(lock: &adapter->nfc_lock); |
2746 | |
2747 | err_parse: |
2748 | kfree(objp: filter); |
2749 | |
2750 | return err; |
2751 | } |
2752 | |
2753 | static int igb_delete_clsflower(struct igb_adapter *adapter, |
2754 | struct flow_cls_offload *cls_flower) |
2755 | { |
2756 | struct igb_nfc_filter *filter; |
2757 | int err; |
2758 | |
2759 | spin_lock(lock: &adapter->nfc_lock); |
2760 | |
2761 | hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node) |
2762 | if (filter->cookie == cls_flower->cookie) |
2763 | break; |
2764 | |
2765 | if (!filter) { |
2766 | err = -ENOENT; |
2767 | goto out; |
2768 | } |
2769 | |
2770 | err = igb_erase_filter(adapter, input: filter); |
2771 | if (err < 0) |
2772 | goto out; |
2773 | |
2774 | hlist_del(n: &filter->nfc_node); |
2775 | kfree(objp: filter); |
2776 | |
2777 | out: |
2778 | spin_unlock(lock: &adapter->nfc_lock); |
2779 | |
2780 | return err; |
2781 | } |
2782 | |
2783 | static int igb_setup_tc_cls_flower(struct igb_adapter *adapter, |
2784 | struct flow_cls_offload *cls_flower) |
2785 | { |
2786 | switch (cls_flower->command) { |
2787 | case FLOW_CLS_REPLACE: |
2788 | return igb_configure_clsflower(adapter, cls_flower); |
2789 | case FLOW_CLS_DESTROY: |
2790 | return igb_delete_clsflower(adapter, cls_flower); |
2791 | case FLOW_CLS_STATS: |
2792 | return -EOPNOTSUPP; |
2793 | default: |
2794 | return -EOPNOTSUPP; |
2795 | } |
2796 | } |
2797 | |
2798 | static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data, |
2799 | void *cb_priv) |
2800 | { |
2801 | struct igb_adapter *adapter = cb_priv; |
2802 | |
2803 | if (!tc_cls_can_offload_and_chain0(dev: adapter->netdev, common: type_data)) |
2804 | return -EOPNOTSUPP; |
2805 | |
2806 | switch (type) { |
2807 | case TC_SETUP_CLSFLOWER: |
2808 | return igb_setup_tc_cls_flower(adapter, cls_flower: type_data); |
2809 | |
2810 | default: |
2811 | return -EOPNOTSUPP; |
2812 | } |
2813 | } |
2814 | |
2815 | static int igb_offload_txtime(struct igb_adapter *adapter, |
2816 | struct tc_etf_qopt_offload *qopt) |
2817 | { |
2818 | struct e1000_hw *hw = &adapter->hw; |
2819 | int err; |
2820 | |
2821 | /* Launchtime offloading is only supported by i210 controller. */ |
2822 | if (hw->mac.type != e1000_i210) |
2823 | return -EOPNOTSUPP; |
2824 | |
2825 | /* Launchtime offloading is only supported by queues 0 and 1. */ |
2826 | if (qopt->queue < 0 || qopt->queue > 1) |
2827 | return -EINVAL; |
2828 | |
2829 | err = igb_save_txtime_params(adapter, queue: qopt->queue, enable: qopt->enable); |
2830 | if (err) |
2831 | return err; |
2832 | |
2833 | igb_offload_apply(adapter, queue: qopt->queue); |
2834 | |
2835 | return 0; |
2836 | } |
2837 | |
2838 | static int igb_tc_query_caps(struct igb_adapter *adapter, |
2839 | struct tc_query_caps_base *base) |
2840 | { |
2841 | switch (base->type) { |
2842 | case TC_SETUP_QDISC_TAPRIO: { |
2843 | struct tc_taprio_caps *caps = base->caps; |
2844 | |
2845 | caps->broken_mqprio = true; |
2846 | |
2847 | return 0; |
2848 | } |
2849 | default: |
2850 | return -EOPNOTSUPP; |
2851 | } |
2852 | } |
2853 | |
2854 | static LIST_HEAD(igb_block_cb_list); |
2855 | |
2856 | static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type, |
2857 | void *type_data) |
2858 | { |
2859 | struct igb_adapter *adapter = netdev_priv(dev); |
2860 | |
2861 | switch (type) { |
2862 | case TC_QUERY_CAPS: |
2863 | return igb_tc_query_caps(adapter, base: type_data); |
2864 | case TC_SETUP_QDISC_CBS: |
2865 | return igb_offload_cbs(adapter, qopt: type_data); |
2866 | case TC_SETUP_BLOCK: |
2867 | return flow_block_cb_setup_simple(f: type_data, |
2868 | driver_list: &igb_block_cb_list, |
2869 | cb: igb_setup_tc_block_cb, |
2870 | cb_ident: adapter, cb_priv: adapter, ingress_only: true); |
2871 | |
2872 | case TC_SETUP_QDISC_ETF: |
2873 | return igb_offload_txtime(adapter, qopt: type_data); |
2874 | |
2875 | default: |
2876 | return -EOPNOTSUPP; |
2877 | } |
2878 | } |
2879 | |
2880 | static int igb_xdp_setup(struct net_device *dev, struct netdev_bpf *bpf) |
2881 | { |
2882 | int i, frame_size = dev->mtu + IGB_ETH_PKT_HDR_PAD; |
2883 | struct igb_adapter *adapter = netdev_priv(dev); |
2884 | struct bpf_prog *prog = bpf->prog, *old_prog; |
2885 | bool running = netif_running(dev); |
2886 | bool need_reset; |
2887 | |
2888 | /* verify igb ring attributes are sufficient for XDP */ |
2889 | for (i = 0; i < adapter->num_rx_queues; i++) { |
2890 | struct igb_ring *ring = adapter->rx_ring[i]; |
2891 | |
2892 | if (frame_size > igb_rx_bufsz(ring)) { |
2893 | NL_SET_ERR_MSG_MOD(bpf->extack, |
2894 | "The RX buffer size is too small for the frame size" ); |
2895 | netdev_warn(dev, format: "XDP RX buffer size %d is too small for the frame size %d\n" , |
2896 | igb_rx_bufsz(ring), frame_size); |
2897 | return -EINVAL; |
2898 | } |
2899 | } |
2900 | |
2901 | old_prog = xchg(&adapter->xdp_prog, prog); |
2902 | need_reset = (!!prog != !!old_prog); |
2903 | |
2904 | /* device is up and bpf is added/removed, must setup the RX queues */ |
2905 | if (need_reset && running) { |
2906 | igb_close(dev); |
2907 | } else { |
2908 | for (i = 0; i < adapter->num_rx_queues; i++) |
2909 | (void)xchg(&adapter->rx_ring[i]->xdp_prog, |
2910 | adapter->xdp_prog); |
2911 | } |
2912 | |
2913 | if (old_prog) |
2914 | bpf_prog_put(prog: old_prog); |
2915 | |
2916 | /* bpf is just replaced, RXQ and MTU are already setup */ |
2917 | if (!need_reset) { |
2918 | return 0; |
2919 | } else { |
2920 | if (prog) |
2921 | xdp_features_set_redirect_target(dev, support_sg: true); |
2922 | else |
2923 | xdp_features_clear_redirect_target(dev); |
2924 | } |
2925 | |
2926 | if (running) |
2927 | igb_open(dev); |
2928 | |
2929 | return 0; |
2930 | } |
2931 | |
2932 | static int igb_xdp(struct net_device *dev, struct netdev_bpf *xdp) |
2933 | { |
2934 | switch (xdp->command) { |
2935 | case XDP_SETUP_PROG: |
2936 | return igb_xdp_setup(dev, bpf: xdp); |
2937 | default: |
2938 | return -EINVAL; |
2939 | } |
2940 | } |
2941 | |
2942 | static void igb_xdp_ring_update_tail(struct igb_ring *ring) |
2943 | { |
2944 | /* Force memory writes to complete before letting h/w know there |
2945 | * are new descriptors to fetch. |
2946 | */ |
2947 | wmb(); |
2948 | writel(val: ring->next_to_use, addr: ring->tail); |
2949 | } |
2950 | |
2951 | static struct igb_ring *igb_xdp_tx_queue_mapping(struct igb_adapter *adapter) |
2952 | { |
2953 | unsigned int r_idx = smp_processor_id(); |
2954 | |
2955 | if (r_idx >= adapter->num_tx_queues) |
2956 | r_idx = r_idx % adapter->num_tx_queues; |
2957 | |
2958 | return adapter->tx_ring[r_idx]; |
2959 | } |
2960 | |
2961 | static int igb_xdp_xmit_back(struct igb_adapter *adapter, struct xdp_buff *xdp) |
2962 | { |
2963 | struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); |
2964 | int cpu = smp_processor_id(); |
2965 | struct igb_ring *tx_ring; |
2966 | struct netdev_queue *nq; |
2967 | u32 ret; |
2968 | |
2969 | if (unlikely(!xdpf)) |
2970 | return IGB_XDP_CONSUMED; |
2971 | |
2972 | /* During program transitions its possible adapter->xdp_prog is assigned |
2973 | * but ring has not been configured yet. In this case simply abort xmit. |
2974 | */ |
2975 | tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL; |
2976 | if (unlikely(!tx_ring)) |
2977 | return IGB_XDP_CONSUMED; |
2978 | |
2979 | nq = txring_txq(tx_ring); |
2980 | __netif_tx_lock(txq: nq, cpu); |
2981 | /* Avoid transmit queue timeout since we share it with the slow path */ |
2982 | txq_trans_cond_update(txq: nq); |
2983 | ret = igb_xmit_xdp_ring(adapter, ring: tx_ring, xdpf); |
2984 | __netif_tx_unlock(txq: nq); |
2985 | |
2986 | return ret; |
2987 | } |
2988 | |
2989 | static int igb_xdp_xmit(struct net_device *dev, int n, |
2990 | struct xdp_frame **frames, u32 flags) |
2991 | { |
2992 | struct igb_adapter *adapter = netdev_priv(dev); |
2993 | int cpu = smp_processor_id(); |
2994 | struct igb_ring *tx_ring; |
2995 | struct netdev_queue *nq; |
2996 | int nxmit = 0; |
2997 | int i; |
2998 | |
2999 | if (unlikely(test_bit(__IGB_DOWN, &adapter->state))) |
3000 | return -ENETDOWN; |
3001 | |
3002 | if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) |
3003 | return -EINVAL; |
3004 | |
3005 | /* During program transitions its possible adapter->xdp_prog is assigned |
3006 | * but ring has not been configured yet. In this case simply abort xmit. |
3007 | */ |
3008 | tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL; |
3009 | if (unlikely(!tx_ring)) |
3010 | return -ENXIO; |
3011 | |
3012 | nq = txring_txq(tx_ring); |
3013 | __netif_tx_lock(txq: nq, cpu); |
3014 | |
3015 | /* Avoid transmit queue timeout since we share it with the slow path */ |
3016 | txq_trans_cond_update(txq: nq); |
3017 | |
3018 | for (i = 0; i < n; i++) { |
3019 | struct xdp_frame *xdpf = frames[i]; |
3020 | int err; |
3021 | |
3022 | err = igb_xmit_xdp_ring(adapter, ring: tx_ring, xdpf); |
3023 | if (err != IGB_XDP_TX) |
3024 | break; |
3025 | nxmit++; |
3026 | } |
3027 | |
3028 | __netif_tx_unlock(txq: nq); |
3029 | |
3030 | if (unlikely(flags & XDP_XMIT_FLUSH)) |
3031 | igb_xdp_ring_update_tail(ring: tx_ring); |
3032 | |
3033 | return nxmit; |
3034 | } |
3035 | |
3036 | static const struct net_device_ops igb_netdev_ops = { |
3037 | .ndo_open = igb_open, |
3038 | .ndo_stop = igb_close, |
3039 | .ndo_start_xmit = igb_xmit_frame, |
3040 | .ndo_get_stats64 = igb_get_stats64, |
3041 | .ndo_set_rx_mode = igb_set_rx_mode, |
3042 | .ndo_set_mac_address = igb_set_mac, |
3043 | .ndo_change_mtu = igb_change_mtu, |
3044 | .ndo_eth_ioctl = igb_ioctl, |
3045 | .ndo_tx_timeout = igb_tx_timeout, |
3046 | .ndo_validate_addr = eth_validate_addr, |
3047 | .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid, |
3048 | .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid, |
3049 | .ndo_set_vf_mac = igb_ndo_set_vf_mac, |
3050 | .ndo_set_vf_vlan = igb_ndo_set_vf_vlan, |
3051 | .ndo_set_vf_rate = igb_ndo_set_vf_bw, |
3052 | .ndo_set_vf_spoofchk = igb_ndo_set_vf_spoofchk, |
3053 | .ndo_set_vf_trust = igb_ndo_set_vf_trust, |
3054 | .ndo_get_vf_config = igb_ndo_get_vf_config, |
3055 | .ndo_fix_features = igb_fix_features, |
3056 | .ndo_set_features = igb_set_features, |
3057 | .ndo_fdb_add = igb_ndo_fdb_add, |
3058 | .ndo_features_check = igb_features_check, |
3059 | .ndo_setup_tc = igb_setup_tc, |
3060 | .ndo_bpf = igb_xdp, |
3061 | .ndo_xdp_xmit = igb_xdp_xmit, |
3062 | }; |
3063 | |
3064 | /** |
3065 | * igb_set_fw_version - Configure version string for ethtool |
3066 | * @adapter: adapter struct |
3067 | **/ |
3068 | void igb_set_fw_version(struct igb_adapter *adapter) |
3069 | { |
3070 | struct e1000_hw *hw = &adapter->hw; |
3071 | struct e1000_fw_version fw; |
3072 | char *lbuf; |
3073 | |
3074 | igb_get_fw_version(hw, fw_vers: &fw); |
3075 | |
3076 | switch (hw->mac.type) { |
3077 | case e1000_i210: |
3078 | case e1000_i211: |
3079 | if (!(igb_get_flash_presence_i210(hw))) { |
3080 | lbuf = kasprintf(GFP_KERNEL, fmt: "%2d.%2d-%d" , |
3081 | fw.invm_major, fw.invm_minor, |
3082 | fw.invm_img_type); |
3083 | break; |
3084 | } |
3085 | fallthrough; |
3086 | default: |
3087 | /* if option rom is valid, display its version too */ |
3088 | if (fw.or_valid) { |
3089 | lbuf = kasprintf(GFP_KERNEL, fmt: "%d.%d, 0x%08x, %d.%d.%d" , |
3090 | fw.eep_major, fw.eep_minor, |
3091 | fw.etrack_id, fw.or_major, fw.or_build, |
3092 | fw.or_patch); |
3093 | /* no option rom */ |
3094 | } else if (fw.etrack_id != 0X0000) { |
3095 | lbuf = kasprintf(GFP_KERNEL, fmt: "%d.%d, 0x%08x" , |
3096 | fw.eep_major, fw.eep_minor, |
3097 | fw.etrack_id); |
3098 | } else { |
3099 | lbuf = kasprintf(GFP_KERNEL, fmt: "%d.%d.%d" , fw.eep_major, |
3100 | fw.eep_minor, fw.eep_build); |
3101 | } |
3102 | break; |
3103 | } |
3104 | |
3105 | /* the truncate happens here if it doesn't fit */ |
3106 | strscpy(p: adapter->fw_version, q: lbuf, size: sizeof(adapter->fw_version)); |
3107 | kfree(objp: lbuf); |
3108 | } |
3109 | |
3110 | /** |
3111 | * igb_init_mas - init Media Autosense feature if enabled in the NVM |
3112 | * |
3113 | * @adapter: adapter struct |
3114 | **/ |
3115 | static void igb_init_mas(struct igb_adapter *adapter) |
3116 | { |
3117 | struct e1000_hw *hw = &adapter->hw; |
3118 | u16 eeprom_data; |
3119 | |
3120 | hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data); |
3121 | switch (hw->bus.func) { |
3122 | case E1000_FUNC_0: |
3123 | if (eeprom_data & IGB_MAS_ENABLE_0) { |
3124 | adapter->flags |= IGB_FLAG_MAS_ENABLE; |
3125 | netdev_info(dev: adapter->netdev, |
3126 | format: "MAS: Enabling Media Autosense for port %d\n" , |
3127 | hw->bus.func); |
3128 | } |
3129 | break; |
3130 | case E1000_FUNC_1: |
3131 | if (eeprom_data & IGB_MAS_ENABLE_1) { |
3132 | adapter->flags |= IGB_FLAG_MAS_ENABLE; |
3133 | netdev_info(dev: adapter->netdev, |
3134 | format: "MAS: Enabling Media Autosense for port %d\n" , |
3135 | hw->bus.func); |
3136 | } |
3137 | break; |
3138 | case E1000_FUNC_2: |
3139 | if (eeprom_data & IGB_MAS_ENABLE_2) { |
3140 | adapter->flags |= IGB_FLAG_MAS_ENABLE; |
3141 | netdev_info(dev: adapter->netdev, |
3142 | format: "MAS: Enabling Media Autosense for port %d\n" , |
3143 | hw->bus.func); |
3144 | } |
3145 | break; |
3146 | case E1000_FUNC_3: |
3147 | if (eeprom_data & IGB_MAS_ENABLE_3) { |
3148 | adapter->flags |= IGB_FLAG_MAS_ENABLE; |
3149 | netdev_info(dev: adapter->netdev, |
3150 | format: "MAS: Enabling Media Autosense for port %d\n" , |
3151 | hw->bus.func); |
3152 | } |
3153 | break; |
3154 | default: |
3155 | /* Shouldn't get here */ |
3156 | netdev_err(dev: adapter->netdev, |
3157 | format: "MAS: Invalid port configuration, returning\n" ); |
3158 | break; |
3159 | } |
3160 | } |
3161 | |
3162 | /** |
3163 | * igb_init_i2c - Init I2C interface |
3164 | * @adapter: pointer to adapter structure |
3165 | **/ |
3166 | static s32 igb_init_i2c(struct igb_adapter *adapter) |
3167 | { |
3168 | s32 status = 0; |
3169 | |
3170 | /* I2C interface supported on i350 devices */ |
3171 | if (adapter->hw.mac.type != e1000_i350) |
3172 | return 0; |
3173 | |
3174 | /* Initialize the i2c bus which is controlled by the registers. |
3175 | * This bus will use the i2c_algo_bit structure that implements |
3176 | * the protocol through toggling of the 4 bits in the register. |
3177 | */ |
3178 | adapter->i2c_adap.owner = THIS_MODULE; |
3179 | adapter->i2c_algo = igb_i2c_algo; |
3180 | adapter->i2c_algo.data = adapter; |
3181 | adapter->i2c_adap.algo_data = &adapter->i2c_algo; |
3182 | adapter->i2c_adap.dev.parent = &adapter->pdev->dev; |
3183 | strscpy(p: adapter->i2c_adap.name, q: "igb BB" , |
3184 | size: sizeof(adapter->i2c_adap.name)); |
3185 | status = i2c_bit_add_bus(&adapter->i2c_adap); |
3186 | return status; |
3187 | } |
3188 | |
3189 | /** |
3190 | * igb_probe - Device Initialization Routine |
3191 | * @pdev: PCI device information struct |
3192 | * @ent: entry in igb_pci_tbl |
3193 | * |
3194 | * Returns 0 on success, negative on failure |
3195 | * |
3196 | * igb_probe initializes an adapter identified by a pci_dev structure. |
3197 | * The OS initialization, configuring of the adapter private structure, |
3198 | * and a hardware reset occur. |
3199 | **/ |
3200 | static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) |
3201 | { |
3202 | struct net_device *netdev; |
3203 | struct igb_adapter *adapter; |
3204 | struct e1000_hw *hw; |
3205 | u16 eeprom_data = 0; |
3206 | s32 ret_val; |
3207 | static int global_quad_port_a; /* global quad port a indication */ |
3208 | const struct e1000_info *ei = igb_info_tbl[ent->driver_data]; |
3209 | u8 part_str[E1000_PBANUM_LENGTH]; |
3210 | int err; |
3211 | |
3212 | /* Catch broken hardware that put the wrong VF device ID in |
3213 | * the PCIe SR-IOV capability. |
3214 | */ |
3215 | if (pdev->is_virtfn) { |
3216 | WARN(1, KERN_ERR "%s (%x:%x) should not be a VF!\n" , |
3217 | pci_name(pdev), pdev->vendor, pdev->device); |
3218 | return -EINVAL; |
3219 | } |
3220 | |
3221 | err = pci_enable_device_mem(dev: pdev); |
3222 | if (err) |
3223 | return err; |
3224 | |
3225 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64)); |
3226 | if (err) { |
3227 | dev_err(&pdev->dev, |
3228 | "No usable DMA configuration, aborting\n" ); |
3229 | goto err_dma; |
3230 | } |
3231 | |
3232 | err = pci_request_mem_regions(pdev, name: igb_driver_name); |
3233 | if (err) |
3234 | goto err_pci_reg; |
3235 | |
3236 | pci_set_master(dev: pdev); |
3237 | pci_save_state(dev: pdev); |
3238 | |
3239 | err = -ENOMEM; |
3240 | netdev = alloc_etherdev_mq(sizeof(struct igb_adapter), |
3241 | IGB_MAX_TX_QUEUES); |
3242 | if (!netdev) |
3243 | goto err_alloc_etherdev; |
3244 | |
3245 | SET_NETDEV_DEV(netdev, &pdev->dev); |
3246 | |
3247 | pci_set_drvdata(pdev, data: netdev); |
3248 | adapter = netdev_priv(dev: netdev); |
3249 | adapter->netdev = netdev; |
3250 | adapter->pdev = pdev; |
3251 | hw = &adapter->hw; |
3252 | hw->back = adapter; |
3253 | adapter->msg_enable = netif_msg_init(debug_value: debug, DEFAULT_MSG_ENABLE); |
3254 | |
3255 | err = -EIO; |
3256 | adapter->io_addr = pci_iomap(dev: pdev, bar: 0, max: 0); |
3257 | if (!adapter->io_addr) |
3258 | goto err_ioremap; |
3259 | /* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */ |
3260 | hw->hw_addr = adapter->io_addr; |
3261 | |
3262 | netdev->netdev_ops = &igb_netdev_ops; |
3263 | igb_set_ethtool_ops(netdev); |
3264 | netdev->watchdog_timeo = 5 * HZ; |
3265 | |
3266 | strscpy(p: netdev->name, q: pci_name(pdev), size: sizeof(netdev->name)); |
3267 | |
3268 | netdev->mem_start = pci_resource_start(pdev, 0); |
3269 | netdev->mem_end = pci_resource_end(pdev, 0); |
3270 | |
3271 | /* PCI config space info */ |
3272 | hw->vendor_id = pdev->vendor; |
3273 | hw->device_id = pdev->device; |
3274 | hw->revision_id = pdev->revision; |
3275 | hw->subsystem_vendor_id = pdev->subsystem_vendor; |
3276 | hw->subsystem_device_id = pdev->subsystem_device; |
3277 | |
3278 | /* Copy the default MAC, PHY and NVM function pointers */ |
3279 | memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops)); |
3280 | memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops)); |
3281 | memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops)); |
3282 | /* Initialize skew-specific constants */ |
3283 | err = ei->get_invariants(hw); |
3284 | if (err) |
3285 | goto err_sw_init; |
3286 | |
3287 | /* setup the private structure */ |
3288 | err = igb_sw_init(adapter); |
3289 | if (err) |
3290 | goto err_sw_init; |
3291 | |
3292 | igb_get_bus_info_pcie(hw); |
3293 | |
3294 | hw->phy.autoneg_wait_to_complete = false; |
3295 | |
3296 | /* Copper options */ |
3297 | if (hw->phy.media_type == e1000_media_type_copper) { |
3298 | hw->phy.mdix = AUTO_ALL_MODES; |
3299 | hw->phy.disable_polarity_correction = false; |
3300 | hw->phy.ms_type = e1000_ms_hw_default; |
3301 | } |
3302 | |
3303 | if (igb_check_reset_block(hw)) |
3304 | dev_info(&pdev->dev, |
3305 | "PHY reset is blocked due to SOL/IDER session.\n" ); |
3306 | |
3307 | /* features is initialized to 0 in allocation, it might have bits |
3308 | * set by igb_sw_init so we should use an or instead of an |
3309 | * assignment. |
3310 | */ |
3311 | netdev->features |= NETIF_F_SG | |
3312 | NETIF_F_TSO | |
3313 | NETIF_F_TSO6 | |
3314 | NETIF_F_RXHASH | |
3315 | NETIF_F_RXCSUM | |
3316 | NETIF_F_HW_CSUM; |
3317 | |
3318 | if (hw->mac.type >= e1000_82576) |
3319 | netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4; |
3320 | |
3321 | if (hw->mac.type >= e1000_i350) |
3322 | netdev->features |= NETIF_F_HW_TC; |
3323 | |
3324 | #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ |
3325 | NETIF_F_GSO_GRE_CSUM | \ |
3326 | NETIF_F_GSO_IPXIP4 | \ |
3327 | NETIF_F_GSO_IPXIP6 | \ |
3328 | NETIF_F_GSO_UDP_TUNNEL | \ |
3329 | NETIF_F_GSO_UDP_TUNNEL_CSUM) |
3330 | |
3331 | netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES; |
3332 | netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES; |
3333 | |
3334 | /* copy netdev features into list of user selectable features */ |
3335 | netdev->hw_features |= netdev->features | |
3336 | NETIF_F_HW_VLAN_CTAG_RX | |
3337 | NETIF_F_HW_VLAN_CTAG_TX | |
3338 | NETIF_F_RXALL; |
3339 | |
3340 | if (hw->mac.type >= e1000_i350) |
3341 | netdev->hw_features |= NETIF_F_NTUPLE; |
3342 | |
3343 | netdev->features |= NETIF_F_HIGHDMA; |
3344 | |
3345 | netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; |
3346 | netdev->mpls_features |= NETIF_F_HW_CSUM; |
3347 | netdev->hw_enc_features |= netdev->vlan_features; |
3348 | |
3349 | /* set this bit last since it cannot be part of vlan_features */ |
3350 | netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | |
3351 | NETIF_F_HW_VLAN_CTAG_RX | |
3352 | NETIF_F_HW_VLAN_CTAG_TX; |
3353 | |
3354 | netdev->priv_flags |= IFF_SUPP_NOFCS; |
3355 | |
3356 | netdev->priv_flags |= IFF_UNICAST_FLT; |
3357 | netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT; |
3358 | |
3359 | /* MTU range: 68 - 9216 */ |
3360 | netdev->min_mtu = ETH_MIN_MTU; |
3361 | netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; |
3362 | |
3363 | adapter->en_mng_pt = igb_enable_mng_pass_thru(hw); |
3364 | |
3365 | /* before reading the NVM, reset the controller to put the device in a |
3366 | * known good starting state |
3367 | */ |
3368 | hw->mac.ops.reset_hw(hw); |
3369 | |
3370 | /* make sure the NVM is good , i211/i210 parts can have special NVM |
3371 | * that doesn't contain a checksum |
3372 | */ |
3373 | switch (hw->mac.type) { |
3374 | case e1000_i210: |
3375 | case e1000_i211: |
3376 | if (igb_get_flash_presence_i210(hw)) { |
3377 | if (hw->nvm.ops.validate(hw) < 0) { |
3378 | dev_err(&pdev->dev, |
3379 | "The NVM Checksum Is Not Valid\n" ); |
3380 | err = -EIO; |
3381 | goto err_eeprom; |
3382 | } |
3383 | } |
3384 | break; |
3385 | default: |
3386 | if (hw->nvm.ops.validate(hw) < 0) { |
3387 | dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n" ); |
3388 | err = -EIO; |
3389 | goto err_eeprom; |
3390 | } |
3391 | break; |
3392 | } |
3393 | |
3394 | if (eth_platform_get_mac_address(dev: &pdev->dev, mac_addr: hw->mac.addr)) { |
3395 | /* copy the MAC address out of the NVM */ |
3396 | if (hw->mac.ops.read_mac_addr(hw)) |
3397 | dev_err(&pdev->dev, "NVM Read Error\n" ); |
3398 | } |
3399 | |
3400 | eth_hw_addr_set(dev: netdev, addr: hw->mac.addr); |
3401 | |
3402 | if (!is_valid_ether_addr(addr: netdev->dev_addr)) { |
3403 | dev_err(&pdev->dev, "Invalid MAC Address\n" ); |
3404 | err = -EIO; |
3405 | goto err_eeprom; |
3406 | } |
3407 | |
3408 | igb_set_default_mac_filter(adapter); |
3409 | |
3410 | /* get firmware version for ethtool -i */ |
3411 | igb_set_fw_version(adapter); |
3412 | |
3413 | /* configure RXPBSIZE and TXPBSIZE */ |
3414 | if (hw->mac.type == e1000_i210) { |
3415 | wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT); |
3416 | wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT); |
3417 | } |
3418 | |
3419 | timer_setup(&adapter->watchdog_timer, igb_watchdog, 0); |
3420 | timer_setup(&adapter->phy_info_timer, igb_update_phy_info, 0); |
3421 | |
3422 | INIT_WORK(&adapter->reset_task, igb_reset_task); |
3423 | INIT_WORK(&adapter->watchdog_task, igb_watchdog_task); |
3424 | |
3425 | /* Initialize link properties that are user-changeable */ |
3426 | adapter->fc_autoneg = true; |
3427 | hw->mac.autoneg = true; |
3428 | hw->phy.autoneg_advertised = 0x2f; |
3429 | |
3430 | hw->fc.requested_mode = e1000_fc_default; |
3431 | hw->fc.current_mode = e1000_fc_default; |
3432 | |
3433 | igb_validate_mdi_setting(hw); |
3434 | |
3435 | /* By default, support wake on port A */ |
3436 | if (hw->bus.func == 0) |
3437 | adapter->flags |= IGB_FLAG_WOL_SUPPORTED; |
3438 | |
3439 | /* Check the NVM for wake support on non-port A ports */ |
3440 | if (hw->mac.type >= e1000_82580) |
3441 | hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A + |
3442 | NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1, |
3443 | &eeprom_data); |
3444 | else if (hw->bus.func == 1) |
3445 | hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); |
3446 | |
3447 | if (eeprom_data & IGB_EEPROM_APME) |
3448 | adapter->flags |= IGB_FLAG_WOL_SUPPORTED; |
3449 | |
3450 | /* now that we have the eeprom settings, apply the special cases where |
3451 | * the eeprom may be wrong or the board simply won't support wake on |
3452 | * lan on a particular port |
3453 | */ |
3454 | switch (pdev->device) { |
3455 | case E1000_DEV_ID_82575GB_QUAD_COPPER: |
3456 | adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; |
3457 | break; |
3458 | case E1000_DEV_ID_82575EB_FIBER_SERDES: |
3459 | case E1000_DEV_ID_82576_FIBER: |
3460 | case E1000_DEV_ID_82576_SERDES: |
3461 | /* Wake events only supported on port A for dual fiber |
3462 | * regardless of eeprom setting |
3463 | */ |
3464 | if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1) |
3465 | adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; |
3466 | break; |
3467 | case E1000_DEV_ID_82576_QUAD_COPPER: |
3468 | case E1000_DEV_ID_82576_QUAD_COPPER_ET2: |
3469 | /* if quad port adapter, disable WoL on all but port A */ |
3470 | if (global_quad_port_a != 0) |
3471 | adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; |
3472 | else |
3473 | adapter->flags |= IGB_FLAG_QUAD_PORT_A; |
3474 | /* Reset for multiple quad port adapters */ |
3475 | if (++global_quad_port_a == 4) |
3476 | global_quad_port_a = 0; |
3477 | break; |
3478 | default: |
3479 | /* If the device can't wake, don't set software support */ |
3480 | if (!device_can_wakeup(dev: &adapter->pdev->dev)) |
3481 | adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED; |
3482 | } |
3483 | |
3484 | /* initialize the wol settings based on the eeprom settings */ |
3485 | if (adapter->flags & IGB_FLAG_WOL_SUPPORTED) |
3486 | adapter->wol |= E1000_WUFC_MAG; |
3487 | |
3488 | /* Some vendors want WoL disabled by default, but still supported */ |
3489 | if ((hw->mac.type == e1000_i350) && |
3490 | (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) { |
3491 | adapter->flags |= IGB_FLAG_WOL_SUPPORTED; |
3492 | adapter->wol = 0; |
3493 | } |
3494 | |
3495 | /* Some vendors want the ability to Use the EEPROM setting as |
3496 | * enable/disable only, and not for capability |
3497 | */ |
3498 | if (((hw->mac.type == e1000_i350) || |
3499 | (hw->mac.type == e1000_i354)) && |
3500 | (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) { |
3501 | adapter->flags |= IGB_FLAG_WOL_SUPPORTED; |
3502 | adapter->wol = 0; |
3503 | } |
3504 | if (hw->mac.type == e1000_i350) { |
3505 | if (((pdev->subsystem_device == 0x5001) || |
3506 | (pdev->subsystem_device == 0x5002)) && |
3507 | (hw->bus.func == 0)) { |
3508 | adapter->flags |= IGB_FLAG_WOL_SUPPORTED; |
3509 | adapter->wol = 0; |
3510 | } |
3511 | if (pdev->subsystem_device == 0x1F52) |
3512 | adapter->flags |= IGB_FLAG_WOL_SUPPORTED; |
3513 | } |
3514 | |
3515 | device_set_wakeup_enable(dev: &adapter->pdev->dev, |
3516 | enable: adapter->flags & IGB_FLAG_WOL_SUPPORTED); |
3517 | |
3518 | /* reset the hardware with the new settings */ |
3519 | igb_reset(adapter); |
3520 | |
3521 | /* Init the I2C interface */ |
3522 | err = igb_init_i2c(adapter); |
3523 | if (err) { |
3524 | dev_err(&pdev->dev, "failed to init i2c interface\n" ); |
3525 | goto err_eeprom; |
3526 | } |
3527 | |
3528 | /* let the f/w know that the h/w is now under the control of the |
3529 | * driver. |
3530 | */ |
3531 | igb_get_hw_control(adapter); |
3532 | |
3533 | strcpy(p: netdev->name, q: "eth%d" ); |
3534 | err = register_netdev(dev: netdev); |
3535 | if (err) |
3536 | goto err_register; |
3537 | |
3538 | /* carrier off reporting is important to ethtool even BEFORE open */ |
3539 | netif_carrier_off(dev: netdev); |
3540 | |
3541 | #ifdef CONFIG_IGB_DCA |
3542 | if (dca_add_requester(dev: &pdev->dev) == 0) { |
3543 | adapter->flags |= IGB_FLAG_DCA_ENABLED; |
3544 | dev_info(&pdev->dev, "DCA enabled\n" ); |
3545 | igb_setup_dca(adapter); |
3546 | } |
3547 | |
3548 | #endif |
3549 | #ifdef CONFIG_IGB_HWMON |
3550 | /* Initialize the thermal sensor on i350 devices. */ |
3551 | if (hw->mac.type == e1000_i350 && hw->bus.func == 0) { |
3552 | u16 ets_word; |
3553 | |
3554 | /* Read the NVM to determine if this i350 device supports an |
3555 | * external thermal sensor. |
3556 | */ |
3557 | hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word); |
3558 | if (ets_word != 0x0000 && ets_word != 0xFFFF) |
3559 | adapter->ets = true; |
3560 | else |
3561 | adapter->ets = false; |
3562 | /* Only enable I2C bit banging if an external thermal |
3563 | * sensor is supported. |
3564 | */ |
3565 | if (adapter->ets) |
3566 | igb_set_i2c_bb(hw); |
3567 | hw->mac.ops.init_thermal_sensor_thresh(hw); |
3568 | if (igb_sysfs_init(adapter)) |
3569 | dev_err(&pdev->dev, |
3570 | "failed to allocate sysfs resources\n" ); |
3571 | } else { |
3572 | adapter->ets = false; |
3573 | } |
3574 | #endif |
3575 | /* Check if Media Autosense is enabled */ |
3576 | adapter->ei = *ei; |
3577 | if (hw->dev_spec._82575.mas_capable) |
3578 | igb_init_mas(adapter); |
3579 | |
3580 | /* do hw tstamp init after resetting */ |
3581 | igb_ptp_init(adapter); |
3582 | |
3583 | dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n" ); |
3584 | /* print bus type/speed/width info, not applicable to i354 */ |
3585 | if (hw->mac.type != e1000_i354) { |
3586 | dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n" , |
3587 | netdev->name, |
3588 | ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" : |
3589 | (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" : |
3590 | "unknown" ), |
3591 | ((hw->bus.width == e1000_bus_width_pcie_x4) ? |
3592 | "Width x4" : |
3593 | (hw->bus.width == e1000_bus_width_pcie_x2) ? |
3594 | "Width x2" : |
3595 | (hw->bus.width == e1000_bus_width_pcie_x1) ? |
3596 | "Width x1" : "unknown" ), netdev->dev_addr); |
3597 | } |
3598 | |
3599 | if ((hw->mac.type == e1000_82576 && |
3600 | rd32(E1000_EECD) & E1000_EECD_PRES) || |
3601 | (hw->mac.type >= e1000_i210 || |
3602 | igb_get_flash_presence_i210(hw))) { |
3603 | ret_val = igb_read_part_string(hw, part_num: part_str, |
3604 | E1000_PBANUM_LENGTH); |
3605 | } else { |
3606 | ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND; |
3607 | } |
3608 | |
3609 | if (ret_val) |
3610 | strcpy(p: part_str, q: "Unknown" ); |
3611 | dev_info(&pdev->dev, "%s: PBA No: %s\n" , netdev->name, part_str); |
3612 | dev_info(&pdev->dev, |
3613 | "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n" , |
3614 | (adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" : |
3615 | (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy" , |
3616 | adapter->num_rx_queues, adapter->num_tx_queues); |
3617 | if (hw->phy.media_type == e1000_media_type_copper) { |
3618 | switch (hw->mac.type) { |
3619 | case e1000_i350: |
3620 | case e1000_i210: |
3621 | case e1000_i211: |
3622 | /* Enable EEE for internal copper PHY devices */ |
3623 | err = igb_set_eee_i350(hw, adv1G: true, adv100M: true); |
3624 | if ((!err) && |
3625 | (!hw->dev_spec._82575.eee_disable)) { |
3626 | adapter->eee_advert = |
3627 | MDIO_EEE_100TX | MDIO_EEE_1000T; |
3628 | adapter->flags |= IGB_FLAG_EEE; |
3629 | } |
3630 | break; |
3631 | case e1000_i354: |
3632 | if ((rd32(E1000_CTRL_EXT) & |
3633 | E1000_CTRL_EXT_LINK_MODE_SGMII)) { |
3634 | err = igb_set_eee_i354(hw, adv1G: true, adv100M: true); |
3635 | if ((!err) && |
3636 | (!hw->dev_spec._82575.eee_disable)) { |
3637 | adapter->eee_advert = |
3638 | MDIO_EEE_100TX | MDIO_EEE_1000T; |
3639 | adapter->flags |= IGB_FLAG_EEE; |
3640 | } |
3641 | } |
3642 | break; |
3643 | default: |
3644 | break; |
3645 | } |
3646 | } |
3647 | |
3648 | dev_pm_set_driver_flags(dev: &pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE); |
3649 | |
3650 | pm_runtime_put_noidle(dev: &pdev->dev); |
3651 | return 0; |
3652 | |
3653 | err_register: |
3654 | igb_release_hw_control(adapter); |
3655 | memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap)); |
3656 | err_eeprom: |
3657 | if (!igb_check_reset_block(hw)) |
3658 | igb_reset_phy(hw); |
3659 | |
3660 | if (hw->flash_address) |
3661 | iounmap(addr: hw->flash_address); |
3662 | err_sw_init: |
3663 | kfree(objp: adapter->mac_table); |
3664 | kfree(objp: adapter->shadow_vfta); |
3665 | igb_clear_interrupt_scheme(adapter); |
3666 | #ifdef CONFIG_PCI_IOV |
3667 | igb_disable_sriov(dev: pdev, reinit: false); |
3668 | #endif |
3669 | pci_iounmap(dev: pdev, adapter->io_addr); |
3670 | err_ioremap: |
3671 | free_netdev(dev: netdev); |
3672 | err_alloc_etherdev: |
3673 | pci_release_mem_regions(pdev); |
3674 | err_pci_reg: |
3675 | err_dma: |
3676 | pci_disable_device(dev: pdev); |
3677 | return err; |
3678 | } |
3679 | |
3680 | #ifdef CONFIG_PCI_IOV |
3681 | static int igb_sriov_reinit(struct pci_dev *dev) |
3682 | { |
3683 | struct net_device *netdev = pci_get_drvdata(pdev: dev); |
3684 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
3685 | struct pci_dev *pdev = adapter->pdev; |
3686 | |
3687 | rtnl_lock(); |
3688 | |
3689 | if (netif_running(dev: netdev)) |
3690 | igb_close(netdev); |
3691 | else |
3692 | igb_reset(adapter); |
3693 | |
3694 | igb_clear_interrupt_scheme(adapter); |
3695 | |
3696 | igb_init_queue_configuration(adapter); |
3697 | |
3698 | if (igb_init_interrupt_scheme(adapter, msix: true)) { |
3699 | rtnl_unlock(); |
3700 | dev_err(&pdev->dev, "Unable to allocate memory for queues\n" ); |
3701 | return -ENOMEM; |
3702 | } |
3703 | |
3704 | if (netif_running(dev: netdev)) |
3705 | igb_open(netdev); |
3706 | |
3707 | rtnl_unlock(); |
3708 | |
3709 | return 0; |
3710 | } |
3711 | |
3712 | static int igb_disable_sriov(struct pci_dev *pdev, bool reinit) |
3713 | { |
3714 | struct net_device *netdev = pci_get_drvdata(pdev); |
3715 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
3716 | struct e1000_hw *hw = &adapter->hw; |
3717 | unsigned long flags; |
3718 | |
3719 | /* reclaim resources allocated to VFs */ |
3720 | if (adapter->vf_data) { |
3721 | /* disable iov and allow time for transactions to clear */ |
3722 | if (pci_vfs_assigned(dev: pdev)) { |
3723 | dev_warn(&pdev->dev, |
3724 | "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n" ); |
3725 | return -EPERM; |
3726 | } else { |
3727 | pci_disable_sriov(dev: pdev); |
3728 | msleep(msecs: 500); |
3729 | } |
3730 | spin_lock_irqsave(&adapter->vfs_lock, flags); |
3731 | kfree(objp: adapter->vf_mac_list); |
3732 | adapter->vf_mac_list = NULL; |
3733 | kfree(objp: adapter->vf_data); |
3734 | adapter->vf_data = NULL; |
3735 | adapter->vfs_allocated_count = 0; |
3736 | spin_unlock_irqrestore(lock: &adapter->vfs_lock, flags); |
3737 | wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ); |
3738 | wrfl(); |
3739 | msleep(msecs: 100); |
3740 | dev_info(&pdev->dev, "IOV Disabled\n" ); |
3741 | |
3742 | /* Re-enable DMA Coalescing flag since IOV is turned off */ |
3743 | adapter->flags |= IGB_FLAG_DMAC; |
3744 | } |
3745 | |
3746 | return reinit ? igb_sriov_reinit(dev: pdev) : 0; |
3747 | } |
3748 | |
3749 | static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs, bool reinit) |
3750 | { |
3751 | struct net_device *netdev = pci_get_drvdata(pdev); |
3752 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
3753 | int old_vfs = pci_num_vf(dev: pdev); |
3754 | struct vf_mac_filter *mac_list; |
3755 | int err = 0; |
3756 | int num_vf_mac_filters, i; |
3757 | |
3758 | if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) { |
3759 | err = -EPERM; |
3760 | goto out; |
3761 | } |
3762 | if (!num_vfs) |
3763 | goto out; |
3764 | |
3765 | if (old_vfs) { |
3766 | dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n" , |
3767 | old_vfs, max_vfs); |
3768 | adapter->vfs_allocated_count = old_vfs; |
3769 | } else |
3770 | adapter->vfs_allocated_count = num_vfs; |
3771 | |
3772 | adapter->vf_data = kcalloc(n: adapter->vfs_allocated_count, |
3773 | size: sizeof(struct vf_data_storage), GFP_KERNEL); |
3774 | |
3775 | /* if allocation failed then we do not support SR-IOV */ |
3776 | if (!adapter->vf_data) { |
3777 | adapter->vfs_allocated_count = 0; |
3778 | err = -ENOMEM; |
3779 | goto out; |
3780 | } |
3781 | |
3782 | /* Due to the limited number of RAR entries calculate potential |
3783 | * number of MAC filters available for the VFs. Reserve entries |
3784 | * for PF default MAC, PF MAC filters and at least one RAR entry |
3785 | * for each VF for VF MAC. |
3786 | */ |
3787 | num_vf_mac_filters = adapter->hw.mac.rar_entry_count - |
3788 | (1 + IGB_PF_MAC_FILTERS_RESERVED + |
3789 | adapter->vfs_allocated_count); |
3790 | |
3791 | adapter->vf_mac_list = kcalloc(n: num_vf_mac_filters, |
3792 | size: sizeof(struct vf_mac_filter), |
3793 | GFP_KERNEL); |
3794 | |
3795 | mac_list = adapter->vf_mac_list; |
3796 | INIT_LIST_HEAD(list: &adapter->vf_macs.l); |
3797 | |
3798 | if (adapter->vf_mac_list) { |
3799 | /* Initialize list of VF MAC filters */ |
3800 | for (i = 0; i < num_vf_mac_filters; i++) { |
3801 | mac_list->vf = -1; |
3802 | mac_list->free = true; |
3803 | list_add(new: &mac_list->l, head: &adapter->vf_macs.l); |
3804 | mac_list++; |
3805 | } |
3806 | } else { |
3807 | /* If we could not allocate memory for the VF MAC filters |
3808 | * we can continue without this feature but warn user. |
3809 | */ |
3810 | dev_err(&pdev->dev, |
3811 | "Unable to allocate memory for VF MAC filter list\n" ); |
3812 | } |
3813 | |
3814 | dev_info(&pdev->dev, "%d VFs allocated\n" , |
3815 | adapter->vfs_allocated_count); |
3816 | for (i = 0; i < adapter->vfs_allocated_count; i++) |
3817 | igb_vf_configure(adapter, vf: i); |
3818 | |
3819 | /* DMA Coalescing is not supported in IOV mode. */ |
3820 | adapter->flags &= ~IGB_FLAG_DMAC; |
3821 | |
3822 | if (reinit) { |
3823 | err = igb_sriov_reinit(dev: pdev); |
3824 | if (err) |
3825 | goto err_out; |
3826 | } |
3827 | |
3828 | /* only call pci_enable_sriov() if no VFs are allocated already */ |
3829 | if (!old_vfs) { |
3830 | err = pci_enable_sriov(dev: pdev, nr_virtfn: adapter->vfs_allocated_count); |
3831 | if (err) |
3832 | goto err_out; |
3833 | } |
3834 | |
3835 | goto out; |
3836 | |
3837 | err_out: |
3838 | kfree(objp: adapter->vf_mac_list); |
3839 | adapter->vf_mac_list = NULL; |
3840 | kfree(objp: adapter->vf_data); |
3841 | adapter->vf_data = NULL; |
3842 | adapter->vfs_allocated_count = 0; |
3843 | out: |
3844 | return err; |
3845 | } |
3846 | |
3847 | #endif |
3848 | /** |
3849 | * igb_remove_i2c - Cleanup I2C interface |
3850 | * @adapter: pointer to adapter structure |
3851 | **/ |
3852 | static void igb_remove_i2c(struct igb_adapter *adapter) |
3853 | { |
3854 | /* free the adapter bus structure */ |
3855 | i2c_del_adapter(adap: &adapter->i2c_adap); |
3856 | } |
3857 | |
3858 | /** |
3859 | * igb_remove - Device Removal Routine |
3860 | * @pdev: PCI device information struct |
3861 | * |
3862 | * igb_remove is called by the PCI subsystem to alert the driver |
3863 | * that it should release a PCI device. The could be caused by a |
3864 | * Hot-Plug event, or because the driver is going to be removed from |
3865 | * memory. |
3866 | **/ |
3867 | static void igb_remove(struct pci_dev *pdev) |
3868 | { |
3869 | struct net_device *netdev = pci_get_drvdata(pdev); |
3870 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
3871 | struct e1000_hw *hw = &adapter->hw; |
3872 | |
3873 | pm_runtime_get_noresume(dev: &pdev->dev); |
3874 | #ifdef CONFIG_IGB_HWMON |
3875 | igb_sysfs_exit(adapter); |
3876 | #endif |
3877 | igb_remove_i2c(adapter); |
3878 | igb_ptp_stop(adapter); |
3879 | /* The watchdog timer may be rescheduled, so explicitly |
3880 | * disable watchdog from being rescheduled. |
3881 | */ |
3882 | set_bit(nr: __IGB_DOWN, addr: &adapter->state); |
3883 | del_timer_sync(timer: &adapter->watchdog_timer); |
3884 | del_timer_sync(timer: &adapter->phy_info_timer); |
3885 | |
3886 | cancel_work_sync(work: &adapter->reset_task); |
3887 | cancel_work_sync(work: &adapter->watchdog_task); |
3888 | |
3889 | #ifdef CONFIG_IGB_DCA |
3890 | if (adapter->flags & IGB_FLAG_DCA_ENABLED) { |
3891 | dev_info(&pdev->dev, "DCA disabled\n" ); |
3892 | dca_remove_requester(dev: &pdev->dev); |
3893 | adapter->flags &= ~IGB_FLAG_DCA_ENABLED; |
3894 | wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE); |
3895 | } |
3896 | #endif |
3897 | |
3898 | /* Release control of h/w to f/w. If f/w is AMT enabled, this |
3899 | * would have already happened in close and is redundant. |
3900 | */ |
3901 | igb_release_hw_control(adapter); |
3902 | |
3903 | #ifdef CONFIG_PCI_IOV |
3904 | igb_disable_sriov(pdev, reinit: false); |
3905 | #endif |
3906 | |
3907 | unregister_netdev(dev: netdev); |
3908 | |
3909 | igb_clear_interrupt_scheme(adapter); |
3910 | |
3911 | pci_iounmap(dev: pdev, adapter->io_addr); |
3912 | if (hw->flash_address) |
3913 | iounmap(addr: hw->flash_address); |
3914 | pci_release_mem_regions(pdev); |
3915 | |
3916 | kfree(objp: adapter->mac_table); |
3917 | kfree(objp: adapter->shadow_vfta); |
3918 | free_netdev(dev: netdev); |
3919 | |
3920 | pci_disable_device(dev: pdev); |
3921 | } |
3922 | |
3923 | /** |
3924 | * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space |
3925 | * @adapter: board private structure to initialize |
3926 | * |
3927 | * This function initializes the vf specific data storage and then attempts to |
3928 | * allocate the VFs. The reason for ordering it this way is because it is much |
3929 | * mor expensive time wise to disable SR-IOV than it is to allocate and free |
3930 | * the memory for the VFs. |
3931 | **/ |
3932 | static void igb_probe_vfs(struct igb_adapter *adapter) |
3933 | { |
3934 | #ifdef CONFIG_PCI_IOV |
3935 | struct pci_dev *pdev = adapter->pdev; |
3936 | struct e1000_hw *hw = &adapter->hw; |
3937 | |
3938 | /* Virtualization features not supported on i210 and 82580 family. */ |
3939 | if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211) || |
3940 | (hw->mac.type == e1000_82580)) |
3941 | return; |
3942 | |
3943 | /* Of the below we really only want the effect of getting |
3944 | * IGB_FLAG_HAS_MSIX set (if available), without which |
3945 | * igb_enable_sriov() has no effect. |
3946 | */ |
3947 | igb_set_interrupt_capability(adapter, msix: true); |
3948 | igb_reset_interrupt_capability(adapter); |
3949 | |
3950 | pci_sriov_set_totalvfs(dev: pdev, numvfs: 7); |
3951 | igb_enable_sriov(pdev, num_vfs: max_vfs, reinit: false); |
3952 | |
3953 | #endif /* CONFIG_PCI_IOV */ |
3954 | } |
3955 | |
3956 | unsigned int (struct igb_adapter *adapter) |
3957 | { |
3958 | struct e1000_hw *hw = &adapter->hw; |
3959 | unsigned int ; |
3960 | |
3961 | /* Determine the maximum number of RSS queues supported. */ |
3962 | switch (hw->mac.type) { |
3963 | case e1000_i211: |
3964 | max_rss_queues = IGB_MAX_RX_QUEUES_I211; |
3965 | break; |
3966 | case e1000_82575: |
3967 | case e1000_i210: |
3968 | max_rss_queues = IGB_MAX_RX_QUEUES_82575; |
3969 | break; |
3970 | case e1000_i350: |
3971 | /* I350 cannot do RSS and SR-IOV at the same time */ |
3972 | if (!!adapter->vfs_allocated_count) { |
3973 | max_rss_queues = 1; |
3974 | break; |
3975 | } |
3976 | fallthrough; |
3977 | case e1000_82576: |
3978 | if (!!adapter->vfs_allocated_count) { |
3979 | max_rss_queues = 2; |
3980 | break; |
3981 | } |
3982 | fallthrough; |
3983 | case e1000_82580: |
3984 | case e1000_i354: |
3985 | default: |
3986 | max_rss_queues = IGB_MAX_RX_QUEUES; |
3987 | break; |
3988 | } |
3989 | |
3990 | return max_rss_queues; |
3991 | } |
3992 | |
3993 | static void igb_init_queue_configuration(struct igb_adapter *adapter) |
3994 | { |
3995 | u32 ; |
3996 | |
3997 | max_rss_queues = igb_get_max_rss_queues(adapter); |
3998 | adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); |
3999 | |
4000 | igb_set_flag_queue_pairs(adapter, max_rss_queues); |
4001 | } |
4002 | |
4003 | void igb_set_flag_queue_pairs(struct igb_adapter *adapter, |
4004 | const u32 ) |
4005 | { |
4006 | struct e1000_hw *hw = &adapter->hw; |
4007 | |
4008 | /* Determine if we need to pair queues. */ |
4009 | switch (hw->mac.type) { |
4010 | case e1000_82575: |
4011 | case e1000_i211: |
4012 | /* Device supports enough interrupts without queue pairing. */ |
4013 | break; |
4014 | case e1000_82576: |
4015 | case e1000_82580: |
4016 | case e1000_i350: |
4017 | case e1000_i354: |
4018 | case e1000_i210: |
4019 | default: |
4020 | /* If rss_queues > half of max_rss_queues, pair the queues in |
4021 | * order to conserve interrupts due to limited supply. |
4022 | */ |
4023 | if (adapter->rss_queues > (max_rss_queues / 2)) |
4024 | adapter->flags |= IGB_FLAG_QUEUE_PAIRS; |
4025 | else |
4026 | adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS; |
4027 | break; |
4028 | } |
4029 | } |
4030 | |
4031 | /** |
4032 | * igb_sw_init - Initialize general software structures (struct igb_adapter) |
4033 | * @adapter: board private structure to initialize |
4034 | * |
4035 | * igb_sw_init initializes the Adapter private data structure. |
4036 | * Fields are initialized based on PCI device information and |
4037 | * OS network device settings (MTU size). |
4038 | **/ |
4039 | static int igb_sw_init(struct igb_adapter *adapter) |
4040 | { |
4041 | struct e1000_hw *hw = &adapter->hw; |
4042 | struct net_device *netdev = adapter->netdev; |
4043 | struct pci_dev *pdev = adapter->pdev; |
4044 | |
4045 | pci_read_config_word(dev: pdev, PCI_COMMAND, val: &hw->bus.pci_cmd_word); |
4046 | |
4047 | /* set default ring sizes */ |
4048 | adapter->tx_ring_count = IGB_DEFAULT_TXD; |
4049 | adapter->rx_ring_count = IGB_DEFAULT_RXD; |
4050 | |
4051 | /* set default ITR values */ |
4052 | adapter->rx_itr_setting = IGB_DEFAULT_ITR; |
4053 | adapter->tx_itr_setting = IGB_DEFAULT_ITR; |
4054 | |
4055 | /* set default work limits */ |
4056 | adapter->tx_work_limit = IGB_DEFAULT_TX_WORK; |
4057 | |
4058 | adapter->max_frame_size = netdev->mtu + IGB_ETH_PKT_HDR_PAD; |
4059 | adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; |
4060 | |
4061 | spin_lock_init(&adapter->nfc_lock); |
4062 | spin_lock_init(&adapter->stats64_lock); |
4063 | |
4064 | /* init spinlock to avoid concurrency of VF resources */ |
4065 | spin_lock_init(&adapter->vfs_lock); |
4066 | #ifdef CONFIG_PCI_IOV |
4067 | switch (hw->mac.type) { |
4068 | case e1000_82576: |
4069 | case e1000_i350: |
4070 | if (max_vfs > 7) { |
4071 | dev_warn(&pdev->dev, |
4072 | "Maximum of 7 VFs per PF, using max\n" ); |
4073 | max_vfs = adapter->vfs_allocated_count = 7; |
4074 | } else |
4075 | adapter->vfs_allocated_count = max_vfs; |
4076 | if (adapter->vfs_allocated_count) |
4077 | dev_warn(&pdev->dev, |
4078 | "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n" ); |
4079 | break; |
4080 | default: |
4081 | break; |
4082 | } |
4083 | #endif /* CONFIG_PCI_IOV */ |
4084 | |
4085 | /* Assume MSI-X interrupts, will be checked during IRQ allocation */ |
4086 | adapter->flags |= IGB_FLAG_HAS_MSIX; |
4087 | |
4088 | adapter->mac_table = kcalloc(n: hw->mac.rar_entry_count, |
4089 | size: sizeof(struct igb_mac_addr), |
4090 | GFP_KERNEL); |
4091 | if (!adapter->mac_table) |
4092 | return -ENOMEM; |
4093 | |
4094 | igb_probe_vfs(adapter); |
4095 | |
4096 | igb_init_queue_configuration(adapter); |
4097 | |
4098 | /* Setup and initialize a copy of the hw vlan table array */ |
4099 | adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, size: sizeof(u32), |
4100 | GFP_KERNEL); |
4101 | if (!adapter->shadow_vfta) |
4102 | return -ENOMEM; |
4103 | |
4104 | /* This call may decrease the number of queues */ |
4105 | if (igb_init_interrupt_scheme(adapter, msix: true)) { |
4106 | dev_err(&pdev->dev, "Unable to allocate memory for queues\n" ); |
4107 | return -ENOMEM; |
4108 | } |
4109 | |
4110 | /* Explicitly disable IRQ since the NIC can be in any state. */ |
4111 | igb_irq_disable(adapter); |
4112 | |
4113 | if (hw->mac.type >= e1000_i350) |
4114 | adapter->flags &= ~IGB_FLAG_DMAC; |
4115 | |
4116 | set_bit(nr: __IGB_DOWN, addr: &adapter->state); |
4117 | return 0; |
4118 | } |
4119 | |
4120 | /** |
4121 | * __igb_open - Called when a network interface is made active |
4122 | * @netdev: network interface device structure |
4123 | * @resuming: indicates whether we are in a resume call |
4124 | * |
4125 | * Returns 0 on success, negative value on failure |
4126 | * |
4127 | * The open entry point is called when a network interface is made |
4128 | * active by the system (IFF_UP). At this point all resources needed |
4129 | * for transmit and receive operations are allocated, the interrupt |
4130 | * handler is registered with the OS, the watchdog timer is started, |
4131 | * and the stack is notified that the interface is ready. |
4132 | **/ |
4133 | static int __igb_open(struct net_device *netdev, bool resuming) |
4134 | { |
4135 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
4136 | struct e1000_hw *hw = &adapter->hw; |
4137 | struct pci_dev *pdev = adapter->pdev; |
4138 | int err; |
4139 | int i; |
4140 | |
4141 | /* disallow open during test */ |
4142 | if (test_bit(__IGB_TESTING, &adapter->state)) { |
4143 | WARN_ON(resuming); |
4144 | return -EBUSY; |
4145 | } |
4146 | |
4147 | if (!resuming) |
4148 | pm_runtime_get_sync(dev: &pdev->dev); |
4149 | |
4150 | netif_carrier_off(dev: netdev); |
4151 | |
4152 | /* allocate transmit descriptors */ |
4153 | err = igb_setup_all_tx_resources(adapter); |
4154 | if (err) |
4155 | goto err_setup_tx; |
4156 | |
4157 | /* allocate receive descriptors */ |
4158 | err = igb_setup_all_rx_resources(adapter); |
4159 | if (err) |
4160 | goto err_setup_rx; |
4161 | |
4162 | igb_power_up_link(adapter); |
4163 | |
4164 | /* before we allocate an interrupt, we must be ready to handle it. |
4165 | * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt |
4166 | * as soon as we call pci_request_irq, so we have to setup our |
4167 | * clean_rx handler before we do so. |
4168 | */ |
4169 | igb_configure(adapter); |
4170 | |
4171 | err = igb_request_irq(adapter); |
4172 | if (err) |
4173 | goto err_req_irq; |
4174 | |
4175 | /* Notify the stack of the actual queue counts. */ |
4176 | err = netif_set_real_num_tx_queues(dev: adapter->netdev, |
4177 | txq: adapter->num_tx_queues); |
4178 | if (err) |
4179 | goto err_set_queues; |
4180 | |
4181 | err = netif_set_real_num_rx_queues(dev: adapter->netdev, |
4182 | rxq: adapter->num_rx_queues); |
4183 | if (err) |
4184 | goto err_set_queues; |
4185 | |
4186 | /* From here on the code is the same as igb_up() */ |
4187 | clear_bit(nr: __IGB_DOWN, addr: &adapter->state); |
4188 | |
4189 | for (i = 0; i < adapter->num_q_vectors; i++) |
4190 | napi_enable(n: &(adapter->q_vector[i]->napi)); |
4191 | |
4192 | /* Clear any pending interrupts. */ |
4193 | rd32(E1000_TSICR); |
4194 | rd32(E1000_ICR); |
4195 | |
4196 | igb_irq_enable(adapter); |
4197 | |
4198 | /* notify VFs that reset has been completed */ |
4199 | if (adapter->vfs_allocated_count) { |
4200 | u32 reg_data = rd32(E1000_CTRL_EXT); |
4201 | |
4202 | reg_data |= E1000_CTRL_EXT_PFRSTD; |
4203 | wr32(E1000_CTRL_EXT, reg_data); |
4204 | } |
4205 | |
4206 | netif_tx_start_all_queues(dev: netdev); |
4207 | |
4208 | if (!resuming) |
4209 | pm_runtime_put(dev: &pdev->dev); |
4210 | |
4211 | /* start the watchdog. */ |
4212 | hw->mac.get_link_status = 1; |
4213 | schedule_work(work: &adapter->watchdog_task); |
4214 | |
4215 | return 0; |
4216 | |
4217 | err_set_queues: |
4218 | igb_free_irq(adapter); |
4219 | err_req_irq: |
4220 | igb_release_hw_control(adapter); |
4221 | igb_power_down_link(adapter); |
4222 | igb_free_all_rx_resources(adapter); |
4223 | err_setup_rx: |
4224 | igb_free_all_tx_resources(adapter); |
4225 | err_setup_tx: |
4226 | igb_reset(adapter); |
4227 | if (!resuming) |
4228 | pm_runtime_put(dev: &pdev->dev); |
4229 | |
4230 | return err; |
4231 | } |
4232 | |
4233 | int igb_open(struct net_device *netdev) |
4234 | { |
4235 | return __igb_open(netdev, resuming: false); |
4236 | } |
4237 | |
4238 | /** |
4239 | * __igb_close - Disables a network interface |
4240 | * @netdev: network interface device structure |
4241 | * @suspending: indicates we are in a suspend call |
4242 | * |
4243 | * Returns 0, this is not allowed to fail |
4244 | * |
4245 | * The close entry point is called when an interface is de-activated |
4246 | * by the OS. The hardware is still under the driver's control, but |
4247 | * needs to be disabled. A global MAC reset is issued to stop the |
4248 | * hardware, and all transmit and receive resources are freed. |
4249 | **/ |
4250 | static int __igb_close(struct net_device *netdev, bool suspending) |
4251 | { |
4252 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
4253 | struct pci_dev *pdev = adapter->pdev; |
4254 | |
4255 | WARN_ON(test_bit(__IGB_RESETTING, &adapter->state)); |
4256 | |
4257 | if (!suspending) |
4258 | pm_runtime_get_sync(dev: &pdev->dev); |
4259 | |
4260 | igb_down(adapter); |
4261 | igb_free_irq(adapter); |
4262 | |
4263 | igb_free_all_tx_resources(adapter); |
4264 | igb_free_all_rx_resources(adapter); |
4265 | |
4266 | if (!suspending) |
4267 | pm_runtime_put_sync(dev: &pdev->dev); |
4268 | return 0; |
4269 | } |
4270 | |
4271 | int igb_close(struct net_device *netdev) |
4272 | { |
4273 | if (netif_device_present(dev: netdev) || netdev->dismantle) |
4274 | return __igb_close(netdev, suspending: false); |
4275 | return 0; |
4276 | } |
4277 | |
4278 | /** |
4279 | * igb_setup_tx_resources - allocate Tx resources (Descriptors) |
4280 | * @tx_ring: tx descriptor ring (for a specific queue) to setup |
4281 | * |
4282 | * Return 0 on success, negative on failure |
4283 | **/ |
4284 | int igb_setup_tx_resources(struct igb_ring *tx_ring) |
4285 | { |
4286 | struct device *dev = tx_ring->dev; |
4287 | int size; |
4288 | |
4289 | size = sizeof(struct igb_tx_buffer) * tx_ring->count; |
4290 | |
4291 | tx_ring->tx_buffer_info = vmalloc(size); |
4292 | if (!tx_ring->tx_buffer_info) |
4293 | goto err; |
4294 | |
4295 | /* round up to nearest 4K */ |
4296 | tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc); |
4297 | tx_ring->size = ALIGN(tx_ring->size, 4096); |
4298 | |
4299 | tx_ring->desc = dma_alloc_coherent(dev, size: tx_ring->size, |
4300 | dma_handle: &tx_ring->dma, GFP_KERNEL); |
4301 | if (!tx_ring->desc) |
4302 | goto err; |
4303 | |
4304 | tx_ring->next_to_use = 0; |
4305 | tx_ring->next_to_clean = 0; |
4306 | |
4307 | return 0; |
4308 | |
4309 | err: |
4310 | vfree(addr: tx_ring->tx_buffer_info); |
4311 | tx_ring->tx_buffer_info = NULL; |
4312 | dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n" ); |
4313 | return -ENOMEM; |
4314 | } |
4315 | |
4316 | /** |
4317 | * igb_setup_all_tx_resources - wrapper to allocate Tx resources |
4318 | * (Descriptors) for all queues |
4319 | * @adapter: board private structure |
4320 | * |
4321 | * Return 0 on success, negative on failure |
4322 | **/ |
4323 | static int igb_setup_all_tx_resources(struct igb_adapter *adapter) |
4324 | { |
4325 | struct pci_dev *pdev = adapter->pdev; |
4326 | int i, err = 0; |
4327 | |
4328 | for (i = 0; i < adapter->num_tx_queues; i++) { |
4329 | err = igb_setup_tx_resources(tx_ring: adapter->tx_ring[i]); |
4330 | if (err) { |
4331 | dev_err(&pdev->dev, |
4332 | "Allocation for Tx Queue %u failed\n" , i); |
4333 | for (i--; i >= 0; i--) |
4334 | igb_free_tx_resources(adapter->tx_ring[i]); |
4335 | break; |
4336 | } |
4337 | } |
4338 | |
4339 | return err; |
4340 | } |
4341 | |
4342 | /** |
4343 | * igb_setup_tctl - configure the transmit control registers |
4344 | * @adapter: Board private structure |
4345 | **/ |
4346 | void igb_setup_tctl(struct igb_adapter *adapter) |
4347 | { |
4348 | struct e1000_hw *hw = &adapter->hw; |
4349 | u32 tctl; |
4350 | |
4351 | /* disable queue 0 which is enabled by default on 82575 and 82576 */ |
4352 | wr32(E1000_TXDCTL(0), 0); |
4353 | |
4354 | /* Program the Transmit Control Register */ |
4355 | tctl = rd32(E1000_TCTL); |
4356 | tctl &= ~E1000_TCTL_CT; |
4357 | tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | |
4358 | (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); |
4359 | |
4360 | igb_config_collision_dist(hw); |
4361 | |
4362 | /* Enable transmits */ |
4363 | tctl |= E1000_TCTL_EN; |
4364 | |
4365 | wr32(E1000_TCTL, tctl); |
4366 | } |
4367 | |
4368 | /** |
4369 | * igb_configure_tx_ring - Configure transmit ring after Reset |
4370 | * @adapter: board private structure |
4371 | * @ring: tx ring to configure |
4372 | * |
4373 | * Configure a transmit ring after a reset. |
4374 | **/ |
4375 | void igb_configure_tx_ring(struct igb_adapter *adapter, |
4376 | struct igb_ring *ring) |
4377 | { |
4378 | struct e1000_hw *hw = &adapter->hw; |
4379 | u32 txdctl = 0; |
4380 | u64 tdba = ring->dma; |
4381 | int reg_idx = ring->reg_idx; |
4382 | |
4383 | wr32(E1000_TDLEN(reg_idx), |
4384 | ring->count * sizeof(union e1000_adv_tx_desc)); |
4385 | wr32(E1000_TDBAL(reg_idx), |
4386 | tdba & 0x00000000ffffffffULL); |
4387 | wr32(E1000_TDBAH(reg_idx), tdba >> 32); |
4388 | |
4389 | ring->tail = adapter->io_addr + E1000_TDT(reg_idx); |
4390 | wr32(E1000_TDH(reg_idx), 0); |
4391 | writel(val: 0, addr: ring->tail); |
4392 | |
4393 | txdctl |= IGB_TX_PTHRESH; |
4394 | txdctl |= IGB_TX_HTHRESH << 8; |
4395 | txdctl |= IGB_TX_WTHRESH << 16; |
4396 | |
4397 | /* reinitialize tx_buffer_info */ |
4398 | memset(ring->tx_buffer_info, 0, |
4399 | sizeof(struct igb_tx_buffer) * ring->count); |
4400 | |
4401 | txdctl |= E1000_TXDCTL_QUEUE_ENABLE; |
4402 | wr32(E1000_TXDCTL(reg_idx), txdctl); |
4403 | } |
4404 | |
4405 | /** |
4406 | * igb_configure_tx - Configure transmit Unit after Reset |
4407 | * @adapter: board private structure |
4408 | * |
4409 | * Configure the Tx unit of the MAC after a reset. |
4410 | **/ |
4411 | static void igb_configure_tx(struct igb_adapter *adapter) |
4412 | { |
4413 | struct e1000_hw *hw = &adapter->hw; |
4414 | int i; |
4415 | |
4416 | /* disable the queues */ |
4417 | for (i = 0; i < adapter->num_tx_queues; i++) |
4418 | wr32(E1000_TXDCTL(adapter->tx_ring[i]->reg_idx), 0); |
4419 | |
4420 | wrfl(); |
4421 | usleep_range(min: 10000, max: 20000); |
4422 | |
4423 | for (i = 0; i < adapter->num_tx_queues; i++) |
4424 | igb_configure_tx_ring(adapter, ring: adapter->tx_ring[i]); |
4425 | } |
4426 | |
4427 | /** |
4428 | * igb_setup_rx_resources - allocate Rx resources (Descriptors) |
4429 | * @rx_ring: Rx descriptor ring (for a specific queue) to setup |
4430 | * |
4431 | * Returns 0 on success, negative on failure |
4432 | **/ |
4433 | int igb_setup_rx_resources(struct igb_ring *rx_ring) |
4434 | { |
4435 | struct igb_adapter *adapter = netdev_priv(dev: rx_ring->netdev); |
4436 | struct device *dev = rx_ring->dev; |
4437 | int size, res; |
4438 | |
4439 | /* XDP RX-queue info */ |
4440 | if (xdp_rxq_info_is_reg(xdp_rxq: &rx_ring->xdp_rxq)) |
4441 | xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq); |
4442 | res = xdp_rxq_info_reg(xdp_rxq: &rx_ring->xdp_rxq, dev: rx_ring->netdev, |
4443 | queue_index: rx_ring->queue_index, napi_id: 0); |
4444 | if (res < 0) { |
4445 | dev_err(dev, "Failed to register xdp_rxq index %u\n" , |
4446 | rx_ring->queue_index); |
4447 | return res; |
4448 | } |
4449 | |
4450 | size = sizeof(struct igb_rx_buffer) * rx_ring->count; |
4451 | |
4452 | rx_ring->rx_buffer_info = vmalloc(size); |
4453 | if (!rx_ring->rx_buffer_info) |
4454 | goto err; |
4455 | |
4456 | /* Round up to nearest 4K */ |
4457 | rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc); |
4458 | rx_ring->size = ALIGN(rx_ring->size, 4096); |
4459 | |
4460 | rx_ring->desc = dma_alloc_coherent(dev, size: rx_ring->size, |
4461 | dma_handle: &rx_ring->dma, GFP_KERNEL); |
4462 | if (!rx_ring->desc) |
4463 | goto err; |
4464 | |
4465 | rx_ring->next_to_alloc = 0; |
4466 | rx_ring->next_to_clean = 0; |
4467 | rx_ring->next_to_use = 0; |
4468 | |
4469 | rx_ring->xdp_prog = adapter->xdp_prog; |
4470 | |
4471 | return 0; |
4472 | |
4473 | err: |
4474 | xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq); |
4475 | vfree(addr: rx_ring->rx_buffer_info); |
4476 | rx_ring->rx_buffer_info = NULL; |
4477 | dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n" ); |
4478 | return -ENOMEM; |
4479 | } |
4480 | |
4481 | /** |
4482 | * igb_setup_all_rx_resources - wrapper to allocate Rx resources |
4483 | * (Descriptors) for all queues |
4484 | * @adapter: board private structure |
4485 | * |
4486 | * Return 0 on success, negative on failure |
4487 | **/ |
4488 | static int igb_setup_all_rx_resources(struct igb_adapter *adapter) |
4489 | { |
4490 | struct pci_dev *pdev = adapter->pdev; |
4491 | int i, err = 0; |
4492 | |
4493 | for (i = 0; i < adapter->num_rx_queues; i++) { |
4494 | err = igb_setup_rx_resources(rx_ring: adapter->rx_ring[i]); |
4495 | if (err) { |
4496 | dev_err(&pdev->dev, |
4497 | "Allocation for Rx Queue %u failed\n" , i); |
4498 | for (i--; i >= 0; i--) |
4499 | igb_free_rx_resources(adapter->rx_ring[i]); |
4500 | break; |
4501 | } |
4502 | } |
4503 | |
4504 | return err; |
4505 | } |
4506 | |
4507 | /** |
4508 | * igb_setup_mrqc - configure the multiple receive queue control registers |
4509 | * @adapter: Board private structure |
4510 | **/ |
4511 | static void igb_setup_mrqc(struct igb_adapter *adapter) |
4512 | { |
4513 | struct e1000_hw *hw = &adapter->hw; |
4514 | u32 mrqc, rxcsum; |
4515 | u32 j, num_rx_queues; |
4516 | u32 [10]; |
4517 | |
4518 | netdev_rss_key_fill(buffer: rss_key, len: sizeof(rss_key)); |
4519 | for (j = 0; j < 10; j++) |
4520 | wr32(E1000_RSSRK(j), rss_key[j]); |
4521 | |
4522 | num_rx_queues = adapter->rss_queues; |
4523 | |
4524 | switch (hw->mac.type) { |
4525 | case e1000_82576: |
4526 | /* 82576 supports 2 RSS queues for SR-IOV */ |
4527 | if (adapter->vfs_allocated_count) |
4528 | num_rx_queues = 2; |
4529 | break; |
4530 | default: |
4531 | break; |
4532 | } |
4533 | |
4534 | if (adapter->rss_indir_tbl_init != num_rx_queues) { |
4535 | for (j = 0; j < IGB_RETA_SIZE; j++) |
4536 | adapter->rss_indir_tbl[j] = |
4537 | (j * num_rx_queues) / IGB_RETA_SIZE; |
4538 | adapter->rss_indir_tbl_init = num_rx_queues; |
4539 | } |
4540 | igb_write_rss_indir_tbl(adapter); |
4541 | |
4542 | /* Disable raw packet checksumming so that RSS hash is placed in |
4543 | * descriptor on writeback. No need to enable TCP/UDP/IP checksum |
4544 | * offloads as they are enabled by default |
4545 | */ |
4546 | rxcsum = rd32(E1000_RXCSUM); |
4547 | rxcsum |= E1000_RXCSUM_PCSD; |
4548 | |
4549 | if (adapter->hw.mac.type >= e1000_82576) |
4550 | /* Enable Receive Checksum Offload for SCTP */ |
4551 | rxcsum |= E1000_RXCSUM_CRCOFL; |
4552 | |
4553 | /* Don't need to set TUOFL or IPOFL, they default to 1 */ |
4554 | wr32(E1000_RXCSUM, rxcsum); |
4555 | |
4556 | /* Generate RSS hash based on packet types, TCP/UDP |
4557 | * port numbers and/or IPv4/v6 src and dst addresses |
4558 | */ |
4559 | mrqc = E1000_MRQC_RSS_FIELD_IPV4 | |
4560 | E1000_MRQC_RSS_FIELD_IPV4_TCP | |
4561 | E1000_MRQC_RSS_FIELD_IPV6 | |
4562 | E1000_MRQC_RSS_FIELD_IPV6_TCP | |
4563 | E1000_MRQC_RSS_FIELD_IPV6_TCP_EX; |
4564 | |
4565 | if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP) |
4566 | mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP; |
4567 | if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP) |
4568 | mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP; |
4569 | |
4570 | /* If VMDq is enabled then we set the appropriate mode for that, else |
4571 | * we default to RSS so that an RSS hash is calculated per packet even |
4572 | * if we are only using one queue |
4573 | */ |
4574 | if (adapter->vfs_allocated_count) { |
4575 | if (hw->mac.type > e1000_82575) { |
4576 | /* Set the default pool for the PF's first queue */ |
4577 | u32 vtctl = rd32(E1000_VT_CTL); |
4578 | |
4579 | vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK | |
4580 | E1000_VT_CTL_DISABLE_DEF_POOL); |
4581 | vtctl |= adapter->vfs_allocated_count << |
4582 | E1000_VT_CTL_DEFAULT_POOL_SHIFT; |
4583 | wr32(E1000_VT_CTL, vtctl); |
4584 | } |
4585 | if (adapter->rss_queues > 1) |
4586 | mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ; |
4587 | else |
4588 | mrqc |= E1000_MRQC_ENABLE_VMDQ; |
4589 | } else { |
4590 | mrqc |= E1000_MRQC_ENABLE_RSS_MQ; |
4591 | } |
4592 | igb_vmm_control(adapter); |
4593 | |
4594 | wr32(E1000_MRQC, mrqc); |
4595 | } |
4596 | |
4597 | /** |
4598 | * igb_setup_rctl - configure the receive control registers |
4599 | * @adapter: Board private structure |
4600 | **/ |
4601 | void igb_setup_rctl(struct igb_adapter *adapter) |
4602 | { |
4603 | struct e1000_hw *hw = &adapter->hw; |
4604 | u32 rctl; |
4605 | |
4606 | rctl = rd32(E1000_RCTL); |
4607 | |
4608 | rctl &= ~(3 << E1000_RCTL_MO_SHIFT); |
4609 | rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC); |
4610 | |
4611 | rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF | |
4612 | (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); |
4613 | |
4614 | /* enable stripping of CRC. It's unlikely this will break BMC |
4615 | * redirection as it did with e1000. Newer features require |
4616 | * that the HW strips the CRC. |
4617 | */ |
4618 | rctl |= E1000_RCTL_SECRC; |
4619 | |
4620 | /* disable store bad packets and clear size bits. */ |
4621 | rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256); |
4622 | |
4623 | /* enable LPE to allow for reception of jumbo frames */ |
4624 | rctl |= E1000_RCTL_LPE; |
4625 | |
4626 | /* disable queue 0 to prevent tail write w/o re-config */ |
4627 | wr32(E1000_RXDCTL(0), 0); |
4628 | |
4629 | /* Attention!!! For SR-IOV PF driver operations you must enable |
4630 | * queue drop for all VF and PF queues to prevent head of line blocking |
4631 | * if an un-trusted VF does not provide descriptors to hardware. |
4632 | */ |
4633 | if (adapter->vfs_allocated_count) { |
4634 | /* set all queue drop enable bits */ |
4635 | wr32(E1000_QDE, ALL_QUEUES); |
4636 | } |
4637 | |
4638 | /* This is useful for sniffing bad packets. */ |
4639 | if (adapter->netdev->features & NETIF_F_RXALL) { |
4640 | /* UPE and MPE will be handled by normal PROMISC logic |
4641 | * in e1000e_set_rx_mode |
4642 | */ |
4643 | rctl |= (E1000_RCTL_SBP | /* Receive bad packets */ |
4644 | E1000_RCTL_BAM | /* RX All Bcast Pkts */ |
4645 | E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */ |
4646 | |
4647 | rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */ |
4648 | E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */ |
4649 | /* Do not mess with E1000_CTRL_VME, it affects transmit as well, |
4650 | * and that breaks VLANs. |
4651 | */ |
4652 | } |
4653 | |
4654 | wr32(E1000_RCTL, rctl); |
4655 | } |
4656 | |
4657 | static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size, |
4658 | int vfn) |
4659 | { |
4660 | struct e1000_hw *hw = &adapter->hw; |
4661 | u32 vmolr; |
4662 | |
4663 | if (size > MAX_JUMBO_FRAME_SIZE) |
4664 | size = MAX_JUMBO_FRAME_SIZE; |
4665 | |
4666 | vmolr = rd32(E1000_VMOLR(vfn)); |
4667 | vmolr &= ~E1000_VMOLR_RLPML_MASK; |
4668 | vmolr |= size | E1000_VMOLR_LPE; |
4669 | wr32(E1000_VMOLR(vfn), vmolr); |
4670 | |
4671 | return 0; |
4672 | } |
4673 | |
4674 | static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter, |
4675 | int vfn, bool enable) |
4676 | { |
4677 | struct e1000_hw *hw = &adapter->hw; |
4678 | u32 val, reg; |
4679 | |
4680 | if (hw->mac.type < e1000_82576) |
4681 | return; |
4682 | |
4683 | if (hw->mac.type == e1000_i350) |
4684 | reg = E1000_DVMOLR(vfn); |
4685 | else |
4686 | reg = E1000_VMOLR(vfn); |
4687 | |
4688 | val = rd32(reg); |
4689 | if (enable) |
4690 | val |= E1000_VMOLR_STRVLAN; |
4691 | else |
4692 | val &= ~(E1000_VMOLR_STRVLAN); |
4693 | wr32(reg, val); |
4694 | } |
4695 | |
4696 | static inline void igb_set_vmolr(struct igb_adapter *adapter, |
4697 | int vfn, bool aupe) |
4698 | { |
4699 | struct e1000_hw *hw = &adapter->hw; |
4700 | u32 vmolr; |
4701 | |
4702 | /* This register exists only on 82576 and newer so if we are older then |
4703 | * we should exit and do nothing |
4704 | */ |
4705 | if (hw->mac.type < e1000_82576) |
4706 | return; |
4707 | |
4708 | vmolr = rd32(E1000_VMOLR(vfn)); |
4709 | if (aupe) |
4710 | vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */ |
4711 | else |
4712 | vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */ |
4713 | |
4714 | /* clear all bits that might not be set */ |
4715 | vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE); |
4716 | |
4717 | if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count) |
4718 | vmolr |= E1000_VMOLR_RSSE; /* enable RSS */ |
4719 | /* for VMDq only allow the VFs and pool 0 to accept broadcast and |
4720 | * multicast packets |
4721 | */ |
4722 | if (vfn <= adapter->vfs_allocated_count) |
4723 | vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */ |
4724 | |
4725 | wr32(E1000_VMOLR(vfn), vmolr); |
4726 | } |
4727 | |
4728 | /** |
4729 | * igb_setup_srrctl - configure the split and replication receive control |
4730 | * registers |
4731 | * @adapter: Board private structure |
4732 | * @ring: receive ring to be configured |
4733 | **/ |
4734 | void igb_setup_srrctl(struct igb_adapter *adapter, struct igb_ring *ring) |
4735 | { |
4736 | struct e1000_hw *hw = &adapter->hw; |
4737 | int reg_idx = ring->reg_idx; |
4738 | u32 srrctl = 0; |
4739 | |
4740 | srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; |
4741 | if (ring_uses_large_buffer(ring)) |
4742 | srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT; |
4743 | else |
4744 | srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; |
4745 | srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; |
4746 | if (hw->mac.type >= e1000_82580) |
4747 | srrctl |= E1000_SRRCTL_TIMESTAMP; |
4748 | /* Only set Drop Enable if VFs allocated, or we are supporting multiple |
4749 | * queues and rx flow control is disabled |
4750 | */ |
4751 | if (adapter->vfs_allocated_count || |
4752 | (!(hw->fc.current_mode & e1000_fc_rx_pause) && |
4753 | adapter->num_rx_queues > 1)) |
4754 | srrctl |= E1000_SRRCTL_DROP_EN; |
4755 | |
4756 | wr32(E1000_SRRCTL(reg_idx), srrctl); |
4757 | } |
4758 | |
4759 | /** |
4760 | * igb_configure_rx_ring - Configure a receive ring after Reset |
4761 | * @adapter: board private structure |
4762 | * @ring: receive ring to be configured |
4763 | * |
4764 | * Configure the Rx unit of the MAC after a reset. |
4765 | **/ |
4766 | void igb_configure_rx_ring(struct igb_adapter *adapter, |
4767 | struct igb_ring *ring) |
4768 | { |
4769 | struct e1000_hw *hw = &adapter->hw; |
4770 | union e1000_adv_rx_desc *rx_desc; |
4771 | u64 rdba = ring->dma; |
4772 | int reg_idx = ring->reg_idx; |
4773 | u32 rxdctl = 0; |
4774 | |
4775 | xdp_rxq_info_unreg_mem_model(xdp_rxq: &ring->xdp_rxq); |
4776 | WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, |
4777 | MEM_TYPE_PAGE_SHARED, NULL)); |
4778 | |
4779 | /* disable the queue */ |
4780 | wr32(E1000_RXDCTL(reg_idx), 0); |
4781 | |
4782 | /* Set DMA base address registers */ |
4783 | wr32(E1000_RDBAL(reg_idx), |
4784 | rdba & 0x00000000ffffffffULL); |
4785 | wr32(E1000_RDBAH(reg_idx), rdba >> 32); |
4786 | wr32(E1000_RDLEN(reg_idx), |
4787 | ring->count * sizeof(union e1000_adv_rx_desc)); |
4788 | |
4789 | /* initialize head and tail */ |
4790 | ring->tail = adapter->io_addr + E1000_RDT(reg_idx); |
4791 | wr32(E1000_RDH(reg_idx), 0); |
4792 | writel(val: 0, addr: ring->tail); |
4793 | |
4794 | /* set descriptor configuration */ |
4795 | igb_setup_srrctl(adapter, ring); |
4796 | |
4797 | /* set filtering for VMDQ pools */ |
4798 | igb_set_vmolr(adapter, vfn: reg_idx & 0x7, aupe: true); |
4799 | |
4800 | rxdctl |= IGB_RX_PTHRESH; |
4801 | rxdctl |= IGB_RX_HTHRESH << 8; |
4802 | rxdctl |= IGB_RX_WTHRESH << 16; |
4803 | |
4804 | /* initialize rx_buffer_info */ |
4805 | memset(ring->rx_buffer_info, 0, |
4806 | sizeof(struct igb_rx_buffer) * ring->count); |
4807 | |
4808 | /* initialize Rx descriptor 0 */ |
4809 | rx_desc = IGB_RX_DESC(ring, 0); |
4810 | rx_desc->wb.upper.length = 0; |
4811 | |
4812 | /* enable receive descriptor fetching */ |
4813 | rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; |
4814 | wr32(E1000_RXDCTL(reg_idx), rxdctl); |
4815 | } |
4816 | |
4817 | static void igb_set_rx_buffer_len(struct igb_adapter *adapter, |
4818 | struct igb_ring *rx_ring) |
4819 | { |
4820 | #if (PAGE_SIZE < 8192) |
4821 | struct e1000_hw *hw = &adapter->hw; |
4822 | #endif |
4823 | |
4824 | /* set build_skb and buffer size flags */ |
4825 | clear_ring_build_skb_enabled(rx_ring); |
4826 | clear_ring_uses_large_buffer(rx_ring); |
4827 | |
4828 | if (adapter->flags & IGB_FLAG_RX_LEGACY) |
4829 | return; |
4830 | |
4831 | set_ring_build_skb_enabled(rx_ring); |
4832 | |
4833 | #if (PAGE_SIZE < 8192) |
4834 | if (adapter->max_frame_size > IGB_MAX_FRAME_BUILD_SKB || |
4835 | rd32(E1000_RCTL) & E1000_RCTL_SBP) |
4836 | set_ring_uses_large_buffer(rx_ring); |
4837 | #endif |
4838 | } |
4839 | |
4840 | /** |
4841 | * igb_configure_rx - Configure receive Unit after Reset |
4842 | * @adapter: board private structure |
4843 | * |
4844 | * Configure the Rx unit of the MAC after a reset. |
4845 | **/ |
4846 | static void igb_configure_rx(struct igb_adapter *adapter) |
4847 | { |
4848 | int i; |
4849 | |
4850 | /* set the correct pool for the PF default MAC address in entry 0 */ |
4851 | igb_set_default_mac_filter(adapter); |
4852 | |
4853 | /* Setup the HW Rx Head and Tail Descriptor Pointers and |
4854 | * the Base and Length of the Rx Descriptor Ring |
4855 | */ |
4856 | for (i = 0; i < adapter->num_rx_queues; i++) { |
4857 | struct igb_ring *rx_ring = adapter->rx_ring[i]; |
4858 | |
4859 | igb_set_rx_buffer_len(adapter, rx_ring); |
4860 | igb_configure_rx_ring(adapter, ring: rx_ring); |
4861 | } |
4862 | } |
4863 | |
4864 | /** |
4865 | * igb_free_tx_resources - Free Tx Resources per Queue |
4866 | * @tx_ring: Tx descriptor ring for a specific queue |
4867 | * |
4868 | * Free all transmit software resources |
4869 | **/ |
4870 | void igb_free_tx_resources(struct igb_ring *tx_ring) |
4871 | { |
4872 | igb_clean_tx_ring(tx_ring); |
4873 | |
4874 | vfree(addr: tx_ring->tx_buffer_info); |
4875 | tx_ring->tx_buffer_info = NULL; |
4876 | |
4877 | /* if not set, then don't free */ |
4878 | if (!tx_ring->desc) |
4879 | return; |
4880 | |
4881 | dma_free_coherent(dev: tx_ring->dev, size: tx_ring->size, |
4882 | cpu_addr: tx_ring->desc, dma_handle: tx_ring->dma); |
4883 | |
4884 | tx_ring->desc = NULL; |
4885 | } |
4886 | |
4887 | /** |
4888 | * igb_free_all_tx_resources - Free Tx Resources for All Queues |
4889 | * @adapter: board private structure |
4890 | * |
4891 | * Free all transmit software resources |
4892 | **/ |
4893 | static void igb_free_all_tx_resources(struct igb_adapter *adapter) |
4894 | { |
4895 | int i; |
4896 | |
4897 | for (i = 0; i < adapter->num_tx_queues; i++) |
4898 | if (adapter->tx_ring[i]) |
4899 | igb_free_tx_resources(tx_ring: adapter->tx_ring[i]); |
4900 | } |
4901 | |
4902 | /** |
4903 | * igb_clean_tx_ring - Free Tx Buffers |
4904 | * @tx_ring: ring to be cleaned |
4905 | **/ |
4906 | static void igb_clean_tx_ring(struct igb_ring *tx_ring) |
4907 | { |
4908 | u16 i = tx_ring->next_to_clean; |
4909 | struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; |
4910 | |
4911 | while (i != tx_ring->next_to_use) { |
4912 | union e1000_adv_tx_desc *eop_desc, *tx_desc; |
4913 | |
4914 | /* Free all the Tx ring sk_buffs or xdp frames */ |
4915 | if (tx_buffer->type == IGB_TYPE_SKB) |
4916 | dev_kfree_skb_any(skb: tx_buffer->skb); |
4917 | else |
4918 | xdp_return_frame(xdpf: tx_buffer->xdpf); |
4919 | |
4920 | /* unmap skb header data */ |
4921 | dma_unmap_single(tx_ring->dev, |
4922 | dma_unmap_addr(tx_buffer, dma), |
4923 | dma_unmap_len(tx_buffer, len), |
4924 | DMA_TO_DEVICE); |
4925 | |
4926 | /* check for eop_desc to determine the end of the packet */ |
4927 | eop_desc = tx_buffer->next_to_watch; |
4928 | tx_desc = IGB_TX_DESC(tx_ring, i); |
4929 | |
4930 | /* unmap remaining buffers */ |
4931 | while (tx_desc != eop_desc) { |
4932 | tx_buffer++; |
4933 | tx_desc++; |
4934 | i++; |
4935 | if (unlikely(i == tx_ring->count)) { |
4936 | i = 0; |
4937 | tx_buffer = tx_ring->tx_buffer_info; |
4938 | tx_desc = IGB_TX_DESC(tx_ring, 0); |
4939 | } |
4940 | |
4941 | /* unmap any remaining paged data */ |
4942 | if (dma_unmap_len(tx_buffer, len)) |
4943 | dma_unmap_page(tx_ring->dev, |
4944 | dma_unmap_addr(tx_buffer, dma), |
4945 | dma_unmap_len(tx_buffer, len), |
4946 | DMA_TO_DEVICE); |
4947 | } |
4948 | |
4949 | tx_buffer->next_to_watch = NULL; |
4950 | |
4951 | /* move us one more past the eop_desc for start of next pkt */ |
4952 | tx_buffer++; |
4953 | i++; |
4954 | if (unlikely(i == tx_ring->count)) { |
4955 | i = 0; |
4956 | tx_buffer = tx_ring->tx_buffer_info; |
4957 | } |
4958 | } |
4959 | |
4960 | /* reset BQL for queue */ |
4961 | netdev_tx_reset_queue(q: txring_txq(tx_ring)); |
4962 | |
4963 | /* reset next_to_use and next_to_clean */ |
4964 | tx_ring->next_to_use = 0; |
4965 | tx_ring->next_to_clean = 0; |
4966 | } |
4967 | |
4968 | /** |
4969 | * igb_clean_all_tx_rings - Free Tx Buffers for all queues |
4970 | * @adapter: board private structure |
4971 | **/ |
4972 | static void igb_clean_all_tx_rings(struct igb_adapter *adapter) |
4973 | { |
4974 | int i; |
4975 | |
4976 | for (i = 0; i < adapter->num_tx_queues; i++) |
4977 | if (adapter->tx_ring[i]) |
4978 | igb_clean_tx_ring(tx_ring: adapter->tx_ring[i]); |
4979 | } |
4980 | |
4981 | /** |
4982 | * igb_free_rx_resources - Free Rx Resources |
4983 | * @rx_ring: ring to clean the resources from |
4984 | * |
4985 | * Free all receive software resources |
4986 | **/ |
4987 | void igb_free_rx_resources(struct igb_ring *rx_ring) |
4988 | { |
4989 | igb_clean_rx_ring(rx_ring); |
4990 | |
4991 | rx_ring->xdp_prog = NULL; |
4992 | xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq); |
4993 | vfree(addr: rx_ring->rx_buffer_info); |
4994 | rx_ring->rx_buffer_info = NULL; |
4995 | |
4996 | /* if not set, then don't free */ |
4997 | if (!rx_ring->desc) |
4998 | return; |
4999 | |
5000 | dma_free_coherent(dev: rx_ring->dev, size: rx_ring->size, |
5001 | cpu_addr: rx_ring->desc, dma_handle: rx_ring->dma); |
5002 | |
5003 | rx_ring->desc = NULL; |
5004 | } |
5005 | |
5006 | /** |
5007 | * igb_free_all_rx_resources - Free Rx Resources for All Queues |
5008 | * @adapter: board private structure |
5009 | * |
5010 | * Free all receive software resources |
5011 | **/ |
5012 | static void igb_free_all_rx_resources(struct igb_adapter *adapter) |
5013 | { |
5014 | int i; |
5015 | |
5016 | for (i = 0; i < adapter->num_rx_queues; i++) |
5017 | if (adapter->rx_ring[i]) |
5018 | igb_free_rx_resources(rx_ring: adapter->rx_ring[i]); |
5019 | } |
5020 | |
5021 | /** |
5022 | * igb_clean_rx_ring - Free Rx Buffers per Queue |
5023 | * @rx_ring: ring to free buffers from |
5024 | **/ |
5025 | static void igb_clean_rx_ring(struct igb_ring *rx_ring) |
5026 | { |
5027 | u16 i = rx_ring->next_to_clean; |
5028 | |
5029 | dev_kfree_skb(rx_ring->skb); |
5030 | rx_ring->skb = NULL; |
5031 | |
5032 | /* Free all the Rx ring sk_buffs */ |
5033 | while (i != rx_ring->next_to_alloc) { |
5034 | struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i]; |
5035 | |
5036 | /* Invalidate cache lines that may have been written to by |
5037 | * device so that we avoid corrupting memory. |
5038 | */ |
5039 | dma_sync_single_range_for_cpu(dev: rx_ring->dev, |
5040 | addr: buffer_info->dma, |
5041 | offset: buffer_info->page_offset, |
5042 | size: igb_rx_bufsz(ring: rx_ring), |
5043 | dir: DMA_FROM_DEVICE); |
5044 | |
5045 | /* free resources associated with mapping */ |
5046 | dma_unmap_page_attrs(dev: rx_ring->dev, |
5047 | addr: buffer_info->dma, |
5048 | igb_rx_pg_size(rx_ring), |
5049 | dir: DMA_FROM_DEVICE, |
5050 | IGB_RX_DMA_ATTR); |
5051 | __page_frag_cache_drain(page: buffer_info->page, |
5052 | count: buffer_info->pagecnt_bias); |
5053 | |
5054 | i++; |
5055 | if (i == rx_ring->count) |
5056 | i = 0; |
5057 | } |
5058 | |
5059 | rx_ring->next_to_alloc = 0; |
5060 | rx_ring->next_to_clean = 0; |
5061 | rx_ring->next_to_use = 0; |
5062 | } |
5063 | |
5064 | /** |
5065 | * igb_clean_all_rx_rings - Free Rx Buffers for all queues |
5066 | * @adapter: board private structure |
5067 | **/ |
5068 | static void igb_clean_all_rx_rings(struct igb_adapter *adapter) |
5069 | { |
5070 | int i; |
5071 | |
5072 | for (i = 0; i < adapter->num_rx_queues; i++) |
5073 | if (adapter->rx_ring[i]) |
5074 | igb_clean_rx_ring(rx_ring: adapter->rx_ring[i]); |
5075 | } |
5076 | |
5077 | /** |
5078 | * igb_set_mac - Change the Ethernet Address of the NIC |
5079 | * @netdev: network interface device structure |
5080 | * @p: pointer to an address structure |
5081 | * |
5082 | * Returns 0 on success, negative on failure |
5083 | **/ |
5084 | static int igb_set_mac(struct net_device *netdev, void *p) |
5085 | { |
5086 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
5087 | struct e1000_hw *hw = &adapter->hw; |
5088 | struct sockaddr *addr = p; |
5089 | |
5090 | if (!is_valid_ether_addr(addr: addr->sa_data)) |
5091 | return -EADDRNOTAVAIL; |
5092 | |
5093 | eth_hw_addr_set(dev: netdev, addr: addr->sa_data); |
5094 | memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); |
5095 | |
5096 | /* set the correct pool for the new PF MAC address in entry 0 */ |
5097 | igb_set_default_mac_filter(adapter); |
5098 | |
5099 | return 0; |
5100 | } |
5101 | |
5102 | /** |
5103 | * igb_write_mc_addr_list - write multicast addresses to MTA |
5104 | * @netdev: network interface device structure |
5105 | * |
5106 | * Writes multicast address list to the MTA hash table. |
5107 | * Returns: -ENOMEM on failure |
5108 | * 0 on no addresses written |
5109 | * X on writing X addresses to MTA |
5110 | **/ |
5111 | static int igb_write_mc_addr_list(struct net_device *netdev) |
5112 | { |
5113 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
5114 | struct e1000_hw *hw = &adapter->hw; |
5115 | struct netdev_hw_addr *ha; |
5116 | u8 *mta_list; |
5117 | int i; |
5118 | |
5119 | if (netdev_mc_empty(netdev)) { |
5120 | /* nothing to program, so clear mc list */ |
5121 | igb_update_mc_addr_list(hw, NULL, mc_addr_count: 0); |
5122 | igb_restore_vf_multicasts(adapter); |
5123 | return 0; |
5124 | } |
5125 | |
5126 | mta_list = kcalloc(netdev_mc_count(netdev), size: 6, GFP_ATOMIC); |
5127 | if (!mta_list) |
5128 | return -ENOMEM; |
5129 | |
5130 | /* The shared function expects a packed array of only addresses. */ |
5131 | i = 0; |
5132 | netdev_for_each_mc_addr(ha, netdev) |
5133 | memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN); |
5134 | |
5135 | igb_update_mc_addr_list(hw, mc_addr_list: mta_list, mc_addr_count: i); |
5136 | kfree(objp: mta_list); |
5137 | |
5138 | return netdev_mc_count(netdev); |
5139 | } |
5140 | |
5141 | static int igb_vlan_promisc_enable(struct igb_adapter *adapter) |
5142 | { |
5143 | struct e1000_hw *hw = &adapter->hw; |
5144 | u32 i, pf_id; |
5145 | |
5146 | switch (hw->mac.type) { |
5147 | case e1000_i210: |
5148 | case e1000_i211: |
5149 | case e1000_i350: |
5150 | /* VLAN filtering needed for VLAN prio filter */ |
5151 | if (adapter->netdev->features & NETIF_F_NTUPLE) |
5152 | break; |
5153 | fallthrough; |
5154 | case e1000_82576: |
5155 | case e1000_82580: |
5156 | case e1000_i354: |
5157 | /* VLAN filtering needed for pool filtering */ |
5158 | if (adapter->vfs_allocated_count) |
5159 | break; |
5160 | fallthrough; |
5161 | default: |
5162 | return 1; |
5163 | } |
5164 | |
5165 | /* We are already in VLAN promisc, nothing to do */ |
5166 | if (adapter->flags & IGB_FLAG_VLAN_PROMISC) |
5167 | return 0; |
5168 | |
5169 | if (!adapter->vfs_allocated_count) |
5170 | goto set_vfta; |
5171 | |
5172 | /* Add PF to all active pools */ |
5173 | pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT; |
5174 | |
5175 | for (i = E1000_VLVF_ARRAY_SIZE; --i;) { |
5176 | u32 vlvf = rd32(E1000_VLVF(i)); |
5177 | |
5178 | vlvf |= BIT(pf_id); |
5179 | wr32(E1000_VLVF(i), vlvf); |
5180 | } |
5181 | |
5182 | set_vfta: |
5183 | /* Set all bits in the VLAN filter table array */ |
5184 | for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;) |
5185 | hw->mac.ops.write_vfta(hw, i, ~0U); |
5186 | |
5187 | /* Set flag so we don't redo unnecessary work */ |
5188 | adapter->flags |= IGB_FLAG_VLAN_PROMISC; |
5189 | |
5190 | return 0; |
5191 | } |
5192 | |
5193 | #define VFTA_BLOCK_SIZE 8 |
5194 | static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset) |
5195 | { |
5196 | struct e1000_hw *hw = &adapter->hw; |
5197 | u32 vfta[VFTA_BLOCK_SIZE] = { 0 }; |
5198 | u32 vid_start = vfta_offset * 32; |
5199 | u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32); |
5200 | u32 i, vid, word, bits, pf_id; |
5201 | |
5202 | /* guarantee that we don't scrub out management VLAN */ |
5203 | vid = adapter->mng_vlan_id; |
5204 | if (vid >= vid_start && vid < vid_end) |
5205 | vfta[(vid - vid_start) / 32] |= BIT(vid % 32); |
5206 | |
5207 | if (!adapter->vfs_allocated_count) |
5208 | goto set_vfta; |
5209 | |
5210 | pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT; |
5211 | |
5212 | for (i = E1000_VLVF_ARRAY_SIZE; --i;) { |
5213 | u32 vlvf = rd32(E1000_VLVF(i)); |
5214 | |
5215 | /* pull VLAN ID from VLVF */ |
5216 | vid = vlvf & VLAN_VID_MASK; |
5217 | |
5218 | /* only concern ourselves with a certain range */ |
5219 | if (vid < vid_start || vid >= vid_end) |
5220 | continue; |
5221 | |
5222 | if (vlvf & E1000_VLVF_VLANID_ENABLE) { |
5223 | /* record VLAN ID in VFTA */ |
5224 | vfta[(vid - vid_start) / 32] |= BIT(vid % 32); |
5225 | |
5226 | /* if PF is part of this then continue */ |
5227 | if (test_bit(vid, adapter->active_vlans)) |
5228 | continue; |
5229 | } |
5230 | |
5231 | /* remove PF from the pool */ |
5232 | bits = ~BIT(pf_id); |
5233 | bits &= rd32(E1000_VLVF(i)); |
5234 | wr32(E1000_VLVF(i), bits); |
5235 | } |
5236 | |
5237 | set_vfta: |
5238 | /* extract values from active_vlans and write back to VFTA */ |
5239 | for (i = VFTA_BLOCK_SIZE; i--;) { |
5240 | vid = (vfta_offset + i) * 32; |
5241 | word = vid / BITS_PER_LONG; |
5242 | bits = vid % BITS_PER_LONG; |
5243 | |
5244 | vfta[i] |= adapter->active_vlans[word] >> bits; |
5245 | |
5246 | hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]); |
5247 | } |
5248 | } |
5249 | |
5250 | static void igb_vlan_promisc_disable(struct igb_adapter *adapter) |
5251 | { |
5252 | u32 i; |
5253 | |
5254 | /* We are not in VLAN promisc, nothing to do */ |
5255 | if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC)) |
5256 | return; |
5257 | |
5258 | /* Set flag so we don't redo unnecessary work */ |
5259 | adapter->flags &= ~IGB_FLAG_VLAN_PROMISC; |
5260 | |
5261 | for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE) |
5262 | igb_scrub_vfta(adapter, vfta_offset: i); |
5263 | } |
5264 | |
5265 | /** |
5266 | * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set |
5267 | * @netdev: network interface device structure |
5268 | * |
5269 | * The set_rx_mode entry point is called whenever the unicast or multicast |
5270 | * address lists or the network interface flags are updated. This routine is |
5271 | * responsible for configuring the hardware for proper unicast, multicast, |
5272 | * promiscuous mode, and all-multi behavior. |
5273 | **/ |
5274 | static void igb_set_rx_mode(struct net_device *netdev) |
5275 | { |
5276 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
5277 | struct e1000_hw *hw = &adapter->hw; |
5278 | unsigned int vfn = adapter->vfs_allocated_count; |
5279 | u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE; |
5280 | int count; |
5281 | |
5282 | /* Check for Promiscuous and All Multicast modes */ |
5283 | if (netdev->flags & IFF_PROMISC) { |
5284 | rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE; |
5285 | vmolr |= E1000_VMOLR_MPME; |
5286 | |
5287 | /* enable use of UTA filter to force packets to default pool */ |
5288 | if (hw->mac.type == e1000_82576) |
5289 | vmolr |= E1000_VMOLR_ROPE; |
5290 | } else { |
5291 | if (netdev->flags & IFF_ALLMULTI) { |
5292 | rctl |= E1000_RCTL_MPE; |
5293 | vmolr |= E1000_VMOLR_MPME; |
5294 | } else { |
5295 | /* Write addresses to the MTA, if the attempt fails |
5296 | * then we should just turn on promiscuous mode so |
5297 | * that we can at least receive multicast traffic |
5298 | */ |
5299 | count = igb_write_mc_addr_list(netdev); |
5300 | if (count < 0) { |
5301 | rctl |= E1000_RCTL_MPE; |
5302 | vmolr |= E1000_VMOLR_MPME; |
5303 | } else if (count) { |
5304 | vmolr |= E1000_VMOLR_ROMPE; |
5305 | } |
5306 | } |
5307 | } |
5308 | |
5309 | /* Write addresses to available RAR registers, if there is not |
5310 | * sufficient space to store all the addresses then enable |
5311 | * unicast promiscuous mode |
5312 | */ |
5313 | if (__dev_uc_sync(dev: netdev, sync: igb_uc_sync, unsync: igb_uc_unsync)) { |
5314 | rctl |= E1000_RCTL_UPE; |
5315 | vmolr |= E1000_VMOLR_ROPE; |
5316 | } |
5317 | |
5318 | /* enable VLAN filtering by default */ |
5319 | rctl |= E1000_RCTL_VFE; |
5320 | |
5321 | /* disable VLAN filtering for modes that require it */ |
5322 | if ((netdev->flags & IFF_PROMISC) || |
5323 | (netdev->features & NETIF_F_RXALL)) { |
5324 | /* if we fail to set all rules then just clear VFE */ |
5325 | if (igb_vlan_promisc_enable(adapter)) |
5326 | rctl &= ~E1000_RCTL_VFE; |
5327 | } else { |
5328 | igb_vlan_promisc_disable(adapter); |
5329 | } |
5330 | |
5331 | /* update state of unicast, multicast, and VLAN filtering modes */ |
5332 | rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE | |
5333 | E1000_RCTL_VFE); |
5334 | wr32(E1000_RCTL, rctl); |
5335 | |
5336 | #if (PAGE_SIZE < 8192) |
5337 | if (!adapter->vfs_allocated_count) { |
5338 | if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB) |
5339 | rlpml = IGB_MAX_FRAME_BUILD_SKB; |
5340 | } |
5341 | #endif |
5342 | wr32(E1000_RLPML, rlpml); |
5343 | |
5344 | /* In order to support SR-IOV and eventually VMDq it is necessary to set |
5345 | * the VMOLR to enable the appropriate modes. Without this workaround |
5346 | * we will have issues with VLAN tag stripping not being done for frames |
5347 | * that are only arriving because we are the default pool |
5348 | */ |
5349 | if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350)) |
5350 | return; |
5351 | |
5352 | /* set UTA to appropriate mode */ |
5353 | igb_set_uta(adapter, set: !!(vmolr & E1000_VMOLR_ROPE)); |
5354 | |
5355 | vmolr |= rd32(E1000_VMOLR(vfn)) & |
5356 | ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE); |
5357 | |
5358 | /* enable Rx jumbo frames, restrict as needed to support build_skb */ |
5359 | vmolr &= ~E1000_VMOLR_RLPML_MASK; |
5360 | #if (PAGE_SIZE < 8192) |
5361 | if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB) |
5362 | vmolr |= IGB_MAX_FRAME_BUILD_SKB; |
5363 | else |
5364 | #endif |
5365 | vmolr |= MAX_JUMBO_FRAME_SIZE; |
5366 | vmolr |= E1000_VMOLR_LPE; |
5367 | |
5368 | wr32(E1000_VMOLR(vfn), vmolr); |
5369 | |
5370 | igb_restore_vf_multicasts(adapter); |
5371 | } |
5372 | |
5373 | static void igb_check_wvbr(struct igb_adapter *adapter) |
5374 | { |
5375 | struct e1000_hw *hw = &adapter->hw; |
5376 | u32 wvbr = 0; |
5377 | |
5378 | switch (hw->mac.type) { |
5379 | case e1000_82576: |
5380 | case e1000_i350: |
5381 | wvbr = rd32(E1000_WVBR); |
5382 | if (!wvbr) |
5383 | return; |
5384 | break; |
5385 | default: |
5386 | break; |
5387 | } |
5388 | |
5389 | adapter->wvbr |= wvbr; |
5390 | } |
5391 | |
5392 | #define IGB_STAGGERED_QUEUE_OFFSET 8 |
5393 | |
5394 | static void igb_spoof_check(struct igb_adapter *adapter) |
5395 | { |
5396 | int j; |
5397 | |
5398 | if (!adapter->wvbr) |
5399 | return; |
5400 | |
5401 | for (j = 0; j < adapter->vfs_allocated_count; j++) { |
5402 | if (adapter->wvbr & BIT(j) || |
5403 | adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) { |
5404 | dev_warn(&adapter->pdev->dev, |
5405 | "Spoof event(s) detected on VF %d\n" , j); |
5406 | adapter->wvbr &= |
5407 | ~(BIT(j) | |
5408 | BIT(j + IGB_STAGGERED_QUEUE_OFFSET)); |
5409 | } |
5410 | } |
5411 | } |
5412 | |
5413 | /* Need to wait a few seconds after link up to get diagnostic information from |
5414 | * the phy |
5415 | */ |
5416 | static void igb_update_phy_info(struct timer_list *t) |
5417 | { |
5418 | struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer); |
5419 | igb_get_phy_info(hw: &adapter->hw); |
5420 | } |
5421 | |
5422 | /** |
5423 | * igb_has_link - check shared code for link and determine up/down |
5424 | * @adapter: pointer to driver private info |
5425 | **/ |
5426 | bool igb_has_link(struct igb_adapter *adapter) |
5427 | { |
5428 | struct e1000_hw *hw = &adapter->hw; |
5429 | bool link_active = false; |
5430 | |
5431 | /* get_link_status is set on LSC (link status) interrupt or |
5432 | * rx sequence error interrupt. get_link_status will stay |
5433 | * false until the e1000_check_for_link establishes link |
5434 | * for copper adapters ONLY |
5435 | */ |
5436 | switch (hw->phy.media_type) { |
5437 | case e1000_media_type_copper: |
5438 | if (!hw->mac.get_link_status) |
5439 | return true; |
5440 | fallthrough; |
5441 | case e1000_media_type_internal_serdes: |
5442 | hw->mac.ops.check_for_link(hw); |
5443 | link_active = !hw->mac.get_link_status; |
5444 | break; |
5445 | default: |
5446 | case e1000_media_type_unknown: |
5447 | break; |
5448 | } |
5449 | |
5450 | if (((hw->mac.type == e1000_i210) || |
5451 | (hw->mac.type == e1000_i211)) && |
5452 | (hw->phy.id == I210_I_PHY_ID)) { |
5453 | if (!netif_carrier_ok(dev: adapter->netdev)) { |
5454 | adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE; |
5455 | } else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) { |
5456 | adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE; |
5457 | adapter->link_check_timeout = jiffies; |
5458 | } |
5459 | } |
5460 | |
5461 | return link_active; |
5462 | } |
5463 | |
5464 | static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event) |
5465 | { |
5466 | bool ret = false; |
5467 | u32 ctrl_ext, thstat; |
5468 | |
5469 | /* check for thermal sensor event on i350 copper only */ |
5470 | if (hw->mac.type == e1000_i350) { |
5471 | thstat = rd32(E1000_THSTAT); |
5472 | ctrl_ext = rd32(E1000_CTRL_EXT); |
5473 | |
5474 | if ((hw->phy.media_type == e1000_media_type_copper) && |
5475 | !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) |
5476 | ret = !!(thstat & event); |
5477 | } |
5478 | |
5479 | return ret; |
5480 | } |
5481 | |
5482 | /** |
5483 | * igb_check_lvmmc - check for malformed packets received |
5484 | * and indicated in LVMMC register |
5485 | * @adapter: pointer to adapter |
5486 | **/ |
5487 | static void igb_check_lvmmc(struct igb_adapter *adapter) |
5488 | { |
5489 | struct e1000_hw *hw = &adapter->hw; |
5490 | u32 lvmmc; |
5491 | |
5492 | lvmmc = rd32(E1000_LVMMC); |
5493 | if (lvmmc) { |
5494 | if (unlikely(net_ratelimit())) { |
5495 | netdev_warn(dev: adapter->netdev, |
5496 | format: "malformed Tx packet detected and dropped, LVMMC:0x%08x\n" , |
5497 | lvmmc); |
5498 | } |
5499 | } |
5500 | } |
5501 | |
5502 | /** |
5503 | * igb_watchdog - Timer Call-back |
5504 | * @t: pointer to timer_list containing our private info pointer |
5505 | **/ |
5506 | static void igb_watchdog(struct timer_list *t) |
5507 | { |
5508 | struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer); |
5509 | /* Do the rest outside of interrupt context */ |
5510 | schedule_work(work: &adapter->watchdog_task); |
5511 | } |
5512 | |
5513 | static void igb_watchdog_task(struct work_struct *work) |
5514 | { |
5515 | struct igb_adapter *adapter = container_of(work, |
5516 | struct igb_adapter, |
5517 | watchdog_task); |
5518 | struct e1000_hw *hw = &adapter->hw; |
5519 | struct e1000_phy_info *phy = &hw->phy; |
5520 | struct net_device *netdev = adapter->netdev; |
5521 | u32 link; |
5522 | int i; |
5523 | u32 connsw; |
5524 | u16 phy_data, retry_count = 20; |
5525 | |
5526 | link = igb_has_link(adapter); |
5527 | |
5528 | if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) { |
5529 | if (time_after(jiffies, (adapter->link_check_timeout + HZ))) |
5530 | adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE; |
5531 | else |
5532 | link = false; |
5533 | } |
5534 | |
5535 | /* Force link down if we have fiber to swap to */ |
5536 | if (adapter->flags & IGB_FLAG_MAS_ENABLE) { |
5537 | if (hw->phy.media_type == e1000_media_type_copper) { |
5538 | connsw = rd32(E1000_CONNSW); |
5539 | if (!(connsw & E1000_CONNSW_AUTOSENSE_EN)) |
5540 | link = 0; |
5541 | } |
5542 | } |
5543 | if (link) { |
5544 | /* Perform a reset if the media type changed. */ |
5545 | if (hw->dev_spec._82575.media_changed) { |
5546 | hw->dev_spec._82575.media_changed = false; |
5547 | adapter->flags |= IGB_FLAG_MEDIA_RESET; |
5548 | igb_reset(adapter); |
5549 | } |
5550 | /* Cancel scheduled suspend requests. */ |
5551 | pm_runtime_resume(dev: netdev->dev.parent); |
5552 | |
5553 | if (!netif_carrier_ok(dev: netdev)) { |
5554 | u32 ctrl; |
5555 | |
5556 | hw->mac.ops.get_speed_and_duplex(hw, |
5557 | &adapter->link_speed, |
5558 | &adapter->link_duplex); |
5559 | |
5560 | ctrl = rd32(E1000_CTRL); |
5561 | /* Links status message must follow this format */ |
5562 | netdev_info(dev: netdev, |
5563 | format: "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n" , |
5564 | netdev->name, |
5565 | adapter->link_speed, |
5566 | adapter->link_duplex == FULL_DUPLEX ? |
5567 | "Full" : "Half" , |
5568 | (ctrl & E1000_CTRL_TFCE) && |
5569 | (ctrl & E1000_CTRL_RFCE) ? "RX/TX" : |
5570 | (ctrl & E1000_CTRL_RFCE) ? "RX" : |
5571 | (ctrl & E1000_CTRL_TFCE) ? "TX" : "None" ); |
5572 | |
5573 | /* disable EEE if enabled */ |
5574 | if ((adapter->flags & IGB_FLAG_EEE) && |
5575 | (adapter->link_duplex == HALF_DUPLEX)) { |
5576 | dev_info(&adapter->pdev->dev, |
5577 | "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n" ); |
5578 | adapter->hw.dev_spec._82575.eee_disable = true; |
5579 | adapter->flags &= ~IGB_FLAG_EEE; |
5580 | } |
5581 | |
5582 | /* check if SmartSpeed worked */ |
5583 | igb_check_downshift(hw); |
5584 | if (phy->speed_downgraded) |
5585 | netdev_warn(dev: netdev, format: "Link Speed was downgraded by SmartSpeed\n" ); |
5586 | |
5587 | /* check for thermal sensor event */ |
5588 | if (igb_thermal_sensor_event(hw, |
5589 | E1000_THSTAT_LINK_THROTTLE)) |
5590 | netdev_info(dev: netdev, format: "The network adapter link speed was downshifted because it overheated\n" ); |
5591 | |
5592 | /* adjust timeout factor according to speed/duplex */ |
5593 | adapter->tx_timeout_factor = 1; |
5594 | switch (adapter->link_speed) { |
5595 | case SPEED_10: |
5596 | adapter->tx_timeout_factor = 14; |
5597 | break; |
5598 | case SPEED_100: |
5599 | /* maybe add some timeout factor ? */ |
5600 | break; |
5601 | } |
5602 | |
5603 | if (adapter->link_speed != SPEED_1000 || |
5604 | !hw->phy.ops.read_reg) |
5605 | goto no_wait; |
5606 | |
5607 | /* wait for Remote receiver status OK */ |
5608 | retry_read_status: |
5609 | if (!igb_read_phy_reg(hw, PHY_1000T_STATUS, |
5610 | data: &phy_data)) { |
5611 | if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && |
5612 | retry_count) { |
5613 | msleep(msecs: 100); |
5614 | retry_count--; |
5615 | goto retry_read_status; |
5616 | } else if (!retry_count) { |
5617 | dev_err(&adapter->pdev->dev, "exceed max 2 second\n" ); |
5618 | } |
5619 | } else { |
5620 | dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n" ); |
5621 | } |
5622 | no_wait: |
5623 | netif_carrier_on(dev: netdev); |
5624 | |
5625 | igb_ping_all_vfs(adapter); |
5626 | igb_check_vf_rate_limit(adapter); |
5627 | |
5628 | /* link state has changed, schedule phy info update */ |
5629 | if (!test_bit(__IGB_DOWN, &adapter->state)) |
5630 | mod_timer(timer: &adapter->phy_info_timer, |
5631 | expires: round_jiffies(j: jiffies + 2 * HZ)); |
5632 | } |
5633 | } else { |
5634 | if (netif_carrier_ok(dev: netdev)) { |
5635 | adapter->link_speed = 0; |
5636 | adapter->link_duplex = 0; |
5637 | |
5638 | /* check for thermal sensor event */ |
5639 | if (igb_thermal_sensor_event(hw, |
5640 | E1000_THSTAT_PWR_DOWN)) { |
5641 | netdev_err(dev: netdev, format: "The network adapter was stopped because it overheated\n" ); |
5642 | } |
5643 | |
5644 | /* Links status message must follow this format */ |
5645 | netdev_info(dev: netdev, format: "igb: %s NIC Link is Down\n" , |
5646 | netdev->name); |
5647 | netif_carrier_off(dev: netdev); |
5648 | |
5649 | igb_ping_all_vfs(adapter); |
5650 | |
5651 | /* link state has changed, schedule phy info update */ |
5652 | if (!test_bit(__IGB_DOWN, &adapter->state)) |
5653 | mod_timer(timer: &adapter->phy_info_timer, |
5654 | expires: round_jiffies(j: jiffies + 2 * HZ)); |
5655 | |
5656 | /* link is down, time to check for alternate media */ |
5657 | if (adapter->flags & IGB_FLAG_MAS_ENABLE) { |
5658 | igb_check_swap_media(adapter); |
5659 | if (adapter->flags & IGB_FLAG_MEDIA_RESET) { |
5660 | schedule_work(work: &adapter->reset_task); |
5661 | /* return immediately */ |
5662 | return; |
5663 | } |
5664 | } |
5665 | pm_schedule_suspend(dev: netdev->dev.parent, |
5666 | MSEC_PER_SEC * 5); |
5667 | |
5668 | /* also check for alternate media here */ |
5669 | } else if (!netif_carrier_ok(dev: netdev) && |
5670 | (adapter->flags & IGB_FLAG_MAS_ENABLE)) { |
5671 | igb_check_swap_media(adapter); |
5672 | if (adapter->flags & IGB_FLAG_MEDIA_RESET) { |
5673 | schedule_work(work: &adapter->reset_task); |
5674 | /* return immediately */ |
5675 | return; |
5676 | } |
5677 | } |
5678 | } |
5679 | |
5680 | spin_lock(lock: &adapter->stats64_lock); |
5681 | igb_update_stats(adapter); |
5682 | spin_unlock(lock: &adapter->stats64_lock); |
5683 | |
5684 | for (i = 0; i < adapter->num_tx_queues; i++) { |
5685 | struct igb_ring *tx_ring = adapter->tx_ring[i]; |
5686 | if (!netif_carrier_ok(dev: netdev)) { |
5687 | /* We've lost link, so the controller stops DMA, |
5688 | * but we've got queued Tx work that's never going |
5689 | * to get done, so reset controller to flush Tx. |
5690 | * (Do the reset outside of interrupt context). |
5691 | */ |
5692 | if (igb_desc_unused(ring: tx_ring) + 1 < tx_ring->count) { |
5693 | adapter->tx_timeout_count++; |
5694 | schedule_work(work: &adapter->reset_task); |
5695 | /* return immediately since reset is imminent */ |
5696 | return; |
5697 | } |
5698 | } |
5699 | |
5700 | /* Force detection of hung controller every watchdog period */ |
5701 | set_bit(nr: IGB_RING_FLAG_TX_DETECT_HANG, addr: &tx_ring->flags); |
5702 | } |
5703 | |
5704 | /* Cause software interrupt to ensure Rx ring is cleaned */ |
5705 | if (adapter->flags & IGB_FLAG_HAS_MSIX) { |
5706 | u32 eics = 0; |
5707 | |
5708 | for (i = 0; i < adapter->num_q_vectors; i++) |
5709 | eics |= adapter->q_vector[i]->eims_value; |
5710 | wr32(E1000_EICS, eics); |
5711 | } else { |
5712 | wr32(E1000_ICS, E1000_ICS_RXDMT0); |
5713 | } |
5714 | |
5715 | igb_spoof_check(adapter); |
5716 | igb_ptp_rx_hang(adapter); |
5717 | igb_ptp_tx_hang(adapter); |
5718 | |
5719 | /* Check LVMMC register on i350/i354 only */ |
5720 | if ((adapter->hw.mac.type == e1000_i350) || |
5721 | (adapter->hw.mac.type == e1000_i354)) |
5722 | igb_check_lvmmc(adapter); |
5723 | |
5724 | /* Reset the timer */ |
5725 | if (!test_bit(__IGB_DOWN, &adapter->state)) { |
5726 | if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) |
5727 | mod_timer(timer: &adapter->watchdog_timer, |
5728 | expires: round_jiffies(j: jiffies + HZ)); |
5729 | else |
5730 | mod_timer(timer: &adapter->watchdog_timer, |
5731 | expires: round_jiffies(j: jiffies + 2 * HZ)); |
5732 | } |
5733 | } |
5734 | |
5735 | enum latency_range { |
5736 | lowest_latency = 0, |
5737 | low_latency = 1, |
5738 | bulk_latency = 2, |
5739 | latency_invalid = 255 |
5740 | }; |
5741 | |
5742 | /** |
5743 | * igb_update_ring_itr - update the dynamic ITR value based on packet size |
5744 | * @q_vector: pointer to q_vector |
5745 | * |
5746 | * Stores a new ITR value based on strictly on packet size. This |
5747 | * algorithm is less sophisticated than that used in igb_update_itr, |
5748 | * due to the difficulty of synchronizing statistics across multiple |
5749 | * receive rings. The divisors and thresholds used by this function |
5750 | * were determined based on theoretical maximum wire speed and testing |
5751 | * data, in order to minimize response time while increasing bulk |
5752 | * throughput. |
5753 | * This functionality is controlled by ethtool's coalescing settings. |
5754 | * NOTE: This function is called only when operating in a multiqueue |
5755 | * receive environment. |
5756 | **/ |
5757 | static void igb_update_ring_itr(struct igb_q_vector *q_vector) |
5758 | { |
5759 | int new_val = q_vector->itr_val; |
5760 | int avg_wire_size = 0; |
5761 | struct igb_adapter *adapter = q_vector->adapter; |
5762 | unsigned int packets; |
5763 | |
5764 | /* For non-gigabit speeds, just fix the interrupt rate at 4000 |
5765 | * ints/sec - ITR timer value of 120 ticks. |
5766 | */ |
5767 | if (adapter->link_speed != SPEED_1000) { |
5768 | new_val = IGB_4K_ITR; |
5769 | goto set_itr_val; |
5770 | } |
5771 | |
5772 | packets = q_vector->rx.total_packets; |
5773 | if (packets) |
5774 | avg_wire_size = q_vector->rx.total_bytes / packets; |
5775 | |
5776 | packets = q_vector->tx.total_packets; |
5777 | if (packets) |
5778 | avg_wire_size = max_t(u32, avg_wire_size, |
5779 | q_vector->tx.total_bytes / packets); |
5780 | |
5781 | /* if avg_wire_size isn't set no work was done */ |
5782 | if (!avg_wire_size) |
5783 | goto clear_counts; |
5784 | |
5785 | /* Add 24 bytes to size to account for CRC, preamble, and gap */ |
5786 | avg_wire_size += 24; |
5787 | |
5788 | /* Don't starve jumbo frames */ |
5789 | avg_wire_size = min(avg_wire_size, 3000); |
5790 | |
5791 | /* Give a little boost to mid-size frames */ |
5792 | if ((avg_wire_size > 300) && (avg_wire_size < 1200)) |
5793 | new_val = avg_wire_size / 3; |
5794 | else |
5795 | new_val = avg_wire_size / 2; |
5796 | |
5797 | /* conservative mode (itr 3) eliminates the lowest_latency setting */ |
5798 | if (new_val < IGB_20K_ITR && |
5799 | ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || |
5800 | (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) |
5801 | new_val = IGB_20K_ITR; |
5802 | |
5803 | set_itr_val: |
5804 | if (new_val != q_vector->itr_val) { |
5805 | q_vector->itr_val = new_val; |
5806 | q_vector->set_itr = 1; |
5807 | } |
5808 | clear_counts: |
5809 | q_vector->rx.total_bytes = 0; |
5810 | q_vector->rx.total_packets = 0; |
5811 | q_vector->tx.total_bytes = 0; |
5812 | q_vector->tx.total_packets = 0; |
5813 | } |
5814 | |
5815 | /** |
5816 | * igb_update_itr - update the dynamic ITR value based on statistics |
5817 | * @q_vector: pointer to q_vector |
5818 | * @ring_container: ring info to update the itr for |
5819 | * |
5820 | * Stores a new ITR value based on packets and byte |
5821 | * counts during the last interrupt. The advantage of per interrupt |
5822 | * computation is faster updates and more accurate ITR for the current |
5823 | * traffic pattern. Constants in this function were computed |
5824 | * based on theoretical maximum wire speed and thresholds were set based |
5825 | * on testing data as well as attempting to minimize response time |
5826 | * while increasing bulk throughput. |
5827 | * This functionality is controlled by ethtool's coalescing settings. |
5828 | * NOTE: These calculations are only valid when operating in a single- |
5829 | * queue environment. |
5830 | **/ |
5831 | static void igb_update_itr(struct igb_q_vector *q_vector, |
5832 | struct igb_ring_container *ring_container) |
5833 | { |
5834 | unsigned int packets = ring_container->total_packets; |
5835 | unsigned int bytes = ring_container->total_bytes; |
5836 | u8 itrval = ring_container->itr; |
5837 | |
5838 | /* no packets, exit with status unchanged */ |
5839 | if (packets == 0) |
5840 | return; |
5841 | |
5842 | switch (itrval) { |
5843 | case lowest_latency: |
5844 | /* handle TSO and jumbo frames */ |
5845 | if (bytes/packets > 8000) |
5846 | itrval = bulk_latency; |
5847 | else if ((packets < 5) && (bytes > 512)) |
5848 | itrval = low_latency; |
5849 | break; |
5850 | case low_latency: /* 50 usec aka 20000 ints/s */ |
5851 | if (bytes > 10000) { |
5852 | /* this if handles the TSO accounting */ |
5853 | if (bytes/packets > 8000) |
5854 | itrval = bulk_latency; |
5855 | else if ((packets < 10) || ((bytes/packets) > 1200)) |
5856 | itrval = bulk_latency; |
5857 | else if ((packets > 35)) |
5858 | itrval = lowest_latency; |
5859 | } else if (bytes/packets > 2000) { |
5860 | itrval = bulk_latency; |
5861 | } else if (packets <= 2 && bytes < 512) { |
5862 | itrval = lowest_latency; |
5863 | } |
5864 | break; |
5865 | case bulk_latency: /* 250 usec aka 4000 ints/s */ |
5866 | if (bytes > 25000) { |
5867 | if (packets > 35) |
5868 | itrval = low_latency; |
5869 | } else if (bytes < 1500) { |
5870 | itrval = low_latency; |
5871 | } |
5872 | break; |
5873 | } |
5874 | |
5875 | /* clear work counters since we have the values we need */ |
5876 | ring_container->total_bytes = 0; |
5877 | ring_container->total_packets = 0; |
5878 | |
5879 | /* write updated itr to ring container */ |
5880 | ring_container->itr = itrval; |
5881 | } |
5882 | |
5883 | static void igb_set_itr(struct igb_q_vector *q_vector) |
5884 | { |
5885 | struct igb_adapter *adapter = q_vector->adapter; |
5886 | u32 new_itr = q_vector->itr_val; |
5887 | u8 current_itr = 0; |
5888 | |
5889 | /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ |
5890 | if (adapter->link_speed != SPEED_1000) { |
5891 | current_itr = 0; |
5892 | new_itr = IGB_4K_ITR; |
5893 | goto set_itr_now; |
5894 | } |
5895 | |
5896 | igb_update_itr(q_vector, ring_container: &q_vector->tx); |
5897 | igb_update_itr(q_vector, ring_container: &q_vector->rx); |
5898 | |
5899 | current_itr = max(q_vector->rx.itr, q_vector->tx.itr); |
5900 | |
5901 | /* conservative mode (itr 3) eliminates the lowest_latency setting */ |
5902 | if (current_itr == lowest_latency && |
5903 | ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || |
5904 | (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) |
5905 | current_itr = low_latency; |
5906 | |
5907 | switch (current_itr) { |
5908 | /* counts and packets in update_itr are dependent on these numbers */ |
5909 | case lowest_latency: |
5910 | new_itr = IGB_70K_ITR; /* 70,000 ints/sec */ |
5911 | break; |
5912 | case low_latency: |
5913 | new_itr = IGB_20K_ITR; /* 20,000 ints/sec */ |
5914 | break; |
5915 | case bulk_latency: |
5916 | new_itr = IGB_4K_ITR; /* 4,000 ints/sec */ |
5917 | break; |
5918 | default: |
5919 | break; |
5920 | } |
5921 | |
5922 | set_itr_now: |
5923 | if (new_itr != q_vector->itr_val) { |
5924 | /* this attempts to bias the interrupt rate towards Bulk |
5925 | * by adding intermediate steps when interrupt rate is |
5926 | * increasing |
5927 | */ |
5928 | new_itr = new_itr > q_vector->itr_val ? |
5929 | max((new_itr * q_vector->itr_val) / |
5930 | (new_itr + (q_vector->itr_val >> 2)), |
5931 | new_itr) : new_itr; |
5932 | /* Don't write the value here; it resets the adapter's |
5933 | * internal timer, and causes us to delay far longer than |
5934 | * we should between interrupts. Instead, we write the ITR |
5935 | * value at the beginning of the next interrupt so the timing |
5936 | * ends up being correct. |
5937 | */ |
5938 | q_vector->itr_val = new_itr; |
5939 | q_vector->set_itr = 1; |
5940 | } |
5941 | } |
5942 | |
5943 | static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, |
5944 | struct igb_tx_buffer *first, |
5945 | u32 vlan_macip_lens, u32 type_tucmd, |
5946 | u32 mss_l4len_idx) |
5947 | { |
5948 | struct e1000_adv_tx_context_desc *context_desc; |
5949 | u16 i = tx_ring->next_to_use; |
5950 | struct timespec64 ts; |
5951 | |
5952 | context_desc = IGB_TX_CTXTDESC(tx_ring, i); |
5953 | |
5954 | i++; |
5955 | tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; |
5956 | |
5957 | /* set bits to identify this as an advanced context descriptor */ |
5958 | type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT; |
5959 | |
5960 | /* For 82575, context index must be unique per ring. */ |
5961 | if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) |
5962 | mss_l4len_idx |= tx_ring->reg_idx << 4; |
5963 | |
5964 | context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); |
5965 | context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); |
5966 | context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); |
5967 | |
5968 | /* We assume there is always a valid tx time available. Invalid times |
5969 | * should have been handled by the upper layers. |
5970 | */ |
5971 | if (tx_ring->launchtime_enable) { |
5972 | ts = ktime_to_timespec64(first->skb->tstamp); |
5973 | skb_txtime_consumed(skb: first->skb); |
5974 | context_desc->seqnum_seed = cpu_to_le32(ts.tv_nsec / 32); |
5975 | } else { |
5976 | context_desc->seqnum_seed = 0; |
5977 | } |
5978 | } |
5979 | |
5980 | static int igb_tso(struct igb_ring *tx_ring, |
5981 | struct igb_tx_buffer *first, |
5982 | u8 *hdr_len) |
5983 | { |
5984 | u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; |
5985 | struct sk_buff *skb = first->skb; |
5986 | union { |
5987 | struct iphdr *v4; |
5988 | struct ipv6hdr *v6; |
5989 | unsigned char *hdr; |
5990 | } ip; |
5991 | union { |
5992 | struct tcphdr *tcp; |
5993 | struct udphdr *udp; |
5994 | unsigned char *hdr; |
5995 | } l4; |
5996 | u32 paylen, l4_offset; |
5997 | int err; |
5998 | |
5999 | if (skb->ip_summed != CHECKSUM_PARTIAL) |
6000 | return 0; |
6001 | |
6002 | if (!skb_is_gso(skb)) |
6003 | return 0; |
6004 | |
6005 | err = skb_cow_head(skb, headroom: 0); |
6006 | if (err < 0) |
6007 | return err; |
6008 | |
6009 | ip.hdr = skb_network_header(skb); |
6010 | l4.hdr = skb_checksum_start(skb); |
6011 | |
6012 | /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ |
6013 | type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ? |
6014 | E1000_ADVTXD_TUCMD_L4T_UDP : E1000_ADVTXD_TUCMD_L4T_TCP; |
6015 | |
6016 | /* initialize outer IP header fields */ |
6017 | if (ip.v4->version == 4) { |
6018 | unsigned char *csum_start = skb_checksum_start(skb); |
6019 | unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); |
6020 | |
6021 | /* IP header will have to cancel out any data that |
6022 | * is not a part of the outer IP header |
6023 | */ |
6024 | ip.v4->check = csum_fold(sum: csum_partial(buff: trans_start, |
6025 | len: csum_start - trans_start, |
6026 | sum: 0)); |
6027 | type_tucmd |= E1000_ADVTXD_TUCMD_IPV4; |
6028 | |
6029 | ip.v4->tot_len = 0; |
6030 | first->tx_flags |= IGB_TX_FLAGS_TSO | |
6031 | IGB_TX_FLAGS_CSUM | |
6032 | IGB_TX_FLAGS_IPV4; |
6033 | } else { |
6034 | ip.v6->payload_len = 0; |
6035 | first->tx_flags |= IGB_TX_FLAGS_TSO | |
6036 | IGB_TX_FLAGS_CSUM; |
6037 | } |
6038 | |
6039 | /* determine offset of inner transport header */ |
6040 | l4_offset = l4.hdr - skb->data; |
6041 | |
6042 | /* remove payload length from inner checksum */ |
6043 | paylen = skb->len - l4_offset; |
6044 | if (type_tucmd & E1000_ADVTXD_TUCMD_L4T_TCP) { |
6045 | /* compute length of segmentation header */ |
6046 | *hdr_len = (l4.tcp->doff * 4) + l4_offset; |
6047 | csum_replace_by_diff(sum: &l4.tcp->check, |
6048 | diff: (__force __wsum)htonl(paylen)); |
6049 | } else { |
6050 | /* compute length of segmentation header */ |
6051 | *hdr_len = sizeof(*l4.udp) + l4_offset; |
6052 | csum_replace_by_diff(sum: &l4.udp->check, |
6053 | diff: (__force __wsum)htonl(paylen)); |
6054 | } |
6055 | |
6056 | /* update gso size and bytecount with header size */ |
6057 | first->gso_segs = skb_shinfo(skb)->gso_segs; |
6058 | first->bytecount += (first->gso_segs - 1) * *hdr_len; |
6059 | |
6060 | /* MSS L4LEN IDX */ |
6061 | mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT; |
6062 | mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT; |
6063 | |
6064 | /* VLAN MACLEN IPLEN */ |
6065 | vlan_macip_lens = l4.hdr - ip.hdr; |
6066 | vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT; |
6067 | vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK; |
6068 | |
6069 | igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, |
6070 | type_tucmd, mss_l4len_idx); |
6071 | |
6072 | return 1; |
6073 | } |
6074 | |
6075 | static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first) |
6076 | { |
6077 | struct sk_buff *skb = first->skb; |
6078 | u32 vlan_macip_lens = 0; |
6079 | u32 type_tucmd = 0; |
6080 | |
6081 | if (skb->ip_summed != CHECKSUM_PARTIAL) { |
6082 | csum_failed: |
6083 | if (!(first->tx_flags & IGB_TX_FLAGS_VLAN) && |
6084 | !tx_ring->launchtime_enable) |
6085 | return; |
6086 | goto no_csum; |
6087 | } |
6088 | |
6089 | switch (skb->csum_offset) { |
6090 | case offsetof(struct tcphdr, check): |
6091 | type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP; |
6092 | fallthrough; |
6093 | case offsetof(struct udphdr, check): |
6094 | break; |
6095 | case offsetof(struct sctphdr, checksum): |
6096 | /* validate that this is actually an SCTP request */ |
6097 | if (skb_csum_is_sctp(skb)) { |
6098 | type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP; |
6099 | break; |
6100 | } |
6101 | fallthrough; |
6102 | default: |
6103 | skb_checksum_help(skb); |
6104 | goto csum_failed; |
6105 | } |
6106 | |
6107 | /* update TX checksum flag */ |
6108 | first->tx_flags |= IGB_TX_FLAGS_CSUM; |
6109 | vlan_macip_lens = skb_checksum_start_offset(skb) - |
6110 | skb_network_offset(skb); |
6111 | no_csum: |
6112 | vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT; |
6113 | vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK; |
6114 | |
6115 | igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, mss_l4len_idx: 0); |
6116 | } |
6117 | |
6118 | #define IGB_SET_FLAG(_input, _flag, _result) \ |
6119 | ((_flag <= _result) ? \ |
6120 | ((u32)(_input & _flag) * (_result / _flag)) : \ |
6121 | ((u32)(_input & _flag) / (_flag / _result))) |
6122 | |
6123 | static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) |
6124 | { |
6125 | /* set type for advanced descriptor with frame checksum insertion */ |
6126 | u32 cmd_type = E1000_ADVTXD_DTYP_DATA | |
6127 | E1000_ADVTXD_DCMD_DEXT | |
6128 | E1000_ADVTXD_DCMD_IFCS; |
6129 | |
6130 | /* set HW vlan bit if vlan is present */ |
6131 | cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN, |
6132 | (E1000_ADVTXD_DCMD_VLE)); |
6133 | |
6134 | /* set segmentation bits for TSO */ |
6135 | cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO, |
6136 | (E1000_ADVTXD_DCMD_TSE)); |
6137 | |
6138 | /* set timestamp bit if present */ |
6139 | cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP, |
6140 | (E1000_ADVTXD_MAC_TSTAMP)); |
6141 | |
6142 | /* insert frame checksum */ |
6143 | cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS); |
6144 | |
6145 | return cmd_type; |
6146 | } |
6147 | |
6148 | static void igb_tx_olinfo_status(struct igb_ring *tx_ring, |
6149 | union e1000_adv_tx_desc *tx_desc, |
6150 | u32 tx_flags, unsigned int paylen) |
6151 | { |
6152 | u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT; |
6153 | |
6154 | /* 82575 requires a unique index per ring */ |
6155 | if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) |
6156 | olinfo_status |= tx_ring->reg_idx << 4; |
6157 | |
6158 | /* insert L4 checksum */ |
6159 | olinfo_status |= IGB_SET_FLAG(tx_flags, |
6160 | IGB_TX_FLAGS_CSUM, |
6161 | (E1000_TXD_POPTS_TXSM << 8)); |
6162 | |
6163 | /* insert IPv4 checksum */ |
6164 | olinfo_status |= IGB_SET_FLAG(tx_flags, |
6165 | IGB_TX_FLAGS_IPV4, |
6166 | (E1000_TXD_POPTS_IXSM << 8)); |
6167 | |
6168 | tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); |
6169 | } |
6170 | |
6171 | static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size) |
6172 | { |
6173 | struct net_device *netdev = tx_ring->netdev; |
6174 | |
6175 | netif_stop_subqueue(dev: netdev, queue_index: tx_ring->queue_index); |
6176 | |
6177 | /* Herbert's original patch had: |
6178 | * smp_mb__after_netif_stop_queue(); |
6179 | * but since that doesn't exist yet, just open code it. |
6180 | */ |
6181 | smp_mb(); |
6182 | |
6183 | /* We need to check again in a case another CPU has just |
6184 | * made room available. |
6185 | */ |
6186 | if (igb_desc_unused(ring: tx_ring) < size) |
6187 | return -EBUSY; |
6188 | |
6189 | /* A reprieve! */ |
6190 | netif_wake_subqueue(dev: netdev, queue_index: tx_ring->queue_index); |
6191 | |
6192 | u64_stats_update_begin(syncp: &tx_ring->tx_syncp2); |
6193 | tx_ring->tx_stats.restart_queue2++; |
6194 | u64_stats_update_end(syncp: &tx_ring->tx_syncp2); |
6195 | |
6196 | return 0; |
6197 | } |
6198 | |
6199 | static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size) |
6200 | { |
6201 | if (igb_desc_unused(ring: tx_ring) >= size) |
6202 | return 0; |
6203 | return __igb_maybe_stop_tx(tx_ring, size); |
6204 | } |
6205 | |
6206 | static int igb_tx_map(struct igb_ring *tx_ring, |
6207 | struct igb_tx_buffer *first, |
6208 | const u8 hdr_len) |
6209 | { |
6210 | struct sk_buff *skb = first->skb; |
6211 | struct igb_tx_buffer *tx_buffer; |
6212 | union e1000_adv_tx_desc *tx_desc; |
6213 | skb_frag_t *frag; |
6214 | dma_addr_t dma; |
6215 | unsigned int data_len, size; |
6216 | u32 tx_flags = first->tx_flags; |
6217 | u32 cmd_type = igb_tx_cmd_type(skb, tx_flags); |
6218 | u16 i = tx_ring->next_to_use; |
6219 | |
6220 | tx_desc = IGB_TX_DESC(tx_ring, i); |
6221 | |
6222 | igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen: skb->len - hdr_len); |
6223 | |
6224 | size = skb_headlen(skb); |
6225 | data_len = skb->data_len; |
6226 | |
6227 | dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); |
6228 | |
6229 | tx_buffer = first; |
6230 | |
6231 | for (frag = &skb_shinfo(skb)->frags[0];; frag++) { |
6232 | if (dma_mapping_error(dev: tx_ring->dev, dma_addr: dma)) |
6233 | goto dma_error; |
6234 | |
6235 | /* record length, and DMA address */ |
6236 | dma_unmap_len_set(tx_buffer, len, size); |
6237 | dma_unmap_addr_set(tx_buffer, dma, dma); |
6238 | |
6239 | tx_desc->read.buffer_addr = cpu_to_le64(dma); |
6240 | |
6241 | while (unlikely(size > IGB_MAX_DATA_PER_TXD)) { |
6242 | tx_desc->read.cmd_type_len = |
6243 | cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD); |
6244 | |
6245 | i++; |
6246 | tx_desc++; |
6247 | if (i == tx_ring->count) { |
6248 | tx_desc = IGB_TX_DESC(tx_ring, 0); |
6249 | i = 0; |
6250 | } |
6251 | tx_desc->read.olinfo_status = 0; |
6252 | |
6253 | dma += IGB_MAX_DATA_PER_TXD; |
6254 | size -= IGB_MAX_DATA_PER_TXD; |
6255 | |
6256 | tx_desc->read.buffer_addr = cpu_to_le64(dma); |
6257 | } |
6258 | |
6259 | if (likely(!data_len)) |
6260 | break; |
6261 | |
6262 | tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size); |
6263 | |
6264 | i++; |
6265 | tx_desc++; |
6266 | if (i == tx_ring->count) { |
6267 | tx_desc = IGB_TX_DESC(tx_ring, 0); |
6268 | i = 0; |
6269 | } |
6270 | tx_desc->read.olinfo_status = 0; |
6271 | |
6272 | size = skb_frag_size(frag); |
6273 | data_len -= size; |
6274 | |
6275 | dma = skb_frag_dma_map(dev: tx_ring->dev, frag, offset: 0, |
6276 | size, dir: DMA_TO_DEVICE); |
6277 | |
6278 | tx_buffer = &tx_ring->tx_buffer_info[i]; |
6279 | } |
6280 | |
6281 | /* write last descriptor with RS and EOP bits */ |
6282 | cmd_type |= size | IGB_TXD_DCMD; |
6283 | tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); |
6284 | |
6285 | netdev_tx_sent_queue(dev_queue: txring_txq(tx_ring), bytes: first->bytecount); |
6286 | |
6287 | /* set the timestamp */ |
6288 | first->time_stamp = jiffies; |
6289 | |
6290 | skb_tx_timestamp(skb); |
6291 | |
6292 | /* Force memory writes to complete before letting h/w know there |
6293 | * are new descriptors to fetch. (Only applicable for weak-ordered |
6294 | * memory model archs, such as IA-64). |
6295 | * |
6296 | * We also need this memory barrier to make certain all of the |
6297 | * status bits have been updated before next_to_watch is written. |
6298 | */ |
6299 | dma_wmb(); |
6300 | |
6301 | /* set next_to_watch value indicating a packet is present */ |
6302 | first->next_to_watch = tx_desc; |
6303 | |
6304 | i++; |
6305 | if (i == tx_ring->count) |
6306 | i = 0; |
6307 | |
6308 | tx_ring->next_to_use = i; |
6309 | |
6310 | /* Make sure there is space in the ring for the next send. */ |
6311 | igb_maybe_stop_tx(tx_ring, DESC_NEEDED); |
6312 | |
6313 | if (netif_xmit_stopped(dev_queue: txring_txq(tx_ring)) || !netdev_xmit_more()) { |
6314 | writel(val: i, addr: tx_ring->tail); |
6315 | } |
6316 | return 0; |
6317 | |
6318 | dma_error: |
6319 | dev_err(tx_ring->dev, "TX DMA map failed\n" ); |
6320 | tx_buffer = &tx_ring->tx_buffer_info[i]; |
6321 | |
6322 | /* clear dma mappings for failed tx_buffer_info map */ |
6323 | while (tx_buffer != first) { |
6324 | if (dma_unmap_len(tx_buffer, len)) |
6325 | dma_unmap_page(tx_ring->dev, |
6326 | dma_unmap_addr(tx_buffer, dma), |
6327 | dma_unmap_len(tx_buffer, len), |
6328 | DMA_TO_DEVICE); |
6329 | dma_unmap_len_set(tx_buffer, len, 0); |
6330 | |
6331 | if (i-- == 0) |
6332 | i += tx_ring->count; |
6333 | tx_buffer = &tx_ring->tx_buffer_info[i]; |
6334 | } |
6335 | |
6336 | if (dma_unmap_len(tx_buffer, len)) |
6337 | dma_unmap_single(tx_ring->dev, |
6338 | dma_unmap_addr(tx_buffer, dma), |
6339 | dma_unmap_len(tx_buffer, len), |
6340 | DMA_TO_DEVICE); |
6341 | dma_unmap_len_set(tx_buffer, len, 0); |
6342 | |
6343 | dev_kfree_skb_any(skb: tx_buffer->skb); |
6344 | tx_buffer->skb = NULL; |
6345 | |
6346 | tx_ring->next_to_use = i; |
6347 | |
6348 | return -1; |
6349 | } |
6350 | |
6351 | int igb_xmit_xdp_ring(struct igb_adapter *adapter, |
6352 | struct igb_ring *tx_ring, |
6353 | struct xdp_frame *xdpf) |
6354 | { |
6355 | struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(frame: xdpf); |
6356 | u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; |
6357 | u16 count, i, index = tx_ring->next_to_use; |
6358 | struct igb_tx_buffer *tx_head = &tx_ring->tx_buffer_info[index]; |
6359 | struct igb_tx_buffer *tx_buffer = tx_head; |
6360 | union e1000_adv_tx_desc *tx_desc = IGB_TX_DESC(tx_ring, index); |
6361 | u32 len = xdpf->len, cmd_type, olinfo_status; |
6362 | void *data = xdpf->data; |
6363 | |
6364 | count = TXD_USE_COUNT(len); |
6365 | for (i = 0; i < nr_frags; i++) |
6366 | count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i])); |
6367 | |
6368 | if (igb_maybe_stop_tx(tx_ring, size: count + 3)) |
6369 | return IGB_XDP_CONSUMED; |
6370 | |
6371 | i = 0; |
6372 | /* record the location of the first descriptor for this packet */ |
6373 | tx_head->bytecount = xdp_get_frame_len(xdpf); |
6374 | tx_head->type = IGB_TYPE_XDP; |
6375 | tx_head->gso_segs = 1; |
6376 | tx_head->xdpf = xdpf; |
6377 | |
6378 | olinfo_status = tx_head->bytecount << E1000_ADVTXD_PAYLEN_SHIFT; |
6379 | /* 82575 requires a unique index per ring */ |
6380 | if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) |
6381 | olinfo_status |= tx_ring->reg_idx << 4; |
6382 | tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); |
6383 | |
6384 | for (;;) { |
6385 | dma_addr_t dma; |
6386 | |
6387 | dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE); |
6388 | if (dma_mapping_error(dev: tx_ring->dev, dma_addr: dma)) |
6389 | goto unmap; |
6390 | |
6391 | /* record length, and DMA address */ |
6392 | dma_unmap_len_set(tx_buffer, len, len); |
6393 | dma_unmap_addr_set(tx_buffer, dma, dma); |
6394 | |
6395 | /* put descriptor type bits */ |
6396 | cmd_type = E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_DEXT | |
6397 | E1000_ADVTXD_DCMD_IFCS | len; |
6398 | |
6399 | tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); |
6400 | tx_desc->read.buffer_addr = cpu_to_le64(dma); |
6401 | |
6402 | tx_buffer->protocol = 0; |
6403 | |
6404 | if (++index == tx_ring->count) |
6405 | index = 0; |
6406 | |
6407 | if (i == nr_frags) |
6408 | break; |
6409 | |
6410 | tx_buffer = &tx_ring->tx_buffer_info[index]; |
6411 | tx_desc = IGB_TX_DESC(tx_ring, index); |
6412 | tx_desc->read.olinfo_status = 0; |
6413 | |
6414 | data = skb_frag_address(frag: &sinfo->frags[i]); |
6415 | len = skb_frag_size(frag: &sinfo->frags[i]); |
6416 | i++; |
6417 | } |
6418 | tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_TXD_DCMD); |
6419 | |
6420 | netdev_tx_sent_queue(dev_queue: txring_txq(tx_ring), bytes: tx_head->bytecount); |
6421 | /* set the timestamp */ |
6422 | tx_head->time_stamp = jiffies; |
6423 | |
6424 | /* Avoid any potential race with xdp_xmit and cleanup */ |
6425 | smp_wmb(); |
6426 | |
6427 | /* set next_to_watch value indicating a packet is present */ |
6428 | tx_head->next_to_watch = tx_desc; |
6429 | tx_ring->next_to_use = index; |
6430 | |
6431 | /* Make sure there is space in the ring for the next send. */ |
6432 | igb_maybe_stop_tx(tx_ring, DESC_NEEDED); |
6433 | |
6434 | if (netif_xmit_stopped(dev_queue: txring_txq(tx_ring)) || !netdev_xmit_more()) |
6435 | writel(val: index, addr: tx_ring->tail); |
6436 | |
6437 | return IGB_XDP_TX; |
6438 | |
6439 | unmap: |
6440 | for (;;) { |
6441 | tx_buffer = &tx_ring->tx_buffer_info[index]; |
6442 | if (dma_unmap_len(tx_buffer, len)) |
6443 | dma_unmap_page(tx_ring->dev, |
6444 | dma_unmap_addr(tx_buffer, dma), |
6445 | dma_unmap_len(tx_buffer, len), |
6446 | DMA_TO_DEVICE); |
6447 | dma_unmap_len_set(tx_buffer, len, 0); |
6448 | if (tx_buffer == tx_head) |
6449 | break; |
6450 | |
6451 | if (!index) |
6452 | index += tx_ring->count; |
6453 | index--; |
6454 | } |
6455 | |
6456 | return IGB_XDP_CONSUMED; |
6457 | } |
6458 | |
6459 | netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb, |
6460 | struct igb_ring *tx_ring) |
6461 | { |
6462 | struct igb_tx_buffer *first; |
6463 | int tso; |
6464 | u32 tx_flags = 0; |
6465 | unsigned short f; |
6466 | u16 count = TXD_USE_COUNT(skb_headlen(skb)); |
6467 | __be16 protocol = vlan_get_protocol(skb); |
6468 | u8 hdr_len = 0; |
6469 | |
6470 | /* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD, |
6471 | * + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD, |
6472 | * + 2 desc gap to keep tail from touching head, |
6473 | * + 1 desc for context descriptor, |
6474 | * otherwise try next time |
6475 | */ |
6476 | for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) |
6477 | count += TXD_USE_COUNT(skb_frag_size( |
6478 | &skb_shinfo(skb)->frags[f])); |
6479 | |
6480 | if (igb_maybe_stop_tx(tx_ring, size: count + 3)) { |
6481 | /* this is a hard error */ |
6482 | return NETDEV_TX_BUSY; |
6483 | } |
6484 | |
6485 | /* record the location of the first descriptor for this packet */ |
6486 | first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; |
6487 | first->type = IGB_TYPE_SKB; |
6488 | first->skb = skb; |
6489 | first->bytecount = skb->len; |
6490 | first->gso_segs = 1; |
6491 | |
6492 | if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { |
6493 | struct igb_adapter *adapter = netdev_priv(dev: tx_ring->netdev); |
6494 | |
6495 | if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && |
6496 | !test_and_set_bit_lock(nr: __IGB_PTP_TX_IN_PROGRESS, |
6497 | addr: &adapter->state)) { |
6498 | skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; |
6499 | tx_flags |= IGB_TX_FLAGS_TSTAMP; |
6500 | |
6501 | adapter->ptp_tx_skb = skb_get(skb); |
6502 | adapter->ptp_tx_start = jiffies; |
6503 | if (adapter->hw.mac.type == e1000_82576) |
6504 | schedule_work(work: &adapter->ptp_tx_work); |
6505 | } else { |
6506 | adapter->tx_hwtstamp_skipped++; |
6507 | } |
6508 | } |
6509 | |
6510 | if (skb_vlan_tag_present(skb)) { |
6511 | tx_flags |= IGB_TX_FLAGS_VLAN; |
6512 | tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT); |
6513 | } |
6514 | |
6515 | /* record initial flags and protocol */ |
6516 | first->tx_flags = tx_flags; |
6517 | first->protocol = protocol; |
6518 | |
6519 | tso = igb_tso(tx_ring, first, hdr_len: &hdr_len); |
6520 | if (tso < 0) |
6521 | goto out_drop; |
6522 | else if (!tso) |
6523 | igb_tx_csum(tx_ring, first); |
6524 | |
6525 | if (igb_tx_map(tx_ring, first, hdr_len)) |
6526 | goto cleanup_tx_tstamp; |
6527 | |
6528 | return NETDEV_TX_OK; |
6529 | |
6530 | out_drop: |
6531 | dev_kfree_skb_any(skb: first->skb); |
6532 | first->skb = NULL; |
6533 | cleanup_tx_tstamp: |
6534 | if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) { |
6535 | struct igb_adapter *adapter = netdev_priv(dev: tx_ring->netdev); |
6536 | |
6537 | dev_kfree_skb_any(skb: adapter->ptp_tx_skb); |
6538 | adapter->ptp_tx_skb = NULL; |
6539 | if (adapter->hw.mac.type == e1000_82576) |
6540 | cancel_work_sync(work: &adapter->ptp_tx_work); |
6541 | clear_bit_unlock(nr: __IGB_PTP_TX_IN_PROGRESS, addr: &adapter->state); |
6542 | } |
6543 | |
6544 | return NETDEV_TX_OK; |
6545 | } |
6546 | |
6547 | static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter, |
6548 | struct sk_buff *skb) |
6549 | { |
6550 | unsigned int r_idx = skb->queue_mapping; |
6551 | |
6552 | if (r_idx >= adapter->num_tx_queues) |
6553 | r_idx = r_idx % adapter->num_tx_queues; |
6554 | |
6555 | return adapter->tx_ring[r_idx]; |
6556 | } |
6557 | |
6558 | static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, |
6559 | struct net_device *netdev) |
6560 | { |
6561 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
6562 | |
6563 | /* The minimum packet size with TCTL.PSP set is 17 so pad the skb |
6564 | * in order to meet this minimum size requirement. |
6565 | */ |
6566 | if (skb_put_padto(skb, len: 17)) |
6567 | return NETDEV_TX_OK; |
6568 | |
6569 | return igb_xmit_frame_ring(skb, tx_ring: igb_tx_queue_mapping(adapter, skb)); |
6570 | } |
6571 | |
6572 | /** |
6573 | * igb_tx_timeout - Respond to a Tx Hang |
6574 | * @netdev: network interface device structure |
6575 | * @txqueue: number of the Tx queue that hung (unused) |
6576 | **/ |
6577 | static void igb_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue) |
6578 | { |
6579 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
6580 | struct e1000_hw *hw = &adapter->hw; |
6581 | |
6582 | /* Do the reset outside of interrupt context */ |
6583 | adapter->tx_timeout_count++; |
6584 | |
6585 | if (hw->mac.type >= e1000_82580) |
6586 | hw->dev_spec._82575.global_device_reset = true; |
6587 | |
6588 | schedule_work(work: &adapter->reset_task); |
6589 | wr32(E1000_EICS, |
6590 | (adapter->eims_enable_mask & ~adapter->eims_other)); |
6591 | } |
6592 | |
6593 | static void igb_reset_task(struct work_struct *work) |
6594 | { |
6595 | struct igb_adapter *adapter; |
6596 | adapter = container_of(work, struct igb_adapter, reset_task); |
6597 | |
6598 | rtnl_lock(); |
6599 | /* If we're already down or resetting, just bail */ |
6600 | if (test_bit(__IGB_DOWN, &adapter->state) || |
6601 | test_bit(__IGB_RESETTING, &adapter->state)) { |
6602 | rtnl_unlock(); |
6603 | return; |
6604 | } |
6605 | |
6606 | igb_dump(adapter); |
6607 | netdev_err(dev: adapter->netdev, format: "Reset adapter\n" ); |
6608 | igb_reinit_locked(adapter); |
6609 | rtnl_unlock(); |
6610 | } |
6611 | |
6612 | /** |
6613 | * igb_get_stats64 - Get System Network Statistics |
6614 | * @netdev: network interface device structure |
6615 | * @stats: rtnl_link_stats64 pointer |
6616 | **/ |
6617 | static void igb_get_stats64(struct net_device *netdev, |
6618 | struct rtnl_link_stats64 *stats) |
6619 | { |
6620 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
6621 | |
6622 | spin_lock(lock: &adapter->stats64_lock); |
6623 | igb_update_stats(adapter); |
6624 | memcpy(stats, &adapter->stats64, sizeof(*stats)); |
6625 | spin_unlock(lock: &adapter->stats64_lock); |
6626 | } |
6627 | |
6628 | /** |
6629 | * igb_change_mtu - Change the Maximum Transfer Unit |
6630 | * @netdev: network interface device structure |
6631 | * @new_mtu: new value for maximum frame size |
6632 | * |
6633 | * Returns 0 on success, negative on failure |
6634 | **/ |
6635 | static int igb_change_mtu(struct net_device *netdev, int new_mtu) |
6636 | { |
6637 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
6638 | int max_frame = new_mtu + IGB_ETH_PKT_HDR_PAD; |
6639 | |
6640 | if (adapter->xdp_prog) { |
6641 | int i; |
6642 | |
6643 | for (i = 0; i < adapter->num_rx_queues; i++) { |
6644 | struct igb_ring *ring = adapter->rx_ring[i]; |
6645 | |
6646 | if (max_frame > igb_rx_bufsz(ring)) { |
6647 | netdev_warn(dev: adapter->netdev, |
6648 | format: "Requested MTU size is not supported with XDP. Max frame size is %d\n" , |
6649 | max_frame); |
6650 | return -EINVAL; |
6651 | } |
6652 | } |
6653 | } |
6654 | |
6655 | /* adjust max frame to be at least the size of a standard frame */ |
6656 | if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) |
6657 | max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; |
6658 | |
6659 | while (test_and_set_bit(nr: __IGB_RESETTING, addr: &adapter->state)) |
6660 | usleep_range(min: 1000, max: 2000); |
6661 | |
6662 | /* igb_down has a dependency on max_frame_size */ |
6663 | adapter->max_frame_size = max_frame; |
6664 | |
6665 | if (netif_running(dev: netdev)) |
6666 | igb_down(adapter); |
6667 | |
6668 | netdev_dbg(netdev, "changing MTU from %d to %d\n" , |
6669 | netdev->mtu, new_mtu); |
6670 | netdev->mtu = new_mtu; |
6671 | |
6672 | if (netif_running(dev: netdev)) |
6673 | igb_up(adapter); |
6674 | else |
6675 | igb_reset(adapter); |
6676 | |
6677 | clear_bit(nr: __IGB_RESETTING, addr: &adapter->state); |
6678 | |
6679 | return 0; |
6680 | } |
6681 | |
6682 | /** |
6683 | * igb_update_stats - Update the board statistics counters |
6684 | * @adapter: board private structure |
6685 | **/ |
6686 | void igb_update_stats(struct igb_adapter *adapter) |
6687 | { |
6688 | struct rtnl_link_stats64 *net_stats = &adapter->stats64; |
6689 | struct e1000_hw *hw = &adapter->hw; |
6690 | struct pci_dev *pdev = adapter->pdev; |
6691 | u32 reg, mpc; |
6692 | int i; |
6693 | u64 bytes, packets; |
6694 | unsigned int start; |
6695 | u64 _bytes, _packets; |
6696 | |
6697 | /* Prevent stats update while adapter is being reset, or if the pci |
6698 | * connection is down. |
6699 | */ |
6700 | if (adapter->link_speed == 0) |
6701 | return; |
6702 | if (pci_channel_offline(pdev)) |
6703 | return; |
6704 | |
6705 | bytes = 0; |
6706 | packets = 0; |
6707 | |
6708 | rcu_read_lock(); |
6709 | for (i = 0; i < adapter->num_rx_queues; i++) { |
6710 | struct igb_ring *ring = adapter->rx_ring[i]; |
6711 | u32 rqdpc = rd32(E1000_RQDPC(i)); |
6712 | if (hw->mac.type >= e1000_i210) |
6713 | wr32(E1000_RQDPC(i), 0); |
6714 | |
6715 | if (rqdpc) { |
6716 | ring->rx_stats.drops += rqdpc; |
6717 | net_stats->rx_fifo_errors += rqdpc; |
6718 | } |
6719 | |
6720 | do { |
6721 | start = u64_stats_fetch_begin(syncp: &ring->rx_syncp); |
6722 | _bytes = ring->rx_stats.bytes; |
6723 | _packets = ring->rx_stats.packets; |
6724 | } while (u64_stats_fetch_retry(syncp: &ring->rx_syncp, start)); |
6725 | bytes += _bytes; |
6726 | packets += _packets; |
6727 | } |
6728 | |
6729 | net_stats->rx_bytes = bytes; |
6730 | net_stats->rx_packets = packets; |
6731 | |
6732 | bytes = 0; |
6733 | packets = 0; |
6734 | for (i = 0; i < adapter->num_tx_queues; i++) { |
6735 | struct igb_ring *ring = adapter->tx_ring[i]; |
6736 | do { |
6737 | start = u64_stats_fetch_begin(syncp: &ring->tx_syncp); |
6738 | _bytes = ring->tx_stats.bytes; |
6739 | _packets = ring->tx_stats.packets; |
6740 | } while (u64_stats_fetch_retry(syncp: &ring->tx_syncp, start)); |
6741 | bytes += _bytes; |
6742 | packets += _packets; |
6743 | } |
6744 | net_stats->tx_bytes = bytes; |
6745 | net_stats->tx_packets = packets; |
6746 | rcu_read_unlock(); |
6747 | |
6748 | /* read stats registers */ |
6749 | adapter->stats.crcerrs += rd32(E1000_CRCERRS); |
6750 | adapter->stats.gprc += rd32(E1000_GPRC); |
6751 | adapter->stats.gorc += rd32(E1000_GORCL); |
6752 | rd32(E1000_GORCH); /* clear GORCL */ |
6753 | adapter->stats.bprc += rd32(E1000_BPRC); |
6754 | adapter->stats.mprc += rd32(E1000_MPRC); |
6755 | adapter->stats.roc += rd32(E1000_ROC); |
6756 | |
6757 | adapter->stats.prc64 += rd32(E1000_PRC64); |
6758 | adapter->stats.prc127 += rd32(E1000_PRC127); |
6759 | adapter->stats.prc255 += rd32(E1000_PRC255); |
6760 | adapter->stats.prc511 += rd32(E1000_PRC511); |
6761 | adapter->stats.prc1023 += rd32(E1000_PRC1023); |
6762 | adapter->stats.prc1522 += rd32(E1000_PRC1522); |
6763 | adapter->stats.symerrs += rd32(E1000_SYMERRS); |
6764 | adapter->stats.sec += rd32(E1000_SEC); |
6765 | |
6766 | mpc = rd32(E1000_MPC); |
6767 | adapter->stats.mpc += mpc; |
6768 | net_stats->rx_fifo_errors += mpc; |
6769 | adapter->stats.scc += rd32(E1000_SCC); |
6770 | adapter->stats.ecol += rd32(E1000_ECOL); |
6771 | adapter->stats.mcc += rd32(E1000_MCC); |
6772 | adapter->stats.latecol += rd32(E1000_LATECOL); |
6773 | adapter->stats.dc += rd32(E1000_DC); |
6774 | adapter->stats.rlec += rd32(E1000_RLEC); |
6775 | adapter->stats.xonrxc += rd32(E1000_XONRXC); |
6776 | adapter->stats.xontxc += rd32(E1000_XONTXC); |
6777 | adapter->stats.xoffrxc += rd32(E1000_XOFFRXC); |
6778 | adapter->stats.xofftxc += rd32(E1000_XOFFTXC); |
6779 | adapter->stats.fcruc += rd32(E1000_FCRUC); |
6780 | adapter->stats.gptc += rd32(E1000_GPTC); |
6781 | adapter->stats.gotc += rd32(E1000_GOTCL); |
6782 | rd32(E1000_GOTCH); /* clear GOTCL */ |
6783 | adapter->stats.rnbc += rd32(E1000_RNBC); |
6784 | adapter->stats.ruc += rd32(E1000_RUC); |
6785 | adapter->stats.rfc += rd32(E1000_RFC); |
6786 | adapter->stats.rjc += rd32(E1000_RJC); |
6787 | adapter->stats.tor += rd32(E1000_TORH); |
6788 | adapter->stats.tot += rd32(E1000_TOTH); |
6789 | adapter->stats.tpr += rd32(E1000_TPR); |
6790 | |
6791 | adapter->stats.ptc64 += rd32(E1000_PTC64); |
6792 | adapter->stats.ptc127 += rd32(E1000_PTC127); |
6793 | adapter->stats.ptc255 += rd32(E1000_PTC255); |
6794 | adapter->stats.ptc511 += rd32(E1000_PTC511); |
6795 | adapter->stats.ptc1023 += rd32(E1000_PTC1023); |
6796 | adapter->stats.ptc1522 += rd32(E1000_PTC1522); |
6797 | |
6798 | adapter->stats.mptc += rd32(E1000_MPTC); |
6799 | adapter->stats.bptc += rd32(E1000_BPTC); |
6800 | |
6801 | adapter->stats.tpt += rd32(E1000_TPT); |
6802 | adapter->stats.colc += rd32(E1000_COLC); |
6803 | |
6804 | adapter->stats.algnerrc += rd32(E1000_ALGNERRC); |
6805 | /* read internal phy specific stats */ |
6806 | reg = rd32(E1000_CTRL_EXT); |
6807 | if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) { |
6808 | adapter->stats.rxerrc += rd32(E1000_RXERRC); |
6809 | |
6810 | /* this stat has invalid values on i210/i211 */ |
6811 | if ((hw->mac.type != e1000_i210) && |
6812 | (hw->mac.type != e1000_i211)) |
6813 | adapter->stats.tncrs += rd32(E1000_TNCRS); |
6814 | } |
6815 | |
6816 | adapter->stats.tsctc += rd32(E1000_TSCTC); |
6817 | adapter->stats.tsctfc += rd32(E1000_TSCTFC); |
6818 | |
6819 | adapter->stats.iac += rd32(E1000_IAC); |
6820 | adapter->stats.icrxoc += rd32(E1000_ICRXOC); |
6821 | adapter->stats.icrxptc += rd32(E1000_ICRXPTC); |
6822 | adapter->stats.icrxatc += rd32(E1000_ICRXATC); |
6823 | adapter->stats.ictxptc += rd32(E1000_ICTXPTC); |
6824 | adapter->stats.ictxatc += rd32(E1000_ICTXATC); |
6825 | adapter->stats.ictxqec += rd32(E1000_ICTXQEC); |
6826 | adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC); |
6827 | adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC); |
6828 | |
6829 | /* Fill out the OS statistics structure */ |
6830 | net_stats->multicast = adapter->stats.mprc; |
6831 | net_stats->collisions = adapter->stats.colc; |
6832 | |
6833 | /* Rx Errors */ |
6834 | |
6835 | /* RLEC on some newer hardware can be incorrect so build |
6836 | * our own version based on RUC and ROC |
6837 | */ |
6838 | net_stats->rx_errors = adapter->stats.rxerrc + |
6839 | adapter->stats.crcerrs + adapter->stats.algnerrc + |
6840 | adapter->stats.ruc + adapter->stats.roc + |
6841 | adapter->stats.cexterr; |
6842 | net_stats->rx_length_errors = adapter->stats.ruc + |
6843 | adapter->stats.roc; |
6844 | net_stats->rx_crc_errors = adapter->stats.crcerrs; |
6845 | net_stats->rx_frame_errors = adapter->stats.algnerrc; |
6846 | net_stats->rx_missed_errors = adapter->stats.mpc; |
6847 | |
6848 | /* Tx Errors */ |
6849 | net_stats->tx_errors = adapter->stats.ecol + |
6850 | adapter->stats.latecol; |
6851 | net_stats->tx_aborted_errors = adapter->stats.ecol; |
6852 | net_stats->tx_window_errors = adapter->stats.latecol; |
6853 | net_stats->tx_carrier_errors = adapter->stats.tncrs; |
6854 | |
6855 | /* Tx Dropped needs to be maintained elsewhere */ |
6856 | |
6857 | /* Management Stats */ |
6858 | adapter->stats.mgptc += rd32(E1000_MGTPTC); |
6859 | adapter->stats.mgprc += rd32(E1000_MGTPRC); |
6860 | adapter->stats.mgpdc += rd32(E1000_MGTPDC); |
6861 | |
6862 | /* OS2BMC Stats */ |
6863 | reg = rd32(E1000_MANC); |
6864 | if (reg & E1000_MANC_EN_BMC2OS) { |
6865 | adapter->stats.o2bgptc += rd32(E1000_O2BGPTC); |
6866 | adapter->stats.o2bspc += rd32(E1000_O2BSPC); |
6867 | adapter->stats.b2ospc += rd32(E1000_B2OSPC); |
6868 | adapter->stats.b2ogprc += rd32(E1000_B2OGPRC); |
6869 | } |
6870 | } |
6871 | |
6872 | static void igb_perout(struct igb_adapter *adapter, int tsintr_tt) |
6873 | { |
6874 | int pin = ptp_find_pin(ptp: adapter->ptp_clock, func: PTP_PF_PEROUT, chan: tsintr_tt); |
6875 | struct e1000_hw *hw = &adapter->hw; |
6876 | struct timespec64 ts; |
6877 | u32 tsauxc; |
6878 | |
6879 | if (pin < 0 || pin >= IGB_N_SDP) |
6880 | return; |
6881 | |
6882 | spin_lock(lock: &adapter->tmreg_lock); |
6883 | |
6884 | if (hw->mac.type == e1000_82580 || |
6885 | hw->mac.type == e1000_i354 || |
6886 | hw->mac.type == e1000_i350) { |
6887 | s64 ns = timespec64_to_ns(ts: &adapter->perout[tsintr_tt].period); |
6888 | u32 systiml, systimh, level_mask, level, rem; |
6889 | u64 systim, now; |
6890 | |
6891 | /* read systim registers in sequence */ |
6892 | rd32(E1000_SYSTIMR); |
6893 | systiml = rd32(E1000_SYSTIML); |
6894 | systimh = rd32(E1000_SYSTIMH); |
6895 | systim = (((u64)(systimh & 0xFF)) << 32) | ((u64)systiml); |
6896 | now = timecounter_cyc2time(tc: &adapter->tc, cycle_tstamp: systim); |
6897 | |
6898 | if (pin < 2) { |
6899 | level_mask = (tsintr_tt == 1) ? 0x80000 : 0x40000; |
6900 | level = (rd32(E1000_CTRL) & level_mask) ? 1 : 0; |
6901 | } else { |
6902 | level_mask = (tsintr_tt == 1) ? 0x80 : 0x40; |
6903 | level = (rd32(E1000_CTRL_EXT) & level_mask) ? 1 : 0; |
6904 | } |
6905 | |
6906 | div_u64_rem(dividend: now, divisor: ns, remainder: &rem); |
6907 | systim = systim + (ns - rem); |
6908 | |
6909 | /* synchronize pin level with rising/falling edges */ |
6910 | div_u64_rem(dividend: now, divisor: ns << 1, remainder: &rem); |
6911 | if (rem < ns) { |
6912 | /* first half of period */ |
6913 | if (level == 0) { |
6914 | /* output is already low, skip this period */ |
6915 | systim += ns; |
6916 | pr_notice("igb: periodic output on %s missed falling edge\n" , |
6917 | adapter->sdp_config[pin].name); |
6918 | } |
6919 | } else { |
6920 | /* second half of period */ |
6921 | if (level == 1) { |
6922 | /* output is already high, skip this period */ |
6923 | systim += ns; |
6924 | pr_notice("igb: periodic output on %s missed rising edge\n" , |
6925 | adapter->sdp_config[pin].name); |
6926 | } |
6927 | } |
6928 | |
6929 | /* for this chip family tv_sec is the upper part of the binary value, |
6930 | * so not seconds |
6931 | */ |
6932 | ts.tv_nsec = (u32)systim; |
6933 | ts.tv_sec = ((u32)(systim >> 32)) & 0xFF; |
6934 | } else { |
6935 | ts = timespec64_add(lhs: adapter->perout[tsintr_tt].start, |
6936 | rhs: adapter->perout[tsintr_tt].period); |
6937 | } |
6938 | |
6939 | /* u32 conversion of tv_sec is safe until y2106 */ |
6940 | wr32((tsintr_tt == 1) ? E1000_TRGTTIML1 : E1000_TRGTTIML0, ts.tv_nsec); |
6941 | wr32((tsintr_tt == 1) ? E1000_TRGTTIMH1 : E1000_TRGTTIMH0, (u32)ts.tv_sec); |
6942 | tsauxc = rd32(E1000_TSAUXC); |
6943 | tsauxc |= TSAUXC_EN_TT0; |
6944 | wr32(E1000_TSAUXC, tsauxc); |
6945 | adapter->perout[tsintr_tt].start = ts; |
6946 | |
6947 | spin_unlock(lock: &adapter->tmreg_lock); |
6948 | } |
6949 | |
6950 | static void igb_extts(struct igb_adapter *adapter, int tsintr_tt) |
6951 | { |
6952 | int pin = ptp_find_pin(ptp: adapter->ptp_clock, func: PTP_PF_EXTTS, chan: tsintr_tt); |
6953 | int auxstmpl = (tsintr_tt == 1) ? E1000_AUXSTMPL1 : E1000_AUXSTMPL0; |
6954 | int auxstmph = (tsintr_tt == 1) ? E1000_AUXSTMPH1 : E1000_AUXSTMPH0; |
6955 | struct e1000_hw *hw = &adapter->hw; |
6956 | struct ptp_clock_event event; |
6957 | struct timespec64 ts; |
6958 | unsigned long flags; |
6959 | |
6960 | if (pin < 0 || pin >= IGB_N_SDP) |
6961 | return; |
6962 | |
6963 | if (hw->mac.type == e1000_82580 || |
6964 | hw->mac.type == e1000_i354 || |
6965 | hw->mac.type == e1000_i350) { |
6966 | u64 ns = rd32(auxstmpl); |
6967 | |
6968 | ns += ((u64)(rd32(auxstmph) & 0xFF)) << 32; |
6969 | spin_lock_irqsave(&adapter->tmreg_lock, flags); |
6970 | ns = timecounter_cyc2time(tc: &adapter->tc, cycle_tstamp: ns); |
6971 | spin_unlock_irqrestore(lock: &adapter->tmreg_lock, flags); |
6972 | ts = ns_to_timespec64(nsec: ns); |
6973 | } else { |
6974 | ts.tv_nsec = rd32(auxstmpl); |
6975 | ts.tv_sec = rd32(auxstmph); |
6976 | } |
6977 | |
6978 | event.type = PTP_CLOCK_EXTTS; |
6979 | event.index = tsintr_tt; |
6980 | event.timestamp = ts.tv_sec * 1000000000ULL + ts.tv_nsec; |
6981 | ptp_clock_event(ptp: adapter->ptp_clock, event: &event); |
6982 | } |
6983 | |
6984 | static void igb_tsync_interrupt(struct igb_adapter *adapter) |
6985 | { |
6986 | struct e1000_hw *hw = &adapter->hw; |
6987 | u32 ack = 0, tsicr = rd32(E1000_TSICR); |
6988 | struct ptp_clock_event event; |
6989 | |
6990 | if (tsicr & TSINTR_SYS_WRAP) { |
6991 | event.type = PTP_CLOCK_PPS; |
6992 | if (adapter->ptp_caps.pps) |
6993 | ptp_clock_event(ptp: adapter->ptp_clock, event: &event); |
6994 | ack |= TSINTR_SYS_WRAP; |
6995 | } |
6996 | |
6997 | if (tsicr & E1000_TSICR_TXTS) { |
6998 | /* retrieve hardware timestamp */ |
6999 | schedule_work(work: &adapter->ptp_tx_work); |
7000 | ack |= E1000_TSICR_TXTS; |
7001 | } |
7002 | |
7003 | if (tsicr & TSINTR_TT0) { |
7004 | igb_perout(adapter, tsintr_tt: 0); |
7005 | ack |= TSINTR_TT0; |
7006 | } |
7007 | |
7008 | if (tsicr & TSINTR_TT1) { |
7009 | igb_perout(adapter, tsintr_tt: 1); |
7010 | ack |= TSINTR_TT1; |
7011 | } |
7012 | |
7013 | if (tsicr & TSINTR_AUTT0) { |
7014 | igb_extts(adapter, tsintr_tt: 0); |
7015 | ack |= TSINTR_AUTT0; |
7016 | } |
7017 | |
7018 | if (tsicr & TSINTR_AUTT1) { |
7019 | igb_extts(adapter, tsintr_tt: 1); |
7020 | ack |= TSINTR_AUTT1; |
7021 | } |
7022 | |
7023 | /* acknowledge the interrupts */ |
7024 | wr32(E1000_TSICR, ack); |
7025 | } |
7026 | |
7027 | static irqreturn_t igb_msix_other(int irq, void *data) |
7028 | { |
7029 | struct igb_adapter *adapter = data; |
7030 | struct e1000_hw *hw = &adapter->hw; |
7031 | u32 icr = rd32(E1000_ICR); |
7032 | /* reading ICR causes bit 31 of EICR to be cleared */ |
7033 | |
7034 | if (icr & E1000_ICR_DRSTA) |
7035 | schedule_work(work: &adapter->reset_task); |
7036 | |
7037 | if (icr & E1000_ICR_DOUTSYNC) { |
7038 | /* HW is reporting DMA is out of sync */ |
7039 | adapter->stats.doosync++; |
7040 | /* The DMA Out of Sync is also indication of a spoof event |
7041 | * in IOV mode. Check the Wrong VM Behavior register to |
7042 | * see if it is really a spoof event. |
7043 | */ |
7044 | igb_check_wvbr(adapter); |
7045 | } |
7046 | |
7047 | /* Check for a mailbox event */ |
7048 | if (icr & E1000_ICR_VMMB) |
7049 | igb_msg_task(adapter); |
7050 | |
7051 | if (icr & E1000_ICR_LSC) { |
7052 | hw->mac.get_link_status = 1; |
7053 | /* guard against interrupt when we're going down */ |
7054 | if (!test_bit(__IGB_DOWN, &adapter->state)) |
7055 | mod_timer(timer: &adapter->watchdog_timer, expires: jiffies + 1); |
7056 | } |
7057 | |
7058 | if (icr & E1000_ICR_TS) |
7059 | igb_tsync_interrupt(adapter); |
7060 | |
7061 | wr32(E1000_EIMS, adapter->eims_other); |
7062 | |
7063 | return IRQ_HANDLED; |
7064 | } |
7065 | |
7066 | static void igb_write_itr(struct igb_q_vector *q_vector) |
7067 | { |
7068 | struct igb_adapter *adapter = q_vector->adapter; |
7069 | u32 itr_val = q_vector->itr_val & 0x7FFC; |
7070 | |
7071 | if (!q_vector->set_itr) |
7072 | return; |
7073 | |
7074 | if (!itr_val) |
7075 | itr_val = 0x4; |
7076 | |
7077 | if (adapter->hw.mac.type == e1000_82575) |
7078 | itr_val |= itr_val << 16; |
7079 | else |
7080 | itr_val |= E1000_EITR_CNT_IGNR; |
7081 | |
7082 | writel(val: itr_val, addr: q_vector->itr_register); |
7083 | q_vector->set_itr = 0; |
7084 | } |
7085 | |
7086 | static irqreturn_t igb_msix_ring(int irq, void *data) |
7087 | { |
7088 | struct igb_q_vector *q_vector = data; |
7089 | |
7090 | /* Write the ITR value calculated from the previous interrupt. */ |
7091 | igb_write_itr(q_vector); |
7092 | |
7093 | napi_schedule(n: &q_vector->napi); |
7094 | |
7095 | return IRQ_HANDLED; |
7096 | } |
7097 | |
7098 | #ifdef CONFIG_IGB_DCA |
7099 | static void igb_update_tx_dca(struct igb_adapter *adapter, |
7100 | struct igb_ring *tx_ring, |
7101 | int cpu) |
7102 | { |
7103 | struct e1000_hw *hw = &adapter->hw; |
7104 | u32 txctrl = dca3_get_tag(dev: tx_ring->dev, cpu); |
7105 | |
7106 | if (hw->mac.type != e1000_82575) |
7107 | txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT; |
7108 | |
7109 | /* We can enable relaxed ordering for reads, but not writes when |
7110 | * DCA is enabled. This is due to a known issue in some chipsets |
7111 | * which will cause the DCA tag to be cleared. |
7112 | */ |
7113 | txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN | |
7114 | E1000_DCA_TXCTRL_DATA_RRO_EN | |
7115 | E1000_DCA_TXCTRL_DESC_DCA_EN; |
7116 | |
7117 | wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl); |
7118 | } |
7119 | |
7120 | static void igb_update_rx_dca(struct igb_adapter *adapter, |
7121 | struct igb_ring *rx_ring, |
7122 | int cpu) |
7123 | { |
7124 | struct e1000_hw *hw = &adapter->hw; |
7125 | u32 rxctrl = dca3_get_tag(dev: &adapter->pdev->dev, cpu); |
7126 | |
7127 | if (hw->mac.type != e1000_82575) |
7128 | rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT; |
7129 | |
7130 | /* We can enable relaxed ordering for reads, but not writes when |
7131 | * DCA is enabled. This is due to a known issue in some chipsets |
7132 | * which will cause the DCA tag to be cleared. |
7133 | */ |
7134 | rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN | |
7135 | E1000_DCA_RXCTRL_DESC_DCA_EN; |
7136 | |
7137 | wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl); |
7138 | } |
7139 | |
7140 | static void igb_update_dca(struct igb_q_vector *q_vector) |
7141 | { |
7142 | struct igb_adapter *adapter = q_vector->adapter; |
7143 | int cpu = get_cpu(); |
7144 | |
7145 | if (q_vector->cpu == cpu) |
7146 | goto out_no_update; |
7147 | |
7148 | if (q_vector->tx.ring) |
7149 | igb_update_tx_dca(adapter, tx_ring: q_vector->tx.ring, cpu); |
7150 | |
7151 | if (q_vector->rx.ring) |
7152 | igb_update_rx_dca(adapter, rx_ring: q_vector->rx.ring, cpu); |
7153 | |
7154 | q_vector->cpu = cpu; |
7155 | out_no_update: |
7156 | put_cpu(); |
7157 | } |
7158 | |
7159 | static void igb_setup_dca(struct igb_adapter *adapter) |
7160 | { |
7161 | struct e1000_hw *hw = &adapter->hw; |
7162 | int i; |
7163 | |
7164 | if (!(adapter->flags & IGB_FLAG_DCA_ENABLED)) |
7165 | return; |
7166 | |
7167 | /* Always use CB2 mode, difference is masked in the CB driver. */ |
7168 | wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2); |
7169 | |
7170 | for (i = 0; i < adapter->num_q_vectors; i++) { |
7171 | adapter->q_vector[i]->cpu = -1; |
7172 | igb_update_dca(q_vector: adapter->q_vector[i]); |
7173 | } |
7174 | } |
7175 | |
7176 | static int __igb_notify_dca(struct device *dev, void *data) |
7177 | { |
7178 | struct net_device *netdev = dev_get_drvdata(dev); |
7179 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
7180 | struct pci_dev *pdev = adapter->pdev; |
7181 | struct e1000_hw *hw = &adapter->hw; |
7182 | unsigned long event = *(unsigned long *)data; |
7183 | |
7184 | switch (event) { |
7185 | case DCA_PROVIDER_ADD: |
7186 | /* if already enabled, don't do it again */ |
7187 | if (adapter->flags & IGB_FLAG_DCA_ENABLED) |
7188 | break; |
7189 | if (dca_add_requester(dev) == 0) { |
7190 | adapter->flags |= IGB_FLAG_DCA_ENABLED; |
7191 | dev_info(&pdev->dev, "DCA enabled\n" ); |
7192 | igb_setup_dca(adapter); |
7193 | break; |
7194 | } |
7195 | fallthrough; /* since DCA is disabled. */ |
7196 | case DCA_PROVIDER_REMOVE: |
7197 | if (adapter->flags & IGB_FLAG_DCA_ENABLED) { |
7198 | /* without this a class_device is left |
7199 | * hanging around in the sysfs model |
7200 | */ |
7201 | dca_remove_requester(dev); |
7202 | dev_info(&pdev->dev, "DCA disabled\n" ); |
7203 | adapter->flags &= ~IGB_FLAG_DCA_ENABLED; |
7204 | wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE); |
7205 | } |
7206 | break; |
7207 | } |
7208 | |
7209 | return 0; |
7210 | } |
7211 | |
7212 | static int igb_notify_dca(struct notifier_block *nb, unsigned long event, |
7213 | void *p) |
7214 | { |
7215 | int ret_val; |
7216 | |
7217 | ret_val = driver_for_each_device(drv: &igb_driver.driver, NULL, data: &event, |
7218 | fn: __igb_notify_dca); |
7219 | |
7220 | return ret_val ? NOTIFY_BAD : NOTIFY_DONE; |
7221 | } |
7222 | #endif /* CONFIG_IGB_DCA */ |
7223 | |
7224 | #ifdef CONFIG_PCI_IOV |
7225 | static int igb_vf_configure(struct igb_adapter *adapter, int vf) |
7226 | { |
7227 | unsigned char mac_addr[ETH_ALEN]; |
7228 | |
7229 | eth_zero_addr(addr: mac_addr); |
7230 | igb_set_vf_mac(adapter, vf, mac_addr); |
7231 | |
7232 | /* By default spoof check is enabled for all VFs */ |
7233 | adapter->vf_data[vf].spoofchk_enabled = true; |
7234 | |
7235 | /* By default VFs are not trusted */ |
7236 | adapter->vf_data[vf].trusted = false; |
7237 | |
7238 | return 0; |
7239 | } |
7240 | |
7241 | #endif |
7242 | static void igb_ping_all_vfs(struct igb_adapter *adapter) |
7243 | { |
7244 | struct e1000_hw *hw = &adapter->hw; |
7245 | u32 ping; |
7246 | int i; |
7247 | |
7248 | for (i = 0 ; i < adapter->vfs_allocated_count; i++) { |
7249 | ping = E1000_PF_CONTROL_MSG; |
7250 | if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS) |
7251 | ping |= E1000_VT_MSGTYPE_CTS; |
7252 | igb_write_mbx(hw, msg: &ping, size: 1, mbx_id: i); |
7253 | } |
7254 | } |
7255 | |
7256 | static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf) |
7257 | { |
7258 | struct e1000_hw *hw = &adapter->hw; |
7259 | u32 vmolr = rd32(E1000_VMOLR(vf)); |
7260 | struct vf_data_storage *vf_data = &adapter->vf_data[vf]; |
7261 | |
7262 | vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC | |
7263 | IGB_VF_FLAG_MULTI_PROMISC); |
7264 | vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME); |
7265 | |
7266 | if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) { |
7267 | vmolr |= E1000_VMOLR_MPME; |
7268 | vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC; |
7269 | *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST; |
7270 | } else { |
7271 | /* if we have hashes and we are clearing a multicast promisc |
7272 | * flag we need to write the hashes to the MTA as this step |
7273 | * was previously skipped |
7274 | */ |
7275 | if (vf_data->num_vf_mc_hashes > 30) { |
7276 | vmolr |= E1000_VMOLR_MPME; |
7277 | } else if (vf_data->num_vf_mc_hashes) { |
7278 | int j; |
7279 | |
7280 | vmolr |= E1000_VMOLR_ROMPE; |
7281 | for (j = 0; j < vf_data->num_vf_mc_hashes; j++) |
7282 | igb_mta_set(hw, hash_value: vf_data->vf_mc_hashes[j]); |
7283 | } |
7284 | } |
7285 | |
7286 | wr32(E1000_VMOLR(vf), vmolr); |
7287 | |
7288 | /* there are flags left unprocessed, likely not supported */ |
7289 | if (*msgbuf & E1000_VT_MSGINFO_MASK) |
7290 | return -EINVAL; |
7291 | |
7292 | return 0; |
7293 | } |
7294 | |
7295 | static int igb_set_vf_multicasts(struct igb_adapter *adapter, |
7296 | u32 *msgbuf, u32 vf) |
7297 | { |
7298 | int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT; |
7299 | u16 *hash_list = (u16 *)&msgbuf[1]; |
7300 | struct vf_data_storage *vf_data = &adapter->vf_data[vf]; |
7301 | int i; |
7302 | |
7303 | /* salt away the number of multicast addresses assigned |
7304 | * to this VF for later use to restore when the PF multi cast |
7305 | * list changes |
7306 | */ |
7307 | vf_data->num_vf_mc_hashes = n; |
7308 | |
7309 | /* only up to 30 hash values supported */ |
7310 | if (n > 30) |
7311 | n = 30; |
7312 | |
7313 | /* store the hashes for later use */ |
7314 | for (i = 0; i < n; i++) |
7315 | vf_data->vf_mc_hashes[i] = hash_list[i]; |
7316 | |
7317 | /* Flush and reset the mta with the new values */ |
7318 | igb_set_rx_mode(netdev: adapter->netdev); |
7319 | |
7320 | return 0; |
7321 | } |
7322 | |
7323 | static void igb_restore_vf_multicasts(struct igb_adapter *adapter) |
7324 | { |
7325 | struct e1000_hw *hw = &adapter->hw; |
7326 | struct vf_data_storage *vf_data; |
7327 | int i, j; |
7328 | |
7329 | for (i = 0; i < adapter->vfs_allocated_count; i++) { |
7330 | u32 vmolr = rd32(E1000_VMOLR(i)); |
7331 | |
7332 | vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME); |
7333 | |
7334 | vf_data = &adapter->vf_data[i]; |
7335 | |
7336 | if ((vf_data->num_vf_mc_hashes > 30) || |
7337 | (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) { |
7338 | vmolr |= E1000_VMOLR_MPME; |
7339 | } else if (vf_data->num_vf_mc_hashes) { |
7340 | vmolr |= E1000_VMOLR_ROMPE; |
7341 | for (j = 0; j < vf_data->num_vf_mc_hashes; j++) |
7342 | igb_mta_set(hw, hash_value: vf_data->vf_mc_hashes[j]); |
7343 | } |
7344 | wr32(E1000_VMOLR(i), vmolr); |
7345 | } |
7346 | } |
7347 | |
7348 | static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf) |
7349 | { |
7350 | struct e1000_hw *hw = &adapter->hw; |
7351 | u32 pool_mask, vlvf_mask, i; |
7352 | |
7353 | /* create mask for VF and other pools */ |
7354 | pool_mask = E1000_VLVF_POOLSEL_MASK; |
7355 | vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf); |
7356 | |
7357 | /* drop PF from pool bits */ |
7358 | pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT + |
7359 | adapter->vfs_allocated_count); |
7360 | |
7361 | /* Find the vlan filter for this id */ |
7362 | for (i = E1000_VLVF_ARRAY_SIZE; i--;) { |
7363 | u32 vlvf = rd32(E1000_VLVF(i)); |
7364 | u32 vfta_mask, vid, vfta; |
7365 | |
7366 | /* remove the vf from the pool */ |
7367 | if (!(vlvf & vlvf_mask)) |
7368 | continue; |
7369 | |
7370 | /* clear out bit from VLVF */ |
7371 | vlvf ^= vlvf_mask; |
7372 | |
7373 | /* if other pools are present, just remove ourselves */ |
7374 | if (vlvf & pool_mask) |
7375 | goto update_vlvfb; |
7376 | |
7377 | /* if PF is present, leave VFTA */ |
7378 | if (vlvf & E1000_VLVF_POOLSEL_MASK) |
7379 | goto update_vlvf; |
7380 | |
7381 | vid = vlvf & E1000_VLVF_VLANID_MASK; |
7382 | vfta_mask = BIT(vid % 32); |
7383 | |
7384 | /* clear bit from VFTA */ |
7385 | vfta = adapter->shadow_vfta[vid / 32]; |
7386 | if (vfta & vfta_mask) |
7387 | hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask); |
7388 | update_vlvf: |
7389 | /* clear pool selection enable */ |
7390 | if (adapter->flags & IGB_FLAG_VLAN_PROMISC) |
7391 | vlvf &= E1000_VLVF_POOLSEL_MASK; |
7392 | else |
7393 | vlvf = 0; |
7394 | update_vlvfb: |
7395 | /* clear pool bits */ |
7396 | wr32(E1000_VLVF(i), vlvf); |
7397 | } |
7398 | } |
7399 | |
7400 | static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan) |
7401 | { |
7402 | u32 vlvf; |
7403 | int idx; |
7404 | |
7405 | /* short cut the special case */ |
7406 | if (vlan == 0) |
7407 | return 0; |
7408 | |
7409 | /* Search for the VLAN id in the VLVF entries */ |
7410 | for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) { |
7411 | vlvf = rd32(E1000_VLVF(idx)); |
7412 | if ((vlvf & VLAN_VID_MASK) == vlan) |
7413 | break; |
7414 | } |
7415 | |
7416 | return idx; |
7417 | } |
7418 | |
7419 | static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid) |
7420 | { |
7421 | struct e1000_hw *hw = &adapter->hw; |
7422 | u32 bits, pf_id; |
7423 | int idx; |
7424 | |
7425 | idx = igb_find_vlvf_entry(hw, vlan: vid); |
7426 | if (!idx) |
7427 | return; |
7428 | |
7429 | /* See if any other pools are set for this VLAN filter |
7430 | * entry other than the PF. |
7431 | */ |
7432 | pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT; |
7433 | bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK; |
7434 | bits &= rd32(E1000_VLVF(idx)); |
7435 | |
7436 | /* Disable the filter so this falls into the default pool. */ |
7437 | if (!bits) { |
7438 | if (adapter->flags & IGB_FLAG_VLAN_PROMISC) |
7439 | wr32(E1000_VLVF(idx), BIT(pf_id)); |
7440 | else |
7441 | wr32(E1000_VLVF(idx), 0); |
7442 | } |
7443 | } |
7444 | |
7445 | static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid, |
7446 | bool add, u32 vf) |
7447 | { |
7448 | int pf_id = adapter->vfs_allocated_count; |
7449 | struct e1000_hw *hw = &adapter->hw; |
7450 | int err; |
7451 | |
7452 | /* If VLAN overlaps with one the PF is currently monitoring make |
7453 | * sure that we are able to allocate a VLVF entry. This may be |
7454 | * redundant but it guarantees PF will maintain visibility to |
7455 | * the VLAN. |
7456 | */ |
7457 | if (add && test_bit(vid, adapter->active_vlans)) { |
7458 | err = igb_vfta_set(hw, vid, vind: pf_id, vlan_on: true, vlvf_bypass: false); |
7459 | if (err) |
7460 | return err; |
7461 | } |
7462 | |
7463 | err = igb_vfta_set(hw, vid, vind: vf, vlan_on: add, vlvf_bypass: false); |
7464 | |
7465 | if (add && !err) |
7466 | return err; |
7467 | |
7468 | /* If we failed to add the VF VLAN or we are removing the VF VLAN |
7469 | * we may need to drop the PF pool bit in order to allow us to free |
7470 | * up the VLVF resources. |
7471 | */ |
7472 | if (test_bit(vid, adapter->active_vlans) || |
7473 | (adapter->flags & IGB_FLAG_VLAN_PROMISC)) |
7474 | igb_update_pf_vlvf(adapter, vid); |
7475 | |
7476 | return err; |
7477 | } |
7478 | |
7479 | static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf) |
7480 | { |
7481 | struct e1000_hw *hw = &adapter->hw; |
7482 | |
7483 | if (vid) |
7484 | wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT)); |
7485 | else |
7486 | wr32(E1000_VMVIR(vf), 0); |
7487 | } |
7488 | |
7489 | static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf, |
7490 | u16 vlan, u8 qos) |
7491 | { |
7492 | int err; |
7493 | |
7494 | err = igb_set_vf_vlan(adapter, vid: vlan, add: true, vf); |
7495 | if (err) |
7496 | return err; |
7497 | |
7498 | igb_set_vmvir(adapter, vid: vlan | (qos << VLAN_PRIO_SHIFT), vf); |
7499 | igb_set_vmolr(adapter, vfn: vf, aupe: !vlan); |
7500 | |
7501 | /* revoke access to previous VLAN */ |
7502 | if (vlan != adapter->vf_data[vf].pf_vlan) |
7503 | igb_set_vf_vlan(adapter, vid: adapter->vf_data[vf].pf_vlan, |
7504 | add: false, vf); |
7505 | |
7506 | adapter->vf_data[vf].pf_vlan = vlan; |
7507 | adapter->vf_data[vf].pf_qos = qos; |
7508 | igb_set_vf_vlan_strip(adapter, vfn: vf, enable: true); |
7509 | dev_info(&adapter->pdev->dev, |
7510 | "Setting VLAN %d, QOS 0x%x on VF %d\n" , vlan, qos, vf); |
7511 | if (test_bit(__IGB_DOWN, &adapter->state)) { |
7512 | dev_warn(&adapter->pdev->dev, |
7513 | "The VF VLAN has been set, but the PF device is not up.\n" ); |
7514 | dev_warn(&adapter->pdev->dev, |
7515 | "Bring the PF device up before attempting to use the VF device.\n" ); |
7516 | } |
7517 | |
7518 | return err; |
7519 | } |
7520 | |
7521 | static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf) |
7522 | { |
7523 | /* Restore tagless access via VLAN 0 */ |
7524 | igb_set_vf_vlan(adapter, vid: 0, add: true, vf); |
7525 | |
7526 | igb_set_vmvir(adapter, vid: 0, vf); |
7527 | igb_set_vmolr(adapter, vfn: vf, aupe: true); |
7528 | |
7529 | /* Remove any PF assigned VLAN */ |
7530 | if (adapter->vf_data[vf].pf_vlan) |
7531 | igb_set_vf_vlan(adapter, vid: adapter->vf_data[vf].pf_vlan, |
7532 | add: false, vf); |
7533 | |
7534 | adapter->vf_data[vf].pf_vlan = 0; |
7535 | adapter->vf_data[vf].pf_qos = 0; |
7536 | igb_set_vf_vlan_strip(adapter, vfn: vf, enable: false); |
7537 | |
7538 | return 0; |
7539 | } |
7540 | |
7541 | static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf, |
7542 | u16 vlan, u8 qos, __be16 vlan_proto) |
7543 | { |
7544 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
7545 | |
7546 | if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7)) |
7547 | return -EINVAL; |
7548 | |
7549 | if (vlan_proto != htons(ETH_P_8021Q)) |
7550 | return -EPROTONOSUPPORT; |
7551 | |
7552 | return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) : |
7553 | igb_disable_port_vlan(adapter, vf); |
7554 | } |
7555 | |
7556 | static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf) |
7557 | { |
7558 | int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT; |
7559 | int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK); |
7560 | int ret; |
7561 | |
7562 | if (adapter->vf_data[vf].pf_vlan) |
7563 | return -1; |
7564 | |
7565 | /* VLAN 0 is a special case, don't allow it to be removed */ |
7566 | if (!vid && !add) |
7567 | return 0; |
7568 | |
7569 | ret = igb_set_vf_vlan(adapter, vid, add: !!add, vf); |
7570 | if (!ret) |
7571 | igb_set_vf_vlan_strip(adapter, vfn: vf, enable: !!vid); |
7572 | return ret; |
7573 | } |
7574 | |
7575 | static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf) |
7576 | { |
7577 | struct vf_data_storage *vf_data = &adapter->vf_data[vf]; |
7578 | |
7579 | /* clear flags - except flag that indicates PF has set the MAC */ |
7580 | vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC; |
7581 | vf_data->last_nack = jiffies; |
7582 | |
7583 | /* reset vlans for device */ |
7584 | igb_clear_vf_vfta(adapter, vf); |
7585 | igb_set_vf_vlan(adapter, vid: vf_data->pf_vlan, add: true, vf); |
7586 | igb_set_vmvir(adapter, vid: vf_data->pf_vlan | |
7587 | (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf); |
7588 | igb_set_vmolr(adapter, vfn: vf, aupe: !vf_data->pf_vlan); |
7589 | igb_set_vf_vlan_strip(adapter, vfn: vf, enable: !!(vf_data->pf_vlan)); |
7590 | |
7591 | /* reset multicast table array for vf */ |
7592 | adapter->vf_data[vf].num_vf_mc_hashes = 0; |
7593 | |
7594 | /* Flush and reset the mta with the new values */ |
7595 | igb_set_rx_mode(netdev: adapter->netdev); |
7596 | } |
7597 | |
7598 | static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf) |
7599 | { |
7600 | unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses; |
7601 | |
7602 | /* clear mac address as we were hotplug removed/added */ |
7603 | if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC)) |
7604 | eth_zero_addr(addr: vf_mac); |
7605 | |
7606 | /* process remaining reset events */ |
7607 | igb_vf_reset(adapter, vf); |
7608 | } |
7609 | |
7610 | static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf) |
7611 | { |
7612 | struct e1000_hw *hw = &adapter->hw; |
7613 | unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses; |
7614 | u32 reg, msgbuf[3] = {}; |
7615 | u8 *addr = (u8 *)(&msgbuf[1]); |
7616 | |
7617 | /* process all the same items cleared in a function level reset */ |
7618 | igb_vf_reset(adapter, vf); |
7619 | |
7620 | /* set vf mac address */ |
7621 | igb_set_vf_mac(adapter, vf, vf_mac); |
7622 | |
7623 | /* enable transmit and receive for vf */ |
7624 | reg = rd32(E1000_VFTE); |
7625 | wr32(E1000_VFTE, reg | BIT(vf)); |
7626 | reg = rd32(E1000_VFRE); |
7627 | wr32(E1000_VFRE, reg | BIT(vf)); |
7628 | |
7629 | adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS; |
7630 | |
7631 | /* reply to reset with ack and vf mac address */ |
7632 | if (!is_zero_ether_addr(addr: vf_mac)) { |
7633 | msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK; |
7634 | memcpy(addr, vf_mac, ETH_ALEN); |
7635 | } else { |
7636 | msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK; |
7637 | } |
7638 | igb_write_mbx(hw, msg: msgbuf, size: 3, mbx_id: vf); |
7639 | } |
7640 | |
7641 | static void igb_flush_mac_table(struct igb_adapter *adapter) |
7642 | { |
7643 | struct e1000_hw *hw = &adapter->hw; |
7644 | int i; |
7645 | |
7646 | for (i = 0; i < hw->mac.rar_entry_count; i++) { |
7647 | adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE; |
7648 | eth_zero_addr(addr: adapter->mac_table[i].addr); |
7649 | adapter->mac_table[i].queue = 0; |
7650 | igb_rar_set_index(adapter, i); |
7651 | } |
7652 | } |
7653 | |
7654 | static int igb_available_rars(struct igb_adapter *adapter, u8 queue) |
7655 | { |
7656 | struct e1000_hw *hw = &adapter->hw; |
7657 | /* do not count rar entries reserved for VFs MAC addresses */ |
7658 | int rar_entries = hw->mac.rar_entry_count - |
7659 | adapter->vfs_allocated_count; |
7660 | int i, count = 0; |
7661 | |
7662 | for (i = 0; i < rar_entries; i++) { |
7663 | /* do not count default entries */ |
7664 | if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) |
7665 | continue; |
7666 | |
7667 | /* do not count "in use" entries for different queues */ |
7668 | if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) && |
7669 | (adapter->mac_table[i].queue != queue)) |
7670 | continue; |
7671 | |
7672 | count++; |
7673 | } |
7674 | |
7675 | return count; |
7676 | } |
7677 | |
7678 | /* Set default MAC address for the PF in the first RAR entry */ |
7679 | static void igb_set_default_mac_filter(struct igb_adapter *adapter) |
7680 | { |
7681 | struct igb_mac_addr *mac_table = &adapter->mac_table[0]; |
7682 | |
7683 | ether_addr_copy(dst: mac_table->addr, src: adapter->hw.mac.addr); |
7684 | mac_table->queue = adapter->vfs_allocated_count; |
7685 | mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE; |
7686 | |
7687 | igb_rar_set_index(adapter, 0); |
7688 | } |
7689 | |
7690 | /* If the filter to be added and an already existing filter express |
7691 | * the same address and address type, it should be possible to only |
7692 | * override the other configurations, for example the queue to steer |
7693 | * traffic. |
7694 | */ |
7695 | static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry, |
7696 | const u8 *addr, const u8 flags) |
7697 | { |
7698 | if (!(entry->state & IGB_MAC_STATE_IN_USE)) |
7699 | return true; |
7700 | |
7701 | if ((entry->state & IGB_MAC_STATE_SRC_ADDR) != |
7702 | (flags & IGB_MAC_STATE_SRC_ADDR)) |
7703 | return false; |
7704 | |
7705 | if (!ether_addr_equal(addr1: addr, addr2: entry->addr)) |
7706 | return false; |
7707 | |
7708 | return true; |
7709 | } |
7710 | |
7711 | /* Add a MAC filter for 'addr' directing matching traffic to 'queue', |
7712 | * 'flags' is used to indicate what kind of match is made, match is by |
7713 | * default for the destination address, if matching by source address |
7714 | * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used. |
7715 | */ |
7716 | static int igb_add_mac_filter_flags(struct igb_adapter *adapter, |
7717 | const u8 *addr, const u8 queue, |
7718 | const u8 flags) |
7719 | { |
7720 | struct e1000_hw *hw = &adapter->hw; |
7721 | int rar_entries = hw->mac.rar_entry_count - |
7722 | adapter->vfs_allocated_count; |
7723 | int i; |
7724 | |
7725 | if (is_zero_ether_addr(addr)) |
7726 | return -EINVAL; |
7727 | |
7728 | /* Search for the first empty entry in the MAC table. |
7729 | * Do not touch entries at the end of the table reserved for the VF MAC |
7730 | * addresses. |
7731 | */ |
7732 | for (i = 0; i < rar_entries; i++) { |
7733 | if (!igb_mac_entry_can_be_used(entry: &adapter->mac_table[i], |
7734 | addr, flags)) |
7735 | continue; |
7736 | |
7737 | ether_addr_copy(dst: adapter->mac_table[i].addr, src: addr); |
7738 | adapter->mac_table[i].queue = queue; |
7739 | adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags; |
7740 | |
7741 | igb_rar_set_index(adapter, i); |
7742 | return i; |
7743 | } |
7744 | |
7745 | return -ENOSPC; |
7746 | } |
7747 | |
7748 | static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr, |
7749 | const u8 queue) |
7750 | { |
7751 | return igb_add_mac_filter_flags(adapter, addr, queue, flags: 0); |
7752 | } |
7753 | |
7754 | /* Remove a MAC filter for 'addr' directing matching traffic to |
7755 | * 'queue', 'flags' is used to indicate what kind of match need to be |
7756 | * removed, match is by default for the destination address, if |
7757 | * matching by source address is to be removed the flag |
7758 | * IGB_MAC_STATE_SRC_ADDR can be used. |
7759 | */ |
7760 | static int igb_del_mac_filter_flags(struct igb_adapter *adapter, |
7761 | const u8 *addr, const u8 queue, |
7762 | const u8 flags) |
7763 | { |
7764 | struct e1000_hw *hw = &adapter->hw; |
7765 | int rar_entries = hw->mac.rar_entry_count - |
7766 | adapter->vfs_allocated_count; |
7767 | int i; |
7768 | |
7769 | if (is_zero_ether_addr(addr)) |
7770 | return -EINVAL; |
7771 | |
7772 | /* Search for matching entry in the MAC table based on given address |
7773 | * and queue. Do not touch entries at the end of the table reserved |
7774 | * for the VF MAC addresses. |
7775 | */ |
7776 | for (i = 0; i < rar_entries; i++) { |
7777 | if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)) |
7778 | continue; |
7779 | if ((adapter->mac_table[i].state & flags) != flags) |
7780 | continue; |
7781 | if (adapter->mac_table[i].queue != queue) |
7782 | continue; |
7783 | if (!ether_addr_equal(addr1: adapter->mac_table[i].addr, addr2: addr)) |
7784 | continue; |
7785 | |
7786 | /* When a filter for the default address is "deleted", |
7787 | * we return it to its initial configuration |
7788 | */ |
7789 | if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) { |
7790 | adapter->mac_table[i].state = |
7791 | IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE; |
7792 | adapter->mac_table[i].queue = |
7793 | adapter->vfs_allocated_count; |
7794 | } else { |
7795 | adapter->mac_table[i].state = 0; |
7796 | adapter->mac_table[i].queue = 0; |
7797 | eth_zero_addr(addr: adapter->mac_table[i].addr); |
7798 | } |
7799 | |
7800 | igb_rar_set_index(adapter, i); |
7801 | return 0; |
7802 | } |
7803 | |
7804 | return -ENOENT; |
7805 | } |
7806 | |
7807 | static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr, |
7808 | const u8 queue) |
7809 | { |
7810 | return igb_del_mac_filter_flags(adapter, addr, queue, flags: 0); |
7811 | } |
7812 | |
7813 | int igb_add_mac_steering_filter(struct igb_adapter *adapter, |
7814 | const u8 *addr, u8 queue, u8 flags) |
7815 | { |
7816 | struct e1000_hw *hw = &adapter->hw; |
7817 | |
7818 | /* In theory, this should be supported on 82575 as well, but |
7819 | * that part wasn't easily accessible during development. |
7820 | */ |
7821 | if (hw->mac.type != e1000_i210) |
7822 | return -EOPNOTSUPP; |
7823 | |
7824 | return igb_add_mac_filter_flags(adapter, addr, queue, |
7825 | IGB_MAC_STATE_QUEUE_STEERING | flags); |
7826 | } |
7827 | |
7828 | int igb_del_mac_steering_filter(struct igb_adapter *adapter, |
7829 | const u8 *addr, u8 queue, u8 flags) |
7830 | { |
7831 | return igb_del_mac_filter_flags(adapter, addr, queue, |
7832 | IGB_MAC_STATE_QUEUE_STEERING | flags); |
7833 | } |
7834 | |
7835 | static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr) |
7836 | { |
7837 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
7838 | int ret; |
7839 | |
7840 | ret = igb_add_mac_filter(adapter, addr, queue: adapter->vfs_allocated_count); |
7841 | |
7842 | return min_t(int, ret, 0); |
7843 | } |
7844 | |
7845 | static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr) |
7846 | { |
7847 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
7848 | |
7849 | igb_del_mac_filter(adapter, addr, queue: adapter->vfs_allocated_count); |
7850 | |
7851 | return 0; |
7852 | } |
7853 | |
7854 | static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf, |
7855 | const u32 info, const u8 *addr) |
7856 | { |
7857 | struct pci_dev *pdev = adapter->pdev; |
7858 | struct vf_data_storage *vf_data = &adapter->vf_data[vf]; |
7859 | struct vf_mac_filter *entry; |
7860 | bool found = false; |
7861 | int ret = 0; |
7862 | |
7863 | if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) && |
7864 | !vf_data->trusted) { |
7865 | dev_warn(&pdev->dev, |
7866 | "VF %d requested MAC filter but is administratively denied\n" , |
7867 | vf); |
7868 | return -EINVAL; |
7869 | } |
7870 | if (!is_valid_ether_addr(addr)) { |
7871 | dev_warn(&pdev->dev, |
7872 | "VF %d attempted to set invalid MAC filter\n" , |
7873 | vf); |
7874 | return -EINVAL; |
7875 | } |
7876 | |
7877 | switch (info) { |
7878 | case E1000_VF_MAC_FILTER_CLR: |
7879 | /* remove all unicast MAC filters related to the current VF */ |
7880 | list_for_each_entry(entry, &adapter->vf_macs.l, l) { |
7881 | if (entry->vf == vf) { |
7882 | entry->vf = -1; |
7883 | entry->free = true; |
7884 | igb_del_mac_filter(adapter, addr: entry->vf_mac, queue: vf); |
7885 | } |
7886 | } |
7887 | break; |
7888 | case E1000_VF_MAC_FILTER_ADD: |
7889 | /* try to find empty slot in the list */ |
7890 | list_for_each_entry(entry, &adapter->vf_macs.l, l) { |
7891 | if (entry->free) { |
7892 | found = true; |
7893 | break; |
7894 | } |
7895 | } |
7896 | |
7897 | if (found) { |
7898 | entry->free = false; |
7899 | entry->vf = vf; |
7900 | ether_addr_copy(dst: entry->vf_mac, src: addr); |
7901 | |
7902 | ret = igb_add_mac_filter(adapter, addr, queue: vf); |
7903 | ret = min_t(int, ret, 0); |
7904 | } else { |
7905 | ret = -ENOSPC; |
7906 | } |
7907 | |
7908 | if (ret == -ENOSPC) |
7909 | dev_warn(&pdev->dev, |
7910 | "VF %d has requested MAC filter but there is no space for it\n" , |
7911 | vf); |
7912 | break; |
7913 | default: |
7914 | ret = -EINVAL; |
7915 | break; |
7916 | } |
7917 | |
7918 | return ret; |
7919 | } |
7920 | |
7921 | static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf) |
7922 | { |
7923 | struct pci_dev *pdev = adapter->pdev; |
7924 | struct vf_data_storage *vf_data = &adapter->vf_data[vf]; |
7925 | u32 info = msg[0] & E1000_VT_MSGINFO_MASK; |
7926 | |
7927 | /* The VF MAC Address is stored in a packed array of bytes |
7928 | * starting at the second 32 bit word of the msg array |
7929 | */ |
7930 | unsigned char *addr = (unsigned char *)&msg[1]; |
7931 | int ret = 0; |
7932 | |
7933 | if (!info) { |
7934 | if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) && |
7935 | !vf_data->trusted) { |
7936 | dev_warn(&pdev->dev, |
7937 | "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n" , |
7938 | vf); |
7939 | return -EINVAL; |
7940 | } |
7941 | |
7942 | if (!is_valid_ether_addr(addr)) { |
7943 | dev_warn(&pdev->dev, |
7944 | "VF %d attempted to set invalid MAC\n" , |
7945 | vf); |
7946 | return -EINVAL; |
7947 | } |
7948 | |
7949 | ret = igb_set_vf_mac(adapter, vf, addr); |
7950 | } else { |
7951 | ret = igb_set_vf_mac_filter(adapter, vf, info, addr); |
7952 | } |
7953 | |
7954 | return ret; |
7955 | } |
7956 | |
7957 | static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf) |
7958 | { |
7959 | struct e1000_hw *hw = &adapter->hw; |
7960 | struct vf_data_storage *vf_data = &adapter->vf_data[vf]; |
7961 | u32 msg = E1000_VT_MSGTYPE_NACK; |
7962 | |
7963 | /* if device isn't clear to send it shouldn't be reading either */ |
7964 | if (!(vf_data->flags & IGB_VF_FLAG_CTS) && |
7965 | time_after(jiffies, vf_data->last_nack + (2 * HZ))) { |
7966 | igb_write_mbx(hw, msg: &msg, size: 1, mbx_id: vf); |
7967 | vf_data->last_nack = jiffies; |
7968 | } |
7969 | } |
7970 | |
7971 | static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf) |
7972 | { |
7973 | struct pci_dev *pdev = adapter->pdev; |
7974 | u32 msgbuf[E1000_VFMAILBOX_SIZE]; |
7975 | struct e1000_hw *hw = &adapter->hw; |
7976 | struct vf_data_storage *vf_data = &adapter->vf_data[vf]; |
7977 | s32 retval; |
7978 | |
7979 | retval = igb_read_mbx(hw, msg: msgbuf, E1000_VFMAILBOX_SIZE, mbx_id: vf, unlock: false); |
7980 | |
7981 | if (retval) { |
7982 | /* if receive failed revoke VF CTS stats and restart init */ |
7983 | dev_err(&pdev->dev, "Error receiving message from VF\n" ); |
7984 | vf_data->flags &= ~IGB_VF_FLAG_CTS; |
7985 | if (!time_after(jiffies, vf_data->last_nack + (2 * HZ))) |
7986 | goto unlock; |
7987 | goto out; |
7988 | } |
7989 | |
7990 | /* this is a message we already processed, do nothing */ |
7991 | if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK)) |
7992 | goto unlock; |
7993 | |
7994 | /* until the vf completes a reset it should not be |
7995 | * allowed to start any configuration. |
7996 | */ |
7997 | if (msgbuf[0] == E1000_VF_RESET) { |
7998 | /* unlocks mailbox */ |
7999 | igb_vf_reset_msg(adapter, vf); |
8000 | return; |
8001 | } |
8002 | |
8003 | if (!(vf_data->flags & IGB_VF_FLAG_CTS)) { |
8004 | if (!time_after(jiffies, vf_data->last_nack + (2 * HZ))) |
8005 | goto unlock; |
8006 | retval = -1; |
8007 | goto out; |
8008 | } |
8009 | |
8010 | switch ((msgbuf[0] & 0xFFFF)) { |
8011 | case E1000_VF_SET_MAC_ADDR: |
8012 | retval = igb_set_vf_mac_addr(adapter, msg: msgbuf, vf); |
8013 | break; |
8014 | case E1000_VF_SET_PROMISC: |
8015 | retval = igb_set_vf_promisc(adapter, msgbuf, vf); |
8016 | break; |
8017 | case E1000_VF_SET_MULTICAST: |
8018 | retval = igb_set_vf_multicasts(adapter, msgbuf, vf); |
8019 | break; |
8020 | case E1000_VF_SET_LPE: |
8021 | retval = igb_set_vf_rlpml(adapter, size: msgbuf[1], vfn: vf); |
8022 | break; |
8023 | case E1000_VF_SET_VLAN: |
8024 | retval = -1; |
8025 | if (vf_data->pf_vlan) |
8026 | dev_warn(&pdev->dev, |
8027 | "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n" , |
8028 | vf); |
8029 | else |
8030 | retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf); |
8031 | break; |
8032 | default: |
8033 | dev_err(&pdev->dev, "Unhandled Msg %08x\n" , msgbuf[0]); |
8034 | retval = -1; |
8035 | break; |
8036 | } |
8037 | |
8038 | msgbuf[0] |= E1000_VT_MSGTYPE_CTS; |
8039 | out: |
8040 | /* notify the VF of the results of what it sent us */ |
8041 | if (retval) |
8042 | msgbuf[0] |= E1000_VT_MSGTYPE_NACK; |
8043 | else |
8044 | msgbuf[0] |= E1000_VT_MSGTYPE_ACK; |
8045 | |
8046 | /* unlocks mailbox */ |
8047 | igb_write_mbx(hw, msg: msgbuf, size: 1, mbx_id: vf); |
8048 | return; |
8049 | |
8050 | unlock: |
8051 | igb_unlock_mbx(hw, mbx_id: vf); |
8052 | } |
8053 | |
8054 | static void igb_msg_task(struct igb_adapter *adapter) |
8055 | { |
8056 | struct e1000_hw *hw = &adapter->hw; |
8057 | unsigned long flags; |
8058 | u32 vf; |
8059 | |
8060 | spin_lock_irqsave(&adapter->vfs_lock, flags); |
8061 | for (vf = 0; vf < adapter->vfs_allocated_count; vf++) { |
8062 | /* process any reset requests */ |
8063 | if (!igb_check_for_rst(hw, mbx_id: vf)) |
8064 | igb_vf_reset_event(adapter, vf); |
8065 | |
8066 | /* process any messages pending */ |
8067 | if (!igb_check_for_msg(hw, mbx_id: vf)) |
8068 | igb_rcv_msg_from_vf(adapter, vf); |
8069 | |
8070 | /* process any acks */ |
8071 | if (!igb_check_for_ack(hw, mbx_id: vf)) |
8072 | igb_rcv_ack_from_vf(adapter, vf); |
8073 | } |
8074 | spin_unlock_irqrestore(lock: &adapter->vfs_lock, flags); |
8075 | } |
8076 | |
8077 | /** |
8078 | * igb_set_uta - Set unicast filter table address |
8079 | * @adapter: board private structure |
8080 | * @set: boolean indicating if we are setting or clearing bits |
8081 | * |
8082 | * The unicast table address is a register array of 32-bit registers. |
8083 | * The table is meant to be used in a way similar to how the MTA is used |
8084 | * however due to certain limitations in the hardware it is necessary to |
8085 | * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous |
8086 | * enable bit to allow vlan tag stripping when promiscuous mode is enabled |
8087 | **/ |
8088 | static void igb_set_uta(struct igb_adapter *adapter, bool set) |
8089 | { |
8090 | struct e1000_hw *hw = &adapter->hw; |
8091 | u32 uta = set ? ~0 : 0; |
8092 | int i; |
8093 | |
8094 | /* we only need to do this if VMDq is enabled */ |
8095 | if (!adapter->vfs_allocated_count) |
8096 | return; |
8097 | |
8098 | for (i = hw->mac.uta_reg_count; i--;) |
8099 | array_wr32(E1000_UTA, i, uta); |
8100 | } |
8101 | |
8102 | /** |
8103 | * igb_intr_msi - Interrupt Handler |
8104 | * @irq: interrupt number |
8105 | * @data: pointer to a network interface device structure |
8106 | **/ |
8107 | static irqreturn_t igb_intr_msi(int irq, void *data) |
8108 | { |
8109 | struct igb_adapter *adapter = data; |
8110 | struct igb_q_vector *q_vector = adapter->q_vector[0]; |
8111 | struct e1000_hw *hw = &adapter->hw; |
8112 | /* read ICR disables interrupts using IAM */ |
8113 | u32 icr = rd32(E1000_ICR); |
8114 | |
8115 | igb_write_itr(q_vector); |
8116 | |
8117 | if (icr & E1000_ICR_DRSTA) |
8118 | schedule_work(work: &adapter->reset_task); |
8119 | |
8120 | if (icr & E1000_ICR_DOUTSYNC) { |
8121 | /* HW is reporting DMA is out of sync */ |
8122 | adapter->stats.doosync++; |
8123 | } |
8124 | |
8125 | if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { |
8126 | hw->mac.get_link_status = 1; |
8127 | if (!test_bit(__IGB_DOWN, &adapter->state)) |
8128 | mod_timer(timer: &adapter->watchdog_timer, expires: jiffies + 1); |
8129 | } |
8130 | |
8131 | if (icr & E1000_ICR_TS) |
8132 | igb_tsync_interrupt(adapter); |
8133 | |
8134 | napi_schedule(n: &q_vector->napi); |
8135 | |
8136 | return IRQ_HANDLED; |
8137 | } |
8138 | |
8139 | /** |
8140 | * igb_intr - Legacy Interrupt Handler |
8141 | * @irq: interrupt number |
8142 | * @data: pointer to a network interface device structure |
8143 | **/ |
8144 | static irqreturn_t igb_intr(int irq, void *data) |
8145 | { |
8146 | struct igb_adapter *adapter = data; |
8147 | struct igb_q_vector *q_vector = adapter->q_vector[0]; |
8148 | struct e1000_hw *hw = &adapter->hw; |
8149 | /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No |
8150 | * need for the IMC write |
8151 | */ |
8152 | u32 icr = rd32(E1000_ICR); |
8153 | |
8154 | /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is |
8155 | * not set, then the adapter didn't send an interrupt |
8156 | */ |
8157 | if (!(icr & E1000_ICR_INT_ASSERTED)) |
8158 | return IRQ_NONE; |
8159 | |
8160 | igb_write_itr(q_vector); |
8161 | |
8162 | if (icr & E1000_ICR_DRSTA) |
8163 | schedule_work(work: &adapter->reset_task); |
8164 | |
8165 | if (icr & E1000_ICR_DOUTSYNC) { |
8166 | /* HW is reporting DMA is out of sync */ |
8167 | adapter->stats.doosync++; |
8168 | } |
8169 | |
8170 | if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { |
8171 | hw->mac.get_link_status = 1; |
8172 | /* guard against interrupt when we're going down */ |
8173 | if (!test_bit(__IGB_DOWN, &adapter->state)) |
8174 | mod_timer(timer: &adapter->watchdog_timer, expires: jiffies + 1); |
8175 | } |
8176 | |
8177 | if (icr & E1000_ICR_TS) |
8178 | igb_tsync_interrupt(adapter); |
8179 | |
8180 | napi_schedule(n: &q_vector->napi); |
8181 | |
8182 | return IRQ_HANDLED; |
8183 | } |
8184 | |
8185 | static void igb_ring_irq_enable(struct igb_q_vector *q_vector) |
8186 | { |
8187 | struct igb_adapter *adapter = q_vector->adapter; |
8188 | struct e1000_hw *hw = &adapter->hw; |
8189 | |
8190 | if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || |
8191 | (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { |
8192 | if ((adapter->num_q_vectors == 1) && !adapter->vf_data) |
8193 | igb_set_itr(q_vector); |
8194 | else |
8195 | igb_update_ring_itr(q_vector); |
8196 | } |
8197 | |
8198 | if (!test_bit(__IGB_DOWN, &adapter->state)) { |
8199 | if (adapter->flags & IGB_FLAG_HAS_MSIX) |
8200 | wr32(E1000_EIMS, q_vector->eims_value); |
8201 | else |
8202 | igb_irq_enable(adapter); |
8203 | } |
8204 | } |
8205 | |
8206 | /** |
8207 | * igb_poll - NAPI Rx polling callback |
8208 | * @napi: napi polling structure |
8209 | * @budget: count of how many packets we should handle |
8210 | **/ |
8211 | static int igb_poll(struct napi_struct *napi, int budget) |
8212 | { |
8213 | struct igb_q_vector *q_vector = container_of(napi, |
8214 | struct igb_q_vector, |
8215 | napi); |
8216 | bool clean_complete = true; |
8217 | int work_done = 0; |
8218 | |
8219 | #ifdef CONFIG_IGB_DCA |
8220 | if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED) |
8221 | igb_update_dca(q_vector); |
8222 | #endif |
8223 | if (q_vector->tx.ring) |
8224 | clean_complete = igb_clean_tx_irq(q_vector, budget); |
8225 | |
8226 | if (q_vector->rx.ring) { |
8227 | int cleaned = igb_clean_rx_irq(q_vector, budget); |
8228 | |
8229 | work_done += cleaned; |
8230 | if (cleaned >= budget) |
8231 | clean_complete = false; |
8232 | } |
8233 | |
8234 | /* If all work not completed, return budget and keep polling */ |
8235 | if (!clean_complete) |
8236 | return budget; |
8237 | |
8238 | /* Exit the polling mode, but don't re-enable interrupts if stack might |
8239 | * poll us due to busy-polling |
8240 | */ |
8241 | if (likely(napi_complete_done(napi, work_done))) |
8242 | igb_ring_irq_enable(q_vector); |
8243 | |
8244 | return work_done; |
8245 | } |
8246 | |
8247 | /** |
8248 | * igb_clean_tx_irq - Reclaim resources after transmit completes |
8249 | * @q_vector: pointer to q_vector containing needed info |
8250 | * @napi_budget: Used to determine if we are in netpoll |
8251 | * |
8252 | * returns true if ring is completely cleaned |
8253 | **/ |
8254 | static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget) |
8255 | { |
8256 | struct igb_adapter *adapter = q_vector->adapter; |
8257 | struct igb_ring *tx_ring = q_vector->tx.ring; |
8258 | struct igb_tx_buffer *tx_buffer; |
8259 | union e1000_adv_tx_desc *tx_desc; |
8260 | unsigned int total_bytes = 0, total_packets = 0; |
8261 | unsigned int budget = q_vector->tx.work_limit; |
8262 | unsigned int i = tx_ring->next_to_clean; |
8263 | |
8264 | if (test_bit(__IGB_DOWN, &adapter->state)) |
8265 | return true; |
8266 | |
8267 | tx_buffer = &tx_ring->tx_buffer_info[i]; |
8268 | tx_desc = IGB_TX_DESC(tx_ring, i); |
8269 | i -= tx_ring->count; |
8270 | |
8271 | do { |
8272 | union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; |
8273 | |
8274 | /* if next_to_watch is not set then there is no work pending */ |
8275 | if (!eop_desc) |
8276 | break; |
8277 | |
8278 | /* prevent any other reads prior to eop_desc */ |
8279 | smp_rmb(); |
8280 | |
8281 | /* if DD is not set pending work has not been completed */ |
8282 | if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD))) |
8283 | break; |
8284 | |
8285 | /* clear next_to_watch to prevent false hangs */ |
8286 | tx_buffer->next_to_watch = NULL; |
8287 | |
8288 | /* update the statistics for this packet */ |
8289 | total_bytes += tx_buffer->bytecount; |
8290 | total_packets += tx_buffer->gso_segs; |
8291 | |
8292 | /* free the skb */ |
8293 | if (tx_buffer->type == IGB_TYPE_SKB) |
8294 | napi_consume_skb(skb: tx_buffer->skb, budget: napi_budget); |
8295 | else |
8296 | xdp_return_frame(xdpf: tx_buffer->xdpf); |
8297 | |
8298 | /* unmap skb header data */ |
8299 | dma_unmap_single(tx_ring->dev, |
8300 | dma_unmap_addr(tx_buffer, dma), |
8301 | dma_unmap_len(tx_buffer, len), |
8302 | DMA_TO_DEVICE); |
8303 | |
8304 | /* clear tx_buffer data */ |
8305 | dma_unmap_len_set(tx_buffer, len, 0); |
8306 | |
8307 | /* clear last DMA location and unmap remaining buffers */ |
8308 | while (tx_desc != eop_desc) { |
8309 | tx_buffer++; |
8310 | tx_desc++; |
8311 | i++; |
8312 | if (unlikely(!i)) { |
8313 | i -= tx_ring->count; |
8314 | tx_buffer = tx_ring->tx_buffer_info; |
8315 | tx_desc = IGB_TX_DESC(tx_ring, 0); |
8316 | } |
8317 | |
8318 | /* unmap any remaining paged data */ |
8319 | if (dma_unmap_len(tx_buffer, len)) { |
8320 | dma_unmap_page(tx_ring->dev, |
8321 | dma_unmap_addr(tx_buffer, dma), |
8322 | dma_unmap_len(tx_buffer, len), |
8323 | DMA_TO_DEVICE); |
8324 | dma_unmap_len_set(tx_buffer, len, 0); |
8325 | } |
8326 | } |
8327 | |
8328 | /* move us one more past the eop_desc for start of next pkt */ |
8329 | tx_buffer++; |
8330 | tx_desc++; |
8331 | i++; |
8332 | if (unlikely(!i)) { |
8333 | i -= tx_ring->count; |
8334 | tx_buffer = tx_ring->tx_buffer_info; |
8335 | tx_desc = IGB_TX_DESC(tx_ring, 0); |
8336 | } |
8337 | |
8338 | /* issue prefetch for next Tx descriptor */ |
8339 | prefetch(tx_desc); |
8340 | |
8341 | /* update budget accounting */ |
8342 | budget--; |
8343 | } while (likely(budget)); |
8344 | |
8345 | netdev_tx_completed_queue(dev_queue: txring_txq(tx_ring), |
8346 | pkts: total_packets, bytes: total_bytes); |
8347 | i += tx_ring->count; |
8348 | tx_ring->next_to_clean = i; |
8349 | u64_stats_update_begin(syncp: &tx_ring->tx_syncp); |
8350 | tx_ring->tx_stats.bytes += total_bytes; |
8351 | tx_ring->tx_stats.packets += total_packets; |
8352 | u64_stats_update_end(syncp: &tx_ring->tx_syncp); |
8353 | q_vector->tx.total_bytes += total_bytes; |
8354 | q_vector->tx.total_packets += total_packets; |
8355 | |
8356 | if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) { |
8357 | struct e1000_hw *hw = &adapter->hw; |
8358 | |
8359 | /* Detect a transmit hang in hardware, this serializes the |
8360 | * check with the clearing of time_stamp and movement of i |
8361 | */ |
8362 | clear_bit(nr: IGB_RING_FLAG_TX_DETECT_HANG, addr: &tx_ring->flags); |
8363 | if (tx_buffer->next_to_watch && |
8364 | time_after(jiffies, tx_buffer->time_stamp + |
8365 | (adapter->tx_timeout_factor * HZ)) && |
8366 | !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) { |
8367 | |
8368 | /* detected Tx unit hang */ |
8369 | dev_err(tx_ring->dev, |
8370 | "Detected Tx Unit Hang\n" |
8371 | " Tx Queue <%d>\n" |
8372 | " TDH <%x>\n" |
8373 | " TDT <%x>\n" |
8374 | " next_to_use <%x>\n" |
8375 | " next_to_clean <%x>\n" |
8376 | "buffer_info[next_to_clean]\n" |
8377 | " time_stamp <%lx>\n" |
8378 | " next_to_watch <%p>\n" |
8379 | " jiffies <%lx>\n" |
8380 | " desc.status <%x>\n" , |
8381 | tx_ring->queue_index, |
8382 | rd32(E1000_TDH(tx_ring->reg_idx)), |
8383 | readl(tx_ring->tail), |
8384 | tx_ring->next_to_use, |
8385 | tx_ring->next_to_clean, |
8386 | tx_buffer->time_stamp, |
8387 | tx_buffer->next_to_watch, |
8388 | jiffies, |
8389 | tx_buffer->next_to_watch->wb.status); |
8390 | netif_stop_subqueue(dev: tx_ring->netdev, |
8391 | queue_index: tx_ring->queue_index); |
8392 | |
8393 | /* we are about to reset, no point in enabling stuff */ |
8394 | return true; |
8395 | } |
8396 | } |
8397 | |
8398 | #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) |
8399 | if (unlikely(total_packets && |
8400 | netif_carrier_ok(tx_ring->netdev) && |
8401 | igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) { |
8402 | /* Make sure that anybody stopping the queue after this |
8403 | * sees the new next_to_clean. |
8404 | */ |
8405 | smp_mb(); |
8406 | if (__netif_subqueue_stopped(dev: tx_ring->netdev, |
8407 | queue_index: tx_ring->queue_index) && |
8408 | !(test_bit(__IGB_DOWN, &adapter->state))) { |
8409 | netif_wake_subqueue(dev: tx_ring->netdev, |
8410 | queue_index: tx_ring->queue_index); |
8411 | |
8412 | u64_stats_update_begin(syncp: &tx_ring->tx_syncp); |
8413 | tx_ring->tx_stats.restart_queue++; |
8414 | u64_stats_update_end(syncp: &tx_ring->tx_syncp); |
8415 | } |
8416 | } |
8417 | |
8418 | return !!budget; |
8419 | } |
8420 | |
8421 | /** |
8422 | * igb_reuse_rx_page - page flip buffer and store it back on the ring |
8423 | * @rx_ring: rx descriptor ring to store buffers on |
8424 | * @old_buff: donor buffer to have page reused |
8425 | * |
8426 | * Synchronizes page for reuse by the adapter |
8427 | **/ |
8428 | static void igb_reuse_rx_page(struct igb_ring *rx_ring, |
8429 | struct igb_rx_buffer *old_buff) |
8430 | { |
8431 | struct igb_rx_buffer *new_buff; |
8432 | u16 nta = rx_ring->next_to_alloc; |
8433 | |
8434 | new_buff = &rx_ring->rx_buffer_info[nta]; |
8435 | |
8436 | /* update, and store next to alloc */ |
8437 | nta++; |
8438 | rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; |
8439 | |
8440 | /* Transfer page from old buffer to new buffer. |
8441 | * Move each member individually to avoid possible store |
8442 | * forwarding stalls. |
8443 | */ |
8444 | new_buff->dma = old_buff->dma; |
8445 | new_buff->page = old_buff->page; |
8446 | new_buff->page_offset = old_buff->page_offset; |
8447 | new_buff->pagecnt_bias = old_buff->pagecnt_bias; |
8448 | } |
8449 | |
8450 | static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer, |
8451 | int rx_buf_pgcnt) |
8452 | { |
8453 | unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; |
8454 | struct page *page = rx_buffer->page; |
8455 | |
8456 | /* avoid re-using remote and pfmemalloc pages */ |
8457 | if (!dev_page_is_reusable(page)) |
8458 | return false; |
8459 | |
8460 | #if (PAGE_SIZE < 8192) |
8461 | /* if we are only owner of page we can reuse it */ |
8462 | if (unlikely((rx_buf_pgcnt - pagecnt_bias) > 1)) |
8463 | return false; |
8464 | #else |
8465 | #define IGB_LAST_OFFSET \ |
8466 | (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048) |
8467 | |
8468 | if (rx_buffer->page_offset > IGB_LAST_OFFSET) |
8469 | return false; |
8470 | #endif |
8471 | |
8472 | /* If we have drained the page fragment pool we need to update |
8473 | * the pagecnt_bias and page count so that we fully restock the |
8474 | * number of references the driver holds. |
8475 | */ |
8476 | if (unlikely(pagecnt_bias == 1)) { |
8477 | page_ref_add(page, USHRT_MAX - 1); |
8478 | rx_buffer->pagecnt_bias = USHRT_MAX; |
8479 | } |
8480 | |
8481 | return true; |
8482 | } |
8483 | |
8484 | /** |
8485 | * igb_add_rx_frag - Add contents of Rx buffer to sk_buff |
8486 | * @rx_ring: rx descriptor ring to transact packets on |
8487 | * @rx_buffer: buffer containing page to add |
8488 | * @skb: sk_buff to place the data into |
8489 | * @size: size of buffer to be added |
8490 | * |
8491 | * This function will add the data contained in rx_buffer->page to the skb. |
8492 | **/ |
8493 | static void igb_add_rx_frag(struct igb_ring *rx_ring, |
8494 | struct igb_rx_buffer *rx_buffer, |
8495 | struct sk_buff *skb, |
8496 | unsigned int size) |
8497 | { |
8498 | #if (PAGE_SIZE < 8192) |
8499 | unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; |
8500 | #else |
8501 | unsigned int truesize = ring_uses_build_skb(rx_ring) ? |
8502 | SKB_DATA_ALIGN(IGB_SKB_PAD + size) : |
8503 | SKB_DATA_ALIGN(size); |
8504 | #endif |
8505 | skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page: rx_buffer->page, |
8506 | off: rx_buffer->page_offset, size, truesize); |
8507 | #if (PAGE_SIZE < 8192) |
8508 | rx_buffer->page_offset ^= truesize; |
8509 | #else |
8510 | rx_buffer->page_offset += truesize; |
8511 | #endif |
8512 | } |
8513 | |
8514 | static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring, |
8515 | struct igb_rx_buffer *rx_buffer, |
8516 | struct xdp_buff *xdp, |
8517 | ktime_t timestamp) |
8518 | { |
8519 | #if (PAGE_SIZE < 8192) |
8520 | unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; |
8521 | #else |
8522 | unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - |
8523 | xdp->data_hard_start); |
8524 | #endif |
8525 | unsigned int size = xdp->data_end - xdp->data; |
8526 | unsigned int headlen; |
8527 | struct sk_buff *skb; |
8528 | |
8529 | /* prefetch first cache line of first page */ |
8530 | net_prefetch(p: xdp->data); |
8531 | |
8532 | /* allocate a skb to store the frags */ |
8533 | skb = napi_alloc_skb(napi: &rx_ring->q_vector->napi, IGB_RX_HDR_LEN); |
8534 | if (unlikely(!skb)) |
8535 | return NULL; |
8536 | |
8537 | if (timestamp) |
8538 | skb_hwtstamps(skb)->hwtstamp = timestamp; |
8539 | |
8540 | /* Determine available headroom for copy */ |
8541 | headlen = size; |
8542 | if (headlen > IGB_RX_HDR_LEN) |
8543 | headlen = eth_get_headlen(dev: skb->dev, data: xdp->data, IGB_RX_HDR_LEN); |
8544 | |
8545 | /* align pull length to size of long to optimize memcpy performance */ |
8546 | memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen, sizeof(long))); |
8547 | |
8548 | /* update all of the pointers */ |
8549 | size -= headlen; |
8550 | if (size) { |
8551 | skb_add_rx_frag(skb, i: 0, page: rx_buffer->page, |
8552 | off: (xdp->data + headlen) - page_address(rx_buffer->page), |
8553 | size, truesize); |
8554 | #if (PAGE_SIZE < 8192) |
8555 | rx_buffer->page_offset ^= truesize; |
8556 | #else |
8557 | rx_buffer->page_offset += truesize; |
8558 | #endif |
8559 | } else { |
8560 | rx_buffer->pagecnt_bias++; |
8561 | } |
8562 | |
8563 | return skb; |
8564 | } |
8565 | |
8566 | static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring, |
8567 | struct igb_rx_buffer *rx_buffer, |
8568 | struct xdp_buff *xdp, |
8569 | ktime_t timestamp) |
8570 | { |
8571 | #if (PAGE_SIZE < 8192) |
8572 | unsigned int truesize = igb_rx_pg_size(rx_ring) / 2; |
8573 | #else |
8574 | unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + |
8575 | SKB_DATA_ALIGN(xdp->data_end - |
8576 | xdp->data_hard_start); |
8577 | #endif |
8578 | unsigned int metasize = xdp->data - xdp->data_meta; |
8579 | struct sk_buff *skb; |
8580 | |
8581 | /* prefetch first cache line of first page */ |
8582 | net_prefetch(p: xdp->data_meta); |
8583 | |
8584 | /* build an skb around the page buffer */ |
8585 | skb = napi_build_skb(data: xdp->data_hard_start, frag_size: truesize); |
8586 | if (unlikely(!skb)) |
8587 | return NULL; |
8588 | |
8589 | /* update pointers within the skb to store the data */ |
8590 | skb_reserve(skb, len: xdp->data - xdp->data_hard_start); |
8591 | __skb_put(skb, len: xdp->data_end - xdp->data); |
8592 | |
8593 | if (metasize) |
8594 | skb_metadata_set(skb, meta_len: metasize); |
8595 | |
8596 | if (timestamp) |
8597 | skb_hwtstamps(skb)->hwtstamp = timestamp; |
8598 | |
8599 | /* update buffer offset */ |
8600 | #if (PAGE_SIZE < 8192) |
8601 | rx_buffer->page_offset ^= truesize; |
8602 | #else |
8603 | rx_buffer->page_offset += truesize; |
8604 | #endif |
8605 | |
8606 | return skb; |
8607 | } |
8608 | |
8609 | static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter, |
8610 | struct igb_ring *rx_ring, |
8611 | struct xdp_buff *xdp) |
8612 | { |
8613 | int err, result = IGB_XDP_PASS; |
8614 | struct bpf_prog *xdp_prog; |
8615 | u32 act; |
8616 | |
8617 | xdp_prog = READ_ONCE(rx_ring->xdp_prog); |
8618 | |
8619 | if (!xdp_prog) |
8620 | goto xdp_out; |
8621 | |
8622 | prefetchw(x: xdp->data_hard_start); /* xdp_frame write */ |
8623 | |
8624 | act = bpf_prog_run_xdp(prog: xdp_prog, xdp); |
8625 | switch (act) { |
8626 | case XDP_PASS: |
8627 | break; |
8628 | case XDP_TX: |
8629 | result = igb_xdp_xmit_back(adapter, xdp); |
8630 | if (result == IGB_XDP_CONSUMED) |
8631 | goto out_failure; |
8632 | break; |
8633 | case XDP_REDIRECT: |
8634 | err = xdp_do_redirect(dev: adapter->netdev, xdp, prog: xdp_prog); |
8635 | if (err) |
8636 | goto out_failure; |
8637 | result = IGB_XDP_REDIR; |
8638 | break; |
8639 | default: |
8640 | bpf_warn_invalid_xdp_action(dev: adapter->netdev, prog: xdp_prog, act); |
8641 | fallthrough; |
8642 | case XDP_ABORTED: |
8643 | out_failure: |
8644 | trace_xdp_exception(dev: rx_ring->netdev, xdp: xdp_prog, act); |
8645 | fallthrough; |
8646 | case XDP_DROP: |
8647 | result = IGB_XDP_CONSUMED; |
8648 | break; |
8649 | } |
8650 | xdp_out: |
8651 | return ERR_PTR(error: -result); |
8652 | } |
8653 | |
8654 | static unsigned int igb_rx_frame_truesize(struct igb_ring *rx_ring, |
8655 | unsigned int size) |
8656 | { |
8657 | unsigned int truesize; |
8658 | |
8659 | #if (PAGE_SIZE < 8192) |
8660 | truesize = igb_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ |
8661 | #else |
8662 | truesize = ring_uses_build_skb(rx_ring) ? |
8663 | SKB_DATA_ALIGN(IGB_SKB_PAD + size) + |
8664 | SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : |
8665 | SKB_DATA_ALIGN(size); |
8666 | #endif |
8667 | return truesize; |
8668 | } |
8669 | |
8670 | static void igb_rx_buffer_flip(struct igb_ring *rx_ring, |
8671 | struct igb_rx_buffer *rx_buffer, |
8672 | unsigned int size) |
8673 | { |
8674 | unsigned int truesize = igb_rx_frame_truesize(rx_ring, size); |
8675 | #if (PAGE_SIZE < 8192) |
8676 | rx_buffer->page_offset ^= truesize; |
8677 | #else |
8678 | rx_buffer->page_offset += truesize; |
8679 | #endif |
8680 | } |
8681 | |
8682 | static inline void igb_rx_checksum(struct igb_ring *ring, |
8683 | union e1000_adv_rx_desc *rx_desc, |
8684 | struct sk_buff *skb) |
8685 | { |
8686 | skb_checksum_none_assert(skb); |
8687 | |
8688 | /* Ignore Checksum bit is set */ |
8689 | if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM)) |
8690 | return; |
8691 | |
8692 | /* Rx checksum disabled via ethtool */ |
8693 | if (!(ring->netdev->features & NETIF_F_RXCSUM)) |
8694 | return; |
8695 | |
8696 | /* TCP/UDP checksum error bit is set */ |
8697 | if (igb_test_staterr(rx_desc, |
8698 | E1000_RXDEXT_STATERR_TCPE | |
8699 | E1000_RXDEXT_STATERR_IPE)) { |
8700 | /* work around errata with sctp packets where the TCPE aka |
8701 | * L4E bit is set incorrectly on 64 byte (60 byte w/o crc) |
8702 | * packets, (aka let the stack check the crc32c) |
8703 | */ |
8704 | if (!((skb->len == 60) && |
8705 | test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) { |
8706 | u64_stats_update_begin(syncp: &ring->rx_syncp); |
8707 | ring->rx_stats.csum_err++; |
8708 | u64_stats_update_end(syncp: &ring->rx_syncp); |
8709 | } |
8710 | /* let the stack verify checksum errors */ |
8711 | return; |
8712 | } |
8713 | /* It must be a TCP or UDP packet with a valid checksum */ |
8714 | if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS | |
8715 | E1000_RXD_STAT_UDPCS)) |
8716 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
8717 | |
8718 | dev_dbg(ring->dev, "cksum success: bits %08X\n" , |
8719 | le32_to_cpu(rx_desc->wb.upper.status_error)); |
8720 | } |
8721 | |
8722 | static inline void igb_rx_hash(struct igb_ring *ring, |
8723 | union e1000_adv_rx_desc *rx_desc, |
8724 | struct sk_buff *skb) |
8725 | { |
8726 | if (ring->netdev->features & NETIF_F_RXHASH) |
8727 | skb_set_hash(skb, |
8728 | le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), |
8729 | type: PKT_HASH_TYPE_L3); |
8730 | } |
8731 | |
8732 | /** |
8733 | * igb_is_non_eop - process handling of non-EOP buffers |
8734 | * @rx_ring: Rx ring being processed |
8735 | * @rx_desc: Rx descriptor for current buffer |
8736 | * |
8737 | * This function updates next to clean. If the buffer is an EOP buffer |
8738 | * this function exits returning false, otherwise it will place the |
8739 | * sk_buff in the next buffer to be chained and return true indicating |
8740 | * that this is in fact a non-EOP buffer. |
8741 | **/ |
8742 | static bool igb_is_non_eop(struct igb_ring *rx_ring, |
8743 | union e1000_adv_rx_desc *rx_desc) |
8744 | { |
8745 | u32 ntc = rx_ring->next_to_clean + 1; |
8746 | |
8747 | /* fetch, update, and store next to clean */ |
8748 | ntc = (ntc < rx_ring->count) ? ntc : 0; |
8749 | rx_ring->next_to_clean = ntc; |
8750 | |
8751 | prefetch(IGB_RX_DESC(rx_ring, ntc)); |
8752 | |
8753 | if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP))) |
8754 | return false; |
8755 | |
8756 | return true; |
8757 | } |
8758 | |
8759 | /** |
8760 | * igb_cleanup_headers - Correct corrupted or empty headers |
8761 | * @rx_ring: rx descriptor ring packet is being transacted on |
8762 | * @rx_desc: pointer to the EOP Rx descriptor |
8763 | * @skb: pointer to current skb being fixed |
8764 | * |
8765 | * Address the case where we are pulling data in on pages only |
8766 | * and as such no data is present in the skb header. |
8767 | * |
8768 | * In addition if skb is not at least 60 bytes we need to pad it so that |
8769 | * it is large enough to qualify as a valid Ethernet frame. |
8770 | * |
8771 | * Returns true if an error was encountered and skb was freed. |
8772 | **/ |
8773 | static bool (struct igb_ring *rx_ring, |
8774 | union e1000_adv_rx_desc *rx_desc, |
8775 | struct sk_buff *skb) |
8776 | { |
8777 | /* XDP packets use error pointer so abort at this point */ |
8778 | if (IS_ERR(ptr: skb)) |
8779 | return true; |
8780 | |
8781 | if (unlikely((igb_test_staterr(rx_desc, |
8782 | E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) { |
8783 | struct net_device *netdev = rx_ring->netdev; |
8784 | if (!(netdev->features & NETIF_F_RXALL)) { |
8785 | dev_kfree_skb_any(skb); |
8786 | return true; |
8787 | } |
8788 | } |
8789 | |
8790 | /* if eth_skb_pad returns an error the skb was freed */ |
8791 | if (eth_skb_pad(skb)) |
8792 | return true; |
8793 | |
8794 | return false; |
8795 | } |
8796 | |
8797 | /** |
8798 | * igb_process_skb_fields - Populate skb header fields from Rx descriptor |
8799 | * @rx_ring: rx descriptor ring packet is being transacted on |
8800 | * @rx_desc: pointer to the EOP Rx descriptor |
8801 | * @skb: pointer to current skb being populated |
8802 | * |
8803 | * This function checks the ring, descriptor, and packet information in |
8804 | * order to populate the hash, checksum, VLAN, timestamp, protocol, and |
8805 | * other fields within the skb. |
8806 | **/ |
8807 | static void igb_process_skb_fields(struct igb_ring *rx_ring, |
8808 | union e1000_adv_rx_desc *rx_desc, |
8809 | struct sk_buff *skb) |
8810 | { |
8811 | struct net_device *dev = rx_ring->netdev; |
8812 | |
8813 | igb_rx_hash(ring: rx_ring, rx_desc, skb); |
8814 | |
8815 | igb_rx_checksum(ring: rx_ring, rx_desc, skb); |
8816 | |
8817 | if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) && |
8818 | !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) |
8819 | igb_ptp_rx_rgtstamp(q_vector: rx_ring->q_vector, skb); |
8820 | |
8821 | if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && |
8822 | igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) { |
8823 | u16 vid; |
8824 | |
8825 | if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) && |
8826 | test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags)) |
8827 | vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan); |
8828 | else |
8829 | vid = le16_to_cpu(rx_desc->wb.upper.vlan); |
8830 | |
8831 | __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci: vid); |
8832 | } |
8833 | |
8834 | skb_record_rx_queue(skb, rx_queue: rx_ring->queue_index); |
8835 | |
8836 | skb->protocol = eth_type_trans(skb, dev: rx_ring->netdev); |
8837 | } |
8838 | |
8839 | static unsigned int igb_rx_offset(struct igb_ring *rx_ring) |
8840 | { |
8841 | return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0; |
8842 | } |
8843 | |
8844 | static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring, |
8845 | const unsigned int size, int *rx_buf_pgcnt) |
8846 | { |
8847 | struct igb_rx_buffer *rx_buffer; |
8848 | |
8849 | rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; |
8850 | *rx_buf_pgcnt = |
8851 | #if (PAGE_SIZE < 8192) |
8852 | page_count(page: rx_buffer->page); |
8853 | #else |
8854 | 0; |
8855 | #endif |
8856 | prefetchw(x: rx_buffer->page); |
8857 | |
8858 | /* we are reusing so sync this buffer for CPU use */ |
8859 | dma_sync_single_range_for_cpu(dev: rx_ring->dev, |
8860 | addr: rx_buffer->dma, |
8861 | offset: rx_buffer->page_offset, |
8862 | size, |
8863 | dir: DMA_FROM_DEVICE); |
8864 | |
8865 | rx_buffer->pagecnt_bias--; |
8866 | |
8867 | return rx_buffer; |
8868 | } |
8869 | |
8870 | static void igb_put_rx_buffer(struct igb_ring *rx_ring, |
8871 | struct igb_rx_buffer *rx_buffer, int rx_buf_pgcnt) |
8872 | { |
8873 | if (igb_can_reuse_rx_page(rx_buffer, rx_buf_pgcnt)) { |
8874 | /* hand second half of page back to the ring */ |
8875 | igb_reuse_rx_page(rx_ring, old_buff: rx_buffer); |
8876 | } else { |
8877 | /* We are not reusing the buffer so unmap it and free |
8878 | * any references we are holding to it |
8879 | */ |
8880 | dma_unmap_page_attrs(dev: rx_ring->dev, addr: rx_buffer->dma, |
8881 | igb_rx_pg_size(rx_ring), dir: DMA_FROM_DEVICE, |
8882 | IGB_RX_DMA_ATTR); |
8883 | __page_frag_cache_drain(page: rx_buffer->page, |
8884 | count: rx_buffer->pagecnt_bias); |
8885 | } |
8886 | |
8887 | /* clear contents of rx_buffer */ |
8888 | rx_buffer->page = NULL; |
8889 | } |
8890 | |
8891 | static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) |
8892 | { |
8893 | struct igb_adapter *adapter = q_vector->adapter; |
8894 | struct igb_ring *rx_ring = q_vector->rx.ring; |
8895 | struct sk_buff *skb = rx_ring->skb; |
8896 | unsigned int total_bytes = 0, total_packets = 0; |
8897 | u16 cleaned_count = igb_desc_unused(ring: rx_ring); |
8898 | unsigned int xdp_xmit = 0; |
8899 | struct xdp_buff xdp; |
8900 | u32 frame_sz = 0; |
8901 | int rx_buf_pgcnt; |
8902 | |
8903 | /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ |
8904 | #if (PAGE_SIZE < 8192) |
8905 | frame_sz = igb_rx_frame_truesize(rx_ring, size: 0); |
8906 | #endif |
8907 | xdp_init_buff(xdp: &xdp, frame_sz, rxq: &rx_ring->xdp_rxq); |
8908 | |
8909 | while (likely(total_packets < budget)) { |
8910 | union e1000_adv_rx_desc *rx_desc; |
8911 | struct igb_rx_buffer *rx_buffer; |
8912 | ktime_t timestamp = 0; |
8913 | int pkt_offset = 0; |
8914 | unsigned int size; |
8915 | void *pktbuf; |
8916 | |
8917 | /* return some buffers to hardware, one at a time is too slow */ |
8918 | if (cleaned_count >= IGB_RX_BUFFER_WRITE) { |
8919 | igb_alloc_rx_buffers(rx_ring, cleaned_count); |
8920 | cleaned_count = 0; |
8921 | } |
8922 | |
8923 | rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean); |
8924 | size = le16_to_cpu(rx_desc->wb.upper.length); |
8925 | if (!size) |
8926 | break; |
8927 | |
8928 | /* This memory barrier is needed to keep us from reading |
8929 | * any other fields out of the rx_desc until we know the |
8930 | * descriptor has been written back |
8931 | */ |
8932 | dma_rmb(); |
8933 | |
8934 | rx_buffer = igb_get_rx_buffer(rx_ring, size, rx_buf_pgcnt: &rx_buf_pgcnt); |
8935 | pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset; |
8936 | |
8937 | /* pull rx packet timestamp if available and valid */ |
8938 | if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) { |
8939 | int ts_hdr_len; |
8940 | |
8941 | ts_hdr_len = igb_ptp_rx_pktstamp(q_vector: rx_ring->q_vector, |
8942 | va: pktbuf, timestamp: ×tamp); |
8943 | |
8944 | pkt_offset += ts_hdr_len; |
8945 | size -= ts_hdr_len; |
8946 | } |
8947 | |
8948 | /* retrieve a buffer from the ring */ |
8949 | if (!skb) { |
8950 | unsigned char *hard_start = pktbuf - igb_rx_offset(rx_ring); |
8951 | unsigned int offset = pkt_offset + igb_rx_offset(rx_ring); |
8952 | |
8953 | xdp_prepare_buff(xdp: &xdp, hard_start, headroom: offset, data_len: size, meta_valid: true); |
8954 | xdp_buff_clear_frags_flag(xdp: &xdp); |
8955 | #if (PAGE_SIZE > 4096) |
8956 | /* At larger PAGE_SIZE, frame_sz depend on len size */ |
8957 | xdp.frame_sz = igb_rx_frame_truesize(rx_ring, size); |
8958 | #endif |
8959 | skb = igb_run_xdp(adapter, rx_ring, xdp: &xdp); |
8960 | } |
8961 | |
8962 | if (IS_ERR(ptr: skb)) { |
8963 | unsigned int xdp_res = -PTR_ERR(ptr: skb); |
8964 | |
8965 | if (xdp_res & (IGB_XDP_TX | IGB_XDP_REDIR)) { |
8966 | xdp_xmit |= xdp_res; |
8967 | igb_rx_buffer_flip(rx_ring, rx_buffer, size); |
8968 | } else { |
8969 | rx_buffer->pagecnt_bias++; |
8970 | } |
8971 | total_packets++; |
8972 | total_bytes += size; |
8973 | } else if (skb) |
8974 | igb_add_rx_frag(rx_ring, rx_buffer, skb, size); |
8975 | else if (ring_uses_build_skb(rx_ring)) |
8976 | skb = igb_build_skb(rx_ring, rx_buffer, xdp: &xdp, |
8977 | timestamp); |
8978 | else |
8979 | skb = igb_construct_skb(rx_ring, rx_buffer, |
8980 | xdp: &xdp, timestamp); |
8981 | |
8982 | /* exit if we failed to retrieve a buffer */ |
8983 | if (!skb) { |
8984 | rx_ring->rx_stats.alloc_failed++; |
8985 | rx_buffer->pagecnt_bias++; |
8986 | break; |
8987 | } |
8988 | |
8989 | igb_put_rx_buffer(rx_ring, rx_buffer, rx_buf_pgcnt); |
8990 | cleaned_count++; |
8991 | |
8992 | /* fetch next buffer in frame if non-eop */ |
8993 | if (igb_is_non_eop(rx_ring, rx_desc)) |
8994 | continue; |
8995 | |
8996 | /* verify the packet layout is correct */ |
8997 | if (igb_cleanup_headers(rx_ring, rx_desc, skb)) { |
8998 | skb = NULL; |
8999 | continue; |
9000 | } |
9001 | |
9002 | /* probably a little skewed due to removing CRC */ |
9003 | total_bytes += skb->len; |
9004 | |
9005 | /* populate checksum, timestamp, VLAN, and protocol */ |
9006 | igb_process_skb_fields(rx_ring, rx_desc, skb); |
9007 | |
9008 | napi_gro_receive(napi: &q_vector->napi, skb); |
9009 | |
9010 | /* reset skb pointer */ |
9011 | skb = NULL; |
9012 | |
9013 | /* update budget accounting */ |
9014 | total_packets++; |
9015 | } |
9016 | |
9017 | /* place incomplete frames back on ring for completion */ |
9018 | rx_ring->skb = skb; |
9019 | |
9020 | if (xdp_xmit & IGB_XDP_REDIR) |
9021 | xdp_do_flush(); |
9022 | |
9023 | if (xdp_xmit & IGB_XDP_TX) { |
9024 | struct igb_ring *tx_ring = igb_xdp_tx_queue_mapping(adapter); |
9025 | |
9026 | igb_xdp_ring_update_tail(ring: tx_ring); |
9027 | } |
9028 | |
9029 | u64_stats_update_begin(syncp: &rx_ring->rx_syncp); |
9030 | rx_ring->rx_stats.packets += total_packets; |
9031 | rx_ring->rx_stats.bytes += total_bytes; |
9032 | u64_stats_update_end(syncp: &rx_ring->rx_syncp); |
9033 | q_vector->rx.total_packets += total_packets; |
9034 | q_vector->rx.total_bytes += total_bytes; |
9035 | |
9036 | if (cleaned_count) |
9037 | igb_alloc_rx_buffers(rx_ring, cleaned_count); |
9038 | |
9039 | return total_packets; |
9040 | } |
9041 | |
9042 | static bool igb_alloc_mapped_page(struct igb_ring *rx_ring, |
9043 | struct igb_rx_buffer *bi) |
9044 | { |
9045 | struct page *page = bi->page; |
9046 | dma_addr_t dma; |
9047 | |
9048 | /* since we are recycling buffers we should seldom need to alloc */ |
9049 | if (likely(page)) |
9050 | return true; |
9051 | |
9052 | /* alloc new page for storage */ |
9053 | page = dev_alloc_pages(order: igb_rx_pg_order(ring: rx_ring)); |
9054 | if (unlikely(!page)) { |
9055 | rx_ring->rx_stats.alloc_failed++; |
9056 | return false; |
9057 | } |
9058 | |
9059 | /* map page for use */ |
9060 | dma = dma_map_page_attrs(dev: rx_ring->dev, page, offset: 0, |
9061 | igb_rx_pg_size(rx_ring), |
9062 | dir: DMA_FROM_DEVICE, |
9063 | IGB_RX_DMA_ATTR); |
9064 | |
9065 | /* if mapping failed free memory back to system since |
9066 | * there isn't much point in holding memory we can't use |
9067 | */ |
9068 | if (dma_mapping_error(dev: rx_ring->dev, dma_addr: dma)) { |
9069 | __free_pages(page, order: igb_rx_pg_order(ring: rx_ring)); |
9070 | |
9071 | rx_ring->rx_stats.alloc_failed++; |
9072 | return false; |
9073 | } |
9074 | |
9075 | bi->dma = dma; |
9076 | bi->page = page; |
9077 | bi->page_offset = igb_rx_offset(rx_ring); |
9078 | page_ref_add(page, USHRT_MAX - 1); |
9079 | bi->pagecnt_bias = USHRT_MAX; |
9080 | |
9081 | return true; |
9082 | } |
9083 | |
9084 | /** |
9085 | * igb_alloc_rx_buffers - Replace used receive buffers |
9086 | * @rx_ring: rx descriptor ring to allocate new receive buffers |
9087 | * @cleaned_count: count of buffers to allocate |
9088 | **/ |
9089 | void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count) |
9090 | { |
9091 | union e1000_adv_rx_desc *rx_desc; |
9092 | struct igb_rx_buffer *bi; |
9093 | u16 i = rx_ring->next_to_use; |
9094 | u16 bufsz; |
9095 | |
9096 | /* nothing to do */ |
9097 | if (!cleaned_count) |
9098 | return; |
9099 | |
9100 | rx_desc = IGB_RX_DESC(rx_ring, i); |
9101 | bi = &rx_ring->rx_buffer_info[i]; |
9102 | i -= rx_ring->count; |
9103 | |
9104 | bufsz = igb_rx_bufsz(ring: rx_ring); |
9105 | |
9106 | do { |
9107 | if (!igb_alloc_mapped_page(rx_ring, bi)) |
9108 | break; |
9109 | |
9110 | /* sync the buffer for use by the device */ |
9111 | dma_sync_single_range_for_device(dev: rx_ring->dev, addr: bi->dma, |
9112 | offset: bi->page_offset, size: bufsz, |
9113 | dir: DMA_FROM_DEVICE); |
9114 | |
9115 | /* Refresh the desc even if buffer_addrs didn't change |
9116 | * because each write-back erases this info. |
9117 | */ |
9118 | rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); |
9119 | |
9120 | rx_desc++; |
9121 | bi++; |
9122 | i++; |
9123 | if (unlikely(!i)) { |
9124 | rx_desc = IGB_RX_DESC(rx_ring, 0); |
9125 | bi = rx_ring->rx_buffer_info; |
9126 | i -= rx_ring->count; |
9127 | } |
9128 | |
9129 | /* clear the length for the next_to_use descriptor */ |
9130 | rx_desc->wb.upper.length = 0; |
9131 | |
9132 | cleaned_count--; |
9133 | } while (cleaned_count); |
9134 | |
9135 | i += rx_ring->count; |
9136 | |
9137 | if (rx_ring->next_to_use != i) { |
9138 | /* record the next descriptor to use */ |
9139 | rx_ring->next_to_use = i; |
9140 | |
9141 | /* update next to alloc since we have filled the ring */ |
9142 | rx_ring->next_to_alloc = i; |
9143 | |
9144 | /* Force memory writes to complete before letting h/w |
9145 | * know there are new descriptors to fetch. (Only |
9146 | * applicable for weak-ordered memory model archs, |
9147 | * such as IA-64). |
9148 | */ |
9149 | dma_wmb(); |
9150 | writel(val: i, addr: rx_ring->tail); |
9151 | } |
9152 | } |
9153 | |
9154 | /** |
9155 | * igb_mii_ioctl - |
9156 | * @netdev: pointer to netdev struct |
9157 | * @ifr: interface structure |
9158 | * @cmd: ioctl command to execute |
9159 | **/ |
9160 | static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) |
9161 | { |
9162 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9163 | struct mii_ioctl_data *data = if_mii(rq: ifr); |
9164 | |
9165 | if (adapter->hw.phy.media_type != e1000_media_type_copper) |
9166 | return -EOPNOTSUPP; |
9167 | |
9168 | switch (cmd) { |
9169 | case SIOCGMIIPHY: |
9170 | data->phy_id = adapter->hw.phy.addr; |
9171 | break; |
9172 | case SIOCGMIIREG: |
9173 | if (igb_read_phy_reg(hw: &adapter->hw, offset: data->reg_num & 0x1F, |
9174 | data: &data->val_out)) |
9175 | return -EIO; |
9176 | break; |
9177 | case SIOCSMIIREG: |
9178 | default: |
9179 | return -EOPNOTSUPP; |
9180 | } |
9181 | return 0; |
9182 | } |
9183 | |
9184 | /** |
9185 | * igb_ioctl - |
9186 | * @netdev: pointer to netdev struct |
9187 | * @ifr: interface structure |
9188 | * @cmd: ioctl command to execute |
9189 | **/ |
9190 | static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) |
9191 | { |
9192 | switch (cmd) { |
9193 | case SIOCGMIIPHY: |
9194 | case SIOCGMIIREG: |
9195 | case SIOCSMIIREG: |
9196 | return igb_mii_ioctl(netdev, ifr, cmd); |
9197 | case SIOCGHWTSTAMP: |
9198 | return igb_ptp_get_ts_config(netdev, ifr); |
9199 | case SIOCSHWTSTAMP: |
9200 | return igb_ptp_set_ts_config(netdev, ifr); |
9201 | default: |
9202 | return -EOPNOTSUPP; |
9203 | } |
9204 | } |
9205 | |
9206 | void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value) |
9207 | { |
9208 | struct igb_adapter *adapter = hw->back; |
9209 | |
9210 | pci_read_config_word(dev: adapter->pdev, where: reg, val: value); |
9211 | } |
9212 | |
9213 | void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value) |
9214 | { |
9215 | struct igb_adapter *adapter = hw->back; |
9216 | |
9217 | pci_write_config_word(dev: adapter->pdev, where: reg, val: *value); |
9218 | } |
9219 | |
9220 | s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value) |
9221 | { |
9222 | struct igb_adapter *adapter = hw->back; |
9223 | |
9224 | if (pcie_capability_read_word(dev: adapter->pdev, pos: reg, val: value)) |
9225 | return -E1000_ERR_CONFIG; |
9226 | |
9227 | return 0; |
9228 | } |
9229 | |
9230 | s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value) |
9231 | { |
9232 | struct igb_adapter *adapter = hw->back; |
9233 | |
9234 | if (pcie_capability_write_word(dev: adapter->pdev, pos: reg, val: *value)) |
9235 | return -E1000_ERR_CONFIG; |
9236 | |
9237 | return 0; |
9238 | } |
9239 | |
9240 | static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features) |
9241 | { |
9242 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9243 | struct e1000_hw *hw = &adapter->hw; |
9244 | u32 ctrl, rctl; |
9245 | bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX); |
9246 | |
9247 | if (enable) { |
9248 | /* enable VLAN tag insert/strip */ |
9249 | ctrl = rd32(E1000_CTRL); |
9250 | ctrl |= E1000_CTRL_VME; |
9251 | wr32(E1000_CTRL, ctrl); |
9252 | |
9253 | /* Disable CFI check */ |
9254 | rctl = rd32(E1000_RCTL); |
9255 | rctl &= ~E1000_RCTL_CFIEN; |
9256 | wr32(E1000_RCTL, rctl); |
9257 | } else { |
9258 | /* disable VLAN tag insert/strip */ |
9259 | ctrl = rd32(E1000_CTRL); |
9260 | ctrl &= ~E1000_CTRL_VME; |
9261 | wr32(E1000_CTRL, ctrl); |
9262 | } |
9263 | |
9264 | igb_set_vf_vlan_strip(adapter, vfn: adapter->vfs_allocated_count, enable); |
9265 | } |
9266 | |
9267 | static int igb_vlan_rx_add_vid(struct net_device *netdev, |
9268 | __be16 proto, u16 vid) |
9269 | { |
9270 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9271 | struct e1000_hw *hw = &adapter->hw; |
9272 | int pf_id = adapter->vfs_allocated_count; |
9273 | |
9274 | /* add the filter since PF can receive vlans w/o entry in vlvf */ |
9275 | if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC)) |
9276 | igb_vfta_set(hw, vid, vind: pf_id, vlan_on: true, vlvf_bypass: !!vid); |
9277 | |
9278 | set_bit(nr: vid, addr: adapter->active_vlans); |
9279 | |
9280 | return 0; |
9281 | } |
9282 | |
9283 | static int igb_vlan_rx_kill_vid(struct net_device *netdev, |
9284 | __be16 proto, u16 vid) |
9285 | { |
9286 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9287 | int pf_id = adapter->vfs_allocated_count; |
9288 | struct e1000_hw *hw = &adapter->hw; |
9289 | |
9290 | /* remove VID from filter table */ |
9291 | if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC)) |
9292 | igb_vfta_set(hw, vid, vind: pf_id, vlan_on: false, vlvf_bypass: true); |
9293 | |
9294 | clear_bit(nr: vid, addr: adapter->active_vlans); |
9295 | |
9296 | return 0; |
9297 | } |
9298 | |
9299 | static void igb_restore_vlan(struct igb_adapter *adapter) |
9300 | { |
9301 | u16 vid = 1; |
9302 | |
9303 | igb_vlan_mode(netdev: adapter->netdev, features: adapter->netdev->features); |
9304 | igb_vlan_rx_add_vid(netdev: adapter->netdev, htons(ETH_P_8021Q), vid: 0); |
9305 | |
9306 | for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID) |
9307 | igb_vlan_rx_add_vid(netdev: adapter->netdev, htons(ETH_P_8021Q), vid); |
9308 | } |
9309 | |
9310 | int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx) |
9311 | { |
9312 | struct pci_dev *pdev = adapter->pdev; |
9313 | struct e1000_mac_info *mac = &adapter->hw.mac; |
9314 | |
9315 | mac->autoneg = 0; |
9316 | |
9317 | /* Make sure dplx is at most 1 bit and lsb of speed is not set |
9318 | * for the switch() below to work |
9319 | */ |
9320 | if ((spd & 1) || (dplx & ~1)) |
9321 | goto err_inval; |
9322 | |
9323 | /* Fiber NIC's only allow 1000 gbps Full duplex |
9324 | * and 100Mbps Full duplex for 100baseFx sfp |
9325 | */ |
9326 | if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { |
9327 | switch (spd + dplx) { |
9328 | case SPEED_10 + DUPLEX_HALF: |
9329 | case SPEED_10 + DUPLEX_FULL: |
9330 | case SPEED_100 + DUPLEX_HALF: |
9331 | goto err_inval; |
9332 | default: |
9333 | break; |
9334 | } |
9335 | } |
9336 | |
9337 | switch (spd + dplx) { |
9338 | case SPEED_10 + DUPLEX_HALF: |
9339 | mac->forced_speed_duplex = ADVERTISE_10_HALF; |
9340 | break; |
9341 | case SPEED_10 + DUPLEX_FULL: |
9342 | mac->forced_speed_duplex = ADVERTISE_10_FULL; |
9343 | break; |
9344 | case SPEED_100 + DUPLEX_HALF: |
9345 | mac->forced_speed_duplex = ADVERTISE_100_HALF; |
9346 | break; |
9347 | case SPEED_100 + DUPLEX_FULL: |
9348 | mac->forced_speed_duplex = ADVERTISE_100_FULL; |
9349 | break; |
9350 | case SPEED_1000 + DUPLEX_FULL: |
9351 | mac->autoneg = 1; |
9352 | adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; |
9353 | break; |
9354 | case SPEED_1000 + DUPLEX_HALF: /* not supported */ |
9355 | default: |
9356 | goto err_inval; |
9357 | } |
9358 | |
9359 | /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */ |
9360 | adapter->hw.phy.mdix = AUTO_ALL_MODES; |
9361 | |
9362 | return 0; |
9363 | |
9364 | err_inval: |
9365 | dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n" ); |
9366 | return -EINVAL; |
9367 | } |
9368 | |
9369 | static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake, |
9370 | bool runtime) |
9371 | { |
9372 | struct net_device *netdev = pci_get_drvdata(pdev); |
9373 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9374 | struct e1000_hw *hw = &adapter->hw; |
9375 | u32 ctrl, rctl, status; |
9376 | u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol; |
9377 | bool wake; |
9378 | |
9379 | rtnl_lock(); |
9380 | netif_device_detach(dev: netdev); |
9381 | |
9382 | if (netif_running(dev: netdev)) |
9383 | __igb_close(netdev, suspending: true); |
9384 | |
9385 | igb_ptp_suspend(adapter); |
9386 | |
9387 | igb_clear_interrupt_scheme(adapter); |
9388 | rtnl_unlock(); |
9389 | |
9390 | status = rd32(E1000_STATUS); |
9391 | if (status & E1000_STATUS_LU) |
9392 | wufc &= ~E1000_WUFC_LNKC; |
9393 | |
9394 | if (wufc) { |
9395 | igb_setup_rctl(adapter); |
9396 | igb_set_rx_mode(netdev); |
9397 | |
9398 | /* turn on all-multi mode if wake on multicast is enabled */ |
9399 | if (wufc & E1000_WUFC_MC) { |
9400 | rctl = rd32(E1000_RCTL); |
9401 | rctl |= E1000_RCTL_MPE; |
9402 | wr32(E1000_RCTL, rctl); |
9403 | } |
9404 | |
9405 | ctrl = rd32(E1000_CTRL); |
9406 | ctrl |= E1000_CTRL_ADVD3WUC; |
9407 | wr32(E1000_CTRL, ctrl); |
9408 | |
9409 | /* Allow time for pending master requests to run */ |
9410 | igb_disable_pcie_master(hw); |
9411 | |
9412 | wr32(E1000_WUC, E1000_WUC_PME_EN); |
9413 | wr32(E1000_WUFC, wufc); |
9414 | } else { |
9415 | wr32(E1000_WUC, 0); |
9416 | wr32(E1000_WUFC, 0); |
9417 | } |
9418 | |
9419 | wake = wufc || adapter->en_mng_pt; |
9420 | if (!wake) |
9421 | igb_power_down_link(adapter); |
9422 | else |
9423 | igb_power_up_link(adapter); |
9424 | |
9425 | if (enable_wake) |
9426 | *enable_wake = wake; |
9427 | |
9428 | /* Release control of h/w to f/w. If f/w is AMT enabled, this |
9429 | * would have already happened in close and is redundant. |
9430 | */ |
9431 | igb_release_hw_control(adapter); |
9432 | |
9433 | pci_disable_device(dev: pdev); |
9434 | |
9435 | return 0; |
9436 | } |
9437 | |
9438 | static void igb_deliver_wake_packet(struct net_device *netdev) |
9439 | { |
9440 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9441 | struct e1000_hw *hw = &adapter->hw; |
9442 | struct sk_buff *skb; |
9443 | u32 wupl; |
9444 | |
9445 | wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK; |
9446 | |
9447 | /* WUPM stores only the first 128 bytes of the wake packet. |
9448 | * Read the packet only if we have the whole thing. |
9449 | */ |
9450 | if ((wupl == 0) || (wupl > E1000_WUPM_BYTES)) |
9451 | return; |
9452 | |
9453 | skb = netdev_alloc_skb_ip_align(dev: netdev, E1000_WUPM_BYTES); |
9454 | if (!skb) |
9455 | return; |
9456 | |
9457 | skb_put(skb, len: wupl); |
9458 | |
9459 | /* Ensure reads are 32-bit aligned */ |
9460 | wupl = roundup(wupl, 4); |
9461 | |
9462 | memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl); |
9463 | |
9464 | skb->protocol = eth_type_trans(skb, dev: netdev); |
9465 | netif_rx(skb); |
9466 | } |
9467 | |
9468 | static int __maybe_unused igb_suspend(struct device *dev) |
9469 | { |
9470 | return __igb_shutdown(to_pci_dev(dev), NULL, runtime: 0); |
9471 | } |
9472 | |
9473 | static int __maybe_unused __igb_resume(struct device *dev, bool rpm) |
9474 | { |
9475 | struct pci_dev *pdev = to_pci_dev(dev); |
9476 | struct net_device *netdev = pci_get_drvdata(pdev); |
9477 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9478 | struct e1000_hw *hw = &adapter->hw; |
9479 | u32 err, val; |
9480 | |
9481 | pci_set_power_state(dev: pdev, PCI_D0); |
9482 | pci_restore_state(dev: pdev); |
9483 | pci_save_state(dev: pdev); |
9484 | |
9485 | if (!pci_device_is_present(pdev)) |
9486 | return -ENODEV; |
9487 | err = pci_enable_device_mem(dev: pdev); |
9488 | if (err) { |
9489 | dev_err(&pdev->dev, |
9490 | "igb: Cannot enable PCI device from suspend\n" ); |
9491 | return err; |
9492 | } |
9493 | pci_set_master(dev: pdev); |
9494 | |
9495 | pci_enable_wake(dev: pdev, PCI_D3hot, enable: 0); |
9496 | pci_enable_wake(dev: pdev, PCI_D3cold, enable: 0); |
9497 | |
9498 | if (igb_init_interrupt_scheme(adapter, msix: true)) { |
9499 | dev_err(&pdev->dev, "Unable to allocate memory for queues\n" ); |
9500 | return -ENOMEM; |
9501 | } |
9502 | |
9503 | igb_reset(adapter); |
9504 | |
9505 | /* let the f/w know that the h/w is now under the control of the |
9506 | * driver. |
9507 | */ |
9508 | igb_get_hw_control(adapter); |
9509 | |
9510 | val = rd32(E1000_WUS); |
9511 | if (val & WAKE_PKT_WUS) |
9512 | igb_deliver_wake_packet(netdev); |
9513 | |
9514 | wr32(E1000_WUS, ~0); |
9515 | |
9516 | if (!rpm) |
9517 | rtnl_lock(); |
9518 | if (!err && netif_running(dev: netdev)) |
9519 | err = __igb_open(netdev, resuming: true); |
9520 | |
9521 | if (!err) |
9522 | netif_device_attach(dev: netdev); |
9523 | if (!rpm) |
9524 | rtnl_unlock(); |
9525 | |
9526 | return err; |
9527 | } |
9528 | |
9529 | static int __maybe_unused igb_resume(struct device *dev) |
9530 | { |
9531 | return __igb_resume(dev, rpm: false); |
9532 | } |
9533 | |
9534 | static int __maybe_unused igb_runtime_idle(struct device *dev) |
9535 | { |
9536 | struct net_device *netdev = dev_get_drvdata(dev); |
9537 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9538 | |
9539 | if (!igb_has_link(adapter)) |
9540 | pm_schedule_suspend(dev, MSEC_PER_SEC * 5); |
9541 | |
9542 | return -EBUSY; |
9543 | } |
9544 | |
9545 | static int __maybe_unused igb_runtime_suspend(struct device *dev) |
9546 | { |
9547 | return __igb_shutdown(to_pci_dev(dev), NULL, runtime: 1); |
9548 | } |
9549 | |
9550 | static int __maybe_unused igb_runtime_resume(struct device *dev) |
9551 | { |
9552 | return __igb_resume(dev, rpm: true); |
9553 | } |
9554 | |
9555 | static void igb_shutdown(struct pci_dev *pdev) |
9556 | { |
9557 | bool wake; |
9558 | |
9559 | __igb_shutdown(pdev, enable_wake: &wake, runtime: 0); |
9560 | |
9561 | if (system_state == SYSTEM_POWER_OFF) { |
9562 | pci_wake_from_d3(dev: pdev, enable: wake); |
9563 | pci_set_power_state(dev: pdev, PCI_D3hot); |
9564 | } |
9565 | } |
9566 | |
9567 | static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs) |
9568 | { |
9569 | #ifdef CONFIG_PCI_IOV |
9570 | int err; |
9571 | |
9572 | if (num_vfs == 0) { |
9573 | return igb_disable_sriov(pdev: dev, reinit: true); |
9574 | } else { |
9575 | err = igb_enable_sriov(pdev: dev, num_vfs, reinit: true); |
9576 | return err ? err : num_vfs; |
9577 | } |
9578 | #endif |
9579 | return 0; |
9580 | } |
9581 | |
9582 | /** |
9583 | * igb_io_error_detected - called when PCI error is detected |
9584 | * @pdev: Pointer to PCI device |
9585 | * @state: The current pci connection state |
9586 | * |
9587 | * This function is called after a PCI bus error affecting |
9588 | * this device has been detected. |
9589 | **/ |
9590 | static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev, |
9591 | pci_channel_state_t state) |
9592 | { |
9593 | struct net_device *netdev = pci_get_drvdata(pdev); |
9594 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9595 | |
9596 | if (state == pci_channel_io_normal) { |
9597 | dev_warn(&pdev->dev, "Non-correctable non-fatal error reported.\n" ); |
9598 | return PCI_ERS_RESULT_CAN_RECOVER; |
9599 | } |
9600 | |
9601 | netif_device_detach(dev: netdev); |
9602 | |
9603 | if (state == pci_channel_io_perm_failure) |
9604 | return PCI_ERS_RESULT_DISCONNECT; |
9605 | |
9606 | if (netif_running(dev: netdev)) |
9607 | igb_down(adapter); |
9608 | pci_disable_device(dev: pdev); |
9609 | |
9610 | /* Request a slot reset. */ |
9611 | return PCI_ERS_RESULT_NEED_RESET; |
9612 | } |
9613 | |
9614 | /** |
9615 | * igb_io_slot_reset - called after the pci bus has been reset. |
9616 | * @pdev: Pointer to PCI device |
9617 | * |
9618 | * Restart the card from scratch, as if from a cold-boot. Implementation |
9619 | * resembles the first-half of the __igb_resume routine. |
9620 | **/ |
9621 | static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev) |
9622 | { |
9623 | struct net_device *netdev = pci_get_drvdata(pdev); |
9624 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9625 | struct e1000_hw *hw = &adapter->hw; |
9626 | pci_ers_result_t result; |
9627 | |
9628 | if (pci_enable_device_mem(dev: pdev)) { |
9629 | dev_err(&pdev->dev, |
9630 | "Cannot re-enable PCI device after reset.\n" ); |
9631 | result = PCI_ERS_RESULT_DISCONNECT; |
9632 | } else { |
9633 | pci_set_master(dev: pdev); |
9634 | pci_restore_state(dev: pdev); |
9635 | pci_save_state(dev: pdev); |
9636 | |
9637 | pci_enable_wake(dev: pdev, PCI_D3hot, enable: 0); |
9638 | pci_enable_wake(dev: pdev, PCI_D3cold, enable: 0); |
9639 | |
9640 | /* In case of PCI error, adapter lose its HW address |
9641 | * so we should re-assign it here. |
9642 | */ |
9643 | hw->hw_addr = adapter->io_addr; |
9644 | |
9645 | igb_reset(adapter); |
9646 | wr32(E1000_WUS, ~0); |
9647 | result = PCI_ERS_RESULT_RECOVERED; |
9648 | } |
9649 | |
9650 | return result; |
9651 | } |
9652 | |
9653 | /** |
9654 | * igb_io_resume - called when traffic can start flowing again. |
9655 | * @pdev: Pointer to PCI device |
9656 | * |
9657 | * This callback is called when the error recovery driver tells us that |
9658 | * its OK to resume normal operation. Implementation resembles the |
9659 | * second-half of the __igb_resume routine. |
9660 | */ |
9661 | static void igb_io_resume(struct pci_dev *pdev) |
9662 | { |
9663 | struct net_device *netdev = pci_get_drvdata(pdev); |
9664 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9665 | |
9666 | if (netif_running(dev: netdev)) { |
9667 | if (igb_up(adapter)) { |
9668 | dev_err(&pdev->dev, "igb_up failed after reset\n" ); |
9669 | return; |
9670 | } |
9671 | } |
9672 | |
9673 | netif_device_attach(dev: netdev); |
9674 | |
9675 | /* let the f/w know that the h/w is now under the control of the |
9676 | * driver. |
9677 | */ |
9678 | igb_get_hw_control(adapter); |
9679 | } |
9680 | |
9681 | /** |
9682 | * igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table |
9683 | * @adapter: Pointer to adapter structure |
9684 | * @index: Index of the RAR entry which need to be synced with MAC table |
9685 | **/ |
9686 | static void igb_rar_set_index(struct igb_adapter *adapter, u32 index) |
9687 | { |
9688 | struct e1000_hw *hw = &adapter->hw; |
9689 | u32 rar_low, rar_high; |
9690 | u8 *addr = adapter->mac_table[index].addr; |
9691 | |
9692 | /* HW expects these to be in network order when they are plugged |
9693 | * into the registers which are little endian. In order to guarantee |
9694 | * that ordering we need to do an leXX_to_cpup here in order to be |
9695 | * ready for the byteswap that occurs with writel |
9696 | */ |
9697 | rar_low = le32_to_cpup(p: (__le32 *)(addr)); |
9698 | rar_high = le16_to_cpup(p: (__le16 *)(addr + 4)); |
9699 | |
9700 | /* Indicate to hardware the Address is Valid. */ |
9701 | if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) { |
9702 | if (is_valid_ether_addr(addr)) |
9703 | rar_high |= E1000_RAH_AV; |
9704 | |
9705 | if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR) |
9706 | rar_high |= E1000_RAH_ASEL_SRC_ADDR; |
9707 | |
9708 | switch (hw->mac.type) { |
9709 | case e1000_82575: |
9710 | case e1000_i210: |
9711 | if (adapter->mac_table[index].state & |
9712 | IGB_MAC_STATE_QUEUE_STEERING) |
9713 | rar_high |= E1000_RAH_QSEL_ENABLE; |
9714 | |
9715 | rar_high |= E1000_RAH_POOL_1 * |
9716 | adapter->mac_table[index].queue; |
9717 | break; |
9718 | default: |
9719 | rar_high |= E1000_RAH_POOL_1 << |
9720 | adapter->mac_table[index].queue; |
9721 | break; |
9722 | } |
9723 | } |
9724 | |
9725 | wr32(E1000_RAL(index), rar_low); |
9726 | wrfl(); |
9727 | wr32(E1000_RAH(index), rar_high); |
9728 | wrfl(); |
9729 | } |
9730 | |
9731 | static int igb_set_vf_mac(struct igb_adapter *adapter, |
9732 | int vf, unsigned char *mac_addr) |
9733 | { |
9734 | struct e1000_hw *hw = &adapter->hw; |
9735 | /* VF MAC addresses start at end of receive addresses and moves |
9736 | * towards the first, as a result a collision should not be possible |
9737 | */ |
9738 | int rar_entry = hw->mac.rar_entry_count - (vf + 1); |
9739 | unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses; |
9740 | |
9741 | ether_addr_copy(dst: vf_mac_addr, src: mac_addr); |
9742 | ether_addr_copy(dst: adapter->mac_table[rar_entry].addr, src: mac_addr); |
9743 | adapter->mac_table[rar_entry].queue = vf; |
9744 | adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE; |
9745 | igb_rar_set_index(adapter, index: rar_entry); |
9746 | |
9747 | return 0; |
9748 | } |
9749 | |
9750 | static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) |
9751 | { |
9752 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9753 | |
9754 | if (vf >= adapter->vfs_allocated_count) |
9755 | return -EINVAL; |
9756 | |
9757 | /* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC |
9758 | * flag and allows to overwrite the MAC via VF netdev. This |
9759 | * is necessary to allow libvirt a way to restore the original |
9760 | * MAC after unbinding vfio-pci and reloading igbvf after shutting |
9761 | * down a VM. |
9762 | */ |
9763 | if (is_zero_ether_addr(addr: mac)) { |
9764 | adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC; |
9765 | dev_info(&adapter->pdev->dev, |
9766 | "remove administratively set MAC on VF %d\n" , |
9767 | vf); |
9768 | } else if (is_valid_ether_addr(addr: mac)) { |
9769 | adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC; |
9770 | dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n" , |
9771 | mac, vf); |
9772 | dev_info(&adapter->pdev->dev, |
9773 | "Reload the VF driver to make this change effective." ); |
9774 | /* Generate additional warning if PF is down */ |
9775 | if (test_bit(__IGB_DOWN, &adapter->state)) { |
9776 | dev_warn(&adapter->pdev->dev, |
9777 | "The VF MAC address has been set, but the PF device is not up.\n" ); |
9778 | dev_warn(&adapter->pdev->dev, |
9779 | "Bring the PF device up before attempting to use the VF device.\n" ); |
9780 | } |
9781 | } else { |
9782 | return -EINVAL; |
9783 | } |
9784 | return igb_set_vf_mac(adapter, vf, mac_addr: mac); |
9785 | } |
9786 | |
9787 | static int igb_link_mbps(int internal_link_speed) |
9788 | { |
9789 | switch (internal_link_speed) { |
9790 | case SPEED_100: |
9791 | return 100; |
9792 | case SPEED_1000: |
9793 | return 1000; |
9794 | default: |
9795 | return 0; |
9796 | } |
9797 | } |
9798 | |
9799 | static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate, |
9800 | int link_speed) |
9801 | { |
9802 | int rf_dec, rf_int; |
9803 | u32 bcnrc_val; |
9804 | |
9805 | if (tx_rate != 0) { |
9806 | /* Calculate the rate factor values to set */ |
9807 | rf_int = link_speed / tx_rate; |
9808 | rf_dec = (link_speed - (rf_int * tx_rate)); |
9809 | rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) / |
9810 | tx_rate; |
9811 | |
9812 | bcnrc_val = E1000_RTTBCNRC_RS_ENA; |
9813 | bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) & |
9814 | E1000_RTTBCNRC_RF_INT_MASK); |
9815 | bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK); |
9816 | } else { |
9817 | bcnrc_val = 0; |
9818 | } |
9819 | |
9820 | wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */ |
9821 | /* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM |
9822 | * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported. |
9823 | */ |
9824 | wr32(E1000_RTTBCNRM, 0x14); |
9825 | wr32(E1000_RTTBCNRC, bcnrc_val); |
9826 | } |
9827 | |
9828 | static void igb_check_vf_rate_limit(struct igb_adapter *adapter) |
9829 | { |
9830 | int actual_link_speed, i; |
9831 | bool reset_rate = false; |
9832 | |
9833 | /* VF TX rate limit was not set or not supported */ |
9834 | if ((adapter->vf_rate_link_speed == 0) || |
9835 | (adapter->hw.mac.type != e1000_82576)) |
9836 | return; |
9837 | |
9838 | actual_link_speed = igb_link_mbps(internal_link_speed: adapter->link_speed); |
9839 | if (actual_link_speed != adapter->vf_rate_link_speed) { |
9840 | reset_rate = true; |
9841 | adapter->vf_rate_link_speed = 0; |
9842 | dev_info(&adapter->pdev->dev, |
9843 | "Link speed has been changed. VF Transmit rate is disabled\n" ); |
9844 | } |
9845 | |
9846 | for (i = 0; i < adapter->vfs_allocated_count; i++) { |
9847 | if (reset_rate) |
9848 | adapter->vf_data[i].tx_rate = 0; |
9849 | |
9850 | igb_set_vf_rate_limit(hw: &adapter->hw, vf: i, |
9851 | tx_rate: adapter->vf_data[i].tx_rate, |
9852 | link_speed: actual_link_speed); |
9853 | } |
9854 | } |
9855 | |
9856 | static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, |
9857 | int min_tx_rate, int max_tx_rate) |
9858 | { |
9859 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9860 | struct e1000_hw *hw = &adapter->hw; |
9861 | int actual_link_speed; |
9862 | |
9863 | if (hw->mac.type != e1000_82576) |
9864 | return -EOPNOTSUPP; |
9865 | |
9866 | if (min_tx_rate) |
9867 | return -EINVAL; |
9868 | |
9869 | actual_link_speed = igb_link_mbps(internal_link_speed: adapter->link_speed); |
9870 | if ((vf >= adapter->vfs_allocated_count) || |
9871 | (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) || |
9872 | (max_tx_rate < 0) || |
9873 | (max_tx_rate > actual_link_speed)) |
9874 | return -EINVAL; |
9875 | |
9876 | adapter->vf_rate_link_speed = actual_link_speed; |
9877 | adapter->vf_data[vf].tx_rate = (u16)max_tx_rate; |
9878 | igb_set_vf_rate_limit(hw, vf, tx_rate: max_tx_rate, link_speed: actual_link_speed); |
9879 | |
9880 | return 0; |
9881 | } |
9882 | |
9883 | static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, |
9884 | bool setting) |
9885 | { |
9886 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9887 | struct e1000_hw *hw = &adapter->hw; |
9888 | u32 reg_val, reg_offset; |
9889 | |
9890 | if (!adapter->vfs_allocated_count) |
9891 | return -EOPNOTSUPP; |
9892 | |
9893 | if (vf >= adapter->vfs_allocated_count) |
9894 | return -EINVAL; |
9895 | |
9896 | reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC; |
9897 | reg_val = rd32(reg_offset); |
9898 | if (setting) |
9899 | reg_val |= (BIT(vf) | |
9900 | BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)); |
9901 | else |
9902 | reg_val &= ~(BIT(vf) | |
9903 | BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)); |
9904 | wr32(reg_offset, reg_val); |
9905 | |
9906 | adapter->vf_data[vf].spoofchk_enabled = setting; |
9907 | return 0; |
9908 | } |
9909 | |
9910 | static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting) |
9911 | { |
9912 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9913 | |
9914 | if (vf >= adapter->vfs_allocated_count) |
9915 | return -EINVAL; |
9916 | if (adapter->vf_data[vf].trusted == setting) |
9917 | return 0; |
9918 | |
9919 | adapter->vf_data[vf].trusted = setting; |
9920 | |
9921 | dev_info(&adapter->pdev->dev, "VF %u is %strusted\n" , |
9922 | vf, setting ? "" : "not " ); |
9923 | return 0; |
9924 | } |
9925 | |
9926 | static int igb_ndo_get_vf_config(struct net_device *netdev, |
9927 | int vf, struct ifla_vf_info *ivi) |
9928 | { |
9929 | struct igb_adapter *adapter = netdev_priv(dev: netdev); |
9930 | if (vf >= adapter->vfs_allocated_count) |
9931 | return -EINVAL; |
9932 | ivi->vf = vf; |
9933 | memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN); |
9934 | ivi->max_tx_rate = adapter->vf_data[vf].tx_rate; |
9935 | ivi->min_tx_rate = 0; |
9936 | ivi->vlan = adapter->vf_data[vf].pf_vlan; |
9937 | ivi->qos = adapter->vf_data[vf].pf_qos; |
9938 | ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled; |
9939 | ivi->trusted = adapter->vf_data[vf].trusted; |
9940 | return 0; |
9941 | } |
9942 | |
9943 | static void igb_vmm_control(struct igb_adapter *adapter) |
9944 | { |
9945 | struct e1000_hw *hw = &adapter->hw; |
9946 | u32 reg; |
9947 | |
9948 | switch (hw->mac.type) { |
9949 | case e1000_82575: |
9950 | case e1000_i210: |
9951 | case e1000_i211: |
9952 | case e1000_i354: |
9953 | default: |
9954 | /* replication is not supported for 82575 */ |
9955 | return; |
9956 | case e1000_82576: |
9957 | /* notify HW that the MAC is adding vlan tags */ |
9958 | reg = rd32(E1000_DTXCTL); |
9959 | reg |= E1000_DTXCTL_VLAN_ADDED; |
9960 | wr32(E1000_DTXCTL, reg); |
9961 | fallthrough; |
9962 | case e1000_82580: |
9963 | /* enable replication vlan tag stripping */ |
9964 | reg = rd32(E1000_RPLOLR); |
9965 | reg |= E1000_RPLOLR_STRVLAN; |
9966 | wr32(E1000_RPLOLR, reg); |
9967 | fallthrough; |
9968 | case e1000_i350: |
9969 | /* none of the above registers are supported by i350 */ |
9970 | break; |
9971 | } |
9972 | |
9973 | if (adapter->vfs_allocated_count) { |
9974 | igb_vmdq_set_loopback_pf(hw, true); |
9975 | igb_vmdq_set_replication_pf(hw, true); |
9976 | igb_vmdq_set_anti_spoofing_pf(hw, true, |
9977 | adapter->vfs_allocated_count); |
9978 | } else { |
9979 | igb_vmdq_set_loopback_pf(hw, false); |
9980 | igb_vmdq_set_replication_pf(hw, false); |
9981 | } |
9982 | } |
9983 | |
9984 | static void igb_init_dmac(struct igb_adapter *adapter, u32 pba) |
9985 | { |
9986 | struct e1000_hw *hw = &adapter->hw; |
9987 | u32 dmac_thr; |
9988 | u16 hwm; |
9989 | u32 reg; |
9990 | |
9991 | if (hw->mac.type > e1000_82580) { |
9992 | if (adapter->flags & IGB_FLAG_DMAC) { |
9993 | /* force threshold to 0. */ |
9994 | wr32(E1000_DMCTXTH, 0); |
9995 | |
9996 | /* DMA Coalescing high water mark needs to be greater |
9997 | * than the Rx threshold. Set hwm to PBA - max frame |
9998 | * size in 16B units, capping it at PBA - 6KB. |
9999 | */ |
10000 | hwm = 64 * (pba - 6); |
10001 | reg = rd32(E1000_FCRTC); |
10002 | reg &= ~E1000_FCRTC_RTH_COAL_MASK; |
10003 | reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) |
10004 | & E1000_FCRTC_RTH_COAL_MASK); |
10005 | wr32(E1000_FCRTC, reg); |
10006 | |
10007 | /* Set the DMA Coalescing Rx threshold to PBA - 2 * max |
10008 | * frame size, capping it at PBA - 10KB. |
10009 | */ |
10010 | dmac_thr = pba - 10; |
10011 | reg = rd32(E1000_DMACR); |
10012 | reg &= ~E1000_DMACR_DMACTHR_MASK; |
10013 | reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT) |
10014 | & E1000_DMACR_DMACTHR_MASK); |
10015 | |
10016 | /* transition to L0x or L1 if available..*/ |
10017 | reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); |
10018 | |
10019 | /* watchdog timer= +-1000 usec in 32usec intervals */ |
10020 | reg |= (1000 >> 5); |
10021 | |
10022 | /* Disable BMC-to-OS Watchdog Enable */ |
10023 | if (hw->mac.type != e1000_i354) |
10024 | reg &= ~E1000_DMACR_DC_BMC2OSW_EN; |
10025 | wr32(E1000_DMACR, reg); |
10026 | |
10027 | /* no lower threshold to disable |
10028 | * coalescing(smart fifb)-UTRESH=0 |
10029 | */ |
10030 | wr32(E1000_DMCRTRH, 0); |
10031 | |
10032 | reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4); |
10033 | |
10034 | wr32(E1000_DMCTLX, reg); |
10035 | |
10036 | /* free space in tx packet buffer to wake from |
10037 | * DMA coal |
10038 | */ |
10039 | wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE - |
10040 | (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6); |
10041 | } |
10042 | |
10043 | if (hw->mac.type >= e1000_i210 || |
10044 | (adapter->flags & IGB_FLAG_DMAC)) { |
10045 | reg = rd32(E1000_PCIEMISC); |
10046 | reg |= E1000_PCIEMISC_LX_DECISION; |
10047 | wr32(E1000_PCIEMISC, reg); |
10048 | } /* endif adapter->dmac is not disabled */ |
10049 | } else if (hw->mac.type == e1000_82580) { |
10050 | u32 reg = rd32(E1000_PCIEMISC); |
10051 | |
10052 | wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION); |
10053 | wr32(E1000_DMACR, 0); |
10054 | } |
10055 | } |
10056 | |
10057 | /** |
10058 | * igb_read_i2c_byte - Reads 8 bit word over I2C |
10059 | * @hw: pointer to hardware structure |
10060 | * @byte_offset: byte offset to read |
10061 | * @dev_addr: device address |
10062 | * @data: value read |
10063 | * |
10064 | * Performs byte read operation over I2C interface at |
10065 | * a specified device address. |
10066 | **/ |
10067 | s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset, |
10068 | u8 dev_addr, u8 *data) |
10069 | { |
10070 | struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw); |
10071 | struct i2c_client *this_client = adapter->i2c_client; |
10072 | s32 status; |
10073 | u16 swfw_mask = 0; |
10074 | |
10075 | if (!this_client) |
10076 | return E1000_ERR_I2C; |
10077 | |
10078 | swfw_mask = E1000_SWFW_PHY0_SM; |
10079 | |
10080 | if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) |
10081 | return E1000_ERR_SWFW_SYNC; |
10082 | |
10083 | status = i2c_smbus_read_byte_data(client: this_client, command: byte_offset); |
10084 | hw->mac.ops.release_swfw_sync(hw, swfw_mask); |
10085 | |
10086 | if (status < 0) |
10087 | return E1000_ERR_I2C; |
10088 | else { |
10089 | *data = status; |
10090 | return 0; |
10091 | } |
10092 | } |
10093 | |
10094 | /** |
10095 | * igb_write_i2c_byte - Writes 8 bit word over I2C |
10096 | * @hw: pointer to hardware structure |
10097 | * @byte_offset: byte offset to write |
10098 | * @dev_addr: device address |
10099 | * @data: value to write |
10100 | * |
10101 | * Performs byte write operation over I2C interface at |
10102 | * a specified device address. |
10103 | **/ |
10104 | s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset, |
10105 | u8 dev_addr, u8 data) |
10106 | { |
10107 | struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw); |
10108 | struct i2c_client *this_client = adapter->i2c_client; |
10109 | s32 status; |
10110 | u16 swfw_mask = E1000_SWFW_PHY0_SM; |
10111 | |
10112 | if (!this_client) |
10113 | return E1000_ERR_I2C; |
10114 | |
10115 | if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) |
10116 | return E1000_ERR_SWFW_SYNC; |
10117 | status = i2c_smbus_write_byte_data(client: this_client, command: byte_offset, value: data); |
10118 | hw->mac.ops.release_swfw_sync(hw, swfw_mask); |
10119 | |
10120 | if (status) |
10121 | return E1000_ERR_I2C; |
10122 | else |
10123 | return 0; |
10124 | |
10125 | } |
10126 | |
10127 | int igb_reinit_queues(struct igb_adapter *adapter) |
10128 | { |
10129 | struct net_device *netdev = adapter->netdev; |
10130 | struct pci_dev *pdev = adapter->pdev; |
10131 | int err = 0; |
10132 | |
10133 | if (netif_running(dev: netdev)) |
10134 | igb_close(netdev); |
10135 | |
10136 | igb_reset_interrupt_capability(adapter); |
10137 | |
10138 | if (igb_init_interrupt_scheme(adapter, msix: true)) { |
10139 | dev_err(&pdev->dev, "Unable to allocate memory for queues\n" ); |
10140 | return -ENOMEM; |
10141 | } |
10142 | |
10143 | if (netif_running(dev: netdev)) |
10144 | err = igb_open(netdev); |
10145 | |
10146 | return err; |
10147 | } |
10148 | |
10149 | static void igb_nfc_filter_exit(struct igb_adapter *adapter) |
10150 | { |
10151 | struct igb_nfc_filter *rule; |
10152 | |
10153 | spin_lock(lock: &adapter->nfc_lock); |
10154 | |
10155 | hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) |
10156 | igb_erase_filter(adapter, input: rule); |
10157 | |
10158 | hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) |
10159 | igb_erase_filter(adapter, input: rule); |
10160 | |
10161 | spin_unlock(lock: &adapter->nfc_lock); |
10162 | } |
10163 | |
10164 | static void igb_nfc_filter_restore(struct igb_adapter *adapter) |
10165 | { |
10166 | struct igb_nfc_filter *rule; |
10167 | |
10168 | spin_lock(lock: &adapter->nfc_lock); |
10169 | |
10170 | hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) |
10171 | igb_add_filter(adapter, input: rule); |
10172 | |
10173 | spin_unlock(lock: &adapter->nfc_lock); |
10174 | } |
10175 | /* igb_main.c */ |
10176 | |