1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright(c) 1999 - 2024 Intel Corporation. */
3
4	#include <linux/types.h>
5	#include <linux/module.h>
6	#include <linux/pci.h>
7	#include <linux/netdevice.h>
8	#include <linux/vmalloc.h>
9	#include <linux/string.h>
10	#include <linux/in.h>
11	#include <linux/interrupt.h>
12	#include <linux/ip.h>
13	#include <linux/tcp.h>
14	#include <linux/sctp.h>
15	#include <linux/pkt_sched.h>
16	#include <linux/ipv6.h>
17	#include <linux/slab.h>
18	#include <net/checksum.h>
19	#include <net/ip6_checksum.h>
20	#include <linux/etherdevice.h>
21	#include <linux/ethtool.h>
22	#include <linux/if.h>
23	#include <linux/if_vlan.h>
24	#include <linux/if_macvlan.h>
25	#include <linux/if_bridge.h>
26	#include <linux/prefetch.h>
27	#include <linux/bpf.h>
28	#include <linux/bpf_trace.h>
29	#include <linux/atomic.h>
30	#include <linux/numa.h>
31	#include <generated/utsrelease.h>
32	#include <scsi/fc/fc_fcoe.h>
33	#include <net/udp_tunnel.h>
34	#include <net/pkt_cls.h>
35	#include <net/tc_act/tc_gact.h>
36	#include <net/tc_act/tc_mirred.h>
37	#include <net/vxlan.h>
38	#include <net/mpls.h>
39	#include <net/netdev_queues.h>
40	#include <net/xdp_sock_drv.h>
41	#include <net/xfrm.h>
42
43	#include "ixgbe.h"
44	#include "ixgbe_common.h"
45	#include "ixgbe_e610.h"
46	#include "ixgbe_dcb_82599.h"
47	#include "ixgbe_mbx.h"
48	#include "ixgbe_phy.h"
49	#include "ixgbe_sriov.h"
50	#include "ixgbe_model.h"
51	#include "ixgbe_txrx_common.h"
52	#include "devlink/devlink.h"
53
54	char ixgbe_driver_name[] = "ixgbe";
55	static const char ixgbe_driver_string[] =
56	"Intel(R) 10 Gigabit PCI Express Network Driver";
57	#ifdef IXGBE_FCOE
58	char ixgbe_default_device_descr[] =
59	"Intel(R) 10 Gigabit Network Connection";
60	#else
61	static char ixgbe_default_device_descr[] =
62	"Intel(R) 10 Gigabit Network Connection";
63	#endif
64	static const char ixgbe_copyright[] =
65	"Copyright (c) 1999-2016 Intel Corporation.";
66
67	static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter";
68
69	static const struct ixgbe_info *ixgbe_info_tbl[] = {
70	[board_82598] = &ixgbe_82598_info,
71	[board_82599] = &ixgbe_82599_info,
72	[board_X540] = &ixgbe_X540_info,
73	[board_X550] = &ixgbe_X550_info,
74	[board_X550EM_x] = &ixgbe_X550EM_x_info,
75	[board_x550em_x_fw] = &ixgbe_x550em_x_fw_info,
76	[board_x550em_a] = &ixgbe_x550em_a_info,
77	[board_x550em_a_fw] = &ixgbe_x550em_a_fw_info,
78	[board_e610] = &ixgbe_e610_info,
79	};
80
81	/* ixgbe_pci_tbl - PCI Device ID Table
82	*
83	* Wildcard entries (PCI_ANY_ID) should come last
84	* Last entry must be all 0s
85	*
86	* { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
87	* Class, Class Mask, private data (not used) }
88	*/
89	static const struct pci_device_id ixgbe_pci_tbl[] = {
90	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
91	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), .vendor: board_82598 },
92	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
93	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), .vendor: board_82598 },
94	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
95	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), .vendor: board_82598 },
96	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
97	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), .vendor: board_82598 },
98	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
99	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), .vendor: board_82598 },
100	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
101	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), .vendor: board_82598 },
102	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
103	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), .vendor: board_82599 },
104	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
105	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), .vendor: board_82599 },
106	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
107	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), .vendor: board_82599 },
108	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
109	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), .vendor: board_82599 },
110	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
111	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), .vendor: board_82599 },
112	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
113	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), .vendor: board_X540 },
114	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
115	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), .vendor: board_82599 },
116	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 },
117	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), .vendor: board_82599 },
118	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
119	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), .vendor: board_X540 },
120	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550},
121	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), .vendor: board_X550},
122	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x},
123	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_XFI), .vendor: board_X550EM_x},
124	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x},
125	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), .vendor: board_X550EM_x},
126	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x},
127	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_1G_T), .vendor: board_x550em_x_fw},
128	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a },
129	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), .vendor: board_x550em_a },
130	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },
131	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), .vendor: board_x550em_a },
132	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },
133	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), .vendor: board_x550em_a},
134	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },
135	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), .vendor: board_x550em_a_fw },
136	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), board_x550em_a_fw },
137	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_BACKPLANE), .vendor: board_e610},
138	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_SFP), board_e610},
139	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_10G_T), .vendor: board_e610},
140	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_2_5G_T), board_e610},
141	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_SGMII), .vendor: board_e610},
142	/* required last entry */
143	{0, }
144	};
145	MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
146
147	#ifdef CONFIG_IXGBE_DCA
148	static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
149	void *p);
150	static struct notifier_block dca_notifier = {
151	.notifier_call = ixgbe_notify_dca,
152	.next = NULL,
153	.priority = 0
154	};
155	#endif
156
157	#ifdef CONFIG_PCI_IOV
158	static unsigned int max_vfs;
159	module_param(max_vfs, uint, 0);
160	MODULE_PARM_DESC(max_vfs,
161	"Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63. (Deprecated)");
162	#endif /* CONFIG_PCI_IOV */
163
164	static bool allow_unsupported_sfp;
165	module_param(allow_unsupported_sfp, bool, 0444);
166	MODULE_PARM_DESC(allow_unsupported_sfp,
167	"Allow unsupported and untested SFP+ modules on 82599-based adapters");
168
169	#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
170	static int debug = -1;
171	module_param(debug, int, 0);
172	MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
173
174	MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
175	MODULE_LICENSE("GPL v2");
176
177	DEFINE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key);
178	EXPORT_SYMBOL(ixgbe_xdp_locking_key);
179
180	static struct workqueue_struct *ixgbe_wq;
181
182	static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev);
183	static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *);
184	static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *);
185	static void ixgbe_watchdog_update_link(struct ixgbe_adapter *);
186
187	static const struct net_device_ops ixgbe_netdev_ops;
188
189	static bool netif_is_ixgbe(struct net_device *dev)
190	{
191	return dev && (dev->netdev_ops == &ixgbe_netdev_ops);
192	}
193
194	static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter,
195	u32 reg, u16 *value)
196	{
197	struct pci_dev *parent_dev;
198	struct pci_bus *parent_bus;
199
200	parent_bus = adapter->pdev->bus->parent;
201	if (!parent_bus)
202	return -1;
203
204	parent_dev = parent_bus->self;
205	if (!parent_dev)
206	return -1;
207
208	if (!pci_is_pcie(dev: parent_dev))
209	return -1;
210
211	pcie_capability_read_word(dev: parent_dev, pos: reg, val: value);
212	if (*value == IXGBE_FAILED_READ_CFG_WORD &&
213	ixgbe_check_cfg_remove(hw: &adapter->hw, pdev: parent_dev))
214	return -1;
215	return 0;
216	}
217
218	static int ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter)
219	{
220	struct ixgbe_hw *hw = &adapter->hw;
221	u16 link_status = 0;
222	int err;
223
224	hw->bus.type = ixgbe_bus_type_pci_express;
225
226	/* Get the negotiated link width and speed from PCI config space of the
227	* parent, as this device is behind a switch
228	*/
229	err = ixgbe_read_pci_cfg_word_parent(adapter, reg: 18, value: &link_status);
230
231	/* assume caller will handle error case */
232	if (err)
233	return err;
234
235	hw->bus.width = ixgbe_convert_bus_width(link_status);
236	hw->bus.speed = ixgbe_convert_bus_speed(link_status);
237
238	return 0;
239	}
240
241	/**
242	* ixgbe_pcie_from_parent - Determine whether PCIe info should come from parent
243	* @hw: hw specific details
244	*
245	* This function is used by probe to determine whether a device's PCI-Express
246	* bandwidth details should be gathered from the parent bus instead of from the
247	* device. Used to ensure that various locations all have the correct device ID
248	* checks.
249	*
250	* Return: true if information should be collected from the parent bus, false
251	* otherwise
252	*/
253	static bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw)
254	{
255	switch (hw->device_id) {
256	case IXGBE_DEV_ID_82599_SFP_SF_QP:
257	case IXGBE_DEV_ID_82599_QSFP_SF_QP:
258	return true;
259	default:
260	return false;
261	}
262	}
263
264	static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,
265	int expected_gts)
266	{
267	struct ixgbe_hw *hw = &adapter->hw;
268	struct pci_dev *pdev;
269
270	/* Some devices are not connected over PCIe and thus do not negotiate
271	* speed. These devices do not have valid bus info, and thus any report
272	* we generate may not be correct.
273	*/
274	if (hw->bus.type == ixgbe_bus_type_internal)
275	return;
276
277	/* determine whether to use the parent device */
278	if (ixgbe_pcie_from_parent(hw: &adapter->hw))
279	pdev = adapter->pdev->bus->parent->self;
280	else
281	pdev = adapter->pdev;
282
283	pcie_print_link_status(dev: pdev);
284	}
285
286	static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
287	{
288	if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
289	!test_bit(__IXGBE_REMOVING, &adapter->state) &&
290	!test_and_set_bit(nr: __IXGBE_SERVICE_SCHED, addr: &adapter->state))
291	queue_work(wq: ixgbe_wq, work: &adapter->service_task);
292	}
293
294	static void ixgbe_remove_adapter(struct ixgbe_hw *hw)
295	{
296	struct ixgbe_adapter *adapter = hw->back;
297
298	if (!hw->hw_addr)
299	return;
300	hw->hw_addr = NULL;
301	e_dev_err("Adapter removed\n");
302	if (test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
303	ixgbe_service_event_schedule(adapter);
304	}
305
306	static u32 ixgbe_check_remove(struct ixgbe_hw *hw, u32 reg)
307	{
308	u8 __iomem *reg_addr;
309	u32 value;
310	int i;
311
312	reg_addr = READ_ONCE(hw->hw_addr);
313	if (ixgbe_removed(addr: reg_addr))
314	return IXGBE_FAILED_READ_REG;
315
316	/* Register read of 0xFFFFFFF can indicate the adapter has been removed,
317	* so perform several status register reads to determine if the adapter
318	* has been removed.
319	*/
320	for (i = 0; i < IXGBE_FAILED_READ_RETRIES; i++) {
321	value = readl(addr: reg_addr + IXGBE_STATUS);
322	if (value != IXGBE_FAILED_READ_REG)
323	break;
324	mdelay(3);
325	}
326
327	if (value == IXGBE_FAILED_READ_REG)
328	ixgbe_remove_adapter(hw);
329	else
330	value = readl(addr: reg_addr + reg);
331	return value;
332	}
333
334	/**
335	* ixgbe_read_reg - Read from device register
336	* @hw: hw specific details
337	* @reg: offset of register to read
338	*
339	* Returns : value read or IXGBE_FAILED_READ_REG if removed
340	*
341	* This function is used to read device registers. It checks for device
342	* removal by confirming any read that returns all ones by checking the
343	* status register value for all ones. This function avoids reading from
344	* the hardware if a removal was previously detected in which case it
345	* returns IXGBE_FAILED_READ_REG (all ones).
346	*/
347	u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)
348	{
349	u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr);
350	u32 value;
351
352	if (ixgbe_removed(addr: reg_addr))
353	return IXGBE_FAILED_READ_REG;
354	if (unlikely(hw->phy.nw_mng_if_sel &
355	IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) {
356	struct ixgbe_adapter *adapter;
357	int i;
358
359	for (i = 0; i < 200; ++i) {
360	value = readl(addr: reg_addr + IXGBE_MAC_SGMII_BUSY);
361	if (likely(!value))
362	goto writes_completed;
363	if (value == IXGBE_FAILED_READ_REG) {
364	ixgbe_remove_adapter(hw);
365	return IXGBE_FAILED_READ_REG;
366	}
367	udelay(usec: 5);
368	}
369
370	adapter = hw->back;
371	e_warn(hw, "register writes incomplete %08x\n", value);
372	}
373
374	writes_completed:
375	value = readl(addr: reg_addr + reg);
376	if (unlikely(value == IXGBE_FAILED_READ_REG))
377	value = ixgbe_check_remove(hw, reg);
378	return value;
379	}
380
381	static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev)
382	{
383	u16 value;
384
385	pci_read_config_word(dev: pdev, PCI_VENDOR_ID, val: &value);
386	if (value == IXGBE_FAILED_READ_CFG_WORD) {
387	ixgbe_remove_adapter(hw);
388	return true;
389	}
390	return false;
391	}
392
393	u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg)
394	{
395	struct ixgbe_adapter *adapter = hw->back;
396	u16 value;
397
398	if (ixgbe_removed(addr: hw->hw_addr))
399	return IXGBE_FAILED_READ_CFG_WORD;
400	pci_read_config_word(dev: adapter->pdev, where: reg, val: &value);
401	if (value == IXGBE_FAILED_READ_CFG_WORD &&
402	ixgbe_check_cfg_remove(hw, pdev: adapter->pdev))
403	return IXGBE_FAILED_READ_CFG_WORD;
404	return value;
405	}
406
407	#ifdef CONFIG_PCI_IOV
408	static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg)
409	{
410	struct ixgbe_adapter *adapter = hw->back;
411	u32 value;
412
413	if (ixgbe_removed(addr: hw->hw_addr))
414	return IXGBE_FAILED_READ_CFG_DWORD;
415	pci_read_config_dword(dev: adapter->pdev, where: reg, val: &value);
416	if (value == IXGBE_FAILED_READ_CFG_DWORD &&
417	ixgbe_check_cfg_remove(hw, pdev: adapter->pdev))
418	return IXGBE_FAILED_READ_CFG_DWORD;
419	return value;
420	}
421	#endif /* CONFIG_PCI_IOV */
422
423	void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value)
424	{
425	struct ixgbe_adapter *adapter = hw->back;
426
427	if (ixgbe_removed(addr: hw->hw_addr))
428	return;
429	pci_write_config_word(dev: adapter->pdev, where: reg, val: value);
430	}
431
432	static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
433	{
434	BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
435
436	/* flush memory to make sure state is correct before next watchdog */
437	smp_mb__before_atomic();
438	clear_bit(nr: __IXGBE_SERVICE_SCHED, addr: &adapter->state);
439	}
440
441	struct ixgbe_reg_info {
442	u32 ofs;
443	char *name;
444	};
445
446	static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
447
448	/* General Registers */
449	{IXGBE_CTRL, "CTRL"},
450	{IXGBE_STATUS, "STATUS"},
451	{IXGBE_CTRL_EXT, "CTRL_EXT"},
452
453	/* Interrupt Registers */
454	{IXGBE_EICR, "EICR"},
455
456	/* RX Registers */
457	{IXGBE_SRRCTL(0), "SRRCTL"},
458	{IXGBE_DCA_RXCTRL(0), "DRXCTL"},
459	{IXGBE_RDLEN(0), "RDLEN"},
460	{IXGBE_RDH(0), "RDH"},
461	{IXGBE_RDT(0), "RDT"},
462	{IXGBE_RXDCTL(0), "RXDCTL"},
463	{IXGBE_RDBAL(0), "RDBAL"},
464	{IXGBE_RDBAH(0), "RDBAH"},
465
466	/* TX Registers */
467	{IXGBE_TDBAL(0), "TDBAL"},
468	{IXGBE_TDBAH(0), "TDBAH"},
469	{IXGBE_TDLEN(0), "TDLEN"},
470	{IXGBE_TDH(0), "TDH"},
471	{IXGBE_TDT(0), "TDT"},
472	{IXGBE_TXDCTL(0), "TXDCTL"},
473
474	/* List Terminator */
475	{ .name = NULL }
476	};
477
478
479	/*
480	* ixgbe_regdump - register printout routine
481	*/
482	static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
483	{
484	int i;
485	char rname[16];
486	u32 regs[64];
487
488	switch (reginfo->ofs) {
489	case IXGBE_SRRCTL(0):
490	for (i = 0; i < 64; i++)
491	regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
492	break;
493	case IXGBE_DCA_RXCTRL(0):
494	for (i = 0; i < 64; i++)
495	regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
496	break;
497	case IXGBE_RDLEN(0):
498	for (i = 0; i < 64; i++)
499	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
500	break;
501	case IXGBE_RDH(0):
502	for (i = 0; i < 64; i++)
503	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
504	break;
505	case IXGBE_RDT(0):
506	for (i = 0; i < 64; i++)
507	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
508	break;
509	case IXGBE_RXDCTL(0):
510	for (i = 0; i < 64; i++)
511	regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
512	break;
513	case IXGBE_RDBAL(0):
514	for (i = 0; i < 64; i++)
515	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
516	break;
517	case IXGBE_RDBAH(0):
518	for (i = 0; i < 64; i++)
519	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
520	break;
521	case IXGBE_TDBAL(0):
522	for (i = 0; i < 64; i++)
523	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
524	break;
525	case IXGBE_TDBAH(0):
526	for (i = 0; i < 64; i++)
527	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
528	break;
529	case IXGBE_TDLEN(0):
530	for (i = 0; i < 64; i++)
531	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
532	break;
533	case IXGBE_TDH(0):
534	for (i = 0; i < 64; i++)
535	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
536	break;
537	case IXGBE_TDT(0):
538	for (i = 0; i < 64; i++)
539	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
540	break;
541	case IXGBE_TXDCTL(0):
542	for (i = 0; i < 64; i++)
543	regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
544	break;
545	default:
546	pr_info("%-15s %08x\n",
547	reginfo->name, IXGBE_READ_REG(hw, reginfo->ofs));
548	return;
549	}
550
551	i = 0;
552	while (i < 64) {
553	int j;
554	char buf[9 * 8 + 1];
555	char *p = buf;
556
557	snprintf(buf: rname, size: 16, fmt: "%s[%d-%d]", reginfo->name, i, i + 7);
558	for (j = 0; j < 8; j++)
559	p += sprintf(buf: p, fmt: " %08x", regs[i++]);
560	pr_err("%-15s%s\n", rname, buf);
561	}
562
563	}
564
565	static void ixgbe_print_buffer(struct ixgbe_ring *ring, int n)
566	{
567	struct ixgbe_tx_buffer *tx_buffer;
568
569	tx_buffer = &ring->tx_buffer_info[ring->next_to_clean];
570	pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",
571	n, ring->next_to_use, ring->next_to_clean,
572	(u64)dma_unmap_addr(tx_buffer, dma),
573	dma_unmap_len(tx_buffer, len),
574	tx_buffer->next_to_watch,
575	(u64)tx_buffer->time_stamp);
576	}
577
578	/*
579	* ixgbe_dump - Print registers, tx-rings and rx-rings
580	*/
581	static void ixgbe_dump(struct ixgbe_adapter *adapter)
582	{
583	struct net_device *netdev = adapter->netdev;
584	struct ixgbe_hw *hw = &adapter->hw;
585	struct ixgbe_reg_info *reginfo;
586	int n = 0;
587	struct ixgbe_ring *ring;
588	struct ixgbe_tx_buffer *tx_buffer;
589	union ixgbe_adv_tx_desc *tx_desc;
590	struct my_u0 { u64 a; u64 b; } *u0;
591	struct ixgbe_ring *rx_ring;
592	union ixgbe_adv_rx_desc *rx_desc;
593	struct ixgbe_rx_buffer *rx_buffer_info;
594	int i = 0;
595
596	if (!netif_msg_hw(adapter))
597	return;
598
599	/* Print netdevice Info */
600	if (netdev) {
601	dev_info(&adapter->pdev->dev, "Net device Info\n");
602	pr_info("Device Name state "
603	"trans_start\n");
604	pr_info("%-15s %016lX %016lX\n",
605	netdev->name,
606	netdev->state,
607	dev_trans_start(netdev));
608	}
609
610	/* Print Registers */
611	dev_info(&adapter->pdev->dev, "Register Dump\n");
612	pr_info(" Register Name Value\n");
613	for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
614	reginfo->name; reginfo++) {
615	ixgbe_regdump(hw, reginfo);
616	}
617
618	/* Print TX Ring Summary */
619	if (!netdev || !netif_running(dev: netdev))
620	return;
621
622	dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
623	pr_info(" %s %s %s %s\n",
624	"Queue [NTU] [NTC] [bi(ntc)->dma ]",
625	"leng", "ntw", "timestamp");
626	for (n = 0; n < adapter->num_tx_queues; n++) {
627	ring = adapter->tx_ring[n];
628	ixgbe_print_buffer(ring, n);
629	}
630
631	for (n = 0; n < adapter->num_xdp_queues; n++) {
632	ring = adapter->xdp_ring[n];
633	ixgbe_print_buffer(ring, n);
634	}
635
636	/* Print TX Rings */
637	if (!netif_msg_tx_done(adapter))
638	goto rx_ring_summary;
639
640	dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
641
642	/* Transmit Descriptor Formats
643	*
644	* 82598 Advanced Transmit Descriptor
645	* +--------------------------------------------------------------+
646	* 0 | Buffer Address [63:0] |
647	* +--------------------------------------------------------------+
648	* 8 | PAYLEN | POPTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |
649	* +--------------------------------------------------------------+
650	* 63 46 45 40 39 36 35 32 31 24 23 20 19 0
651	*
652	* 82598 Advanced Transmit Descriptor (Write-Back Format)
653	* +--------------------------------------------------------------+
654	* 0 | RSV [63:0] |
655	* +--------------------------------------------------------------+
656	* 8 | RSV | STA | NXTSEQ |
657	* +--------------------------------------------------------------+
658	* 63 36 35 32 31 0
659	*
660	* 82599+ Advanced Transmit Descriptor
661	* +--------------------------------------------------------------+
662	* 0 | Buffer Address [63:0] |
663	* +--------------------------------------------------------------+
664	* 8 |PAYLEN |POPTS|CC|IDX |STA |DCMD |DTYP |MAC |RSV |DTALEN |
665	* +--------------------------------------------------------------+
666	* 63 46 45 40 39 38 36 35 32 31 24 23 20 19 18 17 16 15 0
667	*
668	* 82599+ Advanced Transmit Descriptor (Write-Back Format)
669	* +--------------------------------------------------------------+
670	* 0 | RSV [63:0] |
671	* +--------------------------------------------------------------+
672	* 8 | RSV | STA | RSV |
673	* +--------------------------------------------------------------+
674	* 63 36 35 32 31 0
675	*/
676
677	for (n = 0; n < adapter->num_tx_queues; n++) {
678	ring = adapter->tx_ring[n];
679	pr_info("------------------------------------\n");
680	pr_info("TX QUEUE INDEX = %d\n", ring->queue_index);
681	pr_info("------------------------------------\n");
682	pr_info("%s%s %s %s %s %s\n",
683	"T [desc] [address 63:0 ] ",
684	"[PlPOIdStDDt Ln] [bi->dma ] ",
685	"leng", "ntw", "timestamp", "bi->skb");
686
687	for (i = 0; ring->desc && (i < ring->count); i++) {
688	tx_desc = IXGBE_TX_DESC(ring, i);
689	tx_buffer = &ring->tx_buffer_info[i];
690	u0 = (struct my_u0 *)tx_desc;
691	if (dma_unmap_len(tx_buffer, len) > 0) {
692	const char *ring_desc;
693
694	if (i == ring->next_to_use &&
695	i == ring->next_to_clean)
696	ring_desc = " NTC/U";
697	else if (i == ring->next_to_use)
698	ring_desc = " NTU";
699	else if (i == ring->next_to_clean)
700	ring_desc = " NTC";
701	else
702	ring_desc = "";
703	pr_info("T [0x%03X] %016llX %016llX %016llX %08X %p %016llX %p%s",
704	i,
705	le64_to_cpu((__force __le64)u0->a),
706	le64_to_cpu((__force __le64)u0->b),
707	(u64)dma_unmap_addr(tx_buffer, dma),
708	dma_unmap_len(tx_buffer, len),
709	tx_buffer->next_to_watch,
710	(u64)tx_buffer->time_stamp,
711	tx_buffer->skb,
712	ring_desc);
713
714	if (netif_msg_pktdata(adapter) &&
715	tx_buffer->skb)
716	print_hex_dump(KERN_INFO, prefix_str: "",
717	prefix_type: DUMP_PREFIX_ADDRESS, rowsize: 16, groupsize: 1,
718	buf: tx_buffer->skb->data,
719	dma_unmap_len(tx_buffer, len),
720	ascii: true);
721	}
722	}
723	}
724
725	/* Print RX Rings Summary */
726	rx_ring_summary:
727	dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
728	pr_info("Queue [NTU] [NTC]\n");
729	for (n = 0; n < adapter->num_rx_queues; n++) {
730	rx_ring = adapter->rx_ring[n];
731	pr_info("%5d %5X %5X\n",
732	n, rx_ring->next_to_use, rx_ring->next_to_clean);
733	}
734
735	/* Print RX Rings */
736	if (!netif_msg_rx_status(adapter))
737	return;
738
739	dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
740
741	/* Receive Descriptor Formats
742	*
743	* 82598 Advanced Receive Descriptor (Read) Format
744	* 63 1 0
745	* +-----------------------------------------------------+
746	* 0 | Packet Buffer Address [63:1] |A0/NSE|
747	* +----------------------------------------------+------+
748	* 8 | Header Buffer Address [63:1] | DD |
749	* +-----------------------------------------------------+
750	*
751	*
752	* 82598 Advanced Receive Descriptor (Write-Back) Format
753	*
754	* 63 48 47 32 31 30 21 20 16 15 4 3 0
755	* +------------------------------------------------------+
756	* 0 | RSS Hash / |SPH| HDR_LEN | RSV |Packet| RSS |
757	* | Packet | IP | | | | Type | Type |
758	* | Checksum | Ident | | | | | |
759	* +------------------------------------------------------+
760	* 8 | VLAN Tag | Length | Extended Error | Extended Status |
761	* +------------------------------------------------------+
762	* 63 48 47 32 31 20 19 0
763	*
764	* 82599+ Advanced Receive Descriptor (Read) Format
765	* 63 1 0
766	* +-----------------------------------------------------+
767	* 0 | Packet Buffer Address [63:1] |A0/NSE|
768	* +----------------------------------------------+------+
769	* 8 | Header Buffer Address [63:1] | DD |
770	* +-----------------------------------------------------+
771	*
772	*
773	* 82599+ Advanced Receive Descriptor (Write-Back) Format
774	*
775	* 63 48 47 32 31 30 21 20 17 16 4 3 0
776	* +------------------------------------------------------+
777	* 0 |RSS / Frag Checksum|SPH| HDR_LEN |RSC- |Packet| RSS |
778	* |/ RTT / PCoE_PARAM | | | CNT | Type | Type |
779	* |/ Flow Dir Flt ID | | | | | |
780	* +------------------------------------------------------+
781	* 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |
782	* +------------------------------------------------------+
783	* 63 48 47 32 31 20 19 0
784	*/
785
786	for (n = 0; n < adapter->num_rx_queues; n++) {
787	rx_ring = adapter->rx_ring[n];
788	pr_info("------------------------------------\n");
789	pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
790	pr_info("------------------------------------\n");
791	pr_info("%s%s%s\n",
792	"R [desc] [ PktBuf A0] ",
793	"[ HeadBuf DD] [bi->dma ] [bi->skb ] ",
794	"<-- Adv Rx Read format");
795	pr_info("%s%s%s\n",
796	"RWB[desc] [PcsmIpSHl PtRs] ",
797	"[vl er S cks ln] ---------------- [bi->skb ] ",
798	"<-- Adv Rx Write-Back format");
799
800	for (i = 0; i < rx_ring->count; i++) {
801	const char *ring_desc;
802
803	if (i == rx_ring->next_to_use)
804	ring_desc = " NTU";
805	else if (i == rx_ring->next_to_clean)
806	ring_desc = " NTC";
807	else
808	ring_desc = "";
809
810	rx_buffer_info = &rx_ring->rx_buffer_info[i];
811	rx_desc = IXGBE_RX_DESC(rx_ring, i);
812	u0 = (struct my_u0 *)rx_desc;
813	if (rx_desc->wb.upper.length) {
814	/* Descriptor Done */
815	pr_info("RWB[0x%03X] %016llX %016llX ---------------- %p%s\n",
816	i,
817	le64_to_cpu((__force __le64)u0->a),
818	le64_to_cpu((__force __le64)u0->b),
819	rx_buffer_info->skb,
820	ring_desc);
821	} else {
822	pr_info("R [0x%03X] %016llX %016llX %016llX %p%s\n",
823	i,
824	le64_to_cpu((__force __le64)u0->a),
825	le64_to_cpu((__force __le64)u0->b),
826	(u64)rx_buffer_info->dma,
827	rx_buffer_info->skb,
828	ring_desc);
829
830	if (netif_msg_pktdata(adapter) &&
831	rx_buffer_info->dma) {
832	print_hex_dump(KERN_INFO, prefix_str: "",
833	prefix_type: DUMP_PREFIX_ADDRESS, rowsize: 16, groupsize: 1,
834	page_address(rx_buffer_info->page) +
835	rx_buffer_info->page_offset,
836	len: ixgbe_rx_bufsz(ring: rx_ring), ascii: true);
837	}
838	}
839	}
840	}
841	}
842
843	static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
844	{
845	u32 ctrl_ext;
846
847	/* Let firmware take over control of h/w */
848	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
849	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
850	ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
851	}
852
853	static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
854	{
855	u32 ctrl_ext;
856
857	/* Let firmware know the driver has taken over */
858	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
859	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
860	ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
861	}
862
863	/**
864	* ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
865	* @adapter: pointer to adapter struct
866	* @direction: 0 for Rx, 1 for Tx, -1 for other causes
867	* @queue: queue to map the corresponding interrupt to
868	* @msix_vector: the vector to map to the corresponding queue
869	*
870	*/
871	static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
872	u8 queue, u8 msix_vector)
873	{
874	u32 ivar, index;
875	struct ixgbe_hw *hw = &adapter->hw;
876	switch (hw->mac.type) {
877	case ixgbe_mac_82598EB:
878	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
879	if (direction == -1)
880	direction = 0;
881	index = (((direction * 64) + queue) >> 2) & 0x1F;
882	ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
883	ivar &= ~(0xFF << (8 * (queue & 0x3)));
884	ivar |= (msix_vector << (8 * (queue & 0x3)));
885	IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
886	break;
887	case ixgbe_mac_82599EB:
888	case ixgbe_mac_X540:
889	case ixgbe_mac_X550:
890	case ixgbe_mac_X550EM_x:
891	case ixgbe_mac_x550em_a:
892	case ixgbe_mac_e610:
893	if (direction == -1) {
894	/* other causes */
895	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
896	index = ((queue & 1) * 8);
897	ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
898	ivar &= ~(0xFF << index);
899	ivar |= (msix_vector << index);
900	IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
901	break;
902	} else {
903	/* tx or rx causes */
904	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
905	index = ((16 * (queue & 1)) + (8 * direction));
906	ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
907	ivar &= ~(0xFF << index);
908	ivar |= (msix_vector << index);
909	IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
910	break;
911	}
912	default:
913	break;
914	}
915	}
916
917	void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
918	u64 qmask)
919	{
920	u32 mask;
921
922	switch (adapter->hw.mac.type) {
923	case ixgbe_mac_82598EB:
924	mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
925	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
926	break;
927	case ixgbe_mac_82599EB:
928	case ixgbe_mac_X540:
929	case ixgbe_mac_X550:
930	case ixgbe_mac_X550EM_x:
931	case ixgbe_mac_x550em_a:
932	case ixgbe_mac_e610:
933	mask = (qmask & 0xFFFFFFFF);
934	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
935	mask = (qmask >> 32);
936	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
937	break;
938	default:
939	break;
940	}
941	}
942
943	static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
944	{
945	struct ixgbe_hw *hw = &adapter->hw;
946	struct ixgbe_hw_stats *hwstats = &adapter->stats;
947	int i;
948	u32 data;
949
950	if ((hw->fc.current_mode != ixgbe_fc_full) &&
951	(hw->fc.current_mode != ixgbe_fc_rx_pause))
952	return;
953
954	switch (hw->mac.type) {
955	case ixgbe_mac_82598EB:
956	data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
957	break;
958	default:
959	data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
960	}
961	hwstats->lxoffrxc += data;
962
963	/* refill credits (no tx hang) if we received xoff */
964	if (!data)
965	return;
966
967	for (i = 0; i < adapter->num_tx_queues; i++)
968	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED,
969	addr: &adapter->tx_ring[i]->state);
970
971	for (i = 0; i < adapter->num_xdp_queues; i++)
972	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED,
973	addr: &adapter->xdp_ring[i]->state);
974	}
975
976	static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
977	{
978	struct ixgbe_hw *hw = &adapter->hw;
979	struct ixgbe_hw_stats *hwstats = &adapter->stats;
980	u32 xoff[8] = {0};
981	u8 tc;
982	int i;
983	bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
984
985	if (adapter->ixgbe_ieee_pfc)
986	pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
987
988	if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
989	ixgbe_update_xoff_rx_lfc(adapter);
990	return;
991	}
992
993	/* update stats for each tc, only valid with PFC enabled */
994	for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
995	u32 pxoffrxc;
996
997	switch (hw->mac.type) {
998	case ixgbe_mac_82598EB:
999	pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
1000	break;
1001	default:
1002	pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
1003	}
1004	hwstats->pxoffrxc[i] += pxoffrxc;
1005	/* Get the TC for given UP */
1006	tc = netdev_get_prio_tc_map(dev: adapter->netdev, prio: i);
1007	xoff[tc] += pxoffrxc;
1008	}
1009
1010	/* disarm tx queues that have received xoff frames */
1011	for (i = 0; i < adapter->num_tx_queues; i++) {
1012	struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
1013
1014	tc = tx_ring->dcb_tc;
1015	if (xoff[tc])
1016	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED, addr: &tx_ring->state);
1017	}
1018
1019	for (i = 0; i < adapter->num_xdp_queues; i++) {
1020	struct ixgbe_ring *xdp_ring = adapter->xdp_ring[i];
1021
1022	tc = xdp_ring->dcb_tc;
1023	if (xoff[tc])
1024	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED, addr: &xdp_ring->state);
1025	}
1026	}
1027
1028	static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
1029	{
1030	return ring->stats.packets;
1031	}
1032
1033	static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
1034	{
1035	unsigned int head, tail;
1036
1037	head = ring->next_to_clean;
1038	tail = ring->next_to_use;
1039
1040	return ((head <= tail) ? tail : tail + ring->count) - head;
1041	}
1042
1043	static bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
1044	{
1045	u32 tx_done = ixgbe_get_tx_completed(ring: tx_ring);
1046	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
1047	u32 tx_pending = ixgbe_get_tx_pending(ring: tx_ring);
1048
1049	clear_check_for_tx_hang(tx_ring);
1050
1051	/*
1052	* Check for a hung queue, but be thorough. This verifies
1053	* that a transmit has been completed since the previous
1054	* check AND there is at least one packet pending. The
1055	* ARMED bit is set to indicate a potential hang. The
1056	* bit is cleared if a pause frame is received to remove
1057	* false hang detection due to PFC or 802.3x frames. By
1058	* requiring this to fail twice we avoid races with
1059	* pfc clearing the ARMED bit and conditions where we
1060	* run the check_tx_hang logic with a transmit completion
1061	* pending but without time to complete it yet.
1062	*/
1063	if (tx_done_old == tx_done && tx_pending)
1064	/* make sure it is true for two checks in a row */
1065	return test_and_set_bit(nr: __IXGBE_HANG_CHECK_ARMED,
1066	addr: &tx_ring->state);
1067	/* update completed stats and continue */
1068	tx_ring->tx_stats.tx_done_old = tx_done;
1069	/* reset the countdown */
1070	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED, addr: &tx_ring->state);
1071
1072	return false;
1073	}
1074
1075	/**
1076	* ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
1077	* @adapter: driver private struct
1078	**/
1079	static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
1080	{
1081
1082	/* Do the reset outside of interrupt context */
1083	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1084	set_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
1085	e_warn(drv, "initiating reset due to tx timeout\n");
1086	ixgbe_service_event_schedule(adapter);
1087	}
1088	}
1089
1090	/**
1091	* ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate
1092	* @netdev: network interface device structure
1093	* @queue_index: Tx queue to set
1094	* @maxrate: desired maximum transmit bitrate
1095	**/
1096	static int ixgbe_tx_maxrate(struct net_device *netdev,
1097	int queue_index, u32 maxrate)
1098	{
1099	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
1100	struct ixgbe_hw *hw = &adapter->hw;
1101	u32 bcnrc_val = ixgbe_link_mbps(adapter);
1102
1103	if (!maxrate)
1104	return 0;
1105
1106	/* Calculate the rate factor values to set */
1107	bcnrc_val <<= IXGBE_RTTBCNRC_RF_INT_SHIFT;
1108	bcnrc_val /= maxrate;
1109
1110	/* clear everything but the rate factor */
1111	bcnrc_val &= IXGBE_RTTBCNRC_RF_INT_MASK |
1112	IXGBE_RTTBCNRC_RF_DEC_MASK;
1113
1114	/* enable the rate scheduler */
1115	bcnrc_val |= IXGBE_RTTBCNRC_RS_ENA;
1116
1117	IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_index);
1118	IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
1119
1120	return 0;
1121	}
1122
1123	/**
1124	* ixgbe_update_tx_ring_stats - Update Tx ring specific counters
1125	* @tx_ring: ring to update
1126	* @q_vector: queue vector ring belongs to
1127	* @pkts: number of processed packets
1128	* @bytes: number of processed bytes
1129	*/
1130	void ixgbe_update_tx_ring_stats(struct ixgbe_ring *tx_ring,
1131	struct ixgbe_q_vector *q_vector, u64 pkts,
1132	u64 bytes)
1133	{
1134	u64_stats_update_begin(syncp: &tx_ring->syncp);
1135	tx_ring->stats.bytes += bytes;
1136	tx_ring->stats.packets += pkts;
1137	u64_stats_update_end(syncp: &tx_ring->syncp);
1138	q_vector->tx.total_bytes += bytes;
1139	q_vector->tx.total_packets += pkts;
1140	}
1141
1142	/**
1143	* ixgbe_update_rx_ring_stats - Update Rx ring specific counters
1144	* @rx_ring: ring to update
1145	* @q_vector: queue vector ring belongs to
1146	* @pkts: number of processed packets
1147	* @bytes: number of processed bytes
1148	*/
1149	void ixgbe_update_rx_ring_stats(struct ixgbe_ring *rx_ring,
1150	struct ixgbe_q_vector *q_vector, u64 pkts,
1151	u64 bytes)
1152	{
1153	u64_stats_update_begin(syncp: &rx_ring->syncp);
1154	rx_ring->stats.bytes += bytes;
1155	rx_ring->stats.packets += pkts;
1156	u64_stats_update_end(syncp: &rx_ring->syncp);
1157	q_vector->rx.total_bytes += bytes;
1158	q_vector->rx.total_packets += pkts;
1159	}
1160
1161	/**
1162	* ixgbe_clean_tx_irq - Reclaim resources after transmit completes
1163	* @q_vector: structure containing interrupt and ring information
1164	* @tx_ring: tx ring to clean
1165	* @napi_budget: Used to determine if we are in netpoll
1166	**/
1167	static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
1168	struct ixgbe_ring *tx_ring, int napi_budget)
1169	{
1170	struct ixgbe_adapter *adapter = q_vector->adapter;
1171	struct ixgbe_tx_buffer *tx_buffer;
1172	union ixgbe_adv_tx_desc *tx_desc;
1173	unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
1174	unsigned int budget = q_vector->tx.work_limit;
1175	unsigned int i = tx_ring->next_to_clean;
1176	struct netdev_queue *txq;
1177
1178	if (test_bit(__IXGBE_DOWN, &adapter->state))
1179	return true;
1180
1181	tx_buffer = &tx_ring->tx_buffer_info[i];
1182	tx_desc = IXGBE_TX_DESC(tx_ring, i);
1183	i -= tx_ring->count;
1184
1185	do {
1186	union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
1187
1188	/* if next_to_watch is not set then there is no work pending */
1189	if (!eop_desc)
1190	break;
1191
1192	/* prevent any other reads prior to eop_desc */
1193	smp_rmb();
1194
1195	/* if DD is not set pending work has not been completed */
1196	if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
1197	break;
1198
1199	/* clear next_to_watch to prevent false hangs */
1200	tx_buffer->next_to_watch = NULL;
1201
1202	/* update the statistics for this packet */
1203	total_bytes += tx_buffer->bytecount;
1204	total_packets += tx_buffer->gso_segs;
1205	if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
1206	total_ipsec++;
1207
1208	/* free the skb */
1209	if (ring_is_xdp(tx_ring))
1210	xdp_return_frame(xdpf: tx_buffer->xdpf);
1211	else
1212	napi_consume_skb(skb: tx_buffer->skb, budget: napi_budget);
1213
1214	/* unmap skb header data */
1215	dma_unmap_single(tx_ring->dev,
1216	dma_unmap_addr(tx_buffer, dma),
1217	dma_unmap_len(tx_buffer, len),
1218	DMA_TO_DEVICE);
1219
1220	/* clear tx_buffer data */
1221	dma_unmap_len_set(tx_buffer, len, 0);
1222
1223	/* unmap remaining buffers */
1224	while (tx_desc != eop_desc) {
1225	tx_buffer++;
1226	tx_desc++;
1227	i++;
1228	if (unlikely(!i)) {
1229	i -= tx_ring->count;
1230	tx_buffer = tx_ring->tx_buffer_info;
1231	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
1232	}
1233
1234	/* unmap any remaining paged data */
1235	if (dma_unmap_len(tx_buffer, len)) {
1236	dma_unmap_page(tx_ring->dev,
1237	dma_unmap_addr(tx_buffer, dma),
1238	dma_unmap_len(tx_buffer, len),
1239	DMA_TO_DEVICE);
1240	dma_unmap_len_set(tx_buffer, len, 0);
1241	}
1242	}
1243
1244	/* move us one more past the eop_desc for start of next pkt */
1245	tx_buffer++;
1246	tx_desc++;
1247	i++;
1248	if (unlikely(!i)) {
1249	i -= tx_ring->count;
1250	tx_buffer = tx_ring->tx_buffer_info;
1251	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
1252	}
1253
1254	/* issue prefetch for next Tx descriptor */
1255	prefetch(tx_desc);
1256
1257	/* update budget accounting */
1258	budget--;
1259	} while (likely(budget));
1260
1261	i += tx_ring->count;
1262	tx_ring->next_to_clean = i;
1263	ixgbe_update_tx_ring_stats(tx_ring, q_vector, pkts: total_packets,
1264	bytes: total_bytes);
1265	adapter->tx_ipsec += total_ipsec;
1266
1267	if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
1268	/* schedule immediate reset if we believe we hung */
1269	struct ixgbe_hw *hw = &adapter->hw;
1270	e_err(drv, "Detected Tx Unit Hang %s\n"
1271	" Tx Queue <%d>\n"
1272	" TDH, TDT <%x>, <%x>\n"
1273	" next_to_use <%x>\n"
1274	" next_to_clean <%x>\n"
1275	"tx_buffer_info[next_to_clean]\n"
1276	" time_stamp <%lx>\n"
1277	" jiffies <%lx>\n",
1278	ring_is_xdp(tx_ring) ? "(XDP)" : "",
1279	tx_ring->queue_index,
1280	IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
1281	IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
1282	tx_ring->next_to_use, i,
1283	tx_ring->tx_buffer_info[i].time_stamp, jiffies);
1284
1285	if (!ring_is_xdp(tx_ring))
1286	netif_stop_subqueue(dev: tx_ring->netdev,
1287	queue_index: tx_ring->queue_index);
1288
1289	e_info(probe,
1290	"tx hang %d detected on queue %d, resetting adapter\n",
1291	adapter->tx_timeout_count + 1, tx_ring->queue_index);
1292
1293	/* schedule immediate reset if we believe we hung */
1294	ixgbe_tx_timeout_reset(adapter);
1295
1296	/* the adapter is about to reset, no point in enabling stuff */
1297	return true;
1298	}
1299
1300	if (ring_is_xdp(tx_ring))
1301	return !!budget;
1302
1303	#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
1304	txq = netdev_get_tx_queue(dev: tx_ring->netdev, index: tx_ring->queue_index);
1305	if (!__netif_txq_completed_wake(txq, total_packets, total_bytes,
1306	ixgbe_desc_unused(tx_ring),
1307	TX_WAKE_THRESHOLD,
1308	!netif_carrier_ok(tx_ring->netdev) ||
1309	test_bit(__IXGBE_DOWN, &adapter->state)))
1310	++tx_ring->tx_stats.restart_queue;
1311
1312	return !!budget;
1313	}
1314
1315	#ifdef CONFIG_IXGBE_DCA
1316	static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
1317	struct ixgbe_ring *tx_ring,
1318	int cpu)
1319	{
1320	struct ixgbe_hw *hw = &adapter->hw;
1321	u32 txctrl = 0;
1322	u16 reg_offset;
1323
1324	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1325	txctrl = dca3_get_tag(dev: tx_ring->dev, cpu);
1326
1327	switch (hw->mac.type) {
1328	case ixgbe_mac_82598EB:
1329	reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
1330	break;
1331	case ixgbe_mac_82599EB:
1332	case ixgbe_mac_X540:
1333	reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
1334	txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;
1335	break;
1336	default:
1337	/* for unknown hardware do not write register */
1338	return;
1339	}
1340
1341	/*
1342	* We can enable relaxed ordering for reads, but not writes when
1343	* DCA is enabled. This is due to a known issue in some chipsets
1344	* which will cause the DCA tag to be cleared.
1345	*/
1346	txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1347	IXGBE_DCA_TXCTRL_DATA_RRO_EN |
1348	IXGBE_DCA_TXCTRL_DESC_DCA_EN;
1349
1350	IXGBE_WRITE_REG(hw, reg_offset, txctrl);
1351	}
1352
1353	static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
1354	struct ixgbe_ring *rx_ring,
1355	int cpu)
1356	{
1357	struct ixgbe_hw *hw = &adapter->hw;
1358	u32 rxctrl = 0;
1359	u8 reg_idx = rx_ring->reg_idx;
1360
1361	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1362	rxctrl = dca3_get_tag(dev: rx_ring->dev, cpu);
1363
1364	switch (hw->mac.type) {
1365	case ixgbe_mac_82599EB:
1366	case ixgbe_mac_X540:
1367	rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;
1368	break;
1369	default:
1370	break;
1371	}
1372
1373	/*
1374	* We can enable relaxed ordering for reads, but not writes when
1375	* DCA is enabled. This is due to a known issue in some chipsets
1376	* which will cause the DCA tag to be cleared.
1377	*/
1378	rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
1379	IXGBE_DCA_RXCTRL_DATA_DCA_EN |
1380	IXGBE_DCA_RXCTRL_DESC_DCA_EN;
1381
1382	IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
1383	}
1384
1385	static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
1386	{
1387	struct ixgbe_adapter *adapter = q_vector->adapter;
1388	struct ixgbe_ring *ring;
1389	int cpu = get_cpu();
1390
1391	if (q_vector->cpu == cpu)
1392	goto out_no_update;
1393
1394	ixgbe_for_each_ring(ring, q_vector->tx)
1395	ixgbe_update_tx_dca(adapter, tx_ring: ring, cpu);
1396
1397	ixgbe_for_each_ring(ring, q_vector->rx)
1398	ixgbe_update_rx_dca(adapter, rx_ring: ring, cpu);
1399
1400	q_vector->cpu = cpu;
1401	out_no_update:
1402	put_cpu();
1403	}
1404
1405	static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
1406	{
1407	int i;
1408
1409	/* always use CB2 mode, difference is masked in the CB driver */
1410	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1411	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1412	IXGBE_DCA_CTRL_DCA_MODE_CB2);
1413	else
1414	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1415	IXGBE_DCA_CTRL_DCA_DISABLE);
1416
1417	for (i = 0; i < adapter->num_q_vectors; i++) {
1418	adapter->q_vector[i]->cpu = -1;
1419	ixgbe_update_dca(q_vector: adapter->q_vector[i]);
1420	}
1421	}
1422
1423	static int __ixgbe_notify_dca(struct device *dev, void *data)
1424	{
1425	struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
1426	unsigned long event = *(unsigned long *)data;
1427
1428	if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
1429	return 0;
1430
1431	switch (event) {
1432	case DCA_PROVIDER_ADD:
1433	/* if we're already enabled, don't do it again */
1434	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1435	break;
1436	if (dca_add_requester(dev) == 0) {
1437	adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
1438	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1439	IXGBE_DCA_CTRL_DCA_MODE_CB2);
1440	break;
1441	}
1442	fallthrough; /* DCA is disabled. */
1443	case DCA_PROVIDER_REMOVE:
1444	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
1445	dca_remove_requester(dev);
1446	adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
1447	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1448	IXGBE_DCA_CTRL_DCA_DISABLE);
1449	}
1450	break;
1451	}
1452
1453	return 0;
1454	}
1455
1456	#endif /* CONFIG_IXGBE_DCA */
1457
1458	#define IXGBE_RSS_L4_TYPES_MASK \
1459	((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \
1460	(1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \
1461	(1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \
1462	(1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))
1463
1464	static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
1465	union ixgbe_adv_rx_desc *rx_desc,
1466	struct sk_buff *skb)
1467	{
1468	u16 rss_type;
1469
1470	if (!(ring->netdev->features & NETIF_F_RXHASH))
1471	return;
1472
1473	rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
1474	IXGBE_RXDADV_RSSTYPE_MASK;
1475
1476	if (!rss_type)
1477	return;
1478
1479	skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1480	type: (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
1481	PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
1482	}
1483
1484	#ifdef IXGBE_FCOE
1485	/**
1486	* ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
1487	* @ring: structure containing ring specific data
1488	* @rx_desc: advanced rx descriptor
1489	*
1490	* Returns : true if it is FCoE pkt
1491	*/
1492	static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
1493	union ixgbe_adv_rx_desc *rx_desc)
1494	{
1495	__le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1496
1497	return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
1498	((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1499	(cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1500	IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1501	}
1502
1503	#endif /* IXGBE_FCOE */
1504	/**
1505	* ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1506	* @ring: structure containing ring specific data
1507	* @rx_desc: current Rx descriptor being processed
1508	* @skb: skb currently being received and modified
1509	**/
1510	static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
1511	union ixgbe_adv_rx_desc *rx_desc,
1512	struct sk_buff *skb)
1513	{
1514	__le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1515	bool encap_pkt = false;
1516
1517	skb_checksum_none_assert(skb);
1518
1519	/* Rx csum disabled */
1520	if (!(ring->netdev->features & NETIF_F_RXCSUM))
1521	return;
1522
1523	/* check for VXLAN and Geneve packets */
1524	if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) {
1525	encap_pkt = true;
1526	skb->encapsulation = 1;
1527	}
1528
1529	/* if IP and error */
1530	if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
1531	ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
1532	ring->rx_stats.csum_err++;
1533	return;
1534	}
1535
1536	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
1537	return;
1538
1539	if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
1540	/*
1541	* 82599 errata, UDP frames with a 0 checksum can be marked as
1542	* checksum errors.
1543	*/
1544	if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
1545	test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
1546	return;
1547
1548	ring->rx_stats.csum_err++;
1549	return;
1550	}
1551
1552	/* It must be a TCP or UDP packet with a valid checksum */
1553	skb->ip_summed = CHECKSUM_UNNECESSARY;
1554	if (encap_pkt) {
1555	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_OUTERIPCS))
1556	return;
1557
1558	if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_OUTERIPER)) {
1559	skb->ip_summed = CHECKSUM_NONE;
1560	return;
1561	}
1562	/* If we checked the outer header let the stack know */
1563	skb->csum_level = 1;
1564	}
1565	}
1566
1567	static unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring)
1568	{
1569	return ring_uses_build_skb(rx_ring) ? IXGBE_SKB_PAD : 0;
1570	}
1571
1572	static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
1573	struct ixgbe_rx_buffer *bi)
1574	{
1575	struct page *page = bi->page;
1576	dma_addr_t dma;
1577
1578	/* since we are recycling buffers we should seldom need to alloc */
1579	if (likely(page))
1580	return true;
1581
1582	/* alloc new page for storage */
1583	page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring));
1584	if (unlikely(!page)) {
1585	rx_ring->rx_stats.alloc_rx_page_failed++;
1586	return false;
1587	}
1588
1589	/* map page for use */
1590	dma = dma_map_page_attrs(dev: rx_ring->dev, page, offset: 0,
1591	ixgbe_rx_pg_size(rx_ring),
1592	dir: DMA_FROM_DEVICE,
1593	IXGBE_RX_DMA_ATTR);
1594
1595	/*
1596	* if mapping failed free memory back to system since
1597	* there isn't much point in holding memory we can't use
1598	*/
1599	if (dma_mapping_error(dev: rx_ring->dev, dma_addr: dma)) {
1600	__free_pages(page, order: ixgbe_rx_pg_order(ring: rx_ring));
1601
1602	rx_ring->rx_stats.alloc_rx_page_failed++;
1603	return false;
1604	}
1605
1606	bi->dma = dma;
1607	bi->page = page;
1608	bi->page_offset = rx_ring->rx_offset;
1609	page_ref_add(page, USHRT_MAX - 1);
1610	bi->pagecnt_bias = USHRT_MAX;
1611	rx_ring->rx_stats.alloc_rx_page++;
1612
1613	return true;
1614	}
1615
1616	/**
1617	* ixgbe_alloc_rx_buffers - Replace used receive buffers
1618	* @rx_ring: ring to place buffers on
1619	* @cleaned_count: number of buffers to replace
1620	**/
1621	void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1622	{
1623	union ixgbe_adv_rx_desc *rx_desc;
1624	struct ixgbe_rx_buffer *bi;
1625	u16 i = rx_ring->next_to_use;
1626	u16 bufsz;
1627
1628	/* nothing to do */
1629	if (!cleaned_count)
1630	return;
1631
1632	rx_desc = IXGBE_RX_DESC(rx_ring, i);
1633	bi = &rx_ring->rx_buffer_info[i];
1634	i -= rx_ring->count;
1635
1636	bufsz = ixgbe_rx_bufsz(ring: rx_ring);
1637
1638	do {
1639	if (!ixgbe_alloc_mapped_page(rx_ring, bi))
1640	break;
1641
1642	/* sync the buffer for use by the device */
1643	dma_sync_single_range_for_device(dev: rx_ring->dev, addr: bi->dma,
1644	offset: bi->page_offset, size: bufsz,
1645	dir: DMA_FROM_DEVICE);
1646
1647	/*
1648	* Refresh the desc even if buffer_addrs didn't change
1649	* because each write-back erases this info.
1650	*/
1651	rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1652
1653	rx_desc++;
1654	bi++;
1655	i++;
1656	if (unlikely(!i)) {
1657	rx_desc = IXGBE_RX_DESC(rx_ring, 0);
1658	bi = rx_ring->rx_buffer_info;
1659	i -= rx_ring->count;
1660	}
1661
1662	/* clear the length for the next_to_use descriptor */
1663	rx_desc->wb.upper.length = 0;
1664
1665	cleaned_count--;
1666	} while (cleaned_count);
1667
1668	i += rx_ring->count;
1669
1670	if (rx_ring->next_to_use != i) {
1671	rx_ring->next_to_use = i;
1672
1673	/* update next to alloc since we have filled the ring */
1674	rx_ring->next_to_alloc = i;
1675
1676	/* Force memory writes to complete before letting h/w
1677	* know there are new descriptors to fetch. (Only
1678	* applicable for weak-ordered memory model archs,
1679	* such as IA-64).
1680	*/
1681	wmb();
1682	writel(val: i, addr: rx_ring->tail);
1683	}
1684	}
1685
1686	static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
1687	struct sk_buff *skb)
1688	{
1689	u16 hdr_len = skb_headlen(skb);
1690
1691	/* set gso_size to avoid messing up TCP MSS */
1692	skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
1693	IXGBE_CB(skb)->append_cnt);
1694	skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1695	}
1696
1697	static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
1698	struct sk_buff *skb)
1699	{
1700	/* if append_cnt is 0 then frame is not RSC */
1701	if (!IXGBE_CB(skb)->append_cnt)
1702	return;
1703
1704	rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
1705	rx_ring->rx_stats.rsc_flush++;
1706
1707	ixgbe_set_rsc_gso_size(ring: rx_ring, skb);
1708
1709	/* gso_size is computed using append_cnt so always clear it last */
1710	IXGBE_CB(skb)->append_cnt = 0;
1711	}
1712
1713	/**
1714	* ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor
1715	* @rx_ring: rx descriptor ring packet is being transacted on
1716	* @rx_desc: pointer to the EOP Rx descriptor
1717	* @skb: pointer to current skb being populated
1718	*
1719	* This function checks the ring, descriptor, and packet information in
1720	* order to populate the hash, checksum, VLAN, timestamp, protocol, and
1721	* other fields within the skb.
1722	**/
1723	void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
1724	union ixgbe_adv_rx_desc *rx_desc,
1725	struct sk_buff *skb)
1726	{
1727	struct net_device *dev = rx_ring->netdev;
1728	u32 flags = rx_ring->q_vector->adapter->flags;
1729
1730	ixgbe_update_rsc_stats(rx_ring, skb);
1731
1732	ixgbe_rx_hash(ring: rx_ring, rx_desc, skb);
1733
1734	ixgbe_rx_checksum(ring: rx_ring, rx_desc, skb);
1735
1736	if (unlikely(flags & IXGBE_FLAG_RX_HWTSTAMP_ENABLED))
1737	ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb);
1738
1739	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1740	ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
1741	u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1742	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci: vid);
1743	}
1744
1745	if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))
1746	ixgbe_ipsec_rx(rx_ring, rx_desc, skb);
1747
1748	/* record Rx queue, or update MACVLAN statistics */
1749	if (netif_is_ixgbe(dev))
1750	skb_record_rx_queue(skb, rx_queue: rx_ring->queue_index);
1751	else
1752	macvlan_count_rx(vlan: netdev_priv(dev), len: skb->len + ETH_HLEN, success: true,
1753	multicast: false);
1754
1755	skb->protocol = eth_type_trans(skb, dev);
1756	}
1757
1758	void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
1759	struct sk_buff *skb)
1760	{
1761	napi_gro_receive(napi: &q_vector->napi, skb);
1762	}
1763
1764	/**
1765	* ixgbe_is_non_eop - process handling of non-EOP buffers
1766	* @rx_ring: Rx ring being processed
1767	* @rx_desc: Rx descriptor for current buffer
1768	* @skb: Current socket buffer containing buffer in progress
1769	*
1770	* This function updates next to clean. If the buffer is an EOP buffer
1771	* this function exits returning false, otherwise it will place the
1772	* sk_buff in the next buffer to be chained and return true indicating
1773	* that this is in fact a non-EOP buffer.
1774	**/
1775	static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
1776	union ixgbe_adv_rx_desc *rx_desc,
1777	struct sk_buff *skb)
1778	{
1779	u32 ntc = rx_ring->next_to_clean + 1;
1780
1781	/* fetch, update, and store next to clean */
1782	ntc = (ntc < rx_ring->count) ? ntc : 0;
1783	rx_ring->next_to_clean = ntc;
1784
1785	prefetch(IXGBE_RX_DESC(rx_ring, ntc));
1786
1787	/* update RSC append count if present */
1788	if (ring_is_rsc_enabled(rx_ring)) {
1789	__le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
1790	cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);
1791
1792	if (unlikely(rsc_enabled)) {
1793	u32 rsc_cnt = le32_to_cpu(rsc_enabled);
1794
1795	rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
1796	IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;
1797
1798	/* update ntc based on RSC value */
1799	ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
1800	ntc &= IXGBE_RXDADV_NEXTP_MASK;
1801	ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
1802	}
1803	}
1804
1805	/* if we are the last buffer then there is nothing else to do */
1806	if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
1807	return false;
1808
1809	/* place skb in next buffer to be received */
1810	rx_ring->rx_buffer_info[ntc].skb = skb;
1811	rx_ring->rx_stats.non_eop_descs++;
1812
1813	return true;
1814	}
1815
1816	/**
1817	* ixgbe_pull_tail - ixgbe specific version of skb_pull_tail
1818	* @rx_ring: rx descriptor ring packet is being transacted on
1819	* @skb: pointer to current skb being adjusted
1820	*
1821	* This function is an ixgbe specific version of __pskb_pull_tail. The
1822	* main difference between this version and the original function is that
1823	* this function can make several assumptions about the state of things
1824	* that allow for significant optimizations versus the standard function.
1825	* As a result we can do things like drop a frag and maintain an accurate
1826	* truesize for the skb.
1827	*/
1828	static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
1829	struct sk_buff *skb)
1830	{
1831	skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
1832	unsigned char *va;
1833	unsigned int pull_len;
1834
1835	/*
1836	* it is valid to use page_address instead of kmap since we are
1837	* working with pages allocated out of the lomem pool per
1838	* alloc_page(GFP_ATOMIC)
1839	*/
1840	va = skb_frag_address(frag);
1841
1842	/*
1843	* we need the header to contain the greater of either ETH_HLEN or
1844	* 60 bytes if the skb->len is less than 60 for skb_pad.
1845	*/
1846	pull_len = eth_get_headlen(dev: skb->dev, data: va, IXGBE_RX_HDR_SIZE);
1847
1848	/* align pull length to size of long to optimize memcpy performance */
1849	skb_copy_to_linear_data(skb, from: va, ALIGN(pull_len, sizeof(long)));
1850
1851	/* update all of the pointers */
1852	skb_frag_size_sub(frag, delta: pull_len);
1853	skb_frag_off_add(frag, delta: pull_len);
1854	skb->data_len -= pull_len;
1855	skb->tail += pull_len;
1856	}
1857
1858	/**
1859	* ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB
1860	* @rx_ring: rx descriptor ring packet is being transacted on
1861	* @skb: pointer to current skb being updated
1862	*
1863	* This function provides a basic DMA sync up for the first fragment of an
1864	* skb. The reason for doing this is that the first fragment cannot be
1865	* unmapped until we have reached the end of packet descriptor for a buffer
1866	* chain.
1867	*/
1868	static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
1869	struct sk_buff *skb)
1870	{
1871	if (ring_uses_build_skb(rx_ring)) {
1872	unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1;
1873	unsigned long offset = (unsigned long)(skb->data) & mask;
1874
1875	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
1876	IXGBE_CB(skb)->dma,
1877	offset,
1878	size: skb_headlen(skb),
1879	dir: DMA_FROM_DEVICE);
1880	} else {
1881	skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
1882
1883	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
1884	IXGBE_CB(skb)->dma,
1885	offset: skb_frag_off(frag),
1886	size: skb_frag_size(frag),
1887	dir: DMA_FROM_DEVICE);
1888	}
1889
1890	/* If the page was released, just unmap it. */
1891	if (unlikely(IXGBE_CB(skb)->page_released)) {
1892	dma_unmap_page_attrs(dev: rx_ring->dev, IXGBE_CB(skb)->dma,
1893	ixgbe_rx_pg_size(rx_ring),
1894	dir: DMA_FROM_DEVICE,
1895	IXGBE_RX_DMA_ATTR);
1896	}
1897	}
1898
1899	/**
1900	* ixgbe_cleanup_headers - Correct corrupted or empty headers
1901	* @rx_ring: rx descriptor ring packet is being transacted on
1902	* @rx_desc: pointer to the EOP Rx descriptor
1903	* @skb: pointer to current skb being fixed
1904	*
1905	* Check if the skb is valid in the XDP case it will be an error pointer.
1906	* Return true in this case to abort processing and advance to next
1907	* descriptor.
1908	*
1909	* Check for corrupted packet headers caused by senders on the local L2
1910	* embedded NIC switch not setting up their Tx Descriptors right. These
1911	* should be very rare.
1912	*
1913	* Also address the case where we are pulling data in on pages only
1914	* and as such no data is present in the skb header.
1915	*
1916	* In addition if skb is not at least 60 bytes we need to pad it so that
1917	* it is large enough to qualify as a valid Ethernet frame.
1918	*
1919	* Returns true if an error was encountered and skb was freed.
1920	**/
1921	bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
1922	union ixgbe_adv_rx_desc *rx_desc,
1923	struct sk_buff *skb)
1924	{
1925	struct net_device *netdev = rx_ring->netdev;
1926
1927	/* Verify netdev is present, and that packet does not have any
1928	* errors that would be unacceptable to the netdev.
1929	*/
1930	if (!netdev ||
1931	(unlikely(ixgbe_test_staterr(rx_desc,
1932	IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
1933	!(netdev->features & NETIF_F_RXALL)))) {
1934	dev_kfree_skb_any(skb);
1935	return true;
1936	}
1937
1938	/* place header in linear portion of buffer */
1939	if (!skb_headlen(skb))
1940	ixgbe_pull_tail(rx_ring, skb);
1941
1942	#ifdef IXGBE_FCOE
1943	/* do not attempt to pad FCoE Frames as this will disrupt DDP */
1944	if (ixgbe_rx_is_fcoe(ring: rx_ring, rx_desc))
1945	return false;
1946
1947	#endif
1948	/* if eth_skb_pad returns an error the skb was freed */
1949	if (eth_skb_pad(skb))
1950	return true;
1951
1952	return false;
1953	}
1954
1955	/**
1956	* ixgbe_reuse_rx_page - page flip buffer and store it back on the ring
1957	* @rx_ring: rx descriptor ring to store buffers on
1958	* @old_buff: donor buffer to have page reused
1959	*
1960	* Synchronizes page for reuse by the adapter
1961	**/
1962	static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
1963	struct ixgbe_rx_buffer *old_buff)
1964	{
1965	struct ixgbe_rx_buffer *new_buff;
1966	u16 nta = rx_ring->next_to_alloc;
1967
1968	new_buff = &rx_ring->rx_buffer_info[nta];
1969
1970	/* update, and store next to alloc */
1971	nta++;
1972	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1973
1974	/* Transfer page from old buffer to new buffer.
1975	* Move each member individually to avoid possible store
1976	* forwarding stalls and unnecessary copy of skb.
1977	*/
1978	new_buff->dma = old_buff->dma;
1979	new_buff->page = old_buff->page;
1980	new_buff->page_offset = old_buff->page_offset;
1981	new_buff->pagecnt_bias = old_buff->pagecnt_bias;
1982	}
1983
1984	static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer,
1985	int rx_buffer_pgcnt)
1986	{
1987	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1988	struct page *page = rx_buffer->page;
1989
1990	/* avoid re-using remote and pfmemalloc pages */
1991	if (!dev_page_is_reusable(page))
1992	return false;
1993
1994	#if (PAGE_SIZE < 8192)
1995	/* if we are only owner of page we can reuse it */
1996	if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))
1997	return false;
1998	#else
1999	/* The last offset is a bit aggressive in that we assume the
2000	* worst case of FCoE being enabled and using a 3K buffer.
2001	* However this should have minimal impact as the 1K extra is
2002	* still less than one buffer in size.
2003	*/
2004	#define IXGBE_LAST_OFFSET \
2005	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K)
2006	if (rx_buffer->page_offset > IXGBE_LAST_OFFSET)
2007	return false;
2008	#endif
2009
2010	/* If we have drained the page fragment pool we need to update
2011	* the pagecnt_bias and page count so that we fully restock the
2012	* number of references the driver holds.
2013	*/
2014	if (unlikely(pagecnt_bias == 1)) {
2015	page_ref_add(page, USHRT_MAX - 1);
2016	rx_buffer->pagecnt_bias = USHRT_MAX;
2017	}
2018
2019	return true;
2020	}
2021
2022	/**
2023	* ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff
2024	* @rx_ring: rx descriptor ring to transact packets on
2025	* @rx_buffer: buffer containing page to add
2026	* @skb: sk_buff to place the data into
2027	* @size: size of data in rx_buffer
2028	*
2029	* This function will add the data contained in rx_buffer->page to the skb.
2030	* This is done either through a direct copy if the data in the buffer is
2031	* less than the skb header size, otherwise it will just attach the page as
2032	* a frag to the skb.
2033	*
2034	* The function will then update the page offset if necessary and return
2035	* true if the buffer can be reused by the adapter.
2036	**/
2037	static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
2038	struct ixgbe_rx_buffer *rx_buffer,
2039	struct sk_buff *skb,
2040	unsigned int size)
2041	{
2042	#if (PAGE_SIZE < 8192)
2043	unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
2044	#else
2045	unsigned int truesize = rx_ring->rx_offset ?
2046	SKB_DATA_ALIGN(rx_ring->rx_offset + size) :
2047	SKB_DATA_ALIGN(size);
2048	#endif
2049	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page: rx_buffer->page,
2050	off: rx_buffer->page_offset, size, truesize);
2051	#if (PAGE_SIZE < 8192)
2052	rx_buffer->page_offset ^= truesize;
2053	#else
2054	rx_buffer->page_offset += truesize;
2055	#endif
2056	}
2057
2058	static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring,
2059	union ixgbe_adv_rx_desc *rx_desc,
2060	struct sk_buff **skb,
2061	const unsigned int size,
2062	int *rx_buffer_pgcnt)
2063	{
2064	struct ixgbe_rx_buffer *rx_buffer;
2065
2066	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
2067	*rx_buffer_pgcnt =
2068	#if (PAGE_SIZE < 8192)
2069	page_count(page: rx_buffer->page);
2070	#else
2071	0;
2072	#endif
2073	prefetchw(x: rx_buffer->page);
2074	*skb = rx_buffer->skb;
2075
2076	/* Delay unmapping of the first packet. It carries the header
2077	* information, HW may still access the header after the writeback.
2078	* Only unmap it when EOP is reached
2079	*/
2080	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) {
2081	if (!*skb)
2082	goto skip_sync;
2083	} else {
2084	if (*skb)
2085	ixgbe_dma_sync_frag(rx_ring, skb: *skb);
2086	}
2087
2088	/* we are reusing so sync this buffer for CPU use */
2089	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
2090	addr: rx_buffer->dma,
2091	offset: rx_buffer->page_offset,
2092	size,
2093	dir: DMA_FROM_DEVICE);
2094	skip_sync:
2095	rx_buffer->pagecnt_bias--;
2096
2097	return rx_buffer;
2098	}
2099
2100	static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring,
2101	struct ixgbe_rx_buffer *rx_buffer,
2102	struct sk_buff *skb,
2103	int rx_buffer_pgcnt)
2104	{
2105	if (ixgbe_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {
2106	/* hand second half of page back to the ring */
2107	ixgbe_reuse_rx_page(rx_ring, old_buff: rx_buffer);
2108	} else {
2109	if (skb && IXGBE_CB(skb)->dma == rx_buffer->dma) {
2110	/* the page has been released from the ring */
2111	IXGBE_CB(skb)->page_released = true;
2112	} else {
2113	/* we are not reusing the buffer so unmap it */
2114	dma_unmap_page_attrs(dev: rx_ring->dev, addr: rx_buffer->dma,
2115	ixgbe_rx_pg_size(rx_ring),
2116	dir: DMA_FROM_DEVICE,
2117	IXGBE_RX_DMA_ATTR);
2118	}
2119	__page_frag_cache_drain(page: rx_buffer->page,
2120	count: rx_buffer->pagecnt_bias);
2121	}
2122
2123	/* clear contents of rx_buffer */
2124	rx_buffer->page = NULL;
2125	rx_buffer->skb = NULL;
2126	}
2127
2128	static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring,
2129	struct ixgbe_rx_buffer *rx_buffer,
2130	struct xdp_buff *xdp,
2131	union ixgbe_adv_rx_desc *rx_desc)
2132	{
2133	unsigned int size = xdp->data_end - xdp->data;
2134	#if (PAGE_SIZE < 8192)
2135	unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
2136	#else
2137	unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
2138	xdp->data_hard_start);
2139	#endif
2140	struct sk_buff *skb;
2141
2142	/* prefetch first cache line of first page */
2143	net_prefetch(p: xdp->data);
2144
2145	/* Note, we get here by enabling legacy-rx via:
2146	*
2147	* ethtool --set-priv-flags <dev> legacy-rx on
2148	*
2149	* In this mode, we currently get 0 extra XDP headroom as
2150	* opposed to having legacy-rx off, where we process XDP
2151	* packets going to stack via ixgbe_build_skb(). The latter
2152	* provides us currently with 192 bytes of headroom.
2153	*
2154	* For ixgbe_construct_skb() mode it means that the
2155	* xdp->data_meta will always point to xdp->data, since
2156	* the helper cannot expand the head. Should this ever
2157	* change in future for legacy-rx mode on, then lets also
2158	* add xdp->data_meta handling here.
2159	*/
2160
2161	/* allocate a skb to store the frags */
2162	skb = napi_alloc_skb(napi: &rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE);
2163	if (unlikely(!skb))
2164	return NULL;
2165
2166	if (size > IXGBE_RX_HDR_SIZE) {
2167	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
2168	IXGBE_CB(skb)->dma = rx_buffer->dma;
2169
2170	skb_add_rx_frag(skb, i: 0, page: rx_buffer->page,
2171	off: xdp->data - page_address(rx_buffer->page),
2172	size, truesize);
2173	#if (PAGE_SIZE < 8192)
2174	rx_buffer->page_offset ^= truesize;
2175	#else
2176	rx_buffer->page_offset += truesize;
2177	#endif
2178	} else {
2179	memcpy(__skb_put(skb, size),
2180	xdp->data, ALIGN(size, sizeof(long)));
2181	rx_buffer->pagecnt_bias++;
2182	}
2183
2184	return skb;
2185	}
2186
2187	static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring,
2188	struct ixgbe_rx_buffer *rx_buffer,
2189	struct xdp_buff *xdp,
2190	union ixgbe_adv_rx_desc *rx_desc)
2191	{
2192	unsigned int metasize = xdp->data - xdp->data_meta;
2193	#if (PAGE_SIZE < 8192)
2194	unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
2195	#else
2196	unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
2197	SKB_DATA_ALIGN(xdp->data_end -
2198	xdp->data_hard_start);
2199	#endif
2200	struct sk_buff *skb;
2201
2202	/* Prefetch first cache line of first page. If xdp->data_meta
2203	* is unused, this points extactly as xdp->data, otherwise we
2204	* likely have a consumer accessing first few bytes of meta
2205	* data, and then actual data.
2206	*/
2207	net_prefetch(p: xdp->data_meta);
2208
2209	/* build an skb to around the page buffer */
2210	skb = napi_build_skb(data: xdp->data_hard_start, frag_size: truesize);
2211	if (unlikely(!skb))
2212	return NULL;
2213
2214	/* update pointers within the skb to store the data */
2215	skb_reserve(skb, len: xdp->data - xdp->data_hard_start);
2216	__skb_put(skb, len: xdp->data_end - xdp->data);
2217	if (metasize)
2218	skb_metadata_set(skb, meta_len: metasize);
2219
2220	/* record DMA address if this is the start of a chain of buffers */
2221	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
2222	IXGBE_CB(skb)->dma = rx_buffer->dma;
2223
2224	/* update buffer offset */
2225	#if (PAGE_SIZE < 8192)
2226	rx_buffer->page_offset ^= truesize;
2227	#else
2228	rx_buffer->page_offset += truesize;
2229	#endif
2230
2231	return skb;
2232	}
2233
2234	static int ixgbe_run_xdp(struct ixgbe_adapter *adapter,
2235	struct ixgbe_ring *rx_ring,
2236	struct xdp_buff *xdp)
2237	{
2238	int err, result = IXGBE_XDP_PASS;
2239	struct bpf_prog *xdp_prog;
2240	struct ixgbe_ring *ring;
2241	struct xdp_frame *xdpf;
2242	u32 act;
2243
2244	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
2245
2246	if (!xdp_prog)
2247	goto xdp_out;
2248
2249	prefetchw(x: xdp->data_hard_start); /* xdp_frame write */
2250
2251	act = bpf_prog_run_xdp(prog: xdp_prog, xdp);
2252	switch (act) {
2253	case XDP_PASS:
2254	break;
2255	case XDP_TX:
2256	xdpf = xdp_convert_buff_to_frame(xdp);
2257	if (unlikely(!xdpf))
2258	goto out_failure;
2259	ring = ixgbe_determine_xdp_ring(adapter);
2260	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
2261	spin_lock(lock: &ring->tx_lock);
2262	result = ixgbe_xmit_xdp_ring(ring, xdpf);
2263	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
2264	spin_unlock(lock: &ring->tx_lock);
2265	if (result == IXGBE_XDP_CONSUMED)
2266	goto out_failure;
2267	break;
2268	case XDP_REDIRECT:
2269	err = xdp_do_redirect(dev: adapter->netdev, xdp, prog: xdp_prog);
2270	if (err)
2271	goto out_failure;
2272	result = IXGBE_XDP_REDIR;
2273	break;
2274	default:
2275	bpf_warn_invalid_xdp_action(dev: rx_ring->netdev, prog: xdp_prog, act);
2276	fallthrough;
2277	case XDP_ABORTED:
2278	out_failure:
2279	trace_xdp_exception(dev: rx_ring->netdev, xdp: xdp_prog, act);
2280	fallthrough; /* handle aborts by dropping packet */
2281	case XDP_DROP:
2282	result = IXGBE_XDP_CONSUMED;
2283	break;
2284	}
2285	xdp_out:
2286	return result;
2287	}
2288
2289	static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring,
2290	unsigned int size)
2291	{
2292	unsigned int truesize;
2293
2294	#if (PAGE_SIZE < 8192)
2295	truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
2296	#else
2297	truesize = rx_ring->rx_offset ?
2298	SKB_DATA_ALIGN(rx_ring->rx_offset + size) +
2299	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
2300	SKB_DATA_ALIGN(size);
2301	#endif
2302	return truesize;
2303	}
2304
2305	static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring,
2306	struct ixgbe_rx_buffer *rx_buffer,
2307	unsigned int size)
2308	{
2309	unsigned int truesize = ixgbe_rx_frame_truesize(rx_ring, size);
2310	#if (PAGE_SIZE < 8192)
2311	rx_buffer->page_offset ^= truesize;
2312	#else
2313	rx_buffer->page_offset += truesize;
2314	#endif
2315	}
2316
2317	/**
2318	* ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
2319	* @q_vector: structure containing interrupt and ring information
2320	* @rx_ring: rx descriptor ring to transact packets on
2321	* @budget: Total limit on number of packets to process
2322	*
2323	* This function provides a "bounce buffer" approach to Rx interrupt
2324	* processing. The advantage to this is that on systems that have
2325	* expensive overhead for IOMMU access this provides a means of avoiding
2326	* it by maintaining the mapping of the page to the syste.
2327	*
2328	* Returns amount of work completed
2329	**/
2330	static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
2331	struct ixgbe_ring *rx_ring,
2332	const int budget)
2333	{
2334	unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0;
2335	struct ixgbe_adapter *adapter = q_vector->adapter;
2336	#ifdef IXGBE_FCOE
2337	int ddp_bytes;
2338	unsigned int mss = 0;
2339	#endif /* IXGBE_FCOE */
2340	u16 cleaned_count = ixgbe_desc_unused(ring: rx_ring);
2341	unsigned int offset = rx_ring->rx_offset;
2342	unsigned int xdp_xmit = 0;
2343	struct xdp_buff xdp;
2344	int xdp_res = 0;
2345
2346	/* Frame size depend on rx_ring setup when PAGE_SIZE=4K */
2347	#if (PAGE_SIZE < 8192)
2348	frame_sz = ixgbe_rx_frame_truesize(rx_ring, size: 0);
2349	#endif
2350	xdp_init_buff(xdp: &xdp, frame_sz, rxq: &rx_ring->xdp_rxq);
2351
2352	while (likely(total_rx_packets < budget)) {
2353	union ixgbe_adv_rx_desc *rx_desc;
2354	struct ixgbe_rx_buffer *rx_buffer;
2355	struct sk_buff *skb;
2356	int rx_buffer_pgcnt;
2357	unsigned int size;
2358
2359	/* return some buffers to hardware, one at a time is too slow */
2360	if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
2361	ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
2362	cleaned_count = 0;
2363	}
2364
2365	rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
2366	size = le16_to_cpu(rx_desc->wb.upper.length);
2367	if (!size)
2368	break;
2369
2370	/* This memory barrier is needed to keep us from reading
2371	* any other fields out of the rx_desc until we know the
2372	* descriptor has been written back
2373	*/
2374	dma_rmb();
2375
2376	rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, skb: &skb, size, rx_buffer_pgcnt: &rx_buffer_pgcnt);
2377
2378	/* retrieve a buffer from the ring */
2379	if (!skb) {
2380	unsigned char *hard_start;
2381
2382	hard_start = page_address(rx_buffer->page) +
2383	rx_buffer->page_offset - offset;
2384	xdp_prepare_buff(xdp: &xdp, hard_start, headroom: offset, data_len: size, meta_valid: true);
2385	xdp_buff_clear_frags_flag(xdp: &xdp);
2386	#if (PAGE_SIZE > 4096)
2387	/* At larger PAGE_SIZE, frame_sz depend on len size */
2388	xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size);
2389	#endif
2390	xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp: &xdp);
2391	}
2392
2393	if (xdp_res) {
2394	if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) {
2395	xdp_xmit |= xdp_res;
2396	ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size);
2397	} else {
2398	rx_buffer->pagecnt_bias++;
2399	}
2400	total_rx_packets++;
2401	total_rx_bytes += size;
2402	} else if (skb) {
2403	ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size);
2404	} else if (ring_uses_build_skb(rx_ring)) {
2405	skb = ixgbe_build_skb(rx_ring, rx_buffer,
2406	xdp: &xdp, rx_desc);
2407	} else {
2408	skb = ixgbe_construct_skb(rx_ring, rx_buffer,
2409	xdp: &xdp, rx_desc);
2410	}
2411
2412	/* exit if we failed to retrieve a buffer */
2413	if (!xdp_res && !skb) {
2414	rx_ring->rx_stats.alloc_rx_buff_failed++;
2415	rx_buffer->pagecnt_bias++;
2416	break;
2417	}
2418
2419	ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);
2420	cleaned_count++;
2421
2422	/* place incomplete frames back on ring for completion */
2423	if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
2424	continue;
2425
2426	/* verify the packet layout is correct */
2427	if (xdp_res || ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
2428	continue;
2429
2430	/* probably a little skewed due to removing CRC */
2431	total_rx_bytes += skb->len;
2432
2433	/* populate checksum, timestamp, VLAN, and protocol */
2434	ixgbe_process_skb_fields(rx_ring, rx_desc, skb);
2435
2436	#ifdef IXGBE_FCOE
2437	/* if ddp, not passing to ULD unless for FCP_RSP or error */
2438	if (ixgbe_rx_is_fcoe(ring: rx_ring, rx_desc)) {
2439	ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
2440	/* include DDPed FCoE data */
2441	if (ddp_bytes > 0) {
2442	if (!mss) {
2443	mss = rx_ring->netdev->mtu -
2444	sizeof(struct fcoe_hdr) -
2445	sizeof(struct fc_frame_header) -
2446	sizeof(struct fcoe_crc_eof);
2447	if (mss > 512)
2448	mss &= ~511;
2449	}
2450	total_rx_bytes += ddp_bytes;
2451	total_rx_packets += DIV_ROUND_UP(ddp_bytes,
2452	mss);
2453	}
2454	if (!ddp_bytes) {
2455	dev_kfree_skb_any(skb);
2456	continue;
2457	}
2458	}
2459
2460	#endif /* IXGBE_FCOE */
2461	ixgbe_rx_skb(q_vector, skb);
2462
2463	/* update budget accounting */
2464	total_rx_packets++;
2465	}
2466
2467	if (xdp_xmit & IXGBE_XDP_REDIR)
2468	xdp_do_flush();
2469
2470	if (xdp_xmit & IXGBE_XDP_TX) {
2471	struct ixgbe_ring *ring = ixgbe_determine_xdp_ring(adapter);
2472
2473	ixgbe_xdp_ring_update_tail_locked(ring);
2474	}
2475
2476	ixgbe_update_rx_ring_stats(rx_ring, q_vector, pkts: total_rx_packets,
2477	bytes: total_rx_bytes);
2478
2479	return total_rx_packets;
2480	}
2481
2482	/**
2483	* ixgbe_configure_msix - Configure MSI-X hardware
2484	* @adapter: board private structure
2485	*
2486	* ixgbe_configure_msix sets up the hardware to properly generate MSI-X
2487	* interrupts.
2488	**/
2489	static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
2490	{
2491	struct ixgbe_q_vector *q_vector;
2492	int v_idx;
2493	u32 mask;
2494
2495	/* Populate MSIX to EITR Select */
2496	if (adapter->num_vfs > 32) {
2497	u32 eitrsel = BIT(adapter->num_vfs - 32) - 1;
2498	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
2499	}
2500
2501	/*
2502	* Populate the IVAR table and set the ITR values to the
2503	* corresponding register.
2504	*/
2505	for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
2506	struct ixgbe_ring *ring;
2507	q_vector = adapter->q_vector[v_idx];
2508
2509	ixgbe_for_each_ring(ring, q_vector->rx)
2510	ixgbe_set_ivar(adapter, direction: 0, queue: ring->reg_idx, msix_vector: v_idx);
2511
2512	ixgbe_for_each_ring(ring, q_vector->tx)
2513	ixgbe_set_ivar(adapter, direction: 1, queue: ring->reg_idx, msix_vector: v_idx);
2514
2515	ixgbe_write_eitr(q_vector);
2516	}
2517
2518	switch (adapter->hw.mac.type) {
2519	case ixgbe_mac_82598EB:
2520	ixgbe_set_ivar(adapter, direction: -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
2521	msix_vector: v_idx);
2522	break;
2523	case ixgbe_mac_82599EB:
2524	case ixgbe_mac_X540:
2525	case ixgbe_mac_X550:
2526	case ixgbe_mac_X550EM_x:
2527	case ixgbe_mac_x550em_a:
2528	case ixgbe_mac_e610:
2529	ixgbe_set_ivar(adapter, direction: -1, queue: 1, msix_vector: v_idx);
2530	break;
2531	default:
2532	break;
2533	}
2534	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
2535
2536	/* set up to autoclear timer, and the vectors */
2537	mask = IXGBE_EIMS_ENABLE_MASK;
2538	mask &= ~(IXGBE_EIMS_OTHER |
2539	IXGBE_EIMS_MAILBOX |
2540	IXGBE_EIMS_LSC);
2541
2542	if (adapter->hw.mac.type == ixgbe_mac_e610)
2543	mask &= ~IXGBE_EIMS_FW_EVENT;
2544
2545	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
2546	}
2547
2548	/**
2549	* ixgbe_update_itr - update the dynamic ITR value based on statistics
2550	* @q_vector: structure containing interrupt and ring information
2551	* @ring_container: structure containing ring performance data
2552	*
2553	* Stores a new ITR value based on packets and byte
2554	* counts during the last interrupt. The advantage of per interrupt
2555	* computation is faster updates and more accurate ITR for the current
2556	* traffic pattern. Constants in this function were computed
2557	* based on theoretical maximum wire speed and thresholds were set based
2558	* on testing data as well as attempting to minimize response time
2559	* while increasing bulk throughput.
2560	**/
2561	static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
2562	struct ixgbe_ring_container *ring_container)
2563	{
2564	unsigned int itr = IXGBE_ITR_ADAPTIVE_MIN_USECS |
2565	IXGBE_ITR_ADAPTIVE_LATENCY;
2566	unsigned int avg_wire_size, packets, bytes;
2567	unsigned long next_update = jiffies;
2568
2569	/* If we don't have any rings just leave ourselves set for maximum
2570	* possible latency so we take ourselves out of the equation.
2571	*/
2572	if (!ring_container->ring)
2573	return;
2574
2575	/* If we didn't update within up to 1 - 2 jiffies we can assume
2576	* that either packets are coming in so slow there hasn't been
2577	* any work, or that there is so much work that NAPI is dealing
2578	* with interrupt moderation and we don't need to do anything.
2579	*/
2580	if (time_after(next_update, ring_container->next_update))
2581	goto clear_counts;
2582
2583	packets = ring_container->total_packets;
2584
2585	/* We have no packets to actually measure against. This means
2586	* either one of the other queues on this vector is active or
2587	* we are a Tx queue doing TSO with too high of an interrupt rate.
2588	*
2589	* When this occurs just tick up our delay by the minimum value
2590	* and hope that this extra delay will prevent us from being called
2591	* without any work on our queue.
2592	*/
2593	if (!packets) {
2594	itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;
2595	if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)
2596	itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;
2597	itr += ring_container->itr & IXGBE_ITR_ADAPTIVE_LATENCY;
2598	goto clear_counts;
2599	}
2600
2601	bytes = ring_container->total_bytes;
2602
2603	/* If packets are less than 4 or bytes are less than 9000 assume
2604	* insufficient data to use bulk rate limiting approach. We are
2605	* likely latency driven.
2606	*/
2607	if (packets < 4 && bytes < 9000) {
2608	itr = IXGBE_ITR_ADAPTIVE_LATENCY;
2609	goto adjust_by_size;
2610	}
2611
2612	/* Between 4 and 48 we can assume that our current interrupt delay
2613	* is only slightly too low. As such we should increase it by a small
2614	* fixed amount.
2615	*/
2616	if (packets < 48) {
2617	itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;
2618	if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)
2619	itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;
2620	goto clear_counts;
2621	}
2622
2623	/* Between 48 and 96 is our "goldilocks" zone where we are working
2624	* out "just right". Just report that our current ITR is good for us.
2625	*/
2626	if (packets < 96) {
2627	itr = q_vector->itr >> 2;
2628	goto clear_counts;
2629	}
2630
2631	/* If packet count is 96 or greater we are likely looking at a slight
2632	* overrun of the delay we want. Try halving our delay to see if that
2633	* will cut the number of packets in half per interrupt.
2634	*/
2635	if (packets < 256) {
2636	itr = q_vector->itr >> 3;
2637	if (itr < IXGBE_ITR_ADAPTIVE_MIN_USECS)
2638	itr = IXGBE_ITR_ADAPTIVE_MIN_USECS;
2639	goto clear_counts;
2640	}
2641
2642	/* The paths below assume we are dealing with a bulk ITR since number
2643	* of packets is 256 or greater. We are just going to have to compute
2644	* a value and try to bring the count under control, though for smaller
2645	* packet sizes there isn't much we can do as NAPI polling will likely
2646	* be kicking in sooner rather than later.
2647	*/
2648	itr = IXGBE_ITR_ADAPTIVE_BULK;
2649
2650	adjust_by_size:
2651	/* If packet counts are 256 or greater we can assume we have a gross
2652	* overestimation of what the rate should be. Instead of trying to fine
2653	* tune it just use the formula below to try and dial in an exact value
2654	* give the current packet size of the frame.
2655	*/
2656	avg_wire_size = bytes / packets;
2657
2658	/* The following is a crude approximation of:
2659	* wmem_default / (size + overhead) = desired_pkts_per_int
2660	* rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
2661	* (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
2662	*
2663	* Assuming wmem_default is 212992 and overhead is 640 bytes per
2664	* packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
2665	* formula down to
2666	*
2667	* (170 * (size + 24)) / (size + 640) = ITR
2668	*
2669	* We first do some math on the packet size and then finally bitshift
2670	* by 8 after rounding up. We also have to account for PCIe link speed
2671	* difference as ITR scales based on this.
2672	*/
2673	if (avg_wire_size <= 60) {
2674	/* Start at 50k ints/sec */
2675	avg_wire_size = 5120;
2676	} else if (avg_wire_size <= 316) {
2677	/* 50K ints/sec to 16K ints/sec */
2678	avg_wire_size *= 40;
2679	avg_wire_size += 2720;
2680	} else if (avg_wire_size <= 1084) {
2681	/* 16K ints/sec to 9.2K ints/sec */
2682	avg_wire_size *= 15;
2683	avg_wire_size += 11452;
2684	} else if (avg_wire_size < 1968) {
2685	/* 9.2K ints/sec to 8K ints/sec */
2686	avg_wire_size *= 5;
2687	avg_wire_size += 22420;
2688	} else {
2689	/* plateau at a limit of 8K ints/sec */
2690	avg_wire_size = 32256;
2691	}
2692
2693	/* If we are in low latency mode half our delay which doubles the rate
2694	* to somewhere between 100K to 16K ints/sec
2695	*/
2696	if (itr & IXGBE_ITR_ADAPTIVE_LATENCY)
2697	avg_wire_size >>= 1;
2698
2699	/* Resultant value is 256 times larger than it needs to be. This
2700	* gives us room to adjust the value as needed to either increase
2701	* or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.
2702	*
2703	* Use addition as we have already recorded the new latency flag
2704	* for the ITR value.
2705	*/
2706	switch (q_vector->adapter->link_speed) {
2707	case IXGBE_LINK_SPEED_10GB_FULL:
2708	case IXGBE_LINK_SPEED_100_FULL:
2709	default:
2710	itr += DIV_ROUND_UP(avg_wire_size,
2711	IXGBE_ITR_ADAPTIVE_MIN_INC * 256) *
2712	IXGBE_ITR_ADAPTIVE_MIN_INC;
2713	break;
2714	case IXGBE_LINK_SPEED_2_5GB_FULL:
2715	case IXGBE_LINK_SPEED_1GB_FULL:
2716	case IXGBE_LINK_SPEED_10_FULL:
2717	if (avg_wire_size > 8064)
2718	avg_wire_size = 8064;
2719	itr += DIV_ROUND_UP(avg_wire_size,
2720	IXGBE_ITR_ADAPTIVE_MIN_INC * 64) *
2721	IXGBE_ITR_ADAPTIVE_MIN_INC;
2722	break;
2723	}
2724
2725	clear_counts:
2726	/* write back value */
2727	ring_container->itr = itr;
2728
2729	/* next update should occur within next jiffy */
2730	ring_container->next_update = next_update + 1;
2731
2732	ring_container->total_bytes = 0;
2733	ring_container->total_packets = 0;
2734	}
2735
2736	/**
2737	* ixgbe_write_eitr - write EITR register in hardware specific way
2738	* @q_vector: structure containing interrupt and ring information
2739	*
2740	* This function is made to be called by ethtool and by the driver
2741	* when it needs to update EITR registers at runtime. Hardware
2742	* specific quirks/differences are taken care of here.
2743	*/
2744	void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
2745	{
2746	struct ixgbe_adapter *adapter = q_vector->adapter;
2747	struct ixgbe_hw *hw = &adapter->hw;
2748	int v_idx = q_vector->v_idx;
2749	u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
2750
2751	switch (adapter->hw.mac.type) {
2752	case ixgbe_mac_82598EB:
2753	/* must write high and low 16 bits to reset counter */
2754	itr_reg |= (itr_reg << 16);
2755	break;
2756	case ixgbe_mac_82599EB:
2757	case ixgbe_mac_X540:
2758	case ixgbe_mac_X550:
2759	case ixgbe_mac_X550EM_x:
2760	case ixgbe_mac_x550em_a:
2761	case ixgbe_mac_e610:
2762	/*
2763	* set the WDIS bit to not clear the timer bits and cause an
2764	* immediate assertion of the interrupt
2765	*/
2766	itr_reg |= IXGBE_EITR_CNT_WDIS;
2767	break;
2768	default:
2769	break;
2770	}
2771	IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
2772	}
2773
2774	static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
2775	{
2776	u32 new_itr;
2777
2778	ixgbe_update_itr(q_vector, ring_container: &q_vector->tx);
2779	ixgbe_update_itr(q_vector, ring_container: &q_vector->rx);
2780
2781	/* use the smallest value of new ITR delay calculations */
2782	new_itr = min(q_vector->rx.itr, q_vector->tx.itr);
2783
2784	/* Clear latency flag if set, shift into correct position */
2785	new_itr &= ~IXGBE_ITR_ADAPTIVE_LATENCY;
2786	new_itr <<= 2;
2787
2788	if (new_itr != q_vector->itr) {
2789	/* save the algorithm value here */
2790	q_vector->itr = new_itr;
2791
2792	ixgbe_write_eitr(q_vector);
2793	}
2794	}
2795
2796	/**
2797	* ixgbe_check_overtemp_subtask - check for over temperature
2798	* @adapter: pointer to adapter
2799	**/
2800	static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
2801	{
2802	struct ixgbe_hw *hw = &adapter->hw;
2803	u32 eicr = adapter->interrupt_event;
2804
2805	if (test_bit(__IXGBE_DOWN, &adapter->state))
2806	return;
2807
2808	if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
2809	return;
2810
2811	adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2812
2813	switch (hw->device_id) {
2814	case IXGBE_DEV_ID_82599_T3_LOM:
2815	/*
2816	* Since the warning interrupt is for both ports
2817	* we don't have to check if:
2818	* - This interrupt wasn't for our port.
2819	* - We may have missed the interrupt so always have to
2820	* check if we got a LSC
2821	*/
2822	if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) &&
2823	!(eicr & IXGBE_EICR_LSC))
2824	return;
2825
2826	if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
2827	u32 speed;
2828	bool link_up = false;
2829
2830	hw->mac.ops.check_link(hw, &speed, &link_up, false);
2831
2832	if (link_up)
2833	return;
2834	}
2835
2836	/* Check if this is not due to overtemp */
2837	if (!hw->phy.ops.check_overtemp(hw))
2838	return;
2839
2840	break;
2841	case IXGBE_DEV_ID_X550EM_A_1G_T:
2842	case IXGBE_DEV_ID_X550EM_A_1G_T_L:
2843	if (!hw->phy.ops.check_overtemp(hw))
2844	return;
2845	break;
2846	default:
2847	if (adapter->hw.mac.type >= ixgbe_mac_X540)
2848	return;
2849	if (!(eicr & IXGBE_EICR_GPI_SDP0(hw)))
2850	return;
2851	break;
2852	}
2853	e_crit(drv, "%s\n", ixgbe_overheat_msg);
2854
2855	adapter->interrupt_event = 0;
2856	}
2857
2858	static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
2859	{
2860	struct ixgbe_hw *hw = &adapter->hw;
2861
2862	if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
2863	(eicr & IXGBE_EICR_GPI_SDP1(hw))) {
2864	e_crit(probe, "Fan has stopped, replace the adapter\n");
2865	/* write to clear the interrupt */
2866	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
2867	}
2868	}
2869
2870	static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
2871	{
2872	struct ixgbe_hw *hw = &adapter->hw;
2873
2874	if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
2875	return;
2876
2877	switch (adapter->hw.mac.type) {
2878	case ixgbe_mac_82599EB:
2879	/*
2880	* Need to check link state so complete overtemp check
2881	* on service task
2882	*/
2883	if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) ||
2884	(eicr & IXGBE_EICR_LSC)) &&
2885	(!test_bit(__IXGBE_DOWN, &adapter->state))) {
2886	adapter->interrupt_event = eicr;
2887	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2888	ixgbe_service_event_schedule(adapter);
2889	return;
2890	}
2891	return;
2892	case ixgbe_mac_x550em_a:
2893	if (eicr & IXGBE_EICR_GPI_SDP0_X550EM_a) {
2894	adapter->interrupt_event = eicr;
2895	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2896	ixgbe_service_event_schedule(adapter);
2897	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC,
2898	IXGBE_EICR_GPI_SDP0_X550EM_a);
2899	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICR,
2900	IXGBE_EICR_GPI_SDP0_X550EM_a);
2901	}
2902	return;
2903	case ixgbe_mac_X550:
2904	case ixgbe_mac_X540:
2905	if (!(eicr & IXGBE_EICR_TS))
2906	return;
2907	break;
2908	default:
2909	return;
2910	}
2911
2912	e_crit(drv, "%s\n", ixgbe_overheat_msg);
2913	}
2914
2915	static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
2916	{
2917	switch (hw->mac.type) {
2918	case ixgbe_mac_82598EB:
2919	if (hw->phy.type == ixgbe_phy_nl)
2920	return true;
2921	return false;
2922	case ixgbe_mac_82599EB:
2923	case ixgbe_mac_X550EM_x:
2924	case ixgbe_mac_x550em_a:
2925	switch (hw->mac.ops.get_media_type(hw)) {
2926	case ixgbe_media_type_fiber:
2927	case ixgbe_media_type_fiber_qsfp:
2928	return true;
2929	default:
2930	return false;
2931	}
2932	default:
2933	return false;
2934	}
2935	}
2936
2937	static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
2938	{
2939	struct ixgbe_hw *hw = &adapter->hw;
2940	u32 eicr_mask = IXGBE_EICR_GPI_SDP2(hw);
2941
2942	if (!ixgbe_is_sfp(hw))
2943	return;
2944
2945	/* Later MAC's use different SDP */
2946	if (hw->mac.type >= ixgbe_mac_X540)
2947	eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
2948
2949	if (eicr & eicr_mask) {
2950	/* Clear the interrupt */
2951	IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
2952	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2953	adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
2954	adapter->sfp_poll_time = 0;
2955	ixgbe_service_event_schedule(adapter);
2956	}
2957	}
2958
2959	if (adapter->hw.mac.type == ixgbe_mac_82599EB &&
2960	(eicr & IXGBE_EICR_GPI_SDP1(hw))) {
2961	/* Clear the interrupt */
2962	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
2963	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2964	adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
2965	ixgbe_service_event_schedule(adapter);
2966	}
2967	}
2968	}
2969
2970	static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
2971	{
2972	struct ixgbe_hw *hw = &adapter->hw;
2973
2974	adapter->lsc_int++;
2975	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2976	adapter->link_check_timeout = jiffies;
2977	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2978	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
2979	IXGBE_WRITE_FLUSH(hw);
2980	ixgbe_service_event_schedule(adapter);
2981	}
2982	}
2983
2984	/**
2985	* ixgbe_check_phy_fw_load - check if PHY FW load failed
2986	* @adapter: pointer to adapter structure
2987	* @link_cfg_err: bitmap from the link info structure
2988	*
2989	* Check if external PHY FW load failed and print an error message if it did.
2990	*/
2991	static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter,
2992	u8 link_cfg_err)
2993	{
2994	if (!(link_cfg_err & IXGBE_ACI_LINK_EXTERNAL_PHY_LOAD_FAILURE)) {
2995	adapter->flags2 &= ~IXGBE_FLAG2_PHY_FW_LOAD_FAILED;
2996	return;
2997	}
2998
2999	if (adapter->flags2 & IXGBE_FLAG2_PHY_FW_LOAD_FAILED)
3000	return;
3001
3002	if (link_cfg_err & IXGBE_ACI_LINK_EXTERNAL_PHY_LOAD_FAILURE) {
3003	netdev_err(dev: adapter->netdev, format: "Device failed to load the FW for the external PHY. Please download and install the latest NVM for your device and try again\n");
3004	adapter->flags2 |= IXGBE_FLAG2_PHY_FW_LOAD_FAILED;
3005	}
3006	}
3007
3008	/**
3009	* ixgbe_check_module_power - check module power level
3010	* @adapter: pointer to adapter structure
3011	* @link_cfg_err: bitmap from the link info structure
3012	*
3013	* Check module power level returned by a previous call to aci_get_link_info
3014	* and print error messages if module power level is not supported.
3015	*/
3016	static void ixgbe_check_module_power(struct ixgbe_adapter *adapter,
3017	u8 link_cfg_err)
3018	{
3019	/* If module power level is supported, clear the flag. */
3020	if (!(link_cfg_err & (IXGBE_ACI_LINK_INVAL_MAX_POWER_LIMIT |
3021	IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) {
3022	adapter->flags2 &= ~IXGBE_FLAG2_MOD_POWER_UNSUPPORTED;
3023	return;
3024	}
3025
3026	/* If IXGBE_FLAG2_MOD_POWER_UNSUPPORTED was previously set and the
3027	* above block didn't clear this bit, there's nothing to do.
3028	*/
3029	if (adapter->flags2 & IXGBE_FLAG2_MOD_POWER_UNSUPPORTED)
3030	return;
3031
3032	if (link_cfg_err & IXGBE_ACI_LINK_INVAL_MAX_POWER_LIMIT) {
3033	netdev_err(dev: adapter->netdev, format: "The installed module is incompatible with the device's NVM image. Cannot start link.\n");
3034	adapter->flags2 |= IXGBE_FLAG2_MOD_POWER_UNSUPPORTED;
3035	} else if (link_cfg_err & IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED) {
3036	netdev_err(dev: adapter->netdev, format: "The module's power requirements exceed the device's power supply. Cannot start link.\n");
3037	adapter->flags2 |= IXGBE_FLAG2_MOD_POWER_UNSUPPORTED;
3038	}
3039	}
3040
3041	/**
3042	* ixgbe_check_link_cfg_err - check if link configuration failed
3043	* @adapter: pointer to adapter structure
3044	* @link_cfg_err: bitmap from the link info structure
3045	*
3046	* Print if any link configuration failure happens due to the value in the
3047	* link_cfg_err parameter in the link info structure.
3048	*/
3049	static void ixgbe_check_link_cfg_err(struct ixgbe_adapter *adapter,
3050	u8 link_cfg_err)
3051	{
3052	ixgbe_check_module_power(adapter, link_cfg_err);
3053	ixgbe_check_phy_fw_load(adapter, link_cfg_err);
3054	}
3055
3056	/**
3057	* ixgbe_process_link_status_event - process the link event
3058	* @adapter: pointer to adapter structure
3059	* @link_up: true if the physical link is up and false if it is down
3060	* @link_speed: current link speed received from the link event
3061	*
3062	* Return: 0 on success or negative value on failure.
3063	*/
3064	static int
3065	ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up,
3066	u16 link_speed)
3067	{
3068	struct ixgbe_hw *hw = &adapter->hw;
3069	int status;
3070
3071	/* Update the link info structures and re-enable link events,
3072	* don't bail on failure due to other book keeping needed.
3073	*/
3074	status = ixgbe_update_link_info(hw);
3075	if (status)
3076	e_dev_err("Failed to update link status, err %d aq_err %d\n",
3077	status, hw->aci.last_status);
3078
3079	ixgbe_check_link_cfg_err(adapter, link_cfg_err: hw->link.link_info.link_cfg_err);
3080
3081	/* Check if the link state is up after updating link info, and treat
3082	* this event as an UP event since the link is actually UP now.
3083	*/
3084	if (hw->link.link_info.link_info & IXGBE_ACI_LINK_UP)
3085	link_up = true;
3086
3087	/* Turn off PHY if media was removed. */
3088	if (!(adapter->flags2 & IXGBE_FLAG2_NO_MEDIA) &&
3089	!(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE))
3090	adapter->flags2 |= IXGBE_FLAG2_NO_MEDIA;
3091
3092	if (link_up == adapter->link_up &&
3093	link_up == netif_carrier_ok(dev: adapter->netdev) &&
3094	link_speed == adapter->link_speed)
3095	return 0;
3096
3097	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
3098	adapter->link_check_timeout = jiffies;
3099	ixgbe_watchdog_update_link(adapter);
3100
3101	if (link_up)
3102	ixgbe_watchdog_link_is_up(adapter);
3103	else
3104	ixgbe_watchdog_link_is_down(adapter);
3105
3106	return 0;
3107	}
3108
3109	/**
3110	* ixgbe_handle_link_status_event - handle link status event via ACI
3111	* @adapter: pointer to adapter structure
3112	* @e: event structure containing link status info
3113	*/
3114	static void
3115	ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter,
3116	struct ixgbe_aci_event *e)
3117	{
3118	struct ixgbe_aci_cmd_get_link_status_data *link_data;
3119	u16 link_speed;
3120	bool link_up;
3121
3122	link_data = (struct ixgbe_aci_cmd_get_link_status_data *)e->msg_buf;
3123
3124	link_up = !!(link_data->link_info & IXGBE_ACI_LINK_UP);
3125	link_speed = le16_to_cpu(link_data->link_speed);
3126
3127	if (ixgbe_process_link_status_event(adapter, link_up, link_speed))
3128	e_dev_warn("Could not process link status event");
3129	}
3130
3131	/**
3132	* ixgbe_schedule_fw_event - schedule Firmware event
3133	* @adapter: pointer to the adapter structure
3134	*
3135	* If the adapter is not in down, removing or resetting state,
3136	* an event is scheduled.
3137	*/
3138	static void ixgbe_schedule_fw_event(struct ixgbe_adapter *adapter)
3139	{
3140	if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
3141	!test_bit(__IXGBE_REMOVING, &adapter->state) &&
3142	!test_bit(__IXGBE_RESETTING, &adapter->state)) {
3143	adapter->flags2 |= IXGBE_FLAG2_FW_ASYNC_EVENT;
3144	ixgbe_service_event_schedule(adapter);
3145	}
3146	}
3147
3148	/**
3149	* ixgbe_aci_event_cleanup - release msg_buf memory
3150	* @event: pointer to the event holding msg_buf to be released
3151	*
3152	* Clean memory allocated for event's msg_buf. Implements auto memory cleanup.
3153	*/
3154	static void ixgbe_aci_event_cleanup(struct ixgbe_aci_event *event)
3155	{
3156	kfree(objp: event->msg_buf);
3157	}
3158
3159	/**
3160	* ixgbe_handle_fw_event - handle Firmware event
3161	* @adapter: pointer to the adapter structure
3162	*
3163	* Obtain an event from the ACI and then and then process it according to the
3164	* type of the event and the opcode.
3165	*/
3166	static void ixgbe_handle_fw_event(struct ixgbe_adapter *adapter)
3167	{
3168	struct ixgbe_aci_event event __cleanup(ixgbe_aci_event_cleanup);
3169	struct ixgbe_hw *hw = &adapter->hw;
3170	bool pending = false;
3171	int err;
3172
3173	if (adapter->flags2 & IXGBE_FLAG2_FW_ASYNC_EVENT)
3174	adapter->flags2 &= ~IXGBE_FLAG2_FW_ASYNC_EVENT;
3175	event.buf_len = IXGBE_ACI_MAX_BUFFER_SIZE;
3176	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
3177	if (!event.msg_buf)
3178	return;
3179
3180	do {
3181	err = ixgbe_aci_get_event(hw, e: &event, pending: &pending);
3182	if (err)
3183	break;
3184
3185	switch (le16_to_cpu(event.desc.opcode)) {
3186	case ixgbe_aci_opc_get_link_status:
3187	ixgbe_handle_link_status_event(adapter, e: &event);
3188	break;
3189	case ixgbe_aci_opc_temp_tca_event:
3190	e_crit(drv, "%s\n", ixgbe_overheat_msg);
3191	ixgbe_down(adapter);
3192	break;
3193	default:
3194	e_warn(hw, "unknown FW async event captured\n");
3195	break;
3196	}
3197	} while (pending);
3198	}
3199
3200	static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
3201	u64 qmask)
3202	{
3203	struct ixgbe_hw *hw = &adapter->hw;
3204	u32 mask;
3205
3206	switch (hw->mac.type) {
3207	case ixgbe_mac_82598EB:
3208	mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
3209	IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3210	break;
3211	case ixgbe_mac_82599EB:
3212	case ixgbe_mac_X540:
3213	case ixgbe_mac_X550:
3214	case ixgbe_mac_X550EM_x:
3215	case ixgbe_mac_x550em_a:
3216	case ixgbe_mac_e610:
3217	mask = (qmask & 0xFFFFFFFF);
3218	if (mask)
3219	IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
3220	mask = (qmask >> 32);
3221	if (mask)
3222	IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
3223	break;
3224	default:
3225	break;
3226	}
3227	/* skip the flush */
3228	}
3229
3230	/**
3231	* ixgbe_irq_enable - Enable default interrupt generation settings
3232	* @adapter: board private structure
3233	* @queues: enable irqs for queues
3234	* @flush: flush register write
3235	**/
3236	static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
3237	bool flush)
3238	{
3239	struct ixgbe_hw *hw = &adapter->hw;
3240	u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3241
3242	/* don't reenable LSC while waiting for link */
3243	if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
3244	mask &= ~IXGBE_EIMS_LSC;
3245
3246	if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
3247	switch (adapter->hw.mac.type) {
3248	case ixgbe_mac_82599EB:
3249	mask |= IXGBE_EIMS_GPI_SDP0(hw);
3250	break;
3251	case ixgbe_mac_X540:
3252	case ixgbe_mac_X550:
3253	case ixgbe_mac_X550EM_x:
3254	case ixgbe_mac_x550em_a:
3255	mask |= IXGBE_EIMS_TS;
3256	break;
3257	default:
3258	break;
3259	}
3260	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
3261	mask |= IXGBE_EIMS_GPI_SDP1(hw);
3262	switch (adapter->hw.mac.type) {
3263	case ixgbe_mac_82599EB:
3264	mask |= IXGBE_EIMS_GPI_SDP1(hw);
3265	mask |= IXGBE_EIMS_GPI_SDP2(hw);
3266	fallthrough;
3267	case ixgbe_mac_X540:
3268	case ixgbe_mac_X550:
3269	case ixgbe_mac_X550EM_x:
3270	case ixgbe_mac_e610:
3271	mask |= IXGBE_EIMS_FW_EVENT;
3272	fallthrough;
3273	case ixgbe_mac_x550em_a:
3274	if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
3275	adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
3276	adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N)
3277	mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw);
3278	if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t)
3279	mask |= IXGBE_EICR_GPI_SDP0_X540;
3280	mask |= IXGBE_EIMS_ECC;
3281	mask |= IXGBE_EIMS_MAILBOX;
3282	break;
3283	default:
3284	break;
3285	}
3286
3287	if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
3288	!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
3289	mask |= IXGBE_EIMS_FLOW_DIR;
3290
3291	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
3292	if (queues)
3293	ixgbe_irq_enable_queues(adapter, qmask: ~0);
3294	if (flush)
3295	IXGBE_WRITE_FLUSH(&adapter->hw);
3296	}
3297
3298	static irqreturn_t ixgbe_msix_other(int irq, void *data)
3299	{
3300	struct ixgbe_adapter *adapter = data;
3301	struct ixgbe_hw *hw = &adapter->hw;
3302	u32 eicr;
3303
3304	/*
3305	* Workaround for Silicon errata. Use clear-by-write instead
3306	* of clear-by-read. Reading with EICS will return the
3307	* interrupt causes without clearing, which later be done
3308	* with the write to EICR.
3309	*/
3310	eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
3311
3312	/* The lower 16bits of the EICR register are for the queue interrupts
3313	* which should be masked here in order to not accidentally clear them if
3314	* the bits are high when ixgbe_msix_other is called. There is a race
3315	* condition otherwise which results in possible performance loss
3316	* especially if the ixgbe_msix_other interrupt is triggering
3317	* consistently (as it would when PPS is turned on for the X540 device)
3318	*/
3319	eicr &= 0xFFFF0000;
3320
3321	IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3322
3323	if (eicr & IXGBE_EICR_LSC)
3324	ixgbe_check_lsc(adapter);
3325
3326	if (eicr & IXGBE_EICR_MAILBOX)
3327	ixgbe_msg_task(adapter);
3328
3329	if (eicr & IXGBE_EICR_FW_EVENT)
3330	ixgbe_schedule_fw_event(adapter);
3331
3332	switch (hw->mac.type) {
3333	case ixgbe_mac_82599EB:
3334	case ixgbe_mac_X540:
3335	case ixgbe_mac_X550:
3336	case ixgbe_mac_X550EM_x:
3337	case ixgbe_mac_x550em_a:
3338	case ixgbe_mac_e610:
3339	if (hw->phy.type == ixgbe_phy_x550em_ext_t &&
3340	(eicr & IXGBE_EICR_GPI_SDP0_X540)) {
3341	adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT;
3342	ixgbe_service_event_schedule(adapter);
3343	IXGBE_WRITE_REG(hw, IXGBE_EICR,
3344	IXGBE_EICR_GPI_SDP0_X540);
3345	}
3346	if (eicr & IXGBE_EICR_ECC) {
3347	e_info(link, "Received ECC Err, initiating reset\n");
3348	set_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
3349	ixgbe_service_event_schedule(adapter);
3350	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
3351	}
3352	/* Handle Flow Director Full threshold interrupt */
3353	if (eicr & IXGBE_EICR_FLOW_DIR) {
3354	int reinit_count = 0;
3355	int i;
3356	for (i = 0; i < adapter->num_tx_queues; i++) {
3357	struct ixgbe_ring *ring = adapter->tx_ring[i];
3358	if (test_and_clear_bit(nr: __IXGBE_TX_FDIR_INIT_DONE,
3359	addr: &ring->state))
3360	reinit_count++;
3361	}
3362	if (reinit_count) {
3363	/* no more flow director interrupts until after init */
3364	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
3365	adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
3366	ixgbe_service_event_schedule(adapter);
3367	}
3368	}
3369	ixgbe_check_sfp_event(adapter, eicr);
3370	ixgbe_check_overtemp_event(adapter, eicr);
3371	break;
3372	default:
3373	break;
3374	}
3375
3376	ixgbe_check_fan_failure(adapter, eicr);
3377
3378	if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
3379	ixgbe_ptp_check_pps_event(adapter);
3380
3381	/* re-enable the original interrupt state, no lsc, no queues */
3382	if (!test_bit(__IXGBE_DOWN, &adapter->state))
3383	ixgbe_irq_enable(adapter, queues: false, flush: false);
3384
3385	return IRQ_HANDLED;
3386	}
3387
3388	static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
3389	{
3390	struct ixgbe_q_vector *q_vector = data;
3391
3392	/* EIAM disabled interrupts (on this vector) for us */
3393
3394	if (q_vector->rx.ring || q_vector->tx.ring)
3395	napi_schedule_irqoff(n: &q_vector->napi);
3396
3397	return IRQ_HANDLED;
3398	}
3399
3400	/**
3401	* ixgbe_poll - NAPI Rx polling callback
3402	* @napi: structure for representing this polling device
3403	* @budget: how many packets driver is allowed to clean
3404	*
3405	* This function is used for legacy and MSI, NAPI mode
3406	**/
3407	int ixgbe_poll(struct napi_struct *napi, int budget)
3408	{
3409	struct ixgbe_q_vector *q_vector =
3410	container_of(napi, struct ixgbe_q_vector, napi);
3411	struct ixgbe_adapter *adapter = q_vector->adapter;
3412	struct ixgbe_ring *ring;
3413	int per_ring_budget, work_done = 0;
3414	bool clean_complete = true;
3415
3416	#ifdef CONFIG_IXGBE_DCA
3417	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
3418	ixgbe_update_dca(q_vector);
3419	#endif
3420
3421	ixgbe_for_each_ring(ring, q_vector->tx) {
3422	bool wd = ring->xsk_pool ?
3423	ixgbe_clean_xdp_tx_irq(q_vector, tx_ring: ring, napi_budget: budget) :
3424	ixgbe_clean_tx_irq(q_vector, tx_ring: ring, napi_budget: budget);
3425
3426	if (!wd)
3427	clean_complete = false;
3428	}
3429
3430	/* Exit if we are called by netpoll */
3431	if (budget <= 0)
3432	return budget;
3433
3434	/* attempt to distribute budget to each queue fairly, but don't allow
3435	* the budget to go below 1 because we'll exit polling */
3436	if (q_vector->rx.count > 1)
3437	per_ring_budget = max(budget/q_vector->rx.count, 1);
3438	else
3439	per_ring_budget = budget;
3440
3441	ixgbe_for_each_ring(ring, q_vector->rx) {
3442	int cleaned = ring->xsk_pool ?
3443	ixgbe_clean_rx_irq_zc(q_vector, rx_ring: ring,
3444	budget: per_ring_budget) :
3445	ixgbe_clean_rx_irq(q_vector, rx_ring: ring,
3446	budget: per_ring_budget);
3447
3448	work_done += cleaned;
3449	if (cleaned >= per_ring_budget)
3450	clean_complete = false;
3451	}
3452
3453	/* If all work not completed, return budget and keep polling */
3454	if (!clean_complete)
3455	return budget;
3456
3457	/* all work done, exit the polling mode */
3458	if (likely(napi_complete_done(napi, work_done))) {
3459	if (adapter->rx_itr_setting & 1)
3460	ixgbe_set_itr(q_vector);
3461	if (!test_bit(__IXGBE_DOWN, &adapter->state))
3462	ixgbe_irq_enable_queues(adapter,
3463	BIT_ULL(q_vector->v_idx));
3464	}
3465
3466	return min(work_done, budget - 1);
3467	}
3468
3469	/**
3470	* ixgbe_request_msix_irqs - Initialize MSI-X interrupts
3471	* @adapter: board private structure
3472	*
3473	* ixgbe_request_msix_irqs allocates MSI-X vectors and requests
3474	* interrupts from the kernel.
3475	**/
3476	static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
3477	{
3478	struct net_device *netdev = adapter->netdev;
3479	unsigned int ri = 0, ti = 0;
3480	int vector, err;
3481
3482	for (vector = 0; vector < adapter->num_q_vectors; vector++) {
3483	struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
3484	struct msix_entry *entry = &adapter->msix_entries[vector];
3485
3486	if (q_vector->tx.ring && q_vector->rx.ring) {
3487	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
3488	fmt: "%s-TxRx-%u", netdev->name, ri++);
3489	ti++;
3490	} else if (q_vector->rx.ring) {
3491	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
3492	fmt: "%s-rx-%u", netdev->name, ri++);
3493	} else if (q_vector->tx.ring) {
3494	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
3495	fmt: "%s-tx-%u", netdev->name, ti++);
3496	} else {
3497	/* skip this unused q_vector */
3498	continue;
3499	}
3500	err = request_irq(irq: entry->vector, handler: &ixgbe_msix_clean_rings, flags: 0,
3501	name: q_vector->name, dev: q_vector);
3502	if (err) {
3503	e_err(probe, "request_irq failed for MSIX interrupt "
3504	"Error: %d\n", err);
3505	goto free_queue_irqs;
3506	}
3507	/* If Flow Director is enabled, set interrupt affinity */
3508	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
3509	/* assign the mask for this irq */
3510	irq_update_affinity_hint(irq: entry->vector,
3511	m: &q_vector->affinity_mask);
3512	}
3513	}
3514
3515	err = request_irq(irq: adapter->msix_entries[vector].vector,
3516	handler: ixgbe_msix_other, flags: 0, name: netdev->name, dev: adapter);
3517	if (err) {
3518	e_err(probe, "request_irq for msix_other failed: %d\n", err);
3519	goto free_queue_irqs;
3520	}
3521
3522	return 0;
3523
3524	free_queue_irqs:
3525	while (vector) {
3526	vector--;
3527	irq_update_affinity_hint(irq: adapter->msix_entries[vector].vector,
3528	NULL);
3529	free_irq(adapter->msix_entries[vector].vector,
3530	adapter->q_vector[vector]);
3531	}
3532	adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
3533	pci_disable_msix(dev: adapter->pdev);
3534	kfree(objp: adapter->msix_entries);
3535	adapter->msix_entries = NULL;
3536	return err;
3537	}
3538
3539	/**
3540	* ixgbe_intr - legacy mode Interrupt Handler
3541	* @irq: interrupt number
3542	* @data: pointer to a network interface device structure
3543	**/
3544	static irqreturn_t ixgbe_intr(int irq, void *data)
3545	{
3546	struct ixgbe_adapter *adapter = data;
3547	struct ixgbe_hw *hw = &adapter->hw;
3548	struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
3549	u32 eicr;
3550
3551	/*
3552	* Workaround for silicon errata #26 on 82598. Mask the interrupt
3553	* before the read of EICR.
3554	*/
3555	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
3556
3557	/* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
3558	* therefore no explicit interrupt disable is necessary */
3559	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3560	if (!eicr) {
3561	/*
3562	* shared interrupt alert!
3563	* make sure interrupts are enabled because the read will
3564	* have disabled interrupts due to EIAM
3565	* finish the workaround of silicon errata on 82598. Unmask
3566	* the interrupt that we masked before the EICR read.
3567	*/
3568	if (!test_bit(__IXGBE_DOWN, &adapter->state))
3569	ixgbe_irq_enable(adapter, queues: true, flush: true);
3570	return IRQ_NONE; /* Not our interrupt */
3571	}
3572
3573	if (eicr & IXGBE_EICR_LSC)
3574	ixgbe_check_lsc(adapter);
3575
3576	if (eicr & IXGBE_EICR_FW_EVENT)
3577	ixgbe_schedule_fw_event(adapter);
3578
3579	switch (hw->mac.type) {
3580	case ixgbe_mac_82599EB:
3581	ixgbe_check_sfp_event(adapter, eicr);
3582	fallthrough;
3583	case ixgbe_mac_X540:
3584	case ixgbe_mac_X550:
3585	case ixgbe_mac_X550EM_x:
3586	case ixgbe_mac_x550em_a:
3587	case ixgbe_mac_e610:
3588	if (eicr & IXGBE_EICR_ECC) {
3589	e_info(link, "Received ECC Err, initiating reset\n");
3590	set_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
3591	ixgbe_service_event_schedule(adapter);
3592	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
3593	}
3594	ixgbe_check_overtemp_event(adapter, eicr);
3595	break;
3596	default:
3597	break;
3598	}
3599
3600	ixgbe_check_fan_failure(adapter, eicr);
3601	if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
3602	ixgbe_ptp_check_pps_event(adapter);
3603
3604	/* would disable interrupts here but EIAM disabled it */
3605	napi_schedule_irqoff(n: &q_vector->napi);
3606
3607	/*
3608	* re-enable link(maybe) and non-queue interrupts, no flush.
3609	* ixgbe_poll will re-enable the queue interrupts
3610	*/
3611	if (!test_bit(__IXGBE_DOWN, &adapter->state))
3612	ixgbe_irq_enable(adapter, queues: false, flush: false);
3613
3614	return IRQ_HANDLED;
3615	}
3616
3617	/**
3618	* ixgbe_request_irq - initialize interrupts
3619	* @adapter: board private structure
3620	*
3621	* Attempts to configure interrupts using the best available
3622	* capabilities of the hardware and kernel.
3623	**/
3624	static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
3625	{
3626	struct net_device *netdev = adapter->netdev;
3627	int err;
3628
3629	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
3630	err = ixgbe_request_msix_irqs(adapter);
3631	else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
3632	err = request_irq(irq: adapter->pdev->irq, handler: ixgbe_intr, flags: 0,
3633	name: netdev->name, dev: adapter);
3634	else
3635	err = request_irq(irq: adapter->pdev->irq, handler: ixgbe_intr, IRQF_SHARED,
3636	name: netdev->name, dev: adapter);
3637
3638	if (err)
3639	e_err(probe, "request_irq failed, Error %d\n", err);
3640
3641	return err;
3642	}
3643
3644	static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
3645	{
3646	int vector;
3647
3648	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
3649	free_irq(adapter->pdev->irq, adapter);
3650	return;
3651	}
3652
3653	if (!adapter->msix_entries)
3654	return;
3655
3656	for (vector = 0; vector < adapter->num_q_vectors; vector++) {
3657	struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
3658	struct msix_entry *entry = &adapter->msix_entries[vector];
3659
3660	/* free only the irqs that were actually requested */
3661	if (!q_vector->rx.ring && !q_vector->tx.ring)
3662	continue;
3663
3664	/* clear the affinity_mask in the IRQ descriptor */
3665	irq_update_affinity_hint(irq: entry->vector, NULL);
3666
3667	free_irq(entry->vector, q_vector);
3668	}
3669
3670	free_irq(adapter->msix_entries[vector].vector, adapter);
3671	}
3672
3673	/**
3674	* ixgbe_irq_disable - Mask off interrupt generation on the NIC
3675	* @adapter: board private structure
3676	**/
3677	static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
3678	{
3679	switch (adapter->hw.mac.type) {
3680	case ixgbe_mac_82598EB:
3681	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
3682	break;
3683	case ixgbe_mac_82599EB:
3684	case ixgbe_mac_X540:
3685	case ixgbe_mac_X550:
3686	case ixgbe_mac_X550EM_x:
3687	case ixgbe_mac_x550em_a:
3688	case ixgbe_mac_e610:
3689	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
3690	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
3691	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
3692	break;
3693	default:
3694	break;
3695	}
3696	IXGBE_WRITE_FLUSH(&adapter->hw);
3697	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3698	int vector;
3699
3700	for (vector = 0; vector < adapter->num_q_vectors; vector++)
3701	synchronize_irq(irq: adapter->msix_entries[vector].vector);
3702
3703	synchronize_irq(irq: adapter->msix_entries[vector++].vector);
3704	} else {
3705	synchronize_irq(irq: adapter->pdev->irq);
3706	}
3707	}
3708
3709	/**
3710	* ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
3711	* @adapter: board private structure
3712	*
3713	**/
3714	static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
3715	{
3716	struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
3717
3718	ixgbe_write_eitr(q_vector);
3719
3720	ixgbe_set_ivar(adapter, direction: 0, queue: 0, msix_vector: 0);
3721	ixgbe_set_ivar(adapter, direction: 1, queue: 0, msix_vector: 0);
3722
3723	e_info(hw, "Legacy interrupt IVAR setup done\n");
3724	}
3725
3726	/**
3727	* ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
3728	* @adapter: board private structure
3729	* @ring: structure containing ring specific data
3730	*
3731	* Configure the Tx descriptor ring after a reset.
3732	**/
3733	void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
3734	struct ixgbe_ring *ring)
3735	{
3736	struct ixgbe_hw *hw = &adapter->hw;
3737	u64 tdba = ring->dma;
3738	int wait_loop = 10;
3739	u32 txdctl = IXGBE_TXDCTL_ENABLE;
3740	u8 reg_idx = ring->reg_idx;
3741
3742	ring->xsk_pool = NULL;
3743	if (ring_is_xdp(ring))
3744	ring->xsk_pool = ixgbe_xsk_pool(adapter, ring);
3745
3746	/* disable queue to avoid issues while updating state */
3747	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
3748	IXGBE_WRITE_FLUSH(hw);
3749
3750	IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
3751	(tdba & DMA_BIT_MASK(32)));
3752	IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
3753	IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
3754	ring->count * sizeof(union ixgbe_adv_tx_desc));
3755	IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
3756	IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
3757	ring->tail = adapter->io_addr + IXGBE_TDT(reg_idx);
3758
3759	/*
3760	* set WTHRESH to encourage burst writeback, it should not be set
3761	* higher than 1 when:
3762	* - ITR is 0 as it could cause false TX hangs
3763	* - ITR is set to > 100k int/sec and BQL is enabled
3764	*
3765	* In order to avoid issues WTHRESH + PTHRESH should always be equal
3766	* to or less than the number of on chip descriptors, which is
3767	* currently 40.
3768	*/
3769	if (!ring->q_vector || (ring->q_vector->itr < IXGBE_100K_ITR))
3770	txdctl |= 1u << 16; /* WTHRESH = 1 */
3771	else
3772	txdctl |= 8u << 16; /* WTHRESH = 8 */
3773
3774	/*
3775	* Setting PTHRESH to 32 both improves performance
3776	* and avoids a TX hang with DFP enabled
3777	*/
3778	txdctl |= (1u << 8) | /* HTHRESH = 1 */
3779	32; /* PTHRESH = 32 */
3780
3781	/* reinitialize flowdirector state */
3782	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
3783	ring->atr_sample_rate = adapter->atr_sample_rate;
3784	ring->atr_count = 0;
3785	set_bit(nr: __IXGBE_TX_FDIR_INIT_DONE, addr: &ring->state);
3786	} else {
3787	ring->atr_sample_rate = 0;
3788	}
3789
3790	/* initialize XPS */
3791	if (!test_and_set_bit(nr: __IXGBE_TX_XPS_INIT_DONE, addr: &ring->state)) {
3792	struct ixgbe_q_vector *q_vector = ring->q_vector;
3793
3794	if (q_vector)
3795	netif_set_xps_queue(dev: ring->netdev,
3796	mask: &q_vector->affinity_mask,
3797	index: ring->queue_index);
3798	}
3799
3800	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED, addr: &ring->state);
3801
3802	/* reinitialize tx_buffer_info */
3803	memset(ring->tx_buffer_info, 0,
3804	sizeof(struct ixgbe_tx_buffer) * ring->count);
3805
3806	/* enable queue */
3807	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
3808
3809	/* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
3810	if (hw->mac.type == ixgbe_mac_82598EB &&
3811	!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3812	return;
3813
3814	/* poll to verify queue is enabled */
3815	do {
3816	usleep_range(min: 1000, max: 2000);
3817	txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
3818	} while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
3819	if (!wait_loop)
3820	hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
3821	}
3822
3823	static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
3824	{
3825	struct ixgbe_hw *hw = &adapter->hw;
3826	u32 rttdcs, mtqc;
3827	u8 tcs = adapter->hw_tcs;
3828
3829	if (hw->mac.type == ixgbe_mac_82598EB)
3830	return;
3831
3832	/* disable the arbiter while setting MTQC */
3833	rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3834	rttdcs |= IXGBE_RTTDCS_ARBDIS;
3835	IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3836
3837	/* set transmit pool layout */
3838	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
3839	mtqc = IXGBE_MTQC_VT_ENA;
3840	if (tcs > 4)
3841	mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3842	else if (tcs > 1)
3843	mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3844	else if (adapter->ring_feature[RING_F_VMDQ].mask ==
3845	IXGBE_82599_VMDQ_4Q_MASK)
3846	mtqc |= IXGBE_MTQC_32VF;
3847	else
3848	mtqc |= IXGBE_MTQC_64VF;
3849	} else {
3850	if (tcs > 4) {
3851	mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
3852	} else if (tcs > 1) {
3853	mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3854	} else {
3855	u8 max_txq = adapter->num_tx_queues +
3856	adapter->num_xdp_queues;
3857	if (max_txq > 63)
3858	mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
3859	else
3860	mtqc = IXGBE_MTQC_64Q_1PB;
3861	}
3862	}
3863
3864	IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
3865
3866	/* Enable Security TX Buffer IFG for multiple pb */
3867	if (tcs) {
3868	u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
3869	sectx |= IXGBE_SECTX_DCB;
3870	IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
3871	}
3872
3873	/* re-enable the arbiter */
3874	rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3875	IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3876	}
3877
3878	/**
3879	* ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
3880	* @adapter: board private structure
3881	*
3882	* Configure the Tx unit of the MAC after a reset.
3883	**/
3884	static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
3885	{
3886	struct ixgbe_hw *hw = &adapter->hw;
3887	u32 dmatxctl;
3888	u32 i;
3889
3890	ixgbe_setup_mtqc(adapter);
3891
3892	if (hw->mac.type != ixgbe_mac_82598EB) {
3893	/* DMATXCTL.EN must be before Tx queues are enabled */
3894	dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3895	dmatxctl |= IXGBE_DMATXCTL_TE;
3896	IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3897	}
3898
3899	/* Setup the HW Tx Head and Tail descriptor pointers */
3900	for (i = 0; i < adapter->num_tx_queues; i++)
3901	ixgbe_configure_tx_ring(adapter, ring: adapter->tx_ring[i]);
3902	for (i = 0; i < adapter->num_xdp_queues; i++)
3903	ixgbe_configure_tx_ring(adapter, ring: adapter->xdp_ring[i]);
3904	}
3905
3906	static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
3907	struct ixgbe_ring *ring)
3908	{
3909	struct ixgbe_hw *hw = &adapter->hw;
3910	u8 reg_idx = ring->reg_idx;
3911	u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
3912
3913	srrctl |= IXGBE_SRRCTL_DROP_EN;
3914
3915	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
3916	}
3917
3918	static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
3919	struct ixgbe_ring *ring)
3920	{
3921	struct ixgbe_hw *hw = &adapter->hw;
3922	u8 reg_idx = ring->reg_idx;
3923	u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
3924
3925	srrctl &= ~IXGBE_SRRCTL_DROP_EN;
3926
3927	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
3928	}
3929
3930	#ifdef CONFIG_IXGBE_DCB
3931	void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
3932	#else
3933	static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
3934	#endif
3935	{
3936	int i;
3937	bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
3938
3939	if (adapter->ixgbe_ieee_pfc)
3940	pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
3941
3942	/*
3943	* We should set the drop enable bit if:
3944	* SR-IOV is enabled
3945	* or
3946	* Number of Rx queues > 1 and flow control is disabled
3947	*
3948	* This allows us to avoid head of line blocking for security
3949	* and performance reasons.
3950	*/
3951	if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
3952	!(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
3953	for (i = 0; i < adapter->num_rx_queues; i++)
3954	ixgbe_enable_rx_drop(adapter, ring: adapter->rx_ring[i]);
3955	} else {
3956	for (i = 0; i < adapter->num_rx_queues; i++)
3957	ixgbe_disable_rx_drop(adapter, ring: adapter->rx_ring[i]);
3958	}
3959	}
3960
3961	#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3962
3963	static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
3964	struct ixgbe_ring *rx_ring)
3965	{
3966	struct ixgbe_hw *hw = &adapter->hw;
3967	u32 srrctl;
3968	u8 reg_idx = rx_ring->reg_idx;
3969
3970	if (hw->mac.type == ixgbe_mac_82598EB) {
3971	u16 mask = adapter->ring_feature[RING_F_RSS].mask;
3972
3973	/*
3974	* if VMDq is not active we must program one srrctl register
3975	* per RSS queue since we have enabled RDRXCTL.MVMEN
3976	*/
3977	reg_idx &= mask;
3978	}
3979
3980	/* configure header buffer length, needed for RSC */
3981	srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
3982
3983	/* configure the packet buffer length */
3984	if (rx_ring->xsk_pool) {
3985	u32 xsk_buf_len = xsk_pool_get_rx_frame_size(pool: rx_ring->xsk_pool);
3986
3987	/* If the MAC support setting RXDCTL.RLPML, the
3988	* SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and
3989	* RXDCTL.RLPML is set to the actual UMEM buffer
3990	* size. If not, then we are stuck with a 1k buffer
3991	* size resolution. In this case frames larger than
3992	* the UMEM buffer size viewed in a 1k resolution will
3993	* be dropped.
3994	*/
3995	if (hw->mac.type != ixgbe_mac_82599EB)
3996	srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3997	else
3998	srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
3999	} else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) {
4000	srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4001	} else {
4002	srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4003	}
4004
4005	/* configure descriptor type */
4006	srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4007
4008	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
4009	}
4010
4011	/**
4012	* ixgbe_rss_indir_tbl_entries - Return RSS indirection table entries
4013	* @adapter: device handle
4014	*
4015	* - 82598/82599/X540: 128
4016	* - X550(non-SRIOV mode): 512
4017	* - X550(SRIOV mode): 64
4018	*/
4019	u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter)
4020	{
4021	if (adapter->hw.mac.type < ixgbe_mac_X550)
4022	return 128;
4023	else if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
4024	return 64;
4025	else
4026	return 512;
4027	}
4028
4029	/**
4030	* ixgbe_store_key - Write the RSS key to HW
4031	* @adapter: device handle
4032	*
4033	* Write the RSS key stored in adapter.rss_key to HW.
4034	*/
4035	void ixgbe_store_key(struct ixgbe_adapter *adapter)
4036	{
4037	struct ixgbe_hw *hw = &adapter->hw;
4038	int i;
4039
4040	for (i = 0; i < 10; i++)
4041	IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]);
4042	}
4043
4044	/**
4045	* ixgbe_init_rss_key - Initialize adapter RSS key
4046	* @adapter: device handle
4047	*
4048	* Allocates and initializes the RSS key if it is not allocated.
4049	**/
4050	static inline int ixgbe_init_rss_key(struct ixgbe_adapter *adapter)
4051	{
4052	u32 *rss_key;
4053
4054	if (!adapter->rss_key) {
4055	rss_key = kzalloc(IXGBE_RSS_KEY_SIZE, GFP_KERNEL);
4056	if (unlikely(!rss_key))
4057	return -ENOMEM;
4058
4059	netdev_rss_key_fill(buffer: rss_key, IXGBE_RSS_KEY_SIZE);
4060	adapter->rss_key = rss_key;
4061	}
4062
4063	return 0;
4064	}
4065
4066	/**
4067	* ixgbe_store_reta - Write the RETA table to HW
4068	* @adapter: device handle
4069	*
4070	* Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
4071	*/
4072	void ixgbe_store_reta(struct ixgbe_adapter *adapter)
4073	{
4074	u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
4075	struct ixgbe_hw *hw = &adapter->hw;
4076	u32 reta = 0;
4077	u32 indices_multi;
4078	u8 *indir_tbl = adapter->rss_indir_tbl;
4079
4080	/* Fill out the redirection table as follows:
4081	* - 82598: 8 bit wide entries containing pair of 4 bit RSS
4082	* indices.
4083	* - 82599/X540: 8 bit wide entries containing 4 bit RSS index
4084	* - X550: 8 bit wide entries containing 6 bit RSS index
4085	*/
4086	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
4087	indices_multi = 0x11;
4088	else
4089	indices_multi = 0x1;
4090
4091	/* Write redirection table to HW */
4092	for (i = 0; i < reta_entries; i++) {
4093	reta |= indices_multi * indir_tbl[i] << (i & 0x3) * 8;
4094	if ((i & 3) == 3) {
4095	if (i < 128)
4096	IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4097	else
4098	IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
4099	reta);
4100	reta = 0;
4101	}
4102	}
4103	}
4104
4105	/**
4106	* ixgbe_store_vfreta - Write the RETA table to HW (x550 devices in SRIOV mode)
4107	* @adapter: device handle
4108	*
4109	* Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
4110	*/
4111	static void ixgbe_store_vfreta(struct ixgbe_adapter *adapter)
4112	{
4113	u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
4114	struct ixgbe_hw *hw = &adapter->hw;
4115	u32 vfreta = 0;
4116
4117	/* Write redirection table to HW */
4118	for (i = 0; i < reta_entries; i++) {
4119	u16 pool = adapter->num_rx_pools;
4120
4121	vfreta |= (u32)adapter->rss_indir_tbl[i] << (i & 0x3) * 8;
4122	if ((i & 3) != 3)
4123	continue;
4124
4125	while (pool--)
4126	IXGBE_WRITE_REG(hw,
4127	IXGBE_PFVFRETA(i >> 2, VMDQ_P(pool)),
4128	vfreta);
4129	vfreta = 0;
4130	}
4131	}
4132
4133	static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)
4134	{
4135	u32 i, j;
4136	u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
4137	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
4138
4139	/* Program table for at least 4 queues w/ SR-IOV so that VFs can
4140	* make full use of any rings they may have. We will use the
4141	* PSRTYPE register to control how many rings we use within the PF.
4142	*/
4143	if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 4))
4144	rss_i = 4;
4145
4146	/* Fill out hash function seeds */
4147	ixgbe_store_key(adapter);
4148
4149	/* Fill out redirection table */
4150	memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl));
4151
4152	for (i = 0, j = 0; i < reta_entries; i++, j++) {
4153	if (j == rss_i)
4154	j = 0;
4155
4156	adapter->rss_indir_tbl[i] = j;
4157	}
4158
4159	ixgbe_store_reta(adapter);
4160	}
4161
4162	static void ixgbe_setup_vfreta(struct ixgbe_adapter *adapter)
4163	{
4164	struct ixgbe_hw *hw = &adapter->hw;
4165	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
4166	int i, j;
4167
4168	/* Fill out hash function seeds */
4169	for (i = 0; i < 10; i++) {
4170	u16 pool = adapter->num_rx_pools;
4171
4172	while (pool--)
4173	IXGBE_WRITE_REG(hw,
4174	IXGBE_PFVFRSSRK(i, VMDQ_P(pool)),
4175	*(adapter->rss_key + i));
4176	}
4177
4178	/* Fill out the redirection table */
4179	for (i = 0, j = 0; i < 64; i++, j++) {
4180	if (j == rss_i)
4181	j = 0;
4182
4183	adapter->rss_indir_tbl[i] = j;
4184	}
4185
4186	ixgbe_store_vfreta(adapter);
4187	}
4188
4189	static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
4190	{
4191	struct ixgbe_hw *hw = &adapter->hw;
4192	u32 mrqc = 0, rss_field = 0, vfmrqc = 0;
4193	u32 rxcsum;
4194
4195	/* Disable indicating checksum in descriptor, enables RSS hash */
4196	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4197	rxcsum |= IXGBE_RXCSUM_PCSD;
4198	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4199
4200	if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4201	if (adapter->ring_feature[RING_F_RSS].mask)
4202	mrqc = IXGBE_MRQC_RSSEN;
4203	} else {
4204	u8 tcs = adapter->hw_tcs;
4205
4206	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
4207	if (tcs > 4)
4208	mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */
4209	else if (tcs > 1)
4210	mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */
4211	else if (adapter->ring_feature[RING_F_VMDQ].mask ==
4212	IXGBE_82599_VMDQ_4Q_MASK)
4213	mrqc = IXGBE_MRQC_VMDQRSS32EN;
4214	else
4215	mrqc = IXGBE_MRQC_VMDQRSS64EN;
4216
4217	/* Enable L3/L4 for Tx Switched packets only for X550,
4218	* older devices do not support this feature
4219	*/
4220	if (hw->mac.type >= ixgbe_mac_X550)
4221	mrqc |= IXGBE_MRQC_L3L4TXSWEN;
4222	} else {
4223	if (tcs > 4)
4224	mrqc = IXGBE_MRQC_RTRSS8TCEN;
4225	else if (tcs > 1)
4226	mrqc = IXGBE_MRQC_RTRSS4TCEN;
4227	else
4228	mrqc = IXGBE_MRQC_RSSEN;
4229	}
4230	}
4231
4232	/* Perform hash on these packet types */
4233	rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 |
4234	IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
4235	IXGBE_MRQC_RSS_FIELD_IPV6 |
4236	IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
4237
4238	if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)
4239	rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
4240	if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)
4241	rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
4242
4243	if ((hw->mac.type >= ixgbe_mac_X550) &&
4244	(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) {
4245	u16 pool = adapter->num_rx_pools;
4246
4247	/* Enable VF RSS mode */
4248	mrqc |= IXGBE_MRQC_MULTIPLE_RSS;
4249	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4250
4251	/* Setup RSS through the VF registers */
4252	ixgbe_setup_vfreta(adapter);
4253	vfmrqc = IXGBE_MRQC_RSSEN;
4254	vfmrqc |= rss_field;
4255
4256	while (pool--)
4257	IXGBE_WRITE_REG(hw,
4258	IXGBE_PFVFMRQC(VMDQ_P(pool)),
4259	vfmrqc);
4260	} else {
4261	ixgbe_setup_reta(adapter);
4262	mrqc |= rss_field;
4263	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4264	}
4265	}
4266
4267	/**
4268	* ixgbe_configure_rscctl - enable RSC for the indicated ring
4269	* @adapter: address of board private structure
4270	* @ring: structure containing ring specific data
4271	**/
4272	static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
4273	struct ixgbe_ring *ring)
4274	{
4275	struct ixgbe_hw *hw = &adapter->hw;
4276	u32 rscctrl;
4277	u8 reg_idx = ring->reg_idx;
4278
4279	if (!ring_is_rsc_enabled(ring))
4280	return;
4281
4282	rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
4283	rscctrl |= IXGBE_RSCCTL_RSCEN;
4284	/*
4285	* we must limit the number of descriptors so that the
4286	* total size of max desc * buf_len is not greater
4287	* than 65536
4288	*/
4289	rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
4290	IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
4291	}
4292
4293	#define IXGBE_MAX_RX_DESC_POLL 10
4294	static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
4295	struct ixgbe_ring *ring)
4296	{
4297	struct ixgbe_hw *hw = &adapter->hw;
4298	int wait_loop = IXGBE_MAX_RX_DESC_POLL;
4299	u32 rxdctl;
4300	u8 reg_idx = ring->reg_idx;
4301
4302	if (ixgbe_removed(addr: hw->hw_addr))
4303	return;
4304	/* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
4305	if (hw->mac.type == ixgbe_mac_82598EB &&
4306	!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
4307	return;
4308
4309	do {
4310	usleep_range(min: 1000, max: 2000);
4311	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
4312	} while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
4313
4314	if (!wait_loop) {
4315	e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
4316	"the polling period\n", reg_idx);
4317	}
4318	}
4319
4320	void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
4321	struct ixgbe_ring *ring)
4322	{
4323	struct ixgbe_hw *hw = &adapter->hw;
4324	union ixgbe_adv_rx_desc *rx_desc;
4325	u64 rdba = ring->dma;
4326	u32 rxdctl;
4327	u8 reg_idx = ring->reg_idx;
4328
4329	xdp_rxq_info_unreg_mem_model(xdp_rxq: &ring->xdp_rxq);
4330	ring->xsk_pool = ixgbe_xsk_pool(adapter, ring);
4331	if (ring->xsk_pool) {
4332	WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
4333	MEM_TYPE_XSK_BUFF_POOL,
4334	NULL));
4335	xsk_pool_set_rxq_info(pool: ring->xsk_pool, rxq: &ring->xdp_rxq);
4336	} else {
4337	WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
4338	MEM_TYPE_PAGE_SHARED, NULL));
4339	}
4340
4341	/* disable queue to avoid use of these values while updating state */
4342	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
4343	rxdctl &= ~IXGBE_RXDCTL_ENABLE;
4344
4345	/* write value back with RXDCTL.ENABLE bit cleared */
4346	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
4347	IXGBE_WRITE_FLUSH(hw);
4348
4349	IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
4350	IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
4351	IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
4352	ring->count * sizeof(union ixgbe_adv_rx_desc));
4353	/* Force flushing of IXGBE_RDLEN to prevent MDD */
4354	IXGBE_WRITE_FLUSH(hw);
4355
4356	IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
4357	IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
4358	ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx);
4359
4360	ixgbe_configure_srrctl(adapter, rx_ring: ring);
4361	ixgbe_configure_rscctl(adapter, ring);
4362
4363	if (hw->mac.type == ixgbe_mac_82598EB) {
4364	/*
4365	* enable cache line friendly hardware writes:
4366	* PTHRESH=32 descriptors (half the internal cache),
4367	* this also removes ugly rx_no_buffer_count increment
4368	* HTHRESH=4 descriptors (to minimize latency on fetch)
4369	* WTHRESH=8 burst writeback up to two cache lines
4370	*/
4371	rxdctl &= ~0x3FFFFF;
4372	rxdctl |= 0x080420;
4373	#if (PAGE_SIZE < 8192)
4374	/* RXDCTL.RLPML does not work on 82599 */
4375	} else if (hw->mac.type != ixgbe_mac_82599EB) {
4376	rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
4377	IXGBE_RXDCTL_RLPML_EN);
4378
4379	/* Limit the maximum frame size so we don't overrun the skb.
4380	* This can happen in SRIOV mode when the MTU of the VF is
4381	* higher than the MTU of the PF.
4382	*/
4383	if (ring_uses_build_skb(ring) &&
4384	!test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
4385	rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB |
4386	IXGBE_RXDCTL_RLPML_EN;
4387	#endif
4388	}
4389
4390	ring->rx_offset = ixgbe_rx_offset(rx_ring: ring);
4391
4392	if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) {
4393	u32 xsk_buf_len = xsk_pool_get_rx_frame_size(pool: ring->xsk_pool);
4394
4395	rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
4396	IXGBE_RXDCTL_RLPML_EN);
4397	rxdctl |= xsk_buf_len | IXGBE_RXDCTL_RLPML_EN;
4398
4399	ring->rx_buf_len = xsk_buf_len;
4400	}
4401
4402	/* initialize rx_buffer_info */
4403	memset(ring->rx_buffer_info, 0,
4404	sizeof(struct ixgbe_rx_buffer) * ring->count);
4405
4406	/* initialize Rx descriptor 0 */
4407	rx_desc = IXGBE_RX_DESC(ring, 0);
4408	rx_desc->wb.upper.length = 0;
4409
4410	/* enable receive descriptor ring */
4411	rxdctl |= IXGBE_RXDCTL_ENABLE;
4412	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
4413
4414	ixgbe_rx_desc_queue_enable(adapter, ring);
4415	if (ring->xsk_pool)
4416	ixgbe_alloc_rx_buffers_zc(rx_ring: ring, cleaned_count: ixgbe_desc_unused(ring));
4417	else
4418	ixgbe_alloc_rx_buffers(rx_ring: ring, cleaned_count: ixgbe_desc_unused(ring));
4419	}
4420
4421	static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
4422	{
4423	struct ixgbe_hw *hw = &adapter->hw;
4424	int rss_i = adapter->ring_feature[RING_F_RSS].indices;
4425	u16 pool = adapter->num_rx_pools;
4426
4427	/* PSRTYPE must be initialized in non 82598 adapters */
4428	u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4429	IXGBE_PSRTYPE_UDPHDR |
4430	IXGBE_PSRTYPE_IPV4HDR |
4431	IXGBE_PSRTYPE_L2HDR |
4432	IXGBE_PSRTYPE_IPV6HDR;
4433
4434	if (hw->mac.type == ixgbe_mac_82598EB)
4435	return;
4436
4437	if (rss_i > 3)
4438	psrtype |= 2u << 29;
4439	else if (rss_i > 1)
4440	psrtype |= 1u << 29;
4441
4442	while (pool--)
4443	IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype);
4444	}
4445
4446	static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
4447	{
4448	struct ixgbe_hw *hw = &adapter->hw;
4449	u16 pool = adapter->num_rx_pools;
4450	u32 reg_offset, vf_shift, vmolr;
4451	u32 gcr_ext, vmdctl;
4452	int i;
4453
4454	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
4455	return;
4456
4457	vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
4458	vmdctl |= IXGBE_VMD_CTL_VMDQ_EN;
4459	vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
4460	vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT;
4461	vmdctl |= IXGBE_VT_CTL_REPLEN;
4462	IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
4463
4464	/* accept untagged packets until a vlan tag is
4465	* specifically set for the VMDQ queue/pool
4466	*/
4467	vmolr = IXGBE_VMOLR_AUPE;
4468	while (pool--)
4469	IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(pool)), vmolr);
4470
4471	vf_shift = VMDQ_P(0) % 32;
4472	reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0;
4473
4474	/* Enable only the PF's pool for Tx/Rx */
4475	IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), GENMASK(31, vf_shift));
4476	IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1);
4477	IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), GENMASK(31, vf_shift));
4478	IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1);
4479	if (adapter->bridge_mode == BRIDGE_MODE_VEB)
4480	IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
4481
4482	/* Map PF MAC address in RAR Entry 0 to first pool following VFs */
4483	hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0));
4484
4485	/* clear VLAN promisc flag so VFTA will be updated if necessary */
4486	adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;
4487
4488	/*
4489	* Set up VF register offsets for selected VT Mode,
4490	* i.e. 32 or 64 VFs for SR-IOV
4491	*/
4492	switch (adapter->ring_feature[RING_F_VMDQ].mask) {
4493	case IXGBE_82599_VMDQ_8Q_MASK:
4494	gcr_ext = IXGBE_GCR_EXT_VT_MODE_16;
4495	break;
4496	case IXGBE_82599_VMDQ_4Q_MASK:
4497	gcr_ext = IXGBE_GCR_EXT_VT_MODE_32;
4498	break;
4499	default:
4500	gcr_ext = IXGBE_GCR_EXT_VT_MODE_64;
4501	break;
4502	}
4503
4504	IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
4505
4506	for (i = 0; i < adapter->num_vfs; i++) {
4507	/* configure spoof checking */
4508	ixgbe_ndo_set_vf_spoofchk(netdev: adapter->netdev, vf: i,
4509	setting: adapter->vfinfo[i].spoofchk_enabled);
4510
4511	/* Enable/Disable RSS query feature */
4512	ixgbe_ndo_set_vf_rss_query_en(netdev: adapter->netdev, vf: i,
4513	setting: adapter->vfinfo[i].rss_query_enabled);
4514	}
4515	}
4516
4517	static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
4518	{
4519	struct ixgbe_hw *hw = &adapter->hw;
4520	struct net_device *netdev = adapter->netdev;
4521	int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
4522	struct ixgbe_ring *rx_ring;
4523	int i;
4524	u32 mhadd, hlreg0;
4525
4526	#ifdef IXGBE_FCOE
4527	/* adjust max frame to be able to do baby jumbo for FCoE */
4528	if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
4529	(max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
4530	max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
4531
4532	#endif /* IXGBE_FCOE */
4533
4534	/* adjust max frame to be at least the size of a standard frame */
4535	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
4536	max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN);
4537
4538	mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
4539	if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
4540	mhadd &= ~IXGBE_MHADD_MFS_MASK;
4541	mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
4542
4543	IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
4544	}
4545
4546	hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4547	/* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
4548	hlreg0 |= IXGBE_HLREG0_JUMBOEN;
4549	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
4550
4551	/*
4552	* Setup the HW Rx Head and Tail Descriptor Pointers and
4553	* the Base and Length of the Rx Descriptor Ring
4554	*/
4555	for (i = 0; i < adapter->num_rx_queues; i++) {
4556	rx_ring = adapter->rx_ring[i];
4557
4558	clear_ring_rsc_enabled(rx_ring);
4559	clear_bit(nr: __IXGBE_RX_3K_BUFFER, addr: &rx_ring->state);
4560	clear_bit(nr: __IXGBE_RX_BUILD_SKB_ENABLED, addr: &rx_ring->state);
4561
4562	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
4563	set_ring_rsc_enabled(rx_ring);
4564
4565	if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state))
4566	set_bit(nr: __IXGBE_RX_3K_BUFFER, addr: &rx_ring->state);
4567
4568	if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY)
4569	continue;
4570
4571	set_bit(nr: __IXGBE_RX_BUILD_SKB_ENABLED, addr: &rx_ring->state);
4572
4573	#if (PAGE_SIZE < 8192)
4574	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
4575	set_bit(nr: __IXGBE_RX_3K_BUFFER, addr: &rx_ring->state);
4576
4577	if (IXGBE_2K_TOO_SMALL_WITH_PADDING ||
4578	(max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)))
4579	set_bit(nr: __IXGBE_RX_3K_BUFFER, addr: &rx_ring->state);
4580	#endif
4581	}
4582	}
4583
4584	static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
4585	{
4586	struct ixgbe_hw *hw = &adapter->hw;
4587	u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
4588
4589	switch (hw->mac.type) {
4590	case ixgbe_mac_82598EB:
4591	/*
4592	* For VMDq support of different descriptor types or
4593	* buffer sizes through the use of multiple SRRCTL
4594	* registers, RDRXCTL.MVMEN must be set to 1
4595	*
4596	* also, the manual doesn't mention it clearly but DCA hints
4597	* will only use queue 0's tags unless this bit is set. Side
4598	* effects of setting this bit are only that SRRCTL must be
4599	* fully programmed [0..15]
4600	*/
4601	rdrxctl |= IXGBE_RDRXCTL_MVMEN;
4602	break;
4603	case ixgbe_mac_X550:
4604	case ixgbe_mac_X550EM_x:
4605	case ixgbe_mac_x550em_a:
4606	case ixgbe_mac_e610:
4607	if (adapter->num_vfs)
4608	rdrxctl |= IXGBE_RDRXCTL_PSP;
4609	fallthrough;
4610	case ixgbe_mac_82599EB:
4611	case ixgbe_mac_X540:
4612	/* Disable RSC for ACK packets */
4613	IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
4614	(IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
4615	rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
4616	/* hardware requires some bits to be set by default */
4617	rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
4618	rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
4619	break;
4620	default:
4621	/* We should do nothing since we don't know this hardware */
4622	return;
4623	}
4624
4625	IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
4626	}
4627
4628	/**
4629	* ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
4630	* @adapter: board private structure
4631	*
4632	* Configure the Rx unit of the MAC after a reset.
4633	**/
4634	static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
4635	{
4636	struct ixgbe_hw *hw = &adapter->hw;
4637	int i;
4638	u32 rxctrl, rfctl;
4639
4640	/* disable receives while setting up the descriptors */
4641	hw->mac.ops.disable_rx(hw);
4642
4643	ixgbe_setup_psrtype(adapter);
4644	ixgbe_setup_rdrxctl(adapter);
4645
4646	/* RSC Setup */
4647	rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);
4648	rfctl &= ~IXGBE_RFCTL_RSC_DIS;
4649	if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))
4650	rfctl |= IXGBE_RFCTL_RSC_DIS;
4651
4652	/* disable NFS filtering */
4653	rfctl |= (IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS);
4654	IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
4655
4656	/* Program registers for the distribution of queues */
4657	ixgbe_setup_mrqc(adapter);
4658
4659	/* set_rx_buffer_len must be called before ring initialization */
4660	ixgbe_set_rx_buffer_len(adapter);
4661
4662	/*
4663	* Setup the HW Rx Head and Tail Descriptor Pointers and
4664	* the Base and Length of the Rx Descriptor Ring
4665	*/
4666	for (i = 0; i < adapter->num_rx_queues; i++)
4667	ixgbe_configure_rx_ring(adapter, ring: adapter->rx_ring[i]);
4668
4669	rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4670	/* disable drop enable for 82598 parts */
4671	if (hw->mac.type == ixgbe_mac_82598EB)
4672	rxctrl |= IXGBE_RXCTRL_DMBYPS;
4673
4674	/* enable all receives */
4675	rxctrl |= IXGBE_RXCTRL_RXEN;
4676	hw->mac.ops.enable_rx_dma(hw, rxctrl);
4677	}
4678
4679	static int ixgbe_vlan_rx_add_vid(struct net_device *netdev,
4680	__be16 proto, u16 vid)
4681	{
4682	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
4683	struct ixgbe_hw *hw = &adapter->hw;
4684
4685	/* add VID to filter table */
4686	if (!vid || !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4687	hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true, !!vid);
4688
4689	set_bit(nr: vid, addr: adapter->active_vlans);
4690
4691	return 0;
4692	}
4693
4694	static int ixgbe_find_vlvf_entry(struct ixgbe_hw *hw, u32 vlan)
4695	{
4696	u32 vlvf;
4697	int idx;
4698
4699	/* short cut the special case */
4700	if (vlan == 0)
4701	return 0;
4702
4703	/* Search for the vlan id in the VLVF entries */
4704	for (idx = IXGBE_VLVF_ENTRIES; --idx;) {
4705	vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(idx));
4706	if ((vlvf & VLAN_VID_MASK) == vlan)
4707	break;
4708	}
4709
4710	return idx;
4711	}
4712
4713	void ixgbe_update_pf_promisc_vlvf(struct ixgbe_adapter *adapter, u32 vid)
4714	{
4715	struct ixgbe_hw *hw = &adapter->hw;
4716	u32 bits, word;
4717	int idx;
4718
4719	idx = ixgbe_find_vlvf_entry(hw, vlan: vid);
4720	if (!idx)
4721	return;
4722
4723	/* See if any other pools are set for this VLAN filter
4724	* entry other than the PF.
4725	*/
4726	word = idx * 2 + (VMDQ_P(0) / 32);
4727	bits = ~BIT(VMDQ_P(0) % 32);
4728	bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));
4729
4730	/* Disable the filter so this falls into the default pool. */
4731	if (!bits && !IXGBE_READ_REG(hw, IXGBE_VLVFB(word ^ 1))) {
4732	if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4733	IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), 0);
4734	IXGBE_WRITE_REG(hw, IXGBE_VLVF(idx), 0);
4735	}
4736	}
4737
4738	static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev,
4739	__be16 proto, u16 vid)
4740	{
4741	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
4742	struct ixgbe_hw *hw = &adapter->hw;
4743
4744	/* remove VID from filter table */
4745	if (vid && !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4746	hw->mac.ops.set_vfta(hw, vid, VMDQ_P(0), false, true);
4747
4748	clear_bit(nr: vid, addr: adapter->active_vlans);
4749
4750	return 0;
4751	}
4752
4753	/**
4754	* ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
4755	* @adapter: driver data
4756	*/
4757	static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
4758	{
4759	struct ixgbe_hw *hw = &adapter->hw;
4760	u32 vlnctrl;
4761	int i, j;
4762
4763	switch (hw->mac.type) {
4764	case ixgbe_mac_82598EB:
4765	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4766	vlnctrl &= ~IXGBE_VLNCTRL_VME;
4767	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4768	break;
4769	case ixgbe_mac_82599EB:
4770	case ixgbe_mac_X540:
4771	case ixgbe_mac_X550:
4772	case ixgbe_mac_X550EM_x:
4773	case ixgbe_mac_x550em_a:
4774	case ixgbe_mac_e610:
4775	for (i = 0; i < adapter->num_rx_queues; i++) {
4776	struct ixgbe_ring *ring = adapter->rx_ring[i];
4777
4778	if (!netif_is_ixgbe(dev: ring->netdev))
4779	continue;
4780
4781	j = ring->reg_idx;
4782	vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
4783	vlnctrl &= ~IXGBE_RXDCTL_VME;
4784	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
4785	}
4786	break;
4787	default:
4788	break;
4789	}
4790	}
4791
4792	/**
4793	* ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
4794	* @adapter: driver data
4795	*/
4796	static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
4797	{
4798	struct ixgbe_hw *hw = &adapter->hw;
4799	u32 vlnctrl;
4800	int i, j;
4801
4802	switch (hw->mac.type) {
4803	case ixgbe_mac_82598EB:
4804	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4805	vlnctrl |= IXGBE_VLNCTRL_VME;
4806	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4807	break;
4808	case ixgbe_mac_82599EB:
4809	case ixgbe_mac_X540:
4810	case ixgbe_mac_X550:
4811	case ixgbe_mac_X550EM_x:
4812	case ixgbe_mac_x550em_a:
4813	case ixgbe_mac_e610:
4814	for (i = 0; i < adapter->num_rx_queues; i++) {
4815	struct ixgbe_ring *ring = adapter->rx_ring[i];
4816
4817	if (!netif_is_ixgbe(dev: ring->netdev))
4818	continue;
4819
4820	j = ring->reg_idx;
4821	vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
4822	vlnctrl |= IXGBE_RXDCTL_VME;
4823	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
4824	}
4825	break;
4826	default:
4827	break;
4828	}
4829	}
4830
4831	static void ixgbe_vlan_promisc_enable(struct ixgbe_adapter *adapter)
4832	{
4833	struct ixgbe_hw *hw = &adapter->hw;
4834	u32 vlnctrl, i;
4835
4836	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4837
4838	if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) {
4839	/* For VMDq and SR-IOV we must leave VLAN filtering enabled */
4840	vlnctrl |= IXGBE_VLNCTRL_VFE;
4841	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4842	} else {
4843	vlnctrl &= ~IXGBE_VLNCTRL_VFE;
4844	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4845	return;
4846	}
4847
4848	/* Nothing to do for 82598 */
4849	if (hw->mac.type == ixgbe_mac_82598EB)
4850	return;
4851
4852	/* We are already in VLAN promisc, nothing to do */
4853	if (adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)
4854	return;
4855
4856	/* Set flag so we don't redo unnecessary work */
4857	adapter->flags2 |= IXGBE_FLAG2_VLAN_PROMISC;
4858
4859	/* Add PF to all active pools */
4860	for (i = IXGBE_VLVF_ENTRIES; --i;) {
4861	u32 reg_offset = IXGBE_VLVFB(i * 2 + VMDQ_P(0) / 32);
4862	u32 vlvfb = IXGBE_READ_REG(hw, reg_offset);
4863
4864	vlvfb |= BIT(VMDQ_P(0) % 32);
4865	IXGBE_WRITE_REG(hw, reg_offset, vlvfb);
4866	}
4867
4868	/* Set all bits in the VLAN filter table array */
4869	for (i = hw->mac.vft_size; i--;)
4870	IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), ~0U);
4871	}
4872
4873	#define VFTA_BLOCK_SIZE 8
4874	static void ixgbe_scrub_vfta(struct ixgbe_adapter *adapter, u32 vfta_offset)
4875	{
4876	struct ixgbe_hw *hw = &adapter->hw;
4877	u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
4878	u32 vid_start = vfta_offset * 32;
4879	u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
4880	u32 i, vid, word, bits;
4881
4882	for (i = IXGBE_VLVF_ENTRIES; --i;) {
4883	u32 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(i));
4884
4885	/* pull VLAN ID from VLVF */
4886	vid = vlvf & VLAN_VID_MASK;
4887
4888	/* only concern outselves with a certain range */
4889	if (vid < vid_start || vid >= vid_end)
4890	continue;
4891
4892	if (vlvf) {
4893	/* record VLAN ID in VFTA */
4894	vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
4895
4896	/* if PF is part of this then continue */
4897	if (test_bit(vid, adapter->active_vlans))
4898	continue;
4899	}
4900
4901	/* remove PF from the pool */
4902	word = i * 2 + VMDQ_P(0) / 32;
4903	bits = ~BIT(VMDQ_P(0) % 32);
4904	bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));
4905	IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), bits);
4906	}
4907
4908	/* extract values from active_vlans and write back to VFTA */
4909	for (i = VFTA_BLOCK_SIZE; i--;) {
4910	vid = (vfta_offset + i) * 32;
4911	word = vid / BITS_PER_LONG;
4912	bits = vid % BITS_PER_LONG;
4913
4914	vfta[i] |= adapter->active_vlans[word] >> bits;
4915
4916	IXGBE_WRITE_REG(hw, IXGBE_VFTA(vfta_offset + i), vfta[i]);
4917	}
4918	}
4919
4920	static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter)
4921	{
4922	struct ixgbe_hw *hw = &adapter->hw;
4923	u32 vlnctrl, i;
4924
4925	/* Set VLAN filtering to enabled */
4926	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4927	vlnctrl |= IXGBE_VLNCTRL_VFE;
4928	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4929
4930	if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) ||
4931	hw->mac.type == ixgbe_mac_82598EB)
4932	return;
4933
4934	/* We are not in VLAN promisc, nothing to do */
4935	if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4936	return;
4937
4938	/* Set flag so we don't redo unnecessary work */
4939	adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;
4940
4941	for (i = 0; i < hw->mac.vft_size; i += VFTA_BLOCK_SIZE)
4942	ixgbe_scrub_vfta(adapter, vfta_offset: i);
4943	}
4944
4945	static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
4946	{
4947	u16 vid = 1;
4948
4949	ixgbe_vlan_rx_add_vid(netdev: adapter->netdev, htons(ETH_P_8021Q), vid: 0);
4950
4951	for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
4952	ixgbe_vlan_rx_add_vid(netdev: adapter->netdev, htons(ETH_P_8021Q), vid);
4953	}
4954
4955	/**
4956	* ixgbe_write_mc_addr_list - write multicast addresses to MTA
4957	* @netdev: network interface device structure
4958	*
4959	* Writes multicast address list to the MTA hash table.
4960	* Returns: -ENOMEM on failure
4961	* 0 on no addresses written
4962	* X on writing X addresses to MTA
4963	**/
4964	static int ixgbe_write_mc_addr_list(struct net_device *netdev)
4965	{
4966	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
4967	struct ixgbe_hw *hw = &adapter->hw;
4968
4969	if (!netif_running(dev: netdev))
4970	return 0;
4971
4972	if (hw->mac.ops.update_mc_addr_list)
4973	hw->mac.ops.update_mc_addr_list(hw, netdev);
4974	else
4975	return -ENOMEM;
4976
4977	#ifdef CONFIG_PCI_IOV
4978	ixgbe_restore_vf_multicasts(adapter);
4979	#endif
4980
4981	return netdev_mc_count(netdev);
4982	}
4983
4984	#ifdef CONFIG_PCI_IOV
4985	void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter)
4986	{
4987	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
4988	struct ixgbe_hw *hw = &adapter->hw;
4989	int i;
4990
4991	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
4992	mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;
4993
4994	if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
4995	hw->mac.ops.set_rar(hw, i,
4996	mac_table->addr,
4997	mac_table->pool,
4998	IXGBE_RAH_AV);
4999	else
5000	hw->mac.ops.clear_rar(hw, i);
5001	}
5002	}
5003
5004	#endif
5005	static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter)
5006	{
5007	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5008	struct ixgbe_hw *hw = &adapter->hw;
5009	int i;
5010
5011	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5012	if (!(mac_table->state & IXGBE_MAC_STATE_MODIFIED))
5013	continue;
5014
5015	mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;
5016
5017	if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
5018	hw->mac.ops.set_rar(hw, i,
5019	mac_table->addr,
5020	mac_table->pool,
5021	IXGBE_RAH_AV);
5022	else
5023	hw->mac.ops.clear_rar(hw, i);
5024	}
5025	}
5026
5027	static void ixgbe_flush_sw_mac_table(struct ixgbe_adapter *adapter)
5028	{
5029	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5030	struct ixgbe_hw *hw = &adapter->hw;
5031	int i;
5032
5033	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5034	mac_table->state |= IXGBE_MAC_STATE_MODIFIED;
5035	mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;
5036	}
5037
5038	ixgbe_sync_mac_table(adapter);
5039	}
5040
5041	static int ixgbe_available_rars(struct ixgbe_adapter *adapter, u16 pool)
5042	{
5043	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5044	struct ixgbe_hw *hw = &adapter->hw;
5045	int i, count = 0;
5046
5047	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5048	/* do not count default RAR as available */
5049	if (mac_table->state & IXGBE_MAC_STATE_DEFAULT)
5050	continue;
5051
5052	/* only count unused and addresses that belong to us */
5053	if (mac_table->state & IXGBE_MAC_STATE_IN_USE) {
5054	if (mac_table->pool != pool)
5055	continue;
5056	}
5057
5058	count++;
5059	}
5060
5061	return count;
5062	}
5063
5064	/* this function destroys the first RAR entry */
5065	static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter)
5066	{
5067	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5068	struct ixgbe_hw *hw = &adapter->hw;
5069
5070	memcpy(&mac_table->addr, hw->mac.addr, ETH_ALEN);
5071	mac_table->pool = VMDQ_P(0);
5072
5073	mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE;
5074
5075	hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool,
5076	IXGBE_RAH_AV);
5077	}
5078
5079	int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter,
5080	const u8 *addr, u16 pool)
5081	{
5082	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5083	struct ixgbe_hw *hw = &adapter->hw;
5084	int i;
5085
5086	if (is_zero_ether_addr(addr))
5087	return -EINVAL;
5088
5089	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5090	if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
5091	continue;
5092
5093	ether_addr_copy(dst: mac_table->addr, src: addr);
5094	mac_table->pool = pool;
5095
5096	mac_table->state |= IXGBE_MAC_STATE_MODIFIED |
5097	IXGBE_MAC_STATE_IN_USE;
5098
5099	ixgbe_sync_mac_table(adapter);
5100
5101	return i;
5102	}
5103
5104	return -ENOMEM;
5105	}
5106
5107	int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter,
5108	const u8 *addr, u16 pool)
5109	{
5110	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5111	struct ixgbe_hw *hw = &adapter->hw;
5112	int i;
5113
5114	if (is_zero_ether_addr(addr))
5115	return -EINVAL;
5116
5117	/* search table for addr, if found clear IN_USE flag and sync */
5118	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5119	/* we can only delete an entry if it is in use */
5120	if (!(mac_table->state & IXGBE_MAC_STATE_IN_USE))
5121	continue;
5122	/* we only care about entries that belong to the given pool */
5123	if (mac_table->pool != pool)
5124	continue;
5125	/* we only care about a specific MAC address */
5126	if (!ether_addr_equal(addr1: addr, addr2: mac_table->addr))
5127	continue;
5128
5129	mac_table->state |= IXGBE_MAC_STATE_MODIFIED;
5130	mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;
5131
5132	ixgbe_sync_mac_table(adapter);
5133
5134	return 0;
5135	}
5136
5137	return -ENOMEM;
5138	}
5139
5140	static int ixgbe_uc_sync(struct net_device *netdev, const unsigned char *addr)
5141	{
5142	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
5143	int ret;
5144
5145	ret = ixgbe_add_mac_filter(adapter, addr, VMDQ_P(0));
5146
5147	return min_t(int, ret, 0);
5148	}
5149
5150	static int ixgbe_uc_unsync(struct net_device *netdev, const unsigned char *addr)
5151	{
5152	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
5153
5154	ixgbe_del_mac_filter(adapter, addr, VMDQ_P(0));
5155
5156	return 0;
5157	}
5158
5159	/**
5160	* ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
5161	* @netdev: network interface device structure
5162	*
5163	* The set_rx_method entry point is called whenever the unicast/multicast
5164	* address list or the network interface flags are updated. This routine is
5165	* responsible for configuring the hardware for proper unicast, multicast and
5166	* promiscuous mode.
5167	**/
5168	void ixgbe_set_rx_mode(struct net_device *netdev)
5169	{
5170	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
5171	struct ixgbe_hw *hw = &adapter->hw;
5172	u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
5173	netdev_features_t features = netdev->features;
5174	int count;
5175
5176	/* Check for Promiscuous and All Multicast modes */
5177	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5178
5179	/* set all bits that we expect to always be set */
5180	fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */
5181	fctrl |= IXGBE_FCTRL_BAM;
5182	fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
5183	fctrl |= IXGBE_FCTRL_PMCF;
5184
5185	/* clear the bits we are changing the status of */
5186	fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
5187	if (netdev->flags & IFF_PROMISC) {
5188	hw->addr_ctrl.user_set_promisc = true;
5189	fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
5190	vmolr |= IXGBE_VMOLR_MPE;
5191	features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
5192	} else {
5193	if (netdev->flags & IFF_ALLMULTI) {
5194	fctrl |= IXGBE_FCTRL_MPE;
5195	vmolr |= IXGBE_VMOLR_MPE;
5196	}
5197	hw->addr_ctrl.user_set_promisc = false;
5198	}
5199
5200	/*
5201	* Write addresses to available RAR registers, if there is not
5202	* sufficient space to store all the addresses then enable
5203	* unicast promiscuous mode
5204	*/
5205	if (__dev_uc_sync(dev: netdev, sync: ixgbe_uc_sync, unsync: ixgbe_uc_unsync)) {
5206	fctrl |= IXGBE_FCTRL_UPE;
5207	vmolr |= IXGBE_VMOLR_ROPE;
5208	}
5209
5210	/* Write addresses to the MTA, if the attempt fails
5211	* then we should just turn on promiscuous mode so
5212	* that we can at least receive multicast traffic
5213	*/
5214	count = ixgbe_write_mc_addr_list(netdev);
5215	if (count < 0) {
5216	fctrl |= IXGBE_FCTRL_MPE;
5217	vmolr |= IXGBE_VMOLR_MPE;
5218	} else if (count) {
5219	vmolr |= IXGBE_VMOLR_ROMPE;
5220	}
5221
5222	if (hw->mac.type != ixgbe_mac_82598EB) {
5223	vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) &
5224	~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
5225	IXGBE_VMOLR_ROPE);
5226	IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr);
5227	}
5228
5229	/* This is useful for sniffing bad packets. */
5230	if (features & NETIF_F_RXALL) {
5231	/* UPE and MPE will be handled by normal PROMISC logic
5232	* in e1000e_set_rx_mode */
5233	fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */
5234	IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */
5235	IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */
5236
5237	fctrl &= ~(IXGBE_FCTRL_DPF);
5238	/* NOTE: VLAN filtering is disabled by setting PROMISC */
5239	}
5240
5241	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5242
5243	if (features & NETIF_F_HW_VLAN_CTAG_RX)
5244	ixgbe_vlan_strip_enable(adapter);
5245	else
5246	ixgbe_vlan_strip_disable(adapter);
5247
5248	if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
5249	ixgbe_vlan_promisc_disable(adapter);
5250	else
5251	ixgbe_vlan_promisc_enable(adapter);
5252	}
5253
5254	static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
5255	{
5256	int q_idx;
5257
5258	for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
5259	napi_enable(n: &adapter->q_vector[q_idx]->napi);
5260	}
5261
5262	static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
5263	{
5264	int q_idx;
5265
5266	for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
5267	napi_disable(n: &adapter->q_vector[q_idx]->napi);
5268	}
5269
5270	static int ixgbe_udp_tunnel_sync(struct net_device *dev, unsigned int table)
5271	{
5272	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
5273	struct ixgbe_hw *hw = &adapter->hw;
5274	struct udp_tunnel_info ti;
5275
5276	udp_tunnel_nic_get_port(dev, table, idx: 0, ti: &ti);
5277	if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
5278	adapter->vxlan_port = ti.port;
5279	else
5280	adapter->geneve_port = ti.port;
5281
5282	IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL,
5283	ntohs(adapter->vxlan_port) |
5284	ntohs(adapter->geneve_port) <<
5285	IXGBE_VXLANCTRL_GENEVE_UDPPORT_SHIFT);
5286	return 0;
5287	}
5288
5289	static const struct udp_tunnel_nic_info ixgbe_udp_tunnels_x550 = {
5290	.sync_table = ixgbe_udp_tunnel_sync,
5291	.flags = UDP_TUNNEL_NIC_INFO_IPV4_ONLY,
5292	.tables = {
5293	{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
5294	},
5295	};
5296
5297	static const struct udp_tunnel_nic_info ixgbe_udp_tunnels_x550em_a = {
5298	.sync_table = ixgbe_udp_tunnel_sync,
5299	.flags = UDP_TUNNEL_NIC_INFO_IPV4_ONLY,
5300	.tables = {
5301	{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
5302	{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
5303	},
5304	};
5305
5306	#ifdef CONFIG_IXGBE_DCB
5307	/**
5308	* ixgbe_configure_dcb - Configure DCB hardware
5309	* @adapter: ixgbe adapter struct
5310	*
5311	* This is called by the driver on open to configure the DCB hardware.
5312	* This is also called by the gennetlink interface when reconfiguring
5313	* the DCB state.
5314	*/
5315	static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
5316	{
5317	struct ixgbe_hw *hw = &adapter->hw;
5318	int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
5319
5320	if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
5321	if (hw->mac.type == ixgbe_mac_82598EB)
5322	netif_set_tso_max_size(dev: adapter->netdev, size: 65536);
5323	return;
5324	}
5325
5326	if (hw->mac.type == ixgbe_mac_82598EB)
5327	netif_set_tso_max_size(dev: adapter->netdev, size: 32768);
5328
5329	#ifdef IXGBE_FCOE
5330	if (adapter->netdev->fcoe_mtu)
5331	max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
5332	#endif
5333
5334	/* reconfigure the hardware */
5335	if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
5336	ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
5337	DCB_TX_CONFIG);
5338	ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
5339	DCB_RX_CONFIG);
5340	ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
5341	} else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {
5342	ixgbe_dcb_hw_ets(hw: &adapter->hw,
5343	ets: adapter->ixgbe_ieee_ets,
5344	max: max_frame);
5345	ixgbe_dcb_hw_pfc_config(hw: &adapter->hw,
5346	pfc_en: adapter->ixgbe_ieee_pfc->pfc_en,
5347	tc_prio: adapter->ixgbe_ieee_ets->prio_tc);
5348	}
5349
5350	/* Enable RSS Hash per TC */
5351	if (hw->mac.type != ixgbe_mac_82598EB) {
5352	u32 msb = 0;
5353	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1;
5354
5355	while (rss_i) {
5356	msb++;
5357	rss_i >>= 1;
5358	}
5359
5360	/* write msb to all 8 TCs in one write */
5361	IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111);
5362	}
5363	}
5364	#endif
5365
5366	/* Additional bittime to account for IXGBE framing */
5367	#define IXGBE_ETH_FRAMING 20
5368
5369	/**
5370	* ixgbe_hpbthresh - calculate high water mark for flow control
5371	*
5372	* @adapter: board private structure to calculate for
5373	* @pb: packet buffer to calculate
5374	*/
5375	static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)
5376	{
5377	struct ixgbe_hw *hw = &adapter->hw;
5378	struct net_device *dev = adapter->netdev;
5379	int link, tc, kb, marker;
5380	u32 dv_id, rx_pba;
5381
5382	/* Calculate max LAN frame size */
5383	tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;
5384
5385	#ifdef IXGBE_FCOE
5386	/* FCoE traffic class uses FCOE jumbo frames */
5387	if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE &&
5388	(pb == ixgbe_fcoe_get_tc(adapter)))
5389	tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
5390	#endif
5391
5392	/* Calculate delay value for device */
5393	switch (hw->mac.type) {
5394	case ixgbe_mac_X540:
5395	case ixgbe_mac_X550:
5396	case ixgbe_mac_X550EM_x:
5397	case ixgbe_mac_x550em_a:
5398	case ixgbe_mac_e610:
5399	dv_id = IXGBE_DV_X540(link, tc);
5400	break;
5401	default:
5402	dv_id = IXGBE_DV(link, tc);
5403	break;
5404	}
5405
5406	/* Loopback switch introduces additional latency */
5407	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
5408	dv_id += IXGBE_B2BT(tc);
5409
5410	/* Delay value is calculated in bit times convert to KB */
5411	kb = IXGBE_BT2KB(dv_id);
5412	rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;
5413
5414	marker = rx_pba - kb;
5415
5416	/* It is possible that the packet buffer is not large enough
5417	* to provide required headroom. In this case throw an error
5418	* to user and a do the best we can.
5419	*/
5420	if (marker < 0) {
5421	e_warn(drv, "Packet Buffer(%i) can not provide enough"
5422	"headroom to support flow control."
5423	"Decrease MTU or number of traffic classes\n", pb);
5424	marker = tc + 1;
5425	}
5426
5427	return marker;
5428	}
5429
5430	/**
5431	* ixgbe_lpbthresh - calculate low water mark for flow control
5432	*
5433	* @adapter: board private structure to calculate for
5434	* @pb: packet buffer to calculate
5435	*/
5436	static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb)
5437	{
5438	struct ixgbe_hw *hw = &adapter->hw;
5439	struct net_device *dev = adapter->netdev;
5440	int tc;
5441	u32 dv_id;
5442
5443	/* Calculate max LAN frame size */
5444	tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
5445
5446	#ifdef IXGBE_FCOE
5447	/* FCoE traffic class uses FCOE jumbo frames */
5448	if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE &&
5449	(pb == netdev_get_prio_tc_map(dev, prio: adapter->fcoe.up)))
5450	tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
5451	#endif
5452
5453	/* Calculate delay value for device */
5454	switch (hw->mac.type) {
5455	case ixgbe_mac_X540:
5456	case ixgbe_mac_X550:
5457	case ixgbe_mac_X550EM_x:
5458	case ixgbe_mac_x550em_a:
5459	case ixgbe_mac_e610:
5460	dv_id = IXGBE_LOW_DV_X540(tc);
5461	break;
5462	default:
5463	dv_id = IXGBE_LOW_DV(tc);
5464	break;
5465	}
5466
5467	/* Delay value is calculated in bit times convert to KB */
5468	return IXGBE_BT2KB(dv_id);
5469	}
5470
5471	/*
5472	* ixgbe_pbthresh_setup - calculate and setup high low water marks
5473	*/
5474	static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
5475	{
5476	struct ixgbe_hw *hw = &adapter->hw;
5477	int num_tc = adapter->hw_tcs;
5478	int i;
5479
5480	if (!num_tc)
5481	num_tc = 1;
5482
5483	for (i = 0; i < num_tc; i++) {
5484	hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, pb: i);
5485	hw->fc.low_water[i] = ixgbe_lpbthresh(adapter, pb: i);
5486
5487	/* Low water marks must not be larger than high water marks */
5488	if (hw->fc.low_water[i] > hw->fc.high_water[i])
5489	hw->fc.low_water[i] = 0;
5490	}
5491
5492	for (; i < MAX_TRAFFIC_CLASS; i++)
5493	hw->fc.high_water[i] = 0;
5494	}
5495
5496	static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
5497	{
5498	struct ixgbe_hw *hw = &adapter->hw;
5499	int hdrm;
5500	u8 tc = adapter->hw_tcs;
5501
5502	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
5503	adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
5504	hdrm = 32 << adapter->fdir_pballoc;
5505	else
5506	hdrm = 0;
5507
5508	hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
5509	ixgbe_pbthresh_setup(adapter);
5510	}
5511
5512	static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
5513	{
5514	struct ixgbe_hw *hw = &adapter->hw;
5515	struct hlist_node *node2;
5516	struct ixgbe_fdir_filter *filter;
5517	u8 queue;
5518
5519	spin_lock(lock: &adapter->fdir_perfect_lock);
5520
5521	if (!hlist_empty(h: &adapter->fdir_filter_list))
5522	ixgbe_fdir_set_input_mask_82599(hw, input_mask: &adapter->fdir_mask);
5523
5524	hlist_for_each_entry_safe(filter, node2,
5525	&adapter->fdir_filter_list, fdir_node) {
5526	if (filter->action == IXGBE_FDIR_DROP_QUEUE) {
5527	queue = IXGBE_FDIR_DROP_QUEUE;
5528	} else {
5529	u32 ring = ethtool_get_flow_spec_ring(ring_cookie: filter->action);
5530	u8 vf = ethtool_get_flow_spec_ring_vf(ring_cookie: filter->action);
5531
5532	if (!vf && (ring >= adapter->num_rx_queues)) {
5533	e_err(drv, "FDIR restore failed without VF, ring: %u\n",
5534	ring);
5535	continue;
5536	} else if (vf &&
5537	((vf > adapter->num_vfs) ||
5538	ring >= adapter->num_rx_queues_per_pool)) {
5539	e_err(drv, "FDIR restore failed with VF, vf: %hhu, ring: %u\n",
5540	vf, ring);
5541	continue;
5542	}
5543
5544	/* Map the ring onto the absolute queue index */
5545	if (!vf)
5546	queue = adapter->rx_ring[ring]->reg_idx;
5547	else
5548	queue = ((vf - 1) *
5549	adapter->num_rx_queues_per_pool) + ring;
5550	}
5551
5552	ixgbe_fdir_write_perfect_filter_82599(hw,
5553	input: &filter->filter, soft_id: filter->sw_idx, queue);
5554	}
5555
5556	spin_unlock(lock: &adapter->fdir_perfect_lock);
5557	}
5558
5559	/**
5560	* ixgbe_clean_rx_ring - Free Rx Buffers per Queue
5561	* @rx_ring: ring to free buffers from
5562	**/
5563	static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
5564	{
5565	u16 i = rx_ring->next_to_clean;
5566	struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i];
5567
5568	if (rx_ring->xsk_pool) {
5569	ixgbe_xsk_clean_rx_ring(rx_ring);
5570	goto skip_free;
5571	}
5572
5573	/* Free all the Rx ring sk_buffs */
5574	while (i != rx_ring->next_to_alloc) {
5575	if (rx_buffer->skb) {
5576	struct sk_buff *skb = rx_buffer->skb;
5577	if (IXGBE_CB(skb)->page_released)
5578	dma_unmap_page_attrs(dev: rx_ring->dev,
5579	IXGBE_CB(skb)->dma,
5580	ixgbe_rx_pg_size(rx_ring),
5581	dir: DMA_FROM_DEVICE,
5582	IXGBE_RX_DMA_ATTR);
5583	dev_kfree_skb(skb);
5584	}
5585
5586	/* Invalidate cache lines that may have been written to by
5587	* device so that we avoid corrupting memory.
5588	*/
5589	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
5590	addr: rx_buffer->dma,
5591	offset: rx_buffer->page_offset,
5592	size: ixgbe_rx_bufsz(ring: rx_ring),
5593	dir: DMA_FROM_DEVICE);
5594
5595	/* free resources associated with mapping */
5596	dma_unmap_page_attrs(dev: rx_ring->dev, addr: rx_buffer->dma,
5597	ixgbe_rx_pg_size(rx_ring),
5598	dir: DMA_FROM_DEVICE,
5599	IXGBE_RX_DMA_ATTR);
5600	__page_frag_cache_drain(page: rx_buffer->page,
5601	count: rx_buffer->pagecnt_bias);
5602
5603	i++;
5604	rx_buffer++;
5605	if (i == rx_ring->count) {
5606	i = 0;
5607	rx_buffer = rx_ring->rx_buffer_info;
5608	}
5609	}
5610
5611	skip_free:
5612	rx_ring->next_to_alloc = 0;
5613	rx_ring->next_to_clean = 0;
5614	rx_ring->next_to_use = 0;
5615	}
5616
5617	static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter,
5618	struct ixgbe_fwd_adapter *accel)
5619	{
5620	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
5621	int num_tc = netdev_get_num_tc(dev: adapter->netdev);
5622	struct net_device *vdev = accel->netdev;
5623	int i, baseq, err;
5624
5625	baseq = accel->pool * adapter->num_rx_queues_per_pool;
5626	netdev_dbg(vdev, "pool %i:%i queues %i:%i\n",
5627	accel->pool, adapter->num_rx_pools,
5628	baseq, baseq + adapter->num_rx_queues_per_pool);
5629
5630	accel->rx_base_queue = baseq;
5631	accel->tx_base_queue = baseq;
5632
5633	/* record configuration for macvlan interface in vdev */
5634	for (i = 0; i < num_tc; i++)
5635	netdev_bind_sb_channel_queue(dev: adapter->netdev, sb_dev: vdev,
5636	tc: i, count: rss_i, offset: baseq + (rss_i * i));
5637
5638	for (i = 0; i < adapter->num_rx_queues_per_pool; i++)
5639	adapter->rx_ring[baseq + i]->netdev = vdev;
5640
5641	/* Guarantee all rings are updated before we update the
5642	* MAC address filter.
5643	*/
5644	wmb();
5645
5646	/* ixgbe_add_mac_filter will return an index if it succeeds, so we
5647	* need to only treat it as an error value if it is negative.
5648	*/
5649	err = ixgbe_add_mac_filter(adapter, addr: vdev->dev_addr,
5650	VMDQ_P(accel->pool));
5651	if (err >= 0)
5652	return 0;
5653
5654	/* if we cannot add the MAC rule then disable the offload */
5655	macvlan_release_l2fw_offload(dev: vdev);
5656
5657	for (i = 0; i < adapter->num_rx_queues_per_pool; i++)
5658	adapter->rx_ring[baseq + i]->netdev = NULL;
5659
5660	netdev_err(dev: vdev, format: "L2FW offload disabled due to L2 filter error\n");
5661
5662	/* unbind the queues and drop the subordinate channel config */
5663	netdev_unbind_sb_channel(dev: adapter->netdev, sb_dev: vdev);
5664	netdev_set_sb_channel(dev: vdev, channel: 0);
5665
5666	clear_bit(nr: accel->pool, addr: adapter->fwd_bitmask);
5667	kfree(objp: accel);
5668
5669	return err;
5670	}
5671
5672	static int ixgbe_macvlan_up(struct net_device *vdev,
5673	struct netdev_nested_priv *priv)
5674	{
5675	struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data;
5676	struct ixgbe_fwd_adapter *accel;
5677
5678	if (!netif_is_macvlan(dev: vdev))
5679	return 0;
5680
5681	accel = macvlan_accel_priv(dev: vdev);
5682	if (!accel)
5683	return 0;
5684
5685	ixgbe_fwd_ring_up(adapter, accel);
5686
5687	return 0;
5688	}
5689
5690	static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter)
5691	{
5692	struct netdev_nested_priv priv = {
5693	.data = (void *)adapter,
5694	};
5695
5696	netdev_walk_all_upper_dev_rcu(dev: adapter->netdev,
5697	fn: ixgbe_macvlan_up, priv: &priv);
5698	}
5699
5700	static void ixgbe_configure(struct ixgbe_adapter *adapter)
5701	{
5702	struct ixgbe_hw *hw = &adapter->hw;
5703
5704	ixgbe_configure_pb(adapter);
5705	#ifdef CONFIG_IXGBE_DCB
5706	ixgbe_configure_dcb(adapter);
5707	#endif
5708	/*
5709	* We must restore virtualization before VLANs or else
5710	* the VLVF registers will not be populated
5711	*/
5712	ixgbe_configure_virtualization(adapter);
5713
5714	ixgbe_set_rx_mode(netdev: adapter->netdev);
5715	ixgbe_restore_vlan(adapter);
5716	ixgbe_ipsec_restore(adapter);
5717
5718	switch (hw->mac.type) {
5719	case ixgbe_mac_82599EB:
5720	case ixgbe_mac_X540:
5721	hw->mac.ops.disable_rx_buff(hw);
5722	break;
5723	default:
5724	break;
5725	}
5726
5727	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
5728	ixgbe_init_fdir_signature_82599(hw: &adapter->hw,
5729	fdirctrl: adapter->fdir_pballoc);
5730	} else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
5731	ixgbe_init_fdir_perfect_82599(hw: &adapter->hw,
5732	fdirctrl: adapter->fdir_pballoc);
5733	ixgbe_fdir_filter_restore(adapter);
5734	}
5735
5736	switch (hw->mac.type) {
5737	case ixgbe_mac_82599EB:
5738	case ixgbe_mac_X540:
5739	hw->mac.ops.enable_rx_buff(hw);
5740	break;
5741	default:
5742	break;
5743	}
5744
5745	#ifdef CONFIG_IXGBE_DCA
5746	/* configure DCA */
5747	if (adapter->flags & IXGBE_FLAG_DCA_CAPABLE)
5748	ixgbe_setup_dca(adapter);
5749	#endif /* CONFIG_IXGBE_DCA */
5750
5751	#ifdef IXGBE_FCOE
5752	/* configure FCoE L2 filters, redirection table, and Rx control */
5753	ixgbe_configure_fcoe(adapter);
5754
5755	#endif /* IXGBE_FCOE */
5756	ixgbe_configure_tx(adapter);
5757	ixgbe_configure_rx(adapter);
5758	ixgbe_configure_dfwd(adapter);
5759	}
5760
5761	/**
5762	* ixgbe_enable_link_status_events - enable link status events
5763	* @adapter: pointer to the adapter structure
5764	* @mask: event mask to be set
5765	*
5766	* Enables link status events by invoking ixgbe_configure_lse()
5767	*
5768	* Return: the exit code of the operation.
5769	*/
5770	static int ixgbe_enable_link_status_events(struct ixgbe_adapter *adapter,
5771	u16 mask)
5772	{
5773	int err;
5774
5775	err = ixgbe_configure_lse(hw: &adapter->hw, activate: true, mask);
5776	if (err)
5777	return err;
5778
5779	adapter->lse_mask = mask;
5780	return 0;
5781	}
5782
5783	/**
5784	* ixgbe_disable_link_status_events - disable link status events
5785	* @adapter: pointer to the adapter structure
5786	*
5787	* Disables link status events by invoking ixgbe_configure_lse()
5788	*
5789	* Return: the exit code of the operation.
5790	*/
5791	static int ixgbe_disable_link_status_events(struct ixgbe_adapter *adapter)
5792	{
5793	int err;
5794
5795	err = ixgbe_configure_lse(hw: &adapter->hw, activate: false, mask: adapter->lse_mask);
5796	if (err)
5797	return err;
5798
5799	adapter->lse_mask = 0;
5800	return 0;
5801	}
5802
5803	/**
5804	* ixgbe_sfp_link_config - set up SFP+ link
5805	* @adapter: pointer to private adapter struct
5806	**/
5807	static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
5808	{
5809	/*
5810	* We are assuming the worst case scenario here, and that
5811	* is that an SFP was inserted/removed after the reset
5812	* but before SFP detection was enabled. As such the best
5813	* solution is to just start searching as soon as we start
5814	*/
5815	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
5816	adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
5817
5818	adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
5819	adapter->sfp_poll_time = 0;
5820	}
5821
5822	/**
5823	* ixgbe_non_sfp_link_config - set up non-SFP+ link
5824	* @hw: pointer to private hardware struct
5825	*
5826	* Configure non-SFP link.
5827	*
5828	* Return: 0 on success, negative on failure
5829	**/
5830	static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
5831	{
5832	struct ixgbe_adapter *adapter = container_of(hw, struct ixgbe_adapter,
5833	hw);
5834	u16 mask = ~((u16)(IXGBE_ACI_LINK_EVENT_UPDOWN |
5835	IXGBE_ACI_LINK_EVENT_MEDIA_NA |
5836	IXGBE_ACI_LINK_EVENT_MODULE_QUAL_FAIL |
5837	IXGBE_ACI_LINK_EVENT_PHY_FW_LOAD_FAIL));
5838	bool autoneg, link_up = false;
5839	int ret = -EIO;
5840	u32 speed;
5841
5842	if (hw->mac.ops.check_link)
5843	ret = hw->mac.ops.check_link(hw, &speed, &link_up, false);
5844
5845	if (ret)
5846	return ret;
5847
5848	speed = hw->phy.autoneg_advertised;
5849	if (!speed && hw->mac.ops.get_link_capabilities) {
5850	ret = hw->mac.ops.get_link_capabilities(hw, &speed,
5851	&autoneg);
5852	/* remove NBASE-T speeds from default autonegotiation
5853	* to accommodate broken network switches in the field
5854	* which cannot cope with advertised NBASE-T speeds
5855	*/
5856	speed &= ~(IXGBE_LINK_SPEED_5GB_FULL |
5857	IXGBE_LINK_SPEED_2_5GB_FULL);
5858	}
5859
5860	if (ret)
5861	return ret;
5862
5863	if (hw->mac.ops.setup_link) {
5864	if (adapter->hw.mac.type == ixgbe_mac_e610) {
5865	ret = ixgbe_enable_link_status_events(adapter, mask);
5866	if (ret)
5867	return ret;
5868	}
5869	ret = hw->mac.ops.setup_link(hw, speed, link_up);
5870	}
5871
5872	return ret;
5873	}
5874
5875	/**
5876	* ixgbe_check_media_subtask - check for media
5877	* @adapter: pointer to adapter structure
5878	*
5879	* If media is available then initialize PHY user configuration. Configure the
5880	* PHY if the interface is up.
5881	*/
5882	static void ixgbe_check_media_subtask(struct ixgbe_adapter *adapter)
5883	{
5884	struct ixgbe_hw *hw = &adapter->hw;
5885
5886	/* No need to check for media if it's already present */
5887	if (!(adapter->flags2 & IXGBE_FLAG2_NO_MEDIA))
5888	return;
5889
5890	/* Refresh link info and check if media is present */
5891	if (ixgbe_update_link_info(hw))
5892	return;
5893
5894	ixgbe_check_link_cfg_err(adapter, link_cfg_err: hw->link.link_info.link_cfg_err);
5895
5896	if (hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE) {
5897	/* PHY settings are reset on media insertion, reconfigure
5898	* PHY to preserve settings.
5899	*/
5900	if (!(ixgbe_non_sfp_link_config(hw: &adapter->hw)))
5901	adapter->flags2 &= ~IXGBE_FLAG2_NO_MEDIA;
5902
5903	/* A Link Status Event will be generated; the event handler
5904	* will complete bringing the interface up
5905	*/
5906	}
5907	}
5908
5909	/**
5910	* ixgbe_clear_vf_stats_counters - Clear out VF stats after reset
5911	* @adapter: board private structure
5912	*
5913	* On a reset we need to clear out the VF stats or accounting gets
5914	* messed up because they're not clear on read.
5915	**/
5916	static void ixgbe_clear_vf_stats_counters(struct ixgbe_adapter *adapter)
5917	{
5918	struct ixgbe_hw *hw = &adapter->hw;
5919	int i;
5920
5921	for (i = 0; i < adapter->num_vfs; i++) {
5922	adapter->vfinfo[i].last_vfstats.gprc =
5923	IXGBE_READ_REG(hw, IXGBE_PVFGPRC(i));
5924	adapter->vfinfo[i].saved_rst_vfstats.gprc +=
5925	adapter->vfinfo[i].vfstats.gprc;
5926	adapter->vfinfo[i].vfstats.gprc = 0;
5927	adapter->vfinfo[i].last_vfstats.gptc =
5928	IXGBE_READ_REG(hw, IXGBE_PVFGPTC(i));
5929	adapter->vfinfo[i].saved_rst_vfstats.gptc +=
5930	adapter->vfinfo[i].vfstats.gptc;
5931	adapter->vfinfo[i].vfstats.gptc = 0;
5932	adapter->vfinfo[i].last_vfstats.gorc =
5933	IXGBE_READ_REG(hw, IXGBE_PVFGORC_LSB(i));
5934	adapter->vfinfo[i].saved_rst_vfstats.gorc +=
5935	adapter->vfinfo[i].vfstats.gorc;
5936	adapter->vfinfo[i].vfstats.gorc = 0;
5937	adapter->vfinfo[i].last_vfstats.gotc =
5938	IXGBE_READ_REG(hw, IXGBE_PVFGOTC_LSB(i));
5939	adapter->vfinfo[i].saved_rst_vfstats.gotc +=
5940	adapter->vfinfo[i].vfstats.gotc;
5941	adapter->vfinfo[i].vfstats.gotc = 0;
5942	adapter->vfinfo[i].last_vfstats.mprc =
5943	IXGBE_READ_REG(hw, IXGBE_PVFMPRC(i));
5944	adapter->vfinfo[i].saved_rst_vfstats.mprc +=
5945	adapter->vfinfo[i].vfstats.mprc;
5946	adapter->vfinfo[i].vfstats.mprc = 0;
5947	}
5948	}
5949
5950	static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
5951	{
5952	struct ixgbe_hw *hw = &adapter->hw;
5953	u32 gpie = 0;
5954
5955	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
5956	gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
5957	IXGBE_GPIE_OCD;
5958	gpie |= IXGBE_GPIE_EIAME;
5959	/*
5960	* use EIAM to auto-mask when MSI-X interrupt is asserted
5961	* this saves a register write for every interrupt
5962	*/
5963	switch (hw->mac.type) {
5964	case ixgbe_mac_82598EB:
5965	IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
5966	break;
5967	case ixgbe_mac_82599EB:
5968	case ixgbe_mac_X540:
5969	case ixgbe_mac_X550:
5970	case ixgbe_mac_X550EM_x:
5971	case ixgbe_mac_x550em_a:
5972	case ixgbe_mac_e610:
5973	default:
5974	IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
5975	IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
5976	break;
5977	}
5978	} else {
5979	/* legacy interrupts, use EIAM to auto-mask when reading EICR,
5980	* specifically only auto mask tx and rx interrupts */
5981	IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
5982	}
5983
5984	/* XXX: to interrupt immediately for EICS writes, enable this */
5985	/* gpie |= IXGBE_GPIE_EIMEN; */
5986
5987	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
5988	gpie &= ~IXGBE_GPIE_VTMODE_MASK;
5989
5990	switch (adapter->ring_feature[RING_F_VMDQ].mask) {
5991	case IXGBE_82599_VMDQ_8Q_MASK:
5992	gpie |= IXGBE_GPIE_VTMODE_16;
5993	break;
5994	case IXGBE_82599_VMDQ_4Q_MASK:
5995	gpie |= IXGBE_GPIE_VTMODE_32;
5996	break;
5997	default:
5998	gpie |= IXGBE_GPIE_VTMODE_64;
5999	break;
6000	}
6001	}
6002
6003	/* Enable Thermal over heat sensor interrupt */
6004	if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {
6005	switch (adapter->hw.mac.type) {
6006	case ixgbe_mac_82599EB:
6007	gpie |= IXGBE_SDP0_GPIEN_8259X;
6008	break;
6009	default:
6010	break;
6011	}
6012	}
6013
6014	/* Enable fan failure interrupt */
6015	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
6016	gpie |= IXGBE_SDP1_GPIEN(hw);
6017
6018	switch (hw->mac.type) {
6019	case ixgbe_mac_82599EB:
6020	gpie |= IXGBE_SDP1_GPIEN_8259X | IXGBE_SDP2_GPIEN_8259X;
6021	break;
6022	case ixgbe_mac_X550EM_x:
6023	case ixgbe_mac_x550em_a:
6024	gpie |= IXGBE_SDP0_GPIEN_X540;
6025	break;
6026	default:
6027	break;
6028	}
6029
6030	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
6031	}
6032
6033	static void ixgbe_up_complete(struct ixgbe_adapter *adapter)
6034	{
6035	struct ixgbe_hw *hw = &adapter->hw;
6036	int err;
6037	u32 ctrl_ext;
6038
6039	ixgbe_get_hw_control(adapter);
6040	ixgbe_setup_gpie(adapter);
6041
6042	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
6043	ixgbe_configure_msix(adapter);
6044	else
6045	ixgbe_configure_msi_and_legacy(adapter);
6046
6047	/* enable the optics for 82599 SFP+ fiber */
6048	if (hw->mac.ops.enable_tx_laser)
6049	hw->mac.ops.enable_tx_laser(hw);
6050
6051	if (hw->phy.ops.set_phy_power)
6052	hw->phy.ops.set_phy_power(hw, true);
6053
6054	smp_mb__before_atomic();
6055	clear_bit(nr: __IXGBE_DOWN, addr: &adapter->state);
6056	ixgbe_napi_enable_all(adapter);
6057
6058	if (ixgbe_is_sfp(hw)) {
6059	ixgbe_sfp_link_config(adapter);
6060	} else {
6061	err = ixgbe_non_sfp_link_config(hw);
6062	if (err)
6063	e_err(probe, "link_config FAILED %d\n", err);
6064	}
6065
6066	/* clear any pending interrupts, may auto mask */
6067	IXGBE_READ_REG(hw, IXGBE_EICR);
6068	ixgbe_irq_enable(adapter, queues: true, flush: true);
6069
6070	/*
6071	* If this adapter has a fan, check to see if we had a failure
6072	* before we enabled the interrupt.
6073	*/
6074	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
6075	u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
6076	if (esdp & IXGBE_ESDP_SDP1)
6077	e_crit(drv, "Fan has stopped, replace the adapter\n");
6078	}
6079
6080	/* bring the link up in the watchdog, this could race with our first
6081	* link up interrupt but shouldn't be a problem */
6082	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
6083	adapter->link_check_timeout = jiffies;
6084	mod_timer(timer: &adapter->service_timer, expires: jiffies);
6085
6086	ixgbe_clear_vf_stats_counters(adapter);
6087	/* Set PF Reset Done bit so PF/VF Mail Ops can work */
6088	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
6089	ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
6090	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
6091
6092	/* update setting rx tx for all active vfs */
6093	ixgbe_set_all_vfs(adapter);
6094	}
6095
6096	void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
6097	{
6098	/* put off any impending NetWatchDogTimeout */
6099	netif_trans_update(dev: adapter->netdev);
6100
6101	while (test_and_set_bit(nr: __IXGBE_RESETTING, addr: &adapter->state))
6102	usleep_range(min: 1000, max: 2000);
6103	if (adapter->hw.phy.type == ixgbe_phy_fw)
6104	ixgbe_watchdog_link_is_down(adapter);
6105	ixgbe_down(adapter);
6106	/*
6107	* If SR-IOV enabled then wait a bit before bringing the adapter
6108	* back up to give the VFs time to respond to the reset. The
6109	* two second wait is based upon the watchdog timer cycle in
6110	* the VF driver.
6111	*/
6112	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
6113	msleep(msecs: 2000);
6114	ixgbe_up(adapter);
6115	clear_bit(nr: __IXGBE_RESETTING, addr: &adapter->state);
6116	}
6117
6118	void ixgbe_up(struct ixgbe_adapter *adapter)
6119	{
6120	/* hardware has been reset, we need to reload some things */
6121	ixgbe_configure(adapter);
6122
6123	ixgbe_up_complete(adapter);
6124	}
6125
6126	static unsigned long ixgbe_get_completion_timeout(struct ixgbe_adapter *adapter)
6127	{
6128	u16 devctl2;
6129
6130	pcie_capability_read_word(dev: adapter->pdev, PCI_EXP_DEVCTL2, val: &devctl2);
6131
6132	switch (devctl2 & IXGBE_PCIDEVCTRL2_TIMEO_MASK) {
6133	case IXGBE_PCIDEVCTRL2_17_34s:
6134	case IXGBE_PCIDEVCTRL2_4_8s:
6135	/* For now we cap the upper limit on delay to 2 seconds
6136	* as we end up going up to 34 seconds of delay in worst
6137	* case timeout value.
6138	*/
6139	case IXGBE_PCIDEVCTRL2_1_2s:
6140	return 2000000ul; /* 2.0 s */
6141	case IXGBE_PCIDEVCTRL2_260_520ms:
6142	return 520000ul; /* 520 ms */
6143	case IXGBE_PCIDEVCTRL2_65_130ms:
6144	return 130000ul; /* 130 ms */
6145	case IXGBE_PCIDEVCTRL2_16_32ms:
6146	return 32000ul; /* 32 ms */
6147	case IXGBE_PCIDEVCTRL2_1_2ms:
6148	return 2000ul; /* 2 ms */
6149	case IXGBE_PCIDEVCTRL2_50_100us:
6150	return 100ul; /* 100 us */
6151	case IXGBE_PCIDEVCTRL2_16_32ms_def:
6152	return 32000ul; /* 32 ms */
6153	default:
6154	break;
6155	}
6156
6157	/* We shouldn't need to hit this path, but just in case default as
6158	* though completion timeout is not supported and support 32ms.
6159	*/
6160	return 32000ul;
6161	}
6162
6163	void ixgbe_disable_rx(struct ixgbe_adapter *adapter)
6164	{
6165	unsigned long wait_delay, delay_interval;
6166	struct ixgbe_hw *hw = &adapter->hw;
6167	int i, wait_loop;
6168	u32 rxdctl;
6169
6170	/* disable receives */
6171	hw->mac.ops.disable_rx(hw);
6172
6173	if (ixgbe_removed(addr: hw->hw_addr))
6174	return;
6175
6176	/* disable all enabled Rx queues */
6177	for (i = 0; i < adapter->num_rx_queues; i++) {
6178	struct ixgbe_ring *ring = adapter->rx_ring[i];
6179	u8 reg_idx = ring->reg_idx;
6180
6181	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
6182	rxdctl &= ~IXGBE_RXDCTL_ENABLE;
6183	rxdctl |= IXGBE_RXDCTL_SWFLSH;
6184
6185	/* write value back with RXDCTL.ENABLE bit cleared */
6186	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
6187	}
6188
6189	/* RXDCTL.EN may not change on 82598 if link is down, so skip it */
6190	if (hw->mac.type == ixgbe_mac_82598EB &&
6191	!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
6192	return;
6193
6194	/* Determine our minimum delay interval. We will increase this value
6195	* with each subsequent test. This way if the device returns quickly
6196	* we should spend as little time as possible waiting, however as
6197	* the time increases we will wait for larger periods of time.
6198	*
6199	* The trick here is that we increase the interval using the
6200	* following pattern: 1x 3x 5x 7x 9x 11x 13x 15x 17x 19x. The result
6201	* of that wait is that it totals up to 100x whatever interval we
6202	* choose. Since our minimum wait is 100us we can just divide the
6203	* total timeout by 100 to get our minimum delay interval.
6204	*/
6205	delay_interval = ixgbe_get_completion_timeout(adapter) / 100;
6206
6207	wait_loop = IXGBE_MAX_RX_DESC_POLL;
6208	wait_delay = delay_interval;
6209
6210	while (wait_loop--) {
6211	usleep_range(min: wait_delay, max: wait_delay + 10);
6212	wait_delay += delay_interval * 2;
6213	rxdctl = 0;
6214
6215	/* OR together the reading of all the active RXDCTL registers,
6216	* and then test the result. We need the disable to complete
6217	* before we start freeing the memory and invalidating the
6218	* DMA mappings.
6219	*/
6220	for (i = 0; i < adapter->num_rx_queues; i++) {
6221	struct ixgbe_ring *ring = adapter->rx_ring[i];
6222	u8 reg_idx = ring->reg_idx;
6223
6224	rxdctl |= IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
6225	}
6226
6227	if (!(rxdctl & IXGBE_RXDCTL_ENABLE))
6228	return;
6229	}
6230
6231	e_err(drv,
6232	"RXDCTL.ENABLE for one or more queues not cleared within the polling period\n");
6233	}
6234
6235	void ixgbe_disable_tx(struct ixgbe_adapter *adapter)
6236	{
6237	unsigned long wait_delay, delay_interval;
6238	struct ixgbe_hw *hw = &adapter->hw;
6239	int i, wait_loop;
6240	u32 txdctl;
6241
6242	if (ixgbe_removed(addr: hw->hw_addr))
6243	return;
6244
6245	/* disable all enabled Tx queues */
6246	for (i = 0; i < adapter->num_tx_queues; i++) {
6247	struct ixgbe_ring *ring = adapter->tx_ring[i];
6248	u8 reg_idx = ring->reg_idx;
6249
6250	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
6251	}
6252
6253	/* disable all enabled XDP Tx queues */
6254	for (i = 0; i < adapter->num_xdp_queues; i++) {
6255	struct ixgbe_ring *ring = adapter->xdp_ring[i];
6256	u8 reg_idx = ring->reg_idx;
6257
6258	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
6259	}
6260
6261	/* If the link is not up there shouldn't be much in the way of
6262	* pending transactions. Those that are left will be flushed out
6263	* when the reset logic goes through the flush sequence to clean out
6264	* the pending Tx transactions.
6265	*/
6266	if (!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
6267	goto dma_engine_disable;
6268
6269	/* Determine our minimum delay interval. We will increase this value
6270	* with each subsequent test. This way if the device returns quickly
6271	* we should spend as little time as possible waiting, however as
6272	* the time increases we will wait for larger periods of time.
6273	*
6274	* The trick here is that we increase the interval using the
6275	* following pattern: 1x 3x 5x 7x 9x 11x 13x 15x 17x 19x. The result
6276	* of that wait is that it totals up to 100x whatever interval we
6277	* choose. Since our minimum wait is 100us we can just divide the
6278	* total timeout by 100 to get our minimum delay interval.
6279	*/
6280	delay_interval = ixgbe_get_completion_timeout(adapter) / 100;
6281
6282	wait_loop = IXGBE_MAX_RX_DESC_POLL;
6283	wait_delay = delay_interval;
6284
6285	while (wait_loop--) {
6286	usleep_range(min: wait_delay, max: wait_delay + 10);
6287	wait_delay += delay_interval * 2;
6288	txdctl = 0;
6289
6290	/* OR together the reading of all the active TXDCTL registers,
6291	* and then test the result. We need the disable to complete
6292	* before we start freeing the memory and invalidating the
6293	* DMA mappings.
6294	*/
6295	for (i = 0; i < adapter->num_tx_queues; i++) {
6296	struct ixgbe_ring *ring = adapter->tx_ring[i];
6297	u8 reg_idx = ring->reg_idx;
6298
6299	txdctl |= IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
6300	}
6301	for (i = 0; i < adapter->num_xdp_queues; i++) {
6302	struct ixgbe_ring *ring = adapter->xdp_ring[i];
6303	u8 reg_idx = ring->reg_idx;
6304
6305	txdctl |= IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
6306	}
6307
6308	if (!(txdctl & IXGBE_TXDCTL_ENABLE))
6309	goto dma_engine_disable;
6310	}
6311
6312	e_err(drv,
6313	"TXDCTL.ENABLE for one or more queues not cleared within the polling period\n");
6314
6315	dma_engine_disable:
6316	/* Disable the Tx DMA engine on 82599 and later MAC */
6317	switch (hw->mac.type) {
6318	case ixgbe_mac_82599EB:
6319	case ixgbe_mac_X540:
6320	case ixgbe_mac_X550:
6321	case ixgbe_mac_X550EM_x:
6322	case ixgbe_mac_x550em_a:
6323	case ixgbe_mac_e610:
6324	IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
6325	(IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
6326	~IXGBE_DMATXCTL_TE));
6327	fallthrough;
6328	default:
6329	break;
6330	}
6331	}
6332
6333	void ixgbe_reset(struct ixgbe_adapter *adapter)
6334	{
6335	struct ixgbe_hw *hw = &adapter->hw;
6336	struct net_device *netdev = adapter->netdev;
6337	int err;
6338
6339	if (ixgbe_removed(addr: hw->hw_addr))
6340	return;
6341	/* lock SFP init bit to prevent race conditions with the watchdog */
6342	while (test_and_set_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state))
6343	usleep_range(min: 1000, max: 2000);
6344
6345	/* clear all SFP and link config related flags while holding SFP_INIT */
6346	adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
6347	IXGBE_FLAG2_SFP_NEEDS_RESET);
6348	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
6349
6350	err = hw->mac.ops.init_hw(hw);
6351	switch (err) {
6352	case 0:
6353	case -ENOENT:
6354	case -EOPNOTSUPP:
6355	break;
6356	case -EALREADY:
6357	e_dev_err("primary disable timed out\n");
6358	break;
6359	case -EACCES:
6360	/* We are running on a pre-production device, log a warning */
6361	e_dev_warn("This device is a pre-production adapter/LOM. "
6362	"Please be aware there may be issues associated with "
6363	"your hardware. If you are experiencing problems "
6364	"please contact your Intel or hardware "
6365	"representative who provided you with this "
6366	"hardware.\n");
6367	break;
6368	default:
6369	e_dev_err("Hardware Error: %d\n", err);
6370	}
6371
6372	clear_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state);
6373
6374	/* flush entries out of MAC table */
6375	ixgbe_flush_sw_mac_table(adapter);
6376	__dev_uc_unsync(dev: netdev, NULL);
6377
6378	/* do not flush user set addresses */
6379	ixgbe_mac_set_default_filter(adapter);
6380
6381	/* update SAN MAC vmdq pool selection */
6382	if (hw->mac.san_mac_rar_index)
6383	hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
6384
6385	if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
6386	ixgbe_ptp_reset(adapter);
6387
6388	if (hw->phy.ops.set_phy_power) {
6389	if (!netif_running(dev: adapter->netdev) && !adapter->wol)
6390	hw->phy.ops.set_phy_power(hw, false);
6391	else
6392	hw->phy.ops.set_phy_power(hw, true);
6393	}
6394	}
6395
6396	/**
6397	* ixgbe_clean_tx_ring - Free Tx Buffers
6398	* @tx_ring: ring to be cleaned
6399	**/
6400	static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
6401	{
6402	u16 i = tx_ring->next_to_clean;
6403	struct ixgbe_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
6404
6405	if (tx_ring->xsk_pool) {
6406	ixgbe_xsk_clean_tx_ring(tx_ring);
6407	goto out;
6408	}
6409
6410	while (i != tx_ring->next_to_use) {
6411	union ixgbe_adv_tx_desc *eop_desc, *tx_desc;
6412
6413	/* Free all the Tx ring sk_buffs */
6414	if (ring_is_xdp(tx_ring))
6415	xdp_return_frame(xdpf: tx_buffer->xdpf);
6416	else
6417	dev_kfree_skb_any(skb: tx_buffer->skb);
6418
6419	/* unmap skb header data */
6420	dma_unmap_single(tx_ring->dev,
6421	dma_unmap_addr(tx_buffer, dma),
6422	dma_unmap_len(tx_buffer, len),
6423	DMA_TO_DEVICE);
6424
6425	/* check for eop_desc to determine the end of the packet */
6426	eop_desc = tx_buffer->next_to_watch;
6427	tx_desc = IXGBE_TX_DESC(tx_ring, i);
6428
6429	/* unmap remaining buffers */
6430	while (tx_desc != eop_desc) {
6431	tx_buffer++;
6432	tx_desc++;
6433	i++;
6434	if (unlikely(i == tx_ring->count)) {
6435	i = 0;
6436	tx_buffer = tx_ring->tx_buffer_info;
6437	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
6438	}
6439
6440	/* unmap any remaining paged data */
6441	if (dma_unmap_len(tx_buffer, len))
6442	dma_unmap_page(tx_ring->dev,
6443	dma_unmap_addr(tx_buffer, dma),
6444	dma_unmap_len(tx_buffer, len),
6445	DMA_TO_DEVICE);
6446	}
6447
6448	/* move us one more past the eop_desc for start of next pkt */
6449	tx_buffer++;
6450	i++;
6451	if (unlikely(i == tx_ring->count)) {
6452	i = 0;
6453	tx_buffer = tx_ring->tx_buffer_info;
6454	}
6455	}
6456
6457	/* reset BQL for queue */
6458	if (!ring_is_xdp(tx_ring))
6459	netdev_tx_reset_queue(q: txring_txq(ring: tx_ring));
6460
6461	out:
6462	/* reset next_to_use and next_to_clean */
6463	tx_ring->next_to_use = 0;
6464	tx_ring->next_to_clean = 0;
6465	}
6466
6467	/**
6468	* ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
6469	* @adapter: board private structure
6470	**/
6471	static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
6472	{
6473	int i;
6474
6475	for (i = 0; i < adapter->num_rx_queues; i++)
6476	ixgbe_clean_rx_ring(rx_ring: adapter->rx_ring[i]);
6477	}
6478
6479	/**
6480	* ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
6481	* @adapter: board private structure
6482	**/
6483	static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
6484	{
6485	int i;
6486
6487	for (i = 0; i < adapter->num_tx_queues; i++)
6488	ixgbe_clean_tx_ring(tx_ring: adapter->tx_ring[i]);
6489	for (i = 0; i < adapter->num_xdp_queues; i++)
6490	ixgbe_clean_tx_ring(tx_ring: adapter->xdp_ring[i]);
6491	}
6492
6493	static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
6494	{
6495	struct hlist_node *node2;
6496	struct ixgbe_fdir_filter *filter;
6497
6498	spin_lock(lock: &adapter->fdir_perfect_lock);
6499
6500	hlist_for_each_entry_safe(filter, node2,
6501	&adapter->fdir_filter_list, fdir_node) {
6502	hlist_del(n: &filter->fdir_node);
6503	kfree(objp: filter);
6504	}
6505	adapter->fdir_filter_count = 0;
6506
6507	spin_unlock(lock: &adapter->fdir_perfect_lock);
6508	}
6509
6510	void ixgbe_down(struct ixgbe_adapter *adapter)
6511	{
6512	struct net_device *netdev = adapter->netdev;
6513	struct ixgbe_hw *hw = &adapter->hw;
6514	int i;
6515
6516	/* signal that we are down to the interrupt handler */
6517	if (test_and_set_bit(nr: __IXGBE_DOWN, addr: &adapter->state))
6518	return; /* do nothing if already down */
6519
6520	/* Shut off incoming Tx traffic */
6521	netif_tx_stop_all_queues(dev: netdev);
6522
6523	/* call carrier off first to avoid false dev_watchdog timeouts */
6524	netif_carrier_off(dev: netdev);
6525	netif_tx_disable(dev: netdev);
6526
6527	/* Disable Rx */
6528	ixgbe_disable_rx(adapter);
6529
6530	/* synchronize_rcu() needed for pending XDP buffers to drain */
6531	if (adapter->xdp_ring[0])
6532	synchronize_rcu();
6533
6534	ixgbe_irq_disable(adapter);
6535
6536	ixgbe_napi_disable_all(adapter);
6537
6538	clear_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
6539	adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
6540	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
6541
6542	timer_delete_sync(timer: &adapter->service_timer);
6543
6544	if (adapter->num_vfs) {
6545	/* Clear EITR Select mapping */
6546	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
6547
6548	/* Mark all the VFs as inactive */
6549	for (i = 0 ; i < adapter->num_vfs; i++)
6550	adapter->vfinfo[i].clear_to_send = false;
6551
6552	/* update setting rx tx for all active vfs */
6553	ixgbe_set_all_vfs(adapter);
6554	}
6555
6556	/* disable transmits in the hardware now that interrupts are off */
6557	ixgbe_disable_tx(adapter);
6558
6559	if (!pci_channel_offline(pdev: adapter->pdev))
6560	ixgbe_reset(adapter);
6561
6562	/* power down the optics for 82599 SFP+ fiber */
6563	if (hw->mac.ops.disable_tx_laser)
6564	hw->mac.ops.disable_tx_laser(hw);
6565
6566	ixgbe_clean_all_tx_rings(adapter);
6567	ixgbe_clean_all_rx_rings(adapter);
6568	if (adapter->hw.mac.type == ixgbe_mac_e610)
6569	ixgbe_disable_link_status_events(adapter);
6570	}
6571
6572	/**
6573	* ixgbe_set_eee_capable - helper function to determine EEE support on X550
6574	* @adapter: board private structure
6575	*/
6576	static void ixgbe_set_eee_capable(struct ixgbe_adapter *adapter)
6577	{
6578	struct ixgbe_hw *hw = &adapter->hw;
6579
6580	switch (hw->device_id) {
6581	case IXGBE_DEV_ID_X550EM_A_1G_T:
6582	case IXGBE_DEV_ID_X550EM_A_1G_T_L:
6583	if (!hw->phy.eee_speeds_supported)
6584	break;
6585	adapter->flags2 |= IXGBE_FLAG2_EEE_CAPABLE;
6586	if (!hw->phy.eee_speeds_advertised)
6587	break;
6588	adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED;
6589	break;
6590	default:
6591	adapter->flags2 &= ~IXGBE_FLAG2_EEE_CAPABLE;
6592	adapter->flags2 &= ~IXGBE_FLAG2_EEE_ENABLED;
6593	break;
6594	}
6595	}
6596
6597	/**
6598	* ixgbe_tx_timeout - Respond to a Tx Hang
6599	* @netdev: network interface device structure
6600	* @txqueue: queue number that timed out
6601	**/
6602	static void ixgbe_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
6603	{
6604	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
6605
6606	/* Do the reset outside of interrupt context */
6607	ixgbe_tx_timeout_reset(adapter);
6608	}
6609
6610	#ifdef CONFIG_IXGBE_DCB
6611	static void ixgbe_init_dcb(struct ixgbe_adapter *adapter)
6612	{
6613	struct ixgbe_hw *hw = &adapter->hw;
6614	struct tc_configuration *tc;
6615	int j;
6616
6617	switch (hw->mac.type) {
6618	case ixgbe_mac_82598EB:
6619	case ixgbe_mac_82599EB:
6620	adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;
6621	adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;
6622	break;
6623	case ixgbe_mac_X540:
6624	case ixgbe_mac_X550:
6625	case ixgbe_mac_e610:
6626	adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;
6627	adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;
6628	break;
6629	case ixgbe_mac_X550EM_x:
6630	case ixgbe_mac_x550em_a:
6631	default:
6632	adapter->dcb_cfg.num_tcs.pg_tcs = DEF_TRAFFIC_CLASS;
6633	adapter->dcb_cfg.num_tcs.pfc_tcs = DEF_TRAFFIC_CLASS;
6634	break;
6635	}
6636
6637	/* Configure DCB traffic classes */
6638	for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
6639	tc = &adapter->dcb_cfg.tc_config[j];
6640	tc->path[DCB_TX_CONFIG].bwg_id = 0;
6641	tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
6642	tc->path[DCB_RX_CONFIG].bwg_id = 0;
6643	tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
6644	tc->dcb_pfc = pfc_disabled;
6645	}
6646
6647	/* Initialize default user to priority mapping, UPx->TC0 */
6648	tc = &adapter->dcb_cfg.tc_config[0];
6649	tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
6650	tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
6651
6652	adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
6653	adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
6654	adapter->dcb_cfg.pfc_mode_enable = false;
6655	adapter->dcb_set_bitmap = 0x00;
6656	if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE)
6657	adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
6658	memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg,
6659	sizeof(adapter->temp_dcb_cfg));
6660	}
6661	#endif
6662
6663	/**
6664	* ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
6665	* @adapter: board private structure to initialize
6666	* @ii: pointer to ixgbe_info for device
6667	*
6668	* ixgbe_sw_init initializes the Adapter private data structure.
6669	* Fields are initialized based on PCI device information and
6670	* OS network device settings (MTU size).
6671	**/
6672	static int ixgbe_sw_init(struct ixgbe_adapter *adapter,
6673	const struct ixgbe_info *ii)
6674	{
6675	struct ixgbe_hw *hw = &adapter->hw;
6676	struct pci_dev *pdev = adapter->pdev;
6677	unsigned int rss, fdir;
6678	u32 fwsm;
6679	int i;
6680
6681	/* PCI config space info */
6682
6683	hw->vendor_id = pdev->vendor;
6684	hw->device_id = pdev->device;
6685	hw->revision_id = pdev->revision;
6686	hw->subsystem_vendor_id = pdev->subsystem_vendor;
6687	hw->subsystem_device_id = pdev->subsystem_device;
6688
6689	hw->mac.max_link_up_time = IXGBE_LINK_UP_TIME;
6690
6691	/* get_invariants needs the device IDs */
6692	ii->get_invariants(hw);
6693
6694	/* Set common capability flags and settings */
6695	rss = min_t(int, ixgbe_max_rss_indices(adapter), num_online_cpus());
6696	adapter->ring_feature[RING_F_RSS].limit = rss;
6697	adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
6698	adapter->max_q_vectors = MAX_Q_VECTORS_82599;
6699	adapter->atr_sample_rate = 20;
6700	fdir = min_t(int, IXGBE_MAX_FDIR_INDICES, num_online_cpus());
6701	adapter->ring_feature[RING_F_FDIR].limit = fdir;
6702	adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
6703	adapter->ring_feature[RING_F_VMDQ].limit = 1;
6704	#ifdef CONFIG_IXGBE_DCA
6705	adapter->flags |= IXGBE_FLAG_DCA_CAPABLE;
6706	#endif
6707	#ifdef CONFIG_IXGBE_DCB
6708	adapter->flags |= IXGBE_FLAG_DCB_CAPABLE;
6709	adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
6710	#endif
6711	#ifdef IXGBE_FCOE
6712	adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
6713	adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
6714	#ifdef CONFIG_IXGBE_DCB
6715	/* Default traffic class to use for FCoE */
6716	adapter->fcoe.up = IXGBE_FCOE_DEFTC;
6717	#endif /* CONFIG_IXGBE_DCB */
6718	#endif /* IXGBE_FCOE */
6719
6720	/* initialize static ixgbe jump table entries */
6721	adapter->jump_tables[0] = kzalloc(sizeof(*adapter->jump_tables[0]),
6722	GFP_KERNEL);
6723	if (!adapter->jump_tables[0])
6724	return -ENOMEM;
6725	adapter->jump_tables[0]->mat = ixgbe_ipv4_fields;
6726
6727	for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++)
6728	adapter->jump_tables[i] = NULL;
6729
6730	adapter->mac_table = kcalloc(hw->mac.num_rar_entries,
6731	sizeof(struct ixgbe_mac_addr),
6732	GFP_KERNEL);
6733	if (!adapter->mac_table)
6734	return -ENOMEM;
6735
6736	if (ixgbe_init_rss_key(adapter))
6737	return -ENOMEM;
6738
6739	adapter->af_xdp_zc_qps = bitmap_zalloc(IXGBE_MAX_XDP_QS, GFP_KERNEL);
6740	if (!adapter->af_xdp_zc_qps)
6741	return -ENOMEM;
6742
6743	/* Set MAC specific capability flags and exceptions */
6744	switch (hw->mac.type) {
6745	case ixgbe_mac_82598EB:
6746	adapter->flags2 &= ~IXGBE_FLAG2_RSC_CAPABLE;
6747
6748	if (hw->device_id == IXGBE_DEV_ID_82598AT)
6749	adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
6750
6751	adapter->max_q_vectors = MAX_Q_VECTORS_82598;
6752	adapter->ring_feature[RING_F_FDIR].limit = 0;
6753	adapter->atr_sample_rate = 0;
6754	adapter->fdir_pballoc = 0;
6755	#ifdef IXGBE_FCOE
6756	adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
6757	adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
6758	#ifdef CONFIG_IXGBE_DCB
6759	adapter->fcoe.up = 0;
6760	#endif /* IXGBE_DCB */
6761	#endif /* IXGBE_FCOE */
6762	break;
6763	case ixgbe_mac_82599EB:
6764	if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
6765	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
6766	break;
6767	case ixgbe_mac_X540:
6768	fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw));
6769	if (fwsm & IXGBE_FWSM_TS_ENABLED)
6770	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
6771	break;
6772	case ixgbe_mac_x550em_a:
6773	switch (hw->device_id) {
6774	case IXGBE_DEV_ID_X550EM_A_1G_T:
6775	case IXGBE_DEV_ID_X550EM_A_1G_T_L:
6776	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
6777	break;
6778	default:
6779	break;
6780	}
6781	fallthrough;
6782	case ixgbe_mac_X550EM_x:
6783	#ifdef CONFIG_IXGBE_DCB
6784	adapter->flags &= ~IXGBE_FLAG_DCB_CAPABLE;
6785	#endif
6786	#ifdef IXGBE_FCOE
6787	adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
6788	#ifdef CONFIG_IXGBE_DCB
6789	adapter->fcoe.up = 0;
6790	#endif /* IXGBE_DCB */
6791	#endif /* IXGBE_FCOE */
6792	fallthrough;
6793	case ixgbe_mac_X550:
6794	if (hw->mac.type == ixgbe_mac_X550)
6795	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
6796	#ifdef CONFIG_IXGBE_DCA
6797	adapter->flags &= ~IXGBE_FLAG_DCA_CAPABLE;
6798	#endif
6799	break;
6800	default:
6801	break;
6802	}
6803
6804	#ifdef IXGBE_FCOE
6805	/* FCoE support exists, always init the FCoE lock */
6806	spin_lock_init(&adapter->fcoe.lock);
6807
6808	#endif
6809	/* n-tuple support exists, always init our spinlock */
6810	spin_lock_init(&adapter->fdir_perfect_lock);
6811
6812	/* init spinlock to avoid concurrency of VF resources */
6813	spin_lock_init(&adapter->vfs_lock);
6814
6815	#ifdef CONFIG_IXGBE_DCB
6816	ixgbe_init_dcb(adapter);
6817	#endif
6818	ixgbe_init_ipsec_offload(adapter);
6819
6820	/* default flow control settings */
6821	hw->fc.requested_mode = ixgbe_fc_full;
6822	hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */
6823	ixgbe_pbthresh_setup(adapter);
6824	hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
6825	hw->fc.send_xon = true;
6826	hw->fc.disable_fc_autoneg = ixgbe_device_supports_autoneg_fc(hw);
6827
6828	#ifdef CONFIG_PCI_IOV
6829	if (max_vfs > 0)
6830	e_dev_warn("Enabling SR-IOV VFs using the max_vfs module parameter is deprecated - please use the pci sysfs interface instead.\n");
6831
6832	/* assign number of SR-IOV VFs */
6833	if (hw->mac.type != ixgbe_mac_82598EB) {
6834	if (max_vfs > IXGBE_MAX_VFS_DRV_LIMIT) {
6835	max_vfs = 0;
6836	e_dev_warn("max_vfs parameter out of range. Not assigning any SR-IOV VFs\n");
6837	}
6838	}
6839	#endif /* CONFIG_PCI_IOV */
6840
6841	/* enable itr by default in dynamic mode */
6842	adapter->rx_itr_setting = 1;
6843	adapter->tx_itr_setting = 1;
6844
6845	/* set default ring sizes */
6846	adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
6847	adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
6848
6849	/* set default work limits */
6850	adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK;
6851
6852	/* initialize eeprom parameters */
6853	if (hw->eeprom.ops.init_params(hw)) {
6854	e_dev_err("EEPROM initialization failed\n");
6855	return -EIO;
6856	}
6857
6858	/* PF holds first pool slot */
6859	set_bit(nr: 0, addr: adapter->fwd_bitmask);
6860	set_bit(nr: __IXGBE_DOWN, addr: &adapter->state);
6861
6862	/* enable locking for XDP_TX if we have more CPUs than queues */
6863	if (nr_cpu_ids > IXGBE_MAX_XDP_QS)
6864	static_branch_enable(&ixgbe_xdp_locking_key);
6865
6866	return 0;
6867	}
6868
6869	/**
6870	* ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
6871	* @tx_ring: tx descriptor ring (for a specific queue) to setup
6872	*
6873	* Return 0 on success, negative on failure
6874	**/
6875	int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
6876	{
6877	struct device *dev = tx_ring->dev;
6878	int orig_node = dev_to_node(dev);
6879	int ring_node = NUMA_NO_NODE;
6880	int size;
6881
6882	size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
6883
6884	if (tx_ring->q_vector)
6885	ring_node = tx_ring->q_vector->numa_node;
6886
6887	tx_ring->tx_buffer_info = vmalloc_node(size, ring_node);
6888	if (!tx_ring->tx_buffer_info)
6889	tx_ring->tx_buffer_info = vmalloc(size);
6890	if (!tx_ring->tx_buffer_info)
6891	goto err;
6892
6893	/* round up to nearest 4K */
6894	tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
6895	tx_ring->size = ALIGN(tx_ring->size, 4096);
6896
6897	set_dev_node(dev, node: ring_node);
6898	tx_ring->desc = dma_alloc_coherent(dev,
6899	size: tx_ring->size,
6900	dma_handle: &tx_ring->dma,
6901	GFP_KERNEL);
6902	set_dev_node(dev, node: orig_node);
6903	if (!tx_ring->desc)
6904	tx_ring->desc = dma_alloc_coherent(dev, size: tx_ring->size,
6905	dma_handle: &tx_ring->dma, GFP_KERNEL);
6906	if (!tx_ring->desc)
6907	goto err;
6908
6909	tx_ring->next_to_use = 0;
6910	tx_ring->next_to_clean = 0;
6911	return 0;
6912
6913	err:
6914	vfree(addr: tx_ring->tx_buffer_info);
6915	tx_ring->tx_buffer_info = NULL;
6916	dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
6917	return -ENOMEM;
6918	}
6919
6920	/**
6921	* ixgbe_setup_all_tx_resources - allocate all queues Tx resources
6922	* @adapter: board private structure
6923	*
6924	* If this function returns with an error, then it's possible one or
6925	* more of the rings is populated (while the rest are not). It is the
6926	* callers duty to clean those orphaned rings.
6927	*
6928	* Return 0 on success, negative on failure
6929	**/
6930	static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
6931	{
6932	int i, j = 0, err = 0;
6933
6934	for (i = 0; i < adapter->num_tx_queues; i++) {
6935	err = ixgbe_setup_tx_resources(tx_ring: adapter->tx_ring[i]);
6936	if (!err)
6937	continue;
6938
6939	e_err(probe, "Allocation for Tx Queue %u failed\n", i);
6940	goto err_setup_tx;
6941	}
6942	for (j = 0; j < adapter->num_xdp_queues; j++) {
6943	err = ixgbe_setup_tx_resources(tx_ring: adapter->xdp_ring[j]);
6944	if (!err)
6945	continue;
6946
6947	e_err(probe, "Allocation for Tx Queue %u failed\n", j);
6948	goto err_setup_tx;
6949	}
6950
6951	return 0;
6952	err_setup_tx:
6953	/* rewind the index freeing the rings as we go */
6954	while (j--)
6955	ixgbe_free_tx_resources(adapter->xdp_ring[j]);
6956	while (i--)
6957	ixgbe_free_tx_resources(adapter->tx_ring[i]);
6958	return err;
6959	}
6960
6961	static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring)
6962	{
6963	struct ixgbe_q_vector *q_vector = rx_ring->q_vector;
6964
6965	return q_vector ? q_vector->napi.napi_id : 0;
6966	}
6967
6968	/**
6969	* ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
6970	* @adapter: pointer to ixgbe_adapter
6971	* @rx_ring: rx descriptor ring (for a specific queue) to setup
6972	*
6973	* Returns 0 on success, negative on failure
6974	**/
6975	int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter,
6976	struct ixgbe_ring *rx_ring)
6977	{
6978	struct device *dev = rx_ring->dev;
6979	int orig_node = dev_to_node(dev);
6980	int ring_node = NUMA_NO_NODE;
6981	int size;
6982
6983	size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
6984
6985	if (rx_ring->q_vector)
6986	ring_node = rx_ring->q_vector->numa_node;
6987
6988	rx_ring->rx_buffer_info = vmalloc_node(size, ring_node);
6989	if (!rx_ring->rx_buffer_info)
6990	rx_ring->rx_buffer_info = vmalloc(size);
6991	if (!rx_ring->rx_buffer_info)
6992	goto err;
6993
6994	/* Round up to nearest 4K */
6995	rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
6996	rx_ring->size = ALIGN(rx_ring->size, 4096);
6997
6998	set_dev_node(dev, node: ring_node);
6999	rx_ring->desc = dma_alloc_coherent(dev,
7000	size: rx_ring->size,
7001	dma_handle: &rx_ring->dma,
7002	GFP_KERNEL);
7003	set_dev_node(dev, node: orig_node);
7004	if (!rx_ring->desc)
7005	rx_ring->desc = dma_alloc_coherent(dev, size: rx_ring->size,
7006	dma_handle: &rx_ring->dma, GFP_KERNEL);
7007	if (!rx_ring->desc)
7008	goto err;
7009
7010	rx_ring->next_to_clean = 0;
7011	rx_ring->next_to_use = 0;
7012
7013	/* XDP RX-queue info */
7014	if (xdp_rxq_info_reg(xdp_rxq: &rx_ring->xdp_rxq, dev: adapter->netdev,
7015	queue_index: rx_ring->queue_index, napi_id: ixgbe_rx_napi_id(rx_ring)) < 0)
7016	goto err;
7017
7018	WRITE_ONCE(rx_ring->xdp_prog, adapter->xdp_prog);
7019
7020	return 0;
7021	err:
7022	vfree(addr: rx_ring->rx_buffer_info);
7023	rx_ring->rx_buffer_info = NULL;
7024	dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
7025	return -ENOMEM;
7026	}
7027
7028	/**
7029	* ixgbe_setup_all_rx_resources - allocate all queues Rx resources
7030	* @adapter: board private structure
7031	*
7032	* If this function returns with an error, then it's possible one or
7033	* more of the rings is populated (while the rest are not). It is the
7034	* callers duty to clean those orphaned rings.
7035	*
7036	* Return 0 on success, negative on failure
7037	**/
7038	static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
7039	{
7040	int i, err = 0;
7041
7042	for (i = 0; i < adapter->num_rx_queues; i++) {
7043	err = ixgbe_setup_rx_resources(adapter, rx_ring: adapter->rx_ring[i]);
7044	if (!err)
7045	continue;
7046
7047	e_err(probe, "Allocation for Rx Queue %u failed\n", i);
7048	goto err_setup_rx;
7049	}
7050
7051	#ifdef IXGBE_FCOE
7052	err = ixgbe_setup_fcoe_ddp_resources(adapter);
7053	if (!err)
7054	#endif
7055	return 0;
7056	err_setup_rx:
7057	/* rewind the index freeing the rings as we go */
7058	while (i--)
7059	ixgbe_free_rx_resources(adapter->rx_ring[i]);
7060	return err;
7061	}
7062
7063	/**
7064	* ixgbe_free_tx_resources - Free Tx Resources per Queue
7065	* @tx_ring: Tx descriptor ring for a specific queue
7066	*
7067	* Free all transmit software resources
7068	**/
7069	void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
7070	{
7071	ixgbe_clean_tx_ring(tx_ring);
7072
7073	vfree(addr: tx_ring->tx_buffer_info);
7074	tx_ring->tx_buffer_info = NULL;
7075
7076	/* if not set, then don't free */
7077	if (!tx_ring->desc)
7078	return;
7079
7080	dma_free_coherent(dev: tx_ring->dev, size: tx_ring->size,
7081	cpu_addr: tx_ring->desc, dma_handle: tx_ring->dma);
7082
7083	tx_ring->desc = NULL;
7084	}
7085
7086	/**
7087	* ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
7088	* @adapter: board private structure
7089	*
7090	* Free all transmit software resources
7091	**/
7092	static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
7093	{
7094	int i;
7095
7096	for (i = 0; i < adapter->num_tx_queues; i++)
7097	if (adapter->tx_ring[i]->desc)
7098	ixgbe_free_tx_resources(tx_ring: adapter->tx_ring[i]);
7099	for (i = 0; i < adapter->num_xdp_queues; i++)
7100	if (adapter->xdp_ring[i]->desc)
7101	ixgbe_free_tx_resources(tx_ring: adapter->xdp_ring[i]);
7102	}
7103
7104	/**
7105	* ixgbe_free_rx_resources - Free Rx Resources
7106	* @rx_ring: ring to clean the resources from
7107	*
7108	* Free all receive software resources
7109	**/
7110	void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
7111	{
7112	ixgbe_clean_rx_ring(rx_ring);
7113
7114	rx_ring->xdp_prog = NULL;
7115	xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq);
7116	vfree(addr: rx_ring->rx_buffer_info);
7117	rx_ring->rx_buffer_info = NULL;
7118
7119	/* if not set, then don't free */
7120	if (!rx_ring->desc)
7121	return;
7122
7123	dma_free_coherent(dev: rx_ring->dev, size: rx_ring->size,
7124	cpu_addr: rx_ring->desc, dma_handle: rx_ring->dma);
7125
7126	rx_ring->desc = NULL;
7127	}
7128
7129	/**
7130	* ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
7131	* @adapter: board private structure
7132	*
7133	* Free all receive software resources
7134	**/
7135	static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
7136	{
7137	int i;
7138
7139	#ifdef IXGBE_FCOE
7140	ixgbe_free_fcoe_ddp_resources(adapter);
7141
7142	#endif
7143	for (i = 0; i < adapter->num_rx_queues; i++)
7144	if (adapter->rx_ring[i]->desc)
7145	ixgbe_free_rx_resources(rx_ring: adapter->rx_ring[i]);
7146	}
7147
7148	/**
7149	* ixgbe_max_xdp_frame_size - returns the maximum allowed frame size for XDP
7150	* @adapter: device handle, pointer to adapter
7151	*/
7152	static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter)
7153	{
7154	if (PAGE_SIZE >= 8192 || adapter->flags2 & IXGBE_FLAG2_RX_LEGACY)
7155	return IXGBE_RXBUFFER_2K;
7156	else
7157	return IXGBE_RXBUFFER_3K;
7158	}
7159
7160	/**
7161	* ixgbe_change_mtu - Change the Maximum Transfer Unit
7162	* @netdev: network interface device structure
7163	* @new_mtu: new value for maximum frame size
7164	*
7165	* Returns 0 on success, negative on failure
7166	**/
7167	static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
7168	{
7169	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
7170
7171	if (ixgbe_enabled_xdp_adapter(adapter)) {
7172	int new_frame_size = new_mtu + IXGBE_PKT_HDR_PAD;
7173
7174	if (new_frame_size > ixgbe_max_xdp_frame_size(adapter)) {
7175	e_warn(probe, "Requested MTU size is not supported with XDP\n");
7176	return -EINVAL;
7177	}
7178	}
7179
7180	/*
7181	* For 82599EB we cannot allow legacy VFs to enable their receive
7182	* paths when MTU greater than 1500 is configured. So display a
7183	* warning that legacy VFs will be disabled.
7184	*/
7185	if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
7186	(adapter->hw.mac.type == ixgbe_mac_82599EB) &&
7187	(new_mtu > ETH_DATA_LEN))
7188	e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n");
7189
7190	netdev_dbg(netdev, "changing MTU from %d to %d\n",
7191	netdev->mtu, new_mtu);
7192
7193	/* must set new MTU before calling down or up */
7194	WRITE_ONCE(netdev->mtu, new_mtu);
7195
7196	if (netif_running(dev: netdev))
7197	ixgbe_reinit_locked(adapter);
7198
7199	return 0;
7200	}
7201
7202	/**
7203	* ixgbe_open - Called when a network interface is made active
7204	* @netdev: network interface device structure
7205	*
7206	* Returns 0 on success, negative value on failure
7207	*
7208	* The open entry point is called when a network interface is made
7209	* active by the system (IFF_UP). At this point all resources needed
7210	* for transmit and receive operations are allocated, the interrupt
7211	* handler is registered with the OS, the watchdog timer is started,
7212	* and the stack is notified that the interface is ready.
7213	**/
7214	int ixgbe_open(struct net_device *netdev)
7215	{
7216	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
7217	struct ixgbe_hw *hw = &adapter->hw;
7218	int err, queues;
7219
7220	/* disallow open during test */
7221	if (test_bit(__IXGBE_TESTING, &adapter->state))
7222	return -EBUSY;
7223
7224	netif_carrier_off(dev: netdev);
7225
7226	/* allocate transmit descriptors */
7227	err = ixgbe_setup_all_tx_resources(adapter);
7228	if (err)
7229	goto err_setup_tx;
7230
7231	/* allocate receive descriptors */
7232	err = ixgbe_setup_all_rx_resources(adapter);
7233	if (err)
7234	goto err_setup_rx;
7235
7236	ixgbe_configure(adapter);
7237
7238	err = ixgbe_request_irq(adapter);
7239	if (err)
7240	goto err_req_irq;
7241
7242	/* Notify the stack of the actual queue counts. */
7243	queues = adapter->num_tx_queues;
7244	err = netif_set_real_num_tx_queues(dev: netdev, txq: queues);
7245	if (err)
7246	goto err_set_queues;
7247
7248	queues = adapter->num_rx_queues;
7249	err = netif_set_real_num_rx_queues(dev: netdev, rxq: queues);
7250	if (err)
7251	goto err_set_queues;
7252
7253	ixgbe_ptp_init(adapter);
7254
7255	ixgbe_up_complete(adapter);
7256
7257	udp_tunnel_nic_reset_ntf(dev: netdev);
7258	if (adapter->hw.mac.type == ixgbe_mac_e610) {
7259	int err = ixgbe_update_link_info(hw: &adapter->hw);
7260
7261	if (err)
7262	e_dev_err("Failed to update link info, err %d.\n", err);
7263
7264	ixgbe_check_link_cfg_err(adapter,
7265	link_cfg_err: adapter->hw.link.link_info.link_cfg_err);
7266
7267	err = ixgbe_non_sfp_link_config(hw: &adapter->hw);
7268	if (ixgbe_non_sfp_link_config(hw: &adapter->hw))
7269	e_dev_err("Link setup failed, err %d.\n", err);
7270	}
7271
7272	return 0;
7273
7274	err_set_queues:
7275	ixgbe_free_irq(adapter);
7276	err_req_irq:
7277	ixgbe_free_all_rx_resources(adapter);
7278	if (hw->phy.ops.set_phy_power && !adapter->wol)
7279	hw->phy.ops.set_phy_power(&adapter->hw, false);
7280	err_setup_rx:
7281	ixgbe_free_all_tx_resources(adapter);
7282	err_setup_tx:
7283	ixgbe_reset(adapter);
7284
7285	return err;
7286	}
7287
7288	static void ixgbe_close_suspend(struct ixgbe_adapter *adapter)
7289	{
7290	ixgbe_ptp_suspend(adapter);
7291
7292	if (adapter->hw.phy.ops.enter_lplu) {
7293	adapter->hw.phy.reset_disable = true;
7294	ixgbe_down(adapter);
7295	adapter->hw.phy.ops.enter_lplu(&adapter->hw);
7296	adapter->hw.phy.reset_disable = false;
7297	} else {
7298	ixgbe_down(adapter);
7299	}
7300
7301	ixgbe_free_irq(adapter);
7302
7303	ixgbe_free_all_tx_resources(adapter);
7304	ixgbe_free_all_rx_resources(adapter);
7305	}
7306
7307	/**
7308	* ixgbe_close - Disables a network interface
7309	* @netdev: network interface device structure
7310	*
7311	* Returns 0, this is not allowed to fail
7312	*
7313	* The close entry point is called when an interface is de-activated
7314	* by the OS. The hardware is still under the drivers control, but
7315	* needs to be disabled. A global MAC reset is issued to stop the
7316	* hardware, and all transmit and receive resources are freed.
7317	**/
7318	int ixgbe_close(struct net_device *netdev)
7319	{
7320	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
7321
7322	ixgbe_ptp_stop(adapter);
7323
7324	if (netif_device_present(dev: netdev))
7325	ixgbe_close_suspend(adapter);
7326
7327	ixgbe_fdir_filter_exit(adapter);
7328
7329	ixgbe_release_hw_control(adapter);
7330
7331	return 0;
7332	}
7333
7334	static int ixgbe_resume(struct device *dev_d)
7335	{
7336	struct pci_dev *pdev = to_pci_dev(dev_d);
7337	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7338	struct net_device *netdev = adapter->netdev;
7339	u32 err;
7340
7341	adapter->hw.hw_addr = adapter->io_addr;
7342
7343	err = pci_enable_device_mem(dev: pdev);
7344	if (err) {
7345	e_dev_err("Cannot enable PCI device from suspend\n");
7346	return err;
7347	}
7348	smp_mb__before_atomic();
7349	clear_bit(nr: __IXGBE_DISABLED, addr: &adapter->state);
7350	pci_set_master(dev: pdev);
7351
7352	device_wakeup_disable(dev: dev_d);
7353
7354	ixgbe_reset(adapter);
7355
7356	IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
7357
7358	rtnl_lock();
7359	err = ixgbe_init_interrupt_scheme(adapter);
7360	if (!err && netif_running(dev: netdev))
7361	err = ixgbe_open(netdev);
7362
7363
7364	if (!err)
7365	netif_device_attach(dev: netdev);
7366	rtnl_unlock();
7367
7368	return err;
7369	}
7370
7371	static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
7372	{
7373	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7374	struct net_device *netdev = adapter->netdev;
7375	struct ixgbe_hw *hw = &adapter->hw;
7376	u32 ctrl;
7377	u32 wufc = adapter->wol;
7378
7379	rtnl_lock();
7380	netif_device_detach(dev: netdev);
7381
7382	if (netif_running(dev: netdev))
7383	ixgbe_close_suspend(adapter);
7384
7385	ixgbe_clear_interrupt_scheme(adapter);
7386	rtnl_unlock();
7387
7388	if (hw->mac.ops.stop_link_on_d3)
7389	hw->mac.ops.stop_link_on_d3(hw);
7390
7391	if (wufc) {
7392	u32 fctrl;
7393
7394	ixgbe_set_rx_mode(netdev);
7395
7396	/* enable the optics for 82599 SFP+ fiber as we can WoL */
7397	if (hw->mac.ops.enable_tx_laser)
7398	hw->mac.ops.enable_tx_laser(hw);
7399
7400	/* enable the reception of multicast packets */
7401	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
7402	fctrl |= IXGBE_FCTRL_MPE;
7403	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
7404
7405	ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
7406	ctrl |= IXGBE_CTRL_GIO_DIS;
7407	IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
7408
7409	IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
7410	} else {
7411	IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
7412	IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
7413	}
7414
7415	switch (hw->mac.type) {
7416	case ixgbe_mac_82598EB:
7417	pci_wake_from_d3(dev: pdev, enable: false);
7418	break;
7419	case ixgbe_mac_82599EB:
7420	case ixgbe_mac_X540:
7421	case ixgbe_mac_X550:
7422	case ixgbe_mac_X550EM_x:
7423	case ixgbe_mac_x550em_a:
7424	case ixgbe_mac_e610:
7425	pci_wake_from_d3(dev: pdev, enable: !!wufc);
7426	break;
7427	default:
7428	break;
7429	}
7430
7431	*enable_wake = !!wufc;
7432	if (hw->phy.ops.set_phy_power && !*enable_wake)
7433	hw->phy.ops.set_phy_power(hw, false);
7434
7435	ixgbe_release_hw_control(adapter);
7436
7437	if (!test_and_set_bit(nr: __IXGBE_DISABLED, addr: &adapter->state))
7438	pci_disable_device(dev: pdev);
7439
7440	return 0;
7441	}
7442
7443	static int ixgbe_suspend(struct device *dev_d)
7444	{
7445	struct pci_dev *pdev = to_pci_dev(dev_d);
7446	int retval;
7447	bool wake;
7448
7449	retval = __ixgbe_shutdown(pdev, enable_wake: &wake);
7450
7451	device_set_wakeup_enable(dev: dev_d, enable: wake);
7452
7453	return retval;
7454	}
7455
7456	static void ixgbe_shutdown(struct pci_dev *pdev)
7457	{
7458	bool wake;
7459
7460	__ixgbe_shutdown(pdev, enable_wake: &wake);
7461
7462	if (system_state == SYSTEM_POWER_OFF) {
7463	pci_wake_from_d3(dev: pdev, enable: wake);
7464	pci_set_power_state(dev: pdev, PCI_D3hot);
7465	}
7466	}
7467
7468	/**
7469	* ixgbe_update_stats - Update the board statistics counters.
7470	* @adapter: board private structure
7471	**/
7472	void ixgbe_update_stats(struct ixgbe_adapter *adapter)
7473	{
7474	struct net_device *netdev = adapter->netdev;
7475	struct ixgbe_hw *hw = &adapter->hw;
7476	struct ixgbe_hw_stats *hwstats = &adapter->stats;
7477	u64 total_mpc = 0;
7478	u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
7479	u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
7480	u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
7481	u64 alloc_rx_page = 0;
7482	u64 bytes = 0, packets = 0, hw_csum_rx_error = 0;
7483
7484	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
7485	test_bit(__IXGBE_RESETTING, &adapter->state))
7486	return;
7487
7488	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
7489	u64 rsc_count = 0;
7490	u64 rsc_flush = 0;
7491	for (i = 0; i < adapter->num_rx_queues; i++) {
7492	rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
7493	rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
7494	}
7495	adapter->rsc_total_count = rsc_count;
7496	adapter->rsc_total_flush = rsc_flush;
7497	}
7498
7499	for (i = 0; i < adapter->num_rx_queues; i++) {
7500	struct ixgbe_ring *rx_ring = READ_ONCE(adapter->rx_ring[i]);
7501
7502	if (!rx_ring)
7503	continue;
7504	non_eop_descs += rx_ring->rx_stats.non_eop_descs;
7505	alloc_rx_page += rx_ring->rx_stats.alloc_rx_page;
7506	alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
7507	alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
7508	hw_csum_rx_error += rx_ring->rx_stats.csum_err;
7509	bytes += rx_ring->stats.bytes;
7510	packets += rx_ring->stats.packets;
7511	}
7512	adapter->non_eop_descs = non_eop_descs;
7513	adapter->alloc_rx_page = alloc_rx_page;
7514	adapter->alloc_rx_page_failed = alloc_rx_page_failed;
7515	adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
7516	adapter->hw_csum_rx_error = hw_csum_rx_error;
7517	netdev->stats.rx_bytes = bytes;
7518	netdev->stats.rx_packets = packets;
7519
7520	bytes = 0;
7521	packets = 0;
7522	/* gather some stats to the adapter struct that are per queue */
7523	for (i = 0; i < adapter->num_tx_queues; i++) {
7524	struct ixgbe_ring *tx_ring = READ_ONCE(adapter->tx_ring[i]);
7525
7526	if (!tx_ring)
7527	continue;
7528	restart_queue += tx_ring->tx_stats.restart_queue;
7529	tx_busy += tx_ring->tx_stats.tx_busy;
7530	bytes += tx_ring->stats.bytes;
7531	packets += tx_ring->stats.packets;
7532	}
7533	for (i = 0; i < adapter->num_xdp_queues; i++) {
7534	struct ixgbe_ring *xdp_ring = READ_ONCE(adapter->xdp_ring[i]);
7535
7536	if (!xdp_ring)
7537	continue;
7538	restart_queue += xdp_ring->tx_stats.restart_queue;
7539	tx_busy += xdp_ring->tx_stats.tx_busy;
7540	bytes += xdp_ring->stats.bytes;
7541	packets += xdp_ring->stats.packets;
7542	}
7543	adapter->restart_queue = restart_queue;
7544	adapter->tx_busy = tx_busy;
7545	netdev->stats.tx_bytes = bytes;
7546	netdev->stats.tx_packets = packets;
7547
7548	hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
7549
7550	/* 8 register reads */
7551	for (i = 0; i < 8; i++) {
7552	/* for packet buffers not used, the register should read 0 */
7553	mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
7554	missed_rx += mpc;
7555	hwstats->mpc[i] += mpc;
7556	total_mpc += hwstats->mpc[i];
7557	hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
7558	hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
7559	switch (hw->mac.type) {
7560	case ixgbe_mac_82598EB:
7561	hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
7562	hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
7563	hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
7564	hwstats->pxonrxc[i] +=
7565	IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
7566	break;
7567	case ixgbe_mac_82599EB:
7568	case ixgbe_mac_X540:
7569	case ixgbe_mac_X550:
7570	case ixgbe_mac_X550EM_x:
7571	case ixgbe_mac_x550em_a:
7572	case ixgbe_mac_e610:
7573	hwstats->pxonrxc[i] +=
7574	IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
7575	break;
7576	default:
7577	break;
7578	}
7579	}
7580
7581	/*16 register reads */
7582	for (i = 0; i < 16; i++) {
7583	hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
7584	hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
7585	if (hw->mac.type == ixgbe_mac_82599EB ||
7586	hw->mac.type == ixgbe_mac_X540 ||
7587	hw->mac.type == ixgbe_mac_X550 ||
7588	hw->mac.type == ixgbe_mac_X550EM_x ||
7589	hw->mac.type == ixgbe_mac_x550em_a ||
7590	hw->mac.type == ixgbe_mac_e610) {
7591	hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
7592	IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */
7593	hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
7594	IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */
7595	}
7596	}
7597
7598	hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
7599	/* work around hardware counting issue */
7600	hwstats->gprc -= missed_rx;
7601
7602	ixgbe_update_xoff_received(adapter);
7603
7604	/* 82598 hardware only has a 32 bit counter in the high register */
7605	switch (hw->mac.type) {
7606	case ixgbe_mac_82598EB:
7607	hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
7608	hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
7609	hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
7610	hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
7611	break;
7612	case ixgbe_mac_X540:
7613	case ixgbe_mac_X550:
7614	case ixgbe_mac_X550EM_x:
7615	case ixgbe_mac_x550em_a:
7616	case ixgbe_mac_e610:
7617	/* OS2BMC stats are X540 and later */
7618	hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
7619	hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
7620	hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
7621	hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
7622	fallthrough;
7623	case ixgbe_mac_82599EB:
7624	for (i = 0; i < 16; i++)
7625	adapter->hw_rx_no_dma_resources +=
7626	IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
7627	hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
7628	IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
7629	hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
7630	IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
7631	hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
7632	IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
7633	hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
7634	hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
7635	hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
7636	#ifdef IXGBE_FCOE
7637	hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
7638	hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
7639	hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
7640	hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
7641	hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
7642	hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
7643	/* Add up per cpu counters for total ddp aloc fail */
7644	if (adapter->fcoe.ddp_pool) {
7645	struct ixgbe_fcoe *fcoe = &adapter->fcoe;
7646	struct ixgbe_fcoe_ddp_pool *ddp_pool;
7647	unsigned int cpu;
7648	u64 noddp = 0, noddp_ext_buff = 0;
7649	for_each_possible_cpu(cpu) {
7650	ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu);
7651	noddp += ddp_pool->noddp;
7652	noddp_ext_buff += ddp_pool->noddp_ext_buff;
7653	}
7654	hwstats->fcoe_noddp = noddp;
7655	hwstats->fcoe_noddp_ext_buff = noddp_ext_buff;
7656	}
7657	#endif /* IXGBE_FCOE */
7658	break;
7659	default:
7660	break;
7661	}
7662	bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
7663	hwstats->bprc += bprc;
7664	hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
7665	if (hw->mac.type == ixgbe_mac_82598EB)
7666	hwstats->mprc -= bprc;
7667	hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
7668	hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
7669	hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
7670	hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
7671	hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
7672	hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
7673	hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
7674	hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
7675	lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
7676	hwstats->lxontxc += lxon;
7677	lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
7678	hwstats->lxofftxc += lxoff;
7679	hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
7680	hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
7681	/*
7682	* 82598 errata - tx of flow control packets is included in tx counters
7683	*/
7684	xon_off_tot = lxon + lxoff;
7685	hwstats->gptc -= xon_off_tot;
7686	hwstats->mptc -= xon_off_tot;
7687	hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
7688	hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
7689	hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
7690	hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
7691	hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
7692	hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
7693	hwstats->ptc64 -= xon_off_tot;
7694	hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
7695	hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
7696	hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
7697	hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
7698	hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
7699	hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
7700
7701	/* Fill out the OS statistics structure */
7702	netdev->stats.multicast = hwstats->mprc;
7703
7704	/* Rx Errors */
7705	netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
7706	netdev->stats.rx_dropped = 0;
7707	netdev->stats.rx_length_errors = hwstats->rlec;
7708	netdev->stats.rx_crc_errors = hwstats->crcerrs;
7709	netdev->stats.rx_missed_errors = total_mpc;
7710
7711	/* VF Stats Collection - skip while resetting because these
7712	* are not clear on read and otherwise you'll sometimes get
7713	* crazy values.
7714	*/
7715	if (!test_bit(__IXGBE_RESETTING, &adapter->state)) {
7716	for (i = 0; i < adapter->num_vfs; i++) {
7717	UPDATE_VF_COUNTER_32bit(IXGBE_PVFGPRC(i),
7718	adapter->vfinfo[i].last_vfstats.gprc,
7719	adapter->vfinfo[i].vfstats.gprc);
7720	UPDATE_VF_COUNTER_32bit(IXGBE_PVFGPTC(i),
7721	adapter->vfinfo[i].last_vfstats.gptc,
7722	adapter->vfinfo[i].vfstats.gptc);
7723	UPDATE_VF_COUNTER_36bit(IXGBE_PVFGORC_LSB(i),
7724	IXGBE_PVFGORC_MSB(i),
7725	adapter->vfinfo[i].last_vfstats.gorc,
7726	adapter->vfinfo[i].vfstats.gorc);
7727	UPDATE_VF_COUNTER_36bit(IXGBE_PVFGOTC_LSB(i),
7728	IXGBE_PVFGOTC_MSB(i),
7729	adapter->vfinfo[i].last_vfstats.gotc,
7730	adapter->vfinfo[i].vfstats.gotc);
7731	UPDATE_VF_COUNTER_32bit(IXGBE_PVFMPRC(i),
7732	adapter->vfinfo[i].last_vfstats.mprc,
7733	adapter->vfinfo[i].vfstats.mprc);
7734	}
7735	}
7736	}
7737
7738	/**
7739	* ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table
7740	* @adapter: pointer to the device adapter structure
7741	**/
7742	static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
7743	{
7744	struct ixgbe_hw *hw = &adapter->hw;
7745	int i;
7746
7747	if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
7748	return;
7749
7750	adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
7751
7752	/* if interface is down do nothing */
7753	if (test_bit(__IXGBE_DOWN, &adapter->state))
7754	return;
7755
7756	/* do nothing if we are not using signature filters */
7757	if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
7758	return;
7759
7760	adapter->fdir_overflow++;
7761
7762	if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
7763	for (i = 0; i < adapter->num_tx_queues; i++)
7764	set_bit(nr: __IXGBE_TX_FDIR_INIT_DONE,
7765	addr: &(adapter->tx_ring[i]->state));
7766	for (i = 0; i < adapter->num_xdp_queues; i++)
7767	set_bit(nr: __IXGBE_TX_FDIR_INIT_DONE,
7768	addr: &adapter->xdp_ring[i]->state);
7769	/* re-enable flow director interrupts */
7770	IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
7771	} else {
7772	e_err(probe, "failed to finish FDIR re-initialization, "
7773	"ignored adding FDIR ATR filters\n");
7774	}
7775	}
7776
7777	/**
7778	* ixgbe_check_hang_subtask - check for hung queues and dropped interrupts
7779	* @adapter: pointer to the device adapter structure
7780	*
7781	* This function serves two purposes. First it strobes the interrupt lines
7782	* in order to make certain interrupts are occurring. Secondly it sets the
7783	* bits needed to check for TX hangs. As a result we should immediately
7784	* determine if a hang has occurred.
7785	*/
7786	static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
7787	{
7788	struct ixgbe_hw *hw = &adapter->hw;
7789	u64 eics = 0;
7790	int i;
7791
7792	/* If we're down, removing or resetting, just bail */
7793	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
7794	test_bit(__IXGBE_REMOVING, &adapter->state) ||
7795	test_bit(__IXGBE_RESETTING, &adapter->state))
7796	return;
7797
7798	/* Force detection of hung controller */
7799	if (netif_carrier_ok(dev: adapter->netdev)) {
7800	for (i = 0; i < adapter->num_tx_queues; i++)
7801	set_check_for_tx_hang(adapter->tx_ring[i]);
7802	for (i = 0; i < adapter->num_xdp_queues; i++)
7803	set_check_for_tx_hang(adapter->xdp_ring[i]);
7804	}
7805
7806	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
7807	/*
7808	* for legacy and MSI interrupts don't set any bits
7809	* that are enabled for EIAM, because this operation
7810	* would set *both* EIMS and EICS for any bit in EIAM
7811	*/
7812	IXGBE_WRITE_REG(hw, IXGBE_EICS,
7813	(IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
7814	} else {
7815	/* get one bit for every active tx/rx interrupt vector */
7816	for (i = 0; i < adapter->num_q_vectors; i++) {
7817	struct ixgbe_q_vector *qv = adapter->q_vector[i];
7818	if (qv->rx.ring || qv->tx.ring)
7819	eics |= BIT_ULL(i);
7820	}
7821	}
7822
7823	/* Cause software interrupt to ensure rings are cleaned */
7824	ixgbe_irq_rearm_queues(adapter, qmask: eics);
7825	}
7826
7827	/**
7828	* ixgbe_watchdog_update_link - update the link status
7829	* @adapter: pointer to the device adapter structure
7830	**/
7831	static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
7832	{
7833	struct ixgbe_hw *hw = &adapter->hw;
7834	u32 link_speed = adapter->link_speed;
7835	bool link_up = adapter->link_up;
7836	bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
7837
7838	if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
7839	return;
7840
7841	if (hw->mac.ops.check_link) {
7842	hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
7843	} else {
7844	/* always assume link is up, if no check link function */
7845	link_speed = IXGBE_LINK_SPEED_10GB_FULL;
7846	link_up = true;
7847	}
7848
7849	if (adapter->ixgbe_ieee_pfc)
7850	pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
7851
7852	if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) {
7853	hw->mac.ops.fc_enable(hw);
7854	ixgbe_set_rx_drop_en(adapter);
7855	}
7856
7857	if (link_up ||
7858	time_after(jiffies, (adapter->link_check_timeout +
7859	IXGBE_TRY_LINK_TIMEOUT))) {
7860	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
7861	IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
7862	IXGBE_WRITE_FLUSH(hw);
7863	}
7864
7865	adapter->link_up = link_up;
7866	adapter->link_speed = link_speed;
7867	}
7868
7869	static void ixgbe_update_default_up(struct ixgbe_adapter *adapter)
7870	{
7871	#ifdef CONFIG_IXGBE_DCB
7872	struct net_device *netdev = adapter->netdev;
7873	struct dcb_app app = {
7874	.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE,
7875	.protocol = 0,
7876	};
7877	u8 up = 0;
7878
7879	if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)
7880	up = dcb_ieee_getapp_mask(netdev, &app);
7881
7882	adapter->default_up = (up > 1) ? (ffs(up) - 1) : 0;
7883	#endif
7884	}
7885
7886	/**
7887	* ixgbe_watchdog_link_is_up - update netif_carrier status and
7888	* print link up message
7889	* @adapter: pointer to the device adapter structure
7890	**/
7891	static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
7892	{
7893	struct net_device *netdev = adapter->netdev;
7894	struct ixgbe_hw *hw = &adapter->hw;
7895	u32 link_speed = adapter->link_speed;
7896	const char *speed_str;
7897	bool flow_rx, flow_tx;
7898
7899	/* only continue if link was previously down */
7900	if (netif_carrier_ok(dev: netdev))
7901	return;
7902
7903	adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
7904
7905	switch (hw->mac.type) {
7906	case ixgbe_mac_82598EB: {
7907	u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
7908	u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
7909	flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
7910	flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
7911	}
7912	break;
7913	case ixgbe_mac_X540:
7914	case ixgbe_mac_X550:
7915	case ixgbe_mac_X550EM_x:
7916	case ixgbe_mac_x550em_a:
7917	case ixgbe_mac_e610:
7918	case ixgbe_mac_82599EB: {
7919	u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
7920	u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
7921	flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
7922	flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
7923	}
7924	break;
7925	default:
7926	flow_tx = false;
7927	flow_rx = false;
7928	break;
7929	}
7930
7931	adapter->last_rx_ptp_check = jiffies;
7932
7933	if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
7934	ixgbe_ptp_start_cyclecounter(adapter);
7935
7936	switch (link_speed) {
7937	case IXGBE_LINK_SPEED_10GB_FULL:
7938	speed_str = "10 Gbps";
7939	break;
7940	case IXGBE_LINK_SPEED_5GB_FULL:
7941	speed_str = "5 Gbps";
7942	break;
7943	case IXGBE_LINK_SPEED_2_5GB_FULL:
7944	speed_str = "2.5 Gbps";
7945	break;
7946	case IXGBE_LINK_SPEED_1GB_FULL:
7947	speed_str = "1 Gbps";
7948	break;
7949	case IXGBE_LINK_SPEED_100_FULL:
7950	speed_str = "100 Mbps";
7951	break;
7952	case IXGBE_LINK_SPEED_10_FULL:
7953	speed_str = "10 Mbps";
7954	break;
7955	default:
7956	speed_str = "unknown speed";
7957	break;
7958	}
7959	e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str,
7960	((flow_rx && flow_tx) ? "RX/TX" :
7961	(flow_rx ? "RX" :
7962	(flow_tx ? "TX" : "None"))));
7963
7964	netif_carrier_on(dev: netdev);
7965	ixgbe_check_vf_rate_limit(adapter);
7966
7967	/* enable transmits */
7968	netif_tx_wake_all_queues(dev: adapter->netdev);
7969
7970	/* update the default user priority for VFs */
7971	ixgbe_update_default_up(adapter);
7972
7973	/* ping all the active vfs to let them know link has changed */
7974	ixgbe_ping_all_vfs(adapter);
7975	}
7976
7977	/**
7978	* ixgbe_watchdog_link_is_down - update netif_carrier status and
7979	* print link down message
7980	* @adapter: pointer to the adapter structure
7981	**/
7982	static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter)
7983	{
7984	struct net_device *netdev = adapter->netdev;
7985	struct ixgbe_hw *hw = &adapter->hw;
7986
7987	adapter->link_up = false;
7988	adapter->link_speed = 0;
7989
7990	/* only continue if link was up previously */
7991	if (!netif_carrier_ok(dev: netdev))
7992	return;
7993
7994	/* poll for SFP+ cable when link is down */
7995	if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
7996	adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
7997
7998	if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
7999	ixgbe_ptp_start_cyclecounter(adapter);
8000
8001	e_info(drv, "NIC Link is Down\n");
8002	netif_carrier_off(dev: netdev);
8003
8004	/* ping all the active vfs to let them know link has changed */
8005	ixgbe_ping_all_vfs(adapter);
8006	}
8007
8008	static bool ixgbe_ring_tx_pending(struct ixgbe_adapter *adapter)
8009	{
8010	int i;
8011
8012	for (i = 0; i < adapter->num_tx_queues; i++) {
8013	struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
8014
8015	if (tx_ring->next_to_use != tx_ring->next_to_clean)
8016	return true;
8017	}
8018
8019	for (i = 0; i < adapter->num_xdp_queues; i++) {
8020	struct ixgbe_ring *ring = adapter->xdp_ring[i];
8021
8022	if (ring->next_to_use != ring->next_to_clean)
8023	return true;
8024	}
8025
8026	return false;
8027	}
8028
8029	static bool ixgbe_vf_tx_pending(struct ixgbe_adapter *adapter)
8030	{
8031	struct ixgbe_hw *hw = &adapter->hw;
8032	struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
8033	u32 q_per_pool = __ALIGN_MASK(1, ~vmdq->mask);
8034
8035	int i, j;
8036
8037	if (!adapter->num_vfs)
8038	return false;
8039
8040	/* resetting the PF is only needed for MAC before X550 */
8041	if (hw->mac.type >= ixgbe_mac_X550)
8042	return false;
8043
8044	for (i = 0; i < adapter->num_vfs; i++) {
8045	for (j = 0; j < q_per_pool; j++) {
8046	u32 h, t;
8047
8048	h = IXGBE_READ_REG(hw, IXGBE_PVFTDHN(q_per_pool, i, j));
8049	t = IXGBE_READ_REG(hw, IXGBE_PVFTDTN(q_per_pool, i, j));
8050
8051	if (h != t)
8052	return true;
8053	}
8054	}
8055
8056	return false;
8057	}
8058
8059	/**
8060	* ixgbe_watchdog_flush_tx - flush queues on link down
8061	* @adapter: pointer to the device adapter structure
8062	**/
8063	static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
8064	{
8065	if (!netif_carrier_ok(dev: adapter->netdev)) {
8066	if (ixgbe_ring_tx_pending(adapter) ||
8067	ixgbe_vf_tx_pending(adapter)) {
8068	/* We've lost link, so the controller stops DMA,
8069	* but we've got queued Tx work that's never going
8070	* to get done, so reset controller to flush Tx.
8071	* (Do the reset outside of interrupt context).
8072	*/
8073	e_warn(drv, "initiating reset to clear Tx work after link loss\n");
8074	set_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
8075	}
8076	}
8077	}
8078
8079	#ifdef CONFIG_PCI_IOV
8080	static void ixgbe_bad_vf_abort(struct ixgbe_adapter *adapter, u32 vf)
8081	{
8082	struct ixgbe_hw *hw = &adapter->hw;
8083
8084	if (adapter->hw.mac.type == ixgbe_mac_82599EB &&
8085	adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) {
8086	adapter->vfinfo[vf].primary_abort_count++;
8087	if (adapter->vfinfo[vf].primary_abort_count ==
8088	IXGBE_PRIMARY_ABORT_LIMIT) {
8089	ixgbe_set_vf_link_state(adapter, vf,
8090	state: IFLA_VF_LINK_STATE_DISABLE);
8091	adapter->vfinfo[vf].primary_abort_count = 0;
8092
8093	e_info(drv,
8094	"Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on",
8095	hw->bus.func, vf,
8096	adapter->vfinfo[vf].vf_mac_addresses);
8097	}
8098	}
8099	}
8100
8101	static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
8102	{
8103	struct ixgbe_hw *hw = &adapter->hw;
8104	struct pci_dev *pdev = adapter->pdev;
8105	unsigned int vf;
8106	u32 gpc;
8107
8108	if (!(netif_carrier_ok(dev: adapter->netdev)))
8109	return;
8110
8111	gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);
8112	if (gpc) /* If incrementing then no need for the check below */
8113	return;
8114	/* Check to see if a bad DMA write target from an errant or
8115	* malicious VF has caused a PCIe error. If so then we can
8116	* issue a VFLR to the offending VF(s) and then resume without
8117	* requesting a full slot reset.
8118	*/
8119
8120	if (!pdev)
8121	return;
8122
8123	/* check status reg for all VFs owned by this PF */
8124	for (vf = 0; vf < adapter->num_vfs; ++vf) {
8125	struct pci_dev *vfdev = adapter->vfinfo[vf].vfdev;
8126	u16 status_reg;
8127
8128	if (!vfdev)
8129	continue;
8130	pci_read_config_word(dev: vfdev, PCI_STATUS, val: &status_reg);
8131	if (status_reg != IXGBE_FAILED_READ_CFG_WORD &&
8132	status_reg & PCI_STATUS_REC_MASTER_ABORT) {
8133	ixgbe_bad_vf_abort(adapter, vf);
8134	pcie_flr(dev: vfdev);
8135	}
8136	}
8137	}
8138
8139	static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
8140	{
8141	u32 ssvpc;
8142
8143	/* Do not perform spoof check for 82598 or if not in IOV mode */
8144	if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
8145	adapter->num_vfs == 0)
8146	return;
8147
8148	ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
8149
8150	/*
8151	* ssvpc register is cleared on read, if zero then no
8152	* spoofed packets in the last interval.
8153	*/
8154	if (!ssvpc)
8155	return;
8156
8157	e_warn(drv, "%u Spoofed packets detected\n", ssvpc);
8158	}
8159	#else
8160	static void ixgbe_spoof_check(struct ixgbe_adapter __always_unused *adapter)
8161	{
8162	}
8163
8164	static void
8165	ixgbe_check_for_bad_vf(struct ixgbe_adapter __always_unused *adapter)
8166	{
8167	}
8168	#endif /* CONFIG_PCI_IOV */
8169
8170
8171	/**
8172	* ixgbe_watchdog_subtask - check and bring link up
8173	* @adapter: pointer to the device adapter structure
8174	**/
8175	static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
8176	{
8177	/* if interface is down, removing or resetting, do nothing */
8178	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
8179	test_bit(__IXGBE_REMOVING, &adapter->state) ||
8180	test_bit(__IXGBE_RESETTING, &adapter->state))
8181	return;
8182
8183	ixgbe_watchdog_update_link(adapter);
8184
8185	if (adapter->link_up)
8186	ixgbe_watchdog_link_is_up(adapter);
8187	else
8188	ixgbe_watchdog_link_is_down(adapter);
8189
8190	ixgbe_check_for_bad_vf(adapter);
8191	ixgbe_spoof_check(adapter);
8192	ixgbe_update_stats(adapter);
8193
8194	ixgbe_watchdog_flush_tx(adapter);
8195	}
8196
8197	/**
8198	* ixgbe_sfp_detection_subtask - poll for SFP+ cable
8199	* @adapter: the ixgbe adapter structure
8200	**/
8201	static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
8202	{
8203	struct ixgbe_hw *hw = &adapter->hw;
8204	int err;
8205
8206	/* not searching for SFP so there is nothing to do here */
8207	if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
8208	!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
8209	return;
8210
8211	if (adapter->sfp_poll_time &&
8212	time_after(adapter->sfp_poll_time, jiffies))
8213	return; /* If not yet time to poll for SFP */
8214
8215	/* someone else is in init, wait until next service event */
8216	if (test_and_set_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state))
8217	return;
8218
8219	adapter->sfp_poll_time = jiffies + IXGBE_SFP_POLL_JIFFIES - 1;
8220
8221	err = hw->phy.ops.identify_sfp(hw);
8222	if (err == -EOPNOTSUPP)
8223	goto sfp_out;
8224
8225	if (err == -ENOENT) {
8226	/* If no cable is present, then we need to reset
8227	* the next time we find a good cable. */
8228	adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
8229	}
8230
8231	/* exit on error */
8232	if (err)
8233	goto sfp_out;
8234
8235	/* exit if reset not needed */
8236	if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
8237	goto sfp_out;
8238
8239	adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
8240
8241	/*
8242	* A module may be identified correctly, but the EEPROM may not have
8243	* support for that module. setup_sfp() will fail in that case, so
8244	* we should not allow that module to load.
8245	*/
8246	if (hw->mac.type == ixgbe_mac_82598EB)
8247	err = hw->phy.ops.reset(hw);
8248	else
8249	err = hw->mac.ops.setup_sfp(hw);
8250
8251	if (err == -EOPNOTSUPP)
8252	goto sfp_out;
8253
8254	adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
8255	e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
8256
8257	sfp_out:
8258	clear_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state);
8259
8260	if (err == -EOPNOTSUPP &&
8261	adapter->netdev->reg_state == NETREG_REGISTERED) {
8262	e_dev_err("failed to initialize because an unsupported "
8263	"SFP+ module type was detected.\n");
8264	e_dev_err("Reload the driver after installing a "
8265	"supported module.\n");
8266	unregister_netdev(dev: adapter->netdev);
8267	}
8268	}
8269
8270	/**
8271	* ixgbe_sfp_link_config_subtask - set up link SFP after module install
8272	* @adapter: the ixgbe adapter structure
8273	**/
8274	static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
8275	{
8276	struct ixgbe_hw *hw = &adapter->hw;
8277	u32 cap_speed;
8278	u32 speed;
8279	bool autoneg = false;
8280
8281	if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
8282	return;
8283
8284	/* someone else is in init, wait until next service event */
8285	if (test_and_set_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state))
8286	return;
8287
8288	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
8289
8290	hw->mac.ops.get_link_capabilities(hw, &cap_speed, &autoneg);
8291
8292	/* advertise highest capable link speed */
8293	if (!autoneg && (cap_speed & IXGBE_LINK_SPEED_10GB_FULL))
8294	speed = IXGBE_LINK_SPEED_10GB_FULL;
8295	else
8296	speed = cap_speed & (IXGBE_LINK_SPEED_10GB_FULL |
8297	IXGBE_LINK_SPEED_1GB_FULL);
8298
8299	if (hw->mac.ops.setup_link)
8300	hw->mac.ops.setup_link(hw, speed, true);
8301
8302	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
8303	adapter->link_check_timeout = jiffies;
8304	clear_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state);
8305	}
8306
8307	/**
8308	* ixgbe_service_timer - Timer Call-back
8309	* @t: pointer to timer_list structure
8310	**/
8311	static void ixgbe_service_timer(struct timer_list *t)
8312	{
8313	struct ixgbe_adapter *adapter = timer_container_of(adapter, t,
8314	service_timer);
8315	unsigned long next_event_offset;
8316
8317	/* poll faster when waiting for link */
8318	if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
8319	next_event_offset = HZ / 10;
8320	else
8321	next_event_offset = HZ * 2;
8322
8323	/* Reset the timer */
8324	mod_timer(timer: &adapter->service_timer, expires: next_event_offset + jiffies);
8325
8326	ixgbe_service_event_schedule(adapter);
8327	}
8328
8329	static void ixgbe_phy_interrupt_subtask(struct ixgbe_adapter *adapter)
8330	{
8331	struct ixgbe_hw *hw = &adapter->hw;
8332	bool overtemp;
8333
8334	if (!(adapter->flags2 & IXGBE_FLAG2_PHY_INTERRUPT))
8335	return;
8336
8337	adapter->flags2 &= ~IXGBE_FLAG2_PHY_INTERRUPT;
8338
8339	if (!hw->phy.ops.handle_lasi)
8340	return;
8341
8342	hw->phy.ops.handle_lasi(&adapter->hw, &overtemp);
8343	if (overtemp)
8344	e_crit(drv, "%s\n", ixgbe_overheat_msg);
8345	}
8346
8347	static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
8348	{
8349	if (!test_and_clear_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state))
8350	return;
8351
8352	rtnl_lock();
8353	/* If we're already down, removing or resetting, just bail */
8354	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
8355	test_bit(__IXGBE_REMOVING, &adapter->state) ||
8356	test_bit(__IXGBE_RESETTING, &adapter->state)) {
8357	rtnl_unlock();
8358	return;
8359	}
8360
8361	ixgbe_dump(adapter);
8362	netdev_err(dev: adapter->netdev, format: "Reset adapter\n");
8363	adapter->tx_timeout_count++;
8364
8365	ixgbe_reinit_locked(adapter);
8366	rtnl_unlock();
8367	}
8368
8369	static int ixgbe_check_fw_api_mismatch(struct ixgbe_adapter *adapter)
8370	{
8371	struct ixgbe_hw *hw = &adapter->hw;
8372
8373	if (hw->mac.type != ixgbe_mac_e610)
8374	return 0;
8375
8376	if (hw->mac.ops.get_fw_ver && hw->mac.ops.get_fw_ver(hw))
8377	return 0;
8378
8379	if (hw->api_maj_ver > IXGBE_FW_API_VER_MAJOR) {
8380	e_dev_err("The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
8381
8382	adapter->flags2 |= IXGBE_FLAG2_API_MISMATCH;
8383	return -EOPNOTSUPP;
8384	} else if (hw->api_maj_ver == IXGBE_FW_API_VER_MAJOR &&
8385	hw->api_min_ver > IXGBE_FW_API_VER_MINOR + IXGBE_FW_API_VER_DIFF_ALLOWED) {
8386	e_dev_info("The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
8387	adapter->flags2 |= IXGBE_FLAG2_API_MISMATCH;
8388	} else if (hw->api_maj_ver < IXGBE_FW_API_VER_MAJOR ||
8389	hw->api_min_ver < IXGBE_FW_API_VER_MINOR - IXGBE_FW_API_VER_DIFF_ALLOWED) {
8390	e_dev_info("The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
8391	adapter->flags2 |= IXGBE_FLAG2_API_MISMATCH;
8392	}
8393
8394	return 0;
8395	}
8396
8397	/**
8398	* ixgbe_check_fw_error - Check firmware for errors
8399	* @adapter: the adapter private structure
8400	*
8401	* Check firmware errors in register FWSM
8402	*/
8403	static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter)
8404	{
8405	struct ixgbe_hw *hw = &adapter->hw;
8406	u32 fwsm;
8407	int err;
8408
8409	/* read fwsm.ext_err_ind register and log errors */
8410	fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw));
8411
8412	/* skip if E610's FW is reloading, warning in that case may be misleading */
8413	if (fwsm & IXGBE_FWSM_EXT_ERR_IND_MASK ||
8414	(!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610)))
8415	e_dev_warn("Warning firmware error detected FWSM: 0x%08X\n",
8416	fwsm);
8417
8418	if (hw->mac.ops.fw_recovery_mode && hw->mac.ops.fw_recovery_mode(hw)) {
8419	e_dev_err("Firmware recovery mode detected. Limiting functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
8420	return true;
8421	}
8422	if (!(adapter->flags2 & IXGBE_FLAG2_API_MISMATCH)) {
8423	err = ixgbe_check_fw_api_mismatch(adapter);
8424	if (err)
8425	return true;
8426	}
8427
8428	/* return here if FW rollback mode has been already detected */
8429	if (adapter->flags2 & IXGBE_FLAG2_FW_ROLLBACK)
8430	return false;
8431
8432	if (hw->mac.ops.fw_rollback_mode && hw->mac.ops.fw_rollback_mode(hw)) {
8433	struct ixgbe_nvm_info *nvm_info = &adapter->hw.flash.nvm;
8434	char ver_buff[64] = "";
8435
8436	if (hw->mac.ops.get_fw_ver && hw->mac.ops.get_fw_ver(hw))
8437	goto no_version;
8438
8439	if (hw->mac.ops.get_nvm_ver &&
8440	hw->mac.ops.get_nvm_ver(hw, nvm_info))
8441	goto no_version;
8442
8443	snprintf(buf: ver_buff, size: sizeof(ver_buff),
8444	fmt: "Current version is NVM:%x.%x.%x, FW:%d.%d. ",
8445	nvm_info->major, nvm_info->minor, nvm_info->eetrack,
8446	hw->fw_maj_ver, hw->fw_maj_ver);
8447	no_version:
8448	e_dev_warn("Firmware rollback mode detected. %sDevice may exhibit limited functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware rollback mode.",
8449	ver_buff);
8450
8451	adapter->flags2 |= IXGBE_FLAG2_FW_ROLLBACK;
8452	}
8453
8454	return false;
8455	}
8456
8457	static void ixgbe_recovery_service_task(struct work_struct *work)
8458	{
8459	struct ixgbe_adapter *adapter = container_of(work,
8460	struct ixgbe_adapter,
8461	service_task);
8462
8463	ixgbe_handle_fw_event(adapter);
8464	ixgbe_service_event_complete(adapter);
8465
8466	mod_timer(timer: &adapter->service_timer, expires: jiffies + msecs_to_jiffies(m: 100));
8467	}
8468
8469	/**
8470	* ixgbe_service_task - manages and runs subtasks
8471	* @work: pointer to work_struct containing our data
8472	**/
8473	static void ixgbe_service_task(struct work_struct *work)
8474	{
8475	struct ixgbe_adapter *adapter = container_of(work,
8476	struct ixgbe_adapter,
8477	service_task);
8478	if (ixgbe_removed(addr: adapter->hw.hw_addr)) {
8479	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
8480	rtnl_lock();
8481	ixgbe_down(adapter);
8482	rtnl_unlock();
8483	}
8484	ixgbe_service_event_complete(adapter);
8485	return;
8486	}
8487	if (ixgbe_check_fw_error(adapter)) {
8488	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
8489	if (adapter->mii_bus) {
8490	mdiobus_unregister(bus: adapter->mii_bus);
8491	adapter->mii_bus = NULL;
8492	}
8493	unregister_netdev(dev: adapter->netdev);
8494	}
8495	ixgbe_service_event_complete(adapter);
8496	return;
8497	}
8498	if (adapter->hw.mac.type == ixgbe_mac_e610) {
8499	if (adapter->flags2 & IXGBE_FLAG2_FW_ASYNC_EVENT)
8500	ixgbe_handle_fw_event(adapter);
8501	ixgbe_check_media_subtask(adapter);
8502	}
8503	ixgbe_reset_subtask(adapter);
8504	ixgbe_phy_interrupt_subtask(adapter);
8505	ixgbe_sfp_detection_subtask(adapter);
8506	ixgbe_sfp_link_config_subtask(adapter);
8507	ixgbe_check_overtemp_subtask(adapter);
8508	ixgbe_watchdog_subtask(adapter);
8509	ixgbe_fdir_reinit_subtask(adapter);
8510	ixgbe_check_hang_subtask(adapter);
8511
8512	if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) {
8513	ixgbe_ptp_overflow_check(adapter);
8514	if (adapter->flags & IXGBE_FLAG_RX_HWTSTAMP_IN_REGISTER)
8515	ixgbe_ptp_rx_hang(adapter);
8516	ixgbe_ptp_tx_hang(adapter);
8517	}
8518
8519	ixgbe_service_event_complete(adapter);
8520	}
8521
8522	static int ixgbe_tso(struct ixgbe_ring *tx_ring,
8523	struct ixgbe_tx_buffer *first,
8524	u8 *hdr_len,
8525	struct ixgbe_ipsec_tx_data *itd)
8526	{
8527	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
8528	struct sk_buff *skb = first->skb;
8529	union {
8530	struct iphdr *v4;
8531	struct ipv6hdr *v6;
8532	unsigned char *hdr;
8533	} ip;
8534	union {
8535	struct tcphdr *tcp;
8536	struct udphdr *udp;
8537	unsigned char *hdr;
8538	} l4;
8539	u32 paylen, l4_offset;
8540	u32 fceof_saidx = 0;
8541	int err;
8542
8543	if (skb->ip_summed != CHECKSUM_PARTIAL)
8544	return 0;
8545
8546	if (!skb_is_gso(skb))
8547	return 0;
8548
8549	err = skb_cow_head(skb, headroom: 0);
8550	if (err < 0)
8551	return err;
8552
8553	if (eth_p_mpls(eth_type: first->protocol))
8554	ip.hdr = skb_inner_network_header(skb);
8555	else
8556	ip.hdr = skb_network_header(skb);
8557	l4.hdr = skb_checksum_start(skb);
8558
8559	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
8560	type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ?
8561	IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP;
8562
8563	/* initialize outer IP header fields */
8564	if (ip.v4->version == 4) {
8565	unsigned char *csum_start = skb_checksum_start(skb);
8566	unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
8567	int len = csum_start - trans_start;
8568
8569	/* IP header will have to cancel out any data that
8570	* is not a part of the outer IP header, so set to
8571	* a reverse csum if needed, else init check to 0.
8572	*/
8573	ip.v4->check = (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) ?
8574	csum_fold(csum: csum_partial(buff: trans_start,
8575	len, sum: 0)) : 0;
8576	type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
8577
8578	ip.v4->tot_len = 0;
8579	first->tx_flags |= IXGBE_TX_FLAGS_TSO |
8580	IXGBE_TX_FLAGS_CSUM |
8581	IXGBE_TX_FLAGS_IPV4;
8582	} else {
8583	ip.v6->payload_len = 0;
8584	first->tx_flags |= IXGBE_TX_FLAGS_TSO |
8585	IXGBE_TX_FLAGS_CSUM;
8586	}
8587
8588	/* determine offset of inner transport header */
8589	l4_offset = l4.hdr - skb->data;
8590
8591	/* remove payload length from inner checksum */
8592	paylen = skb->len - l4_offset;
8593
8594	if (type_tucmd & IXGBE_ADVTXD_TUCMD_L4T_TCP) {
8595	/* compute length of segmentation header */
8596	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
8597	csum_replace_by_diff(sum: &l4.tcp->check,
8598	diff: (__force __wsum)htonl(paylen));
8599	} else {
8600	/* compute length of segmentation header */
8601	*hdr_len = sizeof(*l4.udp) + l4_offset;
8602	csum_replace_by_diff(sum: &l4.udp->check,
8603	diff: (__force __wsum)htonl(paylen));
8604	}
8605
8606	/* update gso size and bytecount with header size */
8607	first->gso_segs = skb_shinfo(skb)->gso_segs;
8608	first->bytecount += (first->gso_segs - 1) * *hdr_len;
8609
8610	/* mss_l4len_id: use 0 as index for TSO */
8611	mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT;
8612	mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
8613
8614	fceof_saidx |= itd->sa_idx;
8615	type_tucmd |= itd->flags | itd->trailer_len;
8616
8617	/* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
8618	vlan_macip_lens = l4.hdr - ip.hdr;
8619	vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT;
8620	vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
8621
8622	ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd,
8623	mss_l4len_idx);
8624
8625	return 1;
8626	}
8627
8628	static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
8629	struct ixgbe_tx_buffer *first,
8630	struct ixgbe_ipsec_tx_data *itd)
8631	{
8632	struct sk_buff *skb = first->skb;
8633	u32 vlan_macip_lens = 0;
8634	u32 fceof_saidx = 0;
8635	u32 type_tucmd = 0;
8636
8637	if (skb->ip_summed != CHECKSUM_PARTIAL) {
8638	csum_failed:
8639	if (!(first->tx_flags & (IXGBE_TX_FLAGS_HW_VLAN |
8640	IXGBE_TX_FLAGS_CC)))
8641	return;
8642	goto no_csum;
8643	}
8644
8645	switch (skb->csum_offset) {
8646	case offsetof(struct tcphdr, check):
8647	type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
8648	fallthrough;
8649	case offsetof(struct udphdr, check):
8650	break;
8651	case offsetof(struct sctphdr, checksum):
8652	/* validate that this is actually an SCTP request */
8653	if (skb_csum_is_sctp(skb)) {
8654	type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP;
8655	break;
8656	}
8657	fallthrough;
8658	default:
8659	skb_checksum_help(skb);
8660	goto csum_failed;
8661	}
8662
8663	/* update TX checksum flag */
8664	first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
8665	vlan_macip_lens = skb_checksum_start_offset(skb) -
8666	skb_network_offset(skb);
8667	no_csum:
8668	/* vlan_macip_lens: MACLEN, VLAN tag */
8669	vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
8670	vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
8671
8672	fceof_saidx |= itd->sa_idx;
8673	type_tucmd |= itd->flags | itd->trailer_len;
8674
8675	ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd, 0);
8676	}
8677
8678	#define IXGBE_SET_FLAG(_input, _flag, _result) \
8679	((_flag <= _result) ? \
8680	((u32)(_input & _flag) * (_result / _flag)) : \
8681	((u32)(_input & _flag) / (_flag / _result)))
8682
8683	static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
8684	{
8685	/* set type for advanced descriptor with frame checksum insertion */
8686	u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA |
8687	IXGBE_ADVTXD_DCMD_DEXT |
8688	IXGBE_ADVTXD_DCMD_IFCS;
8689
8690	/* set HW vlan bit if vlan is present */
8691	cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN,
8692	IXGBE_ADVTXD_DCMD_VLE);
8693
8694	/* set segmentation enable bits for TSO/FSO */
8695	cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSO,
8696	IXGBE_ADVTXD_DCMD_TSE);
8697
8698	/* set timestamp bit if present */
8699	cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSTAMP,
8700	IXGBE_ADVTXD_MAC_TSTAMP);
8701
8702	/* insert frame checksum */
8703	cmd_type ^= IXGBE_SET_FLAG(skb->no_fcs, 1, IXGBE_ADVTXD_DCMD_IFCS);
8704
8705	return cmd_type;
8706	}
8707
8708	static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
8709	u32 tx_flags, unsigned int paylen)
8710	{
8711	u32 olinfo_status = paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
8712
8713	/* enable L4 checksum for TSO and TX checksum offload */
8714	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
8715	IXGBE_TX_FLAGS_CSUM,
8716	IXGBE_ADVTXD_POPTS_TXSM);
8717
8718	/* enable IPv4 checksum for TSO */
8719	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
8720	IXGBE_TX_FLAGS_IPV4,
8721	IXGBE_ADVTXD_POPTS_IXSM);
8722
8723	/* enable IPsec */
8724	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
8725	IXGBE_TX_FLAGS_IPSEC,
8726	IXGBE_ADVTXD_POPTS_IPSEC);
8727
8728	/*
8729	* Check Context must be set if Tx switch is enabled, which it
8730	* always is for case where virtual functions are running
8731	*/
8732	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
8733	IXGBE_TX_FLAGS_CC,
8734	IXGBE_ADVTXD_CC);
8735
8736	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
8737	}
8738
8739	static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
8740	{
8741	if (!netif_subqueue_try_stop(tx_ring->netdev, tx_ring->queue_index,
8742	ixgbe_desc_unused(tx_ring), size))
8743	return -EBUSY;
8744
8745	++tx_ring->tx_stats.restart_queue;
8746	return 0;
8747	}
8748
8749	static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
8750	{
8751	if (likely(ixgbe_desc_unused(tx_ring) >= size))
8752	return 0;
8753
8754	return __ixgbe_maybe_stop_tx(tx_ring, size);
8755	}
8756
8757	static int ixgbe_tx_map(struct ixgbe_ring *tx_ring,
8758	struct ixgbe_tx_buffer *first,
8759	const u8 hdr_len)
8760	{
8761	struct sk_buff *skb = first->skb;
8762	struct ixgbe_tx_buffer *tx_buffer;
8763	union ixgbe_adv_tx_desc *tx_desc;
8764	skb_frag_t *frag;
8765	dma_addr_t dma;
8766	unsigned int data_len, size;
8767	u32 tx_flags = first->tx_flags;
8768	u32 cmd_type = ixgbe_tx_cmd_type(skb, tx_flags);
8769	u16 i = tx_ring->next_to_use;
8770
8771	tx_desc = IXGBE_TX_DESC(tx_ring, i);
8772
8773	ixgbe_tx_olinfo_status(tx_desc, tx_flags, paylen: skb->len - hdr_len);
8774
8775	size = skb_headlen(skb);
8776	data_len = skb->data_len;
8777
8778	#ifdef IXGBE_FCOE
8779	if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
8780	if (data_len < sizeof(struct fcoe_crc_eof)) {
8781	size -= sizeof(struct fcoe_crc_eof) - data_len;
8782	data_len = 0;
8783	} else {
8784	data_len -= sizeof(struct fcoe_crc_eof);
8785	}
8786	}
8787
8788	#endif
8789	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
8790
8791	tx_buffer = first;
8792
8793	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
8794	if (dma_mapping_error(dev: tx_ring->dev, dma_addr: dma))
8795	goto dma_error;
8796
8797	/* record length, and DMA address */
8798	dma_unmap_len_set(tx_buffer, len, size);
8799	dma_unmap_addr_set(tx_buffer, dma, dma);
8800
8801	tx_desc->read.buffer_addr = cpu_to_le64(dma);
8802
8803	while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
8804	tx_desc->read.cmd_type_len =
8805	cpu_to_le32(cmd_type ^ IXGBE_MAX_DATA_PER_TXD);
8806
8807	i++;
8808	tx_desc++;
8809	if (i == tx_ring->count) {
8810	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
8811	i = 0;
8812	}
8813	tx_desc->read.olinfo_status = 0;
8814
8815	dma += IXGBE_MAX_DATA_PER_TXD;
8816	size -= IXGBE_MAX_DATA_PER_TXD;
8817
8818	tx_desc->read.buffer_addr = cpu_to_le64(dma);
8819	}
8820
8821	if (likely(!data_len))
8822	break;
8823
8824	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
8825
8826	i++;
8827	tx_desc++;
8828	if (i == tx_ring->count) {
8829	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
8830	i = 0;
8831	}
8832	tx_desc->read.olinfo_status = 0;
8833
8834	#ifdef IXGBE_FCOE
8835	size = min_t(unsigned int, data_len, skb_frag_size(frag));
8836	#else
8837	size = skb_frag_size(frag);
8838	#endif
8839	data_len -= size;
8840
8841	dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
8842	DMA_TO_DEVICE);
8843
8844	tx_buffer = &tx_ring->tx_buffer_info[i];
8845	}
8846
8847	/* write last descriptor with RS and EOP bits */
8848	cmd_type |= size | IXGBE_TXD_CMD;
8849	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
8850
8851	netdev_tx_sent_queue(dev_queue: txring_txq(ring: tx_ring), bytes: first->bytecount);
8852
8853	/* set the timestamp */
8854	first->time_stamp = jiffies;
8855
8856	skb_tx_timestamp(skb);
8857
8858	/*
8859	* Force memory writes to complete before letting h/w know there
8860	* are new descriptors to fetch. (Only applicable for weak-ordered
8861	* memory model archs, such as IA-64).
8862	*
8863	* We also need this memory barrier to make certain all of the
8864	* status bits have been updated before next_to_watch is written.
8865	*/
8866	wmb();
8867
8868	/* set next_to_watch value indicating a packet is present */
8869	first->next_to_watch = tx_desc;
8870
8871	i++;
8872	if (i == tx_ring->count)
8873	i = 0;
8874
8875	tx_ring->next_to_use = i;
8876
8877	ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
8878
8879	if (netif_xmit_stopped(dev_queue: txring_txq(ring: tx_ring)) || !netdev_xmit_more()) {
8880	writel(val: i, addr: tx_ring->tail);
8881	}
8882
8883	return 0;
8884	dma_error:
8885	dev_err(tx_ring->dev, "TX DMA map failed\n");
8886
8887	/* clear dma mappings for failed tx_buffer_info map */
8888	for (;;) {
8889	tx_buffer = &tx_ring->tx_buffer_info[i];
8890	if (dma_unmap_len(tx_buffer, len))
8891	dma_unmap_page(tx_ring->dev,
8892	dma_unmap_addr(tx_buffer, dma),
8893	dma_unmap_len(tx_buffer, len),
8894	DMA_TO_DEVICE);
8895	dma_unmap_len_set(tx_buffer, len, 0);
8896	if (tx_buffer == first)
8897	break;
8898	if (i == 0)
8899	i += tx_ring->count;
8900	i--;
8901	}
8902
8903	dev_kfree_skb_any(skb: first->skb);
8904	first->skb = NULL;
8905
8906	tx_ring->next_to_use = i;
8907
8908	return -1;
8909	}
8910
8911	static void ixgbe_atr(struct ixgbe_ring *ring,
8912	struct ixgbe_tx_buffer *first)
8913	{
8914	struct ixgbe_q_vector *q_vector = ring->q_vector;
8915	union ixgbe_atr_hash_dword input = { .dword = 0 };
8916	union ixgbe_atr_hash_dword common = { .dword = 0 };
8917	union {
8918	unsigned char *network;
8919	struct iphdr *ipv4;
8920	struct ipv6hdr *ipv6;
8921	} hdr;
8922	struct tcphdr *th;
8923	unsigned int hlen;
8924	struct sk_buff *skb;
8925	__be16 vlan_id;
8926	int l4_proto;
8927
8928	/* if ring doesn't have a interrupt vector, cannot perform ATR */
8929	if (!q_vector)
8930	return;
8931
8932	/* do nothing if sampling is disabled */
8933	if (!ring->atr_sample_rate)
8934	return;
8935
8936	ring->atr_count++;
8937
8938	/* currently only IPv4/IPv6 with TCP is supported */
8939	if ((first->protocol != htons(ETH_P_IP)) &&
8940	(first->protocol != htons(ETH_P_IPV6)))
8941	return;
8942
8943	/* snag network header to get L4 type and address */
8944	skb = first->skb;
8945	hdr.network = skb_network_header(skb);
8946	if (unlikely(hdr.network <= skb->data))
8947	return;
8948	if (skb->encapsulation &&
8949	first->protocol == htons(ETH_P_IP) &&
8950	hdr.ipv4->protocol == IPPROTO_UDP) {
8951	struct ixgbe_adapter *adapter = q_vector->adapter;
8952
8953	if (unlikely(skb_tail_pointer(skb) < hdr.network +
8954	vxlan_headroom(0)))
8955	return;
8956
8957	/* verify the port is recognized as VXLAN */
8958	if (adapter->vxlan_port &&
8959	udp_hdr(skb)->dest == adapter->vxlan_port)
8960	hdr.network = skb_inner_network_header(skb);
8961
8962	if (adapter->geneve_port &&
8963	udp_hdr(skb)->dest == adapter->geneve_port)
8964	hdr.network = skb_inner_network_header(skb);
8965	}
8966
8967	/* Make sure we have at least [minimum IPv4 header + TCP]
8968	* or [IPv6 header] bytes
8969	*/
8970	if (unlikely(skb_tail_pointer(skb) < hdr.network + 40))
8971	return;
8972
8973	/* Currently only IPv4/IPv6 with TCP is supported */
8974	switch (hdr.ipv4->version) {
8975	case IPVERSION:
8976	/* access ihl as u8 to avoid unaligned access on ia64 */
8977	hlen = (hdr.network[0] & 0x0F) << 2;
8978	l4_proto = hdr.ipv4->protocol;
8979	break;
8980	case 6:
8981	hlen = hdr.network - skb->data;
8982	l4_proto = ipv6_find_hdr(skb, offset: &hlen, IPPROTO_TCP, NULL, NULL);
8983	hlen -= hdr.network - skb->data;
8984	break;
8985	default:
8986	return;
8987	}
8988
8989	if (l4_proto != IPPROTO_TCP)
8990	return;
8991
8992	if (unlikely(skb_tail_pointer(skb) < hdr.network +
8993	hlen + sizeof(struct tcphdr)))
8994	return;
8995
8996	th = (struct tcphdr *)(hdr.network + hlen);
8997
8998	/* skip this packet since the socket is closing */
8999	if (th->fin)
9000	return;
9001
9002	/* sample on all syn packets or once every atr sample count */
9003	if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
9004	return;
9005
9006	/* reset sample count */
9007	ring->atr_count = 0;
9008
9009	vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);
9010
9011	/*
9012	* src and dst are inverted, think how the receiver sees them
9013	*
9014	* The input is broken into two sections, a non-compressed section
9015	* containing vm_pool, vlan_id, and flow_type. The rest of the data
9016	* is XORed together and stored in the compressed dword.
9017	*/
9018	input.formatted.vlan_id = vlan_id;
9019
9020	/*
9021	* since src port and flex bytes occupy the same word XOR them together
9022	* and write the value to source port portion of compressed dword
9023	*/
9024	if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN))
9025	common.port.src ^= th->dest ^ htons(ETH_P_8021Q);
9026	else
9027	common.port.src ^= th->dest ^ first->protocol;
9028	common.port.dst ^= th->source;
9029
9030	switch (hdr.ipv4->version) {
9031	case IPVERSION:
9032	input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
9033	common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
9034	break;
9035	case 6:
9036	input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
9037	common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
9038	hdr.ipv6->saddr.s6_addr32[1] ^
9039	hdr.ipv6->saddr.s6_addr32[2] ^
9040	hdr.ipv6->saddr.s6_addr32[3] ^
9041	hdr.ipv6->daddr.s6_addr32[0] ^
9042	hdr.ipv6->daddr.s6_addr32[1] ^
9043	hdr.ipv6->daddr.s6_addr32[2] ^
9044	hdr.ipv6->daddr.s6_addr32[3];
9045	break;
9046	default:
9047	break;
9048	}
9049
9050	if (hdr.network != skb_network_header(skb))
9051	input.formatted.flow_type |= IXGBE_ATR_L4TYPE_TUNNEL_MASK;
9052
9053	/* This assumes the Rx queue and Tx queue are bound to the same CPU */
9054	ixgbe_fdir_add_signature_filter_82599(hw: &q_vector->adapter->hw,
9055	input, common, queue: ring->queue_index);
9056	}
9057
9058	#ifdef IXGBE_FCOE
9059	static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb,
9060	struct net_device *sb_dev)
9061	{
9062	struct ixgbe_adapter *adapter;
9063	struct ixgbe_ring_feature *f;
9064	int txq;
9065
9066	if (sb_dev) {
9067	u8 tc = netdev_get_prio_tc_map(dev, prio: skb->priority);
9068	struct net_device *vdev = sb_dev;
9069
9070	txq = vdev->tc_to_txq[tc].offset;
9071	txq += reciprocal_scale(val: skb_get_hash(skb),
9072	ep_ro: vdev->tc_to_txq[tc].count);
9073
9074	return txq;
9075	}
9076
9077	/*
9078	* only execute the code below if protocol is FCoE
9079	* or FIP and we have FCoE enabled on the adapter
9080	*/
9081	switch (vlan_get_protocol(skb)) {
9082	case htons(ETH_P_FCOE):
9083	case htons(ETH_P_FIP):
9084	adapter = ixgbe_from_netdev(netdev: dev);
9085
9086	if (!sb_dev && (adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
9087	break;
9088	fallthrough;
9089	default:
9090	return netdev_pick_tx(dev, skb, sb_dev);
9091	}
9092
9093	f = &adapter->ring_feature[RING_F_FCOE];
9094
9095	txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :
9096	smp_processor_id();
9097
9098	while (txq >= f->indices)
9099	txq -= f->indices;
9100
9101	return txq + f->offset;
9102	}
9103
9104	#endif
9105	int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring,
9106	struct xdp_frame *xdpf)
9107	{
9108	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(frame: xdpf);
9109	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
9110	u16 i = 0, index = ring->next_to_use;
9111	struct ixgbe_tx_buffer *tx_head = &ring->tx_buffer_info[index];
9112	struct ixgbe_tx_buffer *tx_buff = tx_head;
9113	union ixgbe_adv_tx_desc *tx_desc = IXGBE_TX_DESC(ring, index);
9114	u32 cmd_type, len = xdpf->len;
9115	void *data = xdpf->data;
9116
9117	if (unlikely(ixgbe_desc_unused(ring) < 1 + nr_frags))
9118	return IXGBE_XDP_CONSUMED;
9119
9120	tx_head->bytecount = xdp_get_frame_len(xdpf);
9121	tx_head->gso_segs = 1;
9122	tx_head->xdpf = xdpf;
9123
9124	tx_desc->read.olinfo_status =
9125	cpu_to_le32(tx_head->bytecount << IXGBE_ADVTXD_PAYLEN_SHIFT);
9126
9127	for (;;) {
9128	dma_addr_t dma;
9129
9130	dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);
9131	if (dma_mapping_error(dev: ring->dev, dma_addr: dma))
9132	goto unmap;
9133
9134	dma_unmap_len_set(tx_buff, len, len);
9135	dma_unmap_addr_set(tx_buff, dma, dma);
9136
9137	cmd_type = IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_DEXT |
9138	IXGBE_ADVTXD_DCMD_IFCS | len;
9139	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
9140	tx_desc->read.buffer_addr = cpu_to_le64(dma);
9141	tx_buff->protocol = 0;
9142
9143	if (++index == ring->count)
9144	index = 0;
9145
9146	if (i == nr_frags)
9147	break;
9148
9149	tx_buff = &ring->tx_buffer_info[index];
9150	tx_desc = IXGBE_TX_DESC(ring, index);
9151	tx_desc->read.olinfo_status = 0;
9152
9153	data = skb_frag_address(frag: &sinfo->frags[i]);
9154	len = skb_frag_size(frag: &sinfo->frags[i]);
9155	i++;
9156	}
9157	/* put descriptor type bits */
9158	tx_desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD);
9159
9160	/* Avoid any potential race with xdp_xmit and cleanup */
9161	smp_wmb();
9162
9163	tx_head->next_to_watch = tx_desc;
9164	ring->next_to_use = index;
9165
9166	return IXGBE_XDP_TX;
9167
9168	unmap:
9169	for (;;) {
9170	tx_buff = &ring->tx_buffer_info[index];
9171	if (dma_unmap_len(tx_buff, len))
9172	dma_unmap_page(ring->dev, dma_unmap_addr(tx_buff, dma),
9173	dma_unmap_len(tx_buff, len),
9174	DMA_TO_DEVICE);
9175	dma_unmap_len_set(tx_buff, len, 0);
9176	if (tx_buff == tx_head)
9177	break;
9178
9179	if (!index)
9180	index += ring->count;
9181	index--;
9182	}
9183
9184	return IXGBE_XDP_CONSUMED;
9185	}
9186
9187	netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
9188	struct ixgbe_adapter *adapter,
9189	struct ixgbe_ring *tx_ring)
9190	{
9191	struct ixgbe_tx_buffer *first;
9192	int tso;
9193	u32 tx_flags = 0;
9194	unsigned short f;
9195	u16 count = TXD_USE_COUNT(skb_headlen(skb));
9196	struct ixgbe_ipsec_tx_data ipsec_tx = { 0 };
9197	__be16 protocol = skb->protocol;
9198	u8 hdr_len = 0;
9199
9200	/*
9201	* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
9202	* + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
9203	* + 2 desc gap to keep tail from touching head,
9204	* + 1 desc for context descriptor,
9205	* otherwise try next time
9206	*/
9207	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
9208	count += TXD_USE_COUNT(skb_frag_size(
9209	&skb_shinfo(skb)->frags[f]));
9210
9211	if (ixgbe_maybe_stop_tx(tx_ring, size: count + 3)) {
9212	tx_ring->tx_stats.tx_busy++;
9213	return NETDEV_TX_BUSY;
9214	}
9215
9216	/* record the location of the first descriptor for this packet */
9217	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
9218	first->skb = skb;
9219	first->bytecount = skb->len;
9220	first->gso_segs = 1;
9221
9222	/* if we have a HW VLAN tag being added default to the HW one */
9223	if (skb_vlan_tag_present(skb)) {
9224	tx_flags |= skb_vlan_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT;
9225	tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
9226	/* else if it is a SW VLAN check the next protocol and store the tag */
9227	} else if (protocol == htons(ETH_P_8021Q)) {
9228	struct vlan_hdr *vhdr, _vhdr;
9229	vhdr = skb_header_pointer(skb, ETH_HLEN, len: sizeof(_vhdr), buffer: &_vhdr);
9230	if (!vhdr)
9231	goto out_drop;
9232
9233	tx_flags |= ntohs(vhdr->h_vlan_TCI) <<
9234	IXGBE_TX_FLAGS_VLAN_SHIFT;
9235	tx_flags |= IXGBE_TX_FLAGS_SW_VLAN;
9236	}
9237	protocol = vlan_get_protocol(skb);
9238
9239	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
9240	adapter->ptp_clock) {
9241	if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
9242	!test_and_set_bit_lock(nr: __IXGBE_PTP_TX_IN_PROGRESS,
9243	addr: &adapter->state)) {
9244	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
9245	tx_flags |= IXGBE_TX_FLAGS_TSTAMP;
9246
9247	/* schedule check for Tx timestamp */
9248	adapter->ptp_tx_skb = skb_get(skb);
9249	adapter->ptp_tx_start = jiffies;
9250	schedule_work(work: &adapter->ptp_tx_work);
9251	} else {
9252	adapter->tx_hwtstamp_skipped++;
9253	}
9254	}
9255
9256	#ifdef CONFIG_PCI_IOV
9257	/*
9258	* Use the l2switch_enable flag - would be false if the DMA
9259	* Tx switch had been disabled.
9260	*/
9261	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
9262	tx_flags |= IXGBE_TX_FLAGS_CC;
9263
9264	#endif
9265	/* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */
9266	if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
9267	((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) ||
9268	(skb->priority != TC_PRIO_CONTROL))) {
9269	tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
9270	tx_flags |= (skb->priority & 0x7) <<
9271	IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT;
9272	if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) {
9273	struct vlan_ethhdr *vhdr;
9274
9275	if (skb_cow_head(skb, headroom: 0))
9276	goto out_drop;
9277	vhdr = skb_vlan_eth_hdr(skb);
9278	vhdr->h_vlan_TCI = htons(tx_flags >>
9279	IXGBE_TX_FLAGS_VLAN_SHIFT);
9280	} else {
9281	tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
9282	}
9283	}
9284
9285	/* record initial flags and protocol */
9286	first->tx_flags = tx_flags;
9287	first->protocol = protocol;
9288
9289	#ifdef IXGBE_FCOE
9290	/* setup tx offload for FCoE */
9291	if ((protocol == htons(ETH_P_FCOE)) &&
9292	(tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) {
9293	tso = ixgbe_fso(tx_ring, first, hdr_len: &hdr_len);
9294	if (tso < 0)
9295	goto out_drop;
9296
9297	goto xmit_fcoe;
9298	}
9299
9300	#endif /* IXGBE_FCOE */
9301
9302	#ifdef CONFIG_IXGBE_IPSEC
9303	if (xfrm_offload(skb) &&
9304	!ixgbe_ipsec_tx(tx_ring, first, itd: &ipsec_tx))
9305	goto out_drop;
9306	#endif
9307	tso = ixgbe_tso(tx_ring, first, hdr_len: &hdr_len, itd: &ipsec_tx);
9308	if (tso < 0)
9309	goto out_drop;
9310	else if (!tso)
9311	ixgbe_tx_csum(tx_ring, first, itd: &ipsec_tx);
9312
9313	/* add the ATR filter if ATR is on */
9314	if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
9315	ixgbe_atr(ring: tx_ring, first);
9316
9317	#ifdef IXGBE_FCOE
9318	xmit_fcoe:
9319	#endif /* IXGBE_FCOE */
9320	if (ixgbe_tx_map(tx_ring, first, hdr_len))
9321	goto cleanup_tx_timestamp;
9322
9323	return NETDEV_TX_OK;
9324
9325	out_drop:
9326	dev_kfree_skb_any(skb: first->skb);
9327	first->skb = NULL;
9328	cleanup_tx_timestamp:
9329	if (unlikely(tx_flags & IXGBE_TX_FLAGS_TSTAMP)) {
9330	dev_kfree_skb_any(skb: adapter->ptp_tx_skb);
9331	adapter->ptp_tx_skb = NULL;
9332	cancel_work_sync(work: &adapter->ptp_tx_work);
9333	clear_bit_unlock(nr: __IXGBE_PTP_TX_IN_PROGRESS, addr: &adapter->state);
9334	}
9335
9336	return NETDEV_TX_OK;
9337	}
9338
9339	static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb,
9340	struct net_device *netdev,
9341	struct ixgbe_ring *ring)
9342	{
9343	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9344	struct ixgbe_ring *tx_ring;
9345
9346	/*
9347	* The minimum packet size for olinfo paylen is 17 so pad the skb
9348	* in order to meet this minimum size requirement.
9349	*/
9350	if (skb_put_padto(skb, len: 17))
9351	return NETDEV_TX_OK;
9352
9353	tx_ring = ring ? ring : adapter->tx_ring[skb_get_queue_mapping(skb)];
9354	if (unlikely(test_bit(__IXGBE_TX_DISABLED, &tx_ring->state)))
9355	return NETDEV_TX_BUSY;
9356
9357	return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
9358	}
9359
9360	static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb,
9361	struct net_device *netdev)
9362	{
9363	return __ixgbe_xmit_frame(skb, netdev, NULL);
9364	}
9365
9366	/**
9367	* ixgbe_set_mac - Change the Ethernet Address of the NIC
9368	* @netdev: network interface device structure
9369	* @p: pointer to an address structure
9370	*
9371	* Returns 0 on success, negative on failure
9372	**/
9373	static int ixgbe_set_mac(struct net_device *netdev, void *p)
9374	{
9375	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9376	struct ixgbe_hw *hw = &adapter->hw;
9377	struct sockaddr *addr = p;
9378
9379	if (!is_valid_ether_addr(addr: addr->sa_data))
9380	return -EADDRNOTAVAIL;
9381
9382	eth_hw_addr_set(dev: netdev, addr: addr->sa_data);
9383	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9384
9385	ixgbe_mac_set_default_filter(adapter);
9386
9387	return 0;
9388	}
9389
9390	static int
9391	ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
9392	{
9393	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9394	struct ixgbe_hw *hw = &adapter->hw;
9395	u16 value;
9396	int rc;
9397
9398	if (adapter->mii_bus) {
9399	int regnum = addr;
9400
9401	if (devad != MDIO_DEVAD_NONE)
9402	return mdiobus_c45_read(bus: adapter->mii_bus, addr: prtad,
9403	devad, regnum);
9404
9405	return mdiobus_read(bus: adapter->mii_bus, addr: prtad, regnum);
9406	}
9407
9408	if (prtad != hw->phy.mdio.prtad)
9409	return -EINVAL;
9410	rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
9411	if (!rc)
9412	rc = value;
9413	return rc;
9414	}
9415
9416	static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
9417	u16 addr, u16 value)
9418	{
9419	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9420	struct ixgbe_hw *hw = &adapter->hw;
9421
9422	if (adapter->mii_bus) {
9423	int regnum = addr;
9424
9425	if (devad != MDIO_DEVAD_NONE)
9426	return mdiobus_c45_write(bus: adapter->mii_bus, addr: prtad, devad,
9427	regnum, val: value);
9428
9429	return mdiobus_write(bus: adapter->mii_bus, addr: prtad, regnum, val: value);
9430	}
9431
9432	if (prtad != hw->phy.mdio.prtad)
9433	return -EINVAL;
9434	return hw->phy.ops.write_reg(hw, addr, devad, value);
9435	}
9436
9437	static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
9438	{
9439	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9440
9441	switch (cmd) {
9442	case SIOCSHWTSTAMP:
9443	return ixgbe_ptp_set_ts_config(adapter, ifr: req);
9444	case SIOCGHWTSTAMP:
9445	return ixgbe_ptp_get_ts_config(adapter, ifr: req);
9446	case SIOCGMIIPHY:
9447	if (!adapter->hw.phy.ops.read_reg)
9448	return -EOPNOTSUPP;
9449	fallthrough;
9450	default:
9451	return mdio_mii_ioctl(mdio: &adapter->hw.phy.mdio, mii_data: if_mii(rq: req), cmd);
9452	}
9453	}
9454
9455	/**
9456	* ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
9457	* netdev->dev_addrs
9458	* @dev: network interface device structure
9459	*
9460	* Returns non-zero on failure
9461	**/
9462	static int ixgbe_add_sanmac_netdev(struct net_device *dev)
9463	{
9464	int err = 0;
9465	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
9466	struct ixgbe_hw *hw = &adapter->hw;
9467
9468	if (is_valid_ether_addr(addr: hw->mac.san_addr)) {
9469	rtnl_lock();
9470	err = dev_addr_add(dev, addr: hw->mac.san_addr, NETDEV_HW_ADDR_T_SAN);
9471	rtnl_unlock();
9472
9473	/* update SAN MAC vmdq pool selection */
9474	hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
9475	}
9476	return err;
9477	}
9478
9479	/**
9480	* ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
9481	* netdev->dev_addrs
9482	* @dev: network interface device structure
9483	*
9484	* Returns non-zero on failure
9485	**/
9486	static int ixgbe_del_sanmac_netdev(struct net_device *dev)
9487	{
9488	int err = 0;
9489	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
9490	struct ixgbe_mac_info *mac = &adapter->hw.mac;
9491
9492	if (is_valid_ether_addr(addr: mac->san_addr)) {
9493	rtnl_lock();
9494	err = dev_addr_del(dev, addr: mac->san_addr, NETDEV_HW_ADDR_T_SAN);
9495	rtnl_unlock();
9496	}
9497	return err;
9498	}
9499
9500	static void ixgbe_get_ring_stats64(struct rtnl_link_stats64 *stats,
9501	struct ixgbe_ring *ring)
9502	{
9503	u64 bytes, packets;
9504	unsigned int start;
9505
9506	if (ring) {
9507	do {
9508	start = u64_stats_fetch_begin(syncp: &ring->syncp);
9509	packets = ring->stats.packets;
9510	bytes = ring->stats.bytes;
9511	} while (u64_stats_fetch_retry(syncp: &ring->syncp, start));
9512	stats->tx_packets += packets;
9513	stats->tx_bytes += bytes;
9514	}
9515	}
9516
9517	static void ixgbe_get_stats64(struct net_device *netdev,
9518	struct rtnl_link_stats64 *stats)
9519	{
9520	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9521	int i;
9522
9523	rcu_read_lock();
9524	for (i = 0; i < adapter->num_rx_queues; i++) {
9525	struct ixgbe_ring *ring = READ_ONCE(adapter->rx_ring[i]);
9526	u64 bytes, packets;
9527	unsigned int start;
9528
9529	if (ring) {
9530	do {
9531	start = u64_stats_fetch_begin(syncp: &ring->syncp);
9532	packets = ring->stats.packets;
9533	bytes = ring->stats.bytes;
9534	} while (u64_stats_fetch_retry(syncp: &ring->syncp, start));
9535	stats->rx_packets += packets;
9536	stats->rx_bytes += bytes;
9537	}
9538	}
9539
9540	for (i = 0; i < adapter->num_tx_queues; i++) {
9541	struct ixgbe_ring *ring = READ_ONCE(adapter->tx_ring[i]);
9542
9543	ixgbe_get_ring_stats64(stats, ring);
9544	}
9545	for (i = 0; i < adapter->num_xdp_queues; i++) {
9546	struct ixgbe_ring *ring = READ_ONCE(adapter->xdp_ring[i]);
9547
9548	ixgbe_get_ring_stats64(stats, ring);
9549	}
9550	rcu_read_unlock();
9551
9552	/* following stats updated by ixgbe_watchdog_task() */
9553	stats->multicast = netdev->stats.multicast;
9554	stats->rx_errors = netdev->stats.rx_errors;
9555	stats->rx_length_errors = netdev->stats.rx_length_errors;
9556	stats->rx_crc_errors = netdev->stats.rx_crc_errors;
9557	stats->rx_missed_errors = netdev->stats.rx_missed_errors;
9558	}
9559
9560	static int ixgbe_ndo_get_vf_stats(struct net_device *netdev, int vf,
9561	struct ifla_vf_stats *vf_stats)
9562	{
9563	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9564
9565	if (vf < 0 || vf >= adapter->num_vfs)
9566	return -EINVAL;
9567
9568	vf_stats->rx_packets = adapter->vfinfo[vf].vfstats.gprc;
9569	vf_stats->rx_bytes = adapter->vfinfo[vf].vfstats.gorc;
9570	vf_stats->tx_packets = adapter->vfinfo[vf].vfstats.gptc;
9571	vf_stats->tx_bytes = adapter->vfinfo[vf].vfstats.gotc;
9572	vf_stats->multicast = adapter->vfinfo[vf].vfstats.mprc;
9573
9574	return 0;
9575	}
9576
9577	#ifdef CONFIG_IXGBE_DCB
9578	/**
9579	* ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.
9580	* @adapter: pointer to ixgbe_adapter
9581	* @tc: number of traffic classes currently enabled
9582	*
9583	* Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm
9584	* 802.1Q priority maps to a packet buffer that exists.
9585	*/
9586	static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)
9587	{
9588	struct ixgbe_hw *hw = &adapter->hw;
9589	u32 reg, rsave;
9590	int i;
9591
9592	/* 82598 have a static priority to TC mapping that can not
9593	* be changed so no validation is needed.
9594	*/
9595	if (hw->mac.type == ixgbe_mac_82598EB)
9596	return;
9597
9598	reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);
9599	rsave = reg;
9600
9601	for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
9602	u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);
9603
9604	/* If up2tc is out of bounds default to zero */
9605	if (up2tc > tc)
9606	reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);
9607	}
9608
9609	if (reg != rsave)
9610	IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);
9611
9612	return;
9613	}
9614
9615	/**
9616	* ixgbe_set_prio_tc_map - Configure netdev prio tc map
9617	* @adapter: Pointer to adapter struct
9618	*
9619	* Populate the netdev user priority to tc map
9620	*/
9621	static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter)
9622	{
9623	struct net_device *dev = adapter->netdev;
9624	struct ixgbe_dcb_config *dcb_cfg = &adapter->dcb_cfg;
9625	struct ieee_ets *ets = adapter->ixgbe_ieee_ets;
9626	u8 prio;
9627
9628	for (prio = 0; prio < MAX_USER_PRIORITY; prio++) {
9629	u8 tc = 0;
9630
9631	if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE)
9632	tc = ixgbe_dcb_get_tc_from_up(dcb_cfg, 0, prio);
9633	else if (ets)
9634	tc = ets->prio_tc[prio];
9635
9636	netdev_set_prio_tc_map(dev, prio, tc);
9637	}
9638	}
9639
9640	#endif /* CONFIG_IXGBE_DCB */
9641	static int ixgbe_reassign_macvlan_pool(struct net_device *vdev,
9642	struct netdev_nested_priv *priv)
9643	{
9644	struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data;
9645	struct ixgbe_fwd_adapter *accel;
9646	int pool;
9647
9648	/* we only care about macvlans... */
9649	if (!netif_is_macvlan(dev: vdev))
9650	return 0;
9651
9652	/* that have hardware offload enabled... */
9653	accel = macvlan_accel_priv(dev: vdev);
9654	if (!accel)
9655	return 0;
9656
9657	/* If we can relocate to a different bit do so */
9658	pool = find_first_zero_bit(addr: adapter->fwd_bitmask, size: adapter->num_rx_pools);
9659	if (pool < adapter->num_rx_pools) {
9660	set_bit(nr: pool, addr: adapter->fwd_bitmask);
9661	accel->pool = pool;
9662	return 0;
9663	}
9664
9665	/* if we cannot find a free pool then disable the offload */
9666	netdev_err(dev: vdev, format: "L2FW offload disabled due to lack of queue resources\n");
9667	macvlan_release_l2fw_offload(dev: vdev);
9668
9669	/* unbind the queues and drop the subordinate channel config */
9670	netdev_unbind_sb_channel(dev: adapter->netdev, sb_dev: vdev);
9671	netdev_set_sb_channel(dev: vdev, channel: 0);
9672
9673	kfree(objp: accel);
9674
9675	return 0;
9676	}
9677
9678	static void ixgbe_defrag_macvlan_pools(struct net_device *dev)
9679	{
9680	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
9681	struct netdev_nested_priv priv = {
9682	.data = (void *)adapter,
9683	};
9684
9685	/* flush any stale bits out of the fwd bitmask */
9686	bitmap_clear(map: adapter->fwd_bitmask, start: 1, nbits: 63);
9687
9688	/* walk through upper devices reassigning pools */
9689	netdev_walk_all_upper_dev_rcu(dev, fn: ixgbe_reassign_macvlan_pool,
9690	priv: &priv);
9691	}
9692
9693	/**
9694	* ixgbe_setup_tc - configure net_device for multiple traffic classes
9695	*
9696	* @dev: net device to configure
9697	* @tc: number of traffic classes to enable
9698	*/
9699	int ixgbe_setup_tc(struct net_device *dev, u8 tc)
9700	{
9701	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
9702	struct ixgbe_hw *hw = &adapter->hw;
9703
9704	/* Hardware supports up to 8 traffic classes */
9705	if (tc > adapter->dcb_cfg.num_tcs.pg_tcs)
9706	return -EINVAL;
9707
9708	if (hw->mac.type == ixgbe_mac_82598EB && tc && tc < MAX_TRAFFIC_CLASS)
9709	return -EINVAL;
9710
9711	/* Hardware has to reinitialize queues and interrupts to
9712	* match packet buffer alignment. Unfortunately, the
9713	* hardware is not flexible enough to do this dynamically.
9714	*/
9715	if (netif_running(dev))
9716	ixgbe_close(netdev: dev);
9717	else
9718	ixgbe_reset(adapter);
9719
9720	ixgbe_clear_interrupt_scheme(adapter);
9721
9722	#ifdef CONFIG_IXGBE_DCB
9723	if (tc) {
9724	if (adapter->xdp_prog) {
9725	e_warn(probe, "DCB is not supported with XDP\n");
9726
9727	ixgbe_init_interrupt_scheme(adapter);
9728	if (netif_running(dev))
9729	ixgbe_open(netdev: dev);
9730	return -EINVAL;
9731	}
9732
9733	netdev_set_num_tc(dev, num_tc: tc);
9734	ixgbe_set_prio_tc_map(adapter);
9735
9736	adapter->hw_tcs = tc;
9737	adapter->flags |= IXGBE_FLAG_DCB_ENABLED;
9738
9739	if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
9740	adapter->last_lfc_mode = adapter->hw.fc.requested_mode;
9741	adapter->hw.fc.requested_mode = ixgbe_fc_none;
9742	}
9743	} else {
9744	netdev_reset_tc(dev);
9745
9746	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
9747	adapter->hw.fc.requested_mode = adapter->last_lfc_mode;
9748
9749	adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
9750	adapter->hw_tcs = tc;
9751
9752	adapter->temp_dcb_cfg.pfc_mode_enable = false;
9753	adapter->dcb_cfg.pfc_mode_enable = false;
9754	}
9755
9756	ixgbe_validate_rtr(adapter, tc);
9757
9758	#endif /* CONFIG_IXGBE_DCB */
9759	ixgbe_init_interrupt_scheme(adapter);
9760
9761	ixgbe_defrag_macvlan_pools(dev);
9762
9763	if (netif_running(dev))
9764	return ixgbe_open(netdev: dev);
9765
9766	return 0;
9767	}
9768
9769	static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter,
9770	struct tc_cls_u32_offload *cls)
9771	{
9772	u32 hdl = cls->knode.handle;
9773	u32 uhtid = TC_U32_USERHTID(cls->knode.handle);
9774	u32 loc = cls->knode.handle & 0xfffff;
9775	int err = 0, i, j;
9776	struct ixgbe_jump_table *jump = NULL;
9777
9778	if (loc > IXGBE_MAX_HW_ENTRIES)
9779	return -EINVAL;
9780
9781	if ((uhtid != 0x800) && (uhtid >= IXGBE_MAX_LINK_HANDLE))
9782	return -EINVAL;
9783
9784	/* Clear this filter in the link data it is associated with */
9785	if (uhtid != 0x800) {
9786	jump = adapter->jump_tables[uhtid];
9787	if (!jump)
9788	return -EINVAL;
9789	if (!test_bit(loc - 1, jump->child_loc_map))
9790	return -EINVAL;
9791	clear_bit(nr: loc - 1, addr: jump->child_loc_map);
9792	}
9793
9794	/* Check if the filter being deleted is a link */
9795	for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) {
9796	jump = adapter->jump_tables[i];
9797	if (jump && jump->link_hdl == hdl) {
9798	/* Delete filters in the hardware in the child hash
9799	* table associated with this link
9800	*/
9801	for (j = 0; j < IXGBE_MAX_HW_ENTRIES; j++) {
9802	if (!test_bit(j, jump->child_loc_map))
9803	continue;
9804	spin_lock(lock: &adapter->fdir_perfect_lock);
9805	err = ixgbe_update_ethtool_fdir_entry(adapter,
9806	NULL,
9807	sw_idx: j + 1);
9808	spin_unlock(lock: &adapter->fdir_perfect_lock);
9809	clear_bit(nr: j, addr: jump->child_loc_map);
9810	}
9811	/* Remove resources for this link */
9812	kfree(objp: jump->input);
9813	kfree(objp: jump->mask);
9814	kfree(objp: jump);
9815	adapter->jump_tables[i] = NULL;
9816	return err;
9817	}
9818	}
9819
9820	spin_lock(lock: &adapter->fdir_perfect_lock);
9821	err = ixgbe_update_ethtool_fdir_entry(adapter, NULL, sw_idx: loc);
9822	spin_unlock(lock: &adapter->fdir_perfect_lock);
9823	return err;
9824	}
9825
9826	static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter,
9827	struct tc_cls_u32_offload *cls)
9828	{
9829	u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);
9830
9831	if (uhtid >= IXGBE_MAX_LINK_HANDLE)
9832	return -EINVAL;
9833
9834	/* This ixgbe devices do not support hash tables at the moment
9835	* so abort when given hash tables.
9836	*/
9837	if (cls->hnode.divisor > 0)
9838	return -EINVAL;
9839
9840	set_bit(nr: uhtid - 1, addr: &adapter->tables);
9841	return 0;
9842	}
9843
9844	static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter,
9845	struct tc_cls_u32_offload *cls)
9846	{
9847	u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);
9848
9849	if (uhtid >= IXGBE_MAX_LINK_HANDLE)
9850	return -EINVAL;
9851
9852	clear_bit(nr: uhtid - 1, addr: &adapter->tables);
9853	return 0;
9854	}
9855
9856	#ifdef CONFIG_NET_CLS_ACT
9857	struct upper_walk_data {
9858	struct ixgbe_adapter *adapter;
9859	u64 action;
9860	int ifindex;
9861	u8 queue;
9862	};
9863
9864	static int get_macvlan_queue(struct net_device *upper,
9865	struct netdev_nested_priv *priv)
9866	{
9867	if (netif_is_macvlan(dev: upper)) {
9868	struct ixgbe_fwd_adapter *vadapter = macvlan_accel_priv(dev: upper);
9869	struct ixgbe_adapter *adapter;
9870	struct upper_walk_data *data;
9871	int ifindex;
9872
9873	data = (struct upper_walk_data *)priv->data;
9874	ifindex = data->ifindex;
9875	adapter = data->adapter;
9876	if (vadapter && upper->ifindex == ifindex) {
9877	data->queue = adapter->rx_ring[vadapter->rx_base_queue]->reg_idx;
9878	data->action = data->queue;
9879	return 1;
9880	}
9881	}
9882
9883	return 0;
9884	}
9885
9886	static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex,
9887	u8 *queue, u64 *action)
9888	{
9889	struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
9890	unsigned int num_vfs = adapter->num_vfs, vf;
9891	struct netdev_nested_priv priv;
9892	struct upper_walk_data data;
9893	struct net_device *upper;
9894
9895	/* redirect to a SRIOV VF */
9896	for (vf = 0; vf < num_vfs; ++vf) {
9897	upper = pci_get_drvdata(pdev: adapter->vfinfo[vf].vfdev);
9898	if (upper->ifindex == ifindex) {
9899	*queue = vf * __ALIGN_MASK(1, ~vmdq->mask);
9900	*action = vf + 1;
9901	*action <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF;
9902	return 0;
9903	}
9904	}
9905
9906	/* redirect to a offloaded macvlan netdev */
9907	data.adapter = adapter;
9908	data.ifindex = ifindex;
9909	data.action = 0;
9910	data.queue = 0;
9911	priv.data = (void *)&data;
9912	if (netdev_walk_all_upper_dev_rcu(dev: adapter->netdev,
9913	fn: get_macvlan_queue, priv: &priv)) {
9914	*action = data.action;
9915	*queue = data.queue;
9916
9917	return 0;
9918	}
9919
9920	return -EINVAL;
9921	}
9922
9923	static int parse_tc_actions(struct ixgbe_adapter *adapter,
9924	struct tcf_exts *exts, u64 *action, u8 *queue)
9925	{
9926	const struct tc_action *a;
9927	int i;
9928
9929	if (!tcf_exts_has_actions(exts))
9930	return -EINVAL;
9931
9932	tcf_exts_for_each_action(i, a, exts) {
9933	/* Drop action */
9934	if (is_tcf_gact_shot(a)) {
9935	*action = IXGBE_FDIR_DROP_QUEUE;
9936	*queue = IXGBE_FDIR_DROP_QUEUE;
9937	return 0;
9938	}
9939
9940	/* Redirect to a VF or a offloaded macvlan */
9941	if (is_tcf_mirred_egress_redirect(a)) {
9942	struct net_device *dev = tcf_mirred_dev(a);
9943
9944	if (!dev)
9945	return -EINVAL;
9946	return handle_redirect_action(adapter, ifindex: dev->ifindex,
9947	queue, action);
9948	}
9949
9950	return -EINVAL;
9951	}
9952
9953	return -EINVAL;
9954	}
9955	#else
9956	static int parse_tc_actions(struct ixgbe_adapter *adapter,
9957	struct tcf_exts *exts, u64 *action, u8 *queue)
9958	{
9959	return -EINVAL;
9960	}
9961	#endif /* CONFIG_NET_CLS_ACT */
9962
9963	static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input,
9964	union ixgbe_atr_input *mask,
9965	struct tc_cls_u32_offload *cls,
9966	struct ixgbe_mat_field *field_ptr,
9967	struct ixgbe_nexthdr *nexthdr)
9968	{
9969	int i, j, off;
9970	__be32 val, m;
9971	bool found_entry = false, found_jump_field = false;
9972
9973	for (i = 0; i < cls->knode.sel->nkeys; i++) {
9974	off = cls->knode.sel->keys[i].off;
9975	val = cls->knode.sel->keys[i].val;
9976	m = cls->knode.sel->keys[i].mask;
9977
9978	for (j = 0; field_ptr[j].val; j++) {
9979	if (field_ptr[j].off == off) {
9980	field_ptr[j].val(input, mask, (__force u32)val,
9981	(__force u32)m);
9982	input->filter.formatted.flow_type |=
9983	field_ptr[j].type;
9984	found_entry = true;
9985	break;
9986	}
9987	}
9988	if (nexthdr) {
9989	if (nexthdr->off == cls->knode.sel->keys[i].off &&
9990	nexthdr->val ==
9991	(__force u32)cls->knode.sel->keys[i].val &&
9992	nexthdr->mask ==
9993	(__force u32)cls->knode.sel->keys[i].mask)
9994	found_jump_field = true;
9995	else
9996	continue;
9997	}
9998	}
9999
10000	if (nexthdr && !found_jump_field)
10001	return -EINVAL;
10002
10003	if (!found_entry)
10004	return 0;
10005
10006	mask->formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK |
10007	IXGBE_ATR_L4TYPE_MASK;
10008
10009	if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4)
10010	mask->formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK;
10011
10012	return 0;
10013	}
10014
10015	static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter,
10016	struct tc_cls_u32_offload *cls)
10017	{
10018	__be16 protocol = cls->common.protocol;
10019	u32 loc = cls->knode.handle & 0xfffff;
10020	struct ixgbe_hw *hw = &adapter->hw;
10021	struct ixgbe_mat_field *field_ptr;
10022	struct ixgbe_fdir_filter *input = NULL;
10023	union ixgbe_atr_input *mask = NULL;
10024	struct ixgbe_jump_table *jump = NULL;
10025	int i, err = -EINVAL;
10026	u8 queue;
10027	u32 uhtid, link_uhtid;
10028
10029	uhtid = TC_U32_USERHTID(cls->knode.handle);
10030	link_uhtid = TC_U32_USERHTID(cls->knode.link_handle);
10031
10032	/* At the moment cls_u32 jumps to network layer and skips past
10033	* L2 headers. The canonical method to match L2 frames is to use
10034	* negative values. However this is error prone at best but really
10035	* just broken because there is no way to "know" what sort of hdr
10036	* is in front of the network layer. Fix cls_u32 to support L2
10037	* headers when needed.
10038	*/
10039	if (protocol != htons(ETH_P_IP))
10040	return err;
10041
10042	if (loc >= ((1024 << adapter->fdir_pballoc) - 2)) {
10043	e_err(drv, "Location out of range\n");
10044	return err;
10045	}
10046
10047	/* cls u32 is a graph starting at root node 0x800. The driver tracks
10048	* links and also the fields used to advance the parser across each
10049	* link (e.g. nexthdr/eat parameters from 'tc'). This way we can map
10050	* the u32 graph onto the hardware parse graph denoted in ixgbe_model.h
10051	* To add support for new nodes update ixgbe_model.h parse structures
10052	* this function _should_ be generic try not to hardcode values here.
10053	*/
10054	if (uhtid == 0x800) {
10055	field_ptr = (adapter->jump_tables[0])->mat;
10056	} else {
10057	if (uhtid >= IXGBE_MAX_LINK_HANDLE)
10058	return err;
10059	if (!adapter->jump_tables[uhtid])
10060	return err;
10061	field_ptr = (adapter->jump_tables[uhtid])->mat;
10062	}
10063
10064	if (!field_ptr)
10065	return err;
10066
10067	/* At this point we know the field_ptr is valid and need to either
10068	* build cls_u32 link or attach filter. Because adding a link to
10069	* a handle that does not exist is invalid and the same for adding
10070	* rules to handles that don't exist.
10071	*/
10072
10073	if (link_uhtid) {
10074	struct ixgbe_nexthdr *nexthdr = ixgbe_ipv4_jumps;
10075
10076	if (link_uhtid >= IXGBE_MAX_LINK_HANDLE)
10077	return err;
10078
10079	if (!test_bit(link_uhtid - 1, &adapter->tables))
10080	return err;
10081
10082	/* Multiple filters as links to the same hash table are not
10083	* supported. To add a new filter with the same next header
10084	* but different match/jump conditions, create a new hash table
10085	* and link to it.
10086	*/
10087	if (adapter->jump_tables[link_uhtid] &&
10088	(adapter->jump_tables[link_uhtid])->link_hdl) {
10089	e_err(drv, "Link filter exists for link: %x\n",
10090	link_uhtid);
10091	return err;
10092	}
10093
10094	for (i = 0; nexthdr[i].jump; i++) {
10095	if (nexthdr[i].o != cls->knode.sel->offoff ||
10096	nexthdr[i].s != cls->knode.sel->offshift ||
10097	nexthdr[i].m !=
10098	(__force u32)cls->knode.sel->offmask)
10099	return err;
10100
10101	jump = kzalloc(sizeof(*jump), GFP_KERNEL);
10102	if (!jump)
10103	return -ENOMEM;
10104	input = kzalloc(sizeof(*input), GFP_KERNEL);
10105	if (!input) {
10106	err = -ENOMEM;
10107	goto free_jump;
10108	}
10109	mask = kzalloc(sizeof(*mask), GFP_KERNEL);
10110	if (!mask) {
10111	err = -ENOMEM;
10112	goto free_input;
10113	}
10114	jump->input = input;
10115	jump->mask = mask;
10116	jump->link_hdl = cls->knode.handle;
10117
10118	err = ixgbe_clsu32_build_input(input, mask, cls,
10119	field_ptr, nexthdr: &nexthdr[i]);
10120	if (!err) {
10121	jump->mat = nexthdr[i].jump;
10122	adapter->jump_tables[link_uhtid] = jump;
10123	break;
10124	} else {
10125	kfree(objp: mask);
10126	kfree(objp: input);
10127	kfree(objp: jump);
10128	}
10129	}
10130	return 0;
10131	}
10132
10133	input = kzalloc(sizeof(*input), GFP_KERNEL);
10134	if (!input)
10135	return -ENOMEM;
10136	mask = kzalloc(sizeof(*mask), GFP_KERNEL);
10137	if (!mask) {
10138	err = -ENOMEM;
10139	goto free_input;
10140	}
10141
10142	if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) {
10143	if ((adapter->jump_tables[uhtid])->input)
10144	memcpy(input, (adapter->jump_tables[uhtid])->input,
10145	sizeof(*input));
10146	if ((adapter->jump_tables[uhtid])->mask)
10147	memcpy(mask, (adapter->jump_tables[uhtid])->mask,
10148	sizeof(*mask));
10149
10150	/* Lookup in all child hash tables if this location is already
10151	* filled with a filter
10152	*/
10153	for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) {
10154	struct ixgbe_jump_table *link = adapter->jump_tables[i];
10155
10156	if (link && (test_bit(loc - 1, link->child_loc_map))) {
10157	e_err(drv, "Filter exists in location: %x\n",
10158	loc);
10159	err = -EINVAL;
10160	goto err_out;
10161	}
10162	}
10163	}
10164	err = ixgbe_clsu32_build_input(input, mask, cls, field_ptr, NULL);
10165	if (err)
10166	goto err_out;
10167
10168	err = parse_tc_actions(adapter, exts: cls->knode.exts, action: &input->action,
10169	queue: &queue);
10170	if (err < 0)
10171	goto err_out;
10172
10173	input->sw_idx = loc;
10174
10175	spin_lock(lock: &adapter->fdir_perfect_lock);
10176
10177	if (hlist_empty(h: &adapter->fdir_filter_list)) {
10178	memcpy(&adapter->fdir_mask, mask, sizeof(*mask));
10179	err = ixgbe_fdir_set_input_mask_82599(hw, input_mask: mask);
10180	if (err)
10181	goto err_out_w_lock;
10182	} else if (memcmp(p: &adapter->fdir_mask, q: mask, size: sizeof(*mask))) {
10183	err = -EINVAL;
10184	goto err_out_w_lock;
10185	}
10186
10187	ixgbe_atr_compute_perfect_hash_82599(input: &input->filter, mask);
10188	err = ixgbe_fdir_write_perfect_filter_82599(hw, input: &input->filter,
10189	soft_id: input->sw_idx, queue);
10190	if (err)
10191	goto err_out_w_lock;
10192
10193	ixgbe_update_ethtool_fdir_entry(adapter, input, sw_idx: input->sw_idx);
10194	spin_unlock(lock: &adapter->fdir_perfect_lock);
10195
10196	if ((uhtid != 0x800) && (adapter->jump_tables[uhtid]))
10197	set_bit(nr: loc - 1, addr: (adapter->jump_tables[uhtid])->child_loc_map);
10198
10199	kfree(objp: mask);
10200	return err;
10201	err_out_w_lock:
10202	spin_unlock(lock: &adapter->fdir_perfect_lock);
10203	err_out:
10204	kfree(objp: mask);
10205	free_input:
10206	kfree(objp: input);
10207	free_jump:
10208	kfree(objp: jump);
10209	return err;
10210	}
10211
10212	static int ixgbe_setup_tc_cls_u32(struct ixgbe_adapter *adapter,
10213	struct tc_cls_u32_offload *cls_u32)
10214	{
10215	switch (cls_u32->command) {
10216	case TC_CLSU32_NEW_KNODE:
10217	case TC_CLSU32_REPLACE_KNODE:
10218	return ixgbe_configure_clsu32(adapter, cls: cls_u32);
10219	case TC_CLSU32_DELETE_KNODE:
10220	return ixgbe_delete_clsu32(adapter, cls: cls_u32);
10221	case TC_CLSU32_NEW_HNODE:
10222	case TC_CLSU32_REPLACE_HNODE:
10223	return ixgbe_configure_clsu32_add_hnode(adapter, cls: cls_u32);
10224	case TC_CLSU32_DELETE_HNODE:
10225	return ixgbe_configure_clsu32_del_hnode(adapter, cls: cls_u32);
10226	default:
10227	return -EOPNOTSUPP;
10228	}
10229	}
10230
10231	static int ixgbe_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
10232	void *cb_priv)
10233	{
10234	struct ixgbe_adapter *adapter = cb_priv;
10235
10236	if (!tc_cls_can_offload_and_chain0(dev: adapter->netdev, common: type_data))
10237	return -EOPNOTSUPP;
10238
10239	switch (type) {
10240	case TC_SETUP_CLSU32:
10241	return ixgbe_setup_tc_cls_u32(adapter, cls_u32: type_data);
10242	default:
10243	return -EOPNOTSUPP;
10244	}
10245	}
10246
10247	static int ixgbe_setup_tc_mqprio(struct net_device *dev,
10248	struct tc_mqprio_qopt *mqprio)
10249	{
10250	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
10251	return ixgbe_setup_tc(dev, tc: mqprio->num_tc);
10252	}
10253
10254	static LIST_HEAD(ixgbe_block_cb_list);
10255
10256	static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type,
10257	void *type_data)
10258	{
10259	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10260
10261	switch (type) {
10262	case TC_SETUP_BLOCK:
10263	return flow_block_cb_setup_simple(f: type_data,
10264	driver_list: &ixgbe_block_cb_list,
10265	cb: ixgbe_setup_tc_block_cb,
10266	cb_ident: adapter, cb_priv: adapter, ingress_only: true);
10267	case TC_SETUP_QDISC_MQPRIO:
10268	return ixgbe_setup_tc_mqprio(dev, mqprio: type_data);
10269	default:
10270	return -EOPNOTSUPP;
10271	}
10272	}
10273
10274	#ifdef CONFIG_PCI_IOV
10275	void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter)
10276	{
10277	struct net_device *netdev = adapter->netdev;
10278
10279	rtnl_lock();
10280	ixgbe_setup_tc(dev: netdev, tc: adapter->hw_tcs);
10281	rtnl_unlock();
10282	}
10283
10284	#endif
10285	void ixgbe_do_reset(struct net_device *netdev)
10286	{
10287	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
10288
10289	if (netif_running(dev: netdev))
10290	ixgbe_reinit_locked(adapter);
10291	else
10292	ixgbe_reset(adapter);
10293	}
10294
10295	static netdev_features_t ixgbe_fix_features(struct net_device *netdev,
10296	netdev_features_t features)
10297	{
10298	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
10299
10300	/* If Rx checksum is disabled, then RSC/LRO should also be disabled */
10301	if (!(features & NETIF_F_RXCSUM))
10302	features &= ~NETIF_F_LRO;
10303
10304	/* Turn off LRO if not RSC capable */
10305	if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE))
10306	features &= ~NETIF_F_LRO;
10307
10308	if (adapter->xdp_prog && (features & NETIF_F_LRO)) {
10309	e_dev_err("LRO is not supported with XDP\n");
10310	features &= ~NETIF_F_LRO;
10311	}
10312
10313	return features;
10314	}
10315
10316	static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter)
10317	{
10318	int rss = min_t(int, ixgbe_max_rss_indices(adapter),
10319	num_online_cpus());
10320
10321	/* go back to full RSS if we're not running SR-IOV */
10322	if (!adapter->ring_feature[RING_F_VMDQ].offset)
10323	adapter->flags &= ~(IXGBE_FLAG_VMDQ_ENABLED |
10324	IXGBE_FLAG_SRIOV_ENABLED);
10325
10326	adapter->ring_feature[RING_F_RSS].limit = rss;
10327	adapter->ring_feature[RING_F_VMDQ].limit = 1;
10328
10329	ixgbe_setup_tc(dev: adapter->netdev, tc: adapter->hw_tcs);
10330	}
10331
10332	static int ixgbe_set_features(struct net_device *netdev,
10333	netdev_features_t features)
10334	{
10335	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
10336	netdev_features_t changed = netdev->features ^ features;
10337	bool need_reset = false;
10338
10339	/* Make sure RSC matches LRO, reset if change */
10340	if (!(features & NETIF_F_LRO)) {
10341	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
10342	need_reset = true;
10343	adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED;
10344	} else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) &&
10345	!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {
10346	if (adapter->rx_itr_setting == 1 ||
10347	adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) {
10348	adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
10349	need_reset = true;
10350	} else if ((changed ^ features) & NETIF_F_LRO) {
10351	e_info(probe, "rx-usecs set too low, "
10352	"disabling RSC\n");
10353	}
10354	}
10355
10356	/*
10357	* Check if Flow Director n-tuple support or hw_tc support was
10358	* enabled or disabled. If the state changed, we need to reset.
10359	*/
10360	if ((features & NETIF_F_NTUPLE) || (features & NETIF_F_HW_TC)) {
10361	/* turn off ATR, enable perfect filters and reset */
10362	if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE))
10363	need_reset = true;
10364
10365	adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
10366	adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
10367	} else {
10368	/* turn off perfect filters, enable ATR and reset */
10369	if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
10370	need_reset = true;
10371
10372	adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
10373
10374	/* We cannot enable ATR if SR-IOV is enabled */
10375	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED ||
10376	/* We cannot enable ATR if we have 2 or more tcs */
10377	(adapter->hw_tcs > 1) ||
10378	/* We cannot enable ATR if RSS is disabled */
10379	(adapter->ring_feature[RING_F_RSS].limit <= 1) ||
10380	/* A sample rate of 0 indicates ATR disabled */
10381	(!adapter->atr_sample_rate))
10382	; /* do nothing not supported */
10383	else /* otherwise supported and set the flag */
10384	adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
10385	}
10386
10387	if (changed & NETIF_F_RXALL)
10388	need_reset = true;
10389
10390	netdev->features = features;
10391
10392	if ((changed & NETIF_F_HW_L2FW_DOFFLOAD) && adapter->num_rx_pools > 1)
10393	ixgbe_reset_l2fw_offload(adapter);
10394	else if (need_reset)
10395	ixgbe_do_reset(netdev);
10396	else if (changed & (NETIF_F_HW_VLAN_CTAG_RX |
10397	NETIF_F_HW_VLAN_CTAG_FILTER))
10398	ixgbe_set_rx_mode(netdev);
10399
10400	return 1;
10401	}
10402
10403	static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
10404	struct net_device *dev,
10405	const unsigned char *addr, u16 vid,
10406	u16 flags, bool *notified,
10407	struct netlink_ext_ack *extack)
10408	{
10409	/* guarantee we can provide a unique filter for the unicast address */
10410	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
10411	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10412	u16 pool = VMDQ_P(0);
10413
10414	if (netdev_uc_count(dev) >= ixgbe_available_rars(adapter, pool))
10415	return -ENOMEM;
10416	}
10417
10418	return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
10419	}
10420
10421	/**
10422	* ixgbe_configure_bridge_mode - set various bridge modes
10423	* @adapter: the private structure
10424	* @mode: requested bridge mode
10425	*
10426	* Configure some settings require for various bridge modes.
10427	**/
10428	static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter,
10429	__u16 mode)
10430	{
10431	struct ixgbe_hw *hw = &adapter->hw;
10432	unsigned int p, num_pools;
10433	u32 vmdctl;
10434
10435	switch (mode) {
10436	case BRIDGE_MODE_VEPA:
10437	/* disable Tx loopback, rely on switch hairpin mode */
10438	IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC, 0);
10439
10440	/* must enable Rx switching replication to allow multicast
10441	* packet reception on all VFs, and to enable source address
10442	* pruning.
10443	*/
10444	vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);
10445	vmdctl |= IXGBE_VT_CTL_REPLEN;
10446	IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
10447
10448	/* enable Rx source address pruning. Note, this requires
10449	* replication to be enabled or else it does nothing.
10450	*/
10451	num_pools = adapter->num_vfs + adapter->num_rx_pools;
10452	for (p = 0; p < num_pools; p++) {
10453	if (hw->mac.ops.set_source_address_pruning)
10454	hw->mac.ops.set_source_address_pruning(hw,
10455	true,
10456	p);
10457	}
10458	break;
10459	case BRIDGE_MODE_VEB:
10460	/* enable Tx loopback for internal VF/PF communication */
10461	IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC,
10462	IXGBE_PFDTXGSWC_VT_LBEN);
10463
10464	/* disable Rx switching replication unless we have SR-IOV
10465	* virtual functions
10466	*/
10467	vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);
10468	if (!adapter->num_vfs)
10469	vmdctl &= ~IXGBE_VT_CTL_REPLEN;
10470	IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
10471
10472	/* disable Rx source address pruning, since we don't expect to
10473	* be receiving external loopback of our transmitted frames.
10474	*/
10475	num_pools = adapter->num_vfs + adapter->num_rx_pools;
10476	for (p = 0; p < num_pools; p++) {
10477	if (hw->mac.ops.set_source_address_pruning)
10478	hw->mac.ops.set_source_address_pruning(hw,
10479	false,
10480	p);
10481	}
10482	break;
10483	default:
10484	return -EINVAL;
10485	}
10486
10487	adapter->bridge_mode = mode;
10488
10489	e_info(drv, "enabling bridge mode: %s\n",
10490	mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10491
10492	return 0;
10493	}
10494
10495	static int ixgbe_ndo_bridge_setlink(struct net_device *dev,
10496	struct nlmsghdr *nlh, u16 flags,
10497	struct netlink_ext_ack *extack)
10498	{
10499	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10500	struct nlattr *attr, *br_spec;
10501	int rem;
10502
10503	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
10504	return -EOPNOTSUPP;
10505
10506	br_spec = nlmsg_find_attr(nlh, hdrlen: sizeof(struct ifinfomsg), attrtype: IFLA_AF_SPEC);
10507	if (!br_spec)
10508	return -EINVAL;
10509
10510	nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {
10511	__u16 mode = nla_get_u16(nla: attr);
10512	int status = ixgbe_configure_bridge_mode(adapter, mode);
10513
10514	if (status)
10515	return status;
10516
10517	break;
10518	}
10519
10520	return 0;
10521	}
10522
10523	static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
10524	struct net_device *dev,
10525	u32 filter_mask, int nlflags)
10526	{
10527	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10528
10529	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
10530	return 0;
10531
10532	return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
10533	mode: adapter->bridge_mode, flags: 0, mask: 0, nlflags,
10534	filter_mask, NULL);
10535	}
10536
10537	static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev)
10538	{
10539	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: pdev);
10540	struct ixgbe_fwd_adapter *accel;
10541	int tcs = adapter->hw_tcs ? : 1;
10542	int pool, err;
10543
10544	if (adapter->xdp_prog) {
10545	e_warn(probe, "L2FW offload is not supported with XDP\n");
10546	return ERR_PTR(error: -EINVAL);
10547	}
10548
10549	/* The hardware supported by ixgbe only filters on the destination MAC
10550	* address. In order to avoid issues we only support offloading modes
10551	* where the hardware can actually provide the functionality.
10552	*/
10553	if (!macvlan_supports_dest_filter(dev: vdev))
10554	return ERR_PTR(error: -EMEDIUMTYPE);
10555
10556	/* We need to lock down the macvlan to be a single queue device so that
10557	* we can reuse the tc_to_txq field in the macvlan netdev to represent
10558	* the queue mapping to our netdev.
10559	*/
10560	if (netif_is_multiqueue(dev: vdev))
10561	return ERR_PTR(error: -ERANGE);
10562
10563	pool = find_first_zero_bit(addr: adapter->fwd_bitmask, size: adapter->num_rx_pools);
10564	if (pool == adapter->num_rx_pools) {
10565	u16 used_pools = adapter->num_vfs + adapter->num_rx_pools;
10566	u16 reserved_pools;
10567
10568	if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
10569	adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) ||
10570	adapter->num_rx_pools > IXGBE_MAX_MACVLANS)
10571	return ERR_PTR(error: -EBUSY);
10572
10573	/* Hardware has a limited number of available pools. Each VF,
10574	* and the PF require a pool. Check to ensure we don't
10575	* attempt to use more then the available number of pools.
10576	*/
10577	if (used_pools >= IXGBE_MAX_VF_FUNCTIONS)
10578	return ERR_PTR(error: -EBUSY);
10579
10580	/* Enable VMDq flag so device will be set in VM mode */
10581	adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED |
10582	IXGBE_FLAG_SRIOV_ENABLED;
10583
10584	/* Try to reserve as many queues per pool as possible,
10585	* we start with the configurations that support 4 queues
10586	* per pools, followed by 2, and then by just 1 per pool.
10587	*/
10588	if (used_pools < 32 && adapter->num_rx_pools < 16)
10589	reserved_pools = min_t(u16,
10590	32 - used_pools,
10591	16 - adapter->num_rx_pools);
10592	else if (adapter->num_rx_pools < 32)
10593	reserved_pools = min_t(u16,
10594	64 - used_pools,
10595	32 - adapter->num_rx_pools);
10596	else
10597	reserved_pools = 64 - used_pools;
10598
10599
10600	if (!reserved_pools)
10601	return ERR_PTR(error: -EBUSY);
10602
10603	adapter->ring_feature[RING_F_VMDQ].limit += reserved_pools;
10604
10605	/* Force reinit of ring allocation with VMDQ enabled */
10606	err = ixgbe_setup_tc(dev: pdev, tc: adapter->hw_tcs);
10607	if (err)
10608	return ERR_PTR(error: err);
10609
10610	if (pool >= adapter->num_rx_pools)
10611	return ERR_PTR(error: -ENOMEM);
10612	}
10613
10614	accel = kzalloc(sizeof(*accel), GFP_KERNEL);
10615	if (!accel)
10616	return ERR_PTR(error: -ENOMEM);
10617
10618	set_bit(nr: pool, addr: adapter->fwd_bitmask);
10619	netdev_set_sb_channel(dev: vdev, channel: pool);
10620	accel->pool = pool;
10621	accel->netdev = vdev;
10622
10623	if (!netif_running(dev: pdev))
10624	return accel;
10625
10626	err = ixgbe_fwd_ring_up(adapter, accel);
10627	if (err)
10628	return ERR_PTR(error: err);
10629
10630	return accel;
10631	}
10632
10633	static void ixgbe_fwd_del(struct net_device *pdev, void *priv)
10634	{
10635	struct ixgbe_fwd_adapter *accel = priv;
10636	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: pdev);
10637	unsigned int rxbase = accel->rx_base_queue;
10638	unsigned int i;
10639
10640	/* delete unicast filter associated with offloaded interface */
10641	ixgbe_del_mac_filter(adapter, addr: accel->netdev->dev_addr,
10642	VMDQ_P(accel->pool));
10643
10644	/* Allow remaining Rx packets to get flushed out of the
10645	* Rx FIFO before we drop the netdev for the ring.
10646	*/
10647	usleep_range(min: 10000, max: 20000);
10648
10649	for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
10650	struct ixgbe_ring *ring = adapter->rx_ring[rxbase + i];
10651	struct ixgbe_q_vector *qv = ring->q_vector;
10652
10653	/* Make sure we aren't processing any packets and clear
10654	* netdev to shut down the ring.
10655	*/
10656	if (netif_running(dev: adapter->netdev))
10657	napi_synchronize(n: &qv->napi);
10658	ring->netdev = NULL;
10659	}
10660
10661	/* unbind the queues and drop the subordinate channel config */
10662	netdev_unbind_sb_channel(dev: pdev, sb_dev: accel->netdev);
10663	netdev_set_sb_channel(dev: accel->netdev, channel: 0);
10664
10665	clear_bit(nr: accel->pool, addr: adapter->fwd_bitmask);
10666	kfree(objp: accel);
10667	}
10668
10669	#define IXGBE_MAX_MAC_HDR_LEN 127
10670	#define IXGBE_MAX_NETWORK_HDR_LEN 511
10671
10672	static netdev_features_t
10673	ixgbe_features_check(struct sk_buff *skb, struct net_device *dev,
10674	netdev_features_t features)
10675	{
10676	unsigned int network_hdr_len, mac_hdr_len;
10677
10678	/* Make certain the headers can be described by a context descriptor */
10679	mac_hdr_len = skb_network_offset(skb);
10680	if (unlikely(mac_hdr_len > IXGBE_MAX_MAC_HDR_LEN))
10681	return features & ~(NETIF_F_HW_CSUM |
10682	NETIF_F_SCTP_CRC |
10683	NETIF_F_GSO_UDP_L4 |
10684	NETIF_F_HW_VLAN_CTAG_TX |
10685	NETIF_F_TSO |
10686	NETIF_F_TSO6);
10687
10688	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
10689	if (unlikely(network_hdr_len > IXGBE_MAX_NETWORK_HDR_LEN))
10690	return features & ~(NETIF_F_HW_CSUM |
10691	NETIF_F_SCTP_CRC |
10692	NETIF_F_GSO_UDP_L4 |
10693	NETIF_F_TSO |
10694	NETIF_F_TSO6);
10695
10696	/* We can only support IPV4 TSO in tunnels if we can mangle the
10697	* inner IP ID field, so strip TSO if MANGLEID is not supported.
10698	* IPsec offoad sets skb->encapsulation but still can handle
10699	* the TSO, so it's the exception.
10700	*/
10701	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) {
10702	#ifdef CONFIG_IXGBE_IPSEC
10703	if (!secpath_exists(skb))
10704	#endif
10705	features &= ~NETIF_F_TSO;
10706	}
10707
10708	return features;
10709	}
10710
10711	static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog)
10712	{
10713	int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
10714	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10715	struct bpf_prog *old_prog;
10716	bool need_reset;
10717	int num_queues;
10718
10719	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
10720	return -EINVAL;
10721
10722	if (adapter->flags & IXGBE_FLAG_DCB_ENABLED)
10723	return -EINVAL;
10724
10725	/* verify ixgbe ring attributes are sufficient for XDP */
10726	for (i = 0; i < adapter->num_rx_queues; i++) {
10727	struct ixgbe_ring *ring = adapter->rx_ring[i];
10728
10729	if (ring_is_rsc_enabled(ring))
10730	return -EINVAL;
10731
10732	if (frame_size > ixgbe_rx_bufsz(ring))
10733	return -EINVAL;
10734	}
10735
10736	/* if the number of cpus is much larger than the maximum of queues,
10737	* we should stop it and then return with ENOMEM like before.
10738	*/
10739	if (nr_cpu_ids > IXGBE_MAX_XDP_QS * 2)
10740	return -ENOMEM;
10741
10742	old_prog = xchg(&adapter->xdp_prog, prog);
10743	need_reset = (!!prog != !!old_prog);
10744
10745	/* If transitioning XDP modes reconfigure rings */
10746	if (need_reset) {
10747	int err;
10748
10749	if (!prog)
10750	/* Wait until ndo_xsk_wakeup completes. */
10751	synchronize_rcu();
10752	err = ixgbe_setup_tc(dev, tc: adapter->hw_tcs);
10753
10754	if (err)
10755	return -EINVAL;
10756	if (!prog)
10757	xdp_features_clear_redirect_target(dev);
10758	} else {
10759	for (i = 0; i < adapter->num_rx_queues; i++) {
10760	WRITE_ONCE(adapter->rx_ring[i]->xdp_prog,
10761	adapter->xdp_prog);
10762	}
10763	}
10764
10765	if (old_prog)
10766	bpf_prog_put(prog: old_prog);
10767
10768	/* Kick start the NAPI context if there is an AF_XDP socket open
10769	* on that queue id. This so that receiving will start.
10770	*/
10771	if (need_reset && prog) {
10772	num_queues = min_t(int, adapter->num_rx_queues,
10773	adapter->num_xdp_queues);
10774	for (i = 0; i < num_queues; i++)
10775	if (adapter->xdp_ring[i]->xsk_pool)
10776	(void)ixgbe_xsk_wakeup(dev: adapter->netdev, queue_id: i,
10777	XDP_WAKEUP_RX);
10778	xdp_features_set_redirect_target(dev, support_sg: true);
10779	}
10780
10781	return 0;
10782	}
10783
10784	static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp)
10785	{
10786	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10787
10788	switch (xdp->command) {
10789	case XDP_SETUP_PROG:
10790	return ixgbe_xdp_setup(dev, prog: xdp->prog);
10791	case XDP_SETUP_XSK_POOL:
10792	return ixgbe_xsk_pool_setup(adapter, pool: xdp->xsk.pool,
10793	qid: xdp->xsk.queue_id);
10794
10795	default:
10796	return -EINVAL;
10797	}
10798	}
10799
10800	void ixgbe_xdp_ring_update_tail(struct ixgbe_ring *ring)
10801	{
10802	/* Force memory writes to complete before letting h/w know there
10803	* are new descriptors to fetch.
10804	*/
10805	wmb();
10806	writel(val: ring->next_to_use, addr: ring->tail);
10807	}
10808
10809	void ixgbe_xdp_ring_update_tail_locked(struct ixgbe_ring *ring)
10810	{
10811	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
10812	spin_lock(lock: &ring->tx_lock);
10813	ixgbe_xdp_ring_update_tail(ring);
10814	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
10815	spin_unlock(lock: &ring->tx_lock);
10816	}
10817
10818	static int ixgbe_xdp_xmit(struct net_device *dev, int n,
10819	struct xdp_frame **frames, u32 flags)
10820	{
10821	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10822	struct ixgbe_ring *ring;
10823	int nxmit = 0;
10824	int i;
10825
10826	if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state)))
10827	return -ENETDOWN;
10828
10829	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
10830	return -EINVAL;
10831
10832	/* During program transitions its possible adapter->xdp_prog is assigned
10833	* but ring has not been configured yet. In this case simply abort xmit.
10834	*/
10835	ring = adapter->xdp_prog ? ixgbe_determine_xdp_ring(adapter) : NULL;
10836	if (unlikely(!ring))
10837	return -ENXIO;
10838
10839	if (unlikely(test_bit(__IXGBE_TX_DISABLED, &ring->state)))
10840	return -ENXIO;
10841
10842	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
10843	spin_lock(lock: &ring->tx_lock);
10844
10845	for (i = 0; i < n; i++) {
10846	struct xdp_frame *xdpf = frames[i];
10847	int err;
10848
10849	err = ixgbe_xmit_xdp_ring(ring, xdpf);
10850	if (err != IXGBE_XDP_TX)
10851	break;
10852	nxmit++;
10853	}
10854
10855	if (unlikely(flags & XDP_XMIT_FLUSH))
10856	ixgbe_xdp_ring_update_tail(ring);
10857
10858	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
10859	spin_unlock(lock: &ring->tx_lock);
10860
10861	return nxmit;
10862	}
10863
10864	static const struct net_device_ops ixgbe_netdev_ops = {
10865	.ndo_open = ixgbe_open,
10866	.ndo_stop = ixgbe_close,
10867	.ndo_start_xmit = ixgbe_xmit_frame,
10868	.ndo_set_rx_mode = ixgbe_set_rx_mode,
10869	.ndo_validate_addr = eth_validate_addr,
10870	.ndo_set_mac_address = ixgbe_set_mac,
10871	.ndo_change_mtu = ixgbe_change_mtu,
10872	.ndo_tx_timeout = ixgbe_tx_timeout,
10873	.ndo_set_tx_maxrate = ixgbe_tx_maxrate,
10874	.ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid,
10875	.ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid,
10876	.ndo_eth_ioctl = ixgbe_ioctl,
10877	.ndo_set_vf_mac = ixgbe_ndo_set_vf_mac,
10878	.ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan,
10879	.ndo_set_vf_rate = ixgbe_ndo_set_vf_bw,
10880	.ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk,
10881	.ndo_set_vf_link_state = ixgbe_ndo_set_vf_link_state,
10882	.ndo_set_vf_rss_query_en = ixgbe_ndo_set_vf_rss_query_en,
10883	.ndo_set_vf_trust = ixgbe_ndo_set_vf_trust,
10884	.ndo_get_vf_config = ixgbe_ndo_get_vf_config,
10885	.ndo_get_vf_stats = ixgbe_ndo_get_vf_stats,
10886	.ndo_get_stats64 = ixgbe_get_stats64,
10887	.ndo_setup_tc = __ixgbe_setup_tc,
10888	#ifdef IXGBE_FCOE
10889	.ndo_select_queue = ixgbe_select_queue,
10890	.ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
10891	.ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
10892	.ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
10893	.ndo_fcoe_enable = ixgbe_fcoe_enable,
10894	.ndo_fcoe_disable = ixgbe_fcoe_disable,
10895	.ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
10896	.ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo,
10897	#endif /* IXGBE_FCOE */
10898	.ndo_set_features = ixgbe_set_features,
10899	.ndo_fix_features = ixgbe_fix_features,
10900	.ndo_fdb_add = ixgbe_ndo_fdb_add,
10901	.ndo_bridge_setlink = ixgbe_ndo_bridge_setlink,
10902	.ndo_bridge_getlink = ixgbe_ndo_bridge_getlink,
10903	.ndo_dfwd_add_station = ixgbe_fwd_add,
10904	.ndo_dfwd_del_station = ixgbe_fwd_del,
10905	.ndo_features_check = ixgbe_features_check,
10906	.ndo_bpf = ixgbe_xdp,
10907	.ndo_xdp_xmit = ixgbe_xdp_xmit,
10908	.ndo_xsk_wakeup = ixgbe_xsk_wakeup,
10909	};
10910
10911	static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter,
10912	struct ixgbe_ring *tx_ring)
10913	{
10914	unsigned long wait_delay, delay_interval;
10915	struct ixgbe_hw *hw = &adapter->hw;
10916	u8 reg_idx = tx_ring->reg_idx;
10917	int wait_loop;
10918	u32 txdctl;
10919
10920	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
10921
10922	/* delay mechanism from ixgbe_disable_tx */
10923	delay_interval = ixgbe_get_completion_timeout(adapter) / 100;
10924
10925	wait_loop = IXGBE_MAX_RX_DESC_POLL;
10926	wait_delay = delay_interval;
10927
10928	while (wait_loop--) {
10929	usleep_range(min: wait_delay, max: wait_delay + 10);
10930	wait_delay += delay_interval * 2;
10931	txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
10932
10933	if (!(txdctl & IXGBE_TXDCTL_ENABLE))
10934	return;
10935	}
10936
10937	e_err(drv, "TXDCTL.ENABLE not cleared within the polling period\n");
10938	}
10939
10940	static void ixgbe_disable_txr(struct ixgbe_adapter *adapter,
10941	struct ixgbe_ring *tx_ring)
10942	{
10943	set_bit(nr: __IXGBE_TX_DISABLED, addr: &tx_ring->state);
10944	ixgbe_disable_txr_hw(adapter, tx_ring);
10945	}
10946
10947	static void ixgbe_disable_rxr_hw(struct ixgbe_adapter *adapter,
10948	struct ixgbe_ring *rx_ring)
10949	{
10950	unsigned long wait_delay, delay_interval;
10951	struct ixgbe_hw *hw = &adapter->hw;
10952	u8 reg_idx = rx_ring->reg_idx;
10953	int wait_loop;
10954	u32 rxdctl;
10955
10956	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
10957	rxdctl &= ~IXGBE_RXDCTL_ENABLE;
10958	rxdctl |= IXGBE_RXDCTL_SWFLSH;
10959
10960	/* write value back with RXDCTL.ENABLE bit cleared */
10961	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
10962
10963	/* RXDCTL.EN may not change on 82598 if link is down, so skip it */
10964	if (hw->mac.type == ixgbe_mac_82598EB &&
10965	!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
10966	return;
10967
10968	/* delay mechanism from ixgbe_disable_rx */
10969	delay_interval = ixgbe_get_completion_timeout(adapter) / 100;
10970
10971	wait_loop = IXGBE_MAX_RX_DESC_POLL;
10972	wait_delay = delay_interval;
10973
10974	while (wait_loop--) {
10975	usleep_range(min: wait_delay, max: wait_delay + 10);
10976	wait_delay += delay_interval * 2;
10977	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
10978
10979	if (!(rxdctl & IXGBE_RXDCTL_ENABLE))
10980	return;
10981	}
10982
10983	e_err(drv, "RXDCTL.ENABLE not cleared within the polling period\n");
10984	}
10985
10986	static void ixgbe_reset_txr_stats(struct ixgbe_ring *tx_ring)
10987	{
10988	memset(&tx_ring->stats, 0, sizeof(tx_ring->stats));
10989	memset(&tx_ring->tx_stats, 0, sizeof(tx_ring->tx_stats));
10990	}
10991
10992	static void ixgbe_reset_rxr_stats(struct ixgbe_ring *rx_ring)
10993	{
10994	memset(&rx_ring->stats, 0, sizeof(rx_ring->stats));
10995	memset(&rx_ring->rx_stats, 0, sizeof(rx_ring->rx_stats));
10996	}
10997
10998	/**
10999	* ixgbe_irq_disable_single - Disable single IRQ vector
11000	* @adapter: adapter structure
11001	* @ring: ring index
11002	**/
11003	static void ixgbe_irq_disable_single(struct ixgbe_adapter *adapter, u32 ring)
11004	{
11005	struct ixgbe_hw *hw = &adapter->hw;
11006	u64 qmask = BIT_ULL(ring);
11007	u32 mask;
11008
11009	switch (adapter->hw.mac.type) {
11010	case ixgbe_mac_82598EB:
11011	mask = qmask & IXGBE_EIMC_RTX_QUEUE;
11012	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, mask);
11013	break;
11014	case ixgbe_mac_82599EB:
11015	case ixgbe_mac_X540:
11016	case ixgbe_mac_X550:
11017	case ixgbe_mac_X550EM_x:
11018	case ixgbe_mac_x550em_a:
11019	mask = (qmask & 0xFFFFFFFF);
11020	if (mask)
11021	IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
11022	mask = (qmask >> 32);
11023	if (mask)
11024	IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
11025	break;
11026	default:
11027	break;
11028	}
11029	IXGBE_WRITE_FLUSH(&adapter->hw);
11030	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
11031	synchronize_irq(irq: adapter->msix_entries[ring].vector);
11032	else
11033	synchronize_irq(irq: adapter->pdev->irq);
11034	}
11035
11036	/**
11037	* ixgbe_txrx_ring_disable - Disable Rx/Tx/XDP Tx rings
11038	* @adapter: adapter structure
11039	* @ring: ring index
11040	*
11041	* This function disables a certain Rx/Tx/XDP Tx ring. The function
11042	* assumes that the netdev is running.
11043	**/
11044	void ixgbe_txrx_ring_disable(struct ixgbe_adapter *adapter, int ring)
11045	{
11046	struct ixgbe_ring *rx_ring, *tx_ring, *xdp_ring;
11047
11048	rx_ring = adapter->rx_ring[ring];
11049	tx_ring = adapter->tx_ring[ring];
11050	xdp_ring = adapter->xdp_ring[ring];
11051
11052	ixgbe_irq_disable_single(adapter, ring);
11053
11054	/* Rx/Tx/XDP Tx share the same napi context. */
11055	napi_disable(n: &rx_ring->q_vector->napi);
11056
11057	ixgbe_disable_txr(adapter, tx_ring);
11058	if (xdp_ring)
11059	ixgbe_disable_txr(adapter, tx_ring: xdp_ring);
11060	ixgbe_disable_rxr_hw(adapter, rx_ring);
11061
11062	if (xdp_ring)
11063	synchronize_rcu();
11064
11065	ixgbe_clean_tx_ring(tx_ring);
11066	if (xdp_ring)
11067	ixgbe_clean_tx_ring(tx_ring: xdp_ring);
11068	ixgbe_clean_rx_ring(rx_ring);
11069
11070	ixgbe_reset_txr_stats(tx_ring);
11071	if (xdp_ring)
11072	ixgbe_reset_txr_stats(tx_ring: xdp_ring);
11073	ixgbe_reset_rxr_stats(rx_ring);
11074	}
11075
11076	/**
11077	* ixgbe_txrx_ring_enable - Enable Rx/Tx/XDP Tx rings
11078	* @adapter: adapter structure
11079	* @ring: ring index
11080	*
11081	* This function enables a certain Rx/Tx/XDP Tx ring. The function
11082	* assumes that the netdev is running.
11083	**/
11084	void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring)
11085	{
11086	struct ixgbe_ring *rx_ring, *tx_ring, *xdp_ring;
11087
11088	rx_ring = adapter->rx_ring[ring];
11089	tx_ring = adapter->tx_ring[ring];
11090	xdp_ring = adapter->xdp_ring[ring];
11091
11092	ixgbe_configure_tx_ring(adapter, ring: tx_ring);
11093	if (xdp_ring)
11094	ixgbe_configure_tx_ring(adapter, ring: xdp_ring);
11095	ixgbe_configure_rx_ring(adapter, ring: rx_ring);
11096
11097	clear_bit(nr: __IXGBE_TX_DISABLED, addr: &tx_ring->state);
11098	if (xdp_ring)
11099	clear_bit(nr: __IXGBE_TX_DISABLED, addr: &xdp_ring->state);
11100
11101	/* Rx/Tx/XDP Tx share the same napi context. */
11102	napi_enable(n: &rx_ring->q_vector->napi);
11103	ixgbe_irq_enable_queues(adapter, BIT_ULL(ring));
11104	IXGBE_WRITE_FLUSH(&adapter->hw);
11105	}
11106
11107	/**
11108	* ixgbe_enumerate_functions - Get the number of ports this device has
11109	* @adapter: adapter structure
11110	*
11111	* This function enumerates the phsyical functions co-located on a single slot,
11112	* in order to determine how many ports a device has. This is most useful in
11113	* determining the required GT/s of PCIe bandwidth necessary for optimal
11114	* performance.
11115	**/
11116	static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter)
11117	{
11118	struct pci_dev *entry, *pdev = adapter->pdev;
11119	int physfns = 0;
11120
11121	/* Some cards can not use the generic count PCIe functions method,
11122	* because they are behind a parent switch, so we hardcode these with
11123	* the correct number of functions.
11124	*/
11125	if (ixgbe_pcie_from_parent(hw: &adapter->hw))
11126	physfns = 4;
11127
11128	list_for_each_entry(entry, &adapter->pdev->bus->devices, bus_list) {
11129	/* don't count virtual functions */
11130	if (entry->is_virtfn)
11131	continue;
11132
11133	/* When the devices on the bus don't all match our device ID,
11134	* we can't reliably determine the correct number of
11135	* functions. This can occur if a function has been direct
11136	* attached to a virtual machine using VT-d, for example. In
11137	* this case, simply return -1 to indicate this.
11138	*/
11139	if ((entry->vendor != pdev->vendor) ||
11140	(entry->device != pdev->device))
11141	return -1;
11142
11143	physfns++;
11144	}
11145
11146	return physfns;
11147	}
11148
11149	/**
11150	* ixgbe_wol_supported - Check whether device supports WoL
11151	* @adapter: the adapter private structure
11152	* @device_id: the device ID
11153	* @subdevice_id: the subsystem device ID
11154	*
11155	* This function is used by probe and ethtool to determine
11156	* which devices have WoL support
11157	*
11158	**/
11159	bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id,
11160	u16 subdevice_id)
11161	{
11162	struct ixgbe_hw *hw = &adapter->hw;
11163	u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK;
11164
11165	/* WOL not supported on 82598 */
11166	if (hw->mac.type == ixgbe_mac_82598EB)
11167	return false;
11168
11169	/* check eeprom to see if WOL is enabled for X540 and newer */
11170	if (hw->mac.type >= ixgbe_mac_X540) {
11171	if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
11172	((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) &&
11173	(hw->bus.func == 0)))
11174	return true;
11175	}
11176
11177	/* WOL is determined based on device IDs for 82599 MACs */
11178	switch (device_id) {
11179	case IXGBE_DEV_ID_82599_SFP:
11180	/* Only these subdevices could supports WOL */
11181	switch (subdevice_id) {
11182	case IXGBE_SUBDEV_ID_82599_560FLR:
11183	case IXGBE_SUBDEV_ID_82599_LOM_SNAP6:
11184	case IXGBE_SUBDEV_ID_82599_SFP_WOL0:
11185	case IXGBE_SUBDEV_ID_82599_SFP_2OCP:
11186	/* only support first port */
11187	if (hw->bus.func != 0)
11188	break;
11189	fallthrough;
11190	case IXGBE_SUBDEV_ID_82599_SP_560FLR:
11191	case IXGBE_SUBDEV_ID_82599_SFP:
11192	case IXGBE_SUBDEV_ID_82599_RNDC:
11193	case IXGBE_SUBDEV_ID_82599_ECNA_DP:
11194	case IXGBE_SUBDEV_ID_82599_SFP_1OCP:
11195	case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM1:
11196	case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM2:
11197	return true;
11198	}
11199	break;
11200	case IXGBE_DEV_ID_82599EN_SFP:
11201	/* Only these subdevices support WOL */
11202	switch (subdevice_id) {
11203	case IXGBE_SUBDEV_ID_82599EN_SFP_OCP1:
11204	return true;
11205	}
11206	break;
11207	case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
11208	/* All except this subdevice support WOL */
11209	if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
11210	return true;
11211	break;
11212	case IXGBE_DEV_ID_82599_KX4:
11213	return true;
11214	default:
11215	break;
11216	}
11217
11218	return false;
11219	}
11220
11221	/**
11222	* ixgbe_set_fw_version_e610 - Set FW version specifically on E610 adapters
11223	* @adapter: the adapter private structure
11224	*
11225	* This function is used by probe and ethtool to determine the FW version to
11226	* format to display. The FW version is taken from the EEPROM/NVM.
11227	*
11228	*/
11229	void ixgbe_set_fw_version_e610(struct ixgbe_adapter *adapter)
11230	{
11231	struct ixgbe_orom_info *orom = &adapter->hw.flash.orom;
11232	struct ixgbe_nvm_info *nvm = &adapter->hw.flash.nvm;
11233
11234	snprintf(buf: adapter->eeprom_id, size: sizeof(adapter->eeprom_id),
11235	fmt: "%x.%02x 0x%x %d.%d.%d", nvm->major, nvm->minor,
11236	nvm->eetrack, orom->major, orom->build, orom->patch);
11237	}
11238
11239	/**
11240	* ixgbe_set_fw_version - Set FW version
11241	* @adapter: the adapter private structure
11242	*
11243	* This function is used by probe and ethtool to determine the FW version to
11244	* format to display. The FW version is taken from the EEPROM/NVM.
11245	*/
11246	static void ixgbe_set_fw_version(struct ixgbe_adapter *adapter)
11247	{
11248	struct ixgbe_hw *hw = &adapter->hw;
11249	struct ixgbe_nvm_version nvm_ver;
11250
11251	if (adapter->hw.mac.type == ixgbe_mac_e610) {
11252	ixgbe_set_fw_version_e610(adapter);
11253	return;
11254	}
11255
11256	ixgbe_get_oem_prod_version(hw, nvm_ver: &nvm_ver);
11257	if (nvm_ver.oem_valid) {
11258	snprintf(buf: adapter->eeprom_id, size: sizeof(adapter->eeprom_id),
11259	fmt: "%x.%x.%x", nvm_ver.oem_major, nvm_ver.oem_minor,
11260	nvm_ver.oem_release);
11261	return;
11262	}
11263
11264	ixgbe_get_etk_id(hw, nvm_ver: &nvm_ver);
11265	ixgbe_get_orom_version(hw, nvm_ver: &nvm_ver);
11266
11267	if (nvm_ver.or_valid) {
11268	snprintf(buf: adapter->eeprom_id, size: sizeof(adapter->eeprom_id),
11269	fmt: "0x%08x, %d.%d.%d", nvm_ver.etk_id, nvm_ver.or_major,
11270	nvm_ver.or_build, nvm_ver.or_patch);
11271	return;
11272	}
11273
11274	/* Set ETrack ID format */
11275	snprintf(buf: adapter->eeprom_id, size: sizeof(adapter->eeprom_id),
11276	fmt: "0x%08x", nvm_ver.etk_id);
11277	}
11278
11279	/**
11280	* ixgbe_recovery_probe - Handle FW recovery mode during probe
11281	* @adapter: the adapter private structure
11282	*
11283	* Perform limited driver initialization when FW error is detected.
11284	*
11285	* Return: 0 on successful probe for E610, -EIO if recovery mode is detected
11286	* for non-E610 adapter, error status code on any other case.
11287	*/
11288	static int ixgbe_recovery_probe(struct ixgbe_adapter *adapter)
11289	{
11290	struct net_device *netdev = adapter->netdev;
11291	struct pci_dev *pdev = adapter->pdev;
11292	struct ixgbe_hw *hw = &adapter->hw;
11293	bool disable_dev;
11294	int err = -EIO;
11295
11296	if (hw->mac.type != ixgbe_mac_e610)
11297	goto clean_up_probe;
11298
11299	ixgbe_get_hw_control(adapter);
11300	mutex_init(&hw->aci.lock);
11301	err = ixgbe_get_flash_data(hw: &adapter->hw);
11302	if (err)
11303	goto shutdown_aci;
11304
11305	timer_setup(&adapter->service_timer, ixgbe_service_timer, 0);
11306	INIT_WORK(&adapter->service_task, ixgbe_recovery_service_task);
11307	set_bit(nr: __IXGBE_SERVICE_INITED, addr: &adapter->state);
11308	clear_bit(nr: __IXGBE_SERVICE_SCHED, addr: &adapter->state);
11309
11310	if (hw->mac.ops.get_bus_info)
11311	hw->mac.ops.get_bus_info(hw);
11312
11313	pci_set_drvdata(pdev, data: adapter);
11314	/* We are creating devlink interface so NIC can be managed,
11315	* e.g. new NVM image loaded
11316	*/
11317	devl_lock(devlink: adapter->devlink);
11318	ixgbe_devlink_register_port(adapter);
11319	SET_NETDEV_DEVLINK_PORT(adapter->netdev,
11320	&adapter->devlink_port);
11321	ixgbe_devlink_init_regions(adapter);
11322	devl_register(devlink: adapter->devlink);
11323	devl_unlock(devlink: adapter->devlink);
11324
11325	return 0;
11326	shutdown_aci:
11327	mutex_destroy(lock: &adapter->hw.aci.lock);
11328	ixgbe_release_hw_control(adapter);
11329	devlink_free(devlink: adapter->devlink);
11330	clean_up_probe:
11331	disable_dev = !test_and_set_bit(nr: __IXGBE_DISABLED, addr: &adapter->state);
11332	free_netdev(dev: netdev);
11333	pci_release_mem_regions(pdev);
11334	if (disable_dev)
11335	pci_disable_device(dev: pdev);
11336	return err;
11337	}
11338
11339	/**
11340	* ixgbe_probe - Device Initialization Routine
11341	* @pdev: PCI device information struct
11342	* @ent: entry in ixgbe_pci_tbl
11343	*
11344	* Returns 0 on success, negative on failure
11345	*
11346	* ixgbe_probe initializes an adapter identified by a pci_dev structure.
11347	* The OS initialization, configuring of the adapter private structure,
11348	* and a hardware reset occur.
11349	**/
11350	static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
11351	{
11352	struct net_device *netdev;
11353	struct ixgbe_netdevice_priv *netdev_priv_wrapper;
11354	struct ixgbe_adapter *adapter = NULL;
11355	struct ixgbe_hw *hw;
11356	const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
11357	unsigned int indices = MAX_TX_QUEUES;
11358	u8 part_str[IXGBE_PBANUM_LENGTH];
11359	int i, err, expected_gts;
11360	bool disable_dev = false;
11361	#ifdef IXGBE_FCOE
11362	u16 device_caps;
11363	#endif
11364	u32 eec;
11365
11366	/* Catch broken hardware that put the wrong VF device ID in
11367	* the PCIe SR-IOV capability.
11368	*/
11369	if (pdev->is_virtfn) {
11370	WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
11371	pci_name(pdev), pdev->vendor, pdev->device);
11372	return -EINVAL;
11373	}
11374
11375	err = pci_enable_device_mem(dev: pdev);
11376	if (err)
11377	return err;
11378
11379	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
11380	if (err) {
11381	dev_err(&pdev->dev,
11382	"No usable DMA configuration, aborting\n");
11383	goto err_dma;
11384	}
11385
11386	err = pci_request_mem_regions(pdev, name: ixgbe_driver_name);
11387	if (err) {
11388	dev_err(&pdev->dev,
11389	"pci_request_selected_regions failed 0x%x\n", err);
11390	goto err_pci_reg;
11391	}
11392
11393	pci_set_master(dev: pdev);
11394	pci_save_state(dev: pdev);
11395
11396	if (ii->mac == ixgbe_mac_82598EB) {
11397	#ifdef CONFIG_IXGBE_DCB
11398	/* 8 TC w/ 4 queues per TC */
11399	indices = 4 * MAX_TRAFFIC_CLASS;
11400	#else
11401	indices = IXGBE_MAX_RSS_INDICES;
11402	#endif
11403	} else if (ii->mac == ixgbe_mac_e610) {
11404	indices = IXGBE_MAX_RSS_INDICES_X550;
11405	}
11406
11407	adapter = ixgbe_allocate_devlink(dev: &pdev->dev);
11408	if (IS_ERR(ptr: adapter)) {
11409	err = PTR_ERR(ptr: adapter);
11410	goto err_devlink;
11411	}
11412
11413	netdev = alloc_etherdev_mq(sizeof(*netdev_priv_wrapper), indices);
11414	if (!netdev) {
11415	err = -ENOMEM;
11416	goto err_alloc_etherdev;
11417	}
11418
11419	SET_NETDEV_DEV(netdev, &pdev->dev);
11420
11421	netdev_priv_wrapper = netdev_priv(dev: netdev);
11422	netdev_priv_wrapper->adapter = adapter;
11423
11424	adapter->netdev = netdev;
11425	adapter->pdev = pdev;
11426	hw = &adapter->hw;
11427	hw->back = adapter;
11428	adapter->msg_enable = netif_msg_init(debug_value: debug, DEFAULT_MSG_ENABLE);
11429
11430	hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
11431	pci_resource_len(pdev, 0));
11432	adapter->io_addr = hw->hw_addr;
11433	if (!hw->hw_addr) {
11434	err = -EIO;
11435	goto err_ioremap;
11436	}
11437
11438	/* Setup hw api */
11439	hw->mac.ops = *ii->mac_ops;
11440	hw->mac.type = ii->mac;
11441	hw->mvals = ii->mvals;
11442	if (ii->link_ops)
11443	hw->link.ops = *ii->link_ops;
11444
11445	/* EEPROM */
11446	hw->eeprom.ops = *ii->eeprom_ops;
11447	eec = IXGBE_READ_REG(hw, IXGBE_EEC(hw));
11448	if (ixgbe_removed(addr: hw->hw_addr)) {
11449	err = -EIO;
11450	goto err_ioremap;
11451	}
11452	/* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
11453	if (!(eec & BIT(8)))
11454	hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;
11455
11456	/* PHY */
11457	hw->phy.ops = *ii->phy_ops;
11458	hw->phy.sfp_type = ixgbe_sfp_type_unknown;
11459	/* ixgbe_identify_phy_generic will set prtad and mmds properly */
11460	hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
11461	hw->phy.mdio.mmds = 0;
11462	hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
11463	hw->phy.mdio.dev = netdev;
11464	hw->phy.mdio.mdio_read = ixgbe_mdio_read;
11465	hw->phy.mdio.mdio_write = ixgbe_mdio_write;
11466
11467	netdev->netdev_ops = &ixgbe_netdev_ops;
11468	ixgbe_set_ethtool_ops(netdev);
11469	netdev->watchdog_timeo = 5 * HZ;
11470	strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
11471
11472	/* setup the private structure */
11473	err = ixgbe_sw_init(adapter, ii);
11474	if (err)
11475	goto err_sw_init;
11476
11477	/* Make sure the SWFW semaphore is in a valid state */
11478	if (hw->mac.ops.init_swfw_sync)
11479	hw->mac.ops.init_swfw_sync(hw);
11480
11481	if (ixgbe_check_fw_error(adapter))
11482	return ixgbe_recovery_probe(adapter);
11483
11484	if (adapter->hw.mac.type == ixgbe_mac_e610) {
11485	err = ixgbe_get_caps(hw: &adapter->hw);
11486	if (err)
11487	dev_err(&pdev->dev, "ixgbe_get_caps failed %d\n", err);
11488
11489	err = ixgbe_get_flash_data(hw: &adapter->hw);
11490	if (err)
11491	goto err_sw_init;
11492	}
11493
11494	if (adapter->hw.mac.type == ixgbe_mac_82599EB)
11495	adapter->flags2 |= IXGBE_FLAG2_AUTO_DISABLE_VF;
11496
11497	switch (adapter->hw.mac.type) {
11498	case ixgbe_mac_X550:
11499	case ixgbe_mac_X550EM_x:
11500	case ixgbe_mac_e610:
11501	netdev->udp_tunnel_nic_info = &ixgbe_udp_tunnels_x550;
11502	break;
11503	case ixgbe_mac_x550em_a:
11504	netdev->udp_tunnel_nic_info = &ixgbe_udp_tunnels_x550em_a;
11505	break;
11506	default:
11507	break;
11508	}
11509
11510	/* Make it possible the adapter to be woken up via WOL */
11511	switch (adapter->hw.mac.type) {
11512	case ixgbe_mac_82599EB:
11513	case ixgbe_mac_X540:
11514	case ixgbe_mac_X550:
11515	case ixgbe_mac_X550EM_x:
11516	case ixgbe_mac_x550em_a:
11517	case ixgbe_mac_e610:
11518	IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
11519	break;
11520	default:
11521	break;
11522	}
11523
11524	/*
11525	* If there is a fan on this device and it has failed log the
11526	* failure.
11527	*/
11528	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
11529	u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
11530	if (esdp & IXGBE_ESDP_SDP1)
11531	e_crit(probe, "Fan has stopped, replace the adapter\n");
11532	}
11533
11534	if (allow_unsupported_sfp)
11535	hw->allow_unsupported_sfp = allow_unsupported_sfp;
11536
11537	/* reset_hw fills in the perm_addr as well */
11538	hw->phy.reset_if_overtemp = true;
11539	err = hw->mac.ops.reset_hw(hw);
11540	hw->phy.reset_if_overtemp = false;
11541	ixgbe_set_eee_capable(adapter);
11542	if (err == -ENOENT) {
11543	err = 0;
11544	} else if (err == -EOPNOTSUPP) {
11545	e_dev_err("failed to load because an unsupported SFP+ or QSFP module type was detected.\n");
11546	e_dev_err("Reload the driver after installing a supported module.\n");
11547	goto err_sw_init;
11548	} else if (err) {
11549	e_dev_err("HW Init failed: %d\n", err);
11550	goto err_sw_init;
11551	}
11552
11553	#ifdef CONFIG_PCI_IOV
11554	/* SR-IOV not supported on the 82598 */
11555	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
11556	goto skip_sriov;
11557	/* Mailbox */
11558	ixgbe_init_mbx_params_pf(hw);
11559	hw->mbx.ops = ii->mbx_ops;
11560	pci_sriov_set_totalvfs(dev: pdev, IXGBE_MAX_VFS_DRV_LIMIT);
11561	ixgbe_enable_sriov(adapter, max_vfs);
11562	skip_sriov:
11563
11564	#endif
11565	netdev->features = NETIF_F_SG |
11566	NETIF_F_TSO |
11567	NETIF_F_TSO6 |
11568	NETIF_F_RXHASH |
11569	NETIF_F_RXCSUM |
11570	NETIF_F_HW_CSUM;
11571
11572	#define IXGBE_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
11573	NETIF_F_GSO_GRE_CSUM | \
11574	NETIF_F_GSO_IPXIP4 | \
11575	NETIF_F_GSO_IPXIP6 | \
11576	NETIF_F_GSO_UDP_TUNNEL | \
11577	NETIF_F_GSO_UDP_TUNNEL_CSUM)
11578
11579	netdev->gso_partial_features = IXGBE_GSO_PARTIAL_FEATURES;
11580	netdev->features |= NETIF_F_GSO_PARTIAL |
11581	IXGBE_GSO_PARTIAL_FEATURES;
11582
11583	if (hw->mac.type >= ixgbe_mac_82599EB)
11584	netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4;
11585
11586	#ifdef CONFIG_IXGBE_IPSEC
11587	#define IXGBE_ESP_FEATURES (NETIF_F_HW_ESP | \
11588	NETIF_F_HW_ESP_TX_CSUM | \
11589	NETIF_F_GSO_ESP)
11590
11591	if (adapter->ipsec)
11592	netdev->features |= IXGBE_ESP_FEATURES;
11593	#endif
11594	/* copy netdev features into list of user selectable features */
11595	netdev->hw_features |= netdev->features |
11596	NETIF_F_HW_VLAN_CTAG_FILTER |
11597	NETIF_F_HW_VLAN_CTAG_RX |
11598	NETIF_F_HW_VLAN_CTAG_TX |
11599	NETIF_F_RXALL |
11600	NETIF_F_HW_L2FW_DOFFLOAD;
11601
11602	if (hw->mac.type >= ixgbe_mac_82599EB)
11603	netdev->hw_features |= NETIF_F_NTUPLE |
11604	NETIF_F_HW_TC;
11605
11606	netdev->features |= NETIF_F_HIGHDMA;
11607
11608	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
11609	netdev->hw_enc_features |= netdev->vlan_features;
11610	netdev->mpls_features |= NETIF_F_SG |
11611	NETIF_F_TSO |
11612	NETIF_F_TSO6 |
11613	NETIF_F_HW_CSUM;
11614	netdev->mpls_features |= IXGBE_GSO_PARTIAL_FEATURES;
11615
11616	/* set this bit last since it cannot be part of vlan_features */
11617	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
11618	NETIF_F_HW_VLAN_CTAG_RX |
11619	NETIF_F_HW_VLAN_CTAG_TX;
11620
11621	netdev->priv_flags |= IFF_UNICAST_FLT;
11622	netdev->priv_flags |= IFF_SUPP_NOFCS;
11623
11624	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
11625	NETDEV_XDP_ACT_XSK_ZEROCOPY;
11626
11627	/* MTU range: 68 - 9710 */
11628	netdev->min_mtu = ETH_MIN_MTU;
11629	netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
11630
11631	#ifdef CONFIG_IXGBE_DCB
11632	if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE)
11633	netdev->dcbnl_ops = &ixgbe_dcbnl_ops;
11634	#endif
11635
11636	#ifdef IXGBE_FCOE
11637	if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
11638	unsigned int fcoe_l;
11639
11640	if (hw->mac.ops.get_device_caps) {
11641	hw->mac.ops.get_device_caps(hw, &device_caps);
11642	if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
11643	adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
11644	}
11645
11646
11647	fcoe_l = min_t(int, IXGBE_FCRETA_SIZE, num_online_cpus());
11648	adapter->ring_feature[RING_F_FCOE].limit = fcoe_l;
11649
11650	netdev->features |= NETIF_F_FSO |
11651	NETIF_F_FCOE_CRC;
11652
11653	netdev->vlan_features |= NETIF_F_FSO |
11654	NETIF_F_FCOE_CRC;
11655	}
11656	#endif /* IXGBE_FCOE */
11657	if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
11658	netdev->hw_features |= NETIF_F_LRO;
11659	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
11660	netdev->features |= NETIF_F_LRO;
11661
11662	/* make sure the EEPROM is good */
11663	if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
11664	e_dev_err("The EEPROM Checksum Is Not Valid\n");
11665	err = -EIO;
11666	goto err_sw_init;
11667	}
11668
11669	eth_platform_get_mac_address(dev: &adapter->pdev->dev,
11670	mac_addr: adapter->hw.mac.perm_addr);
11671
11672	eth_hw_addr_set(dev: netdev, addr: hw->mac.perm_addr);
11673
11674	if (!is_valid_ether_addr(addr: netdev->dev_addr)) {
11675	e_dev_err("invalid MAC address\n");
11676	err = -EIO;
11677	goto err_sw_init;
11678	}
11679
11680	/* Set hw->mac.addr to permanent MAC address */
11681	ether_addr_copy(dst: hw->mac.addr, src: hw->mac.perm_addr);
11682	ixgbe_mac_set_default_filter(adapter);
11683
11684	if (hw->mac.type == ixgbe_mac_e610)
11685	mutex_init(&hw->aci.lock);
11686	timer_setup(&adapter->service_timer, ixgbe_service_timer, 0);
11687
11688	if (ixgbe_removed(addr: hw->hw_addr)) {
11689	err = -EIO;
11690	goto err_sw_init;
11691	}
11692	INIT_WORK(&adapter->service_task, ixgbe_service_task);
11693	set_bit(nr: __IXGBE_SERVICE_INITED, addr: &adapter->state);
11694	clear_bit(nr: __IXGBE_SERVICE_SCHED, addr: &adapter->state);
11695
11696	err = ixgbe_init_interrupt_scheme(adapter);
11697	if (err)
11698	goto err_sw_init;
11699
11700	for (i = 0; i < adapter->num_rx_queues; i++)
11701	u64_stats_init(syncp: &adapter->rx_ring[i]->syncp);
11702	for (i = 0; i < adapter->num_tx_queues; i++)
11703	u64_stats_init(syncp: &adapter->tx_ring[i]->syncp);
11704	for (i = 0; i < adapter->num_xdp_queues; i++)
11705	u64_stats_init(syncp: &adapter->xdp_ring[i]->syncp);
11706
11707	/* WOL not supported for all devices */
11708	adapter->wol = 0;
11709	hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap);
11710	hw->wol_enabled = ixgbe_wol_supported(adapter, device_id: pdev->device,
11711	subdevice_id: pdev->subsystem_device);
11712	if (hw->wol_enabled)
11713	adapter->wol = IXGBE_WUFC_MAG;
11714
11715	device_set_wakeup_enable(dev: &adapter->pdev->dev, enable: adapter->wol);
11716
11717	/* save off EEPROM version number */
11718	ixgbe_set_fw_version(adapter);
11719
11720	/* pick up the PCI bus settings for reporting later */
11721	if (ixgbe_pcie_from_parent(hw))
11722	ixgbe_get_parent_bus_info(adapter);
11723	else
11724	hw->mac.ops.get_bus_info(hw);
11725
11726	/* calculate the expected PCIe bandwidth required for optimal
11727	* performance. Note that some older parts will never have enough
11728	* bandwidth due to being older generation PCIe parts. We clamp these
11729	* parts to ensure no warning is displayed if it can't be fixed.
11730	*/
11731	switch (hw->mac.type) {
11732	case ixgbe_mac_82598EB:
11733	expected_gts = min(ixgbe_enumerate_functions(adapter) * 10, 16);
11734	break;
11735	default:
11736	expected_gts = ixgbe_enumerate_functions(adapter) * 10;
11737	break;
11738	}
11739
11740	/* don't check link if we failed to enumerate functions */
11741	if (expected_gts > 0)
11742	ixgbe_check_minimum_link(adapter, expected_gts);
11743
11744	err = hw->eeprom.ops.read_pba_string(hw, part_str, sizeof(part_str));
11745	if (err)
11746	strscpy(part_str, "Unknown", sizeof(part_str));
11747	if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
11748	e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
11749	hw->mac.type, hw->phy.type, hw->phy.sfp_type,
11750	part_str);
11751	else
11752	e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
11753	hw->mac.type, hw->phy.type, part_str);
11754
11755	e_dev_info("%pM\n", netdev->dev_addr);
11756
11757	/* reset the hardware with the new settings */
11758	err = hw->mac.ops.start_hw(hw);
11759	if (err == -EACCES) {
11760	/* We are running on a pre-production device, log a warning */
11761	e_dev_warn("This device is a pre-production adapter/LOM. "
11762	"Please be aware there may be issues associated "
11763	"with your hardware. If you are experiencing "
11764	"problems please contact your Intel or hardware "
11765	"representative who provided you with this "
11766	"hardware.\n");
11767	}
11768	strcpy(p: netdev->name, q: "eth%d");
11769	pci_set_drvdata(pdev, data: adapter);
11770
11771	devl_lock(devlink: adapter->devlink);
11772	ixgbe_devlink_register_port(adapter);
11773	SET_NETDEV_DEVLINK_PORT(adapter->netdev, &adapter->devlink_port);
11774
11775	err = register_netdev(dev: netdev);
11776	if (err)
11777	goto err_register;
11778
11779
11780	/* power down the optics for 82599 SFP+ fiber */
11781	if (hw->mac.ops.disable_tx_laser)
11782	hw->mac.ops.disable_tx_laser(hw);
11783
11784	/* carrier off reporting is important to ethtool even BEFORE open */
11785	netif_carrier_off(dev: netdev);
11786
11787	#ifdef CONFIG_IXGBE_DCA
11788	if (dca_add_requester(dev: &pdev->dev) == 0) {
11789	adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
11790	ixgbe_setup_dca(adapter);
11791	}
11792	#endif
11793	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
11794	e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
11795	for (i = 0; i < adapter->num_vfs; i++)
11796	ixgbe_vf_configuration(pdev, event_mask: (i | 0x10000000));
11797	}
11798
11799	/* firmware requires driver version to be 0xFFFFFFFF
11800	* since os does not support feature
11801	*/
11802	if (hw->mac.ops.set_fw_drv_ver)
11803	hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF, 0xFF,
11804	sizeof(UTS_RELEASE) - 1,
11805	UTS_RELEASE);
11806
11807	/* add san mac addr to netdev */
11808	ixgbe_add_sanmac_netdev(dev: netdev);
11809
11810	e_dev_info("%s\n", ixgbe_default_device_descr);
11811
11812	#ifdef CONFIG_IXGBE_HWMON
11813	if (ixgbe_sysfs_init(adapter))
11814	e_err(probe, "failed to allocate sysfs resources\n");
11815	#endif /* CONFIG_IXGBE_HWMON */
11816
11817	ixgbe_dbg_adapter_init(adapter);
11818
11819	/* setup link for SFP devices with MNG FW, else wait for IXGBE_UP */
11820	if (ixgbe_mng_enabled(hw) && ixgbe_is_sfp(hw) && hw->mac.ops.setup_link)
11821	hw->mac.ops.setup_link(hw,
11822	IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL,
11823	true);
11824
11825	err = ixgbe_mii_bus_init(hw);
11826	if (err)
11827	goto err_netdev;
11828
11829	ixgbe_devlink_init_regions(adapter);
11830	devl_register(devlink: adapter->devlink);
11831	devl_unlock(devlink: adapter->devlink);
11832	return 0;
11833
11834	err_netdev:
11835	unregister_netdev(dev: netdev);
11836	err_register:
11837	devl_port_unregister(devlink_port: &adapter->devlink_port);
11838	devl_unlock(devlink: adapter->devlink);
11839	ixgbe_release_hw_control(adapter);
11840	ixgbe_clear_interrupt_scheme(adapter);
11841	if (hw->mac.type == ixgbe_mac_e610)
11842	mutex_destroy(lock: &adapter->hw.aci.lock);
11843	err_sw_init:
11844	ixgbe_disable_sriov(adapter);
11845	adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
11846	iounmap(addr: adapter->io_addr);
11847	kfree(objp: adapter->jump_tables[0]);
11848	kfree(objp: adapter->mac_table);
11849	kfree(objp: adapter->rss_key);
11850	bitmap_free(bitmap: adapter->af_xdp_zc_qps);
11851	err_ioremap:
11852	disable_dev = !test_and_set_bit(nr: __IXGBE_DISABLED, addr: &adapter->state);
11853	free_netdev(dev: netdev);
11854	err_alloc_etherdev:
11855	devlink_free(devlink: adapter->devlink);
11856	pci_release_mem_regions(pdev);
11857	err_devlink:
11858	err_pci_reg:
11859	err_dma:
11860	if (!adapter || disable_dev)
11861	pci_disable_device(dev: pdev);
11862	return err;
11863	}
11864
11865	/**
11866	* ixgbe_remove - Device Removal Routine
11867	* @pdev: PCI device information struct
11868	*
11869	* ixgbe_remove is called by the PCI subsystem to alert the driver
11870	* that it should release a PCI device. The could be caused by a
11871	* Hot-Plug event, or because the driver is going to be removed from
11872	* memory.
11873	**/
11874	static void ixgbe_remove(struct pci_dev *pdev)
11875	{
11876	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
11877	struct net_device *netdev;
11878	bool disable_dev;
11879	int i;
11880
11881	/* if !adapter then we already cleaned up in probe */
11882	if (!adapter)
11883	return;
11884
11885	netdev = adapter->netdev;
11886	devl_lock(devlink: adapter->devlink);
11887	devl_unregister(devlink: adapter->devlink);
11888	ixgbe_devlink_destroy_regions(adapter);
11889	ixgbe_dbg_adapter_exit(adapter);
11890
11891	set_bit(nr: __IXGBE_REMOVING, addr: &adapter->state);
11892	cancel_work_sync(work: &adapter->service_task);
11893
11894	if (adapter->hw.mac.type == ixgbe_mac_e610) {
11895	ixgbe_disable_link_status_events(adapter);
11896	mutex_destroy(lock: &adapter->hw.aci.lock);
11897	}
11898
11899	if (adapter->mii_bus)
11900	mdiobus_unregister(bus: adapter->mii_bus);
11901
11902	#ifdef CONFIG_IXGBE_DCA
11903	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
11904	adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
11905	dca_remove_requester(dev: &pdev->dev);
11906	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
11907	IXGBE_DCA_CTRL_DCA_DISABLE);
11908	}
11909
11910	#endif
11911	#ifdef CONFIG_IXGBE_HWMON
11912	ixgbe_sysfs_exit(adapter);
11913	#endif /* CONFIG_IXGBE_HWMON */
11914
11915	/* remove the added san mac */
11916	ixgbe_del_sanmac_netdev(dev: netdev);
11917
11918	#ifdef CONFIG_PCI_IOV
11919	ixgbe_disable_sriov(adapter);
11920	#endif
11921	if (netdev->reg_state == NETREG_REGISTERED)
11922	unregister_netdev(dev: netdev);
11923
11924	devl_port_unregister(devlink_port: &adapter->devlink_port);
11925	devl_unlock(devlink: adapter->devlink);
11926	devlink_free(devlink: adapter->devlink);
11927
11928	ixgbe_stop_ipsec_offload(adapter);
11929	ixgbe_clear_interrupt_scheme(adapter);
11930
11931	ixgbe_release_hw_control(adapter);
11932
11933	#ifdef CONFIG_DCB
11934	kfree(objp: adapter->ixgbe_ieee_pfc);
11935	kfree(objp: adapter->ixgbe_ieee_ets);
11936
11937	#endif
11938	iounmap(addr: adapter->io_addr);
11939	pci_release_mem_regions(pdev);
11940
11941	e_dev_info("complete\n");
11942
11943	for (i = 0; i < IXGBE_MAX_LINK_HANDLE; i++) {
11944	if (adapter->jump_tables[i]) {
11945	kfree(objp: adapter->jump_tables[i]->input);
11946	kfree(objp: adapter->jump_tables[i]->mask);
11947	}
11948	kfree(objp: adapter->jump_tables[i]);
11949	}
11950
11951	kfree(objp: adapter->mac_table);
11952	kfree(objp: adapter->rss_key);
11953	bitmap_free(bitmap: adapter->af_xdp_zc_qps);
11954	disable_dev = !test_and_set_bit(nr: __IXGBE_DISABLED, addr: &adapter->state);
11955	free_netdev(dev: netdev);
11956
11957	if (disable_dev)
11958	pci_disable_device(dev: pdev);
11959	}
11960
11961	/**
11962	* ixgbe_io_error_detected - called when PCI error is detected
11963	* @pdev: Pointer to PCI device
11964	* @state: The current pci connection state
11965	*
11966	* This function is called after a PCI bus error affecting
11967	* this device has been detected.
11968	*/
11969	static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
11970	pci_channel_state_t state)
11971	{
11972	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
11973	struct net_device *netdev = adapter->netdev;
11974
11975	#ifdef CONFIG_PCI_IOV
11976	struct ixgbe_hw *hw = &adapter->hw;
11977	struct pci_dev *bdev, *vfdev;
11978	u32 dw0, dw1, dw2, dw3;
11979	int vf, pos;
11980	u16 req_id, pf_func;
11981
11982	if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
11983	adapter->num_vfs == 0)
11984	goto skip_bad_vf_detection;
11985
11986	bdev = pdev->bus->self;
11987	while (bdev && (pci_pcie_type(dev: bdev) != PCI_EXP_TYPE_ROOT_PORT))
11988	bdev = bdev->bus->self;
11989
11990	if (!bdev)
11991	goto skip_bad_vf_detection;
11992
11993	pos = pci_find_ext_capability(dev: bdev, PCI_EXT_CAP_ID_ERR);
11994	if (!pos)
11995	goto skip_bad_vf_detection;
11996
11997	dw0 = ixgbe_read_pci_cfg_dword(hw, reg: pos + PCI_ERR_HEADER_LOG);
11998	dw1 = ixgbe_read_pci_cfg_dword(hw, reg: pos + PCI_ERR_HEADER_LOG + 4);
11999	dw2 = ixgbe_read_pci_cfg_dword(hw, reg: pos + PCI_ERR_HEADER_LOG + 8);
12000	dw3 = ixgbe_read_pci_cfg_dword(hw, reg: pos + PCI_ERR_HEADER_LOG + 12);
12001	if (ixgbe_removed(addr: hw->hw_addr))
12002	goto skip_bad_vf_detection;
12003
12004	req_id = dw1 >> 16;
12005	/* On the 82599 if bit 7 of the requestor ID is set then it's a VF */
12006	if (!(req_id & 0x0080))
12007	goto skip_bad_vf_detection;
12008
12009	pf_func = req_id & 0x01;
12010	if ((pf_func & 1) == (pdev->devfn & 1)) {
12011	unsigned int device_id;
12012
12013	vf = FIELD_GET(0x7F, req_id);
12014	e_dev_err("VF %d has caused a PCIe error\n", vf);
12015	e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
12016	"%8.8x\tdw3: %8.8x\n",
12017	dw0, dw1, dw2, dw3);
12018	switch (adapter->hw.mac.type) {
12019	case ixgbe_mac_82599EB:
12020	device_id = IXGBE_82599_VF_DEVICE_ID;
12021	break;
12022	case ixgbe_mac_X540:
12023	device_id = IXGBE_X540_VF_DEVICE_ID;
12024	break;
12025	case ixgbe_mac_X550:
12026	device_id = IXGBE_DEV_ID_X550_VF;
12027	break;
12028	case ixgbe_mac_X550EM_x:
12029	device_id = IXGBE_DEV_ID_X550EM_X_VF;
12030	break;
12031	case ixgbe_mac_x550em_a:
12032	device_id = IXGBE_DEV_ID_X550EM_A_VF;
12033	break;
12034	case ixgbe_mac_e610:
12035	device_id = IXGBE_DEV_ID_E610_VF;
12036	break;
12037	default:
12038	device_id = 0;
12039	break;
12040	}
12041
12042	/* Find the pci device of the offending VF */
12043	vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, device: device_id, NULL);
12044	while (vfdev) {
12045	if (vfdev->devfn == (req_id & 0xFF))
12046	break;
12047	vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
12048	device: device_id, from: vfdev);
12049	}
12050	/*
12051	* There's a slim chance the VF could have been hot plugged,
12052	* so if it is no longer present we don't need to issue the
12053	* VFLR. Just clean up the AER in that case.
12054	*/
12055	if (vfdev) {
12056	pcie_flr(dev: vfdev);
12057	/* Free device reference count */
12058	pci_dev_put(dev: vfdev);
12059	}
12060	}
12061
12062	/*
12063	* Even though the error may have occurred on the other port
12064	* we still need to increment the vf error reference count for
12065	* both ports because the I/O resume function will be called
12066	* for both of them.
12067	*/
12068	adapter->vferr_refcount++;
12069
12070	return PCI_ERS_RESULT_RECOVERED;
12071
12072	skip_bad_vf_detection:
12073	#endif /* CONFIG_PCI_IOV */
12074	if (!test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
12075	return PCI_ERS_RESULT_DISCONNECT;
12076
12077	if (!netif_device_present(dev: netdev))
12078	return PCI_ERS_RESULT_DISCONNECT;
12079
12080	rtnl_lock();
12081	netif_device_detach(dev: netdev);
12082
12083	if (netif_running(dev: netdev))
12084	ixgbe_close_suspend(adapter);
12085
12086	if (state == pci_channel_io_perm_failure) {
12087	rtnl_unlock();
12088	return PCI_ERS_RESULT_DISCONNECT;
12089	}
12090
12091	if (!test_and_set_bit(nr: __IXGBE_DISABLED, addr: &adapter->state))
12092	pci_disable_device(dev: pdev);
12093	rtnl_unlock();
12094
12095	/* Request a slot reset. */
12096	return PCI_ERS_RESULT_NEED_RESET;
12097	}
12098
12099	/**
12100	* ixgbe_io_slot_reset - called after the pci bus has been reset.
12101	* @pdev: Pointer to PCI device
12102	*
12103	* Restart the card from scratch, as if from a cold-boot.
12104	*/
12105	static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
12106	{
12107	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
12108	pci_ers_result_t result;
12109
12110	if (pci_enable_device_mem(dev: pdev)) {
12111	e_err(probe, "Cannot re-enable PCI device after reset.\n");
12112	result = PCI_ERS_RESULT_DISCONNECT;
12113	} else {
12114	smp_mb__before_atomic();
12115	clear_bit(nr: __IXGBE_DISABLED, addr: &adapter->state);
12116	adapter->hw.hw_addr = adapter->io_addr;
12117	pci_set_master(dev: pdev);
12118	pci_restore_state(dev: pdev);
12119	pci_save_state(dev: pdev);
12120
12121	pci_wake_from_d3(dev: pdev, enable: false);
12122
12123	ixgbe_reset(adapter);
12124	IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
12125	result = PCI_ERS_RESULT_RECOVERED;
12126	}
12127
12128	return result;
12129	}
12130
12131	/**
12132	* ixgbe_io_resume - called when traffic can start flowing again.
12133	* @pdev: Pointer to PCI device
12134	*
12135	* This callback is called when the error recovery driver tells us that
12136	* its OK to resume normal operation.
12137	*/
12138	static void ixgbe_io_resume(struct pci_dev *pdev)
12139	{
12140	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
12141	struct net_device *netdev = adapter->netdev;
12142
12143	#ifdef CONFIG_PCI_IOV
12144	if (adapter->vferr_refcount) {
12145	e_info(drv, "Resuming after VF err\n");
12146	adapter->vferr_refcount--;
12147	return;
12148	}
12149
12150	#endif
12151	rtnl_lock();
12152	if (netif_running(dev: netdev))
12153	ixgbe_open(netdev);
12154
12155	netif_device_attach(dev: netdev);
12156	rtnl_unlock();
12157	}
12158
12159	static const struct pci_error_handlers ixgbe_err_handler = {
12160	.error_detected = ixgbe_io_error_detected,
12161	.slot_reset = ixgbe_io_slot_reset,
12162	.resume = ixgbe_io_resume,
12163	};
12164
12165	static DEFINE_SIMPLE_DEV_PM_OPS(ixgbe_pm_ops, ixgbe_suspend, ixgbe_resume);
12166
12167	static struct pci_driver ixgbe_driver = {
12168	.name = ixgbe_driver_name,
12169	.id_table = ixgbe_pci_tbl,
12170	.probe = ixgbe_probe,
12171	.remove = ixgbe_remove,
12172	.driver.pm = pm_sleep_ptr(&ixgbe_pm_ops),
12173	.shutdown = ixgbe_shutdown,
12174	.sriov_configure = ixgbe_pci_sriov_configure,
12175	.err_handler = &ixgbe_err_handler
12176	};
12177
12178	/**
12179	* ixgbe_init_module - Driver Registration Routine
12180	*
12181	* ixgbe_init_module is the first routine called when the driver is
12182	* loaded. All it does is register with the PCI subsystem.
12183	**/
12184	static int __init ixgbe_init_module(void)
12185	{
12186	int ret;
12187	pr_info("%s\n", ixgbe_driver_string);
12188	pr_info("%s\n", ixgbe_copyright);
12189
12190	ixgbe_wq = create_singlethread_workqueue(ixgbe_driver_name);
12191	if (!ixgbe_wq) {
12192	pr_err("%s: Failed to create workqueue\n", ixgbe_driver_name);
12193	return -ENOMEM;
12194	}
12195
12196	ixgbe_dbg_init();
12197
12198	ret = pci_register_driver(&ixgbe_driver);
12199	if (ret) {
12200	destroy_workqueue(wq: ixgbe_wq);
12201	ixgbe_dbg_exit();
12202	return ret;
12203	}
12204
12205	#ifdef CONFIG_IXGBE_DCA
12206	dca_register_notify(nb: &dca_notifier);
12207	#endif
12208
12209	return 0;
12210	}
12211
12212	module_init(ixgbe_init_module);
12213
12214	/**
12215	* ixgbe_exit_module - Driver Exit Cleanup Routine
12216	*
12217	* ixgbe_exit_module is called just before the driver is removed
12218	* from memory.
12219	**/
12220	static void __exit ixgbe_exit_module(void)
12221	{
12222	#ifdef CONFIG_IXGBE_DCA
12223	dca_unregister_notify(nb: &dca_notifier);
12224	#endif
12225	pci_unregister_driver(dev: &ixgbe_driver);
12226
12227	ixgbe_dbg_exit();
12228	if (ixgbe_wq) {
12229	destroy_workqueue(wq: ixgbe_wq);
12230	ixgbe_wq = NULL;
12231	}
12232	}
12233
12234	#ifdef CONFIG_IXGBE_DCA
12235	static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
12236	void *p)
12237	{
12238	int ret_val;
12239
12240	ret_val = driver_for_each_device(drv: &ixgbe_driver.driver, NULL, data: &event,
12241	fn: __ixgbe_notify_dca);
12242
12243	return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
12244	}
12245
12246	#endif /* CONFIG_IXGBE_DCA */
12247
12248	module_exit(ixgbe_exit_module);
12249
12250	/* ixgbe_main.c */
12251