1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright(c) 1999 - 2024 Intel Corporation. */
3
4	#include <linux/types.h>
5	#include <linux/module.h>
6	#include <linux/pci.h>
7	#include <linux/netdevice.h>
8	#include <linux/vmalloc.h>
9	#include <linux/string.h>
10	#include <linux/in.h>
11	#include <linux/interrupt.h>
12	#include <linux/iopoll.h>
13	#include <linux/ip.h>
14	#include <linux/tcp.h>
15	#include <linux/sctp.h>
16	#include <linux/pkt_sched.h>
17	#include <linux/ipv6.h>
18	#include <linux/slab.h>
19	#include <net/checksum.h>
20	#include <net/ip6_checksum.h>
21	#include <linux/etherdevice.h>
22	#include <linux/ethtool.h>
23	#include <linux/if.h>
24	#include <linux/if_vlan.h>
25	#include <linux/if_macvlan.h>
26	#include <linux/if_bridge.h>
27	#include <linux/prefetch.h>
28	#include <linux/bpf.h>
29	#include <linux/bpf_trace.h>
30	#include <linux/atomic.h>
31	#include <linux/numa.h>
32	#include <generated/utsrelease.h>
33	#include <scsi/fc/fc_fcoe.h>
34	#include <net/udp_tunnel.h>
35	#include <net/pkt_cls.h>
36	#include <net/tc_act/tc_gact.h>
37	#include <net/tc_act/tc_mirred.h>
38	#include <net/vxlan.h>
39	#include <net/mpls.h>
40	#include <net/netdev_queues.h>
41	#include <net/xdp_sock_drv.h>
42	#include <net/xfrm.h>
43
44	#include "ixgbe.h"
45	#include "ixgbe_common.h"
46	#include "ixgbe_e610.h"
47	#include "ixgbe_dcb_82599.h"
48	#include "ixgbe_mbx.h"
49	#include "ixgbe_phy.h"
50	#include "ixgbe_sriov.h"
51	#include "ixgbe_model.h"
52	#include "ixgbe_txrx_common.h"
53	#include "devlink/devlink.h"
54
55	char ixgbe_driver_name[] = "ixgbe";
56	static const char ixgbe_driver_string[] =
57	"Intel(R) 10 Gigabit PCI Express Network Driver";
58	#ifdef IXGBE_FCOE
59	char ixgbe_default_device_descr[] =
60	"Intel(R) 10 Gigabit Network Connection";
61	#else
62	static char ixgbe_default_device_descr[] =
63	"Intel(R) 10 Gigabit Network Connection";
64	#endif
65	static const char ixgbe_copyright[] =
66	"Copyright (c) 1999-2016 Intel Corporation.";
67
68	static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter";
69
70	static const struct ixgbe_info *ixgbe_info_tbl[] = {
71	[board_82598] = &ixgbe_82598_info,
72	[board_82599] = &ixgbe_82599_info,
73	[board_X540] = &ixgbe_X540_info,
74	[board_X550] = &ixgbe_X550_info,
75	[board_X550EM_x] = &ixgbe_X550EM_x_info,
76	[board_x550em_x_fw] = &ixgbe_x550em_x_fw_info,
77	[board_x550em_a] = &ixgbe_x550em_a_info,
78	[board_x550em_a_fw] = &ixgbe_x550em_a_fw_info,
79	[board_e610] = &ixgbe_e610_info,
80	};
81
82	/* ixgbe_pci_tbl - PCI Device ID Table
83	*
84	* Wildcard entries (PCI_ANY_ID) should come last
85	* Last entry must be all 0s
86	*
87	* { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
88	* Class, Class Mask, private data (not used) }
89	*/
90	static const struct pci_device_id ixgbe_pci_tbl[] = {
91	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
92	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
93	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
94	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
95	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
96	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
97	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
98	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
99	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
100	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
101	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
102	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
103	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
104	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
105	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
106	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
107	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
108	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
109	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
110	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
111	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
112	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
113	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
114	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
115	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
116	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
117	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 },
118	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
119	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
120	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
121	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550},
122	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), board_X550},
123	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x},
124	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_XFI), board_X550EM_x},
125	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x},
126	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), board_X550EM_x},
127	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x},
128	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_1G_T), board_x550em_x_fw},
129	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a },
130	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), board_x550em_a },
131	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },
132	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a },
133	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },
134	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), board_x550em_a},
135	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },
136	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), board_x550em_a_fw },
137	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), board_x550em_a_fw },
138	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_BACKPLANE), board_e610},
139	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_SFP), board_e610},
140	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_10G_T), board_e610},
141	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_2_5G_T), board_e610},
142	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_SGMII), board_e610},
143	/* required last entry */
144	{0, }
145	};
146	MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
147
148	#ifdef CONFIG_IXGBE_DCA
149	static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
150	void *p);
151	static struct notifier_block dca_notifier = {
152	.notifier_call = ixgbe_notify_dca,
153	.next = NULL,
154	.priority = 0
155	};
156	#endif
157
158	#ifdef CONFIG_PCI_IOV
159	static unsigned int max_vfs;
160	module_param(max_vfs, uint, 0);
161	MODULE_PARM_DESC(max_vfs,
162	"Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63. (Deprecated)");
163	#endif /* CONFIG_PCI_IOV */
164
165	static bool allow_unsupported_sfp;
166	module_param(allow_unsupported_sfp, bool, 0444);
167	MODULE_PARM_DESC(allow_unsupported_sfp,
168	"Allow unsupported and untested SFP+ modules on 82599-based adapters");
169
170	#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
171	static int debug = -1;
172	module_param(debug, int, 0);
173	MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
174
175	MODULE_IMPORT_NS("LIBIE_FWLOG");
176	MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
177	MODULE_LICENSE("GPL v2");
178
179	DEFINE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key);
180	EXPORT_SYMBOL(ixgbe_xdp_locking_key);
181
182	static struct workqueue_struct *ixgbe_wq;
183
184	static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev);
185	static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *);
186	static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *);
187	static void ixgbe_watchdog_update_link(struct ixgbe_adapter *);
188
189	static const struct net_device_ops ixgbe_netdev_ops;
190
191	static bool netif_is_ixgbe(struct net_device *dev)
192	{
193	return dev && (dev->netdev_ops == &ixgbe_netdev_ops);
194	}
195
196	static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter,
197	u32 reg, u16 *value)
198	{
199	struct pci_dev *parent_dev;
200	struct pci_bus *parent_bus;
201
202	parent_bus = adapter->pdev->bus->parent;
203	if (!parent_bus)
204	return -1;
205
206	parent_dev = parent_bus->self;
207	if (!parent_dev)
208	return -1;
209
210	if (!pci_is_pcie(dev: parent_dev))
211	return -1;
212
213	pcie_capability_read_word(dev: parent_dev, pos: reg, val: value);
214	if (*value == IXGBE_FAILED_READ_CFG_WORD &&
215	ixgbe_check_cfg_remove(hw: &adapter->hw, pdev: parent_dev))
216	return -1;
217	return 0;
218	}
219
220	static int ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter)
221	{
222	struct ixgbe_hw *hw = &adapter->hw;
223	u16 link_status = 0;
224	int err;
225
226	hw->bus.type = ixgbe_bus_type_pci_express;
227
228	/* Get the negotiated link width and speed from PCI config space of the
229	* parent, as this device is behind a switch
230	*/
231	err = ixgbe_read_pci_cfg_word_parent(adapter, reg: 18, value: &link_status);
232
233	/* assume caller will handle error case */
234	if (err)
235	return err;
236
237	hw->bus.width = ixgbe_convert_bus_width(link_status);
238	hw->bus.speed = ixgbe_convert_bus_speed(link_status);
239
240	return 0;
241	}
242
243	/**
244	* ixgbe_pcie_from_parent - Determine whether PCIe info should come from parent
245	* @hw: hw specific details
246	*
247	* This function is used by probe to determine whether a device's PCI-Express
248	* bandwidth details should be gathered from the parent bus instead of from the
249	* device. Used to ensure that various locations all have the correct device ID
250	* checks.
251	*
252	* Return: true if information should be collected from the parent bus, false
253	* otherwise
254	*/
255	static bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw)
256	{
257	switch (hw->device_id) {
258	case IXGBE_DEV_ID_82599_SFP_SF_QP:
259	case IXGBE_DEV_ID_82599_QSFP_SF_QP:
260	return true;
261	default:
262	return false;
263	}
264	}
265
266	static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,
267	int expected_gts)
268	{
269	struct ixgbe_hw *hw = &adapter->hw;
270	struct pci_dev *pdev;
271
272	/* Some devices are not connected over PCIe and thus do not negotiate
273	* speed. These devices do not have valid bus info, and thus any report
274	* we generate may not be correct.
275	*/
276	if (hw->bus.type == ixgbe_bus_type_internal)
277	return;
278
279	/* determine whether to use the parent device */
280	if (ixgbe_pcie_from_parent(hw: &adapter->hw))
281	pdev = adapter->pdev->bus->parent->self;
282	else
283	pdev = adapter->pdev;
284
285	pcie_print_link_status(dev: pdev);
286	}
287
288	static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
289	{
290	if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
291	!test_bit(__IXGBE_REMOVING, &adapter->state) &&
292	!test_and_set_bit(nr: __IXGBE_SERVICE_SCHED, addr: &adapter->state))
293	queue_work(wq: ixgbe_wq, work: &adapter->service_task);
294	}
295
296	static void ixgbe_remove_adapter(struct ixgbe_hw *hw)
297	{
298	struct ixgbe_adapter *adapter = hw->back;
299
300	if (!hw->hw_addr)
301	return;
302	hw->hw_addr = NULL;
303	e_dev_err("Adapter removed\n");
304	if (test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
305	ixgbe_service_event_schedule(adapter);
306	}
307
308	static u32 ixgbe_check_remove(struct ixgbe_hw *hw, u32 reg)
309	{
310	u8 __iomem *reg_addr;
311	u32 value;
312	int i;
313
314	reg_addr = READ_ONCE(hw->hw_addr);
315	if (ixgbe_removed(addr: reg_addr))
316	return IXGBE_FAILED_READ_REG;
317
318	/* Register read of 0xFFFFFFF can indicate the adapter has been removed,
319	* so perform several status register reads to determine if the adapter
320	* has been removed.
321	*/
322	for (i = 0; i < IXGBE_FAILED_READ_RETRIES; i++) {
323	value = readl(addr: reg_addr + IXGBE_STATUS);
324	if (value != IXGBE_FAILED_READ_REG)
325	break;
326	mdelay(3);
327	}
328
329	if (value == IXGBE_FAILED_READ_REG)
330	ixgbe_remove_adapter(hw);
331	else
332	value = readl(addr: reg_addr + reg);
333	return value;
334	}
335
336	/**
337	* ixgbe_read_reg - Read from device register
338	* @hw: hw specific details
339	* @reg: offset of register to read
340	*
341	* Returns : value read or IXGBE_FAILED_READ_REG if removed
342	*
343	* This function is used to read device registers. It checks for device
344	* removal by confirming any read that returns all ones by checking the
345	* status register value for all ones. This function avoids reading from
346	* the hardware if a removal was previously detected in which case it
347	* returns IXGBE_FAILED_READ_REG (all ones).
348	*/
349	u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)
350	{
351	u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr);
352	u32 value;
353
354	if (ixgbe_removed(addr: reg_addr))
355	return IXGBE_FAILED_READ_REG;
356	if (unlikely(hw->phy.nw_mng_if_sel &
357	IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) {
358	struct ixgbe_adapter *adapter;
359	int i;
360
361	for (i = 0; i < 200; ++i) {
362	value = readl(addr: reg_addr + IXGBE_MAC_SGMII_BUSY);
363	if (likely(!value))
364	goto writes_completed;
365	if (value == IXGBE_FAILED_READ_REG) {
366	ixgbe_remove_adapter(hw);
367	return IXGBE_FAILED_READ_REG;
368	}
369	udelay(usec: 5);
370	}
371
372	adapter = hw->back;
373	e_warn(hw, "register writes incomplete %08x\n", value);
374	}
375
376	writes_completed:
377	value = readl(addr: reg_addr + reg);
378	if (unlikely(value == IXGBE_FAILED_READ_REG))
379	value = ixgbe_check_remove(hw, reg);
380	return value;
381	}
382
383	static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev)
384	{
385	u16 value;
386
387	pci_read_config_word(dev: pdev, PCI_VENDOR_ID, val: &value);
388	if (value == IXGBE_FAILED_READ_CFG_WORD) {
389	ixgbe_remove_adapter(hw);
390	return true;
391	}
392	return false;
393	}
394
395	u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg)
396	{
397	struct ixgbe_adapter *adapter = hw->back;
398	u16 value;
399
400	if (ixgbe_removed(addr: hw->hw_addr))
401	return IXGBE_FAILED_READ_CFG_WORD;
402	pci_read_config_word(dev: adapter->pdev, where: reg, val: &value);
403	if (value == IXGBE_FAILED_READ_CFG_WORD &&
404	ixgbe_check_cfg_remove(hw, pdev: adapter->pdev))
405	return IXGBE_FAILED_READ_CFG_WORD;
406	return value;
407	}
408
409	#ifdef CONFIG_PCI_IOV
410	static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg)
411	{
412	struct ixgbe_adapter *adapter = hw->back;
413	u32 value;
414
415	if (ixgbe_removed(addr: hw->hw_addr))
416	return IXGBE_FAILED_READ_CFG_DWORD;
417	pci_read_config_dword(dev: adapter->pdev, where: reg, val: &value);
418	if (value == IXGBE_FAILED_READ_CFG_DWORD &&
419	ixgbe_check_cfg_remove(hw, pdev: adapter->pdev))
420	return IXGBE_FAILED_READ_CFG_DWORD;
421	return value;
422	}
423	#endif /* CONFIG_PCI_IOV */
424
425	void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value)
426	{
427	struct ixgbe_adapter *adapter = hw->back;
428
429	if (ixgbe_removed(addr: hw->hw_addr))
430	return;
431	pci_write_config_word(dev: adapter->pdev, where: reg, val: value);
432	}
433
434	static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
435	{
436	BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
437
438	/* flush memory to make sure state is correct before next watchdog */
439	smp_mb__before_atomic();
440	clear_bit(nr: __IXGBE_SERVICE_SCHED, addr: &adapter->state);
441	}
442
443	struct ixgbe_reg_info {
444	u32 ofs;
445	char *name;
446	};
447
448	static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
449
450	/* General Registers */
451	{IXGBE_CTRL, "CTRL"},
452	{IXGBE_STATUS, "STATUS"},
453	{IXGBE_CTRL_EXT, "CTRL_EXT"},
454
455	/* Interrupt Registers */
456	{IXGBE_EICR, "EICR"},
457
458	/* RX Registers */
459	{IXGBE_SRRCTL(0), "SRRCTL"},
460	{IXGBE_DCA_RXCTRL(0), "DRXCTL"},
461	{IXGBE_RDLEN(0), "RDLEN"},
462	{IXGBE_RDH(0), "RDH"},
463	{IXGBE_RDT(0), "RDT"},
464	{IXGBE_RXDCTL(0), "RXDCTL"},
465	{IXGBE_RDBAL(0), "RDBAL"},
466	{IXGBE_RDBAH(0), "RDBAH"},
467
468	/* TX Registers */
469	{IXGBE_TDBAL(0), "TDBAL"},
470	{IXGBE_TDBAH(0), "TDBAH"},
471	{IXGBE_TDLEN(0), "TDLEN"},
472	{IXGBE_TDH(0), "TDH"},
473	{IXGBE_TDT(0), "TDT"},
474	{IXGBE_TXDCTL(0), "TXDCTL"},
475
476	/* List Terminator */
477	{ .name = NULL }
478	};
479
480
481	/*
482	* ixgbe_regdump - register printout routine
483	*/
484	static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
485	{
486	int i;
487	char rname[16];
488	u32 regs[64];
489
490	switch (reginfo->ofs) {
491	case IXGBE_SRRCTL(0):
492	for (i = 0; i < 64; i++)
493	regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
494	break;
495	case IXGBE_DCA_RXCTRL(0):
496	for (i = 0; i < 64; i++)
497	regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
498	break;
499	case IXGBE_RDLEN(0):
500	for (i = 0; i < 64; i++)
501	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
502	break;
503	case IXGBE_RDH(0):
504	for (i = 0; i < 64; i++)
505	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
506	break;
507	case IXGBE_RDT(0):
508	for (i = 0; i < 64; i++)
509	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
510	break;
511	case IXGBE_RXDCTL(0):
512	for (i = 0; i < 64; i++)
513	regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
514	break;
515	case IXGBE_RDBAL(0):
516	for (i = 0; i < 64; i++)
517	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
518	break;
519	case IXGBE_RDBAH(0):
520	for (i = 0; i < 64; i++)
521	regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
522	break;
523	case IXGBE_TDBAL(0):
524	for (i = 0; i < 64; i++)
525	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
526	break;
527	case IXGBE_TDBAH(0):
528	for (i = 0; i < 64; i++)
529	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
530	break;
531	case IXGBE_TDLEN(0):
532	for (i = 0; i < 64; i++)
533	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
534	break;
535	case IXGBE_TDH(0):
536	for (i = 0; i < 64; i++)
537	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
538	break;
539	case IXGBE_TDT(0):
540	for (i = 0; i < 64; i++)
541	regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
542	break;
543	case IXGBE_TXDCTL(0):
544	for (i = 0; i < 64; i++)
545	regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
546	break;
547	default:
548	pr_info("%-15s %08x\n",
549	reginfo->name, IXGBE_READ_REG(hw, reginfo->ofs));
550	return;
551	}
552
553	i = 0;
554	while (i < 64) {
555	int j;
556	char buf[9 * 8 + 1];
557	char *p = buf;
558
559	snprintf(buf: rname, size: 16, fmt: "%s[%d-%d]", reginfo->name, i, i + 7);
560	for (j = 0; j < 8; j++)
561	p += sprintf(buf: p, fmt: " %08x", regs[i++]);
562	pr_err("%-15s%s\n", rname, buf);
563	}
564
565	}
566
567	static void ixgbe_print_buffer(struct ixgbe_ring *ring, int n)
568	{
569	struct ixgbe_tx_buffer *tx_buffer;
570
571	tx_buffer = &ring->tx_buffer_info[ring->next_to_clean];
572	pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",
573	n, ring->next_to_use, ring->next_to_clean,
574	(u64)dma_unmap_addr(tx_buffer, dma),
575	dma_unmap_len(tx_buffer, len),
576	tx_buffer->next_to_watch,
577	(u64)tx_buffer->time_stamp);
578	}
579
580	/*
581	* ixgbe_dump - Print registers, tx-rings and rx-rings
582	*/
583	static void ixgbe_dump(struct ixgbe_adapter *adapter)
584	{
585	struct net_device *netdev = adapter->netdev;
586	struct ixgbe_hw *hw = &adapter->hw;
587	struct ixgbe_reg_info *reginfo;
588	int n = 0;
589	struct ixgbe_ring *ring;
590	struct ixgbe_tx_buffer *tx_buffer;
591	union ixgbe_adv_tx_desc *tx_desc;
592	struct my_u0 { u64 a; u64 b; } *u0;
593	struct ixgbe_ring *rx_ring;
594	union ixgbe_adv_rx_desc *rx_desc;
595	struct ixgbe_rx_buffer *rx_buffer_info;
596	int i = 0;
597
598	if (!netif_msg_hw(adapter))
599	return;
600
601	/* Print netdevice Info */
602	if (netdev) {
603	dev_info(&adapter->pdev->dev, "Net device Info\n");
604	pr_info("Device Name state "
605	"trans_start\n");
606	pr_info("%-15s %016lX %016lX\n",
607	netdev->name,
608	netdev->state,
609	dev_trans_start(netdev));
610	}
611
612	/* Print Registers */
613	dev_info(&adapter->pdev->dev, "Register Dump\n");
614	pr_info(" Register Name Value\n");
615	for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
616	reginfo->name; reginfo++) {
617	ixgbe_regdump(hw, reginfo);
618	}
619
620	/* Print TX Ring Summary */
621	if (!netdev || !netif_running(dev: netdev))
622	return;
623
624	dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
625	pr_info(" %s %s %s %s\n",
626	"Queue [NTU] [NTC] [bi(ntc)->dma ]",
627	"leng", "ntw", "timestamp");
628	for (n = 0; n < adapter->num_tx_queues; n++) {
629	ring = adapter->tx_ring[n];
630	ixgbe_print_buffer(ring, n);
631	}
632
633	for (n = 0; n < adapter->num_xdp_queues; n++) {
634	ring = adapter->xdp_ring[n];
635	ixgbe_print_buffer(ring, n);
636	}
637
638	/* Print TX Rings */
639	if (!netif_msg_tx_done(adapter))
640	goto rx_ring_summary;
641
642	dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
643
644	/* Transmit Descriptor Formats
645	*
646	* 82598 Advanced Transmit Descriptor
647	* +--------------------------------------------------------------+
648	* 0 | Buffer Address [63:0] |
649	* +--------------------------------------------------------------+
650	* 8 | PAYLEN | POPTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |
651	* +--------------------------------------------------------------+
652	* 63 46 45 40 39 36 35 32 31 24 23 20 19 0
653	*
654	* 82598 Advanced Transmit Descriptor (Write-Back Format)
655	* +--------------------------------------------------------------+
656	* 0 | RSV [63:0] |
657	* +--------------------------------------------------------------+
658	* 8 | RSV | STA | NXTSEQ |
659	* +--------------------------------------------------------------+
660	* 63 36 35 32 31 0
661	*
662	* 82599+ Advanced Transmit Descriptor
663	* +--------------------------------------------------------------+
664	* 0 | Buffer Address [63:0] |
665	* +--------------------------------------------------------------+
666	* 8 |PAYLEN |POPTS|CC|IDX |STA |DCMD |DTYP |MAC |RSV |DTALEN |
667	* +--------------------------------------------------------------+
668	* 63 46 45 40 39 38 36 35 32 31 24 23 20 19 18 17 16 15 0
669	*
670	* 82599+ Advanced Transmit Descriptor (Write-Back Format)
671	* +--------------------------------------------------------------+
672	* 0 | RSV [63:0] |
673	* +--------------------------------------------------------------+
674	* 8 | RSV | STA | RSV |
675	* +--------------------------------------------------------------+
676	* 63 36 35 32 31 0
677	*/
678
679	for (n = 0; n < adapter->num_tx_queues; n++) {
680	ring = adapter->tx_ring[n];
681	pr_info("------------------------------------\n");
682	pr_info("TX QUEUE INDEX = %d\n", ring->queue_index);
683	pr_info("------------------------------------\n");
684	pr_info("%s%s %s %s %s %s\n",
685	"T [desc] [address 63:0 ] ",
686	"[PlPOIdStDDt Ln] [bi->dma ] ",
687	"leng", "ntw", "timestamp", "bi->skb");
688
689	for (i = 0; ring->desc && (i < ring->count); i++) {
690	tx_desc = IXGBE_TX_DESC(ring, i);
691	tx_buffer = &ring->tx_buffer_info[i];
692	u0 = (struct my_u0 *)tx_desc;
693	if (dma_unmap_len(tx_buffer, len) > 0) {
694	const char *ring_desc;
695
696	if (i == ring->next_to_use &&
697	i == ring->next_to_clean)
698	ring_desc = " NTC/U";
699	else if (i == ring->next_to_use)
700	ring_desc = " NTU";
701	else if (i == ring->next_to_clean)
702	ring_desc = " NTC";
703	else
704	ring_desc = "";
705	pr_info("T [0x%03X] %016llX %016llX %016llX %08X %p %016llX %p%s",
706	i,
707	le64_to_cpu((__force __le64)u0->a),
708	le64_to_cpu((__force __le64)u0->b),
709	(u64)dma_unmap_addr(tx_buffer, dma),
710	dma_unmap_len(tx_buffer, len),
711	tx_buffer->next_to_watch,
712	(u64)tx_buffer->time_stamp,
713	tx_buffer->skb,
714	ring_desc);
715
716	if (netif_msg_pktdata(adapter) &&
717	tx_buffer->skb)
718	print_hex_dump(KERN_INFO, prefix_str: "",
719	prefix_type: DUMP_PREFIX_ADDRESS, rowsize: 16, groupsize: 1,
720	buf: tx_buffer->skb->data,
721	dma_unmap_len(tx_buffer, len),
722	ascii: true);
723	}
724	}
725	}
726
727	/* Print RX Rings Summary */
728	rx_ring_summary:
729	dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
730	pr_info("Queue [NTU] [NTC]\n");
731	for (n = 0; n < adapter->num_rx_queues; n++) {
732	rx_ring = adapter->rx_ring[n];
733	pr_info("%5d %5X %5X\n",
734	n, rx_ring->next_to_use, rx_ring->next_to_clean);
735	}
736
737	/* Print RX Rings */
738	if (!netif_msg_rx_status(adapter))
739	return;
740
741	dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
742
743	/* Receive Descriptor Formats
744	*
745	* 82598 Advanced Receive Descriptor (Read) Format
746	* 63 1 0
747	* +-----------------------------------------------------+
748	* 0 | Packet Buffer Address [63:1] |A0/NSE|
749	* +----------------------------------------------+------+
750	* 8 | Header Buffer Address [63:1] | DD |
751	* +-----------------------------------------------------+
752	*
753	*
754	* 82598 Advanced Receive Descriptor (Write-Back) Format
755	*
756	* 63 48 47 32 31 30 21 20 16 15 4 3 0
757	* +------------------------------------------------------+
758	* 0 | RSS Hash / |SPH| HDR_LEN | RSV |Packet| RSS |
759	* | Packet | IP | | | | Type | Type |
760	* | Checksum | Ident | | | | | |
761	* +------------------------------------------------------+
762	* 8 | VLAN Tag | Length | Extended Error | Extended Status |
763	* +------------------------------------------------------+
764	* 63 48 47 32 31 20 19 0
765	*
766	* 82599+ Advanced Receive Descriptor (Read) Format
767	* 63 1 0
768	* +-----------------------------------------------------+
769	* 0 | Packet Buffer Address [63:1] |A0/NSE|
770	* +----------------------------------------------+------+
771	* 8 | Header Buffer Address [63:1] | DD |
772	* +-----------------------------------------------------+
773	*
774	*
775	* 82599+ Advanced Receive Descriptor (Write-Back) Format
776	*
777	* 63 48 47 32 31 30 21 20 17 16 4 3 0
778	* +------------------------------------------------------+
779	* 0 |RSS / Frag Checksum|SPH| HDR_LEN |RSC- |Packet| RSS |
780	* |/ RTT / PCoE_PARAM | | | CNT | Type | Type |
781	* |/ Flow Dir Flt ID | | | | | |
782	* +------------------------------------------------------+
783	* 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |
784	* +------------------------------------------------------+
785	* 63 48 47 32 31 20 19 0
786	*/
787
788	for (n = 0; n < adapter->num_rx_queues; n++) {
789	rx_ring = adapter->rx_ring[n];
790	pr_info("------------------------------------\n");
791	pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
792	pr_info("------------------------------------\n");
793	pr_info("%s%s%s\n",
794	"R [desc] [ PktBuf A0] ",
795	"[ HeadBuf DD] [bi->dma ] [bi->skb ] ",
796	"<-- Adv Rx Read format");
797	pr_info("%s%s%s\n",
798	"RWB[desc] [PcsmIpSHl PtRs] ",
799	"[vl er S cks ln] ---------------- [bi->skb ] ",
800	"<-- Adv Rx Write-Back format");
801
802	for (i = 0; i < rx_ring->count; i++) {
803	const char *ring_desc;
804
805	if (i == rx_ring->next_to_use)
806	ring_desc = " NTU";
807	else if (i == rx_ring->next_to_clean)
808	ring_desc = " NTC";
809	else
810	ring_desc = "";
811
812	rx_buffer_info = &rx_ring->rx_buffer_info[i];
813	rx_desc = IXGBE_RX_DESC(rx_ring, i);
814	u0 = (struct my_u0 *)rx_desc;
815	if (rx_desc->wb.upper.length) {
816	/* Descriptor Done */
817	pr_info("RWB[0x%03X] %016llX %016llX ---------------- %p%s\n",
818	i,
819	le64_to_cpu((__force __le64)u0->a),
820	le64_to_cpu((__force __le64)u0->b),
821	rx_buffer_info->skb,
822	ring_desc);
823	} else {
824	pr_info("R [0x%03X] %016llX %016llX %016llX %p%s\n",
825	i,
826	le64_to_cpu((__force __le64)u0->a),
827	le64_to_cpu((__force __le64)u0->b),
828	(u64)rx_buffer_info->dma,
829	rx_buffer_info->skb,
830	ring_desc);
831
832	if (netif_msg_pktdata(adapter) &&
833	rx_buffer_info->dma) {
834	print_hex_dump(KERN_INFO, prefix_str: "",
835	prefix_type: DUMP_PREFIX_ADDRESS, rowsize: 16, groupsize: 1,
836	page_address(rx_buffer_info->page) +
837	rx_buffer_info->page_offset,
838	len: ixgbe_rx_bufsz(ring: rx_ring), ascii: true);
839	}
840	}
841	}
842	}
843	}
844
845	static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
846	{
847	u32 ctrl_ext;
848
849	/* Let firmware take over control of h/w */
850	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
851	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
852	ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
853	}
854
855	static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
856	{
857	u32 ctrl_ext;
858
859	/* Let firmware know the driver has taken over */
860	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
861	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
862	ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
863	}
864
865	/**
866	* ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
867	* @adapter: pointer to adapter struct
868	* @direction: 0 for Rx, 1 for Tx, -1 for other causes
869	* @queue: queue to map the corresponding interrupt to
870	* @msix_vector: the vector to map to the corresponding queue
871	*
872	*/
873	static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
874	u8 queue, u8 msix_vector)
875	{
876	u32 ivar, index;
877	struct ixgbe_hw *hw = &adapter->hw;
878	switch (hw->mac.type) {
879	case ixgbe_mac_82598EB:
880	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
881	if (direction == -1)
882	direction = 0;
883	index = (((direction * 64) + queue) >> 2) & 0x1F;
884	ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
885	ivar &= ~(0xFF << (8 * (queue & 0x3)));
886	ivar |= (msix_vector << (8 * (queue & 0x3)));
887	IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
888	break;
889	case ixgbe_mac_82599EB:
890	case ixgbe_mac_X540:
891	case ixgbe_mac_X550:
892	case ixgbe_mac_X550EM_x:
893	case ixgbe_mac_x550em_a:
894	case ixgbe_mac_e610:
895	if (direction == -1) {
896	/* other causes */
897	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
898	index = ((queue & 1) * 8);
899	ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
900	ivar &= ~(0xFF << index);
901	ivar |= (msix_vector << index);
902	IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
903	break;
904	} else {
905	/* tx or rx causes */
906	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
907	index = ((16 * (queue & 1)) + (8 * direction));
908	ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
909	ivar &= ~(0xFF << index);
910	ivar |= (msix_vector << index);
911	IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
912	break;
913	}
914	default:
915	break;
916	}
917	}
918
919	void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
920	u64 qmask)
921	{
922	u32 mask;
923
924	switch (adapter->hw.mac.type) {
925	case ixgbe_mac_82598EB:
926	mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
927	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
928	break;
929	case ixgbe_mac_82599EB:
930	case ixgbe_mac_X540:
931	case ixgbe_mac_X550:
932	case ixgbe_mac_X550EM_x:
933	case ixgbe_mac_x550em_a:
934	case ixgbe_mac_e610:
935	mask = (qmask & 0xFFFFFFFF);
936	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
937	mask = (qmask >> 32);
938	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
939	break;
940	default:
941	break;
942	}
943	}
944
945	static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
946	{
947	struct ixgbe_hw *hw = &adapter->hw;
948	struct ixgbe_hw_stats *hwstats = &adapter->stats;
949	int i;
950	u32 data;
951
952	if ((hw->fc.current_mode != ixgbe_fc_full) &&
953	(hw->fc.current_mode != ixgbe_fc_rx_pause))
954	return;
955
956	switch (hw->mac.type) {
957	case ixgbe_mac_82598EB:
958	data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
959	break;
960	default:
961	data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
962	}
963	hwstats->lxoffrxc += data;
964
965	/* refill credits (no tx hang) if we received xoff */
966	if (!data)
967	return;
968
969	for (i = 0; i < adapter->num_tx_queues; i++)
970	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED,
971	addr: &adapter->tx_ring[i]->state);
972	}
973
974	static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
975	{
976	struct ixgbe_hw *hw = &adapter->hw;
977	struct ixgbe_hw_stats *hwstats = &adapter->stats;
978	u32 xoff[8] = {0};
979	u8 tc;
980	int i;
981	bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
982
983	if (adapter->ixgbe_ieee_pfc)
984	pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
985
986	if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
987	ixgbe_update_xoff_rx_lfc(adapter);
988	return;
989	}
990
991	/* update stats for each tc, only valid with PFC enabled */
992	for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
993	u32 pxoffrxc;
994
995	switch (hw->mac.type) {
996	case ixgbe_mac_82598EB:
997	pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
998	break;
999	default:
1000	pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
1001	}
1002	hwstats->pxoffrxc[i] += pxoffrxc;
1003	/* Get the TC for given UP */
1004	tc = netdev_get_prio_tc_map(dev: adapter->netdev, prio: i);
1005	xoff[tc] += pxoffrxc;
1006	}
1007
1008	/* disarm tx queues that have received xoff frames */
1009	for (i = 0; i < adapter->num_tx_queues; i++) {
1010	struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
1011
1012	tc = tx_ring->dcb_tc;
1013	if (xoff[tc])
1014	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED, addr: &tx_ring->state);
1015	}
1016
1017	for (i = 0; i < adapter->num_xdp_queues; i++) {
1018	struct ixgbe_ring *xdp_ring = adapter->xdp_ring[i];
1019
1020	tc = xdp_ring->dcb_tc;
1021	if (xoff[tc])
1022	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED, addr: &xdp_ring->state);
1023	}
1024	}
1025
1026	static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
1027	{
1028	return ring->stats.packets;
1029	}
1030
1031	static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
1032	{
1033	unsigned int head, tail;
1034
1035	head = ring->next_to_clean;
1036	tail = ring->next_to_use;
1037
1038	return ((head <= tail) ? tail : tail + ring->count) - head;
1039	}
1040
1041	/**
1042	* ixgbe_get_vf_idx - provide VF index number based on queue index
1043	* @adapter: pointer to the adapter struct
1044	* @queue: Tx queue identifier
1045	* @vf: output VF index
1046	*
1047	* Provide VF index number associated to the input queue.
1048	*
1049	* Returns: 0 if VF provided or error number.
1050	*/
1051	static int ixgbe_get_vf_idx(struct ixgbe_adapter *adapter, u16 queue, u16 *vf)
1052	{
1053	struct ixgbe_hw *hw = &adapter->hw;
1054	u8 queue_count;
1055	u32 reg;
1056
1057	if (queue >= adapter->num_tx_queues)
1058	return -EINVAL;
1059
1060	/* Determine number of queues by checking
1061	* number of virtual functions
1062	*/
1063	reg = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
1064	switch (reg & IXGBE_GCR_EXT_VT_MODE_MASK) {
1065	case IXGBE_GCR_EXT_VT_MODE_64:
1066	queue_count = IXGBE_64VFS_QUEUES;
1067	break;
1068	case IXGBE_GCR_EXT_VT_MODE_32:
1069	queue_count = IXGBE_32VFS_QUEUES;
1070	break;
1071	case IXGBE_GCR_EXT_VT_MODE_16:
1072	queue_count = IXGBE_16VFS_QUEUES;
1073	break;
1074	default:
1075	return -EINVAL;
1076	}
1077
1078	*vf = queue / queue_count;
1079
1080	return 0;
1081	}
1082
1083	static bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
1084	{
1085	u32 tx_done = ixgbe_get_tx_completed(ring: tx_ring);
1086	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
1087	u32 tx_pending = ixgbe_get_tx_pending(ring: tx_ring);
1088
1089	clear_check_for_tx_hang(tx_ring);
1090
1091	/*
1092	* Check for a hung queue, but be thorough. This verifies
1093	* that a transmit has been completed since the previous
1094	* check AND there is at least one packet pending. The
1095	* ARMED bit is set to indicate a potential hang. The
1096	* bit is cleared if a pause frame is received to remove
1097	* false hang detection due to PFC or 802.3x frames. By
1098	* requiring this to fail twice we avoid races with
1099	* pfc clearing the ARMED bit and conditions where we
1100	* run the check_tx_hang logic with a transmit completion
1101	* pending but without time to complete it yet.
1102	*/
1103	if (tx_done_old == tx_done && tx_pending)
1104	/* make sure it is true for two checks in a row */
1105	return test_and_set_bit(nr: __IXGBE_HANG_CHECK_ARMED,
1106	addr: &tx_ring->state);
1107	/* update completed stats and continue */
1108	tx_ring->tx_stats.tx_done_old = tx_done;
1109	/* reset the countdown */
1110	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED, addr: &tx_ring->state);
1111
1112	return false;
1113	}
1114
1115	/**
1116	* ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
1117	* @adapter: driver private struct
1118	**/
1119	static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
1120	{
1121
1122	/* Do the reset outside of interrupt context */
1123	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1124	set_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
1125	e_warn(drv, "initiating reset due to tx timeout\n");
1126	ixgbe_service_event_schedule(adapter);
1127	}
1128	}
1129
1130	/**
1131	* ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate
1132	* @netdev: network interface device structure
1133	* @queue_index: Tx queue to set
1134	* @maxrate: desired maximum transmit bitrate
1135	**/
1136	static int ixgbe_tx_maxrate(struct net_device *netdev,
1137	int queue_index, u32 maxrate)
1138	{
1139	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
1140	struct ixgbe_hw *hw = &adapter->hw;
1141	u32 bcnrc_val = ixgbe_link_mbps(adapter);
1142
1143	if (!maxrate)
1144	return 0;
1145
1146	/* Calculate the rate factor values to set */
1147	bcnrc_val <<= IXGBE_RTTBCNRC_RF_INT_SHIFT;
1148	bcnrc_val /= maxrate;
1149
1150	/* clear everything but the rate factor */
1151	bcnrc_val &= IXGBE_RTTBCNRC_RF_INT_MASK |
1152	IXGBE_RTTBCNRC_RF_DEC_MASK;
1153
1154	/* enable the rate scheduler */
1155	bcnrc_val |= IXGBE_RTTBCNRC_RS_ENA;
1156
1157	IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_index);
1158	IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
1159
1160	return 0;
1161	}
1162
1163	/**
1164	* ixgbe_update_tx_ring_stats - Update Tx ring specific counters
1165	* @tx_ring: ring to update
1166	* @q_vector: queue vector ring belongs to
1167	* @pkts: number of processed packets
1168	* @bytes: number of processed bytes
1169	*/
1170	void ixgbe_update_tx_ring_stats(struct ixgbe_ring *tx_ring,
1171	struct ixgbe_q_vector *q_vector, u64 pkts,
1172	u64 bytes)
1173	{
1174	u64_stats_update_begin(syncp: &tx_ring->syncp);
1175	tx_ring->stats.bytes += bytes;
1176	tx_ring->stats.packets += pkts;
1177	u64_stats_update_end(syncp: &tx_ring->syncp);
1178	q_vector->tx.total_bytes += bytes;
1179	q_vector->tx.total_packets += pkts;
1180	}
1181
1182	/**
1183	* ixgbe_update_rx_ring_stats - Update Rx ring specific counters
1184	* @rx_ring: ring to update
1185	* @q_vector: queue vector ring belongs to
1186	* @pkts: number of processed packets
1187	* @bytes: number of processed bytes
1188	*/
1189	void ixgbe_update_rx_ring_stats(struct ixgbe_ring *rx_ring,
1190	struct ixgbe_q_vector *q_vector, u64 pkts,
1191	u64 bytes)
1192	{
1193	u64_stats_update_begin(syncp: &rx_ring->syncp);
1194	rx_ring->stats.bytes += bytes;
1195	rx_ring->stats.packets += pkts;
1196	u64_stats_update_end(syncp: &rx_ring->syncp);
1197	q_vector->rx.total_bytes += bytes;
1198	q_vector->rx.total_packets += pkts;
1199	}
1200
1201	/**
1202	* ixgbe_pf_handle_tx_hang - handle Tx hang on PF
1203	* @tx_ring: tx ring number
1204	* @next: next ring
1205	*
1206	* Prints a message containing details about the tx hang.
1207	*/
1208	static void ixgbe_pf_handle_tx_hang(struct ixgbe_ring *tx_ring,
1209	unsigned int next)
1210	{
1211	struct ixgbe_adapter *adapter = netdev_priv(dev: tx_ring->netdev);
1212	struct ixgbe_hw *hw = &adapter->hw;
1213
1214	e_err(drv, "Detected Tx Unit Hang\n"
1215	" Tx Queue <%d>\n"
1216	" TDH, TDT <%x>, <%x>\n"
1217	" next_to_use <%x>\n"
1218	" next_to_clean <%x>\n"
1219	"tx_buffer_info[next_to_clean]\n"
1220	" time_stamp <%lx>\n"
1221	" jiffies <%lx>\n",
1222	tx_ring->queue_index,
1223	IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
1224	IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
1225	tx_ring->next_to_use, next,
1226	tx_ring->tx_buffer_info[next].time_stamp, jiffies);
1227
1228	netif_stop_subqueue(dev: tx_ring->netdev,
1229	queue_index: tx_ring->queue_index);
1230	}
1231
1232	/**
1233	* ixgbe_vf_handle_tx_hang - handle Tx hang on VF
1234	* @adapter: structure containing ring specific data
1235	* @vf: VF index
1236	*
1237	* Print a message containing details about malicious driver detection.
1238	* Set malicious VF link down if the detection happened several times.
1239	*/
1240	static void ixgbe_vf_handle_tx_hang(struct ixgbe_adapter *adapter, u16 vf)
1241	{
1242	struct ixgbe_hw *hw = &adapter->hw;
1243
1244	if (adapter->hw.mac.type != ixgbe_mac_e610)
1245	return;
1246
1247	e_warn(drv,
1248	"Malicious Driver Detection tx hang detected on PF %d VF %d MAC: %pM",
1249	hw->bus.func, vf, adapter->vfinfo[vf].vf_mac_addresses);
1250
1251	adapter->tx_hang_count[vf]++;
1252	if (adapter->tx_hang_count[vf] == IXGBE_MAX_TX_VF_HANGS) {
1253	ixgbe_set_vf_link_state(adapter, vf,
1254	state: IFLA_VF_LINK_STATE_DISABLE);
1255	adapter->tx_hang_count[vf] = 0;
1256	}
1257	}
1258
1259	static u32 ixgbe_poll_tx_icache(struct ixgbe_hw *hw, u16 queue, u16 idx)
1260	{
1261	IXGBE_WRITE_REG(hw, IXGBE_TXDESCIC, queue * idx);
1262	return IXGBE_READ_REG(hw, IXGBE_TXDESCIC);
1263	}
1264
1265	/**
1266	* ixgbe_check_illegal_queue - search for queue with illegal packet
1267	* @adapter: structure containing ring specific data
1268	* @queue: queue index
1269	*
1270	* Check if tx descriptor connected with input queue
1271	* contains illegal packet.
1272	*
1273	* Returns: true if queue contain illegal packet.
1274	*/
1275	static bool ixgbe_check_illegal_queue(struct ixgbe_adapter *adapter,
1276	u16 queue)
1277	{
1278	u32 hdr_len_reg, mss_len_reg, type_reg;
1279	struct ixgbe_hw *hw = &adapter->hw;
1280	u32 mss_len, header_len, reg;
1281
1282	for (u16 i = 0; i < IXGBE_MAX_TX_DESCRIPTORS; i++) {
1283	/* HW will clear bit IXGBE_TXDESCIC_READY when address
1284	* is written to address field. HW will set this bit
1285	* when iCache read is done, and data is ready at TIC_DWx.
1286	* Set descriptor address.
1287	*/
1288	read_poll_timeout(ixgbe_poll_tx_icache, reg,
1289	!(reg & IXGBE_TXDESCIC_READY), 0, 0, false,
1290	hw, queue, i);
1291
1292	/* read tx descriptor access registers */
1293	hdr_len_reg = IXGBE_READ_REG(hw, IXGBE_TIC_DW2(IXGBE_VLAN_MACIP_LENS_REG));
1294	type_reg = IXGBE_READ_REG(hw, IXGBE_TIC_DW2(IXGBE_TYPE_TUCMD_MLHL));
1295	mss_len_reg = IXGBE_READ_REG(hw, IXGBE_TIC_DW2(IXGBE_MSS_L4LEN_IDX));
1296
1297	/* check if Advanced Context Descriptor */
1298	if (FIELD_GET(IXGBE_ADVTXD_DTYP_MASK, type_reg) !=
1299	IXGBE_ADVTXD_DTYP_CTXT)
1300	continue;
1301
1302	/* check for illegal MSS and Header length */
1303	mss_len = FIELD_GET(IXGBE_ADVTXD_MSS_MASK, mss_len_reg);
1304	header_len = FIELD_GET(IXGBE_ADVTXD_HEADER_LEN_MASK,
1305	hdr_len_reg);
1306	if ((mss_len + header_len) > SZ_16K) {
1307	e_warn(probe, "mss len + header len too long\n");
1308	return true;
1309	}
1310	}
1311
1312	return false;
1313	}
1314
1315	/**
1316	* ixgbe_handle_mdd_event - handle mdd event
1317	* @adapter: structure containing ring specific data
1318	* @tx_ring: tx descriptor ring to handle
1319	*
1320	* Reset VF driver if malicious vf detected or
1321	* illegal packet in an any queue detected.
1322	*/
1323	static void ixgbe_handle_mdd_event(struct ixgbe_adapter *adapter,
1324	struct ixgbe_ring *tx_ring)
1325	{
1326	u16 vf, q;
1327
1328	if (adapter->vfinfo && ixgbe_check_mdd_event(adapter)) {
1329	/* vf mdd info and malicious vf detected */
1330	if (!ixgbe_get_vf_idx(adapter, queue: tx_ring->queue_index, vf: &vf))
1331	ixgbe_vf_handle_tx_hang(adapter, vf);
1332	} else {
1333	/* malicious vf not detected */
1334	for (q = 0; q < IXGBE_MAX_TX_QUEUES; q++) {
1335	if (ixgbe_check_illegal_queue(adapter, queue: q) &&
1336	!ixgbe_get_vf_idx(adapter, queue: q, vf: &vf))
1337	/* illegal queue detected */
1338	ixgbe_vf_handle_tx_hang(adapter, vf);
1339	}
1340	}
1341	}
1342
1343	/**
1344	* ixgbe_clean_tx_irq - Reclaim resources after transmit completes
1345	* @q_vector: structure containing interrupt and ring information
1346	* @tx_ring: tx ring to clean
1347	* @napi_budget: Used to determine if we are in netpoll
1348	**/
1349	static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
1350	struct ixgbe_ring *tx_ring, int napi_budget)
1351	{
1352	struct ixgbe_adapter *adapter = q_vector->adapter;
1353	struct ixgbe_tx_buffer *tx_buffer;
1354	union ixgbe_adv_tx_desc *tx_desc;
1355	unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
1356	unsigned int budget = q_vector->tx.work_limit;
1357	unsigned int i = tx_ring->next_to_clean;
1358	struct netdev_queue *txq;
1359
1360	if (test_bit(__IXGBE_DOWN, &adapter->state))
1361	return true;
1362
1363	tx_buffer = &tx_ring->tx_buffer_info[i];
1364	tx_desc = IXGBE_TX_DESC(tx_ring, i);
1365	i -= tx_ring->count;
1366
1367	do {
1368	union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
1369
1370	/* if next_to_watch is not set then there is no work pending */
1371	if (!eop_desc)
1372	break;
1373
1374	/* prevent any other reads prior to eop_desc */
1375	smp_rmb();
1376
1377	/* if DD is not set pending work has not been completed */
1378	if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
1379	break;
1380
1381	/* clear next_to_watch to prevent false hangs */
1382	tx_buffer->next_to_watch = NULL;
1383
1384	/* update the statistics for this packet */
1385	total_bytes += tx_buffer->bytecount;
1386	total_packets += tx_buffer->gso_segs;
1387	if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
1388	total_ipsec++;
1389
1390	/* free the skb */
1391	if (ring_is_xdp(tx_ring))
1392	xdp_return_frame(xdpf: tx_buffer->xdpf);
1393	else
1394	napi_consume_skb(skb: tx_buffer->skb, budget: napi_budget);
1395
1396	/* unmap skb header data */
1397	dma_unmap_single(tx_ring->dev,
1398	dma_unmap_addr(tx_buffer, dma),
1399	dma_unmap_len(tx_buffer, len),
1400	DMA_TO_DEVICE);
1401
1402	/* clear tx_buffer data */
1403	dma_unmap_len_set(tx_buffer, len, 0);
1404
1405	/* unmap remaining buffers */
1406	while (tx_desc != eop_desc) {
1407	tx_buffer++;
1408	tx_desc++;
1409	i++;
1410	if (unlikely(!i)) {
1411	i -= tx_ring->count;
1412	tx_buffer = tx_ring->tx_buffer_info;
1413	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
1414	}
1415
1416	/* unmap any remaining paged data */
1417	if (dma_unmap_len(tx_buffer, len)) {
1418	dma_unmap_page(tx_ring->dev,
1419	dma_unmap_addr(tx_buffer, dma),
1420	dma_unmap_len(tx_buffer, len),
1421	DMA_TO_DEVICE);
1422	dma_unmap_len_set(tx_buffer, len, 0);
1423	}
1424	}
1425
1426	/* move us one more past the eop_desc for start of next pkt */
1427	tx_buffer++;
1428	tx_desc++;
1429	i++;
1430	if (unlikely(!i)) {
1431	i -= tx_ring->count;
1432	tx_buffer = tx_ring->tx_buffer_info;
1433	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
1434	}
1435
1436	/* issue prefetch for next Tx descriptor */
1437	prefetch(tx_desc);
1438
1439	/* update budget accounting */
1440	budget--;
1441	} while (likely(budget));
1442
1443	i += tx_ring->count;
1444	tx_ring->next_to_clean = i;
1445	ixgbe_update_tx_ring_stats(tx_ring, q_vector, pkts: total_packets,
1446	bytes: total_bytes);
1447	adapter->tx_ipsec += total_ipsec;
1448
1449	if (ring_is_xdp(tx_ring))
1450	return !!budget;
1451
1452	if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
1453	if (adapter->hw.mac.type == ixgbe_mac_e610)
1454	ixgbe_handle_mdd_event(adapter, tx_ring);
1455
1456	ixgbe_pf_handle_tx_hang(tx_ring, next: i);
1457
1458	e_info(probe,
1459	"tx hang %d detected on queue %d, resetting adapter\n",
1460	adapter->tx_timeout_count + 1, tx_ring->queue_index);
1461
1462	/* schedule immediate reset if we believe we hung */
1463	ixgbe_tx_timeout_reset(adapter);
1464
1465	/* the adapter is about to reset, no point in enabling stuff */
1466	return true;
1467	}
1468
1469	#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
1470	txq = netdev_get_tx_queue(dev: tx_ring->netdev, index: tx_ring->queue_index);
1471	if (!__netif_txq_completed_wake(txq, total_packets, total_bytes,
1472	ixgbe_desc_unused(tx_ring),
1473	TX_WAKE_THRESHOLD,
1474	!netif_carrier_ok(tx_ring->netdev) ||
1475	test_bit(__IXGBE_DOWN, &adapter->state)))
1476	++tx_ring->tx_stats.restart_queue;
1477
1478	return !!budget;
1479	}
1480
1481	#ifdef CONFIG_IXGBE_DCA
1482	static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
1483	struct ixgbe_ring *tx_ring,
1484	int cpu)
1485	{
1486	struct ixgbe_hw *hw = &adapter->hw;
1487	u32 txctrl = 0;
1488	u16 reg_offset;
1489
1490	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1491	txctrl = dca3_get_tag(dev: tx_ring->dev, cpu);
1492
1493	switch (hw->mac.type) {
1494	case ixgbe_mac_82598EB:
1495	reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
1496	break;
1497	case ixgbe_mac_82599EB:
1498	case ixgbe_mac_X540:
1499	reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
1500	txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;
1501	break;
1502	default:
1503	/* for unknown hardware do not write register */
1504	return;
1505	}
1506
1507	/*
1508	* We can enable relaxed ordering for reads, but not writes when
1509	* DCA is enabled. This is due to a known issue in some chipsets
1510	* which will cause the DCA tag to be cleared.
1511	*/
1512	txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1513	IXGBE_DCA_TXCTRL_DATA_RRO_EN |
1514	IXGBE_DCA_TXCTRL_DESC_DCA_EN;
1515
1516	IXGBE_WRITE_REG(hw, reg_offset, txctrl);
1517	}
1518
1519	static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
1520	struct ixgbe_ring *rx_ring,
1521	int cpu)
1522	{
1523	struct ixgbe_hw *hw = &adapter->hw;
1524	u32 rxctrl = 0;
1525	u8 reg_idx = rx_ring->reg_idx;
1526
1527	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1528	rxctrl = dca3_get_tag(dev: rx_ring->dev, cpu);
1529
1530	switch (hw->mac.type) {
1531	case ixgbe_mac_82599EB:
1532	case ixgbe_mac_X540:
1533	rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;
1534	break;
1535	default:
1536	break;
1537	}
1538
1539	/*
1540	* We can enable relaxed ordering for reads, but not writes when
1541	* DCA is enabled. This is due to a known issue in some chipsets
1542	* which will cause the DCA tag to be cleared.
1543	*/
1544	rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
1545	IXGBE_DCA_RXCTRL_DATA_DCA_EN |
1546	IXGBE_DCA_RXCTRL_DESC_DCA_EN;
1547
1548	IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
1549	}
1550
1551	static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
1552	{
1553	struct ixgbe_adapter *adapter = q_vector->adapter;
1554	struct ixgbe_ring *ring;
1555	int cpu = get_cpu();
1556
1557	if (q_vector->cpu == cpu)
1558	goto out_no_update;
1559
1560	ixgbe_for_each_ring(ring, q_vector->tx)
1561	ixgbe_update_tx_dca(adapter, tx_ring: ring, cpu);
1562
1563	ixgbe_for_each_ring(ring, q_vector->rx)
1564	ixgbe_update_rx_dca(adapter, rx_ring: ring, cpu);
1565
1566	q_vector->cpu = cpu;
1567	out_no_update:
1568	put_cpu();
1569	}
1570
1571	static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
1572	{
1573	int i;
1574
1575	/* always use CB2 mode, difference is masked in the CB driver */
1576	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1577	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1578	IXGBE_DCA_CTRL_DCA_MODE_CB2);
1579	else
1580	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1581	IXGBE_DCA_CTRL_DCA_DISABLE);
1582
1583	for (i = 0; i < adapter->num_q_vectors; i++) {
1584	adapter->q_vector[i]->cpu = -1;
1585	ixgbe_update_dca(q_vector: adapter->q_vector[i]);
1586	}
1587	}
1588
1589	static int __ixgbe_notify_dca(struct device *dev, void *data)
1590	{
1591	struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
1592	unsigned long event = *(unsigned long *)data;
1593
1594	if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
1595	return 0;
1596
1597	switch (event) {
1598	case DCA_PROVIDER_ADD:
1599	/* if we're already enabled, don't do it again */
1600	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1601	break;
1602	if (dca_add_requester(dev) == 0) {
1603	adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
1604	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1605	IXGBE_DCA_CTRL_DCA_MODE_CB2);
1606	break;
1607	}
1608	fallthrough; /* DCA is disabled. */
1609	case DCA_PROVIDER_REMOVE:
1610	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
1611	dca_remove_requester(dev);
1612	adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
1613	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
1614	IXGBE_DCA_CTRL_DCA_DISABLE);
1615	}
1616	break;
1617	}
1618
1619	return 0;
1620	}
1621
1622	#endif /* CONFIG_IXGBE_DCA */
1623
1624	#define IXGBE_RSS_L4_TYPES_MASK \
1625	((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \
1626	(1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \
1627	(1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \
1628	(1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))
1629
1630	static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
1631	union ixgbe_adv_rx_desc *rx_desc,
1632	struct sk_buff *skb)
1633	{
1634	u16 rss_type;
1635
1636	if (!(ring->netdev->features & NETIF_F_RXHASH))
1637	return;
1638
1639	rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
1640	IXGBE_RXDADV_RSSTYPE_MASK;
1641
1642	if (!rss_type)
1643	return;
1644
1645	skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1646	type: (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
1647	PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
1648	}
1649
1650	#ifdef IXGBE_FCOE
1651	/**
1652	* ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
1653	* @ring: structure containing ring specific data
1654	* @rx_desc: advanced rx descriptor
1655	*
1656	* Returns : true if it is FCoE pkt
1657	*/
1658	static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
1659	union ixgbe_adv_rx_desc *rx_desc)
1660	{
1661	__le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1662
1663	return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
1664	((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1665	(cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1666	IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1667	}
1668
1669	#endif /* IXGBE_FCOE */
1670	/**
1671	* ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1672	* @ring: structure containing ring specific data
1673	* @rx_desc: current Rx descriptor being processed
1674	* @skb: skb currently being received and modified
1675	**/
1676	static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
1677	union ixgbe_adv_rx_desc *rx_desc,
1678	struct sk_buff *skb)
1679	{
1680	__le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1681	bool encap_pkt = false;
1682
1683	skb_checksum_none_assert(skb);
1684
1685	/* Rx csum disabled */
1686	if (!(ring->netdev->features & NETIF_F_RXCSUM))
1687	return;
1688
1689	/* check for VXLAN and Geneve packets */
1690	if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) {
1691	encap_pkt = true;
1692	skb->encapsulation = 1;
1693	}
1694
1695	/* if IP and error */
1696	if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
1697	ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
1698	ring->rx_stats.csum_err++;
1699	return;
1700	}
1701
1702	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
1703	return;
1704
1705	if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
1706	/*
1707	* 82599 errata, UDP frames with a 0 checksum can be marked as
1708	* checksum errors.
1709	*/
1710	if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
1711	test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
1712	return;
1713
1714	ring->rx_stats.csum_err++;
1715	return;
1716	}
1717
1718	/* It must be a TCP or UDP packet with a valid checksum */
1719	skb->ip_summed = CHECKSUM_UNNECESSARY;
1720	if (encap_pkt) {
1721	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_OUTERIPCS))
1722	return;
1723
1724	if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_OUTERIPER)) {
1725	skb->ip_summed = CHECKSUM_NONE;
1726	return;
1727	}
1728	/* If we checked the outer header let the stack know */
1729	skb->csum_level = 1;
1730	}
1731	}
1732
1733	static unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring)
1734	{
1735	return ring_uses_build_skb(rx_ring) ? IXGBE_SKB_PAD : 0;
1736	}
1737
1738	static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
1739	struct ixgbe_rx_buffer *bi)
1740	{
1741	struct page *page = bi->page;
1742	dma_addr_t dma;
1743
1744	/* since we are recycling buffers we should seldom need to alloc */
1745	if (likely(page))
1746	return true;
1747
1748	/* alloc new page for storage */
1749	page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring));
1750	if (unlikely(!page)) {
1751	rx_ring->rx_stats.alloc_rx_page_failed++;
1752	return false;
1753	}
1754
1755	/* map page for use */
1756	dma = dma_map_page_attrs(dev: rx_ring->dev, page, offset: 0,
1757	ixgbe_rx_pg_size(rx_ring),
1758	dir: DMA_FROM_DEVICE,
1759	IXGBE_RX_DMA_ATTR);
1760
1761	/*
1762	* if mapping failed free memory back to system since
1763	* there isn't much point in holding memory we can't use
1764	*/
1765	if (dma_mapping_error(dev: rx_ring->dev, dma_addr: dma)) {
1766	__free_pages(page, order: ixgbe_rx_pg_order(ring: rx_ring));
1767
1768	rx_ring->rx_stats.alloc_rx_page_failed++;
1769	return false;
1770	}
1771
1772	bi->dma = dma;
1773	bi->page = page;
1774	bi->page_offset = rx_ring->rx_offset;
1775	page_ref_add(page, USHRT_MAX - 1);
1776	bi->pagecnt_bias = USHRT_MAX;
1777	rx_ring->rx_stats.alloc_rx_page++;
1778
1779	return true;
1780	}
1781
1782	/**
1783	* ixgbe_alloc_rx_buffers - Replace used receive buffers
1784	* @rx_ring: ring to place buffers on
1785	* @cleaned_count: number of buffers to replace
1786	**/
1787	void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1788	{
1789	union ixgbe_adv_rx_desc *rx_desc;
1790	struct ixgbe_rx_buffer *bi;
1791	u16 i = rx_ring->next_to_use;
1792	u16 bufsz;
1793
1794	/* nothing to do */
1795	if (!cleaned_count)
1796	return;
1797
1798	rx_desc = IXGBE_RX_DESC(rx_ring, i);
1799	bi = &rx_ring->rx_buffer_info[i];
1800	i -= rx_ring->count;
1801
1802	bufsz = ixgbe_rx_bufsz(ring: rx_ring);
1803
1804	do {
1805	if (!ixgbe_alloc_mapped_page(rx_ring, bi))
1806	break;
1807
1808	/* sync the buffer for use by the device */
1809	dma_sync_single_range_for_device(dev: rx_ring->dev, addr: bi->dma,
1810	offset: bi->page_offset, size: bufsz,
1811	dir: DMA_FROM_DEVICE);
1812
1813	/*
1814	* Refresh the desc even if buffer_addrs didn't change
1815	* because each write-back erases this info.
1816	*/
1817	rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1818
1819	rx_desc++;
1820	bi++;
1821	i++;
1822	if (unlikely(!i)) {
1823	rx_desc = IXGBE_RX_DESC(rx_ring, 0);
1824	bi = rx_ring->rx_buffer_info;
1825	i -= rx_ring->count;
1826	}
1827
1828	/* clear the length for the next_to_use descriptor */
1829	rx_desc->wb.upper.length = 0;
1830
1831	cleaned_count--;
1832	} while (cleaned_count);
1833
1834	i += rx_ring->count;
1835
1836	if (rx_ring->next_to_use != i) {
1837	rx_ring->next_to_use = i;
1838
1839	/* update next to alloc since we have filled the ring */
1840	rx_ring->next_to_alloc = i;
1841
1842	/* Force memory writes to complete before letting h/w
1843	* know there are new descriptors to fetch. (Only
1844	* applicable for weak-ordered memory model archs,
1845	* such as IA-64).
1846	*/
1847	wmb();
1848	writel(val: i, addr: rx_ring->tail);
1849	}
1850	}
1851
1852	static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
1853	struct sk_buff *skb)
1854	{
1855	u16 hdr_len = skb_headlen(skb);
1856
1857	/* set gso_size to avoid messing up TCP MSS */
1858	skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
1859	IXGBE_CB(skb)->append_cnt);
1860	skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1861	}
1862
1863	static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
1864	struct sk_buff *skb)
1865	{
1866	/* if append_cnt is 0 then frame is not RSC */
1867	if (!IXGBE_CB(skb)->append_cnt)
1868	return;
1869
1870	rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
1871	rx_ring->rx_stats.rsc_flush++;
1872
1873	ixgbe_set_rsc_gso_size(ring: rx_ring, skb);
1874
1875	/* gso_size is computed using append_cnt so always clear it last */
1876	IXGBE_CB(skb)->append_cnt = 0;
1877	}
1878
1879	/**
1880	* ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor
1881	* @rx_ring: rx descriptor ring packet is being transacted on
1882	* @rx_desc: pointer to the EOP Rx descriptor
1883	* @skb: pointer to current skb being populated
1884	*
1885	* This function checks the ring, descriptor, and packet information in
1886	* order to populate the hash, checksum, VLAN, timestamp, protocol, and
1887	* other fields within the skb.
1888	**/
1889	void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
1890	union ixgbe_adv_rx_desc *rx_desc,
1891	struct sk_buff *skb)
1892	{
1893	struct net_device *dev = rx_ring->netdev;
1894	u32 flags = rx_ring->q_vector->adapter->flags;
1895
1896	ixgbe_update_rsc_stats(rx_ring, skb);
1897
1898	ixgbe_rx_hash(ring: rx_ring, rx_desc, skb);
1899
1900	ixgbe_rx_checksum(ring: rx_ring, rx_desc, skb);
1901
1902	if (unlikely(flags & IXGBE_FLAG_RX_HWTSTAMP_ENABLED))
1903	ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb);
1904
1905	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1906	ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
1907	u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1908	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci: vid);
1909	}
1910
1911	if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))
1912	ixgbe_ipsec_rx(rx_ring, rx_desc, skb);
1913
1914	/* record Rx queue, or update MACVLAN statistics */
1915	if (netif_is_ixgbe(dev))
1916	skb_record_rx_queue(skb, rx_queue: rx_ring->queue_index);
1917	else
1918	macvlan_count_rx(vlan: netdev_priv(dev), len: skb->len + ETH_HLEN, success: true,
1919	multicast: false);
1920
1921	skb->protocol = eth_type_trans(skb, dev);
1922	}
1923
1924	void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
1925	struct sk_buff *skb)
1926	{
1927	napi_gro_receive(napi: &q_vector->napi, skb);
1928	}
1929
1930	/**
1931	* ixgbe_is_non_eop - process handling of non-EOP buffers
1932	* @rx_ring: Rx ring being processed
1933	* @rx_desc: Rx descriptor for current buffer
1934	* @skb: Current socket buffer containing buffer in progress
1935	*
1936	* This function updates next to clean. If the buffer is an EOP buffer
1937	* this function exits returning false, otherwise it will place the
1938	* sk_buff in the next buffer to be chained and return true indicating
1939	* that this is in fact a non-EOP buffer.
1940	**/
1941	static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
1942	union ixgbe_adv_rx_desc *rx_desc,
1943	struct sk_buff *skb)
1944	{
1945	u32 ntc = rx_ring->next_to_clean + 1;
1946
1947	/* fetch, update, and store next to clean */
1948	ntc = (ntc < rx_ring->count) ? ntc : 0;
1949	rx_ring->next_to_clean = ntc;
1950
1951	prefetch(IXGBE_RX_DESC(rx_ring, ntc));
1952
1953	/* update RSC append count if present */
1954	if (ring_is_rsc_enabled(rx_ring)) {
1955	__le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
1956	cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);
1957
1958	if (unlikely(rsc_enabled)) {
1959	u32 rsc_cnt = le32_to_cpu(rsc_enabled);
1960
1961	rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
1962	IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;
1963
1964	/* update ntc based on RSC value */
1965	ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
1966	ntc &= IXGBE_RXDADV_NEXTP_MASK;
1967	ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
1968	}
1969	}
1970
1971	/* if we are the last buffer then there is nothing else to do */
1972	if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
1973	return false;
1974
1975	/* place skb in next buffer to be received */
1976	rx_ring->rx_buffer_info[ntc].skb = skb;
1977	rx_ring->rx_stats.non_eop_descs++;
1978
1979	return true;
1980	}
1981
1982	/**
1983	* ixgbe_pull_tail - ixgbe specific version of skb_pull_tail
1984	* @rx_ring: rx descriptor ring packet is being transacted on
1985	* @skb: pointer to current skb being adjusted
1986	*
1987	* This function is an ixgbe specific version of __pskb_pull_tail. The
1988	* main difference between this version and the original function is that
1989	* this function can make several assumptions about the state of things
1990	* that allow for significant optimizations versus the standard function.
1991	* As a result we can do things like drop a frag and maintain an accurate
1992	* truesize for the skb.
1993	*/
1994	static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
1995	struct sk_buff *skb)
1996	{
1997	skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
1998	unsigned char *va;
1999	unsigned int pull_len;
2000
2001	/*
2002	* it is valid to use page_address instead of kmap since we are
2003	* working with pages allocated out of the lomem pool per
2004	* alloc_page(GFP_ATOMIC)
2005	*/
2006	va = skb_frag_address(frag);
2007
2008	/*
2009	* we need the header to contain the greater of either ETH_HLEN or
2010	* 60 bytes if the skb->len is less than 60 for skb_pad.
2011	*/
2012	pull_len = eth_get_headlen(dev: skb->dev, data: va, IXGBE_RX_HDR_SIZE);
2013
2014	/* align pull length to size of long to optimize memcpy performance */
2015	skb_copy_to_linear_data(skb, from: va, ALIGN(pull_len, sizeof(long)));
2016
2017	/* update all of the pointers */
2018	skb_frag_size_sub(frag, delta: pull_len);
2019	skb_frag_off_add(frag, delta: pull_len);
2020	skb->data_len -= pull_len;
2021	skb->tail += pull_len;
2022	}
2023
2024	/**
2025	* ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB
2026	* @rx_ring: rx descriptor ring packet is being transacted on
2027	* @skb: pointer to current skb being updated
2028	*
2029	* This function provides a basic DMA sync up for the first fragment of an
2030	* skb. The reason for doing this is that the first fragment cannot be
2031	* unmapped until we have reached the end of packet descriptor for a buffer
2032	* chain.
2033	*/
2034	static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
2035	struct sk_buff *skb)
2036	{
2037	if (ring_uses_build_skb(rx_ring)) {
2038	unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1;
2039	unsigned long offset = (unsigned long)(skb->data) & mask;
2040
2041	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
2042	IXGBE_CB(skb)->dma,
2043	offset,
2044	size: skb_headlen(skb),
2045	dir: DMA_FROM_DEVICE);
2046	} else {
2047	skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
2048
2049	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
2050	IXGBE_CB(skb)->dma,
2051	offset: skb_frag_off(frag),
2052	size: skb_frag_size(frag),
2053	dir: DMA_FROM_DEVICE);
2054	}
2055
2056	/* If the page was released, just unmap it. */
2057	if (unlikely(IXGBE_CB(skb)->page_released)) {
2058	dma_unmap_page_attrs(dev: rx_ring->dev, IXGBE_CB(skb)->dma,
2059	ixgbe_rx_pg_size(rx_ring),
2060	dir: DMA_FROM_DEVICE,
2061	IXGBE_RX_DMA_ATTR);
2062	}
2063	}
2064
2065	/**
2066	* ixgbe_cleanup_headers - Correct corrupted or empty headers
2067	* @rx_ring: rx descriptor ring packet is being transacted on
2068	* @rx_desc: pointer to the EOP Rx descriptor
2069	* @skb: pointer to current skb being fixed
2070	*
2071	* Check if the skb is valid in the XDP case it will be an error pointer.
2072	* Return true in this case to abort processing and advance to next
2073	* descriptor.
2074	*
2075	* Check for corrupted packet headers caused by senders on the local L2
2076	* embedded NIC switch not setting up their Tx Descriptors right. These
2077	* should be very rare.
2078	*
2079	* Also address the case where we are pulling data in on pages only
2080	* and as such no data is present in the skb header.
2081	*
2082	* In addition if skb is not at least 60 bytes we need to pad it so that
2083	* it is large enough to qualify as a valid Ethernet frame.
2084	*
2085	* Returns true if an error was encountered and skb was freed.
2086	**/
2087	bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
2088	union ixgbe_adv_rx_desc *rx_desc,
2089	struct sk_buff *skb)
2090	{
2091	struct net_device *netdev = rx_ring->netdev;
2092
2093	/* Verify netdev is present, and that packet does not have any
2094	* errors that would be unacceptable to the netdev.
2095	*/
2096	if (!netdev ||
2097	(unlikely(ixgbe_test_staterr(rx_desc,
2098	IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
2099	!(netdev->features & NETIF_F_RXALL)))) {
2100	dev_kfree_skb_any(skb);
2101	return true;
2102	}
2103
2104	/* place header in linear portion of buffer */
2105	if (!skb_headlen(skb))
2106	ixgbe_pull_tail(rx_ring, skb);
2107
2108	#ifdef IXGBE_FCOE
2109	/* do not attempt to pad FCoE Frames as this will disrupt DDP */
2110	if (ixgbe_rx_is_fcoe(ring: rx_ring, rx_desc))
2111	return false;
2112
2113	#endif
2114	/* if eth_skb_pad returns an error the skb was freed */
2115	if (eth_skb_pad(skb))
2116	return true;
2117
2118	return false;
2119	}
2120
2121	/**
2122	* ixgbe_reuse_rx_page - page flip buffer and store it back on the ring
2123	* @rx_ring: rx descriptor ring to store buffers on
2124	* @old_buff: donor buffer to have page reused
2125	*
2126	* Synchronizes page for reuse by the adapter
2127	**/
2128	static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
2129	struct ixgbe_rx_buffer *old_buff)
2130	{
2131	struct ixgbe_rx_buffer *new_buff;
2132	u16 nta = rx_ring->next_to_alloc;
2133
2134	new_buff = &rx_ring->rx_buffer_info[nta];
2135
2136	/* update, and store next to alloc */
2137	nta++;
2138	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
2139
2140	/* Transfer page from old buffer to new buffer.
2141	* Move each member individually to avoid possible store
2142	* forwarding stalls and unnecessary copy of skb.
2143	*/
2144	new_buff->dma = old_buff->dma;
2145	new_buff->page = old_buff->page;
2146	new_buff->page_offset = old_buff->page_offset;
2147	new_buff->pagecnt_bias = old_buff->pagecnt_bias;
2148	}
2149
2150	static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer,
2151	int rx_buffer_pgcnt)
2152	{
2153	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
2154	struct page *page = rx_buffer->page;
2155
2156	/* avoid re-using remote and pfmemalloc pages */
2157	if (!dev_page_is_reusable(page))
2158	return false;
2159
2160	#if (PAGE_SIZE < 8192)
2161	/* if we are only owner of page we can reuse it */
2162	if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))
2163	return false;
2164	#else
2165	/* The last offset is a bit aggressive in that we assume the
2166	* worst case of FCoE being enabled and using a 3K buffer.
2167	* However this should have minimal impact as the 1K extra is
2168	* still less than one buffer in size.
2169	*/
2170	#define IXGBE_LAST_OFFSET \
2171	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K)
2172	if (rx_buffer->page_offset > IXGBE_LAST_OFFSET)
2173	return false;
2174	#endif
2175
2176	/* If we have drained the page fragment pool we need to update
2177	* the pagecnt_bias and page count so that we fully restock the
2178	* number of references the driver holds.
2179	*/
2180	if (unlikely(pagecnt_bias == 1)) {
2181	page_ref_add(page, USHRT_MAX - 1);
2182	rx_buffer->pagecnt_bias = USHRT_MAX;
2183	}
2184
2185	return true;
2186	}
2187
2188	/**
2189	* ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff
2190	* @rx_ring: rx descriptor ring to transact packets on
2191	* @rx_buffer: buffer containing page to add
2192	* @skb: sk_buff to place the data into
2193	* @size: size of data in rx_buffer
2194	*
2195	* This function will add the data contained in rx_buffer->page to the skb.
2196	* This is done either through a direct copy if the data in the buffer is
2197	* less than the skb header size, otherwise it will just attach the page as
2198	* a frag to the skb.
2199	*
2200	* The function will then update the page offset if necessary and return
2201	* true if the buffer can be reused by the adapter.
2202	**/
2203	static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
2204	struct ixgbe_rx_buffer *rx_buffer,
2205	struct sk_buff *skb,
2206	unsigned int size)
2207	{
2208	#if (PAGE_SIZE < 8192)
2209	unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
2210	#else
2211	unsigned int truesize = rx_ring->rx_offset ?
2212	SKB_DATA_ALIGN(rx_ring->rx_offset + size) :
2213	SKB_DATA_ALIGN(size);
2214	#endif
2215	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page: rx_buffer->page,
2216	off: rx_buffer->page_offset, size, truesize);
2217	#if (PAGE_SIZE < 8192)
2218	rx_buffer->page_offset ^= truesize;
2219	#else
2220	rx_buffer->page_offset += truesize;
2221	#endif
2222	}
2223
2224	static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring,
2225	union ixgbe_adv_rx_desc *rx_desc,
2226	struct sk_buff **skb,
2227	const unsigned int size,
2228	int *rx_buffer_pgcnt)
2229	{
2230	struct ixgbe_rx_buffer *rx_buffer;
2231
2232	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
2233	*rx_buffer_pgcnt =
2234	#if (PAGE_SIZE < 8192)
2235	page_count(page: rx_buffer->page);
2236	#else
2237	0;
2238	#endif
2239	prefetchw(x: rx_buffer->page);
2240	*skb = rx_buffer->skb;
2241
2242	/* Delay unmapping of the first packet. It carries the header
2243	* information, HW may still access the header after the writeback.
2244	* Only unmap it when EOP is reached
2245	*/
2246	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) {
2247	if (!*skb)
2248	goto skip_sync;
2249	} else {
2250	if (*skb)
2251	ixgbe_dma_sync_frag(rx_ring, skb: *skb);
2252	}
2253
2254	/* we are reusing so sync this buffer for CPU use */
2255	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
2256	addr: rx_buffer->dma,
2257	offset: rx_buffer->page_offset,
2258	size,
2259	dir: DMA_FROM_DEVICE);
2260	skip_sync:
2261	rx_buffer->pagecnt_bias--;
2262
2263	return rx_buffer;
2264	}
2265
2266	static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring,
2267	struct ixgbe_rx_buffer *rx_buffer,
2268	struct sk_buff *skb,
2269	int rx_buffer_pgcnt)
2270	{
2271	if (ixgbe_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {
2272	/* hand second half of page back to the ring */
2273	ixgbe_reuse_rx_page(rx_ring, old_buff: rx_buffer);
2274	} else {
2275	if (skb && IXGBE_CB(skb)->dma == rx_buffer->dma) {
2276	/* the page has been released from the ring */
2277	IXGBE_CB(skb)->page_released = true;
2278	} else {
2279	/* we are not reusing the buffer so unmap it */
2280	dma_unmap_page_attrs(dev: rx_ring->dev, addr: rx_buffer->dma,
2281	ixgbe_rx_pg_size(rx_ring),
2282	dir: DMA_FROM_DEVICE,
2283	IXGBE_RX_DMA_ATTR);
2284	}
2285	__page_frag_cache_drain(page: rx_buffer->page,
2286	count: rx_buffer->pagecnt_bias);
2287	}
2288
2289	/* clear contents of rx_buffer */
2290	rx_buffer->page = NULL;
2291	rx_buffer->skb = NULL;
2292	}
2293
2294	static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring,
2295	struct ixgbe_rx_buffer *rx_buffer,
2296	struct xdp_buff *xdp,
2297	union ixgbe_adv_rx_desc *rx_desc)
2298	{
2299	unsigned int size = xdp->data_end - xdp->data;
2300	#if (PAGE_SIZE < 8192)
2301	unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
2302	#else
2303	unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
2304	xdp->data_hard_start);
2305	#endif
2306	struct sk_buff *skb;
2307
2308	/* prefetch first cache line of first page */
2309	net_prefetch(p: xdp->data);
2310
2311	/* Note, we get here by enabling legacy-rx via:
2312	*
2313	* ethtool --set-priv-flags <dev> legacy-rx on
2314	*
2315	* In this mode, we currently get 0 extra XDP headroom as
2316	* opposed to having legacy-rx off, where we process XDP
2317	* packets going to stack via ixgbe_build_skb(). The latter
2318	* provides us currently with 192 bytes of headroom.
2319	*
2320	* For ixgbe_construct_skb() mode it means that the
2321	* xdp->data_meta will always point to xdp->data, since
2322	* the helper cannot expand the head. Should this ever
2323	* change in future for legacy-rx mode on, then lets also
2324	* add xdp->data_meta handling here.
2325	*/
2326
2327	/* allocate a skb to store the frags */
2328	skb = napi_alloc_skb(napi: &rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE);
2329	if (unlikely(!skb))
2330	return NULL;
2331
2332	if (size > IXGBE_RX_HDR_SIZE) {
2333	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
2334	IXGBE_CB(skb)->dma = rx_buffer->dma;
2335
2336	skb_add_rx_frag(skb, i: 0, page: rx_buffer->page,
2337	off: xdp->data - page_address(rx_buffer->page),
2338	size, truesize);
2339	#if (PAGE_SIZE < 8192)
2340	rx_buffer->page_offset ^= truesize;
2341	#else
2342	rx_buffer->page_offset += truesize;
2343	#endif
2344	} else {
2345	memcpy(__skb_put(skb, size),
2346	xdp->data, ALIGN(size, sizeof(long)));
2347	rx_buffer->pagecnt_bias++;
2348	}
2349
2350	return skb;
2351	}
2352
2353	static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring,
2354	struct ixgbe_rx_buffer *rx_buffer,
2355	struct xdp_buff *xdp,
2356	union ixgbe_adv_rx_desc *rx_desc)
2357	{
2358	unsigned int metasize = xdp->data - xdp->data_meta;
2359	#if (PAGE_SIZE < 8192)
2360	unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;
2361	#else
2362	unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
2363	SKB_DATA_ALIGN(xdp->data_end -
2364	xdp->data_hard_start);
2365	#endif
2366	struct sk_buff *skb;
2367
2368	/* Prefetch first cache line of first page. If xdp->data_meta
2369	* is unused, this points exactly as xdp->data, otherwise we
2370	* likely have a consumer accessing first few bytes of meta
2371	* data, and then actual data.
2372	*/
2373	net_prefetch(p: xdp->data_meta);
2374
2375	/* build an skb to around the page buffer */
2376	skb = napi_build_skb(data: xdp->data_hard_start, frag_size: truesize);
2377	if (unlikely(!skb))
2378	return NULL;
2379
2380	/* update pointers within the skb to store the data */
2381	skb_reserve(skb, len: xdp->data - xdp->data_hard_start);
2382	__skb_put(skb, len: xdp->data_end - xdp->data);
2383	if (metasize)
2384	skb_metadata_set(skb, meta_len: metasize);
2385
2386	/* record DMA address if this is the start of a chain of buffers */
2387	if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
2388	IXGBE_CB(skb)->dma = rx_buffer->dma;
2389
2390	/* update buffer offset */
2391	#if (PAGE_SIZE < 8192)
2392	rx_buffer->page_offset ^= truesize;
2393	#else
2394	rx_buffer->page_offset += truesize;
2395	#endif
2396
2397	return skb;
2398	}
2399
2400	static int ixgbe_run_xdp(struct ixgbe_adapter *adapter,
2401	struct ixgbe_ring *rx_ring,
2402	struct xdp_buff *xdp)
2403	{
2404	int err, result = IXGBE_XDP_PASS;
2405	struct bpf_prog *xdp_prog;
2406	struct ixgbe_ring *ring;
2407	struct xdp_frame *xdpf;
2408	u32 act;
2409
2410	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
2411
2412	if (!xdp_prog)
2413	goto xdp_out;
2414
2415	prefetchw(x: xdp->data_hard_start); /* xdp_frame write */
2416
2417	act = bpf_prog_run_xdp(prog: xdp_prog, xdp);
2418	switch (act) {
2419	case XDP_PASS:
2420	break;
2421	case XDP_TX:
2422	xdpf = xdp_convert_buff_to_frame(xdp);
2423	if (unlikely(!xdpf))
2424	goto out_failure;
2425	ring = ixgbe_determine_xdp_ring(adapter);
2426	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
2427	spin_lock(lock: &ring->tx_lock);
2428	result = ixgbe_xmit_xdp_ring(ring, xdpf);
2429	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
2430	spin_unlock(lock: &ring->tx_lock);
2431	if (result == IXGBE_XDP_CONSUMED)
2432	goto out_failure;
2433	break;
2434	case XDP_REDIRECT:
2435	err = xdp_do_redirect(dev: adapter->netdev, xdp, prog: xdp_prog);
2436	if (err)
2437	goto out_failure;
2438	result = IXGBE_XDP_REDIR;
2439	break;
2440	default:
2441	bpf_warn_invalid_xdp_action(dev: rx_ring->netdev, prog: xdp_prog, act);
2442	fallthrough;
2443	case XDP_ABORTED:
2444	out_failure:
2445	trace_xdp_exception(dev: rx_ring->netdev, xdp: xdp_prog, act);
2446	fallthrough; /* handle aborts by dropping packet */
2447	case XDP_DROP:
2448	result = IXGBE_XDP_CONSUMED;
2449	break;
2450	}
2451	xdp_out:
2452	return result;
2453	}
2454
2455	static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring,
2456	unsigned int size)
2457	{
2458	unsigned int truesize;
2459
2460	#if (PAGE_SIZE < 8192)
2461	truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
2462	#else
2463	truesize = rx_ring->rx_offset ?
2464	SKB_DATA_ALIGN(rx_ring->rx_offset + size) +
2465	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
2466	SKB_DATA_ALIGN(size);
2467	#endif
2468	return truesize;
2469	}
2470
2471	static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring,
2472	struct ixgbe_rx_buffer *rx_buffer,
2473	unsigned int size)
2474	{
2475	unsigned int truesize = ixgbe_rx_frame_truesize(rx_ring, size);
2476	#if (PAGE_SIZE < 8192)
2477	rx_buffer->page_offset ^= truesize;
2478	#else
2479	rx_buffer->page_offset += truesize;
2480	#endif
2481	}
2482
2483	/**
2484	* ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
2485	* @q_vector: structure containing interrupt and ring information
2486	* @rx_ring: rx descriptor ring to transact packets on
2487	* @budget: Total limit on number of packets to process
2488	*
2489	* This function provides a "bounce buffer" approach to Rx interrupt
2490	* processing. The advantage to this is that on systems that have
2491	* expensive overhead for IOMMU access this provides a means of avoiding
2492	* it by maintaining the mapping of the page to the system.
2493	*
2494	* Returns amount of work completed
2495	**/
2496	static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
2497	struct ixgbe_ring *rx_ring,
2498	const int budget)
2499	{
2500	unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0;
2501	struct ixgbe_adapter *adapter = q_vector->adapter;
2502	#ifdef IXGBE_FCOE
2503	int ddp_bytes;
2504	unsigned int mss = 0;
2505	#endif /* IXGBE_FCOE */
2506	u16 cleaned_count = ixgbe_desc_unused(ring: rx_ring);
2507	unsigned int offset = rx_ring->rx_offset;
2508	unsigned int xdp_xmit = 0;
2509	struct xdp_buff xdp;
2510	int xdp_res = 0;
2511
2512	/* Frame size depend on rx_ring setup when PAGE_SIZE=4K */
2513	#if (PAGE_SIZE < 8192)
2514	frame_sz = ixgbe_rx_frame_truesize(rx_ring, size: 0);
2515	#endif
2516	xdp_init_buff(xdp: &xdp, frame_sz, rxq: &rx_ring->xdp_rxq);
2517
2518	while (likely(total_rx_packets < budget)) {
2519	union ixgbe_adv_rx_desc *rx_desc;
2520	struct ixgbe_rx_buffer *rx_buffer;
2521	struct sk_buff *skb;
2522	int rx_buffer_pgcnt;
2523	unsigned int size;
2524
2525	/* return some buffers to hardware, one at a time is too slow */
2526	if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
2527	ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
2528	cleaned_count = 0;
2529	}
2530
2531	rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
2532	size = le16_to_cpu(rx_desc->wb.upper.length);
2533	if (!size)
2534	break;
2535
2536	/* This memory barrier is needed to keep us from reading
2537	* any other fields out of the rx_desc until we know the
2538	* descriptor has been written back
2539	*/
2540	dma_rmb();
2541
2542	rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, skb: &skb, size, rx_buffer_pgcnt: &rx_buffer_pgcnt);
2543
2544	/* retrieve a buffer from the ring */
2545	if (!skb) {
2546	unsigned char *hard_start;
2547
2548	hard_start = page_address(rx_buffer->page) +
2549	rx_buffer->page_offset - offset;
2550	xdp_prepare_buff(xdp: &xdp, hard_start, headroom: offset, data_len: size, meta_valid: true);
2551	xdp_buff_clear_frags_flag(xdp: &xdp);
2552	#if (PAGE_SIZE > 4096)
2553	/* At larger PAGE_SIZE, frame_sz depend on len size */
2554	xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size);
2555	#endif
2556	xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp: &xdp);
2557	}
2558
2559	if (xdp_res) {
2560	if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) {
2561	xdp_xmit |= xdp_res;
2562	ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size);
2563	} else {
2564	rx_buffer->pagecnt_bias++;
2565	}
2566	total_rx_packets++;
2567	total_rx_bytes += size;
2568	} else if (skb) {
2569	ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size);
2570	} else if (ring_uses_build_skb(rx_ring)) {
2571	skb = ixgbe_build_skb(rx_ring, rx_buffer,
2572	xdp: &xdp, rx_desc);
2573	} else {
2574	skb = ixgbe_construct_skb(rx_ring, rx_buffer,
2575	xdp: &xdp, rx_desc);
2576	}
2577
2578	/* exit if we failed to retrieve a buffer */
2579	if (!xdp_res && !skb) {
2580	rx_ring->rx_stats.alloc_rx_buff_failed++;
2581	rx_buffer->pagecnt_bias++;
2582	break;
2583	}
2584
2585	ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);
2586	cleaned_count++;
2587
2588	/* place incomplete frames back on ring for completion */
2589	if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
2590	continue;
2591
2592	/* verify the packet layout is correct */
2593	if (xdp_res || ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
2594	continue;
2595
2596	/* probably a little skewed due to removing CRC */
2597	total_rx_bytes += skb->len;
2598
2599	/* populate checksum, timestamp, VLAN, and protocol */
2600	ixgbe_process_skb_fields(rx_ring, rx_desc, skb);
2601
2602	#ifdef IXGBE_FCOE
2603	/* if ddp, not passing to ULD unless for FCP_RSP or error */
2604	if (ixgbe_rx_is_fcoe(ring: rx_ring, rx_desc)) {
2605	ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
2606	/* include DDPed FCoE data */
2607	if (ddp_bytes > 0) {
2608	if (!mss) {
2609	mss = rx_ring->netdev->mtu -
2610	sizeof(struct fcoe_hdr) -
2611	sizeof(struct fc_frame_header) -
2612	sizeof(struct fcoe_crc_eof);
2613	if (mss > 512)
2614	mss &= ~511;
2615	}
2616	total_rx_bytes += ddp_bytes;
2617	total_rx_packets += DIV_ROUND_UP(ddp_bytes,
2618	mss);
2619	}
2620	if (!ddp_bytes) {
2621	dev_kfree_skb_any(skb);
2622	continue;
2623	}
2624	}
2625
2626	#endif /* IXGBE_FCOE */
2627	ixgbe_rx_skb(q_vector, skb);
2628
2629	/* update budget accounting */
2630	total_rx_packets++;
2631	}
2632
2633	if (xdp_xmit & IXGBE_XDP_REDIR)
2634	xdp_do_flush();
2635
2636	if (xdp_xmit & IXGBE_XDP_TX) {
2637	struct ixgbe_ring *ring = ixgbe_determine_xdp_ring(adapter);
2638
2639	ixgbe_xdp_ring_update_tail_locked(ring);
2640	}
2641
2642	ixgbe_update_rx_ring_stats(rx_ring, q_vector, pkts: total_rx_packets,
2643	bytes: total_rx_bytes);
2644
2645	return total_rx_packets;
2646	}
2647
2648	/**
2649	* ixgbe_configure_msix - Configure MSI-X hardware
2650	* @adapter: board private structure
2651	*
2652	* ixgbe_configure_msix sets up the hardware to properly generate MSI-X
2653	* interrupts.
2654	**/
2655	static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
2656	{
2657	struct ixgbe_q_vector *q_vector;
2658	int v_idx;
2659	u32 mask;
2660
2661	/* Populate MSIX to EITR Select */
2662	if (adapter->num_vfs > 32) {
2663	u32 eitrsel = BIT(adapter->num_vfs - 32) - 1;
2664	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
2665	}
2666
2667	/*
2668	* Populate the IVAR table and set the ITR values to the
2669	* corresponding register.
2670	*/
2671	for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
2672	struct ixgbe_ring *ring;
2673	q_vector = adapter->q_vector[v_idx];
2674
2675	ixgbe_for_each_ring(ring, q_vector->rx)
2676	ixgbe_set_ivar(adapter, direction: 0, queue: ring->reg_idx, msix_vector: v_idx);
2677
2678	ixgbe_for_each_ring(ring, q_vector->tx)
2679	ixgbe_set_ivar(adapter, direction: 1, queue: ring->reg_idx, msix_vector: v_idx);
2680
2681	ixgbe_write_eitr(q_vector);
2682	}
2683
2684	switch (adapter->hw.mac.type) {
2685	case ixgbe_mac_82598EB:
2686	ixgbe_set_ivar(adapter, direction: -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
2687	msix_vector: v_idx);
2688	break;
2689	case ixgbe_mac_82599EB:
2690	case ixgbe_mac_X540:
2691	case ixgbe_mac_X550:
2692	case ixgbe_mac_X550EM_x:
2693	case ixgbe_mac_x550em_a:
2694	case ixgbe_mac_e610:
2695	ixgbe_set_ivar(adapter, direction: -1, queue: 1, msix_vector: v_idx);
2696	break;
2697	default:
2698	break;
2699	}
2700	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
2701
2702	/* set up to autoclear timer, and the vectors */
2703	mask = IXGBE_EIMS_ENABLE_MASK;
2704	mask &= ~(IXGBE_EIMS_OTHER |
2705	IXGBE_EIMS_MAILBOX |
2706	IXGBE_EIMS_LSC);
2707
2708	if (adapter->hw.mac.type == ixgbe_mac_e610)
2709	mask &= ~IXGBE_EIMS_FW_EVENT;
2710
2711	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
2712	}
2713
2714	/**
2715	* ixgbe_update_itr - update the dynamic ITR value based on statistics
2716	* @q_vector: structure containing interrupt and ring information
2717	* @ring_container: structure containing ring performance data
2718	*
2719	* Stores a new ITR value based on packets and byte
2720	* counts during the last interrupt. The advantage of per interrupt
2721	* computation is faster updates and more accurate ITR for the current
2722	* traffic pattern. Constants in this function were computed
2723	* based on theoretical maximum wire speed and thresholds were set based
2724	* on testing data as well as attempting to minimize response time
2725	* while increasing bulk throughput.
2726	**/
2727	static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
2728	struct ixgbe_ring_container *ring_container)
2729	{
2730	unsigned int itr = IXGBE_ITR_ADAPTIVE_MIN_USECS |
2731	IXGBE_ITR_ADAPTIVE_LATENCY;
2732	unsigned int avg_wire_size, packets, bytes;
2733	unsigned long next_update = jiffies;
2734
2735	/* If we don't have any rings just leave ourselves set for maximum
2736	* possible latency so we take ourselves out of the equation.
2737	*/
2738	if (!ring_container->ring)
2739	return;
2740
2741	/* If we didn't update within up to 1 - 2 jiffies we can assume
2742	* that either packets are coming in so slow there hasn't been
2743	* any work, or that there is so much work that NAPI is dealing
2744	* with interrupt moderation and we don't need to do anything.
2745	*/
2746	if (time_after(next_update, ring_container->next_update))
2747	goto clear_counts;
2748
2749	packets = ring_container->total_packets;
2750
2751	/* We have no packets to actually measure against. This means
2752	* either one of the other queues on this vector is active or
2753	* we are a Tx queue doing TSO with too high of an interrupt rate.
2754	*
2755	* When this occurs just tick up our delay by the minimum value
2756	* and hope that this extra delay will prevent us from being called
2757	* without any work on our queue.
2758	*/
2759	if (!packets) {
2760	itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;
2761	if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)
2762	itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;
2763	itr += ring_container->itr & IXGBE_ITR_ADAPTIVE_LATENCY;
2764	goto clear_counts;
2765	}
2766
2767	bytes = ring_container->total_bytes;
2768
2769	/* If packets are less than 4 or bytes are less than 9000 assume
2770	* insufficient data to use bulk rate limiting approach. We are
2771	* likely latency driven.
2772	*/
2773	if (packets < 4 && bytes < 9000) {
2774	itr = IXGBE_ITR_ADAPTIVE_LATENCY;
2775	goto adjust_by_size;
2776	}
2777
2778	/* Between 4 and 48 we can assume that our current interrupt delay
2779	* is only slightly too low. As such we should increase it by a small
2780	* fixed amount.
2781	*/
2782	if (packets < 48) {
2783	itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;
2784	if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)
2785	itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;
2786	goto clear_counts;
2787	}
2788
2789	/* Between 48 and 96 is our "goldilocks" zone where we are working
2790	* out "just right". Just report that our current ITR is good for us.
2791	*/
2792	if (packets < 96) {
2793	itr = q_vector->itr >> 2;
2794	goto clear_counts;
2795	}
2796
2797	/* If packet count is 96 or greater we are likely looking at a slight
2798	* overrun of the delay we want. Try halving our delay to see if that
2799	* will cut the number of packets in half per interrupt.
2800	*/
2801	if (packets < 256) {
2802	itr = q_vector->itr >> 3;
2803	if (itr < IXGBE_ITR_ADAPTIVE_MIN_USECS)
2804	itr = IXGBE_ITR_ADAPTIVE_MIN_USECS;
2805	goto clear_counts;
2806	}
2807
2808	/* The paths below assume we are dealing with a bulk ITR since number
2809	* of packets is 256 or greater. We are just going to have to compute
2810	* a value and try to bring the count under control, though for smaller
2811	* packet sizes there isn't much we can do as NAPI polling will likely
2812	* be kicking in sooner rather than later.
2813	*/
2814	itr = IXGBE_ITR_ADAPTIVE_BULK;
2815
2816	adjust_by_size:
2817	/* If packet counts are 256 or greater we can assume we have a gross
2818	* overestimation of what the rate should be. Instead of trying to fine
2819	* tune it just use the formula below to try and dial in an exact value
2820	* give the current packet size of the frame.
2821	*/
2822	avg_wire_size = bytes / packets;
2823
2824	/* The following is a crude approximation of:
2825	* wmem_default / (size + overhead) = desired_pkts_per_int
2826	* rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
2827	* (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
2828	*
2829	* Assuming wmem_default is 212992 and overhead is 640 bytes per
2830	* packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
2831	* formula down to
2832	*
2833	* (170 * (size + 24)) / (size + 640) = ITR
2834	*
2835	* We first do some math on the packet size and then finally bitshift
2836	* by 8 after rounding up. We also have to account for PCIe link speed
2837	* difference as ITR scales based on this.
2838	*/
2839	if (avg_wire_size <= 60) {
2840	/* Start at 50k ints/sec */
2841	avg_wire_size = 5120;
2842	} else if (avg_wire_size <= 316) {
2843	/* 50K ints/sec to 16K ints/sec */
2844	avg_wire_size *= 40;
2845	avg_wire_size += 2720;
2846	} else if (avg_wire_size <= 1084) {
2847	/* 16K ints/sec to 9.2K ints/sec */
2848	avg_wire_size *= 15;
2849	avg_wire_size += 11452;
2850	} else if (avg_wire_size < 1968) {
2851	/* 9.2K ints/sec to 8K ints/sec */
2852	avg_wire_size *= 5;
2853	avg_wire_size += 22420;
2854	} else {
2855	/* plateau at a limit of 8K ints/sec */
2856	avg_wire_size = 32256;
2857	}
2858
2859	/* If we are in low latency mode half our delay which doubles the rate
2860	* to somewhere between 100K to 16K ints/sec
2861	*/
2862	if (itr & IXGBE_ITR_ADAPTIVE_LATENCY)
2863	avg_wire_size >>= 1;
2864
2865	/* Resultant value is 256 times larger than it needs to be. This
2866	* gives us room to adjust the value as needed to either increase
2867	* or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.
2868	*
2869	* Use addition as we have already recorded the new latency flag
2870	* for the ITR value.
2871	*/
2872	switch (q_vector->adapter->link_speed) {
2873	case IXGBE_LINK_SPEED_10GB_FULL:
2874	case IXGBE_LINK_SPEED_100_FULL:
2875	default:
2876	itr += DIV_ROUND_UP(avg_wire_size,
2877	IXGBE_ITR_ADAPTIVE_MIN_INC * 256) *
2878	IXGBE_ITR_ADAPTIVE_MIN_INC;
2879	break;
2880	case IXGBE_LINK_SPEED_2_5GB_FULL:
2881	case IXGBE_LINK_SPEED_1GB_FULL:
2882	case IXGBE_LINK_SPEED_10_FULL:
2883	if (avg_wire_size > 8064)
2884	avg_wire_size = 8064;
2885	itr += DIV_ROUND_UP(avg_wire_size,
2886	IXGBE_ITR_ADAPTIVE_MIN_INC * 64) *
2887	IXGBE_ITR_ADAPTIVE_MIN_INC;
2888	break;
2889	}
2890
2891	clear_counts:
2892	/* write back value */
2893	ring_container->itr = itr;
2894
2895	/* next update should occur within next jiffy */
2896	ring_container->next_update = next_update + 1;
2897
2898	ring_container->total_bytes = 0;
2899	ring_container->total_packets = 0;
2900	}
2901
2902	/**
2903	* ixgbe_write_eitr - write EITR register in hardware specific way
2904	* @q_vector: structure containing interrupt and ring information
2905	*
2906	* This function is made to be called by ethtool and by the driver
2907	* when it needs to update EITR registers at runtime. Hardware
2908	* specific quirks/differences are taken care of here.
2909	*/
2910	void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
2911	{
2912	struct ixgbe_adapter *adapter = q_vector->adapter;
2913	struct ixgbe_hw *hw = &adapter->hw;
2914	int v_idx = q_vector->v_idx;
2915	u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
2916
2917	switch (adapter->hw.mac.type) {
2918	case ixgbe_mac_82598EB:
2919	/* must write high and low 16 bits to reset counter */
2920	itr_reg |= (itr_reg << 16);
2921	break;
2922	case ixgbe_mac_82599EB:
2923	case ixgbe_mac_X540:
2924	case ixgbe_mac_X550:
2925	case ixgbe_mac_X550EM_x:
2926	case ixgbe_mac_x550em_a:
2927	case ixgbe_mac_e610:
2928	/*
2929	* set the WDIS bit to not clear the timer bits and cause an
2930	* immediate assertion of the interrupt
2931	*/
2932	itr_reg |= IXGBE_EITR_CNT_WDIS;
2933	break;
2934	default:
2935	break;
2936	}
2937	IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
2938	}
2939
2940	static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
2941	{
2942	u32 new_itr;
2943
2944	ixgbe_update_itr(q_vector, ring_container: &q_vector->tx);
2945	ixgbe_update_itr(q_vector, ring_container: &q_vector->rx);
2946
2947	/* use the smallest value of new ITR delay calculations */
2948	new_itr = min(q_vector->rx.itr, q_vector->tx.itr);
2949
2950	/* Clear latency flag if set, shift into correct position */
2951	new_itr &= ~IXGBE_ITR_ADAPTIVE_LATENCY;
2952	new_itr <<= 2;
2953
2954	if (new_itr != q_vector->itr) {
2955	/* save the algorithm value here */
2956	q_vector->itr = new_itr;
2957
2958	ixgbe_write_eitr(q_vector);
2959	}
2960	}
2961
2962	/**
2963	* ixgbe_check_overtemp_subtask - check for over temperature
2964	* @adapter: pointer to adapter
2965	**/
2966	static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
2967	{
2968	struct ixgbe_hw *hw = &adapter->hw;
2969	u32 eicr = adapter->interrupt_event;
2970
2971	if (test_bit(__IXGBE_DOWN, &adapter->state))
2972	return;
2973
2974	if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
2975	return;
2976
2977	adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2978
2979	switch (hw->device_id) {
2980	case IXGBE_DEV_ID_82599_T3_LOM:
2981	/*
2982	* Since the warning interrupt is for both ports
2983	* we don't have to check if:
2984	* - This interrupt wasn't for our port.
2985	* - We may have missed the interrupt so always have to
2986	* check if we got a LSC
2987	*/
2988	if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) &&
2989	!(eicr & IXGBE_EICR_LSC))
2990	return;
2991
2992	if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
2993	u32 speed;
2994	bool link_up = false;
2995
2996	hw->mac.ops.check_link(hw, &speed, &link_up, false);
2997
2998	if (link_up)
2999	return;
3000	}
3001
3002	/* Check if this is not due to overtemp */
3003	if (!hw->phy.ops.check_overtemp(hw))
3004	return;
3005
3006	break;
3007	case IXGBE_DEV_ID_X550EM_A_1G_T:
3008	case IXGBE_DEV_ID_X550EM_A_1G_T_L:
3009	if (!hw->phy.ops.check_overtemp(hw))
3010	return;
3011	break;
3012	default:
3013	if (adapter->hw.mac.type >= ixgbe_mac_X540)
3014	return;
3015	if (!(eicr & IXGBE_EICR_GPI_SDP0(hw)))
3016	return;
3017	break;
3018	}
3019	e_crit(drv, "%s\n", ixgbe_overheat_msg);
3020
3021	adapter->interrupt_event = 0;
3022	}
3023
3024	static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
3025	{
3026	struct ixgbe_hw *hw = &adapter->hw;
3027
3028	if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
3029	(eicr & IXGBE_EICR_GPI_SDP1(hw))) {
3030	e_crit(probe, "Fan has stopped, replace the adapter\n");
3031	/* write to clear the interrupt */
3032	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
3033	}
3034	}
3035
3036	static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
3037	{
3038	struct ixgbe_hw *hw = &adapter->hw;
3039
3040	if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
3041	return;
3042
3043	switch (adapter->hw.mac.type) {
3044	case ixgbe_mac_82599EB:
3045	/*
3046	* Need to check link state so complete overtemp check
3047	* on service task
3048	*/
3049	if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) ||
3050	(eicr & IXGBE_EICR_LSC)) &&
3051	(!test_bit(__IXGBE_DOWN, &adapter->state))) {
3052	adapter->interrupt_event = eicr;
3053	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
3054	ixgbe_service_event_schedule(adapter);
3055	return;
3056	}
3057	return;
3058	case ixgbe_mac_x550em_a:
3059	if (eicr & IXGBE_EICR_GPI_SDP0_X550EM_a) {
3060	adapter->interrupt_event = eicr;
3061	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
3062	ixgbe_service_event_schedule(adapter);
3063	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC,
3064	IXGBE_EICR_GPI_SDP0_X550EM_a);
3065	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICR,
3066	IXGBE_EICR_GPI_SDP0_X550EM_a);
3067	}
3068	return;
3069	case ixgbe_mac_X550:
3070	case ixgbe_mac_X540:
3071	if (!(eicr & IXGBE_EICR_TS))
3072	return;
3073	break;
3074	default:
3075	return;
3076	}
3077
3078	e_crit(drv, "%s\n", ixgbe_overheat_msg);
3079	}
3080
3081	static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
3082	{
3083	switch (hw->mac.type) {
3084	case ixgbe_mac_82598EB:
3085	if (hw->phy.type == ixgbe_phy_nl)
3086	return true;
3087	return false;
3088	case ixgbe_mac_82599EB:
3089	case ixgbe_mac_X550EM_x:
3090	case ixgbe_mac_x550em_a:
3091	switch (hw->mac.ops.get_media_type(hw)) {
3092	case ixgbe_media_type_fiber:
3093	case ixgbe_media_type_fiber_qsfp:
3094	return true;
3095	default:
3096	return false;
3097	}
3098	default:
3099	return false;
3100	}
3101	}
3102
3103	static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
3104	{
3105	struct ixgbe_hw *hw = &adapter->hw;
3106	u32 eicr_mask = IXGBE_EICR_GPI_SDP2(hw);
3107
3108	if (!ixgbe_is_sfp(hw))
3109	return;
3110
3111	/* Later MAC's use different SDP */
3112	if (hw->mac.type >= ixgbe_mac_X540)
3113	eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
3114
3115	if (eicr & eicr_mask) {
3116	/* Clear the interrupt */
3117	IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);
3118	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
3119	adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
3120	adapter->sfp_poll_time = 0;
3121	ixgbe_service_event_schedule(adapter);
3122	}
3123	}
3124
3125	if (adapter->hw.mac.type == ixgbe_mac_82599EB &&
3126	(eicr & IXGBE_EICR_GPI_SDP1(hw))) {
3127	/* Clear the interrupt */
3128	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));
3129	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
3130	adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
3131	ixgbe_service_event_schedule(adapter);
3132	}
3133	}
3134	}
3135
3136	static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
3137	{
3138	struct ixgbe_hw *hw = &adapter->hw;
3139
3140	adapter->lsc_int++;
3141	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
3142	adapter->link_check_timeout = jiffies;
3143	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
3144	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
3145	IXGBE_WRITE_FLUSH(hw);
3146	ixgbe_service_event_schedule(adapter);
3147	}
3148	}
3149
3150	/**
3151	* ixgbe_check_phy_fw_load - check if PHY FW load failed
3152	* @adapter: pointer to adapter structure
3153	* @link_cfg_err: bitmap from the link info structure
3154	*
3155	* Check if external PHY FW load failed and print an error message if it did.
3156	*/
3157	static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter,
3158	u8 link_cfg_err)
3159	{
3160	if (!(link_cfg_err & IXGBE_ACI_LINK_EXTERNAL_PHY_LOAD_FAILURE)) {
3161	adapter->flags2 &= ~IXGBE_FLAG2_PHY_FW_LOAD_FAILED;
3162	return;
3163	}
3164
3165	if (adapter->flags2 & IXGBE_FLAG2_PHY_FW_LOAD_FAILED)
3166	return;
3167
3168	if (link_cfg_err & IXGBE_ACI_LINK_EXTERNAL_PHY_LOAD_FAILURE) {
3169	netdev_err(dev: adapter->netdev, format: "Device failed to load the FW for the external PHY. Please download and install the latest NVM for your device and try again\n");
3170	adapter->flags2 |= IXGBE_FLAG2_PHY_FW_LOAD_FAILED;
3171	}
3172	}
3173
3174	/**
3175	* ixgbe_check_module_power - check module power level
3176	* @adapter: pointer to adapter structure
3177	* @link_cfg_err: bitmap from the link info structure
3178	*
3179	* Check module power level returned by a previous call to aci_get_link_info
3180	* and print error messages if module power level is not supported.
3181	*/
3182	static void ixgbe_check_module_power(struct ixgbe_adapter *adapter,
3183	u8 link_cfg_err)
3184	{
3185	/* If module power level is supported, clear the flag. */
3186	if (!(link_cfg_err & (IXGBE_ACI_LINK_INVAL_MAX_POWER_LIMIT |
3187	IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) {
3188	adapter->flags2 &= ~IXGBE_FLAG2_MOD_POWER_UNSUPPORTED;
3189	return;
3190	}
3191
3192	/* If IXGBE_FLAG2_MOD_POWER_UNSUPPORTED was previously set and the
3193	* above block didn't clear this bit, there's nothing to do.
3194	*/
3195	if (adapter->flags2 & IXGBE_FLAG2_MOD_POWER_UNSUPPORTED)
3196	return;
3197
3198	if (link_cfg_err & IXGBE_ACI_LINK_INVAL_MAX_POWER_LIMIT) {
3199	netdev_err(dev: adapter->netdev, format: "The installed module is incompatible with the device's NVM image. Cannot start link.\n");
3200	adapter->flags2 |= IXGBE_FLAG2_MOD_POWER_UNSUPPORTED;
3201	} else if (link_cfg_err & IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED) {
3202	netdev_err(dev: adapter->netdev, format: "The module's power requirements exceed the device's power supply. Cannot start link.\n");
3203	adapter->flags2 |= IXGBE_FLAG2_MOD_POWER_UNSUPPORTED;
3204	}
3205	}
3206
3207	/**
3208	* ixgbe_check_link_cfg_err - check if link configuration failed
3209	* @adapter: pointer to adapter structure
3210	* @link_cfg_err: bitmap from the link info structure
3211	*
3212	* Print if any link configuration failure happens due to the value in the
3213	* link_cfg_err parameter in the link info structure.
3214	*/
3215	static void ixgbe_check_link_cfg_err(struct ixgbe_adapter *adapter,
3216	u8 link_cfg_err)
3217	{
3218	ixgbe_check_module_power(adapter, link_cfg_err);
3219	ixgbe_check_phy_fw_load(adapter, link_cfg_err);
3220	}
3221
3222	/**
3223	* ixgbe_process_link_status_event - process the link event
3224	* @adapter: pointer to adapter structure
3225	* @link_up: true if the physical link is up and false if it is down
3226	* @link_speed: current link speed received from the link event
3227	*
3228	* Return: 0 on success or negative value on failure.
3229	*/
3230	static int
3231	ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up,
3232	u16 link_speed)
3233	{
3234	struct ixgbe_hw *hw = &adapter->hw;
3235	int status;
3236
3237	/* Update the link info structures and re-enable link events,
3238	* don't bail on failure due to other book keeping needed.
3239	*/
3240	status = ixgbe_update_link_info(hw);
3241	if (status)
3242	e_dev_err("Failed to update link status, err %d aq_err %d\n",
3243	status, hw->aci.last_status);
3244
3245	ixgbe_check_link_cfg_err(adapter, link_cfg_err: hw->link.link_info.link_cfg_err);
3246
3247	/* Check if the link state is up after updating link info, and treat
3248	* this event as an UP event since the link is actually UP now.
3249	*/
3250	if (hw->link.link_info.link_info & IXGBE_ACI_LINK_UP)
3251	link_up = true;
3252
3253	/* Turn off PHY if media was removed. */
3254	if (!(adapter->flags2 & IXGBE_FLAG2_NO_MEDIA) &&
3255	!(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE))
3256	adapter->flags2 |= IXGBE_FLAG2_NO_MEDIA;
3257
3258	if (link_up == adapter->link_up &&
3259	link_up == netif_carrier_ok(dev: adapter->netdev) &&
3260	link_speed == adapter->link_speed)
3261	return 0;
3262
3263	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
3264	adapter->link_check_timeout = jiffies;
3265	ixgbe_watchdog_update_link(adapter);
3266
3267	if (link_up)
3268	ixgbe_watchdog_link_is_up(adapter);
3269	else
3270	ixgbe_watchdog_link_is_down(adapter);
3271
3272	return 0;
3273	}
3274
3275	/**
3276	* ixgbe_handle_link_status_event - handle link status event via ACI
3277	* @adapter: pointer to adapter structure
3278	* @e: event structure containing link status info
3279	*/
3280	static void
3281	ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter,
3282	struct ixgbe_aci_event *e)
3283	{
3284	struct ixgbe_aci_cmd_get_link_status_data *link_data;
3285	u16 link_speed;
3286	bool link_up;
3287
3288	link_data = (struct ixgbe_aci_cmd_get_link_status_data *)e->msg_buf;
3289
3290	link_up = !!(link_data->link_info & IXGBE_ACI_LINK_UP);
3291	link_speed = le16_to_cpu(link_data->link_speed);
3292
3293	if (ixgbe_process_link_status_event(adapter, link_up, link_speed))
3294	e_dev_warn("Could not process link status event");
3295	}
3296
3297	/**
3298	* ixgbe_schedule_fw_event - schedule Firmware event
3299	* @adapter: pointer to the adapter structure
3300	*
3301	* If the adapter is not in down, removing or resetting state,
3302	* an event is scheduled.
3303	*/
3304	static void ixgbe_schedule_fw_event(struct ixgbe_adapter *adapter)
3305	{
3306	if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
3307	!test_bit(__IXGBE_REMOVING, &adapter->state) &&
3308	!test_bit(__IXGBE_RESETTING, &adapter->state)) {
3309	adapter->flags2 |= IXGBE_FLAG2_FW_ASYNC_EVENT;
3310	ixgbe_service_event_schedule(adapter);
3311	}
3312	}
3313
3314	/**
3315	* ixgbe_aci_event_cleanup - release msg_buf memory
3316	* @event: pointer to the event holding msg_buf to be released
3317	*
3318	* Clean memory allocated for event's msg_buf. Implements auto memory cleanup.
3319	*/
3320	static void ixgbe_aci_event_cleanup(struct ixgbe_aci_event *event)
3321	{
3322	kfree(objp: event->msg_buf);
3323	}
3324
3325	/**
3326	* ixgbe_handle_fw_event - handle Firmware event
3327	* @adapter: pointer to the adapter structure
3328	*
3329	* Obtain an event from the ACI and then and then process it according to the
3330	* type of the event and the opcode.
3331	*/
3332	static void ixgbe_handle_fw_event(struct ixgbe_adapter *adapter)
3333	{
3334	struct ixgbe_aci_event event __cleanup(ixgbe_aci_event_cleanup);
3335	struct ixgbe_hw *hw = &adapter->hw;
3336	bool pending = false;
3337	int err;
3338
3339	if (adapter->flags2 & IXGBE_FLAG2_FW_ASYNC_EVENT)
3340	adapter->flags2 &= ~IXGBE_FLAG2_FW_ASYNC_EVENT;
3341	event.buf_len = IXGBE_ACI_MAX_BUFFER_SIZE;
3342	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
3343	if (!event.msg_buf)
3344	return;
3345
3346	do {
3347	err = ixgbe_aci_get_event(hw, e: &event, pending: &pending);
3348	if (err)
3349	break;
3350
3351	switch (le16_to_cpu(event.desc.opcode)) {
3352	case ixgbe_aci_opc_get_link_status:
3353	ixgbe_handle_link_status_event(adapter, e: &event);
3354	break;
3355	case ixgbe_aci_opc_temp_tca_event:
3356	e_crit(drv, "%s\n", ixgbe_overheat_msg);
3357	ixgbe_down(adapter);
3358	break;
3359	case libie_aqc_opc_fw_logs_event:
3360	libie_get_fwlog_data(fwlog: &hw->fwlog, buf: event.msg_buf,
3361	le16_to_cpu(event.desc.datalen));
3362	break;
3363	default:
3364	e_warn(hw, "unknown FW async event captured\n");
3365	break;
3366	}
3367	} while (pending);
3368	}
3369
3370	static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
3371	u64 qmask)
3372	{
3373	struct ixgbe_hw *hw = &adapter->hw;
3374	u32 mask;
3375
3376	switch (hw->mac.type) {
3377	case ixgbe_mac_82598EB:
3378	mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
3379	IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3380	break;
3381	case ixgbe_mac_82599EB:
3382	case ixgbe_mac_X540:
3383	case ixgbe_mac_X550:
3384	case ixgbe_mac_X550EM_x:
3385	case ixgbe_mac_x550em_a:
3386	case ixgbe_mac_e610:
3387	mask = (qmask & 0xFFFFFFFF);
3388	if (mask)
3389	IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
3390	mask = (qmask >> 32);
3391	if (mask)
3392	IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
3393	break;
3394	default:
3395	break;
3396	}
3397	/* skip the flush */
3398	}
3399
3400	/**
3401	* ixgbe_irq_enable - Enable default interrupt generation settings
3402	* @adapter: board private structure
3403	* @queues: enable irqs for queues
3404	* @flush: flush register write
3405	**/
3406	static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
3407	bool flush)
3408	{
3409	struct ixgbe_hw *hw = &adapter->hw;
3410	u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3411
3412	/* don't reenable LSC while waiting for link */
3413	if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
3414	mask &= ~IXGBE_EIMS_LSC;
3415
3416	if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
3417	switch (adapter->hw.mac.type) {
3418	case ixgbe_mac_82599EB:
3419	mask |= IXGBE_EIMS_GPI_SDP0(hw);
3420	break;
3421	case ixgbe_mac_X540:
3422	case ixgbe_mac_X550:
3423	case ixgbe_mac_X550EM_x:
3424	case ixgbe_mac_x550em_a:
3425	mask |= IXGBE_EIMS_TS;
3426	break;
3427	default:
3428	break;
3429	}
3430	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
3431	mask |= IXGBE_EIMS_GPI_SDP1(hw);
3432	switch (adapter->hw.mac.type) {
3433	case ixgbe_mac_82599EB:
3434	mask |= IXGBE_EIMS_GPI_SDP1(hw);
3435	mask |= IXGBE_EIMS_GPI_SDP2(hw);
3436	fallthrough;
3437	case ixgbe_mac_X540:
3438	case ixgbe_mac_X550:
3439	case ixgbe_mac_X550EM_x:
3440	case ixgbe_mac_e610:
3441	mask |= IXGBE_EIMS_FW_EVENT;
3442	fallthrough;
3443	case ixgbe_mac_x550em_a:
3444	if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
3445	adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
3446	adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N)
3447	mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw);
3448	if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t)
3449	mask |= IXGBE_EICR_GPI_SDP0_X540;
3450	mask |= IXGBE_EIMS_ECC;
3451	mask |= IXGBE_EIMS_MAILBOX;
3452	break;
3453	default:
3454	break;
3455	}
3456
3457	if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
3458	!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
3459	mask |= IXGBE_EIMS_FLOW_DIR;
3460
3461	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
3462	if (queues)
3463	ixgbe_irq_enable_queues(adapter, qmask: ~0);
3464	if (flush)
3465	IXGBE_WRITE_FLUSH(&adapter->hw);
3466	}
3467
3468	static irqreturn_t ixgbe_msix_other(int irq, void *data)
3469	{
3470	struct ixgbe_adapter *adapter = data;
3471	struct ixgbe_hw *hw = &adapter->hw;
3472	u32 eicr;
3473
3474	/*
3475	* Workaround for Silicon errata. Use clear-by-write instead
3476	* of clear-by-read. Reading with EICS will return the
3477	* interrupt causes without clearing, which later be done
3478	* with the write to EICR.
3479	*/
3480	eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
3481
3482	/* The lower 16bits of the EICR register are for the queue interrupts
3483	* which should be masked here in order to not accidentally clear them if
3484	* the bits are high when ixgbe_msix_other is called. There is a race
3485	* condition otherwise which results in possible performance loss
3486	* especially if the ixgbe_msix_other interrupt is triggering
3487	* consistently (as it would when PPS is turned on for the X540 device)
3488	*/
3489	eicr &= 0xFFFF0000;
3490
3491	IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
3492
3493	if (eicr & IXGBE_EICR_LSC)
3494	ixgbe_check_lsc(adapter);
3495
3496	if (eicr & IXGBE_EICR_MAILBOX)
3497	ixgbe_msg_task(adapter);
3498
3499	if (eicr & IXGBE_EICR_FW_EVENT)
3500	ixgbe_schedule_fw_event(adapter);
3501
3502	switch (hw->mac.type) {
3503	case ixgbe_mac_82599EB:
3504	case ixgbe_mac_X540:
3505	case ixgbe_mac_X550:
3506	case ixgbe_mac_X550EM_x:
3507	case ixgbe_mac_x550em_a:
3508	case ixgbe_mac_e610:
3509	if (hw->phy.type == ixgbe_phy_x550em_ext_t &&
3510	(eicr & IXGBE_EICR_GPI_SDP0_X540)) {
3511	adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT;
3512	ixgbe_service_event_schedule(adapter);
3513	IXGBE_WRITE_REG(hw, IXGBE_EICR,
3514	IXGBE_EICR_GPI_SDP0_X540);
3515	}
3516	if (eicr & IXGBE_EICR_ECC) {
3517	e_info(link, "Received ECC Err, initiating reset\n");
3518	set_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
3519	ixgbe_service_event_schedule(adapter);
3520	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
3521	}
3522	/* Handle Flow Director Full threshold interrupt */
3523	if (eicr & IXGBE_EICR_FLOW_DIR) {
3524	int reinit_count = 0;
3525	int i;
3526	for (i = 0; i < adapter->num_tx_queues; i++) {
3527	struct ixgbe_ring *ring = adapter->tx_ring[i];
3528	if (test_and_clear_bit(nr: __IXGBE_TX_FDIR_INIT_DONE,
3529	addr: &ring->state))
3530	reinit_count++;
3531	}
3532	if (reinit_count) {
3533	/* no more flow director interrupts until after init */
3534	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
3535	adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
3536	ixgbe_service_event_schedule(adapter);
3537	}
3538	}
3539	ixgbe_check_sfp_event(adapter, eicr);
3540	ixgbe_check_overtemp_event(adapter, eicr);
3541	break;
3542	default:
3543	break;
3544	}
3545
3546	ixgbe_check_fan_failure(adapter, eicr);
3547
3548	if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
3549	ixgbe_ptp_check_pps_event(adapter);
3550
3551	/* re-enable the original interrupt state, no lsc, no queues */
3552	if (!test_bit(__IXGBE_DOWN, &adapter->state))
3553	ixgbe_irq_enable(adapter, queues: false, flush: false);
3554
3555	return IRQ_HANDLED;
3556	}
3557
3558	static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
3559	{
3560	struct ixgbe_q_vector *q_vector = data;
3561
3562	/* EIAM disabled interrupts (on this vector) for us */
3563
3564	if (q_vector->rx.ring || q_vector->tx.ring)
3565	napi_schedule_irqoff(n: &q_vector->napi);
3566
3567	return IRQ_HANDLED;
3568	}
3569
3570	/**
3571	* ixgbe_poll - NAPI Rx polling callback
3572	* @napi: structure for representing this polling device
3573	* @budget: how many packets driver is allowed to clean
3574	*
3575	* This function is used for legacy and MSI, NAPI mode
3576	**/
3577	int ixgbe_poll(struct napi_struct *napi, int budget)
3578	{
3579	struct ixgbe_q_vector *q_vector =
3580	container_of(napi, struct ixgbe_q_vector, napi);
3581	struct ixgbe_adapter *adapter = q_vector->adapter;
3582	struct ixgbe_ring *ring;
3583	int per_ring_budget, work_done = 0;
3584	bool clean_complete = true;
3585
3586	#ifdef CONFIG_IXGBE_DCA
3587	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
3588	ixgbe_update_dca(q_vector);
3589	#endif
3590
3591	ixgbe_for_each_ring(ring, q_vector->tx) {
3592	bool wd = ring->xsk_pool ?
3593	ixgbe_clean_xdp_tx_irq(q_vector, tx_ring: ring, napi_budget: budget) :
3594	ixgbe_clean_tx_irq(q_vector, tx_ring: ring, napi_budget: budget);
3595
3596	if (!wd)
3597	clean_complete = false;
3598	}
3599
3600	/* Exit if we are called by netpoll */
3601	if (budget <= 0)
3602	return budget;
3603
3604	/* attempt to distribute budget to each queue fairly, but don't allow
3605	* the budget to go below 1 because we'll exit polling */
3606	if (q_vector->rx.count > 1)
3607	per_ring_budget = max(budget/q_vector->rx.count, 1);
3608	else
3609	per_ring_budget = budget;
3610
3611	ixgbe_for_each_ring(ring, q_vector->rx) {
3612	int cleaned = ring->xsk_pool ?
3613	ixgbe_clean_rx_irq_zc(q_vector, rx_ring: ring,
3614	budget: per_ring_budget) :
3615	ixgbe_clean_rx_irq(q_vector, rx_ring: ring,
3616	budget: per_ring_budget);
3617
3618	work_done += cleaned;
3619	if (cleaned >= per_ring_budget)
3620	clean_complete = false;
3621	}
3622
3623	/* If all work not completed, return budget and keep polling */
3624	if (!clean_complete)
3625	return budget;
3626
3627	/* all work done, exit the polling mode */
3628	if (likely(napi_complete_done(napi, work_done))) {
3629	if (adapter->rx_itr_setting & 1)
3630	ixgbe_set_itr(q_vector);
3631	if (!test_bit(__IXGBE_DOWN, &adapter->state))
3632	ixgbe_irq_enable_queues(adapter,
3633	BIT_ULL(q_vector->v_idx));
3634	}
3635
3636	return min(work_done, budget - 1);
3637	}
3638
3639	/**
3640	* ixgbe_request_msix_irqs - Initialize MSI-X interrupts
3641	* @adapter: board private structure
3642	*
3643	* ixgbe_request_msix_irqs allocates MSI-X vectors and requests
3644	* interrupts from the kernel.
3645	**/
3646	static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
3647	{
3648	struct net_device *netdev = adapter->netdev;
3649	unsigned int ri = 0, ti = 0;
3650	int vector, err;
3651
3652	for (vector = 0; vector < adapter->num_q_vectors; vector++) {
3653	struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
3654	struct msix_entry *entry = &adapter->msix_entries[vector];
3655
3656	if (q_vector->tx.ring && q_vector->rx.ring) {
3657	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
3658	fmt: "%s-TxRx-%u", netdev->name, ri++);
3659	ti++;
3660	} else if (q_vector->rx.ring) {
3661	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
3662	fmt: "%s-rx-%u", netdev->name, ri++);
3663	} else if (q_vector->tx.ring) {
3664	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
3665	fmt: "%s-tx-%u", netdev->name, ti++);
3666	} else {
3667	/* skip this unused q_vector */
3668	continue;
3669	}
3670	err = request_irq(irq: entry->vector, handler: &ixgbe_msix_clean_rings, flags: 0,
3671	name: q_vector->name, dev: q_vector);
3672	if (err) {
3673	e_err(probe, "request_irq failed for MSIX interrupt "
3674	"Error: %d\n", err);
3675	goto free_queue_irqs;
3676	}
3677	/* If Flow Director is enabled, set interrupt affinity */
3678	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
3679	/* assign the mask for this irq */
3680	irq_update_affinity_hint(irq: entry->vector,
3681	m: &q_vector->affinity_mask);
3682	}
3683	}
3684
3685	err = request_irq(irq: adapter->msix_entries[vector].vector,
3686	handler: ixgbe_msix_other, flags: 0, name: netdev->name, dev: adapter);
3687	if (err) {
3688	e_err(probe, "request_irq for msix_other failed: %d\n", err);
3689	goto free_queue_irqs;
3690	}
3691
3692	return 0;
3693
3694	free_queue_irqs:
3695	while (vector) {
3696	vector--;
3697	irq_update_affinity_hint(irq: adapter->msix_entries[vector].vector,
3698	NULL);
3699	free_irq(adapter->msix_entries[vector].vector,
3700	adapter->q_vector[vector]);
3701	}
3702	adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
3703	pci_disable_msix(dev: adapter->pdev);
3704	kfree(objp: adapter->msix_entries);
3705	adapter->msix_entries = NULL;
3706	return err;
3707	}
3708
3709	/**
3710	* ixgbe_intr - legacy mode Interrupt Handler
3711	* @irq: interrupt number
3712	* @data: pointer to a network interface device structure
3713	**/
3714	static irqreturn_t ixgbe_intr(int irq, void *data)
3715	{
3716	struct ixgbe_adapter *adapter = data;
3717	struct ixgbe_hw *hw = &adapter->hw;
3718	struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
3719	u32 eicr;
3720
3721	/*
3722	* Workaround for silicon errata #26 on 82598. Mask the interrupt
3723	* before the read of EICR.
3724	*/
3725	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
3726
3727	/* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
3728	* therefore no explicit interrupt disable is necessary */
3729	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3730	if (!eicr) {
3731	/*
3732	* shared interrupt alert!
3733	* make sure interrupts are enabled because the read will
3734	* have disabled interrupts due to EIAM
3735	* finish the workaround of silicon errata on 82598. Unmask
3736	* the interrupt that we masked before the EICR read.
3737	*/
3738	if (!test_bit(__IXGBE_DOWN, &adapter->state))
3739	ixgbe_irq_enable(adapter, queues: true, flush: true);
3740	return IRQ_NONE; /* Not our interrupt */
3741	}
3742
3743	if (eicr & IXGBE_EICR_LSC)
3744	ixgbe_check_lsc(adapter);
3745
3746	if (eicr & IXGBE_EICR_FW_EVENT)
3747	ixgbe_schedule_fw_event(adapter);
3748
3749	switch (hw->mac.type) {
3750	case ixgbe_mac_82599EB:
3751	ixgbe_check_sfp_event(adapter, eicr);
3752	fallthrough;
3753	case ixgbe_mac_X540:
3754	case ixgbe_mac_X550:
3755	case ixgbe_mac_X550EM_x:
3756	case ixgbe_mac_x550em_a:
3757	case ixgbe_mac_e610:
3758	if (eicr & IXGBE_EICR_ECC) {
3759	e_info(link, "Received ECC Err, initiating reset\n");
3760	set_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
3761	ixgbe_service_event_schedule(adapter);
3762	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
3763	}
3764	ixgbe_check_overtemp_event(adapter, eicr);
3765	break;
3766	default:
3767	break;
3768	}
3769
3770	ixgbe_check_fan_failure(adapter, eicr);
3771	if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
3772	ixgbe_ptp_check_pps_event(adapter);
3773
3774	/* would disable interrupts here but EIAM disabled it */
3775	napi_schedule_irqoff(n: &q_vector->napi);
3776
3777	/*
3778	* re-enable link(maybe) and non-queue interrupts, no flush.
3779	* ixgbe_poll will re-enable the queue interrupts
3780	*/
3781	if (!test_bit(__IXGBE_DOWN, &adapter->state))
3782	ixgbe_irq_enable(adapter, queues: false, flush: false);
3783
3784	return IRQ_HANDLED;
3785	}
3786
3787	/**
3788	* ixgbe_request_irq - initialize interrupts
3789	* @adapter: board private structure
3790	*
3791	* Attempts to configure interrupts using the best available
3792	* capabilities of the hardware and kernel.
3793	**/
3794	static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
3795	{
3796	struct net_device *netdev = adapter->netdev;
3797	int err;
3798
3799	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
3800	err = ixgbe_request_msix_irqs(adapter);
3801	else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
3802	err = request_irq(irq: adapter->pdev->irq, handler: ixgbe_intr, flags: 0,
3803	name: netdev->name, dev: adapter);
3804	else
3805	err = request_irq(irq: adapter->pdev->irq, handler: ixgbe_intr, IRQF_SHARED,
3806	name: netdev->name, dev: adapter);
3807
3808	if (err)
3809	e_err(probe, "request_irq failed, Error %d\n", err);
3810
3811	return err;
3812	}
3813
3814	static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
3815	{
3816	int vector;
3817
3818	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
3819	free_irq(adapter->pdev->irq, adapter);
3820	return;
3821	}
3822
3823	if (!adapter->msix_entries)
3824	return;
3825
3826	for (vector = 0; vector < adapter->num_q_vectors; vector++) {
3827	struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
3828	struct msix_entry *entry = &adapter->msix_entries[vector];
3829
3830	/* free only the irqs that were actually requested */
3831	if (!q_vector->rx.ring && !q_vector->tx.ring)
3832	continue;
3833
3834	/* clear the affinity_mask in the IRQ descriptor */
3835	irq_update_affinity_hint(irq: entry->vector, NULL);
3836
3837	free_irq(entry->vector, q_vector);
3838	}
3839
3840	free_irq(adapter->msix_entries[vector].vector, adapter);
3841	}
3842
3843	/**
3844	* ixgbe_irq_disable - Mask off interrupt generation on the NIC
3845	* @adapter: board private structure
3846	**/
3847	static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
3848	{
3849	switch (adapter->hw.mac.type) {
3850	case ixgbe_mac_82598EB:
3851	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
3852	break;
3853	case ixgbe_mac_82599EB:
3854	case ixgbe_mac_X540:
3855	case ixgbe_mac_X550:
3856	case ixgbe_mac_X550EM_x:
3857	case ixgbe_mac_x550em_a:
3858	case ixgbe_mac_e610:
3859	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
3860	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
3861	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
3862	break;
3863	default:
3864	break;
3865	}
3866	IXGBE_WRITE_FLUSH(&adapter->hw);
3867	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3868	int vector;
3869
3870	for (vector = 0; vector < adapter->num_q_vectors; vector++)
3871	synchronize_irq(irq: adapter->msix_entries[vector].vector);
3872
3873	synchronize_irq(irq: adapter->msix_entries[vector++].vector);
3874	} else {
3875	synchronize_irq(irq: adapter->pdev->irq);
3876	}
3877	}
3878
3879	/**
3880	* ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
3881	* @adapter: board private structure
3882	*
3883	**/
3884	static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
3885	{
3886	struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
3887
3888	ixgbe_write_eitr(q_vector);
3889
3890	ixgbe_set_ivar(adapter, direction: 0, queue: 0, msix_vector: 0);
3891	ixgbe_set_ivar(adapter, direction: 1, queue: 0, msix_vector: 0);
3892
3893	e_info(hw, "Legacy interrupt IVAR setup done\n");
3894	}
3895
3896	/**
3897	* ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
3898	* @adapter: board private structure
3899	* @ring: structure containing ring specific data
3900	*
3901	* Configure the Tx descriptor ring after a reset.
3902	**/
3903	void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
3904	struct ixgbe_ring *ring)
3905	{
3906	struct ixgbe_hw *hw = &adapter->hw;
3907	u64 tdba = ring->dma;
3908	int wait_loop = 10;
3909	u32 txdctl = IXGBE_TXDCTL_ENABLE;
3910	u8 reg_idx = ring->reg_idx;
3911
3912	ring->xsk_pool = NULL;
3913	if (ring_is_xdp(ring))
3914	ring->xsk_pool = ixgbe_xsk_pool(adapter, ring);
3915
3916	/* disable queue to avoid issues while updating state */
3917	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
3918	IXGBE_WRITE_FLUSH(hw);
3919
3920	IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
3921	(tdba & DMA_BIT_MASK(32)));
3922	IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
3923	IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
3924	ring->count * sizeof(union ixgbe_adv_tx_desc));
3925	IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
3926	IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
3927	ring->tail = adapter->io_addr + IXGBE_TDT(reg_idx);
3928
3929	/*
3930	* set WTHRESH to encourage burst writeback, it should not be set
3931	* higher than 1 when:
3932	* - ITR is 0 as it could cause false TX hangs
3933	* - ITR is set to > 100k int/sec and BQL is enabled
3934	*
3935	* In order to avoid issues WTHRESH + PTHRESH should always be equal
3936	* to or less than the number of on chip descriptors, which is
3937	* currently 40.
3938	*/
3939	if (!ring->q_vector || (ring->q_vector->itr < IXGBE_100K_ITR))
3940	txdctl |= 1u << 16; /* WTHRESH = 1 */
3941	else
3942	txdctl |= 8u << 16; /* WTHRESH = 8 */
3943
3944	/*
3945	* Setting PTHRESH to 32 both improves performance
3946	* and avoids a TX hang with DFP enabled
3947	*/
3948	txdctl |= (1u << 8) | /* HTHRESH = 1 */
3949	32; /* PTHRESH = 32 */
3950
3951	/* reinitialize flowdirector state */
3952	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
3953	ring->atr_sample_rate = adapter->atr_sample_rate;
3954	ring->atr_count = 0;
3955	set_bit(nr: __IXGBE_TX_FDIR_INIT_DONE, addr: &ring->state);
3956	} else {
3957	ring->atr_sample_rate = 0;
3958	}
3959
3960	/* initialize XPS */
3961	if (!test_and_set_bit(nr: __IXGBE_TX_XPS_INIT_DONE, addr: &ring->state)) {
3962	struct ixgbe_q_vector *q_vector = ring->q_vector;
3963
3964	if (q_vector)
3965	netif_set_xps_queue(dev: ring->netdev,
3966	mask: &q_vector->affinity_mask,
3967	index: ring->queue_index);
3968	}
3969
3970	clear_bit(nr: __IXGBE_HANG_CHECK_ARMED, addr: &ring->state);
3971
3972	/* reinitialize tx_buffer_info */
3973	memset(ring->tx_buffer_info, 0,
3974	sizeof(struct ixgbe_tx_buffer) * ring->count);
3975
3976	/* enable queue */
3977	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
3978
3979	/* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
3980	if (hw->mac.type == ixgbe_mac_82598EB &&
3981	!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3982	return;
3983
3984	/* poll to verify queue is enabled */
3985	do {
3986	usleep_range(min: 1000, max: 2000);
3987	txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
3988	} while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
3989	if (!wait_loop)
3990	hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
3991	}
3992
3993	static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
3994	{
3995	struct ixgbe_hw *hw = &adapter->hw;
3996	u32 rttdcs, mtqc;
3997	u8 tcs = adapter->hw_tcs;
3998
3999	if (hw->mac.type == ixgbe_mac_82598EB)
4000	return;
4001
4002	/* disable the arbiter while setting MTQC */
4003	rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
4004	rttdcs |= IXGBE_RTTDCS_ARBDIS;
4005	IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4006
4007	/* set transmit pool layout */
4008	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
4009	mtqc = IXGBE_MTQC_VT_ENA;
4010	if (tcs > 4)
4011	mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
4012	else if (tcs > 1)
4013	mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
4014	else if (adapter->ring_feature[RING_F_VMDQ].mask ==
4015	IXGBE_82599_VMDQ_4Q_MASK)
4016	mtqc |= IXGBE_MTQC_32VF;
4017	else
4018	mtqc |= IXGBE_MTQC_64VF;
4019	} else {
4020	if (tcs > 4) {
4021	mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
4022	} else if (tcs > 1) {
4023	mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
4024	} else {
4025	u8 max_txq = adapter->num_tx_queues +
4026	adapter->num_xdp_queues;
4027	if (max_txq > 63)
4028	mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
4029	else
4030	mtqc = IXGBE_MTQC_64Q_1PB;
4031	}
4032	}
4033
4034	IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
4035
4036	/* Enable Security TX Buffer IFG for multiple pb */
4037	if (tcs) {
4038	u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
4039	sectx |= IXGBE_SECTX_DCB;
4040	IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
4041	}
4042
4043	/* re-enable the arbiter */
4044	rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
4045	IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
4046	}
4047
4048	/**
4049	* ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
4050	* @adapter: board private structure
4051	*
4052	* Configure the Tx unit of the MAC after a reset.
4053	**/
4054	static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
4055	{
4056	struct ixgbe_hw *hw = &adapter->hw;
4057	u32 dmatxctl;
4058	u32 i;
4059
4060	ixgbe_setup_mtqc(adapter);
4061
4062	if (hw->mac.type != ixgbe_mac_82598EB) {
4063	/* DMATXCTL.EN must be before Tx queues are enabled */
4064	dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
4065	dmatxctl |= IXGBE_DMATXCTL_TE;
4066	IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
4067	}
4068
4069	/* Setup the HW Tx Head and Tail descriptor pointers */
4070	for (i = 0; i < adapter->num_tx_queues; i++)
4071	ixgbe_configure_tx_ring(adapter, ring: adapter->tx_ring[i]);
4072	for (i = 0; i < adapter->num_xdp_queues; i++)
4073	ixgbe_configure_tx_ring(adapter, ring: adapter->xdp_ring[i]);
4074	}
4075
4076	static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
4077	struct ixgbe_ring *ring)
4078	{
4079	struct ixgbe_hw *hw = &adapter->hw;
4080	u8 reg_idx = ring->reg_idx;
4081	u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
4082
4083	srrctl |= IXGBE_SRRCTL_DROP_EN;
4084
4085	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
4086	}
4087
4088	static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
4089	struct ixgbe_ring *ring)
4090	{
4091	struct ixgbe_hw *hw = &adapter->hw;
4092	u8 reg_idx = ring->reg_idx;
4093	u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
4094
4095	srrctl &= ~IXGBE_SRRCTL_DROP_EN;
4096
4097	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
4098	}
4099
4100	#ifdef CONFIG_IXGBE_DCB
4101	void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
4102	#else
4103	static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
4104	#endif
4105	{
4106	bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
4107	struct ixgbe_hw *hw = &adapter->hw;
4108	int i;
4109
4110	if (hw->mac.ops.disable_mdd)
4111	hw->mac.ops.disable_mdd(hw);
4112
4113	if (adapter->ixgbe_ieee_pfc)
4114	pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
4115
4116	/*
4117	* We should set the drop enable bit if:
4118	* SR-IOV is enabled
4119	* or
4120	* Number of Rx queues > 1 and flow control is disabled
4121	*
4122	* This allows us to avoid head of line blocking for security
4123	* and performance reasons.
4124	*/
4125	if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
4126	!(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
4127	for (i = 0; i < adapter->num_rx_queues; i++)
4128	ixgbe_enable_rx_drop(adapter, ring: adapter->rx_ring[i]);
4129	} else {
4130	for (i = 0; i < adapter->num_rx_queues; i++)
4131	ixgbe_disable_rx_drop(adapter, ring: adapter->rx_ring[i]);
4132	}
4133
4134	if (hw->mac.ops.enable_mdd)
4135	hw->mac.ops.enable_mdd(hw);
4136	}
4137
4138	#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4139
4140	static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
4141	struct ixgbe_ring *rx_ring)
4142	{
4143	struct ixgbe_hw *hw = &adapter->hw;
4144	u32 srrctl;
4145	u8 reg_idx = rx_ring->reg_idx;
4146
4147	if (hw->mac.type == ixgbe_mac_82598EB) {
4148	u16 mask = adapter->ring_feature[RING_F_RSS].mask;
4149
4150	/*
4151	* if VMDq is not active we must program one srrctl register
4152	* per RSS queue since we have enabled RDRXCTL.MVMEN
4153	*/
4154	reg_idx &= mask;
4155	}
4156
4157	/* configure header buffer length, needed for RSC */
4158	srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
4159
4160	/* configure the packet buffer length */
4161	if (rx_ring->xsk_pool) {
4162	u32 xsk_buf_len = xsk_pool_get_rx_frame_size(pool: rx_ring->xsk_pool);
4163
4164	/* If the MAC support setting RXDCTL.RLPML, the
4165	* SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and
4166	* RXDCTL.RLPML is set to the actual UMEM buffer
4167	* size. If not, then we are stuck with a 1k buffer
4168	* size resolution. In this case frames larger than
4169	* the UMEM buffer size viewed in a 1k resolution will
4170	* be dropped.
4171	*/
4172	if (hw->mac.type != ixgbe_mac_82599EB)
4173	srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4174	else
4175	srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4176	} else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) {
4177	srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4178	} else {
4179	srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4180	}
4181
4182	/* configure descriptor type */
4183	srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4184
4185	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
4186	}
4187
4188	/**
4189	* ixgbe_rss_indir_tbl_entries - Return RSS indirection table entries
4190	* @adapter: device handle
4191	*
4192	* - 82598/82599/X540: 128
4193	* - X550(non-SRIOV mode): 512
4194	* - X550(SRIOV mode): 64
4195	*/
4196	u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter)
4197	{
4198	if (adapter->hw.mac.type < ixgbe_mac_X550)
4199	return 128;
4200	else if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
4201	return 64;
4202	else
4203	return 512;
4204	}
4205
4206	/**
4207	* ixgbe_store_key - Write the RSS key to HW
4208	* @adapter: device handle
4209	*
4210	* Write the RSS key stored in adapter.rss_key to HW.
4211	*/
4212	void ixgbe_store_key(struct ixgbe_adapter *adapter)
4213	{
4214	struct ixgbe_hw *hw = &adapter->hw;
4215	int i;
4216
4217	for (i = 0; i < 10; i++)
4218	IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]);
4219	}
4220
4221	/**
4222	* ixgbe_init_rss_key - Initialize adapter RSS key
4223	* @adapter: device handle
4224	*
4225	* Allocates and initializes the RSS key if it is not allocated.
4226	**/
4227	static inline int ixgbe_init_rss_key(struct ixgbe_adapter *adapter)
4228	{
4229	u32 *rss_key;
4230
4231	if (!adapter->rss_key) {
4232	rss_key = kzalloc(IXGBE_RSS_KEY_SIZE, GFP_KERNEL);
4233	if (unlikely(!rss_key))
4234	return -ENOMEM;
4235
4236	netdev_rss_key_fill(buffer: rss_key, IXGBE_RSS_KEY_SIZE);
4237	adapter->rss_key = rss_key;
4238	}
4239
4240	return 0;
4241	}
4242
4243	/**
4244	* ixgbe_store_reta - Write the RETA table to HW
4245	* @adapter: device handle
4246	*
4247	* Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
4248	*/
4249	void ixgbe_store_reta(struct ixgbe_adapter *adapter)
4250	{
4251	u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
4252	struct ixgbe_hw *hw = &adapter->hw;
4253	u32 reta = 0;
4254	u32 indices_multi;
4255	u8 *indir_tbl = adapter->rss_indir_tbl;
4256
4257	/* Fill out the redirection table as follows:
4258	* - 82598: 8 bit wide entries containing pair of 4 bit RSS
4259	* indices.
4260	* - 82599/X540: 8 bit wide entries containing 4 bit RSS index
4261	* - X550: 8 bit wide entries containing 6 bit RSS index
4262	*/
4263	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
4264	indices_multi = 0x11;
4265	else
4266	indices_multi = 0x1;
4267
4268	/* Write redirection table to HW */
4269	for (i = 0; i < reta_entries; i++) {
4270	reta |= indices_multi * indir_tbl[i] << (i & 0x3) * 8;
4271	if ((i & 3) == 3) {
4272	if (i < 128)
4273	IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4274	else
4275	IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
4276	reta);
4277	reta = 0;
4278	}
4279	}
4280	}
4281
4282	/**
4283	* ixgbe_store_vfreta - Write the RETA table to HW (x550 devices in SRIOV mode)
4284	* @adapter: device handle
4285	*
4286	* Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.
4287	*/
4288	static void ixgbe_store_vfreta(struct ixgbe_adapter *adapter)
4289	{
4290	u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
4291	struct ixgbe_hw *hw = &adapter->hw;
4292	u32 vfreta = 0;
4293
4294	/* Write redirection table to HW */
4295	for (i = 0; i < reta_entries; i++) {
4296	u16 pool = adapter->num_rx_pools;
4297
4298	vfreta |= (u32)adapter->rss_indir_tbl[i] << (i & 0x3) * 8;
4299	if ((i & 3) != 3)
4300	continue;
4301
4302	while (pool--)
4303	IXGBE_WRITE_REG(hw,
4304	IXGBE_PFVFRETA(i >> 2, VMDQ_P(pool)),
4305	vfreta);
4306	vfreta = 0;
4307	}
4308	}
4309
4310	static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)
4311	{
4312	u32 i, j;
4313	u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
4314	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
4315
4316	/* Program table for at least 4 queues w/ SR-IOV so that VFs can
4317	* make full use of any rings they may have. We will use the
4318	* PSRTYPE register to control how many rings we use within the PF.
4319	*/
4320	if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 4))
4321	rss_i = 4;
4322
4323	/* Fill out hash function seeds */
4324	ixgbe_store_key(adapter);
4325
4326	/* Fill out redirection table */
4327	memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl));
4328
4329	for (i = 0, j = 0; i < reta_entries; i++, j++) {
4330	if (j == rss_i)
4331	j = 0;
4332
4333	adapter->rss_indir_tbl[i] = j;
4334	}
4335
4336	ixgbe_store_reta(adapter);
4337	}
4338
4339	static void ixgbe_setup_vfreta(struct ixgbe_adapter *adapter)
4340	{
4341	struct ixgbe_hw *hw = &adapter->hw;
4342	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
4343	int i, j;
4344
4345	/* Fill out hash function seeds */
4346	for (i = 0; i < 10; i++) {
4347	u16 pool = adapter->num_rx_pools;
4348
4349	while (pool--)
4350	IXGBE_WRITE_REG(hw,
4351	IXGBE_PFVFRSSRK(i, VMDQ_P(pool)),
4352	*(adapter->rss_key + i));
4353	}
4354
4355	/* Fill out the redirection table */
4356	for (i = 0, j = 0; i < 64; i++, j++) {
4357	if (j == rss_i)
4358	j = 0;
4359
4360	adapter->rss_indir_tbl[i] = j;
4361	}
4362
4363	ixgbe_store_vfreta(adapter);
4364	}
4365
4366	static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
4367	{
4368	struct ixgbe_hw *hw = &adapter->hw;
4369	u32 mrqc = 0, rss_field = 0, vfmrqc = 0;
4370	u32 rxcsum;
4371
4372	/* Disable indicating checksum in descriptor, enables RSS hash */
4373	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4374	rxcsum |= IXGBE_RXCSUM_PCSD;
4375	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4376
4377	if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4378	if (adapter->ring_feature[RING_F_RSS].mask)
4379	mrqc = IXGBE_MRQC_RSSEN;
4380	} else {
4381	u8 tcs = adapter->hw_tcs;
4382
4383	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
4384	if (tcs > 4)
4385	mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */
4386	else if (tcs > 1)
4387	mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */
4388	else if (adapter->ring_feature[RING_F_VMDQ].mask ==
4389	IXGBE_82599_VMDQ_4Q_MASK)
4390	mrqc = IXGBE_MRQC_VMDQRSS32EN;
4391	else
4392	mrqc = IXGBE_MRQC_VMDQRSS64EN;
4393
4394	/* Enable L3/L4 for Tx Switched packets only for X550,
4395	* older devices do not support this feature
4396	*/
4397	if (hw->mac.type >= ixgbe_mac_X550)
4398	mrqc |= IXGBE_MRQC_L3L4TXSWEN;
4399	} else {
4400	if (tcs > 4)
4401	mrqc = IXGBE_MRQC_RTRSS8TCEN;
4402	else if (tcs > 1)
4403	mrqc = IXGBE_MRQC_RTRSS4TCEN;
4404	else
4405	mrqc = IXGBE_MRQC_RSSEN;
4406	}
4407	}
4408
4409	/* Perform hash on these packet types */
4410	rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 |
4411	IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
4412	IXGBE_MRQC_RSS_FIELD_IPV6 |
4413	IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
4414
4415	if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)
4416	rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
4417	if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)
4418	rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
4419
4420	if ((hw->mac.type >= ixgbe_mac_X550) &&
4421	(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) {
4422	u16 pool = adapter->num_rx_pools;
4423
4424	/* Enable VF RSS mode */
4425	mrqc |= IXGBE_MRQC_MULTIPLE_RSS;
4426	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4427
4428	/* Setup RSS through the VF registers */
4429	ixgbe_setup_vfreta(adapter);
4430	vfmrqc = IXGBE_MRQC_RSSEN;
4431	vfmrqc |= rss_field;
4432
4433	while (pool--)
4434	IXGBE_WRITE_REG(hw,
4435	IXGBE_PFVFMRQC(VMDQ_P(pool)),
4436	vfmrqc);
4437	} else {
4438	ixgbe_setup_reta(adapter);
4439	mrqc |= rss_field;
4440	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4441	}
4442	}
4443
4444	/**
4445	* ixgbe_configure_rscctl - enable RSC for the indicated ring
4446	* @adapter: address of board private structure
4447	* @ring: structure containing ring specific data
4448	**/
4449	static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
4450	struct ixgbe_ring *ring)
4451	{
4452	struct ixgbe_hw *hw = &adapter->hw;
4453	u32 rscctrl;
4454	u8 reg_idx = ring->reg_idx;
4455
4456	if (!ring_is_rsc_enabled(ring))
4457	return;
4458
4459	rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
4460	rscctrl |= IXGBE_RSCCTL_RSCEN;
4461	/*
4462	* we must limit the number of descriptors so that the
4463	* total size of max desc * buf_len is not greater
4464	* than 65536
4465	*/
4466	rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
4467	IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
4468	}
4469
4470	#define IXGBE_MAX_RX_DESC_POLL 10
4471	static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
4472	struct ixgbe_ring *ring)
4473	{
4474	struct ixgbe_hw *hw = &adapter->hw;
4475	int wait_loop = IXGBE_MAX_RX_DESC_POLL;
4476	u32 rxdctl;
4477	u8 reg_idx = ring->reg_idx;
4478
4479	if (ixgbe_removed(addr: hw->hw_addr))
4480	return;
4481	/* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
4482	if (hw->mac.type == ixgbe_mac_82598EB &&
4483	!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
4484	return;
4485
4486	do {
4487	usleep_range(min: 1000, max: 2000);
4488	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
4489	} while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
4490
4491	if (!wait_loop) {
4492	e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
4493	"the polling period\n", reg_idx);
4494	}
4495	}
4496
4497	void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
4498	struct ixgbe_ring *ring)
4499	{
4500	struct ixgbe_hw *hw = &adapter->hw;
4501	union ixgbe_adv_rx_desc *rx_desc;
4502	u64 rdba = ring->dma;
4503	u32 rxdctl;
4504	u8 reg_idx = ring->reg_idx;
4505
4506	xdp_rxq_info_unreg_mem_model(xdp_rxq: &ring->xdp_rxq);
4507	ring->xsk_pool = ixgbe_xsk_pool(adapter, ring);
4508	if (ring->xsk_pool) {
4509	WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
4510	MEM_TYPE_XSK_BUFF_POOL,
4511	NULL));
4512	xsk_pool_set_rxq_info(pool: ring->xsk_pool, rxq: &ring->xdp_rxq);
4513	} else {
4514	WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
4515	MEM_TYPE_PAGE_SHARED, NULL));
4516	}
4517
4518	/* disable queue to avoid use of these values while updating state */
4519	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
4520	rxdctl &= ~IXGBE_RXDCTL_ENABLE;
4521
4522	/* write value back with RXDCTL.ENABLE bit cleared */
4523	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
4524	IXGBE_WRITE_FLUSH(hw);
4525
4526	IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
4527	IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
4528	IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
4529	ring->count * sizeof(union ixgbe_adv_rx_desc));
4530	/* Force flushing of IXGBE_RDLEN to prevent MDD */
4531	IXGBE_WRITE_FLUSH(hw);
4532
4533	IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
4534	IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
4535	ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx);
4536
4537	ixgbe_configure_srrctl(adapter, rx_ring: ring);
4538	ixgbe_configure_rscctl(adapter, ring);
4539
4540	if (hw->mac.type == ixgbe_mac_82598EB) {
4541	/*
4542	* enable cache line friendly hardware writes:
4543	* PTHRESH=32 descriptors (half the internal cache),
4544	* this also removes ugly rx_no_buffer_count increment
4545	* HTHRESH=4 descriptors (to minimize latency on fetch)
4546	* WTHRESH=8 burst writeback up to two cache lines
4547	*/
4548	rxdctl &= ~0x3FFFFF;
4549	rxdctl |= 0x080420;
4550	#if (PAGE_SIZE < 8192)
4551	/* RXDCTL.RLPML does not work on 82599 */
4552	} else if (hw->mac.type != ixgbe_mac_82599EB) {
4553	rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
4554	IXGBE_RXDCTL_RLPML_EN);
4555
4556	/* Limit the maximum frame size so we don't overrun the skb.
4557	* This can happen in SRIOV mode when the MTU of the VF is
4558	* higher than the MTU of the PF.
4559	*/
4560	if (ring_uses_build_skb(ring) &&
4561	!test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
4562	rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB |
4563	IXGBE_RXDCTL_RLPML_EN;
4564	#endif
4565	}
4566
4567	ring->rx_offset = ixgbe_rx_offset(rx_ring: ring);
4568
4569	if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) {
4570	u32 xsk_buf_len = xsk_pool_get_rx_frame_size(pool: ring->xsk_pool);
4571
4572	rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
4573	IXGBE_RXDCTL_RLPML_EN);
4574	rxdctl |= xsk_buf_len | IXGBE_RXDCTL_RLPML_EN;
4575
4576	ring->rx_buf_len = xsk_buf_len;
4577	}
4578
4579	/* initialize rx_buffer_info */
4580	memset(ring->rx_buffer_info, 0,
4581	sizeof(struct ixgbe_rx_buffer) * ring->count);
4582
4583	/* initialize Rx descriptor 0 */
4584	rx_desc = IXGBE_RX_DESC(ring, 0);
4585	rx_desc->wb.upper.length = 0;
4586
4587	/* enable receive descriptor ring */
4588	rxdctl |= IXGBE_RXDCTL_ENABLE;
4589	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
4590
4591	ixgbe_rx_desc_queue_enable(adapter, ring);
4592	if (ring->xsk_pool)
4593	ixgbe_alloc_rx_buffers_zc(rx_ring: ring, cleaned_count: ixgbe_desc_unused(ring));
4594	else
4595	ixgbe_alloc_rx_buffers(rx_ring: ring, cleaned_count: ixgbe_desc_unused(ring));
4596	}
4597
4598	static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
4599	{
4600	struct ixgbe_hw *hw = &adapter->hw;
4601	int rss_i = adapter->ring_feature[RING_F_RSS].indices;
4602	u16 pool = adapter->num_rx_pools;
4603
4604	/* PSRTYPE must be initialized in non 82598 adapters */
4605	u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4606	IXGBE_PSRTYPE_UDPHDR |
4607	IXGBE_PSRTYPE_IPV4HDR |
4608	IXGBE_PSRTYPE_L2HDR |
4609	IXGBE_PSRTYPE_IPV6HDR;
4610
4611	if (hw->mac.type == ixgbe_mac_82598EB)
4612	return;
4613
4614	if (rss_i > 3)
4615	psrtype |= 2u << 29;
4616	else if (rss_i > 1)
4617	psrtype |= 1u << 29;
4618
4619	while (pool--)
4620	IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype);
4621	}
4622
4623	static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
4624	{
4625	struct ixgbe_hw *hw = &adapter->hw;
4626	u16 pool = adapter->num_rx_pools;
4627	u32 reg_offset, vf_shift, vmolr;
4628	u32 gcr_ext, vmdctl;
4629	int i;
4630
4631	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
4632	return;
4633
4634	vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
4635	vmdctl |= IXGBE_VMD_CTL_VMDQ_EN;
4636	vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
4637	vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT;
4638	vmdctl |= IXGBE_VT_CTL_REPLEN;
4639	IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
4640
4641	/* accept untagged packets until a vlan tag is
4642	* specifically set for the VMDQ queue/pool
4643	*/
4644	vmolr = IXGBE_VMOLR_AUPE;
4645	while (pool--)
4646	IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(pool)), vmolr);
4647
4648	vf_shift = VMDQ_P(0) % 32;
4649	reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0;
4650
4651	/* Enable only the PF's pool for Tx/Rx */
4652	IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), GENMASK(31, vf_shift));
4653	IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1);
4654	IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), GENMASK(31, vf_shift));
4655	IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1);
4656	if (adapter->bridge_mode == BRIDGE_MODE_VEB)
4657	IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
4658
4659	/* Map PF MAC address in RAR Entry 0 to first pool following VFs */
4660	hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0));
4661
4662	/* clear VLAN promisc flag so VFTA will be updated if necessary */
4663	adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;
4664
4665	/*
4666	* Set up VF register offsets for selected VT Mode,
4667	* i.e. 32 or 64 VFs for SR-IOV
4668	*/
4669	switch (adapter->ring_feature[RING_F_VMDQ].mask) {
4670	case IXGBE_82599_VMDQ_8Q_MASK:
4671	gcr_ext = IXGBE_GCR_EXT_VT_MODE_16;
4672	break;
4673	case IXGBE_82599_VMDQ_4Q_MASK:
4674	gcr_ext = IXGBE_GCR_EXT_VT_MODE_32;
4675	break;
4676	default:
4677	gcr_ext = IXGBE_GCR_EXT_VT_MODE_64;
4678	break;
4679	}
4680
4681	IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
4682
4683	for (i = 0; i < adapter->num_vfs; i++) {
4684	/* configure spoof checking */
4685	ixgbe_ndo_set_vf_spoofchk(netdev: adapter->netdev, vf: i,
4686	setting: adapter->vfinfo[i].spoofchk_enabled);
4687
4688	/* Enable/Disable RSS query feature */
4689	ixgbe_ndo_set_vf_rss_query_en(netdev: adapter->netdev, vf: i,
4690	setting: adapter->vfinfo[i].rss_query_enabled);
4691	}
4692	}
4693
4694	static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
4695	{
4696	struct ixgbe_hw *hw = &adapter->hw;
4697	struct net_device *netdev = adapter->netdev;
4698	int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
4699	struct ixgbe_ring *rx_ring;
4700	int i;
4701	u32 mhadd, hlreg0;
4702
4703	#ifdef IXGBE_FCOE
4704	/* adjust max frame to be able to do baby jumbo for FCoE */
4705	if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
4706	(max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
4707	max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
4708
4709	#endif /* IXGBE_FCOE */
4710
4711	/* adjust max frame to be at least the size of a standard frame */
4712	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
4713	max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN);
4714
4715	mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
4716	if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
4717	mhadd &= ~IXGBE_MHADD_MFS_MASK;
4718	mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
4719
4720	IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
4721	}
4722
4723	hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4724	/* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
4725	hlreg0 |= IXGBE_HLREG0_JUMBOEN;
4726	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
4727
4728	/*
4729	* Setup the HW Rx Head and Tail Descriptor Pointers and
4730	* the Base and Length of the Rx Descriptor Ring
4731	*/
4732	for (i = 0; i < adapter->num_rx_queues; i++) {
4733	rx_ring = adapter->rx_ring[i];
4734
4735	clear_ring_rsc_enabled(rx_ring);
4736	clear_bit(nr: __IXGBE_RX_3K_BUFFER, addr: &rx_ring->state);
4737	clear_bit(nr: __IXGBE_RX_BUILD_SKB_ENABLED, addr: &rx_ring->state);
4738
4739	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
4740	set_ring_rsc_enabled(rx_ring);
4741
4742	if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state))
4743	set_bit(nr: __IXGBE_RX_3K_BUFFER, addr: &rx_ring->state);
4744
4745	if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY)
4746	continue;
4747
4748	set_bit(nr: __IXGBE_RX_BUILD_SKB_ENABLED, addr: &rx_ring->state);
4749
4750	#if (PAGE_SIZE < 8192)
4751	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
4752	set_bit(nr: __IXGBE_RX_3K_BUFFER, addr: &rx_ring->state);
4753
4754	if (IXGBE_2K_TOO_SMALL_WITH_PADDING ||
4755	(max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)))
4756	set_bit(nr: __IXGBE_RX_3K_BUFFER, addr: &rx_ring->state);
4757	#endif
4758	}
4759	}
4760
4761	static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
4762	{
4763	struct ixgbe_hw *hw = &adapter->hw;
4764	u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
4765
4766	switch (hw->mac.type) {
4767	case ixgbe_mac_82598EB:
4768	/*
4769	* For VMDq support of different descriptor types or
4770	* buffer sizes through the use of multiple SRRCTL
4771	* registers, RDRXCTL.MVMEN must be set to 1
4772	*
4773	* also, the manual doesn't mention it clearly but DCA hints
4774	* will only use queue 0's tags unless this bit is set. Side
4775	* effects of setting this bit are only that SRRCTL must be
4776	* fully programmed [0..15]
4777	*/
4778	rdrxctl |= IXGBE_RDRXCTL_MVMEN;
4779	break;
4780	case ixgbe_mac_X550:
4781	case ixgbe_mac_X550EM_x:
4782	case ixgbe_mac_x550em_a:
4783	case ixgbe_mac_e610:
4784	if (adapter->num_vfs)
4785	rdrxctl |= IXGBE_RDRXCTL_PSP;
4786	fallthrough;
4787	case ixgbe_mac_82599EB:
4788	case ixgbe_mac_X540:
4789	/* Disable RSC for ACK packets */
4790	IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
4791	(IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
4792	rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
4793	/* hardware requires some bits to be set by default */
4794	rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
4795	rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
4796	break;
4797	default:
4798	/* We should do nothing since we don't know this hardware */
4799	return;
4800	}
4801
4802	IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
4803	}
4804
4805	/**
4806	* ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
4807	* @adapter: board private structure
4808	*
4809	* Configure the Rx unit of the MAC after a reset.
4810	**/
4811	static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
4812	{
4813	struct ixgbe_hw *hw = &adapter->hw;
4814	int i;
4815	u32 rxctrl, rfctl;
4816
4817	/* disable receives while setting up the descriptors */
4818	hw->mac.ops.disable_rx(hw);
4819
4820	ixgbe_setup_psrtype(adapter);
4821	ixgbe_setup_rdrxctl(adapter);
4822
4823	/* RSC Setup */
4824	rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);
4825	rfctl &= ~IXGBE_RFCTL_RSC_DIS;
4826	if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))
4827	rfctl |= IXGBE_RFCTL_RSC_DIS;
4828
4829	/* disable NFS filtering */
4830	rfctl |= (IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS);
4831	IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);
4832
4833	/* Program registers for the distribution of queues */
4834	ixgbe_setup_mrqc(adapter);
4835
4836	/* set_rx_buffer_len must be called before ring initialization */
4837	ixgbe_set_rx_buffer_len(adapter);
4838
4839	/*
4840	* Setup the HW Rx Head and Tail Descriptor Pointers and
4841	* the Base and Length of the Rx Descriptor Ring
4842	*/
4843	for (i = 0; i < adapter->num_rx_queues; i++)
4844	ixgbe_configure_rx_ring(adapter, ring: adapter->rx_ring[i]);
4845
4846	rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4847	/* disable drop enable for 82598 parts */
4848	if (hw->mac.type == ixgbe_mac_82598EB)
4849	rxctrl |= IXGBE_RXCTRL_DMBYPS;
4850
4851	/* enable all receives */
4852	rxctrl |= IXGBE_RXCTRL_RXEN;
4853	hw->mac.ops.enable_rx_dma(hw, rxctrl);
4854	}
4855
4856	static int ixgbe_vlan_rx_add_vid(struct net_device *netdev,
4857	__be16 proto, u16 vid)
4858	{
4859	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
4860	struct ixgbe_hw *hw = &adapter->hw;
4861
4862	/* add VID to filter table */
4863	if (!vid || !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4864	hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true, !!vid);
4865
4866	set_bit(nr: vid, addr: adapter->active_vlans);
4867
4868	return 0;
4869	}
4870
4871	static int ixgbe_find_vlvf_entry(struct ixgbe_hw *hw, u32 vlan)
4872	{
4873	u32 vlvf;
4874	int idx;
4875
4876	/* short cut the special case */
4877	if (vlan == 0)
4878	return 0;
4879
4880	/* Search for the vlan id in the VLVF entries */
4881	for (idx = IXGBE_VLVF_ENTRIES; --idx;) {
4882	vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(idx));
4883	if ((vlvf & VLAN_VID_MASK) == vlan)
4884	break;
4885	}
4886
4887	return idx;
4888	}
4889
4890	void ixgbe_update_pf_promisc_vlvf(struct ixgbe_adapter *adapter, u32 vid)
4891	{
4892	struct ixgbe_hw *hw = &adapter->hw;
4893	u32 bits, word;
4894	int idx;
4895
4896	idx = ixgbe_find_vlvf_entry(hw, vlan: vid);
4897	if (!idx)
4898	return;
4899
4900	/* See if any other pools are set for this VLAN filter
4901	* entry other than the PF.
4902	*/
4903	word = idx * 2 + (VMDQ_P(0) / 32);
4904	bits = ~BIT(VMDQ_P(0) % 32);
4905	bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));
4906
4907	/* Disable the filter so this falls into the default pool. */
4908	if (!bits && !IXGBE_READ_REG(hw, IXGBE_VLVFB(word ^ 1))) {
4909	if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4910	IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), 0);
4911	IXGBE_WRITE_REG(hw, IXGBE_VLVF(idx), 0);
4912	}
4913	}
4914
4915	static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev,
4916	__be16 proto, u16 vid)
4917	{
4918	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
4919	struct ixgbe_hw *hw = &adapter->hw;
4920
4921	/* remove VID from filter table */
4922	if (vid && !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
4923	hw->mac.ops.set_vfta(hw, vid, VMDQ_P(0), false, true);
4924
4925	clear_bit(nr: vid, addr: adapter->active_vlans);
4926
4927	return 0;
4928	}
4929
4930	/**
4931	* ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
4932	* @adapter: driver data
4933	*/
4934	static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
4935	{
4936	struct ixgbe_hw *hw = &adapter->hw;
4937	u32 vlnctrl;
4938	int i, j;
4939
4940	switch (hw->mac.type) {
4941	case ixgbe_mac_82598EB:
4942	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4943	vlnctrl &= ~IXGBE_VLNCTRL_VME;
4944	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4945	break;
4946	case ixgbe_mac_82599EB:
4947	case ixgbe_mac_X540:
4948	case ixgbe_mac_X550:
4949	case ixgbe_mac_X550EM_x:
4950	case ixgbe_mac_x550em_a:
4951	case ixgbe_mac_e610:
4952	for (i = 0; i < adapter->num_rx_queues; i++) {
4953	struct ixgbe_ring *ring = adapter->rx_ring[i];
4954
4955	if (!netif_is_ixgbe(dev: ring->netdev))
4956	continue;
4957
4958	j = ring->reg_idx;
4959	vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
4960	vlnctrl &= ~IXGBE_RXDCTL_VME;
4961	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
4962	}
4963	break;
4964	default:
4965	break;
4966	}
4967	}
4968
4969	/**
4970	* ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
4971	* @adapter: driver data
4972	*/
4973	static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
4974	{
4975	struct ixgbe_hw *hw = &adapter->hw;
4976	u32 vlnctrl;
4977	int i, j;
4978
4979	switch (hw->mac.type) {
4980	case ixgbe_mac_82598EB:
4981	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4982	vlnctrl |= IXGBE_VLNCTRL_VME;
4983	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
4984	break;
4985	case ixgbe_mac_82599EB:
4986	case ixgbe_mac_X540:
4987	case ixgbe_mac_X550:
4988	case ixgbe_mac_X550EM_x:
4989	case ixgbe_mac_x550em_a:
4990	case ixgbe_mac_e610:
4991	for (i = 0; i < adapter->num_rx_queues; i++) {
4992	struct ixgbe_ring *ring = adapter->rx_ring[i];
4993
4994	if (!netif_is_ixgbe(dev: ring->netdev))
4995	continue;
4996
4997	j = ring->reg_idx;
4998	vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
4999	vlnctrl |= IXGBE_RXDCTL_VME;
5000	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
5001	}
5002	break;
5003	default:
5004	break;
5005	}
5006	}
5007
5008	static void ixgbe_vlan_promisc_enable(struct ixgbe_adapter *adapter)
5009	{
5010	struct ixgbe_hw *hw = &adapter->hw;
5011	u32 vlnctrl, i;
5012
5013	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
5014
5015	if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) {
5016	/* For VMDq and SR-IOV we must leave VLAN filtering enabled */
5017	vlnctrl |= IXGBE_VLNCTRL_VFE;
5018	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
5019	} else {
5020	vlnctrl &= ~IXGBE_VLNCTRL_VFE;
5021	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
5022	return;
5023	}
5024
5025	/* Nothing to do for 82598 */
5026	if (hw->mac.type == ixgbe_mac_82598EB)
5027	return;
5028
5029	/* We are already in VLAN promisc, nothing to do */
5030	if (adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)
5031	return;
5032
5033	/* Set flag so we don't redo unnecessary work */
5034	adapter->flags2 |= IXGBE_FLAG2_VLAN_PROMISC;
5035
5036	/* Add PF to all active pools */
5037	for (i = IXGBE_VLVF_ENTRIES; --i;) {
5038	u32 reg_offset = IXGBE_VLVFB(i * 2 + VMDQ_P(0) / 32);
5039	u32 vlvfb = IXGBE_READ_REG(hw, reg_offset);
5040
5041	vlvfb |= BIT(VMDQ_P(0) % 32);
5042	IXGBE_WRITE_REG(hw, reg_offset, vlvfb);
5043	}
5044
5045	/* Set all bits in the VLAN filter table array */
5046	for (i = hw->mac.vft_size; i--;)
5047	IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), ~0U);
5048	}
5049
5050	#define VFTA_BLOCK_SIZE 8
5051	static void ixgbe_scrub_vfta(struct ixgbe_adapter *adapter, u32 vfta_offset)
5052	{
5053	struct ixgbe_hw *hw = &adapter->hw;
5054	u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
5055	u32 vid_start = vfta_offset * 32;
5056	u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
5057	u32 i, vid, word, bits;
5058
5059	for (i = IXGBE_VLVF_ENTRIES; --i;) {
5060	u32 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(i));
5061
5062	/* pull VLAN ID from VLVF */
5063	vid = vlvf & VLAN_VID_MASK;
5064
5065	/* only concern ourselves with a certain range */
5066	if (vid < vid_start || vid >= vid_end)
5067	continue;
5068
5069	if (vlvf) {
5070	/* record VLAN ID in VFTA */
5071	vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
5072
5073	/* if PF is part of this then continue */
5074	if (test_bit(vid, adapter->active_vlans))
5075	continue;
5076	}
5077
5078	/* remove PF from the pool */
5079	word = i * 2 + VMDQ_P(0) / 32;
5080	bits = ~BIT(VMDQ_P(0) % 32);
5081	bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));
5082	IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), bits);
5083	}
5084
5085	/* extract values from active_vlans and write back to VFTA */
5086	for (i = VFTA_BLOCK_SIZE; i--;) {
5087	vid = (vfta_offset + i) * 32;
5088	word = vid / BITS_PER_LONG;
5089	bits = vid % BITS_PER_LONG;
5090
5091	vfta[i] |= adapter->active_vlans[word] >> bits;
5092
5093	IXGBE_WRITE_REG(hw, IXGBE_VFTA(vfta_offset + i), vfta[i]);
5094	}
5095	}
5096
5097	static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter)
5098	{
5099	struct ixgbe_hw *hw = &adapter->hw;
5100	u32 vlnctrl, i;
5101
5102	/* Set VLAN filtering to enabled */
5103	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
5104	vlnctrl |= IXGBE_VLNCTRL_VFE;
5105	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
5106
5107	if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) ||
5108	hw->mac.type == ixgbe_mac_82598EB)
5109	return;
5110
5111	/* We are not in VLAN promisc, nothing to do */
5112	if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))
5113	return;
5114
5115	/* Set flag so we don't redo unnecessary work */
5116	adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;
5117
5118	for (i = 0; i < hw->mac.vft_size; i += VFTA_BLOCK_SIZE)
5119	ixgbe_scrub_vfta(adapter, vfta_offset: i);
5120	}
5121
5122	static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
5123	{
5124	u16 vid = 1;
5125
5126	ixgbe_vlan_rx_add_vid(netdev: adapter->netdev, htons(ETH_P_8021Q), vid: 0);
5127
5128	for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
5129	ixgbe_vlan_rx_add_vid(netdev: adapter->netdev, htons(ETH_P_8021Q), vid);
5130	}
5131
5132	/**
5133	* ixgbe_write_mc_addr_list - write multicast addresses to MTA
5134	* @netdev: network interface device structure
5135	*
5136	* Writes multicast address list to the MTA hash table.
5137	* Returns: -ENOMEM on failure
5138	* 0 on no addresses written
5139	* X on writing X addresses to MTA
5140	**/
5141	static int ixgbe_write_mc_addr_list(struct net_device *netdev)
5142	{
5143	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
5144	struct ixgbe_hw *hw = &adapter->hw;
5145
5146	if (!netif_running(dev: netdev))
5147	return 0;
5148
5149	if (hw->mac.ops.update_mc_addr_list)
5150	hw->mac.ops.update_mc_addr_list(hw, netdev);
5151	else
5152	return -ENOMEM;
5153
5154	#ifdef CONFIG_PCI_IOV
5155	ixgbe_restore_vf_multicasts(adapter);
5156	#endif
5157
5158	return netdev_mc_count(netdev);
5159	}
5160
5161	#ifdef CONFIG_PCI_IOV
5162	void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter)
5163	{
5164	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5165	struct ixgbe_hw *hw = &adapter->hw;
5166	int i;
5167
5168	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5169	mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;
5170
5171	if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
5172	hw->mac.ops.set_rar(hw, i,
5173	mac_table->addr,
5174	mac_table->pool,
5175	IXGBE_RAH_AV);
5176	else
5177	hw->mac.ops.clear_rar(hw, i);
5178	}
5179	}
5180
5181	#endif
5182	static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter)
5183	{
5184	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5185	struct ixgbe_hw *hw = &adapter->hw;
5186	int i;
5187
5188	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5189	if (!(mac_table->state & IXGBE_MAC_STATE_MODIFIED))
5190	continue;
5191
5192	mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;
5193
5194	if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
5195	hw->mac.ops.set_rar(hw, i,
5196	mac_table->addr,
5197	mac_table->pool,
5198	IXGBE_RAH_AV);
5199	else
5200	hw->mac.ops.clear_rar(hw, i);
5201	}
5202	}
5203
5204	static void ixgbe_flush_sw_mac_table(struct ixgbe_adapter *adapter)
5205	{
5206	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5207	struct ixgbe_hw *hw = &adapter->hw;
5208	int i;
5209
5210	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5211	mac_table->state |= IXGBE_MAC_STATE_MODIFIED;
5212	mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;
5213	}
5214
5215	ixgbe_sync_mac_table(adapter);
5216	}
5217
5218	static int ixgbe_available_rars(struct ixgbe_adapter *adapter, u16 pool)
5219	{
5220	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5221	struct ixgbe_hw *hw = &adapter->hw;
5222	int i, count = 0;
5223
5224	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5225	/* do not count default RAR as available */
5226	if (mac_table->state & IXGBE_MAC_STATE_DEFAULT)
5227	continue;
5228
5229	/* only count unused and addresses that belong to us */
5230	if (mac_table->state & IXGBE_MAC_STATE_IN_USE) {
5231	if (mac_table->pool != pool)
5232	continue;
5233	}
5234
5235	count++;
5236	}
5237
5238	return count;
5239	}
5240
5241	/* this function destroys the first RAR entry */
5242	static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter)
5243	{
5244	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5245	struct ixgbe_hw *hw = &adapter->hw;
5246
5247	memcpy(&mac_table->addr, hw->mac.addr, ETH_ALEN);
5248	mac_table->pool = VMDQ_P(0);
5249
5250	mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE;
5251
5252	hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool,
5253	IXGBE_RAH_AV);
5254	}
5255
5256	int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter,
5257	const u8 *addr, u16 pool)
5258	{
5259	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5260	struct ixgbe_hw *hw = &adapter->hw;
5261	int i;
5262
5263	if (is_zero_ether_addr(addr))
5264	return -EINVAL;
5265
5266	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5267	if (mac_table->state & IXGBE_MAC_STATE_IN_USE)
5268	continue;
5269
5270	ether_addr_copy(dst: mac_table->addr, src: addr);
5271	mac_table->pool = pool;
5272
5273	mac_table->state |= IXGBE_MAC_STATE_MODIFIED |
5274	IXGBE_MAC_STATE_IN_USE;
5275
5276	ixgbe_sync_mac_table(adapter);
5277
5278	return i;
5279	}
5280
5281	return -ENOMEM;
5282	}
5283
5284	int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter,
5285	const u8 *addr, u16 pool)
5286	{
5287	struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];
5288	struct ixgbe_hw *hw = &adapter->hw;
5289	int i;
5290
5291	if (is_zero_ether_addr(addr))
5292	return -EINVAL;
5293
5294	/* search table for addr, if found clear IN_USE flag and sync */
5295	for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {
5296	/* we can only delete an entry if it is in use */
5297	if (!(mac_table->state & IXGBE_MAC_STATE_IN_USE))
5298	continue;
5299	/* we only care about entries that belong to the given pool */
5300	if (mac_table->pool != pool)
5301	continue;
5302	/* we only care about a specific MAC address */
5303	if (!ether_addr_equal(addr1: addr, addr2: mac_table->addr))
5304	continue;
5305
5306	mac_table->state |= IXGBE_MAC_STATE_MODIFIED;
5307	mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;
5308
5309	ixgbe_sync_mac_table(adapter);
5310
5311	return 0;
5312	}
5313
5314	return -ENOMEM;
5315	}
5316
5317	static int ixgbe_uc_sync(struct net_device *netdev, const unsigned char *addr)
5318	{
5319	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
5320	int ret;
5321
5322	ret = ixgbe_add_mac_filter(adapter, addr, VMDQ_P(0));
5323
5324	return min_t(int, ret, 0);
5325	}
5326
5327	static int ixgbe_uc_unsync(struct net_device *netdev, const unsigned char *addr)
5328	{
5329	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
5330
5331	ixgbe_del_mac_filter(adapter, addr, VMDQ_P(0));
5332
5333	return 0;
5334	}
5335
5336	/**
5337	* ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
5338	* @netdev: network interface device structure
5339	*
5340	* The set_rx_method entry point is called whenever the unicast/multicast
5341	* address list or the network interface flags are updated. This routine is
5342	* responsible for configuring the hardware for proper unicast, multicast and
5343	* promiscuous mode.
5344	**/
5345	void ixgbe_set_rx_mode(struct net_device *netdev)
5346	{
5347	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
5348	struct ixgbe_hw *hw = &adapter->hw;
5349	u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
5350	netdev_features_t features = netdev->features;
5351	int count;
5352
5353	/* Check for Promiscuous and All Multicast modes */
5354	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5355
5356	/* set all bits that we expect to always be set */
5357	fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */
5358	fctrl |= IXGBE_FCTRL_BAM;
5359	fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
5360	fctrl |= IXGBE_FCTRL_PMCF;
5361
5362	/* clear the bits we are changing the status of */
5363	fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
5364	if (netdev->flags & IFF_PROMISC) {
5365	hw->addr_ctrl.user_set_promisc = true;
5366	fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
5367	vmolr |= IXGBE_VMOLR_MPE;
5368	features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
5369	} else {
5370	if (netdev->flags & IFF_ALLMULTI) {
5371	fctrl |= IXGBE_FCTRL_MPE;
5372	vmolr |= IXGBE_VMOLR_MPE;
5373	}
5374	hw->addr_ctrl.user_set_promisc = false;
5375	}
5376
5377	/*
5378	* Write addresses to available RAR registers, if there is not
5379	* sufficient space to store all the addresses then enable
5380	* unicast promiscuous mode
5381	*/
5382	if (__dev_uc_sync(dev: netdev, sync: ixgbe_uc_sync, unsync: ixgbe_uc_unsync)) {
5383	fctrl |= IXGBE_FCTRL_UPE;
5384	vmolr |= IXGBE_VMOLR_ROPE;
5385	}
5386
5387	/* Write addresses to the MTA, if the attempt fails
5388	* then we should just turn on promiscuous mode so
5389	* that we can at least receive multicast traffic
5390	*/
5391	count = ixgbe_write_mc_addr_list(netdev);
5392	if (count < 0) {
5393	fctrl |= IXGBE_FCTRL_MPE;
5394	vmolr |= IXGBE_VMOLR_MPE;
5395	} else if (count) {
5396	vmolr |= IXGBE_VMOLR_ROMPE;
5397	}
5398
5399	if (hw->mac.type != ixgbe_mac_82598EB) {
5400	vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) &
5401	~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
5402	IXGBE_VMOLR_ROPE);
5403	IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr);
5404	}
5405
5406	/* This is useful for sniffing bad packets. */
5407	if (features & NETIF_F_RXALL) {
5408	/* UPE and MPE will be handled by normal PROMISC logic
5409	* in e1000e_set_rx_mode */
5410	fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */
5411	IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */
5412	IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */
5413
5414	fctrl &= ~(IXGBE_FCTRL_DPF);
5415	/* NOTE: VLAN filtering is disabled by setting PROMISC */
5416	}
5417
5418	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5419
5420	if (features & NETIF_F_HW_VLAN_CTAG_RX)
5421	ixgbe_vlan_strip_enable(adapter);
5422	else
5423	ixgbe_vlan_strip_disable(adapter);
5424
5425	if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
5426	ixgbe_vlan_promisc_disable(adapter);
5427	else
5428	ixgbe_vlan_promisc_enable(adapter);
5429	}
5430
5431	static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
5432	{
5433	int q_idx;
5434
5435	for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
5436	napi_enable(n: &adapter->q_vector[q_idx]->napi);
5437	}
5438
5439	static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
5440	{
5441	int q_idx;
5442
5443	for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)
5444	napi_disable(n: &adapter->q_vector[q_idx]->napi);
5445	}
5446
5447	static int ixgbe_udp_tunnel_sync(struct net_device *dev, unsigned int table)
5448	{
5449	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
5450	struct ixgbe_hw *hw = &adapter->hw;
5451	struct udp_tunnel_info ti;
5452
5453	udp_tunnel_nic_get_port(dev, table, idx: 0, ti: &ti);
5454	if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
5455	adapter->vxlan_port = ti.port;
5456	else
5457	adapter->geneve_port = ti.port;
5458
5459	IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL,
5460	ntohs(adapter->vxlan_port) |
5461	ntohs(adapter->geneve_port) <<
5462	IXGBE_VXLANCTRL_GENEVE_UDPPORT_SHIFT);
5463	return 0;
5464	}
5465
5466	static const struct udp_tunnel_nic_info ixgbe_udp_tunnels_x550 = {
5467	.sync_table = ixgbe_udp_tunnel_sync,
5468	.flags = UDP_TUNNEL_NIC_INFO_IPV4_ONLY,
5469	.tables = {
5470	{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
5471	},
5472	};
5473
5474	static const struct udp_tunnel_nic_info ixgbe_udp_tunnels_x550em_a = {
5475	.sync_table = ixgbe_udp_tunnel_sync,
5476	.flags = UDP_TUNNEL_NIC_INFO_IPV4_ONLY,
5477	.tables = {
5478	{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
5479	{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
5480	},
5481	};
5482
5483	#ifdef CONFIG_IXGBE_DCB
5484	/**
5485	* ixgbe_configure_dcb - Configure DCB hardware
5486	* @adapter: ixgbe adapter struct
5487	*
5488	* This is called by the driver on open to configure the DCB hardware.
5489	* This is also called by the gennetlink interface when reconfiguring
5490	* the DCB state.
5491	*/
5492	static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
5493	{
5494	struct ixgbe_hw *hw = &adapter->hw;
5495	int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
5496
5497	if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
5498	if (hw->mac.type == ixgbe_mac_82598EB)
5499	netif_set_tso_max_size(dev: adapter->netdev, size: 65536);
5500	return;
5501	}
5502
5503	if (hw->mac.type == ixgbe_mac_82598EB)
5504	netif_set_tso_max_size(dev: adapter->netdev, size: 32768);
5505
5506	#ifdef IXGBE_FCOE
5507	if (adapter->netdev->fcoe_mtu)
5508	max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
5509	#endif
5510
5511	/* reconfigure the hardware */
5512	if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
5513	ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
5514	DCB_TX_CONFIG);
5515	ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
5516	DCB_RX_CONFIG);
5517	ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
5518	} else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {
5519	ixgbe_dcb_hw_ets(hw: &adapter->hw,
5520	ets: adapter->ixgbe_ieee_ets,
5521	max: max_frame);
5522	ixgbe_dcb_hw_pfc_config(hw: &adapter->hw,
5523	pfc_en: adapter->ixgbe_ieee_pfc->pfc_en,
5524	tc_prio: adapter->ixgbe_ieee_ets->prio_tc);
5525	}
5526
5527	/* Enable RSS Hash per TC */
5528	if (hw->mac.type != ixgbe_mac_82598EB) {
5529	u32 msb = 0;
5530	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1;
5531
5532	while (rss_i) {
5533	msb++;
5534	rss_i >>= 1;
5535	}
5536
5537	/* write msb to all 8 TCs in one write */
5538	IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111);
5539	}
5540	}
5541	#endif
5542
5543	/* Additional bittime to account for IXGBE framing */
5544	#define IXGBE_ETH_FRAMING 20
5545
5546	/**
5547	* ixgbe_hpbthresh - calculate high water mark for flow control
5548	*
5549	* @adapter: board private structure to calculate for
5550	* @pb: packet buffer to calculate
5551	*/
5552	static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)
5553	{
5554	struct ixgbe_hw *hw = &adapter->hw;
5555	struct net_device *dev = adapter->netdev;
5556	int link, tc, kb, marker;
5557	u32 dv_id, rx_pba;
5558
5559	/* Calculate max LAN frame size */
5560	tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;
5561
5562	#ifdef IXGBE_FCOE
5563	/* FCoE traffic class uses FCOE jumbo frames */
5564	if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE &&
5565	(pb == ixgbe_fcoe_get_tc(adapter)))
5566	tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
5567	#endif
5568
5569	/* Calculate delay value for device */
5570	switch (hw->mac.type) {
5571	case ixgbe_mac_X540:
5572	case ixgbe_mac_X550:
5573	case ixgbe_mac_X550EM_x:
5574	case ixgbe_mac_x550em_a:
5575	case ixgbe_mac_e610:
5576	dv_id = IXGBE_DV_X540(link, tc);
5577	break;
5578	default:
5579	dv_id = IXGBE_DV(link, tc);
5580	break;
5581	}
5582
5583	/* Loopback switch introduces additional latency */
5584	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
5585	dv_id += IXGBE_B2BT(tc);
5586
5587	/* Delay value is calculated in bit times convert to KB */
5588	kb = IXGBE_BT2KB(dv_id);
5589	rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;
5590
5591	marker = rx_pba - kb;
5592
5593	/* It is possible that the packet buffer is not large enough
5594	* to provide required headroom. In this case throw an error
5595	* to user and a do the best we can.
5596	*/
5597	if (marker < 0) {
5598	e_warn(drv, "Packet Buffer(%i) can not provide enough"
5599	"headroom to support flow control."
5600	"Decrease MTU or number of traffic classes\n", pb);
5601	marker = tc + 1;
5602	}
5603
5604	return marker;
5605	}
5606
5607	/**
5608	* ixgbe_lpbthresh - calculate low water mark for flow control
5609	*
5610	* @adapter: board private structure to calculate for
5611	* @pb: packet buffer to calculate
5612	*/
5613	static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb)
5614	{
5615	struct ixgbe_hw *hw = &adapter->hw;
5616	struct net_device *dev = adapter->netdev;
5617	int tc;
5618	u32 dv_id;
5619
5620	/* Calculate max LAN frame size */
5621	tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
5622
5623	#ifdef IXGBE_FCOE
5624	/* FCoE traffic class uses FCOE jumbo frames */
5625	if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE &&
5626	(pb == netdev_get_prio_tc_map(dev, prio: adapter->fcoe.up)))
5627	tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;
5628	#endif
5629
5630	/* Calculate delay value for device */
5631	switch (hw->mac.type) {
5632	case ixgbe_mac_X540:
5633	case ixgbe_mac_X550:
5634	case ixgbe_mac_X550EM_x:
5635	case ixgbe_mac_x550em_a:
5636	case ixgbe_mac_e610:
5637	dv_id = IXGBE_LOW_DV_X540(tc);
5638	break;
5639	default:
5640	dv_id = IXGBE_LOW_DV(tc);
5641	break;
5642	}
5643
5644	/* Delay value is calculated in bit times convert to KB */
5645	return IXGBE_BT2KB(dv_id);
5646	}
5647
5648	/*
5649	* ixgbe_pbthresh_setup - calculate and setup high low water marks
5650	*/
5651	static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
5652	{
5653	struct ixgbe_hw *hw = &adapter->hw;
5654	int num_tc = adapter->hw_tcs;
5655	int i;
5656
5657	if (!num_tc)
5658	num_tc = 1;
5659
5660	for (i = 0; i < num_tc; i++) {
5661	hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, pb: i);
5662	hw->fc.low_water[i] = ixgbe_lpbthresh(adapter, pb: i);
5663
5664	/* Low water marks must not be larger than high water marks */
5665	if (hw->fc.low_water[i] > hw->fc.high_water[i])
5666	hw->fc.low_water[i] = 0;
5667	}
5668
5669	for (; i < MAX_TRAFFIC_CLASS; i++)
5670	hw->fc.high_water[i] = 0;
5671	}
5672
5673	static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
5674	{
5675	struct ixgbe_hw *hw = &adapter->hw;
5676	int hdrm;
5677	u8 tc = adapter->hw_tcs;
5678
5679	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
5680	adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
5681	hdrm = 32 << adapter->fdir_pballoc;
5682	else
5683	hdrm = 0;
5684
5685	hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);
5686	ixgbe_pbthresh_setup(adapter);
5687	}
5688
5689	static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
5690	{
5691	struct ixgbe_hw *hw = &adapter->hw;
5692	struct hlist_node *node2;
5693	struct ixgbe_fdir_filter *filter;
5694	u8 queue;
5695
5696	spin_lock(lock: &adapter->fdir_perfect_lock);
5697
5698	if (!hlist_empty(h: &adapter->fdir_filter_list))
5699	ixgbe_fdir_set_input_mask_82599(hw, input_mask: &adapter->fdir_mask);
5700
5701	hlist_for_each_entry_safe(filter, node2,
5702	&adapter->fdir_filter_list, fdir_node) {
5703	if (filter->action == IXGBE_FDIR_DROP_QUEUE) {
5704	queue = IXGBE_FDIR_DROP_QUEUE;
5705	} else {
5706	u32 ring = ethtool_get_flow_spec_ring(ring_cookie: filter->action);
5707	u8 vf = ethtool_get_flow_spec_ring_vf(ring_cookie: filter->action);
5708
5709	if (!vf && (ring >= adapter->num_rx_queues)) {
5710	e_err(drv, "FDIR restore failed without VF, ring: %u\n",
5711	ring);
5712	continue;
5713	} else if (vf &&
5714	((vf > adapter->num_vfs) ||
5715	ring >= adapter->num_rx_queues_per_pool)) {
5716	e_err(drv, "FDIR restore failed with VF, vf: %hhu, ring: %u\n",
5717	vf, ring);
5718	continue;
5719	}
5720
5721	/* Map the ring onto the absolute queue index */
5722	if (!vf)
5723	queue = adapter->rx_ring[ring]->reg_idx;
5724	else
5725	queue = ((vf - 1) *
5726	adapter->num_rx_queues_per_pool) + ring;
5727	}
5728
5729	ixgbe_fdir_write_perfect_filter_82599(hw,
5730	input: &filter->filter, soft_id: filter->sw_idx, queue);
5731	}
5732
5733	spin_unlock(lock: &adapter->fdir_perfect_lock);
5734	}
5735
5736	/**
5737	* ixgbe_clean_rx_ring - Free Rx Buffers per Queue
5738	* @rx_ring: ring to free buffers from
5739	**/
5740	static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
5741	{
5742	u16 i = rx_ring->next_to_clean;
5743	struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i];
5744
5745	if (rx_ring->xsk_pool) {
5746	ixgbe_xsk_clean_rx_ring(rx_ring);
5747	goto skip_free;
5748	}
5749
5750	/* Free all the Rx ring sk_buffs */
5751	while (i != rx_ring->next_to_alloc) {
5752	if (rx_buffer->skb) {
5753	struct sk_buff *skb = rx_buffer->skb;
5754	if (IXGBE_CB(skb)->page_released)
5755	dma_unmap_page_attrs(dev: rx_ring->dev,
5756	IXGBE_CB(skb)->dma,
5757	ixgbe_rx_pg_size(rx_ring),
5758	dir: DMA_FROM_DEVICE,
5759	IXGBE_RX_DMA_ATTR);
5760	dev_kfree_skb(skb);
5761	}
5762
5763	/* Invalidate cache lines that may have been written to by
5764	* device so that we avoid corrupting memory.
5765	*/
5766	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
5767	addr: rx_buffer->dma,
5768	offset: rx_buffer->page_offset,
5769	size: ixgbe_rx_bufsz(ring: rx_ring),
5770	dir: DMA_FROM_DEVICE);
5771
5772	/* free resources associated with mapping */
5773	dma_unmap_page_attrs(dev: rx_ring->dev, addr: rx_buffer->dma,
5774	ixgbe_rx_pg_size(rx_ring),
5775	dir: DMA_FROM_DEVICE,
5776	IXGBE_RX_DMA_ATTR);
5777	__page_frag_cache_drain(page: rx_buffer->page,
5778	count: rx_buffer->pagecnt_bias);
5779
5780	i++;
5781	rx_buffer++;
5782	if (i == rx_ring->count) {
5783	i = 0;
5784	rx_buffer = rx_ring->rx_buffer_info;
5785	}
5786	}
5787
5788	skip_free:
5789	rx_ring->next_to_alloc = 0;
5790	rx_ring->next_to_clean = 0;
5791	rx_ring->next_to_use = 0;
5792	}
5793
5794	static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter,
5795	struct ixgbe_fwd_adapter *accel)
5796	{
5797	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
5798	int num_tc = netdev_get_num_tc(dev: adapter->netdev);
5799	struct net_device *vdev = accel->netdev;
5800	int i, baseq, err;
5801
5802	baseq = accel->pool * adapter->num_rx_queues_per_pool;
5803	netdev_dbg(vdev, "pool %i:%i queues %i:%i\n",
5804	accel->pool, adapter->num_rx_pools,
5805	baseq, baseq + adapter->num_rx_queues_per_pool);
5806
5807	accel->rx_base_queue = baseq;
5808	accel->tx_base_queue = baseq;
5809
5810	/* record configuration for macvlan interface in vdev */
5811	for (i = 0; i < num_tc; i++)
5812	netdev_bind_sb_channel_queue(dev: adapter->netdev, sb_dev: vdev,
5813	tc: i, count: rss_i, offset: baseq + (rss_i * i));
5814
5815	for (i = 0; i < adapter->num_rx_queues_per_pool; i++)
5816	adapter->rx_ring[baseq + i]->netdev = vdev;
5817
5818	/* Guarantee all rings are updated before we update the
5819	* MAC address filter.
5820	*/
5821	wmb();
5822
5823	/* ixgbe_add_mac_filter will return an index if it succeeds, so we
5824	* need to only treat it as an error value if it is negative.
5825	*/
5826	err = ixgbe_add_mac_filter(adapter, addr: vdev->dev_addr,
5827	VMDQ_P(accel->pool));
5828	if (err >= 0)
5829	return 0;
5830
5831	/* if we cannot add the MAC rule then disable the offload */
5832	macvlan_release_l2fw_offload(dev: vdev);
5833
5834	for (i = 0; i < adapter->num_rx_queues_per_pool; i++)
5835	adapter->rx_ring[baseq + i]->netdev = NULL;
5836
5837	netdev_err(dev: vdev, format: "L2FW offload disabled due to L2 filter error\n");
5838
5839	/* unbind the queues and drop the subordinate channel config */
5840	netdev_unbind_sb_channel(dev: adapter->netdev, sb_dev: vdev);
5841	netdev_set_sb_channel(dev: vdev, channel: 0);
5842
5843	clear_bit(nr: accel->pool, addr: adapter->fwd_bitmask);
5844	kfree(objp: accel);
5845
5846	return err;
5847	}
5848
5849	static int ixgbe_macvlan_up(struct net_device *vdev,
5850	struct netdev_nested_priv *priv)
5851	{
5852	struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data;
5853	struct ixgbe_fwd_adapter *accel;
5854
5855	if (!netif_is_macvlan(dev: vdev))
5856	return 0;
5857
5858	accel = macvlan_accel_priv(dev: vdev);
5859	if (!accel)
5860	return 0;
5861
5862	ixgbe_fwd_ring_up(adapter, accel);
5863
5864	return 0;
5865	}
5866
5867	static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter)
5868	{
5869	struct netdev_nested_priv priv = {
5870	.data = (void *)adapter,
5871	};
5872
5873	netdev_walk_all_upper_dev_rcu(dev: adapter->netdev,
5874	fn: ixgbe_macvlan_up, priv: &priv);
5875	}
5876
5877	static void ixgbe_configure(struct ixgbe_adapter *adapter)
5878	{
5879	struct ixgbe_hw *hw = &adapter->hw;
5880
5881	ixgbe_configure_pb(adapter);
5882	#ifdef CONFIG_IXGBE_DCB
5883	ixgbe_configure_dcb(adapter);
5884	#endif
5885	/*
5886	* We must restore virtualization before VLANs or else
5887	* the VLVF registers will not be populated
5888	*/
5889	ixgbe_configure_virtualization(adapter);
5890
5891	ixgbe_set_rx_mode(netdev: adapter->netdev);
5892	ixgbe_restore_vlan(adapter);
5893	ixgbe_ipsec_restore(adapter);
5894
5895	switch (hw->mac.type) {
5896	case ixgbe_mac_82599EB:
5897	case ixgbe_mac_X540:
5898	hw->mac.ops.disable_rx_buff(hw);
5899	break;
5900	default:
5901	break;
5902	}
5903
5904	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
5905	ixgbe_init_fdir_signature_82599(hw: &adapter->hw,
5906	fdirctrl: adapter->fdir_pballoc);
5907	} else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
5908	ixgbe_init_fdir_perfect_82599(hw: &adapter->hw,
5909	fdirctrl: adapter->fdir_pballoc);
5910	ixgbe_fdir_filter_restore(adapter);
5911	}
5912
5913	switch (hw->mac.type) {
5914	case ixgbe_mac_82599EB:
5915	case ixgbe_mac_X540:
5916	hw->mac.ops.enable_rx_buff(hw);
5917	break;
5918	default:
5919	break;
5920	}
5921
5922	#ifdef CONFIG_IXGBE_DCA
5923	/* configure DCA */
5924	if (adapter->flags & IXGBE_FLAG_DCA_CAPABLE)
5925	ixgbe_setup_dca(adapter);
5926	#endif /* CONFIG_IXGBE_DCA */
5927
5928	#ifdef IXGBE_FCOE
5929	/* configure FCoE L2 filters, redirection table, and Rx control */
5930	ixgbe_configure_fcoe(adapter);
5931
5932	#endif /* IXGBE_FCOE */
5933	ixgbe_configure_tx(adapter);
5934	ixgbe_configure_rx(adapter);
5935	ixgbe_configure_dfwd(adapter);
5936	}
5937
5938	/**
5939	* ixgbe_enable_link_status_events - enable link status events
5940	* @adapter: pointer to the adapter structure
5941	* @mask: event mask to be set
5942	*
5943	* Enables link status events by invoking ixgbe_configure_lse()
5944	*
5945	* Return: the exit code of the operation.
5946	*/
5947	static int ixgbe_enable_link_status_events(struct ixgbe_adapter *adapter,
5948	u16 mask)
5949	{
5950	int err;
5951
5952	err = ixgbe_configure_lse(hw: &adapter->hw, activate: true, mask);
5953	if (err)
5954	return err;
5955
5956	adapter->lse_mask = mask;
5957	return 0;
5958	}
5959
5960	/**
5961	* ixgbe_disable_link_status_events - disable link status events
5962	* @adapter: pointer to the adapter structure
5963	*
5964	* Disables link status events by invoking ixgbe_configure_lse()
5965	*
5966	* Return: the exit code of the operation.
5967	*/
5968	static int ixgbe_disable_link_status_events(struct ixgbe_adapter *adapter)
5969	{
5970	int err;
5971
5972	err = ixgbe_configure_lse(hw: &adapter->hw, activate: false, mask: adapter->lse_mask);
5973	if (err)
5974	return err;
5975
5976	adapter->lse_mask = 0;
5977	return 0;
5978	}
5979
5980	/**
5981	* ixgbe_sfp_link_config - set up SFP+ link
5982	* @adapter: pointer to private adapter struct
5983	**/
5984	static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
5985	{
5986	/*
5987	* We are assuming the worst case scenario here, and that
5988	* is that an SFP was inserted/removed after the reset
5989	* but before SFP detection was enabled. As such the best
5990	* solution is to just start searching as soon as we start
5991	*/
5992	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
5993	adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
5994
5995	adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
5996	adapter->sfp_poll_time = 0;
5997	}
5998
5999	/**
6000	* ixgbe_non_sfp_link_config - set up non-SFP+ link
6001	* @hw: pointer to private hardware struct
6002	*
6003	* Configure non-SFP link.
6004	*
6005	* Return: 0 on success, negative on failure
6006	**/
6007	static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
6008	{
6009	struct ixgbe_adapter *adapter = container_of(hw, struct ixgbe_adapter,
6010	hw);
6011	u16 mask = ~((u16)(IXGBE_ACI_LINK_EVENT_UPDOWN |
6012	IXGBE_ACI_LINK_EVENT_MEDIA_NA |
6013	IXGBE_ACI_LINK_EVENT_MODULE_QUAL_FAIL |
6014	IXGBE_ACI_LINK_EVENT_PHY_FW_LOAD_FAIL));
6015	bool autoneg, link_up = false;
6016	int ret = -EIO;
6017	u32 speed;
6018
6019	if (hw->mac.ops.check_link)
6020	ret = hw->mac.ops.check_link(hw, &speed, &link_up, false);
6021
6022	if (ret)
6023	return ret;
6024
6025	speed = hw->phy.autoneg_advertised;
6026	if (!speed && hw->mac.ops.get_link_capabilities) {
6027	ret = hw->mac.ops.get_link_capabilities(hw, &speed,
6028	&autoneg);
6029	/* remove NBASE-T speeds from default autonegotiation
6030	* to accommodate broken network switches in the field
6031	* which cannot cope with advertised NBASE-T speeds
6032	*/
6033	speed &= ~(IXGBE_LINK_SPEED_5GB_FULL |
6034	IXGBE_LINK_SPEED_2_5GB_FULL);
6035	}
6036
6037	if (ret)
6038	return ret;
6039
6040	if (hw->mac.ops.setup_link) {
6041	if (adapter->hw.mac.type == ixgbe_mac_e610) {
6042	ret = ixgbe_enable_link_status_events(adapter, mask);
6043	if (ret)
6044	return ret;
6045	}
6046	ret = hw->mac.ops.setup_link(hw, speed, link_up);
6047	}
6048
6049	return ret;
6050	}
6051
6052	/**
6053	* ixgbe_check_media_subtask - check for media
6054	* @adapter: pointer to adapter structure
6055	*
6056	* If media is available then initialize PHY user configuration. Configure the
6057	* PHY if the interface is up.
6058	*/
6059	static void ixgbe_check_media_subtask(struct ixgbe_adapter *adapter)
6060	{
6061	struct ixgbe_hw *hw = &adapter->hw;
6062
6063	/* No need to check for media if it's already present */
6064	if (!(adapter->flags2 & IXGBE_FLAG2_NO_MEDIA))
6065	return;
6066
6067	/* Refresh link info and check if media is present */
6068	if (ixgbe_update_link_info(hw))
6069	return;
6070
6071	ixgbe_check_link_cfg_err(adapter, link_cfg_err: hw->link.link_info.link_cfg_err);
6072
6073	if (hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE) {
6074	/* PHY settings are reset on media insertion, reconfigure
6075	* PHY to preserve settings.
6076	*/
6077	if (!(ixgbe_non_sfp_link_config(hw: &adapter->hw)))
6078	adapter->flags2 &= ~IXGBE_FLAG2_NO_MEDIA;
6079
6080	/* A Link Status Event will be generated; the event handler
6081	* will complete bringing the interface up
6082	*/
6083	}
6084	}
6085
6086	/**
6087	* ixgbe_clear_vf_stats_counters - Clear out VF stats after reset
6088	* @adapter: board private structure
6089	*
6090	* On a reset we need to clear out the VF stats or accounting gets
6091	* messed up because they're not clear on read.
6092	**/
6093	static void ixgbe_clear_vf_stats_counters(struct ixgbe_adapter *adapter)
6094	{
6095	struct ixgbe_hw *hw = &adapter->hw;
6096	int i;
6097
6098	for (i = 0; i < adapter->num_vfs; i++) {
6099	adapter->vfinfo[i].last_vfstats.gprc =
6100	IXGBE_READ_REG(hw, IXGBE_PVFGPRC(i));
6101	adapter->vfinfo[i].saved_rst_vfstats.gprc +=
6102	adapter->vfinfo[i].vfstats.gprc;
6103	adapter->vfinfo[i].vfstats.gprc = 0;
6104	adapter->vfinfo[i].last_vfstats.gptc =
6105	IXGBE_READ_REG(hw, IXGBE_PVFGPTC(i));
6106	adapter->vfinfo[i].saved_rst_vfstats.gptc +=
6107	adapter->vfinfo[i].vfstats.gptc;
6108	adapter->vfinfo[i].vfstats.gptc = 0;
6109	adapter->vfinfo[i].last_vfstats.gorc =
6110	IXGBE_READ_REG(hw, IXGBE_PVFGORC_LSB(i));
6111	adapter->vfinfo[i].saved_rst_vfstats.gorc +=
6112	adapter->vfinfo[i].vfstats.gorc;
6113	adapter->vfinfo[i].vfstats.gorc = 0;
6114	adapter->vfinfo[i].last_vfstats.gotc =
6115	IXGBE_READ_REG(hw, IXGBE_PVFGOTC_LSB(i));
6116	adapter->vfinfo[i].saved_rst_vfstats.gotc +=
6117	adapter->vfinfo[i].vfstats.gotc;
6118	adapter->vfinfo[i].vfstats.gotc = 0;
6119	adapter->vfinfo[i].last_vfstats.mprc =
6120	IXGBE_READ_REG(hw, IXGBE_PVFMPRC(i));
6121	adapter->vfinfo[i].saved_rst_vfstats.mprc +=
6122	adapter->vfinfo[i].vfstats.mprc;
6123	adapter->vfinfo[i].vfstats.mprc = 0;
6124	}
6125	}
6126
6127	static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
6128	{
6129	struct ixgbe_hw *hw = &adapter->hw;
6130	u32 gpie = 0;
6131
6132	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
6133	gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
6134	IXGBE_GPIE_OCD;
6135	gpie |= IXGBE_GPIE_EIAME;
6136	/*
6137	* use EIAM to auto-mask when MSI-X interrupt is asserted
6138	* this saves a register write for every interrupt
6139	*/
6140	switch (hw->mac.type) {
6141	case ixgbe_mac_82598EB:
6142	IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
6143	break;
6144	case ixgbe_mac_82599EB:
6145	case ixgbe_mac_X540:
6146	case ixgbe_mac_X550:
6147	case ixgbe_mac_X550EM_x:
6148	case ixgbe_mac_x550em_a:
6149	case ixgbe_mac_e610:
6150	default:
6151	IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
6152	IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
6153	break;
6154	}
6155	} else {
6156	/* legacy interrupts, use EIAM to auto-mask when reading EICR,
6157	* specifically only auto mask tx and rx interrupts */
6158	IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
6159	}
6160
6161	/* XXX: to interrupt immediately for EICS writes, enable this */
6162	/* gpie |= IXGBE_GPIE_EIMEN; */
6163
6164	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
6165	gpie &= ~IXGBE_GPIE_VTMODE_MASK;
6166
6167	switch (adapter->ring_feature[RING_F_VMDQ].mask) {
6168	case IXGBE_82599_VMDQ_8Q_MASK:
6169	gpie |= IXGBE_GPIE_VTMODE_16;
6170	break;
6171	case IXGBE_82599_VMDQ_4Q_MASK:
6172	gpie |= IXGBE_GPIE_VTMODE_32;
6173	break;
6174	default:
6175	gpie |= IXGBE_GPIE_VTMODE_64;
6176	break;
6177	}
6178	}
6179
6180	/* Enable Thermal over heat sensor interrupt */
6181	if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {
6182	switch (adapter->hw.mac.type) {
6183	case ixgbe_mac_82599EB:
6184	gpie |= IXGBE_SDP0_GPIEN_8259X;
6185	break;
6186	default:
6187	break;
6188	}
6189	}
6190
6191	/* Enable fan failure interrupt */
6192	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
6193	gpie |= IXGBE_SDP1_GPIEN(hw);
6194
6195	switch (hw->mac.type) {
6196	case ixgbe_mac_82599EB:
6197	gpie |= IXGBE_SDP1_GPIEN_8259X | IXGBE_SDP2_GPIEN_8259X;
6198	break;
6199	case ixgbe_mac_X550EM_x:
6200	case ixgbe_mac_x550em_a:
6201	gpie |= IXGBE_SDP0_GPIEN_X540;
6202	break;
6203	default:
6204	break;
6205	}
6206
6207	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
6208	}
6209
6210	static void ixgbe_up_complete(struct ixgbe_adapter *adapter)
6211	{
6212	struct ixgbe_hw *hw = &adapter->hw;
6213	int err;
6214	u32 ctrl_ext;
6215
6216	ixgbe_get_hw_control(adapter);
6217	ixgbe_setup_gpie(adapter);
6218
6219	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
6220	ixgbe_configure_msix(adapter);
6221	else
6222	ixgbe_configure_msi_and_legacy(adapter);
6223
6224	/* enable the optics for 82599 SFP+ fiber */
6225	if (hw->mac.ops.enable_tx_laser)
6226	hw->mac.ops.enable_tx_laser(hw);
6227
6228	if (hw->phy.ops.set_phy_power)
6229	hw->phy.ops.set_phy_power(hw, true);
6230
6231	smp_mb__before_atomic();
6232	clear_bit(nr: __IXGBE_DOWN, addr: &adapter->state);
6233	ixgbe_napi_enable_all(adapter);
6234
6235	if (ixgbe_is_sfp(hw)) {
6236	ixgbe_sfp_link_config(adapter);
6237	} else {
6238	err = ixgbe_non_sfp_link_config(hw);
6239	if (err)
6240	e_err(probe, "link_config FAILED %d\n", err);
6241	}
6242
6243	/* clear any pending interrupts, may auto mask */
6244	IXGBE_READ_REG(hw, IXGBE_EICR);
6245	ixgbe_irq_enable(adapter, queues: true, flush: true);
6246
6247	/*
6248	* If this adapter has a fan, check to see if we had a failure
6249	* before we enabled the interrupt.
6250	*/
6251	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
6252	u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
6253	if (esdp & IXGBE_ESDP_SDP1)
6254	e_crit(drv, "Fan has stopped, replace the adapter\n");
6255	}
6256
6257	/* bring the link up in the watchdog, this could race with our first
6258	* link up interrupt but shouldn't be a problem */
6259	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
6260	adapter->link_check_timeout = jiffies;
6261	mod_timer(timer: &adapter->service_timer, expires: jiffies);
6262
6263	ixgbe_clear_vf_stats_counters(adapter);
6264	/* Set PF Reset Done bit so PF/VF Mail Ops can work */
6265	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
6266	ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
6267	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
6268
6269	/* update setting rx tx for all active vfs */
6270	ixgbe_set_all_vfs(adapter);
6271	}
6272
6273	void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
6274	{
6275	/* put off any impending NetWatchDogTimeout */
6276	netif_trans_update(dev: adapter->netdev);
6277
6278	while (test_and_set_bit(nr: __IXGBE_RESETTING, addr: &adapter->state))
6279	usleep_range(min: 1000, max: 2000);
6280	if (adapter->hw.phy.type == ixgbe_phy_fw)
6281	ixgbe_watchdog_link_is_down(adapter);
6282	ixgbe_down(adapter);
6283	/*
6284	* If SR-IOV enabled then wait a bit before bringing the adapter
6285	* back up to give the VFs time to respond to the reset. The
6286	* two second wait is based upon the watchdog timer cycle in
6287	* the VF driver.
6288	*/
6289	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
6290	msleep(msecs: 2000);
6291	ixgbe_up(adapter);
6292	clear_bit(nr: __IXGBE_RESETTING, addr: &adapter->state);
6293	}
6294
6295	void ixgbe_up(struct ixgbe_adapter *adapter)
6296	{
6297	/* hardware has been reset, we need to reload some things */
6298	ixgbe_configure(adapter);
6299
6300	ixgbe_up_complete(adapter);
6301	}
6302
6303	static unsigned long ixgbe_get_completion_timeout(struct ixgbe_adapter *adapter)
6304	{
6305	u16 devctl2;
6306
6307	pcie_capability_read_word(dev: adapter->pdev, PCI_EXP_DEVCTL2, val: &devctl2);
6308
6309	switch (devctl2 & IXGBE_PCIDEVCTRL2_TIMEO_MASK) {
6310	case IXGBE_PCIDEVCTRL2_17_34s:
6311	case IXGBE_PCIDEVCTRL2_4_8s:
6312	/* For now we cap the upper limit on delay to 2 seconds
6313	* as we end up going up to 34 seconds of delay in worst
6314	* case timeout value.
6315	*/
6316	case IXGBE_PCIDEVCTRL2_1_2s:
6317	return 2000000ul; /* 2.0 s */
6318	case IXGBE_PCIDEVCTRL2_260_520ms:
6319	return 520000ul; /* 520 ms */
6320	case IXGBE_PCIDEVCTRL2_65_130ms:
6321	return 130000ul; /* 130 ms */
6322	case IXGBE_PCIDEVCTRL2_16_32ms:
6323	return 32000ul; /* 32 ms */
6324	case IXGBE_PCIDEVCTRL2_1_2ms:
6325	return 2000ul; /* 2 ms */
6326	case IXGBE_PCIDEVCTRL2_50_100us:
6327	return 100ul; /* 100 us */
6328	case IXGBE_PCIDEVCTRL2_16_32ms_def:
6329	return 32000ul; /* 32 ms */
6330	default:
6331	break;
6332	}
6333
6334	/* We shouldn't need to hit this path, but just in case default as
6335	* though completion timeout is not supported and support 32ms.
6336	*/
6337	return 32000ul;
6338	}
6339
6340	void ixgbe_disable_rx(struct ixgbe_adapter *adapter)
6341	{
6342	unsigned long wait_delay, delay_interval;
6343	struct ixgbe_hw *hw = &adapter->hw;
6344	int i, wait_loop;
6345	u32 rxdctl;
6346
6347	/* disable receives */
6348	hw->mac.ops.disable_rx(hw);
6349
6350	if (ixgbe_removed(addr: hw->hw_addr))
6351	return;
6352
6353	/* disable all enabled Rx queues */
6354	for (i = 0; i < adapter->num_rx_queues; i++) {
6355	struct ixgbe_ring *ring = adapter->rx_ring[i];
6356	u8 reg_idx = ring->reg_idx;
6357
6358	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
6359	rxdctl &= ~IXGBE_RXDCTL_ENABLE;
6360	rxdctl |= IXGBE_RXDCTL_SWFLSH;
6361
6362	/* write value back with RXDCTL.ENABLE bit cleared */
6363	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
6364	}
6365
6366	/* RXDCTL.EN may not change on 82598 if link is down, so skip it */
6367	if (hw->mac.type == ixgbe_mac_82598EB &&
6368	!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
6369	return;
6370
6371	/* Determine our minimum delay interval. We will increase this value
6372	* with each subsequent test. This way if the device returns quickly
6373	* we should spend as little time as possible waiting, however as
6374	* the time increases we will wait for larger periods of time.
6375	*
6376	* The trick here is that we increase the interval using the
6377	* following pattern: 1x 3x 5x 7x 9x 11x 13x 15x 17x 19x. The result
6378	* of that wait is that it totals up to 100x whatever interval we
6379	* choose. Since our minimum wait is 100us we can just divide the
6380	* total timeout by 100 to get our minimum delay interval.
6381	*/
6382	delay_interval = ixgbe_get_completion_timeout(adapter) / 100;
6383
6384	wait_loop = IXGBE_MAX_RX_DESC_POLL;
6385	wait_delay = delay_interval;
6386
6387	while (wait_loop--) {
6388	usleep_range(min: wait_delay, max: wait_delay + 10);
6389	wait_delay += delay_interval * 2;
6390	rxdctl = 0;
6391
6392	/* OR together the reading of all the active RXDCTL registers,
6393	* and then test the result. We need the disable to complete
6394	* before we start freeing the memory and invalidating the
6395	* DMA mappings.
6396	*/
6397	for (i = 0; i < adapter->num_rx_queues; i++) {
6398	struct ixgbe_ring *ring = adapter->rx_ring[i];
6399	u8 reg_idx = ring->reg_idx;
6400
6401	rxdctl |= IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
6402	}
6403
6404	if (!(rxdctl & IXGBE_RXDCTL_ENABLE))
6405	return;
6406	}
6407
6408	e_err(drv,
6409	"RXDCTL.ENABLE for one or more queues not cleared within the polling period\n");
6410	}
6411
6412	void ixgbe_disable_tx(struct ixgbe_adapter *adapter)
6413	{
6414	unsigned long wait_delay, delay_interval;
6415	struct ixgbe_hw *hw = &adapter->hw;
6416	int i, wait_loop;
6417	u32 txdctl;
6418
6419	if (ixgbe_removed(addr: hw->hw_addr))
6420	return;
6421
6422	/* disable all enabled Tx queues */
6423	for (i = 0; i < adapter->num_tx_queues; i++) {
6424	struct ixgbe_ring *ring = adapter->tx_ring[i];
6425	u8 reg_idx = ring->reg_idx;
6426
6427	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
6428	}
6429
6430	/* disable all enabled XDP Tx queues */
6431	for (i = 0; i < adapter->num_xdp_queues; i++) {
6432	struct ixgbe_ring *ring = adapter->xdp_ring[i];
6433	u8 reg_idx = ring->reg_idx;
6434
6435	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
6436	}
6437
6438	/* If the link is not up there shouldn't be much in the way of
6439	* pending transactions. Those that are left will be flushed out
6440	* when the reset logic goes through the flush sequence to clean out
6441	* the pending Tx transactions.
6442	*/
6443	if (!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
6444	goto dma_engine_disable;
6445
6446	/* Determine our minimum delay interval. We will increase this value
6447	* with each subsequent test. This way if the device returns quickly
6448	* we should spend as little time as possible waiting, however as
6449	* the time increases we will wait for larger periods of time.
6450	*
6451	* The trick here is that we increase the interval using the
6452	* following pattern: 1x 3x 5x 7x 9x 11x 13x 15x 17x 19x. The result
6453	* of that wait is that it totals up to 100x whatever interval we
6454	* choose. Since our minimum wait is 100us we can just divide the
6455	* total timeout by 100 to get our minimum delay interval.
6456	*/
6457	delay_interval = ixgbe_get_completion_timeout(adapter) / 100;
6458
6459	wait_loop = IXGBE_MAX_RX_DESC_POLL;
6460	wait_delay = delay_interval;
6461
6462	while (wait_loop--) {
6463	usleep_range(min: wait_delay, max: wait_delay + 10);
6464	wait_delay += delay_interval * 2;
6465	txdctl = 0;
6466
6467	/* OR together the reading of all the active TXDCTL registers,
6468	* and then test the result. We need the disable to complete
6469	* before we start freeing the memory and invalidating the
6470	* DMA mappings.
6471	*/
6472	for (i = 0; i < adapter->num_tx_queues; i++) {
6473	struct ixgbe_ring *ring = adapter->tx_ring[i];
6474	u8 reg_idx = ring->reg_idx;
6475
6476	txdctl |= IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
6477	}
6478	for (i = 0; i < adapter->num_xdp_queues; i++) {
6479	struct ixgbe_ring *ring = adapter->xdp_ring[i];
6480	u8 reg_idx = ring->reg_idx;
6481
6482	txdctl |= IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
6483	}
6484
6485	if (!(txdctl & IXGBE_TXDCTL_ENABLE))
6486	goto dma_engine_disable;
6487	}
6488
6489	e_err(drv,
6490	"TXDCTL.ENABLE for one or more queues not cleared within the polling period\n");
6491
6492	dma_engine_disable:
6493	/* Disable the Tx DMA engine on 82599 and later MAC */
6494	switch (hw->mac.type) {
6495	case ixgbe_mac_82599EB:
6496	case ixgbe_mac_X540:
6497	case ixgbe_mac_X550:
6498	case ixgbe_mac_X550EM_x:
6499	case ixgbe_mac_x550em_a:
6500	case ixgbe_mac_e610:
6501	IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
6502	(IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
6503	~IXGBE_DMATXCTL_TE));
6504	fallthrough;
6505	default:
6506	break;
6507	}
6508	}
6509
6510	void ixgbe_reset(struct ixgbe_adapter *adapter)
6511	{
6512	struct ixgbe_hw *hw = &adapter->hw;
6513	struct net_device *netdev = adapter->netdev;
6514	int err;
6515
6516	if (ixgbe_removed(addr: hw->hw_addr))
6517	return;
6518	/* lock SFP init bit to prevent race conditions with the watchdog */
6519	while (test_and_set_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state))
6520	usleep_range(min: 1000, max: 2000);
6521
6522	/* clear all SFP and link config related flags while holding SFP_INIT */
6523	adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
6524	IXGBE_FLAG2_SFP_NEEDS_RESET);
6525	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
6526
6527	err = hw->mac.ops.init_hw(hw);
6528	switch (err) {
6529	case 0:
6530	case -ENOENT:
6531	case -EOPNOTSUPP:
6532	break;
6533	case -EALREADY:
6534	e_dev_err("primary disable timed out\n");
6535	break;
6536	case -EACCES:
6537	/* We are running on a pre-production device, log a warning */
6538	e_dev_warn("This device is a pre-production adapter/LOM. "
6539	"Please be aware there may be issues associated with "
6540	"your hardware. If you are experiencing problems "
6541	"please contact your Intel or hardware "
6542	"representative who provided you with this "
6543	"hardware.\n");
6544	break;
6545	default:
6546	e_dev_err("Hardware Error: %d\n", err);
6547	}
6548
6549	clear_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state);
6550
6551	/* flush entries out of MAC table */
6552	ixgbe_flush_sw_mac_table(adapter);
6553	__dev_uc_unsync(dev: netdev, NULL);
6554
6555	/* do not flush user set addresses */
6556	ixgbe_mac_set_default_filter(adapter);
6557
6558	/* update SAN MAC vmdq pool selection */
6559	if (hw->mac.san_mac_rar_index)
6560	hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
6561
6562	if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
6563	ixgbe_ptp_reset(adapter);
6564
6565	if (hw->phy.ops.set_phy_power) {
6566	if (!netif_running(dev: adapter->netdev) && !adapter->wol)
6567	hw->phy.ops.set_phy_power(hw, false);
6568	else
6569	hw->phy.ops.set_phy_power(hw, true);
6570	}
6571	}
6572
6573	/**
6574	* ixgbe_clean_tx_ring - Free Tx Buffers
6575	* @tx_ring: ring to be cleaned
6576	**/
6577	static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
6578	{
6579	u16 i = tx_ring->next_to_clean;
6580	struct ixgbe_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
6581
6582	if (tx_ring->xsk_pool) {
6583	ixgbe_xsk_clean_tx_ring(tx_ring);
6584	goto out;
6585	}
6586
6587	while (i != tx_ring->next_to_use) {
6588	union ixgbe_adv_tx_desc *eop_desc, *tx_desc;
6589
6590	/* Free all the Tx ring sk_buffs */
6591	if (ring_is_xdp(tx_ring))
6592	xdp_return_frame(xdpf: tx_buffer->xdpf);
6593	else
6594	dev_kfree_skb_any(skb: tx_buffer->skb);
6595
6596	/* unmap skb header data */
6597	dma_unmap_single(tx_ring->dev,
6598	dma_unmap_addr(tx_buffer, dma),
6599	dma_unmap_len(tx_buffer, len),
6600	DMA_TO_DEVICE);
6601
6602	/* check for eop_desc to determine the end of the packet */
6603	eop_desc = tx_buffer->next_to_watch;
6604	tx_desc = IXGBE_TX_DESC(tx_ring, i);
6605
6606	/* unmap remaining buffers */
6607	while (tx_desc != eop_desc) {
6608	tx_buffer++;
6609	tx_desc++;
6610	i++;
6611	if (unlikely(i == tx_ring->count)) {
6612	i = 0;
6613	tx_buffer = tx_ring->tx_buffer_info;
6614	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
6615	}
6616
6617	/* unmap any remaining paged data */
6618	if (dma_unmap_len(tx_buffer, len))
6619	dma_unmap_page(tx_ring->dev,
6620	dma_unmap_addr(tx_buffer, dma),
6621	dma_unmap_len(tx_buffer, len),
6622	DMA_TO_DEVICE);
6623	}
6624
6625	/* move us one more past the eop_desc for start of next pkt */
6626	tx_buffer++;
6627	i++;
6628	if (unlikely(i == tx_ring->count)) {
6629	i = 0;
6630	tx_buffer = tx_ring->tx_buffer_info;
6631	}
6632	}
6633
6634	/* reset BQL for queue */
6635	if (!ring_is_xdp(tx_ring))
6636	netdev_tx_reset_queue(q: txring_txq(ring: tx_ring));
6637
6638	out:
6639	/* reset next_to_use and next_to_clean */
6640	tx_ring->next_to_use = 0;
6641	tx_ring->next_to_clean = 0;
6642	}
6643
6644	/**
6645	* ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
6646	* @adapter: board private structure
6647	**/
6648	static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
6649	{
6650	int i;
6651
6652	for (i = 0; i < adapter->num_rx_queues; i++)
6653	ixgbe_clean_rx_ring(rx_ring: adapter->rx_ring[i]);
6654	}
6655
6656	/**
6657	* ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
6658	* @adapter: board private structure
6659	**/
6660	static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
6661	{
6662	int i;
6663
6664	for (i = 0; i < adapter->num_tx_queues; i++)
6665	ixgbe_clean_tx_ring(tx_ring: adapter->tx_ring[i]);
6666	for (i = 0; i < adapter->num_xdp_queues; i++)
6667	ixgbe_clean_tx_ring(tx_ring: adapter->xdp_ring[i]);
6668	}
6669
6670	static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
6671	{
6672	struct hlist_node *node2;
6673	struct ixgbe_fdir_filter *filter;
6674
6675	spin_lock(lock: &adapter->fdir_perfect_lock);
6676
6677	hlist_for_each_entry_safe(filter, node2,
6678	&adapter->fdir_filter_list, fdir_node) {
6679	hlist_del(n: &filter->fdir_node);
6680	kfree(objp: filter);
6681	}
6682	adapter->fdir_filter_count = 0;
6683
6684	spin_unlock(lock: &adapter->fdir_perfect_lock);
6685	}
6686
6687	void ixgbe_down(struct ixgbe_adapter *adapter)
6688	{
6689	struct net_device *netdev = adapter->netdev;
6690	struct ixgbe_hw *hw = &adapter->hw;
6691	int i;
6692
6693	/* signal that we are down to the interrupt handler */
6694	if (test_and_set_bit(nr: __IXGBE_DOWN, addr: &adapter->state))
6695	return; /* do nothing if already down */
6696
6697	/* Shut off incoming Tx traffic */
6698	netif_tx_stop_all_queues(dev: netdev);
6699
6700	/* call carrier off first to avoid false dev_watchdog timeouts */
6701	netif_carrier_off(dev: netdev);
6702	netif_tx_disable(dev: netdev);
6703
6704	/* Disable Rx */
6705	ixgbe_disable_rx(adapter);
6706
6707	/* synchronize_rcu() needed for pending XDP buffers to drain */
6708	if (adapter->xdp_ring[0])
6709	synchronize_rcu();
6710
6711	ixgbe_irq_disable(adapter);
6712
6713	ixgbe_napi_disable_all(adapter);
6714
6715	clear_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
6716	adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
6717	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
6718
6719	timer_delete_sync(timer: &adapter->service_timer);
6720
6721	if (adapter->num_vfs) {
6722	/* Clear EITR Select mapping */
6723	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
6724
6725	/* Mark all the VFs as inactive */
6726	for (i = 0 ; i < adapter->num_vfs; i++)
6727	adapter->vfinfo[i].clear_to_send = false;
6728
6729	/* update setting rx tx for all active vfs */
6730	ixgbe_set_all_vfs(adapter);
6731	}
6732
6733	/* disable transmits in the hardware now that interrupts are off */
6734	ixgbe_disable_tx(adapter);
6735
6736	if (!pci_channel_offline(pdev: adapter->pdev))
6737	ixgbe_reset(adapter);
6738
6739	/* power down the optics for 82599 SFP+ fiber */
6740	if (hw->mac.ops.disable_tx_laser)
6741	hw->mac.ops.disable_tx_laser(hw);
6742
6743	ixgbe_clean_all_tx_rings(adapter);
6744	ixgbe_clean_all_rx_rings(adapter);
6745	if (adapter->hw.mac.type == ixgbe_mac_e610)
6746	ixgbe_disable_link_status_events(adapter);
6747	}
6748
6749	/**
6750	* ixgbe_set_eee_capable - helper function to determine EEE support on X550
6751	* @adapter: board private structure
6752	*/
6753	static void ixgbe_set_eee_capable(struct ixgbe_adapter *adapter)
6754	{
6755	struct ixgbe_hw *hw = &adapter->hw;
6756
6757	switch (hw->device_id) {
6758	case IXGBE_DEV_ID_X550EM_A_1G_T:
6759	case IXGBE_DEV_ID_X550EM_A_1G_T_L:
6760	if (!hw->phy.eee_speeds_supported)
6761	break;
6762	adapter->flags2 |= IXGBE_FLAG2_EEE_CAPABLE;
6763	if (!hw->phy.eee_speeds_advertised)
6764	break;
6765	adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED;
6766	break;
6767	default:
6768	adapter->flags2 &= ~IXGBE_FLAG2_EEE_CAPABLE;
6769	adapter->flags2 &= ~IXGBE_FLAG2_EEE_ENABLED;
6770	break;
6771	}
6772	}
6773
6774	/**
6775	* ixgbe_tx_timeout - Respond to a Tx Hang
6776	* @netdev: network interface device structure
6777	* @txqueue: queue number that timed out
6778	**/
6779	static void ixgbe_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
6780	{
6781	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
6782
6783	/* Do the reset outside of interrupt context */
6784	ixgbe_tx_timeout_reset(adapter);
6785	}
6786
6787	#ifdef CONFIG_IXGBE_DCB
6788	static void ixgbe_init_dcb(struct ixgbe_adapter *adapter)
6789	{
6790	struct ixgbe_hw *hw = &adapter->hw;
6791	struct tc_configuration *tc;
6792	int j;
6793
6794	switch (hw->mac.type) {
6795	case ixgbe_mac_82598EB:
6796	case ixgbe_mac_82599EB:
6797	adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;
6798	adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;
6799	break;
6800	case ixgbe_mac_X540:
6801	case ixgbe_mac_X550:
6802	case ixgbe_mac_e610:
6803	adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;
6804	adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;
6805	break;
6806	case ixgbe_mac_X550EM_x:
6807	case ixgbe_mac_x550em_a:
6808	default:
6809	adapter->dcb_cfg.num_tcs.pg_tcs = DEF_TRAFFIC_CLASS;
6810	adapter->dcb_cfg.num_tcs.pfc_tcs = DEF_TRAFFIC_CLASS;
6811	break;
6812	}
6813
6814	/* Configure DCB traffic classes */
6815	for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
6816	tc = &adapter->dcb_cfg.tc_config[j];
6817	tc->path[DCB_TX_CONFIG].bwg_id = 0;
6818	tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
6819	tc->path[DCB_RX_CONFIG].bwg_id = 0;
6820	tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
6821	tc->dcb_pfc = pfc_disabled;
6822	}
6823
6824	/* Initialize default user to priority mapping, UPx->TC0 */
6825	tc = &adapter->dcb_cfg.tc_config[0];
6826	tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
6827	tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
6828
6829	adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
6830	adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
6831	adapter->dcb_cfg.pfc_mode_enable = false;
6832	adapter->dcb_set_bitmap = 0x00;
6833	if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE)
6834	adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
6835	memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg,
6836	sizeof(adapter->temp_dcb_cfg));
6837	}
6838	#endif
6839
6840	/**
6841	* ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
6842	* @adapter: board private structure to initialize
6843	* @ii: pointer to ixgbe_info for device
6844	*
6845	* ixgbe_sw_init initializes the Adapter private data structure.
6846	* Fields are initialized based on PCI device information and
6847	* OS network device settings (MTU size).
6848	**/
6849	static int ixgbe_sw_init(struct ixgbe_adapter *adapter,
6850	const struct ixgbe_info *ii)
6851	{
6852	struct ixgbe_hw *hw = &adapter->hw;
6853	struct pci_dev *pdev = adapter->pdev;
6854	unsigned int rss, fdir;
6855	u32 fwsm;
6856	int i;
6857
6858	/* PCI config space info */
6859
6860	hw->vendor_id = pdev->vendor;
6861	hw->device_id = pdev->device;
6862	hw->revision_id = pdev->revision;
6863	hw->subsystem_vendor_id = pdev->subsystem_vendor;
6864	hw->subsystem_device_id = pdev->subsystem_device;
6865
6866	hw->mac.max_link_up_time = IXGBE_LINK_UP_TIME;
6867
6868	/* get_invariants needs the device IDs */
6869	ii->get_invariants(hw);
6870
6871	/* Set common capability flags and settings */
6872	rss = min_t(int, ixgbe_max_rss_indices(adapter), num_online_cpus());
6873	adapter->ring_feature[RING_F_RSS].limit = rss;
6874	adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
6875	adapter->max_q_vectors = MAX_Q_VECTORS_82599;
6876	adapter->atr_sample_rate = 20;
6877	fdir = min_t(int, IXGBE_MAX_FDIR_INDICES, num_online_cpus());
6878	adapter->ring_feature[RING_F_FDIR].limit = fdir;
6879	adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
6880	adapter->ring_feature[RING_F_VMDQ].limit = 1;
6881	#ifdef CONFIG_IXGBE_DCA
6882	adapter->flags |= IXGBE_FLAG_DCA_CAPABLE;
6883	#endif
6884	#ifdef CONFIG_IXGBE_DCB
6885	adapter->flags |= IXGBE_FLAG_DCB_CAPABLE;
6886	adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
6887	#endif
6888	#ifdef IXGBE_FCOE
6889	adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
6890	adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
6891	#ifdef CONFIG_IXGBE_DCB
6892	/* Default traffic class to use for FCoE */
6893	adapter->fcoe.up = IXGBE_FCOE_DEFTC;
6894	#endif /* CONFIG_IXGBE_DCB */
6895	#endif /* IXGBE_FCOE */
6896
6897	/* initialize static ixgbe jump table entries */
6898	adapter->jump_tables[0] = kzalloc(sizeof(*adapter->jump_tables[0]),
6899	GFP_KERNEL);
6900	if (!adapter->jump_tables[0])
6901	return -ENOMEM;
6902	adapter->jump_tables[0]->mat = ixgbe_ipv4_fields;
6903
6904	for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++)
6905	adapter->jump_tables[i] = NULL;
6906
6907	adapter->mac_table = kcalloc(hw->mac.num_rar_entries,
6908	sizeof(struct ixgbe_mac_addr),
6909	GFP_KERNEL);
6910	if (!adapter->mac_table)
6911	return -ENOMEM;
6912
6913	if (ixgbe_init_rss_key(adapter))
6914	return -ENOMEM;
6915
6916	adapter->af_xdp_zc_qps = bitmap_zalloc(IXGBE_MAX_XDP_QS, GFP_KERNEL);
6917	if (!adapter->af_xdp_zc_qps)
6918	return -ENOMEM;
6919
6920	/* Set MAC specific capability flags and exceptions */
6921	switch (hw->mac.type) {
6922	case ixgbe_mac_82598EB:
6923	adapter->flags2 &= ~IXGBE_FLAG2_RSC_CAPABLE;
6924
6925	if (hw->device_id == IXGBE_DEV_ID_82598AT)
6926	adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
6927
6928	adapter->max_q_vectors = MAX_Q_VECTORS_82598;
6929	adapter->ring_feature[RING_F_FDIR].limit = 0;
6930	adapter->atr_sample_rate = 0;
6931	adapter->fdir_pballoc = 0;
6932	#ifdef IXGBE_FCOE
6933	adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
6934	adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
6935	#ifdef CONFIG_IXGBE_DCB
6936	adapter->fcoe.up = 0;
6937	#endif /* IXGBE_DCB */
6938	#endif /* IXGBE_FCOE */
6939	break;
6940	case ixgbe_mac_82599EB:
6941	if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
6942	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
6943	break;
6944	case ixgbe_mac_X540:
6945	fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw));
6946	if (fwsm & IXGBE_FWSM_TS_ENABLED)
6947	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
6948	break;
6949	case ixgbe_mac_x550em_a:
6950	switch (hw->device_id) {
6951	case IXGBE_DEV_ID_X550EM_A_1G_T:
6952	case IXGBE_DEV_ID_X550EM_A_1G_T_L:
6953	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
6954	break;
6955	default:
6956	break;
6957	}
6958	fallthrough;
6959	case ixgbe_mac_X550EM_x:
6960	#ifdef CONFIG_IXGBE_DCB
6961	adapter->flags &= ~IXGBE_FLAG_DCB_CAPABLE;
6962	#endif
6963	#ifdef IXGBE_FCOE
6964	adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
6965	#ifdef CONFIG_IXGBE_DCB
6966	adapter->fcoe.up = 0;
6967	#endif /* IXGBE_DCB */
6968	#endif /* IXGBE_FCOE */
6969	fallthrough;
6970	case ixgbe_mac_X550:
6971	if (hw->mac.type == ixgbe_mac_X550)
6972	adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
6973	#ifdef CONFIG_IXGBE_DCA
6974	adapter->flags &= ~IXGBE_FLAG_DCA_CAPABLE;
6975	#endif
6976	break;
6977	default:
6978	break;
6979	}
6980
6981	/* Make sure the SWFW semaphore is in a valid state */
6982	if (hw->mac.ops.init_swfw_sync)
6983	hw->mac.ops.init_swfw_sync(hw);
6984
6985	if (hw->mac.type == ixgbe_mac_e610)
6986	mutex_init(&hw->aci.lock);
6987
6988	#ifdef IXGBE_FCOE
6989	/* FCoE support exists, always init the FCoE lock */
6990	spin_lock_init(&adapter->fcoe.lock);
6991
6992	#endif
6993	/* n-tuple support exists, always init our spinlock */
6994	spin_lock_init(&adapter->fdir_perfect_lock);
6995
6996	/* init spinlock to avoid concurrency of VF resources */
6997	spin_lock_init(&adapter->vfs_lock);
6998
6999	#ifdef CONFIG_IXGBE_DCB
7000	ixgbe_init_dcb(adapter);
7001	#endif
7002	ixgbe_init_ipsec_offload(adapter);
7003
7004	/* default flow control settings */
7005	hw->fc.requested_mode = ixgbe_fc_full;
7006	hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */
7007	ixgbe_pbthresh_setup(adapter);
7008	hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
7009	hw->fc.send_xon = true;
7010	hw->fc.disable_fc_autoneg = ixgbe_device_supports_autoneg_fc(hw);
7011
7012	#ifdef CONFIG_PCI_IOV
7013	if (max_vfs > 0)
7014	e_dev_warn("Enabling SR-IOV VFs using the max_vfs module parameter is deprecated - please use the pci sysfs interface instead.\n");
7015
7016	/* assign number of SR-IOV VFs */
7017	if (hw->mac.type != ixgbe_mac_82598EB) {
7018	if (max_vfs > IXGBE_MAX_VFS_DRV_LIMIT) {
7019	max_vfs = 0;
7020	e_dev_warn("max_vfs parameter out of range. Not assigning any SR-IOV VFs\n");
7021	}
7022	}
7023	#endif /* CONFIG_PCI_IOV */
7024
7025	/* enable itr by default in dynamic mode */
7026	adapter->rx_itr_setting = 1;
7027	adapter->tx_itr_setting = 1;
7028
7029	/* set default ring sizes */
7030	adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
7031	adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
7032
7033	/* set default work limits */
7034	adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK;
7035
7036	/* initialize eeprom parameters */
7037	if (hw->eeprom.ops.init_params(hw)) {
7038	e_dev_err("EEPROM initialization failed\n");
7039	return -EIO;
7040	}
7041
7042	/* PF holds first pool slot */
7043	set_bit(nr: 0, addr: adapter->fwd_bitmask);
7044	set_bit(nr: __IXGBE_DOWN, addr: &adapter->state);
7045
7046	/* enable locking for XDP_TX if we have more CPUs than queues */
7047	if (nr_cpu_ids > IXGBE_MAX_XDP_QS)
7048	static_branch_enable(&ixgbe_xdp_locking_key);
7049
7050	return 0;
7051	}
7052
7053	/**
7054	* ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
7055	* @tx_ring: tx descriptor ring (for a specific queue) to setup
7056	*
7057	* Return 0 on success, negative on failure
7058	**/
7059	int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
7060	{
7061	struct device *dev = tx_ring->dev;
7062	int orig_node = dev_to_node(dev);
7063	int ring_node = NUMA_NO_NODE;
7064	int size;
7065
7066	size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
7067
7068	if (tx_ring->q_vector)
7069	ring_node = tx_ring->q_vector->numa_node;
7070
7071	tx_ring->tx_buffer_info = vmalloc_node(size, ring_node);
7072	if (!tx_ring->tx_buffer_info)
7073	tx_ring->tx_buffer_info = vmalloc(size);
7074	if (!tx_ring->tx_buffer_info)
7075	goto err;
7076
7077	/* round up to nearest 4K */
7078	tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
7079	tx_ring->size = ALIGN(tx_ring->size, 4096);
7080
7081	set_dev_node(dev, node: ring_node);
7082	tx_ring->desc = dma_alloc_coherent(dev,
7083	size: tx_ring->size,
7084	dma_handle: &tx_ring->dma,
7085	GFP_KERNEL);
7086	set_dev_node(dev, node: orig_node);
7087	if (!tx_ring->desc)
7088	tx_ring->desc = dma_alloc_coherent(dev, size: tx_ring->size,
7089	dma_handle: &tx_ring->dma, GFP_KERNEL);
7090	if (!tx_ring->desc)
7091	goto err;
7092
7093	tx_ring->next_to_use = 0;
7094	tx_ring->next_to_clean = 0;
7095	return 0;
7096
7097	err:
7098	vfree(addr: tx_ring->tx_buffer_info);
7099	tx_ring->tx_buffer_info = NULL;
7100	dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
7101	return -ENOMEM;
7102	}
7103
7104	/**
7105	* ixgbe_setup_all_tx_resources - allocate all queues Tx resources
7106	* @adapter: board private structure
7107	*
7108	* If this function returns with an error, then it's possible one or
7109	* more of the rings is populated (while the rest are not). It is the
7110	* callers duty to clean those orphaned rings.
7111	*
7112	* Return 0 on success, negative on failure
7113	**/
7114	static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
7115	{
7116	int i, j = 0, err = 0;
7117
7118	for (i = 0; i < adapter->num_tx_queues; i++) {
7119	err = ixgbe_setup_tx_resources(tx_ring: adapter->tx_ring[i]);
7120	if (!err)
7121	continue;
7122
7123	e_err(probe, "Allocation for Tx Queue %u failed\n", i);
7124	goto err_setup_tx;
7125	}
7126	for (j = 0; j < adapter->num_xdp_queues; j++) {
7127	err = ixgbe_setup_tx_resources(tx_ring: adapter->xdp_ring[j]);
7128	if (!err)
7129	continue;
7130
7131	e_err(probe, "Allocation for Tx Queue %u failed\n", j);
7132	goto err_setup_tx;
7133	}
7134
7135	return 0;
7136	err_setup_tx:
7137	/* rewind the index freeing the rings as we go */
7138	while (j--)
7139	ixgbe_free_tx_resources(adapter->xdp_ring[j]);
7140	while (i--)
7141	ixgbe_free_tx_resources(adapter->tx_ring[i]);
7142	return err;
7143	}
7144
7145	static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring)
7146	{
7147	struct ixgbe_q_vector *q_vector = rx_ring->q_vector;
7148
7149	return q_vector ? q_vector->napi.napi_id : 0;
7150	}
7151
7152	/**
7153	* ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
7154	* @adapter: pointer to ixgbe_adapter
7155	* @rx_ring: rx descriptor ring (for a specific queue) to setup
7156	*
7157	* Returns 0 on success, negative on failure
7158	**/
7159	int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter,
7160	struct ixgbe_ring *rx_ring)
7161	{
7162	struct device *dev = rx_ring->dev;
7163	int orig_node = dev_to_node(dev);
7164	int ring_node = NUMA_NO_NODE;
7165	int size;
7166
7167	size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
7168
7169	if (rx_ring->q_vector)
7170	ring_node = rx_ring->q_vector->numa_node;
7171
7172	rx_ring->rx_buffer_info = vmalloc_node(size, ring_node);
7173	if (!rx_ring->rx_buffer_info)
7174	rx_ring->rx_buffer_info = vmalloc(size);
7175	if (!rx_ring->rx_buffer_info)
7176	goto err;
7177
7178	/* Round up to nearest 4K */
7179	rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
7180	rx_ring->size = ALIGN(rx_ring->size, 4096);
7181
7182	set_dev_node(dev, node: ring_node);
7183	rx_ring->desc = dma_alloc_coherent(dev,
7184	size: rx_ring->size,
7185	dma_handle: &rx_ring->dma,
7186	GFP_KERNEL);
7187	set_dev_node(dev, node: orig_node);
7188	if (!rx_ring->desc)
7189	rx_ring->desc = dma_alloc_coherent(dev, size: rx_ring->size,
7190	dma_handle: &rx_ring->dma, GFP_KERNEL);
7191	if (!rx_ring->desc)
7192	goto err;
7193
7194	rx_ring->next_to_clean = 0;
7195	rx_ring->next_to_use = 0;
7196
7197	/* XDP RX-queue info */
7198	if (xdp_rxq_info_reg(xdp_rxq: &rx_ring->xdp_rxq, dev: adapter->netdev,
7199	queue_index: rx_ring->queue_index, napi_id: ixgbe_rx_napi_id(rx_ring)) < 0)
7200	goto err;
7201
7202	WRITE_ONCE(rx_ring->xdp_prog, adapter->xdp_prog);
7203
7204	return 0;
7205	err:
7206	vfree(addr: rx_ring->rx_buffer_info);
7207	rx_ring->rx_buffer_info = NULL;
7208	dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
7209	return -ENOMEM;
7210	}
7211
7212	/**
7213	* ixgbe_setup_all_rx_resources - allocate all queues Rx resources
7214	* @adapter: board private structure
7215	*
7216	* If this function returns with an error, then it's possible one or
7217	* more of the rings is populated (while the rest are not). It is the
7218	* callers duty to clean those orphaned rings.
7219	*
7220	* Return 0 on success, negative on failure
7221	**/
7222	static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
7223	{
7224	int i, err = 0;
7225
7226	for (i = 0; i < adapter->num_rx_queues; i++) {
7227	err = ixgbe_setup_rx_resources(adapter, rx_ring: adapter->rx_ring[i]);
7228	if (!err)
7229	continue;
7230
7231	e_err(probe, "Allocation for Rx Queue %u failed\n", i);
7232	goto err_setup_rx;
7233	}
7234
7235	#ifdef IXGBE_FCOE
7236	err = ixgbe_setup_fcoe_ddp_resources(adapter);
7237	if (!err)
7238	#endif
7239	return 0;
7240	err_setup_rx:
7241	/* rewind the index freeing the rings as we go */
7242	while (i--)
7243	ixgbe_free_rx_resources(adapter->rx_ring[i]);
7244	return err;
7245	}
7246
7247	/**
7248	* ixgbe_free_tx_resources - Free Tx Resources per Queue
7249	* @tx_ring: Tx descriptor ring for a specific queue
7250	*
7251	* Free all transmit software resources
7252	**/
7253	void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
7254	{
7255	ixgbe_clean_tx_ring(tx_ring);
7256
7257	vfree(addr: tx_ring->tx_buffer_info);
7258	tx_ring->tx_buffer_info = NULL;
7259
7260	/* if not set, then don't free */
7261	if (!tx_ring->desc)
7262	return;
7263
7264	dma_free_coherent(dev: tx_ring->dev, size: tx_ring->size,
7265	cpu_addr: tx_ring->desc, dma_handle: tx_ring->dma);
7266
7267	tx_ring->desc = NULL;
7268	}
7269
7270	/**
7271	* ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
7272	* @adapter: board private structure
7273	*
7274	* Free all transmit software resources
7275	**/
7276	static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
7277	{
7278	int i;
7279
7280	for (i = 0; i < adapter->num_tx_queues; i++)
7281	if (adapter->tx_ring[i]->desc)
7282	ixgbe_free_tx_resources(tx_ring: adapter->tx_ring[i]);
7283	for (i = 0; i < adapter->num_xdp_queues; i++)
7284	if (adapter->xdp_ring[i]->desc)
7285	ixgbe_free_tx_resources(tx_ring: adapter->xdp_ring[i]);
7286	}
7287
7288	/**
7289	* ixgbe_free_rx_resources - Free Rx Resources
7290	* @rx_ring: ring to clean the resources from
7291	*
7292	* Free all receive software resources
7293	**/
7294	void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
7295	{
7296	ixgbe_clean_rx_ring(rx_ring);
7297
7298	rx_ring->xdp_prog = NULL;
7299	xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq);
7300	vfree(addr: rx_ring->rx_buffer_info);
7301	rx_ring->rx_buffer_info = NULL;
7302
7303	/* if not set, then don't free */
7304	if (!rx_ring->desc)
7305	return;
7306
7307	dma_free_coherent(dev: rx_ring->dev, size: rx_ring->size,
7308	cpu_addr: rx_ring->desc, dma_handle: rx_ring->dma);
7309
7310	rx_ring->desc = NULL;
7311	}
7312
7313	/**
7314	* ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
7315	* @adapter: board private structure
7316	*
7317	* Free all receive software resources
7318	**/
7319	static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
7320	{
7321	int i;
7322
7323	#ifdef IXGBE_FCOE
7324	ixgbe_free_fcoe_ddp_resources(adapter);
7325
7326	#endif
7327	for (i = 0; i < adapter->num_rx_queues; i++)
7328	if (adapter->rx_ring[i]->desc)
7329	ixgbe_free_rx_resources(rx_ring: adapter->rx_ring[i]);
7330	}
7331
7332	/**
7333	* ixgbe_max_xdp_frame_size - returns the maximum allowed frame size for XDP
7334	* @adapter: device handle, pointer to adapter
7335	*/
7336	static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter)
7337	{
7338	if (PAGE_SIZE >= 8192 || adapter->flags2 & IXGBE_FLAG2_RX_LEGACY)
7339	return IXGBE_RXBUFFER_2K;
7340	else
7341	return IXGBE_RXBUFFER_3K;
7342	}
7343
7344	/**
7345	* ixgbe_change_mtu - Change the Maximum Transfer Unit
7346	* @netdev: network interface device structure
7347	* @new_mtu: new value for maximum frame size
7348	*
7349	* Returns 0 on success, negative on failure
7350	**/
7351	static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
7352	{
7353	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
7354
7355	if (ixgbe_enabled_xdp_adapter(adapter)) {
7356	int new_frame_size = new_mtu + IXGBE_PKT_HDR_PAD;
7357
7358	if (new_frame_size > ixgbe_max_xdp_frame_size(adapter)) {
7359	e_warn(probe, "Requested MTU size is not supported with XDP\n");
7360	return -EINVAL;
7361	}
7362	}
7363
7364	/*
7365	* For 82599EB we cannot allow legacy VFs to enable their receive
7366	* paths when MTU greater than 1500 is configured. So display a
7367	* warning that legacy VFs will be disabled.
7368	*/
7369	if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
7370	(adapter->hw.mac.type == ixgbe_mac_82599EB) &&
7371	(new_mtu > ETH_DATA_LEN))
7372	e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n");
7373
7374	netdev_dbg(netdev, "changing MTU from %d to %d\n",
7375	netdev->mtu, new_mtu);
7376
7377	/* must set new MTU before calling down or up */
7378	WRITE_ONCE(netdev->mtu, new_mtu);
7379
7380	if (netif_running(dev: netdev))
7381	ixgbe_reinit_locked(adapter);
7382
7383	return 0;
7384	}
7385
7386	/**
7387	* ixgbe_open - Called when a network interface is made active
7388	* @netdev: network interface device structure
7389	*
7390	* Returns 0 on success, negative value on failure
7391	*
7392	* The open entry point is called when a network interface is made
7393	* active by the system (IFF_UP). At this point all resources needed
7394	* for transmit and receive operations are allocated, the interrupt
7395	* handler is registered with the OS, the watchdog timer is started,
7396	* and the stack is notified that the interface is ready.
7397	**/
7398	int ixgbe_open(struct net_device *netdev)
7399	{
7400	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
7401	struct ixgbe_hw *hw = &adapter->hw;
7402	int err, queues;
7403
7404	/* disallow open during test */
7405	if (test_bit(__IXGBE_TESTING, &adapter->state))
7406	return -EBUSY;
7407
7408	netif_carrier_off(dev: netdev);
7409
7410	/* allocate transmit descriptors */
7411	err = ixgbe_setup_all_tx_resources(adapter);
7412	if (err)
7413	goto err_setup_tx;
7414
7415	/* allocate receive descriptors */
7416	err = ixgbe_setup_all_rx_resources(adapter);
7417	if (err)
7418	goto err_setup_rx;
7419
7420	ixgbe_configure(adapter);
7421
7422	err = ixgbe_request_irq(adapter);
7423	if (err)
7424	goto err_req_irq;
7425
7426	/* Notify the stack of the actual queue counts. */
7427	queues = adapter->num_tx_queues;
7428	err = netif_set_real_num_tx_queues(dev: netdev, txq: queues);
7429	if (err)
7430	goto err_set_queues;
7431
7432	queues = adapter->num_rx_queues;
7433	err = netif_set_real_num_rx_queues(dev: netdev, rxq: queues);
7434	if (err)
7435	goto err_set_queues;
7436
7437	ixgbe_ptp_init(adapter);
7438
7439	ixgbe_up_complete(adapter);
7440
7441	udp_tunnel_nic_reset_ntf(dev: netdev);
7442	if (adapter->hw.mac.type == ixgbe_mac_e610) {
7443	int err = ixgbe_update_link_info(hw: &adapter->hw);
7444
7445	if (err)
7446	e_dev_err("Failed to update link info, err %d.\n", err);
7447
7448	ixgbe_check_link_cfg_err(adapter,
7449	link_cfg_err: adapter->hw.link.link_info.link_cfg_err);
7450
7451	err = ixgbe_non_sfp_link_config(hw: &adapter->hw);
7452	if (err)
7453	e_dev_err("Link setup failed, err %d.\n", err);
7454	}
7455
7456	return 0;
7457
7458	err_set_queues:
7459	ixgbe_free_irq(adapter);
7460	err_req_irq:
7461	ixgbe_free_all_rx_resources(adapter);
7462	if (hw->phy.ops.set_phy_power && !adapter->wol)
7463	hw->phy.ops.set_phy_power(&adapter->hw, false);
7464	err_setup_rx:
7465	ixgbe_free_all_tx_resources(adapter);
7466	err_setup_tx:
7467	ixgbe_reset(adapter);
7468
7469	return err;
7470	}
7471
7472	static void ixgbe_close_suspend(struct ixgbe_adapter *adapter)
7473	{
7474	ixgbe_ptp_suspend(adapter);
7475
7476	if (adapter->hw.phy.ops.enter_lplu) {
7477	adapter->hw.phy.reset_disable = true;
7478	ixgbe_down(adapter);
7479	adapter->hw.phy.ops.enter_lplu(&adapter->hw);
7480	adapter->hw.phy.reset_disable = false;
7481	} else {
7482	ixgbe_down(adapter);
7483	}
7484
7485	ixgbe_free_irq(adapter);
7486
7487	ixgbe_free_all_tx_resources(adapter);
7488	ixgbe_free_all_rx_resources(adapter);
7489	}
7490
7491	/**
7492	* ixgbe_close - Disables a network interface
7493	* @netdev: network interface device structure
7494	*
7495	* Returns 0, this is not allowed to fail
7496	*
7497	* The close entry point is called when an interface is de-activated
7498	* by the OS. The hardware is still under the drivers control, but
7499	* needs to be disabled. A global MAC reset is issued to stop the
7500	* hardware, and all transmit and receive resources are freed.
7501	**/
7502	int ixgbe_close(struct net_device *netdev)
7503	{
7504	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
7505
7506	ixgbe_ptp_stop(adapter);
7507
7508	if (netif_device_present(dev: netdev))
7509	ixgbe_close_suspend(adapter);
7510
7511	ixgbe_fdir_filter_exit(adapter);
7512
7513	ixgbe_release_hw_control(adapter);
7514
7515	return 0;
7516	}
7517
7518	static int ixgbe_resume(struct device *dev_d)
7519	{
7520	struct pci_dev *pdev = to_pci_dev(dev_d);
7521	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7522	struct net_device *netdev = adapter->netdev;
7523	u32 err;
7524
7525	adapter->hw.hw_addr = adapter->io_addr;
7526
7527	err = pci_enable_device_mem(dev: pdev);
7528	if (err) {
7529	e_dev_err("Cannot enable PCI device from suspend\n");
7530	return err;
7531	}
7532	smp_mb__before_atomic();
7533	clear_bit(nr: __IXGBE_DISABLED, addr: &adapter->state);
7534	pci_set_master(dev: pdev);
7535
7536	device_wakeup_disable(dev: dev_d);
7537
7538	ixgbe_reset(adapter);
7539
7540	IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
7541
7542	rtnl_lock();
7543	err = ixgbe_init_interrupt_scheme(adapter);
7544	if (!err && netif_running(dev: netdev))
7545	err = ixgbe_open(netdev);
7546
7547
7548	if (!err)
7549	netif_device_attach(dev: netdev);
7550	rtnl_unlock();
7551
7552	return err;
7553	}
7554
7555	static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
7556	{
7557	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7558	struct net_device *netdev = adapter->netdev;
7559	struct ixgbe_hw *hw = &adapter->hw;
7560	u32 ctrl;
7561	u32 wufc = adapter->wol;
7562
7563	rtnl_lock();
7564	netif_device_detach(dev: netdev);
7565
7566	if (netif_running(dev: netdev))
7567	ixgbe_close_suspend(adapter);
7568
7569	ixgbe_clear_interrupt_scheme(adapter);
7570	rtnl_unlock();
7571
7572	if (hw->mac.ops.stop_link_on_d3)
7573	hw->mac.ops.stop_link_on_d3(hw);
7574
7575	if (wufc) {
7576	u32 fctrl;
7577
7578	ixgbe_set_rx_mode(netdev);
7579
7580	/* enable the optics for 82599 SFP+ fiber as we can WoL */
7581	if (hw->mac.ops.enable_tx_laser)
7582	hw->mac.ops.enable_tx_laser(hw);
7583
7584	/* enable the reception of multicast packets */
7585	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
7586	fctrl |= IXGBE_FCTRL_MPE;
7587	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
7588
7589	ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
7590	ctrl |= IXGBE_CTRL_GIO_DIS;
7591	IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
7592
7593	IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
7594	} else {
7595	IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
7596	IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
7597	}
7598
7599	switch (hw->mac.type) {
7600	case ixgbe_mac_82598EB:
7601	pci_wake_from_d3(dev: pdev, enable: false);
7602	break;
7603	case ixgbe_mac_82599EB:
7604	case ixgbe_mac_X540:
7605	case ixgbe_mac_X550:
7606	case ixgbe_mac_X550EM_x:
7607	case ixgbe_mac_x550em_a:
7608	case ixgbe_mac_e610:
7609	pci_wake_from_d3(dev: pdev, enable: !!wufc);
7610	break;
7611	default:
7612	break;
7613	}
7614
7615	*enable_wake = !!wufc;
7616	if (hw->phy.ops.set_phy_power && !*enable_wake)
7617	hw->phy.ops.set_phy_power(hw, false);
7618
7619	ixgbe_release_hw_control(adapter);
7620
7621	if (!test_and_set_bit(nr: __IXGBE_DISABLED, addr: &adapter->state))
7622	pci_disable_device(dev: pdev);
7623
7624	return 0;
7625	}
7626
7627	static int ixgbe_suspend(struct device *dev_d)
7628	{
7629	struct pci_dev *pdev = to_pci_dev(dev_d);
7630	int retval;
7631	bool wake;
7632
7633	retval = __ixgbe_shutdown(pdev, enable_wake: &wake);
7634
7635	device_set_wakeup_enable(dev: dev_d, enable: wake);
7636
7637	return retval;
7638	}
7639
7640	static void ixgbe_shutdown(struct pci_dev *pdev)
7641	{
7642	bool wake;
7643
7644	__ixgbe_shutdown(pdev, enable_wake: &wake);
7645
7646	if (system_state == SYSTEM_POWER_OFF) {
7647	pci_wake_from_d3(dev: pdev, enable: wake);
7648	pci_set_power_state(dev: pdev, PCI_D3hot);
7649	}
7650	}
7651
7652	/**
7653	* ixgbe_update_stats - Update the board statistics counters.
7654	* @adapter: board private structure
7655	**/
7656	void ixgbe_update_stats(struct ixgbe_adapter *adapter)
7657	{
7658	struct net_device *netdev = adapter->netdev;
7659	struct ixgbe_hw *hw = &adapter->hw;
7660	struct ixgbe_hw_stats *hwstats = &adapter->stats;
7661	u64 total_mpc = 0;
7662	u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
7663	u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
7664	u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
7665	u64 alloc_rx_page = 0;
7666	u64 bytes = 0, packets = 0, hw_csum_rx_error = 0;
7667
7668	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
7669	test_bit(__IXGBE_RESETTING, &adapter->state))
7670	return;
7671
7672	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
7673	u64 rsc_count = 0;
7674	u64 rsc_flush = 0;
7675	for (i = 0; i < adapter->num_rx_queues; i++) {
7676	rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
7677	rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
7678	}
7679	adapter->rsc_total_count = rsc_count;
7680	adapter->rsc_total_flush = rsc_flush;
7681	}
7682
7683	for (i = 0; i < adapter->num_rx_queues; i++) {
7684	struct ixgbe_ring *rx_ring = READ_ONCE(adapter->rx_ring[i]);
7685
7686	if (!rx_ring)
7687	continue;
7688	non_eop_descs += rx_ring->rx_stats.non_eop_descs;
7689	alloc_rx_page += rx_ring->rx_stats.alloc_rx_page;
7690	alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
7691	alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
7692	hw_csum_rx_error += rx_ring->rx_stats.csum_err;
7693	bytes += rx_ring->stats.bytes;
7694	packets += rx_ring->stats.packets;
7695	}
7696	adapter->non_eop_descs = non_eop_descs;
7697	adapter->alloc_rx_page = alloc_rx_page;
7698	adapter->alloc_rx_page_failed = alloc_rx_page_failed;
7699	adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
7700	adapter->hw_csum_rx_error = hw_csum_rx_error;
7701	netdev->stats.rx_bytes = bytes;
7702	netdev->stats.rx_packets = packets;
7703
7704	bytes = 0;
7705	packets = 0;
7706	/* gather some stats to the adapter struct that are per queue */
7707	for (i = 0; i < adapter->num_tx_queues; i++) {
7708	struct ixgbe_ring *tx_ring = READ_ONCE(adapter->tx_ring[i]);
7709
7710	if (!tx_ring)
7711	continue;
7712	restart_queue += tx_ring->tx_stats.restart_queue;
7713	tx_busy += tx_ring->tx_stats.tx_busy;
7714	bytes += tx_ring->stats.bytes;
7715	packets += tx_ring->stats.packets;
7716	}
7717	for (i = 0; i < adapter->num_xdp_queues; i++) {
7718	struct ixgbe_ring *xdp_ring = READ_ONCE(adapter->xdp_ring[i]);
7719
7720	if (!xdp_ring)
7721	continue;
7722	restart_queue += xdp_ring->tx_stats.restart_queue;
7723	tx_busy += xdp_ring->tx_stats.tx_busy;
7724	bytes += xdp_ring->stats.bytes;
7725	packets += xdp_ring->stats.packets;
7726	}
7727	adapter->restart_queue = restart_queue;
7728	adapter->tx_busy = tx_busy;
7729	netdev->stats.tx_bytes = bytes;
7730	netdev->stats.tx_packets = packets;
7731
7732	hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
7733
7734	/* 8 register reads */
7735	for (i = 0; i < 8; i++) {
7736	/* for packet buffers not used, the register should read 0 */
7737	mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
7738	missed_rx += mpc;
7739	hwstats->mpc[i] += mpc;
7740	total_mpc += hwstats->mpc[i];
7741	hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
7742	hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
7743	switch (hw->mac.type) {
7744	case ixgbe_mac_82598EB:
7745	hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
7746	hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
7747	hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
7748	hwstats->pxonrxc[i] +=
7749	IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
7750	break;
7751	case ixgbe_mac_82599EB:
7752	case ixgbe_mac_X540:
7753	case ixgbe_mac_X550:
7754	case ixgbe_mac_X550EM_x:
7755	case ixgbe_mac_x550em_a:
7756	case ixgbe_mac_e610:
7757	hwstats->pxonrxc[i] +=
7758	IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
7759	break;
7760	default:
7761	break;
7762	}
7763	}
7764
7765	/*16 register reads */
7766	for (i = 0; i < 16; i++) {
7767	hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
7768	hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
7769	if (hw->mac.type == ixgbe_mac_82599EB ||
7770	hw->mac.type == ixgbe_mac_X540 ||
7771	hw->mac.type == ixgbe_mac_X550 ||
7772	hw->mac.type == ixgbe_mac_X550EM_x ||
7773	hw->mac.type == ixgbe_mac_x550em_a ||
7774	hw->mac.type == ixgbe_mac_e610) {
7775	hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
7776	IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */
7777	hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
7778	IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */
7779	}
7780	}
7781
7782	hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
7783	/* work around hardware counting issue */
7784	hwstats->gprc -= missed_rx;
7785
7786	ixgbe_update_xoff_received(adapter);
7787
7788	/* 82598 hardware only has a 32 bit counter in the high register */
7789	switch (hw->mac.type) {
7790	case ixgbe_mac_82598EB:
7791	hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
7792	hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
7793	hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
7794	hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
7795	break;
7796	case ixgbe_mac_X540:
7797	case ixgbe_mac_X550:
7798	case ixgbe_mac_X550EM_x:
7799	case ixgbe_mac_x550em_a:
7800	case ixgbe_mac_e610:
7801	/* OS2BMC stats are X540 and later */
7802	hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
7803	hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
7804	hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
7805	hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
7806	fallthrough;
7807	case ixgbe_mac_82599EB:
7808	for (i = 0; i < 16; i++)
7809	adapter->hw_rx_no_dma_resources +=
7810	IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
7811	hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
7812	IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
7813	hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
7814	IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
7815	hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
7816	IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
7817	hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
7818	hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
7819	hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
7820	#ifdef IXGBE_FCOE
7821	hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
7822	hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
7823	hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
7824	hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
7825	hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
7826	hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
7827	/* Add up per cpu counters for total ddp aloc fail */
7828	if (adapter->fcoe.ddp_pool) {
7829	struct ixgbe_fcoe *fcoe = &adapter->fcoe;
7830	struct ixgbe_fcoe_ddp_pool *ddp_pool;
7831	unsigned int cpu;
7832	u64 noddp = 0, noddp_ext_buff = 0;
7833	for_each_possible_cpu(cpu) {
7834	ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu);
7835	noddp += ddp_pool->noddp;
7836	noddp_ext_buff += ddp_pool->noddp_ext_buff;
7837	}
7838	hwstats->fcoe_noddp = noddp;
7839	hwstats->fcoe_noddp_ext_buff = noddp_ext_buff;
7840	}
7841	#endif /* IXGBE_FCOE */
7842	break;
7843	default:
7844	break;
7845	}
7846	bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
7847	hwstats->bprc += bprc;
7848	hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
7849	if (hw->mac.type == ixgbe_mac_82598EB)
7850	hwstats->mprc -= bprc;
7851	hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
7852	hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
7853	hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
7854	hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
7855	hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
7856	hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
7857	hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
7858	hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
7859	lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
7860	hwstats->lxontxc += lxon;
7861	lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
7862	hwstats->lxofftxc += lxoff;
7863	hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
7864	hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
7865	/*
7866	* 82598 errata - tx of flow control packets is included in tx counters
7867	*/
7868	xon_off_tot = lxon + lxoff;
7869	hwstats->gptc -= xon_off_tot;
7870	hwstats->mptc -= xon_off_tot;
7871	hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
7872	hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
7873	hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
7874	hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
7875	hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
7876	hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
7877	hwstats->ptc64 -= xon_off_tot;
7878	hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
7879	hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
7880	hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
7881	hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
7882	hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
7883	hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
7884
7885	/* Fill out the OS statistics structure */
7886	netdev->stats.multicast = hwstats->mprc;
7887
7888	/* Rx Errors */
7889	netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
7890	netdev->stats.rx_dropped = 0;
7891	netdev->stats.rx_length_errors = hwstats->rlec;
7892	netdev->stats.rx_crc_errors = hwstats->crcerrs;
7893	netdev->stats.rx_missed_errors = total_mpc;
7894
7895	/* VF Stats Collection - skip while resetting because these
7896	* are not clear on read and otherwise you'll sometimes get
7897	* crazy values.
7898	*/
7899	if (!test_bit(__IXGBE_RESETTING, &adapter->state)) {
7900	for (i = 0; i < adapter->num_vfs; i++) {
7901	UPDATE_VF_COUNTER_32bit(IXGBE_PVFGPRC(i),
7902	adapter->vfinfo[i].last_vfstats.gprc,
7903	adapter->vfinfo[i].vfstats.gprc);
7904	UPDATE_VF_COUNTER_32bit(IXGBE_PVFGPTC(i),
7905	adapter->vfinfo[i].last_vfstats.gptc,
7906	adapter->vfinfo[i].vfstats.gptc);
7907	UPDATE_VF_COUNTER_36bit(IXGBE_PVFGORC_LSB(i),
7908	IXGBE_PVFGORC_MSB(i),
7909	adapter->vfinfo[i].last_vfstats.gorc,
7910	adapter->vfinfo[i].vfstats.gorc);
7911	UPDATE_VF_COUNTER_36bit(IXGBE_PVFGOTC_LSB(i),
7912	IXGBE_PVFGOTC_MSB(i),
7913	adapter->vfinfo[i].last_vfstats.gotc,
7914	adapter->vfinfo[i].vfstats.gotc);
7915	UPDATE_VF_COUNTER_32bit(IXGBE_PVFMPRC(i),
7916	adapter->vfinfo[i].last_vfstats.mprc,
7917	adapter->vfinfo[i].vfstats.mprc);
7918	}
7919	}
7920	}
7921
7922	/**
7923	* ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table
7924	* @adapter: pointer to the device adapter structure
7925	**/
7926	static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
7927	{
7928	struct ixgbe_hw *hw = &adapter->hw;
7929	int i;
7930
7931	if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
7932	return;
7933
7934	adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
7935
7936	/* if interface is down do nothing */
7937	if (test_bit(__IXGBE_DOWN, &adapter->state))
7938	return;
7939
7940	/* do nothing if we are not using signature filters */
7941	if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
7942	return;
7943
7944	adapter->fdir_overflow++;
7945
7946	if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
7947	for (i = 0; i < adapter->num_tx_queues; i++)
7948	set_bit(nr: __IXGBE_TX_FDIR_INIT_DONE,
7949	addr: &(adapter->tx_ring[i]->state));
7950	for (i = 0; i < adapter->num_xdp_queues; i++)
7951	set_bit(nr: __IXGBE_TX_FDIR_INIT_DONE,
7952	addr: &adapter->xdp_ring[i]->state);
7953	/* re-enable flow director interrupts */
7954	IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
7955	} else {
7956	e_err(probe, "failed to finish FDIR re-initialization, "
7957	"ignored adding FDIR ATR filters\n");
7958	}
7959	}
7960
7961	/**
7962	* ixgbe_check_hang_subtask - check for hung queues and dropped interrupts
7963	* @adapter: pointer to the device adapter structure
7964	*
7965	* This function serves two purposes. First it strobes the interrupt lines
7966	* in order to make certain interrupts are occurring. Secondly it sets the
7967	* bits needed to check for TX hangs. As a result we should immediately
7968	* determine if a hang has occurred.
7969	*/
7970	static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
7971	{
7972	struct ixgbe_hw *hw = &adapter->hw;
7973	u64 eics = 0;
7974	int i;
7975
7976	/* If we're down, removing or resetting, just bail */
7977	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
7978	test_bit(__IXGBE_REMOVING, &adapter->state) ||
7979	test_bit(__IXGBE_RESETTING, &adapter->state))
7980	return;
7981
7982	/* Force detection of hung controller */
7983	if (netif_carrier_ok(dev: adapter->netdev))
7984	for (i = 0; i < adapter->num_tx_queues; i++)
7985	set_check_for_tx_hang(adapter->tx_ring[i]);
7986
7987	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
7988	/*
7989	* for legacy and MSI interrupts don't set any bits
7990	* that are enabled for EIAM, because this operation
7991	* would set *both* EIMS and EICS for any bit in EIAM
7992	*/
7993	IXGBE_WRITE_REG(hw, IXGBE_EICS,
7994	(IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
7995	} else {
7996	/* get one bit for every active tx/rx interrupt vector */
7997	for (i = 0; i < adapter->num_q_vectors; i++) {
7998	struct ixgbe_q_vector *qv = adapter->q_vector[i];
7999	if (qv->rx.ring || qv->tx.ring)
8000	eics |= BIT_ULL(i);
8001	}
8002	}
8003
8004	/* Cause software interrupt to ensure rings are cleaned */
8005	ixgbe_irq_rearm_queues(adapter, qmask: eics);
8006	}
8007
8008	/**
8009	* ixgbe_watchdog_update_link - update the link status
8010	* @adapter: pointer to the device adapter structure
8011	**/
8012	static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
8013	{
8014	struct ixgbe_hw *hw = &adapter->hw;
8015	u32 link_speed = adapter->link_speed;
8016	bool link_up = adapter->link_up;
8017	bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
8018
8019	if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
8020	return;
8021
8022	if (hw->mac.ops.check_link) {
8023	hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
8024	} else {
8025	/* always assume link is up, if no check link function */
8026	link_speed = IXGBE_LINK_SPEED_10GB_FULL;
8027	link_up = true;
8028	}
8029
8030	if (adapter->ixgbe_ieee_pfc)
8031	pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
8032
8033	if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) {
8034	hw->mac.ops.fc_enable(hw);
8035	ixgbe_set_rx_drop_en(adapter);
8036	}
8037
8038	if (link_up ||
8039	time_after(jiffies, (adapter->link_check_timeout +
8040	IXGBE_TRY_LINK_TIMEOUT))) {
8041	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
8042	IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
8043	IXGBE_WRITE_FLUSH(hw);
8044	}
8045
8046	adapter->link_up = link_up;
8047	adapter->link_speed = link_speed;
8048	}
8049
8050	static void ixgbe_update_default_up(struct ixgbe_adapter *adapter)
8051	{
8052	#ifdef CONFIG_IXGBE_DCB
8053	struct net_device *netdev = adapter->netdev;
8054	struct dcb_app app = {
8055	.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE,
8056	.protocol = 0,
8057	};
8058	u8 up = 0;
8059
8060	if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)
8061	up = dcb_ieee_getapp_mask(netdev, &app);
8062
8063	adapter->default_up = (up > 1) ? (ffs(up) - 1) : 0;
8064	#endif
8065	}
8066
8067	/**
8068	* ixgbe_watchdog_link_is_up - update netif_carrier status and
8069	* print link up message
8070	* @adapter: pointer to the device adapter structure
8071	**/
8072	static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
8073	{
8074	struct net_device *netdev = adapter->netdev;
8075	struct ixgbe_hw *hw = &adapter->hw;
8076	u32 link_speed = adapter->link_speed;
8077	const char *speed_str;
8078	bool flow_rx, flow_tx;
8079
8080	/* only continue if link was previously down */
8081	if (netif_carrier_ok(dev: netdev))
8082	return;
8083
8084	adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
8085
8086	switch (hw->mac.type) {
8087	case ixgbe_mac_82598EB: {
8088	u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
8089	u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
8090	flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
8091	flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
8092	}
8093	break;
8094	case ixgbe_mac_X540:
8095	case ixgbe_mac_X550:
8096	case ixgbe_mac_X550EM_x:
8097	case ixgbe_mac_x550em_a:
8098	case ixgbe_mac_e610:
8099	case ixgbe_mac_82599EB: {
8100	u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
8101	u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
8102	flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
8103	flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
8104	}
8105	break;
8106	default:
8107	flow_tx = false;
8108	flow_rx = false;
8109	break;
8110	}
8111
8112	adapter->last_rx_ptp_check = jiffies;
8113
8114	if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
8115	ixgbe_ptp_start_cyclecounter(adapter);
8116
8117	switch (link_speed) {
8118	case IXGBE_LINK_SPEED_10GB_FULL:
8119	speed_str = "10 Gbps";
8120	break;
8121	case IXGBE_LINK_SPEED_5GB_FULL:
8122	speed_str = "5 Gbps";
8123	break;
8124	case IXGBE_LINK_SPEED_2_5GB_FULL:
8125	speed_str = "2.5 Gbps";
8126	break;
8127	case IXGBE_LINK_SPEED_1GB_FULL:
8128	speed_str = "1 Gbps";
8129	break;
8130	case IXGBE_LINK_SPEED_100_FULL:
8131	speed_str = "100 Mbps";
8132	break;
8133	case IXGBE_LINK_SPEED_10_FULL:
8134	speed_str = "10 Mbps";
8135	break;
8136	default:
8137	speed_str = "unknown speed";
8138	break;
8139	}
8140	e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str,
8141	((flow_rx && flow_tx) ? "RX/TX" :
8142	(flow_rx ? "RX" :
8143	(flow_tx ? "TX" : "None"))));
8144
8145	netif_carrier_on(dev: netdev);
8146	ixgbe_check_vf_rate_limit(adapter);
8147
8148	if (adapter->num_vfs && hw->mac.ops.enable_mdd)
8149	hw->mac.ops.enable_mdd(hw);
8150
8151	/* enable transmits */
8152	netif_tx_wake_all_queues(dev: adapter->netdev);
8153
8154	/* update the default user priority for VFs */
8155	ixgbe_update_default_up(adapter);
8156
8157	/* ping all the active vfs to let them know link has changed */
8158	ixgbe_ping_all_vfs(adapter);
8159	}
8160
8161	/**
8162	* ixgbe_watchdog_link_is_down - update netif_carrier status and
8163	* print link down message
8164	* @adapter: pointer to the adapter structure
8165	**/
8166	static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter)
8167	{
8168	struct net_device *netdev = adapter->netdev;
8169	struct ixgbe_hw *hw = &adapter->hw;
8170
8171	adapter->link_up = false;
8172	adapter->link_speed = 0;
8173
8174	/* only continue if link was up previously */
8175	if (!netif_carrier_ok(dev: netdev))
8176	return;
8177
8178	adapter->link_down_events++;
8179
8180	/* poll for SFP+ cable when link is down */
8181	if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
8182	adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
8183
8184	if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
8185	ixgbe_ptp_start_cyclecounter(adapter);
8186
8187	e_info(drv, "NIC Link is Down\n");
8188	netif_carrier_off(dev: netdev);
8189
8190	/* ping all the active vfs to let them know link has changed */
8191	ixgbe_ping_all_vfs(adapter);
8192	}
8193
8194	static bool ixgbe_ring_tx_pending(struct ixgbe_adapter *adapter)
8195	{
8196	int i;
8197
8198	for (i = 0; i < adapter->num_tx_queues; i++) {
8199	struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
8200
8201	if (tx_ring->next_to_use != tx_ring->next_to_clean)
8202	return true;
8203	}
8204
8205	return false;
8206	}
8207
8208	static bool ixgbe_vf_tx_pending(struct ixgbe_adapter *adapter)
8209	{
8210	struct ixgbe_hw *hw = &adapter->hw;
8211	struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
8212	u32 q_per_pool = __ALIGN_MASK(1, ~vmdq->mask);
8213
8214	int i, j;
8215
8216	if (!adapter->num_vfs)
8217	return false;
8218
8219	/* resetting the PF is only needed for MAC before X550 */
8220	if (hw->mac.type >= ixgbe_mac_X550)
8221	return false;
8222
8223	for (i = 0; i < adapter->num_vfs; i++) {
8224	for (j = 0; j < q_per_pool; j++) {
8225	u32 h, t;
8226
8227	h = IXGBE_READ_REG(hw, IXGBE_PVFTDHN(q_per_pool, i, j));
8228	t = IXGBE_READ_REG(hw, IXGBE_PVFTDTN(q_per_pool, i, j));
8229
8230	if (h != t)
8231	return true;
8232	}
8233	}
8234
8235	return false;
8236	}
8237
8238	/**
8239	* ixgbe_watchdog_flush_tx - flush queues on link down
8240	* @adapter: pointer to the device adapter structure
8241	**/
8242	static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
8243	{
8244	if (!netif_carrier_ok(dev: adapter->netdev)) {
8245	if (ixgbe_ring_tx_pending(adapter) ||
8246	ixgbe_vf_tx_pending(adapter)) {
8247	/* We've lost link, so the controller stops DMA,
8248	* but we've got queued Tx work that's never going
8249	* to get done, so reset controller to flush Tx.
8250	* (Do the reset outside of interrupt context).
8251	*/
8252	e_warn(drv, "initiating reset to clear Tx work after link loss\n");
8253	set_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state);
8254	}
8255	}
8256	}
8257
8258	#ifdef CONFIG_PCI_IOV
8259	static void ixgbe_bad_vf_abort(struct ixgbe_adapter *adapter, u32 vf)
8260	{
8261	struct ixgbe_hw *hw = &adapter->hw;
8262
8263	if (adapter->hw.mac.type == ixgbe_mac_82599EB &&
8264	adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) {
8265	adapter->vfinfo[vf].primary_abort_count++;
8266	if (adapter->vfinfo[vf].primary_abort_count ==
8267	IXGBE_PRIMARY_ABORT_LIMIT) {
8268	ixgbe_set_vf_link_state(adapter, vf,
8269	state: IFLA_VF_LINK_STATE_DISABLE);
8270	adapter->vfinfo[vf].primary_abort_count = 0;
8271
8272	e_info(drv,
8273	"Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on",
8274	hw->bus.func, vf,
8275	adapter->vfinfo[vf].vf_mac_addresses);
8276	}
8277	}
8278	}
8279
8280	static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)
8281	{
8282	struct ixgbe_hw *hw = &adapter->hw;
8283	struct pci_dev *pdev = adapter->pdev;
8284	unsigned int vf;
8285	u32 gpc;
8286
8287	if (!(netif_carrier_ok(dev: adapter->netdev)))
8288	return;
8289
8290	gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);
8291	if (gpc) /* If incrementing then no need for the check below */
8292	return;
8293	/* Check to see if a bad DMA write target from an errant or
8294	* malicious VF has caused a PCIe error. If so then we can
8295	* issue a VFLR to the offending VF(s) and then resume without
8296	* requesting a full slot reset.
8297	*/
8298
8299	if (!pdev)
8300	return;
8301
8302	/* check status reg for all VFs owned by this PF */
8303	for (vf = 0; vf < adapter->num_vfs; ++vf) {
8304	struct pci_dev *vfdev = adapter->vfinfo[vf].vfdev;
8305	u16 status_reg;
8306
8307	if (!vfdev)
8308	continue;
8309	pci_read_config_word(dev: vfdev, PCI_STATUS, val: &status_reg);
8310	if (status_reg != IXGBE_FAILED_READ_CFG_WORD &&
8311	status_reg & PCI_STATUS_REC_MASTER_ABORT) {
8312	ixgbe_bad_vf_abort(adapter, vf);
8313	pcie_flr(dev: vfdev);
8314	}
8315	}
8316	}
8317
8318	static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
8319	{
8320	u32 ssvpc;
8321
8322	/* Do not perform spoof check for 82598 or if not in IOV mode */
8323	if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
8324	adapter->num_vfs == 0)
8325	return;
8326
8327	ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
8328
8329	/*
8330	* ssvpc register is cleared on read, if zero then no
8331	* spoofed packets in the last interval.
8332	*/
8333	if (!ssvpc)
8334	return;
8335
8336	e_warn(drv, "%u Spoofed packets detected\n", ssvpc);
8337	}
8338	#else
8339	static void ixgbe_spoof_check(struct ixgbe_adapter __always_unused *adapter)
8340	{
8341	}
8342
8343	static void
8344	ixgbe_check_for_bad_vf(struct ixgbe_adapter __always_unused *adapter)
8345	{
8346	}
8347	#endif /* CONFIG_PCI_IOV */
8348
8349
8350	/**
8351	* ixgbe_watchdog_subtask - check and bring link up
8352	* @adapter: pointer to the device adapter structure
8353	**/
8354	static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
8355	{
8356	/* if interface is down, removing or resetting, do nothing */
8357	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
8358	test_bit(__IXGBE_REMOVING, &adapter->state) ||
8359	test_bit(__IXGBE_RESETTING, &adapter->state))
8360	return;
8361
8362	ixgbe_watchdog_update_link(adapter);
8363
8364	if (adapter->link_up)
8365	ixgbe_watchdog_link_is_up(adapter);
8366	else
8367	ixgbe_watchdog_link_is_down(adapter);
8368
8369	ixgbe_check_for_bad_vf(adapter);
8370	ixgbe_spoof_check(adapter);
8371	ixgbe_update_stats(adapter);
8372
8373	ixgbe_watchdog_flush_tx(adapter);
8374	}
8375
8376	/**
8377	* ixgbe_sfp_detection_subtask - poll for SFP+ cable
8378	* @adapter: the ixgbe adapter structure
8379	**/
8380	static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
8381	{
8382	struct ixgbe_hw *hw = &adapter->hw;
8383	int err;
8384
8385	/* not searching for SFP so there is nothing to do here */
8386	if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
8387	!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
8388	return;
8389
8390	if (adapter->sfp_poll_time &&
8391	time_after(adapter->sfp_poll_time, jiffies))
8392	return; /* If not yet time to poll for SFP */
8393
8394	/* someone else is in init, wait until next service event */
8395	if (test_and_set_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state))
8396	return;
8397
8398	adapter->sfp_poll_time = jiffies + IXGBE_SFP_POLL_JIFFIES - 1;
8399
8400	err = hw->phy.ops.identify_sfp(hw);
8401	if (err == -EOPNOTSUPP)
8402	goto sfp_out;
8403
8404	if (err == -ENOENT) {
8405	/* If no cable is present, then we need to reset
8406	* the next time we find a good cable. */
8407	adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
8408	}
8409
8410	/* exit on error */
8411	if (err)
8412	goto sfp_out;
8413
8414	/* exit if reset not needed */
8415	if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
8416	goto sfp_out;
8417
8418	adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
8419
8420	/*
8421	* A module may be identified correctly, but the EEPROM may not have
8422	* support for that module. setup_sfp() will fail in that case, so
8423	* we should not allow that module to load.
8424	*/
8425	if (hw->mac.type == ixgbe_mac_82598EB)
8426	err = hw->phy.ops.reset(hw);
8427	else
8428	err = hw->mac.ops.setup_sfp(hw);
8429
8430	if (err == -EOPNOTSUPP)
8431	goto sfp_out;
8432
8433	adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
8434	e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
8435
8436	sfp_out:
8437	clear_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state);
8438
8439	if (err == -EOPNOTSUPP &&
8440	adapter->netdev->reg_state == NETREG_REGISTERED) {
8441	e_dev_err("failed to initialize because an unsupported "
8442	"SFP+ module type was detected.\n");
8443	e_dev_err("Reload the driver after installing a "
8444	"supported module.\n");
8445	unregister_netdev(dev: adapter->netdev);
8446	}
8447	}
8448
8449	/**
8450	* ixgbe_sfp_link_config_subtask - set up link SFP after module install
8451	* @adapter: the ixgbe adapter structure
8452	**/
8453	static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
8454	{
8455	struct ixgbe_hw *hw = &adapter->hw;
8456	u32 cap_speed;
8457	u32 speed;
8458	bool autoneg = false;
8459
8460	if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
8461	return;
8462
8463	/* someone else is in init, wait until next service event */
8464	if (test_and_set_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state))
8465	return;
8466
8467	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
8468
8469	hw->mac.ops.get_link_capabilities(hw, &cap_speed, &autoneg);
8470
8471	/* advertise highest capable link speed */
8472	if (!autoneg && (cap_speed & IXGBE_LINK_SPEED_10GB_FULL))
8473	speed = IXGBE_LINK_SPEED_10GB_FULL;
8474	else
8475	speed = cap_speed & (IXGBE_LINK_SPEED_10GB_FULL |
8476	IXGBE_LINK_SPEED_1GB_FULL);
8477
8478	if (hw->mac.ops.setup_link)
8479	hw->mac.ops.setup_link(hw, speed, true);
8480
8481	adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
8482	adapter->link_check_timeout = jiffies;
8483	clear_bit(nr: __IXGBE_IN_SFP_INIT, addr: &adapter->state);
8484	}
8485
8486	/**
8487	* ixgbe_service_timer - Timer Call-back
8488	* @t: pointer to timer_list structure
8489	**/
8490	static void ixgbe_service_timer(struct timer_list *t)
8491	{
8492	struct ixgbe_adapter *adapter = timer_container_of(adapter, t,
8493	service_timer);
8494	unsigned long next_event_offset;
8495
8496	/* poll faster when waiting for link */
8497	if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
8498	next_event_offset = HZ / 10;
8499	else
8500	next_event_offset = HZ * 2;
8501
8502	/* Reset the timer */
8503	mod_timer(timer: &adapter->service_timer, expires: next_event_offset + jiffies);
8504
8505	ixgbe_service_event_schedule(adapter);
8506	}
8507
8508	static void ixgbe_phy_interrupt_subtask(struct ixgbe_adapter *adapter)
8509	{
8510	struct ixgbe_hw *hw = &adapter->hw;
8511	bool overtemp;
8512
8513	if (!(adapter->flags2 & IXGBE_FLAG2_PHY_INTERRUPT))
8514	return;
8515
8516	adapter->flags2 &= ~IXGBE_FLAG2_PHY_INTERRUPT;
8517
8518	if (!hw->phy.ops.handle_lasi)
8519	return;
8520
8521	hw->phy.ops.handle_lasi(&adapter->hw, &overtemp);
8522	if (overtemp)
8523	e_crit(drv, "%s\n", ixgbe_overheat_msg);
8524	}
8525
8526	static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
8527	{
8528	if (!test_and_clear_bit(nr: __IXGBE_RESET_REQUESTED, addr: &adapter->state))
8529	return;
8530
8531	rtnl_lock();
8532	/* If we're already down, removing or resetting, just bail */
8533	if (test_bit(__IXGBE_DOWN, &adapter->state) ||
8534	test_bit(__IXGBE_REMOVING, &adapter->state) ||
8535	test_bit(__IXGBE_RESETTING, &adapter->state)) {
8536	rtnl_unlock();
8537	return;
8538	}
8539
8540	ixgbe_dump(adapter);
8541	netdev_err(dev: adapter->netdev, format: "Reset adapter\n");
8542	adapter->tx_timeout_count++;
8543
8544	ixgbe_reinit_locked(adapter);
8545	rtnl_unlock();
8546	}
8547
8548	static int ixgbe_check_fw_api_mismatch(struct ixgbe_adapter *adapter)
8549	{
8550	struct ixgbe_hw *hw = &adapter->hw;
8551
8552	if (hw->mac.type != ixgbe_mac_e610)
8553	return 0;
8554
8555	if (hw->mac.ops.get_fw_ver && hw->mac.ops.get_fw_ver(hw))
8556	return 0;
8557
8558	if (hw->api_maj_ver > IXGBE_FW_API_VER_MAJOR) {
8559	e_dev_err("The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
8560
8561	adapter->flags2 |= IXGBE_FLAG2_API_MISMATCH;
8562	return -EOPNOTSUPP;
8563	} else if (hw->api_maj_ver == IXGBE_FW_API_VER_MAJOR &&
8564	hw->api_min_ver > IXGBE_FW_API_VER_MINOR + IXGBE_FW_API_VER_DIFF_ALLOWED) {
8565	e_dev_info("The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
8566	adapter->flags2 |= IXGBE_FLAG2_API_MISMATCH;
8567	} else if (hw->api_maj_ver < IXGBE_FW_API_VER_MAJOR ||
8568	hw->api_min_ver < IXGBE_FW_API_VER_MINOR - IXGBE_FW_API_VER_DIFF_ALLOWED) {
8569	e_dev_info("The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
8570	adapter->flags2 |= IXGBE_FLAG2_API_MISMATCH;
8571	}
8572
8573	return 0;
8574	}
8575
8576	/**
8577	* ixgbe_check_fw_error - Check firmware for errors
8578	* @adapter: the adapter private structure
8579	*
8580	* Check firmware errors in register FWSM
8581	*/
8582	static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter)
8583	{
8584	struct ixgbe_hw *hw = &adapter->hw;
8585	u32 fwsm;
8586	int err;
8587
8588	/* read fwsm.ext_err_ind register and log errors */
8589	fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw));
8590
8591	/* skip if E610's FW is reloading, warning in that case may be misleading */
8592	if (fwsm & IXGBE_FWSM_EXT_ERR_IND_MASK ||
8593	(!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610)))
8594	e_dev_warn("Warning firmware error detected FWSM: 0x%08X\n",
8595	fwsm);
8596
8597	if (hw->mac.ops.fw_recovery_mode && hw->mac.ops.fw_recovery_mode(hw)) {
8598	e_dev_err("Firmware recovery mode detected. Limiting functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
8599	return true;
8600	}
8601	if (!(adapter->flags2 & IXGBE_FLAG2_API_MISMATCH)) {
8602	err = ixgbe_check_fw_api_mismatch(adapter);
8603	if (err)
8604	return true;
8605	}
8606
8607	/* return here if FW rollback mode has been already detected */
8608	if (adapter->flags2 & IXGBE_FLAG2_FW_ROLLBACK)
8609	return false;
8610
8611	if (hw->mac.ops.fw_rollback_mode && hw->mac.ops.fw_rollback_mode(hw)) {
8612	struct ixgbe_nvm_info *nvm_info = &adapter->hw.flash.nvm;
8613	char ver_buff[64] = "";
8614
8615	if (hw->mac.ops.get_fw_ver && hw->mac.ops.get_fw_ver(hw))
8616	goto no_version;
8617
8618	if (hw->mac.ops.get_nvm_ver &&
8619	hw->mac.ops.get_nvm_ver(hw, nvm_info))
8620	goto no_version;
8621
8622	snprintf(buf: ver_buff, size: sizeof(ver_buff),
8623	fmt: "Current version is NVM:%x.%x.%x, FW:%d.%d. ",
8624	nvm_info->major, nvm_info->minor, nvm_info->eetrack,
8625	hw->fw_maj_ver, hw->fw_maj_ver);
8626	no_version:
8627	e_dev_warn("Firmware rollback mode detected. %sDevice may exhibit limited functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware rollback mode.",
8628	ver_buff);
8629
8630	adapter->flags2 |= IXGBE_FLAG2_FW_ROLLBACK;
8631	}
8632
8633	return false;
8634	}
8635
8636	static void ixgbe_recovery_service_task(struct work_struct *work)
8637	{
8638	struct ixgbe_adapter *adapter = container_of(work,
8639	struct ixgbe_adapter,
8640	service_task);
8641
8642	ixgbe_handle_fw_event(adapter);
8643	ixgbe_service_event_complete(adapter);
8644
8645	mod_timer(timer: &adapter->service_timer, expires: jiffies + msecs_to_jiffies(m: 100));
8646	}
8647
8648	/**
8649	* ixgbe_service_task - manages and runs subtasks
8650	* @work: pointer to work_struct containing our data
8651	**/
8652	static void ixgbe_service_task(struct work_struct *work)
8653	{
8654	struct ixgbe_adapter *adapter = container_of(work,
8655	struct ixgbe_adapter,
8656	service_task);
8657	if (ixgbe_removed(addr: adapter->hw.hw_addr)) {
8658	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
8659	rtnl_lock();
8660	ixgbe_down(adapter);
8661	rtnl_unlock();
8662	}
8663	ixgbe_service_event_complete(adapter);
8664	return;
8665	}
8666	if (ixgbe_check_fw_error(adapter)) {
8667	if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
8668	if (adapter->mii_bus) {
8669	mdiobus_unregister(bus: adapter->mii_bus);
8670	adapter->mii_bus = NULL;
8671	}
8672	unregister_netdev(dev: adapter->netdev);
8673	}
8674	ixgbe_service_event_complete(adapter);
8675	return;
8676	}
8677	if (adapter->hw.mac.type == ixgbe_mac_e610) {
8678	if (adapter->flags2 & IXGBE_FLAG2_FW_ASYNC_EVENT)
8679	ixgbe_handle_fw_event(adapter);
8680	ixgbe_check_media_subtask(adapter);
8681	}
8682	ixgbe_reset_subtask(adapter);
8683	ixgbe_phy_interrupt_subtask(adapter);
8684	ixgbe_sfp_detection_subtask(adapter);
8685	ixgbe_sfp_link_config_subtask(adapter);
8686	ixgbe_check_overtemp_subtask(adapter);
8687	ixgbe_watchdog_subtask(adapter);
8688	ixgbe_fdir_reinit_subtask(adapter);
8689	ixgbe_check_hang_subtask(adapter);
8690
8691	if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) {
8692	ixgbe_ptp_overflow_check(adapter);
8693	if (adapter->flags & IXGBE_FLAG_RX_HWTSTAMP_IN_REGISTER)
8694	ixgbe_ptp_rx_hang(adapter);
8695	ixgbe_ptp_tx_hang(adapter);
8696	}
8697
8698	ixgbe_service_event_complete(adapter);
8699	}
8700
8701	static int ixgbe_tso(struct ixgbe_ring *tx_ring,
8702	struct ixgbe_tx_buffer *first,
8703	u8 *hdr_len,
8704	struct ixgbe_ipsec_tx_data *itd)
8705	{
8706	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
8707	struct sk_buff *skb = first->skb;
8708	union {
8709	struct iphdr *v4;
8710	struct ipv6hdr *v6;
8711	unsigned char *hdr;
8712	} ip;
8713	union {
8714	struct tcphdr *tcp;
8715	struct udphdr *udp;
8716	unsigned char *hdr;
8717	} l4;
8718	u32 paylen, l4_offset;
8719	u32 fceof_saidx = 0;
8720	int err;
8721
8722	if (skb->ip_summed != CHECKSUM_PARTIAL)
8723	return 0;
8724
8725	if (!skb_is_gso(skb))
8726	return 0;
8727
8728	err = skb_cow_head(skb, headroom: 0);
8729	if (err < 0)
8730	return err;
8731
8732	if (eth_p_mpls(eth_type: first->protocol))
8733	ip.hdr = skb_inner_network_header(skb);
8734	else
8735	ip.hdr = skb_network_header(skb);
8736	l4.hdr = skb_checksum_start(skb);
8737
8738	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
8739	type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ?
8740	IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP;
8741
8742	/* initialize outer IP header fields */
8743	if (ip.v4->version == 4) {
8744	unsigned char *csum_start = skb_checksum_start(skb);
8745	unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
8746	int len = csum_start - trans_start;
8747
8748	/* IP header will have to cancel out any data that
8749	* is not a part of the outer IP header, so set to
8750	* a reverse csum if needed, else init check to 0.
8751	*/
8752	ip.v4->check = (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) ?
8753	csum_fold(csum: csum_partial(buff: trans_start,
8754	len, sum: 0)) : 0;
8755	type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
8756
8757	ip.v4->tot_len = 0;
8758	first->tx_flags |= IXGBE_TX_FLAGS_TSO |
8759	IXGBE_TX_FLAGS_CSUM |
8760	IXGBE_TX_FLAGS_IPV4;
8761	} else {
8762	ip.v6->payload_len = 0;
8763	first->tx_flags |= IXGBE_TX_FLAGS_TSO |
8764	IXGBE_TX_FLAGS_CSUM;
8765	}
8766
8767	/* determine offset of inner transport header */
8768	l4_offset = l4.hdr - skb->data;
8769
8770	/* remove payload length from inner checksum */
8771	paylen = skb->len - l4_offset;
8772
8773	if (type_tucmd & IXGBE_ADVTXD_TUCMD_L4T_TCP) {
8774	/* compute length of segmentation header */
8775	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
8776	csum_replace_by_diff(sum: &l4.tcp->check,
8777	diff: (__force __wsum)htonl(paylen));
8778	} else {
8779	/* compute length of segmentation header */
8780	*hdr_len = sizeof(*l4.udp) + l4_offset;
8781	csum_replace_by_diff(sum: &l4.udp->check,
8782	diff: (__force __wsum)htonl(paylen));
8783	}
8784
8785	/* update gso size and bytecount with header size */
8786	first->gso_segs = skb_shinfo(skb)->gso_segs;
8787	first->bytecount += (first->gso_segs - 1) * *hdr_len;
8788
8789	/* mss_l4len_id: use 0 as index for TSO */
8790	mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT;
8791	mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
8792
8793	fceof_saidx |= itd->sa_idx;
8794	type_tucmd |= itd->flags | itd->trailer_len;
8795
8796	/* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
8797	vlan_macip_lens = l4.hdr - ip.hdr;
8798	vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT;
8799	vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
8800
8801	ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd,
8802	mss_l4len_idx);
8803
8804	return 1;
8805	}
8806
8807	static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
8808	struct ixgbe_tx_buffer *first,
8809	struct ixgbe_ipsec_tx_data *itd)
8810	{
8811	struct sk_buff *skb = first->skb;
8812	u32 vlan_macip_lens = 0;
8813	u32 fceof_saidx = 0;
8814	u32 type_tucmd = 0;
8815
8816	if (skb->ip_summed != CHECKSUM_PARTIAL) {
8817	csum_failed:
8818	if (!(first->tx_flags & (IXGBE_TX_FLAGS_HW_VLAN |
8819	IXGBE_TX_FLAGS_CC)))
8820	return;
8821	goto no_csum;
8822	}
8823
8824	switch (skb->csum_offset) {
8825	case offsetof(struct tcphdr, check):
8826	type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
8827	fallthrough;
8828	case offsetof(struct udphdr, check):
8829	break;
8830	case offsetof(struct sctphdr, checksum):
8831	/* validate that this is actually an SCTP request */
8832	if (skb_csum_is_sctp(skb)) {
8833	type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP;
8834	break;
8835	}
8836	fallthrough;
8837	default:
8838	skb_checksum_help(skb);
8839	goto csum_failed;
8840	}
8841
8842	/* update TX checksum flag */
8843	first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
8844	vlan_macip_lens = skb_checksum_start_offset(skb) -
8845	skb_network_offset(skb);
8846	no_csum:
8847	/* vlan_macip_lens: MACLEN, VLAN tag */
8848	vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
8849	vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
8850
8851	fceof_saidx |= itd->sa_idx;
8852	type_tucmd |= itd->flags | itd->trailer_len;
8853
8854	ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd, 0);
8855	}
8856
8857	#define IXGBE_SET_FLAG(_input, _flag, _result) \
8858	((_flag <= _result) ? \
8859	((u32)(_input & _flag) * (_result / _flag)) : \
8860	((u32)(_input & _flag) / (_flag / _result)))
8861
8862	static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
8863	{
8864	/* set type for advanced descriptor with frame checksum insertion */
8865	u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA |
8866	IXGBE_ADVTXD_DCMD_DEXT |
8867	IXGBE_ADVTXD_DCMD_IFCS;
8868
8869	/* set HW vlan bit if vlan is present */
8870	cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN,
8871	IXGBE_ADVTXD_DCMD_VLE);
8872
8873	/* set segmentation enable bits for TSO/FSO */
8874	cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSO,
8875	IXGBE_ADVTXD_DCMD_TSE);
8876
8877	/* set timestamp bit if present */
8878	cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSTAMP,
8879	IXGBE_ADVTXD_MAC_TSTAMP);
8880
8881	/* insert frame checksum */
8882	cmd_type ^= IXGBE_SET_FLAG(skb->no_fcs, 1, IXGBE_ADVTXD_DCMD_IFCS);
8883
8884	return cmd_type;
8885	}
8886
8887	static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
8888	u32 tx_flags, unsigned int paylen)
8889	{
8890	u32 olinfo_status = paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
8891
8892	/* enable L4 checksum for TSO and TX checksum offload */
8893	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
8894	IXGBE_TX_FLAGS_CSUM,
8895	IXGBE_ADVTXD_POPTS_TXSM);
8896
8897	/* enable IPv4 checksum for TSO */
8898	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
8899	IXGBE_TX_FLAGS_IPV4,
8900	IXGBE_ADVTXD_POPTS_IXSM);
8901
8902	/* enable IPsec */
8903	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
8904	IXGBE_TX_FLAGS_IPSEC,
8905	IXGBE_ADVTXD_POPTS_IPSEC);
8906
8907	/*
8908	* Check Context must be set if Tx switch is enabled, which it
8909	* always is for case where virtual functions are running
8910	*/
8911	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
8912	IXGBE_TX_FLAGS_CC,
8913	IXGBE_ADVTXD_CC);
8914
8915	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
8916	}
8917
8918	static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
8919	{
8920	if (!netif_subqueue_try_stop(tx_ring->netdev, tx_ring->queue_index,
8921	ixgbe_desc_unused(tx_ring), size))
8922	return -EBUSY;
8923
8924	++tx_ring->tx_stats.restart_queue;
8925	return 0;
8926	}
8927
8928	static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
8929	{
8930	if (likely(ixgbe_desc_unused(tx_ring) >= size))
8931	return 0;
8932
8933	return __ixgbe_maybe_stop_tx(tx_ring, size);
8934	}
8935
8936	static int ixgbe_tx_map(struct ixgbe_ring *tx_ring,
8937	struct ixgbe_tx_buffer *first,
8938	const u8 hdr_len)
8939	{
8940	struct sk_buff *skb = first->skb;
8941	struct ixgbe_tx_buffer *tx_buffer;
8942	union ixgbe_adv_tx_desc *tx_desc;
8943	skb_frag_t *frag;
8944	dma_addr_t dma;
8945	unsigned int data_len, size;
8946	u32 tx_flags = first->tx_flags;
8947	u32 cmd_type = ixgbe_tx_cmd_type(skb, tx_flags);
8948	u16 i = tx_ring->next_to_use;
8949
8950	tx_desc = IXGBE_TX_DESC(tx_ring, i);
8951
8952	ixgbe_tx_olinfo_status(tx_desc, tx_flags, paylen: skb->len - hdr_len);
8953
8954	size = skb_headlen(skb);
8955	data_len = skb->data_len;
8956
8957	#ifdef IXGBE_FCOE
8958	if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
8959	if (data_len < sizeof(struct fcoe_crc_eof)) {
8960	size -= sizeof(struct fcoe_crc_eof) - data_len;
8961	data_len = 0;
8962	} else {
8963	data_len -= sizeof(struct fcoe_crc_eof);
8964	}
8965	}
8966
8967	#endif
8968	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
8969
8970	tx_buffer = first;
8971
8972	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
8973	if (dma_mapping_error(dev: tx_ring->dev, dma_addr: dma))
8974	goto dma_error;
8975
8976	/* record length, and DMA address */
8977	dma_unmap_len_set(tx_buffer, len, size);
8978	dma_unmap_addr_set(tx_buffer, dma, dma);
8979
8980	tx_desc->read.buffer_addr = cpu_to_le64(dma);
8981
8982	while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
8983	tx_desc->read.cmd_type_len =
8984	cpu_to_le32(cmd_type ^ IXGBE_MAX_DATA_PER_TXD);
8985
8986	i++;
8987	tx_desc++;
8988	if (i == tx_ring->count) {
8989	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
8990	i = 0;
8991	}
8992	tx_desc->read.olinfo_status = 0;
8993
8994	dma += IXGBE_MAX_DATA_PER_TXD;
8995	size -= IXGBE_MAX_DATA_PER_TXD;
8996
8997	tx_desc->read.buffer_addr = cpu_to_le64(dma);
8998	}
8999
9000	if (likely(!data_len))
9001	break;
9002
9003	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
9004
9005	i++;
9006	tx_desc++;
9007	if (i == tx_ring->count) {
9008	tx_desc = IXGBE_TX_DESC(tx_ring, 0);
9009	i = 0;
9010	}
9011	tx_desc->read.olinfo_status = 0;
9012
9013	#ifdef IXGBE_FCOE
9014	size = min_t(unsigned int, data_len, skb_frag_size(frag));
9015	#else
9016	size = skb_frag_size(frag);
9017	#endif
9018	data_len -= size;
9019
9020	dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
9021	DMA_TO_DEVICE);
9022
9023	tx_buffer = &tx_ring->tx_buffer_info[i];
9024	}
9025
9026	/* write last descriptor with RS and EOP bits */
9027	cmd_type |= size | IXGBE_TXD_CMD;
9028	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
9029
9030	netdev_tx_sent_queue(dev_queue: txring_txq(ring: tx_ring), bytes: first->bytecount);
9031
9032	/* set the timestamp */
9033	first->time_stamp = jiffies;
9034
9035	skb_tx_timestamp(skb);
9036
9037	/*
9038	* Force memory writes to complete before letting h/w know there
9039	* are new descriptors to fetch. (Only applicable for weak-ordered
9040	* memory model archs, such as IA-64).
9041	*
9042	* We also need this memory barrier to make certain all of the
9043	* status bits have been updated before next_to_watch is written.
9044	*/
9045	wmb();
9046
9047	/* set next_to_watch value indicating a packet is present */
9048	first->next_to_watch = tx_desc;
9049
9050	i++;
9051	if (i == tx_ring->count)
9052	i = 0;
9053
9054	tx_ring->next_to_use = i;
9055
9056	ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
9057
9058	if (netif_xmit_stopped(dev_queue: txring_txq(ring: tx_ring)) || !netdev_xmit_more()) {
9059	writel(val: i, addr: tx_ring->tail);
9060	}
9061
9062	return 0;
9063	dma_error:
9064	dev_err(tx_ring->dev, "TX DMA map failed\n");
9065
9066	/* clear dma mappings for failed tx_buffer_info map */
9067	for (;;) {
9068	tx_buffer = &tx_ring->tx_buffer_info[i];
9069	if (dma_unmap_len(tx_buffer, len))
9070	dma_unmap_page(tx_ring->dev,
9071	dma_unmap_addr(tx_buffer, dma),
9072	dma_unmap_len(tx_buffer, len),
9073	DMA_TO_DEVICE);
9074	dma_unmap_len_set(tx_buffer, len, 0);
9075	if (tx_buffer == first)
9076	break;
9077	if (i == 0)
9078	i += tx_ring->count;
9079	i--;
9080	}
9081
9082	dev_kfree_skb_any(skb: first->skb);
9083	first->skb = NULL;
9084
9085	tx_ring->next_to_use = i;
9086
9087	return -1;
9088	}
9089
9090	static void ixgbe_atr(struct ixgbe_ring *ring,
9091	struct ixgbe_tx_buffer *first)
9092	{
9093	struct ixgbe_q_vector *q_vector = ring->q_vector;
9094	union ixgbe_atr_hash_dword input = { .dword = 0 };
9095	union ixgbe_atr_hash_dword common = { .dword = 0 };
9096	union {
9097	unsigned char *network;
9098	struct iphdr *ipv4;
9099	struct ipv6hdr *ipv6;
9100	} hdr;
9101	struct tcphdr *th;
9102	unsigned int hlen;
9103	struct sk_buff *skb;
9104	__be16 vlan_id;
9105	int l4_proto;
9106
9107	/* if ring doesn't have a interrupt vector, cannot perform ATR */
9108	if (!q_vector)
9109	return;
9110
9111	/* do nothing if sampling is disabled */
9112	if (!ring->atr_sample_rate)
9113	return;
9114
9115	ring->atr_count++;
9116
9117	/* currently only IPv4/IPv6 with TCP is supported */
9118	if ((first->protocol != htons(ETH_P_IP)) &&
9119	(first->protocol != htons(ETH_P_IPV6)))
9120	return;
9121
9122	/* snag network header to get L4 type and address */
9123	skb = first->skb;
9124	hdr.network = skb_network_header(skb);
9125	if (unlikely(hdr.network <= skb->data))
9126	return;
9127	if (skb->encapsulation &&
9128	first->protocol == htons(ETH_P_IP) &&
9129	hdr.ipv4->protocol == IPPROTO_UDP) {
9130	struct ixgbe_adapter *adapter = q_vector->adapter;
9131
9132	if (unlikely(skb_tail_pointer(skb) < hdr.network +
9133	vxlan_headroom(0)))
9134	return;
9135
9136	/* verify the port is recognized as VXLAN */
9137	if (adapter->vxlan_port &&
9138	udp_hdr(skb)->dest == adapter->vxlan_port)
9139	hdr.network = skb_inner_network_header(skb);
9140
9141	if (adapter->geneve_port &&
9142	udp_hdr(skb)->dest == adapter->geneve_port)
9143	hdr.network = skb_inner_network_header(skb);
9144	}
9145
9146	/* Make sure we have at least [minimum IPv4 header + TCP]
9147	* or [IPv6 header] bytes
9148	*/
9149	if (unlikely(skb_tail_pointer(skb) < hdr.network + 40))
9150	return;
9151
9152	/* Currently only IPv4/IPv6 with TCP is supported */
9153	switch (hdr.ipv4->version) {
9154	case IPVERSION:
9155	/* access ihl as u8 to avoid unaligned access on ia64 */
9156	hlen = (hdr.network[0] & 0x0F) << 2;
9157	l4_proto = hdr.ipv4->protocol;
9158	break;
9159	case 6:
9160	hlen = hdr.network - skb->data;
9161	l4_proto = ipv6_find_hdr(skb, offset: &hlen, IPPROTO_TCP, NULL, NULL);
9162	hlen -= hdr.network - skb->data;
9163	break;
9164	default:
9165	return;
9166	}
9167
9168	if (l4_proto != IPPROTO_TCP)
9169	return;
9170
9171	if (unlikely(skb_tail_pointer(skb) < hdr.network +
9172	hlen + sizeof(struct tcphdr)))
9173	return;
9174
9175	th = (struct tcphdr *)(hdr.network + hlen);
9176
9177	/* skip this packet since the socket is closing */
9178	if (th->fin)
9179	return;
9180
9181	/* sample on all syn packets or once every atr sample count */
9182	if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
9183	return;
9184
9185	/* reset sample count */
9186	ring->atr_count = 0;
9187
9188	vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);
9189
9190	/*
9191	* src and dst are inverted, think how the receiver sees them
9192	*
9193	* The input is broken into two sections, a non-compressed section
9194	* containing vm_pool, vlan_id, and flow_type. The rest of the data
9195	* is XORed together and stored in the compressed dword.
9196	*/
9197	input.formatted.vlan_id = vlan_id;
9198
9199	/*
9200	* since src port and flex bytes occupy the same word XOR them together
9201	* and write the value to source port portion of compressed dword
9202	*/
9203	if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN))
9204	common.port.src ^= th->dest ^ htons(ETH_P_8021Q);
9205	else
9206	common.port.src ^= th->dest ^ first->protocol;
9207	common.port.dst ^= th->source;
9208
9209	switch (hdr.ipv4->version) {
9210	case IPVERSION:
9211	input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
9212	common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
9213	break;
9214	case 6:
9215	input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
9216	common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
9217	hdr.ipv6->saddr.s6_addr32[1] ^
9218	hdr.ipv6->saddr.s6_addr32[2] ^
9219	hdr.ipv6->saddr.s6_addr32[3] ^
9220	hdr.ipv6->daddr.s6_addr32[0] ^
9221	hdr.ipv6->daddr.s6_addr32[1] ^
9222	hdr.ipv6->daddr.s6_addr32[2] ^
9223	hdr.ipv6->daddr.s6_addr32[3];
9224	break;
9225	default:
9226	break;
9227	}
9228
9229	if (hdr.network != skb_network_header(skb))
9230	input.formatted.flow_type |= IXGBE_ATR_L4TYPE_TUNNEL_MASK;
9231
9232	/* This assumes the Rx queue and Tx queue are bound to the same CPU */
9233	ixgbe_fdir_add_signature_filter_82599(hw: &q_vector->adapter->hw,
9234	input, common, queue: ring->queue_index);
9235	}
9236
9237	#ifdef IXGBE_FCOE
9238	static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb,
9239	struct net_device *sb_dev)
9240	{
9241	struct ixgbe_adapter *adapter;
9242	struct ixgbe_ring_feature *f;
9243	int txq;
9244
9245	if (sb_dev) {
9246	u8 tc = netdev_get_prio_tc_map(dev, prio: skb->priority);
9247	struct net_device *vdev = sb_dev;
9248
9249	txq = vdev->tc_to_txq[tc].offset;
9250	txq += reciprocal_scale(val: skb_get_hash(skb),
9251	ep_ro: vdev->tc_to_txq[tc].count);
9252
9253	return txq;
9254	}
9255
9256	/*
9257	* only execute the code below if protocol is FCoE
9258	* or FIP and we have FCoE enabled on the adapter
9259	*/
9260	switch (vlan_get_protocol(skb)) {
9261	case htons(ETH_P_FCOE):
9262	case htons(ETH_P_FIP):
9263	adapter = ixgbe_from_netdev(netdev: dev);
9264
9265	if (!sb_dev && (adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
9266	break;
9267	fallthrough;
9268	default:
9269	return netdev_pick_tx(dev, skb, sb_dev);
9270	}
9271
9272	f = &adapter->ring_feature[RING_F_FCOE];
9273
9274	txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :
9275	smp_processor_id();
9276
9277	while (txq >= f->indices)
9278	txq -= f->indices;
9279
9280	return txq + f->offset;
9281	}
9282
9283	#endif
9284	int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring,
9285	struct xdp_frame *xdpf)
9286	{
9287	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(frame: xdpf);
9288	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
9289	u16 i = 0, index = ring->next_to_use;
9290	struct ixgbe_tx_buffer *tx_head = &ring->tx_buffer_info[index];
9291	struct ixgbe_tx_buffer *tx_buff = tx_head;
9292	union ixgbe_adv_tx_desc *tx_desc = IXGBE_TX_DESC(ring, index);
9293	u32 cmd_type, len = xdpf->len;
9294	void *data = xdpf->data;
9295
9296	if (unlikely(ixgbe_desc_unused(ring) < 1 + nr_frags))
9297	return IXGBE_XDP_CONSUMED;
9298
9299	tx_head->bytecount = xdp_get_frame_len(xdpf);
9300	tx_head->gso_segs = 1;
9301	tx_head->xdpf = xdpf;
9302
9303	tx_desc->read.olinfo_status =
9304	cpu_to_le32(tx_head->bytecount << IXGBE_ADVTXD_PAYLEN_SHIFT);
9305
9306	for (;;) {
9307	dma_addr_t dma;
9308
9309	dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);
9310	if (dma_mapping_error(dev: ring->dev, dma_addr: dma))
9311	goto unmap;
9312
9313	dma_unmap_len_set(tx_buff, len, len);
9314	dma_unmap_addr_set(tx_buff, dma, dma);
9315
9316	cmd_type = IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_DEXT |
9317	IXGBE_ADVTXD_DCMD_IFCS | len;
9318	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
9319	tx_desc->read.buffer_addr = cpu_to_le64(dma);
9320	tx_buff->protocol = 0;
9321
9322	if (++index == ring->count)
9323	index = 0;
9324
9325	if (i == nr_frags)
9326	break;
9327
9328	tx_buff = &ring->tx_buffer_info[index];
9329	tx_desc = IXGBE_TX_DESC(ring, index);
9330	tx_desc->read.olinfo_status = 0;
9331
9332	data = skb_frag_address(frag: &sinfo->frags[i]);
9333	len = skb_frag_size(frag: &sinfo->frags[i]);
9334	i++;
9335	}
9336	/* put descriptor type bits */
9337	tx_desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD);
9338
9339	/* Avoid any potential race with xdp_xmit and cleanup */
9340	smp_wmb();
9341
9342	tx_head->next_to_watch = tx_desc;
9343	ring->next_to_use = index;
9344
9345	return IXGBE_XDP_TX;
9346
9347	unmap:
9348	for (;;) {
9349	tx_buff = &ring->tx_buffer_info[index];
9350	if (dma_unmap_len(tx_buff, len))
9351	dma_unmap_page(ring->dev, dma_unmap_addr(tx_buff, dma),
9352	dma_unmap_len(tx_buff, len),
9353	DMA_TO_DEVICE);
9354	dma_unmap_len_set(tx_buff, len, 0);
9355	if (tx_buff == tx_head)
9356	break;
9357
9358	if (!index)
9359	index += ring->count;
9360	index--;
9361	}
9362
9363	return IXGBE_XDP_CONSUMED;
9364	}
9365
9366	netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
9367	struct ixgbe_adapter *adapter,
9368	struct ixgbe_ring *tx_ring)
9369	{
9370	struct ixgbe_tx_buffer *first;
9371	int tso;
9372	u32 tx_flags = 0;
9373	unsigned short f;
9374	u16 count = TXD_USE_COUNT(skb_headlen(skb));
9375	struct ixgbe_ipsec_tx_data ipsec_tx = { 0 };
9376	__be16 protocol = skb->protocol;
9377	u8 hdr_len = 0;
9378
9379	/*
9380	* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
9381	* + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
9382	* + 2 desc gap to keep tail from touching head,
9383	* + 1 desc for context descriptor,
9384	* otherwise try next time
9385	*/
9386	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
9387	count += TXD_USE_COUNT(skb_frag_size(
9388	&skb_shinfo(skb)->frags[f]));
9389
9390	if (ixgbe_maybe_stop_tx(tx_ring, size: count + 3)) {
9391	tx_ring->tx_stats.tx_busy++;
9392	return NETDEV_TX_BUSY;
9393	}
9394
9395	/* record the location of the first descriptor for this packet */
9396	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
9397	first->skb = skb;
9398	first->bytecount = skb->len;
9399	first->gso_segs = 1;
9400
9401	/* if we have a HW VLAN tag being added default to the HW one */
9402	if (skb_vlan_tag_present(skb)) {
9403	tx_flags |= skb_vlan_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT;
9404	tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
9405	/* else if it is a SW VLAN check the next protocol and store the tag */
9406	} else if (protocol == htons(ETH_P_8021Q)) {
9407	struct vlan_hdr *vhdr, _vhdr;
9408	vhdr = skb_header_pointer(skb, ETH_HLEN, len: sizeof(_vhdr), buffer: &_vhdr);
9409	if (!vhdr)
9410	goto out_drop;
9411
9412	tx_flags |= ntohs(vhdr->h_vlan_TCI) <<
9413	IXGBE_TX_FLAGS_VLAN_SHIFT;
9414	tx_flags |= IXGBE_TX_FLAGS_SW_VLAN;
9415	}
9416	protocol = vlan_get_protocol(skb);
9417
9418	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
9419	adapter->ptp_clock) {
9420	if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
9421	!test_and_set_bit_lock(nr: __IXGBE_PTP_TX_IN_PROGRESS,
9422	addr: &adapter->state)) {
9423	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
9424	tx_flags |= IXGBE_TX_FLAGS_TSTAMP;
9425
9426	/* schedule check for Tx timestamp */
9427	adapter->ptp_tx_skb = skb_get(skb);
9428	adapter->ptp_tx_start = jiffies;
9429	schedule_work(work: &adapter->ptp_tx_work);
9430	} else {
9431	adapter->tx_hwtstamp_skipped++;
9432	}
9433	}
9434
9435	#ifdef CONFIG_PCI_IOV
9436	/*
9437	* Use the l2switch_enable flag - would be false if the DMA
9438	* Tx switch had been disabled.
9439	*/
9440	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
9441	tx_flags |= IXGBE_TX_FLAGS_CC;
9442
9443	#endif
9444	/* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */
9445	if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
9446	((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) ||
9447	(skb->priority != TC_PRIO_CONTROL))) {
9448	tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
9449	tx_flags |= (skb->priority & 0x7) <<
9450	IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT;
9451	if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) {
9452	struct vlan_ethhdr *vhdr;
9453
9454	if (skb_cow_head(skb, headroom: 0))
9455	goto out_drop;
9456	vhdr = skb_vlan_eth_hdr(skb);
9457	vhdr->h_vlan_TCI = htons(tx_flags >>
9458	IXGBE_TX_FLAGS_VLAN_SHIFT);
9459	} else {
9460	tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;
9461	}
9462	}
9463
9464	/* record initial flags and protocol */
9465	first->tx_flags = tx_flags;
9466	first->protocol = protocol;
9467
9468	#ifdef IXGBE_FCOE
9469	/* setup tx offload for FCoE */
9470	if ((protocol == htons(ETH_P_FCOE)) &&
9471	(tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) {
9472	tso = ixgbe_fso(tx_ring, first, hdr_len: &hdr_len);
9473	if (tso < 0)
9474	goto out_drop;
9475
9476	goto xmit_fcoe;
9477	}
9478
9479	#endif /* IXGBE_FCOE */
9480
9481	#ifdef CONFIG_IXGBE_IPSEC
9482	if (xfrm_offload(skb) &&
9483	!ixgbe_ipsec_tx(tx_ring, first, itd: &ipsec_tx))
9484	goto out_drop;
9485	#endif
9486	tso = ixgbe_tso(tx_ring, first, hdr_len: &hdr_len, itd: &ipsec_tx);
9487	if (tso < 0)
9488	goto out_drop;
9489	else if (!tso)
9490	ixgbe_tx_csum(tx_ring, first, itd: &ipsec_tx);
9491
9492	/* add the ATR filter if ATR is on */
9493	if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
9494	ixgbe_atr(ring: tx_ring, first);
9495
9496	#ifdef IXGBE_FCOE
9497	xmit_fcoe:
9498	#endif /* IXGBE_FCOE */
9499	if (ixgbe_tx_map(tx_ring, first, hdr_len))
9500	goto cleanup_tx_timestamp;
9501
9502	return NETDEV_TX_OK;
9503
9504	out_drop:
9505	dev_kfree_skb_any(skb: first->skb);
9506	first->skb = NULL;
9507	cleanup_tx_timestamp:
9508	if (unlikely(tx_flags & IXGBE_TX_FLAGS_TSTAMP)) {
9509	dev_kfree_skb_any(skb: adapter->ptp_tx_skb);
9510	adapter->ptp_tx_skb = NULL;
9511	cancel_work_sync(work: &adapter->ptp_tx_work);
9512	clear_bit_unlock(nr: __IXGBE_PTP_TX_IN_PROGRESS, addr: &adapter->state);
9513	}
9514
9515	return NETDEV_TX_OK;
9516	}
9517
9518	static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb,
9519	struct net_device *netdev,
9520	struct ixgbe_ring *ring)
9521	{
9522	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9523	struct ixgbe_ring *tx_ring;
9524
9525	/*
9526	* The minimum packet size for olinfo paylen is 17 so pad the skb
9527	* in order to meet this minimum size requirement.
9528	*/
9529	if (skb_put_padto(skb, len: 17))
9530	return NETDEV_TX_OK;
9531
9532	tx_ring = ring ? ring : adapter->tx_ring[skb_get_queue_mapping(skb)];
9533	if (unlikely(test_bit(__IXGBE_TX_DISABLED, &tx_ring->state)))
9534	return NETDEV_TX_BUSY;
9535
9536	return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
9537	}
9538
9539	static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb,
9540	struct net_device *netdev)
9541	{
9542	return __ixgbe_xmit_frame(skb, netdev, NULL);
9543	}
9544
9545	/**
9546	* ixgbe_set_mac - Change the Ethernet Address of the NIC
9547	* @netdev: network interface device structure
9548	* @p: pointer to an address structure
9549	*
9550	* Returns 0 on success, negative on failure
9551	**/
9552	static int ixgbe_set_mac(struct net_device *netdev, void *p)
9553	{
9554	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9555	struct ixgbe_hw *hw = &adapter->hw;
9556	struct sockaddr *addr = p;
9557
9558	if (!is_valid_ether_addr(addr: addr->sa_data))
9559	return -EADDRNOTAVAIL;
9560
9561	eth_hw_addr_set(dev: netdev, addr: addr->sa_data);
9562	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
9563
9564	ixgbe_mac_set_default_filter(adapter);
9565
9566	return 0;
9567	}
9568
9569	static int
9570	ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
9571	{
9572	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9573	struct ixgbe_hw *hw = &adapter->hw;
9574	u16 value;
9575	int rc;
9576
9577	if (adapter->mii_bus) {
9578	int regnum = addr;
9579
9580	if (devad != MDIO_DEVAD_NONE)
9581	return mdiobus_c45_read(bus: adapter->mii_bus, addr: prtad,
9582	devad, regnum);
9583
9584	return mdiobus_read(bus: adapter->mii_bus, addr: prtad, regnum);
9585	}
9586
9587	if (prtad != hw->phy.mdio.prtad)
9588	return -EINVAL;
9589	rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
9590	if (!rc)
9591	rc = value;
9592	return rc;
9593	}
9594
9595	static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
9596	u16 addr, u16 value)
9597	{
9598	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9599	struct ixgbe_hw *hw = &adapter->hw;
9600
9601	if (adapter->mii_bus) {
9602	int regnum = addr;
9603
9604	if (devad != MDIO_DEVAD_NONE)
9605	return mdiobus_c45_write(bus: adapter->mii_bus, addr: prtad, devad,
9606	regnum, val: value);
9607
9608	return mdiobus_write(bus: adapter->mii_bus, addr: prtad, regnum, val: value);
9609	}
9610
9611	if (prtad != hw->phy.mdio.prtad)
9612	return -EINVAL;
9613	return hw->phy.ops.write_reg(hw, addr, devad, value);
9614	}
9615
9616	static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
9617	{
9618	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9619
9620	switch (cmd) {
9621	case SIOCGMIIPHY:
9622	if (!adapter->hw.phy.ops.read_reg)
9623	return -EOPNOTSUPP;
9624	fallthrough;
9625	default:
9626	return mdio_mii_ioctl(mdio: &adapter->hw.phy.mdio, mii_data: if_mii(rq: req), cmd);
9627	}
9628	}
9629
9630	/**
9631	* ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
9632	* netdev->dev_addrs
9633	* @dev: network interface device structure
9634	*
9635	* Returns non-zero on failure
9636	**/
9637	static int ixgbe_add_sanmac_netdev(struct net_device *dev)
9638	{
9639	int err = 0;
9640	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
9641	struct ixgbe_hw *hw = &adapter->hw;
9642
9643	if (is_valid_ether_addr(addr: hw->mac.san_addr)) {
9644	rtnl_lock();
9645	err = dev_addr_add(dev, addr: hw->mac.san_addr, NETDEV_HW_ADDR_T_SAN);
9646	rtnl_unlock();
9647
9648	/* update SAN MAC vmdq pool selection */
9649	hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));
9650	}
9651	return err;
9652	}
9653
9654	/**
9655	* ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
9656	* netdev->dev_addrs
9657	* @dev: network interface device structure
9658	*
9659	* Returns non-zero on failure
9660	**/
9661	static int ixgbe_del_sanmac_netdev(struct net_device *dev)
9662	{
9663	int err = 0;
9664	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
9665	struct ixgbe_mac_info *mac = &adapter->hw.mac;
9666
9667	if (is_valid_ether_addr(addr: mac->san_addr)) {
9668	rtnl_lock();
9669	err = dev_addr_del(dev, addr: mac->san_addr, NETDEV_HW_ADDR_T_SAN);
9670	rtnl_unlock();
9671	}
9672	return err;
9673	}
9674
9675	static void ixgbe_get_ring_stats64(struct rtnl_link_stats64 *stats,
9676	struct ixgbe_ring *ring)
9677	{
9678	u64 bytes, packets;
9679	unsigned int start;
9680
9681	if (ring) {
9682	do {
9683	start = u64_stats_fetch_begin(syncp: &ring->syncp);
9684	packets = ring->stats.packets;
9685	bytes = ring->stats.bytes;
9686	} while (u64_stats_fetch_retry(syncp: &ring->syncp, start));
9687	stats->tx_packets += packets;
9688	stats->tx_bytes += bytes;
9689	}
9690	}
9691
9692	static void ixgbe_get_stats64(struct net_device *netdev,
9693	struct rtnl_link_stats64 *stats)
9694	{
9695	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9696	int i;
9697
9698	rcu_read_lock();
9699	for (i = 0; i < adapter->num_rx_queues; i++) {
9700	struct ixgbe_ring *ring = READ_ONCE(adapter->rx_ring[i]);
9701	u64 bytes, packets;
9702	unsigned int start;
9703
9704	if (ring) {
9705	do {
9706	start = u64_stats_fetch_begin(syncp: &ring->syncp);
9707	packets = ring->stats.packets;
9708	bytes = ring->stats.bytes;
9709	} while (u64_stats_fetch_retry(syncp: &ring->syncp, start));
9710	stats->rx_packets += packets;
9711	stats->rx_bytes += bytes;
9712	}
9713	}
9714
9715	for (i = 0; i < adapter->num_tx_queues; i++) {
9716	struct ixgbe_ring *ring = READ_ONCE(adapter->tx_ring[i]);
9717
9718	ixgbe_get_ring_stats64(stats, ring);
9719	}
9720	for (i = 0; i < adapter->num_xdp_queues; i++) {
9721	struct ixgbe_ring *ring = READ_ONCE(adapter->xdp_ring[i]);
9722
9723	ixgbe_get_ring_stats64(stats, ring);
9724	}
9725	rcu_read_unlock();
9726
9727	/* following stats updated by ixgbe_watchdog_task() */
9728	stats->multicast = netdev->stats.multicast;
9729	stats->rx_errors = netdev->stats.rx_errors;
9730	stats->rx_length_errors = netdev->stats.rx_length_errors;
9731	stats->rx_crc_errors = netdev->stats.rx_crc_errors;
9732	stats->rx_missed_errors = netdev->stats.rx_missed_errors;
9733	}
9734
9735	static int ixgbe_ndo_get_vf_stats(struct net_device *netdev, int vf,
9736	struct ifla_vf_stats *vf_stats)
9737	{
9738	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
9739
9740	if (vf < 0 || vf >= adapter->num_vfs)
9741	return -EINVAL;
9742
9743	vf_stats->rx_packets = adapter->vfinfo[vf].vfstats.gprc;
9744	vf_stats->rx_bytes = adapter->vfinfo[vf].vfstats.gorc;
9745	vf_stats->tx_packets = adapter->vfinfo[vf].vfstats.gptc;
9746	vf_stats->tx_bytes = adapter->vfinfo[vf].vfstats.gotc;
9747	vf_stats->multicast = adapter->vfinfo[vf].vfstats.mprc;
9748
9749	return 0;
9750	}
9751
9752	#ifdef CONFIG_IXGBE_DCB
9753	/**
9754	* ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.
9755	* @adapter: pointer to ixgbe_adapter
9756	* @tc: number of traffic classes currently enabled
9757	*
9758	* Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm
9759	* 802.1Q priority maps to a packet buffer that exists.
9760	*/
9761	static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)
9762	{
9763	struct ixgbe_hw *hw = &adapter->hw;
9764	u32 reg, rsave;
9765	int i;
9766
9767	/* 82598 have a static priority to TC mapping that can not
9768	* be changed so no validation is needed.
9769	*/
9770	if (hw->mac.type == ixgbe_mac_82598EB)
9771	return;
9772
9773	reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);
9774	rsave = reg;
9775
9776	for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
9777	u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);
9778
9779	/* If up2tc is out of bounds default to zero */
9780	if (up2tc > tc)
9781	reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);
9782	}
9783
9784	if (reg != rsave)
9785	IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);
9786
9787	return;
9788	}
9789
9790	/**
9791	* ixgbe_set_prio_tc_map - Configure netdev prio tc map
9792	* @adapter: Pointer to adapter struct
9793	*
9794	* Populate the netdev user priority to tc map
9795	*/
9796	static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter)
9797	{
9798	struct net_device *dev = adapter->netdev;
9799	struct ixgbe_dcb_config *dcb_cfg = &adapter->dcb_cfg;
9800	struct ieee_ets *ets = adapter->ixgbe_ieee_ets;
9801	u8 prio;
9802
9803	for (prio = 0; prio < MAX_USER_PRIORITY; prio++) {
9804	u8 tc = 0;
9805
9806	if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE)
9807	tc = ixgbe_dcb_get_tc_from_up(dcb_cfg, 0, prio);
9808	else if (ets)
9809	tc = ets->prio_tc[prio];
9810
9811	netdev_set_prio_tc_map(dev, prio, tc);
9812	}
9813	}
9814
9815	#endif /* CONFIG_IXGBE_DCB */
9816	static int ixgbe_reassign_macvlan_pool(struct net_device *vdev,
9817	struct netdev_nested_priv *priv)
9818	{
9819	struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data;
9820	struct ixgbe_fwd_adapter *accel;
9821	int pool;
9822
9823	/* we only care about macvlans... */
9824	if (!netif_is_macvlan(dev: vdev))
9825	return 0;
9826
9827	/* that have hardware offload enabled... */
9828	accel = macvlan_accel_priv(dev: vdev);
9829	if (!accel)
9830	return 0;
9831
9832	/* If we can relocate to a different bit do so */
9833	pool = find_first_zero_bit(addr: adapter->fwd_bitmask, size: adapter->num_rx_pools);
9834	if (pool < adapter->num_rx_pools) {
9835	set_bit(nr: pool, addr: adapter->fwd_bitmask);
9836	accel->pool = pool;
9837	return 0;
9838	}
9839
9840	/* if we cannot find a free pool then disable the offload */
9841	netdev_err(dev: vdev, format: "L2FW offload disabled due to lack of queue resources\n");
9842	macvlan_release_l2fw_offload(dev: vdev);
9843
9844	/* unbind the queues and drop the subordinate channel config */
9845	netdev_unbind_sb_channel(dev: adapter->netdev, sb_dev: vdev);
9846	netdev_set_sb_channel(dev: vdev, channel: 0);
9847
9848	kfree(objp: accel);
9849
9850	return 0;
9851	}
9852
9853	static void ixgbe_defrag_macvlan_pools(struct net_device *dev)
9854	{
9855	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
9856	struct netdev_nested_priv priv = {
9857	.data = (void *)adapter,
9858	};
9859
9860	/* flush any stale bits out of the fwd bitmask */
9861	bitmap_clear(map: adapter->fwd_bitmask, start: 1, nbits: 63);
9862
9863	/* walk through upper devices reassigning pools */
9864	netdev_walk_all_upper_dev_rcu(dev, fn: ixgbe_reassign_macvlan_pool,
9865	priv: &priv);
9866	}
9867
9868	/**
9869	* ixgbe_setup_tc - configure net_device for multiple traffic classes
9870	*
9871	* @dev: net device to configure
9872	* @tc: number of traffic classes to enable
9873	*/
9874	int ixgbe_setup_tc(struct net_device *dev, u8 tc)
9875	{
9876	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
9877	struct ixgbe_hw *hw = &adapter->hw;
9878
9879	/* Hardware supports up to 8 traffic classes */
9880	if (tc > adapter->dcb_cfg.num_tcs.pg_tcs)
9881	return -EINVAL;
9882
9883	if (hw->mac.type == ixgbe_mac_82598EB && tc && tc < MAX_TRAFFIC_CLASS)
9884	return -EINVAL;
9885
9886	/* Hardware has to reinitialize queues and interrupts to
9887	* match packet buffer alignment. Unfortunately, the
9888	* hardware is not flexible enough to do this dynamically.
9889	*/
9890	if (netif_running(dev))
9891	ixgbe_close(netdev: dev);
9892	else
9893	ixgbe_reset(adapter);
9894
9895	ixgbe_clear_interrupt_scheme(adapter);
9896
9897	#ifdef CONFIG_IXGBE_DCB
9898	if (tc) {
9899	if (adapter->xdp_prog) {
9900	e_warn(probe, "DCB is not supported with XDP\n");
9901
9902	ixgbe_init_interrupt_scheme(adapter);
9903	if (netif_running(dev))
9904	ixgbe_open(netdev: dev);
9905	return -EINVAL;
9906	}
9907
9908	netdev_set_num_tc(dev, num_tc: tc);
9909	ixgbe_set_prio_tc_map(adapter);
9910
9911	adapter->hw_tcs = tc;
9912	adapter->flags |= IXGBE_FLAG_DCB_ENABLED;
9913
9914	if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
9915	adapter->last_lfc_mode = adapter->hw.fc.requested_mode;
9916	adapter->hw.fc.requested_mode = ixgbe_fc_none;
9917	}
9918	} else {
9919	netdev_reset_tc(dev);
9920
9921	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
9922	adapter->hw.fc.requested_mode = adapter->last_lfc_mode;
9923
9924	adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
9925	adapter->hw_tcs = tc;
9926
9927	adapter->temp_dcb_cfg.pfc_mode_enable = false;
9928	adapter->dcb_cfg.pfc_mode_enable = false;
9929	}
9930
9931	ixgbe_validate_rtr(adapter, tc);
9932
9933	#endif /* CONFIG_IXGBE_DCB */
9934	ixgbe_init_interrupt_scheme(adapter);
9935
9936	ixgbe_defrag_macvlan_pools(dev);
9937
9938	if (netif_running(dev))
9939	return ixgbe_open(netdev: dev);
9940
9941	return 0;
9942	}
9943
9944	static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter,
9945	struct tc_cls_u32_offload *cls)
9946	{
9947	u32 hdl = cls->knode.handle;
9948	u32 uhtid = TC_U32_USERHTID(cls->knode.handle);
9949	u32 loc = cls->knode.handle & 0xfffff;
9950	int err = 0, i, j;
9951	struct ixgbe_jump_table *jump = NULL;
9952
9953	if (loc > IXGBE_MAX_HW_ENTRIES)
9954	return -EINVAL;
9955
9956	if ((uhtid != 0x800) && (uhtid >= IXGBE_MAX_LINK_HANDLE))
9957	return -EINVAL;
9958
9959	/* Clear this filter in the link data it is associated with */
9960	if (uhtid != 0x800) {
9961	jump = adapter->jump_tables[uhtid];
9962	if (!jump)
9963	return -EINVAL;
9964	if (!test_bit(loc - 1, jump->child_loc_map))
9965	return -EINVAL;
9966	clear_bit(nr: loc - 1, addr: jump->child_loc_map);
9967	}
9968
9969	/* Check if the filter being deleted is a link */
9970	for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) {
9971	jump = adapter->jump_tables[i];
9972	if (jump && jump->link_hdl == hdl) {
9973	/* Delete filters in the hardware in the child hash
9974	* table associated with this link
9975	*/
9976	for (j = 0; j < IXGBE_MAX_HW_ENTRIES; j++) {
9977	if (!test_bit(j, jump->child_loc_map))
9978	continue;
9979	spin_lock(lock: &adapter->fdir_perfect_lock);
9980	err = ixgbe_update_ethtool_fdir_entry(adapter,
9981	NULL,
9982	sw_idx: j + 1);
9983	spin_unlock(lock: &adapter->fdir_perfect_lock);
9984	clear_bit(nr: j, addr: jump->child_loc_map);
9985	}
9986	/* Remove resources for this link */
9987	kfree(objp: jump->input);
9988	kfree(objp: jump->mask);
9989	kfree(objp: jump);
9990	adapter->jump_tables[i] = NULL;
9991	return err;
9992	}
9993	}
9994
9995	spin_lock(lock: &adapter->fdir_perfect_lock);
9996	err = ixgbe_update_ethtool_fdir_entry(adapter, NULL, sw_idx: loc);
9997	spin_unlock(lock: &adapter->fdir_perfect_lock);
9998	return err;
9999	}
10000
10001	static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter,
10002	struct tc_cls_u32_offload *cls)
10003	{
10004	u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);
10005
10006	if (uhtid >= IXGBE_MAX_LINK_HANDLE)
10007	return -EINVAL;
10008
10009	/* This ixgbe devices do not support hash tables at the moment
10010	* so abort when given hash tables.
10011	*/
10012	if (cls->hnode.divisor > 0)
10013	return -EINVAL;
10014
10015	set_bit(nr: uhtid - 1, addr: &adapter->tables);
10016	return 0;
10017	}
10018
10019	static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter,
10020	struct tc_cls_u32_offload *cls)
10021	{
10022	u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);
10023
10024	if (uhtid >= IXGBE_MAX_LINK_HANDLE)
10025	return -EINVAL;
10026
10027	clear_bit(nr: uhtid - 1, addr: &adapter->tables);
10028	return 0;
10029	}
10030
10031	#ifdef CONFIG_NET_CLS_ACT
10032	struct upper_walk_data {
10033	struct ixgbe_adapter *adapter;
10034	u64 action;
10035	int ifindex;
10036	u8 queue;
10037	};
10038
10039	static int get_macvlan_queue(struct net_device *upper,
10040	struct netdev_nested_priv *priv)
10041	{
10042	if (netif_is_macvlan(dev: upper)) {
10043	struct ixgbe_fwd_adapter *vadapter = macvlan_accel_priv(dev: upper);
10044	struct ixgbe_adapter *adapter;
10045	struct upper_walk_data *data;
10046	int ifindex;
10047
10048	data = (struct upper_walk_data *)priv->data;
10049	ifindex = data->ifindex;
10050	adapter = data->adapter;
10051	if (vadapter && upper->ifindex == ifindex) {
10052	data->queue = adapter->rx_ring[vadapter->rx_base_queue]->reg_idx;
10053	data->action = data->queue;
10054	return 1;
10055	}
10056	}
10057
10058	return 0;
10059	}
10060
10061	static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex,
10062	u8 *queue, u64 *action)
10063	{
10064	struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
10065	unsigned int num_vfs = adapter->num_vfs, vf;
10066	struct netdev_nested_priv priv;
10067	struct upper_walk_data data;
10068	struct net_device *upper;
10069
10070	/* redirect to a SRIOV VF */
10071	for (vf = 0; vf < num_vfs; ++vf) {
10072	upper = pci_get_drvdata(pdev: adapter->vfinfo[vf].vfdev);
10073	if (upper->ifindex == ifindex) {
10074	*queue = vf * __ALIGN_MASK(1, ~vmdq->mask);
10075	*action = vf + 1;
10076	*action <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF;
10077	return 0;
10078	}
10079	}
10080
10081	/* redirect to a offloaded macvlan netdev */
10082	data.adapter = adapter;
10083	data.ifindex = ifindex;
10084	data.action = 0;
10085	data.queue = 0;
10086	priv.data = (void *)&data;
10087	if (netdev_walk_all_upper_dev_rcu(dev: adapter->netdev,
10088	fn: get_macvlan_queue, priv: &priv)) {
10089	*action = data.action;
10090	*queue = data.queue;
10091
10092	return 0;
10093	}
10094
10095	return -EINVAL;
10096	}
10097
10098	static int parse_tc_actions(struct ixgbe_adapter *adapter,
10099	struct tcf_exts *exts, u64 *action, u8 *queue)
10100	{
10101	const struct tc_action *a;
10102	int i;
10103
10104	if (!tcf_exts_has_actions(exts))
10105	return -EINVAL;
10106
10107	tcf_exts_for_each_action(i, a, exts) {
10108	/* Drop action */
10109	if (is_tcf_gact_shot(a)) {
10110	*action = IXGBE_FDIR_DROP_QUEUE;
10111	*queue = IXGBE_FDIR_DROP_QUEUE;
10112	return 0;
10113	}
10114
10115	/* Redirect to a VF or a offloaded macvlan */
10116	if (is_tcf_mirred_egress_redirect(a)) {
10117	struct net_device *dev = tcf_mirred_dev(a);
10118
10119	if (!dev)
10120	return -EINVAL;
10121	return handle_redirect_action(adapter, ifindex: dev->ifindex,
10122	queue, action);
10123	}
10124
10125	return -EINVAL;
10126	}
10127
10128	return -EINVAL;
10129	}
10130	#else
10131	static int parse_tc_actions(struct ixgbe_adapter *adapter,
10132	struct tcf_exts *exts, u64 *action, u8 *queue)
10133	{
10134	return -EINVAL;
10135	}
10136	#endif /* CONFIG_NET_CLS_ACT */
10137
10138	static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input,
10139	union ixgbe_atr_input *mask,
10140	struct tc_cls_u32_offload *cls,
10141	struct ixgbe_mat_field *field_ptr,
10142	struct ixgbe_nexthdr *nexthdr)
10143	{
10144	int i, j, off;
10145	__be32 val, m;
10146	bool found_entry = false, found_jump_field = false;
10147
10148	for (i = 0; i < cls->knode.sel->nkeys; i++) {
10149	off = cls->knode.sel->keys[i].off;
10150	val = cls->knode.sel->keys[i].val;
10151	m = cls->knode.sel->keys[i].mask;
10152
10153	for (j = 0; field_ptr[j].val; j++) {
10154	if (field_ptr[j].off == off) {
10155	field_ptr[j].val(input, mask, (__force u32)val,
10156	(__force u32)m);
10157	input->filter.formatted.flow_type |=
10158	field_ptr[j].type;
10159	found_entry = true;
10160	break;
10161	}
10162	}
10163	if (nexthdr) {
10164	if (nexthdr->off == cls->knode.sel->keys[i].off &&
10165	nexthdr->val ==
10166	(__force u32)cls->knode.sel->keys[i].val &&
10167	nexthdr->mask ==
10168	(__force u32)cls->knode.sel->keys[i].mask)
10169	found_jump_field = true;
10170	else
10171	continue;
10172	}
10173	}
10174
10175	if (nexthdr && !found_jump_field)
10176	return -EINVAL;
10177
10178	if (!found_entry)
10179	return 0;
10180
10181	mask->formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK |
10182	IXGBE_ATR_L4TYPE_MASK;
10183
10184	if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4)
10185	mask->formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK;
10186
10187	return 0;
10188	}
10189
10190	static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter,
10191	struct tc_cls_u32_offload *cls)
10192	{
10193	__be16 protocol = cls->common.protocol;
10194	u32 loc = cls->knode.handle & 0xfffff;
10195	struct ixgbe_hw *hw = &adapter->hw;
10196	struct ixgbe_mat_field *field_ptr;
10197	struct ixgbe_fdir_filter *input = NULL;
10198	union ixgbe_atr_input *mask = NULL;
10199	struct ixgbe_jump_table *jump = NULL;
10200	int i, err = -EINVAL;
10201	u8 queue;
10202	u32 uhtid, link_uhtid;
10203
10204	uhtid = TC_U32_USERHTID(cls->knode.handle);
10205	link_uhtid = TC_U32_USERHTID(cls->knode.link_handle);
10206
10207	/* At the moment cls_u32 jumps to network layer and skips past
10208	* L2 headers. The canonical method to match L2 frames is to use
10209	* negative values. However this is error prone at best but really
10210	* just broken because there is no way to "know" what sort of hdr
10211	* is in front of the network layer. Fix cls_u32 to support L2
10212	* headers when needed.
10213	*/
10214	if (protocol != htons(ETH_P_IP))
10215	return err;
10216
10217	if (loc >= ((1024 << adapter->fdir_pballoc) - 2)) {
10218	e_err(drv, "Location out of range\n");
10219	return err;
10220	}
10221
10222	/* cls u32 is a graph starting at root node 0x800. The driver tracks
10223	* links and also the fields used to advance the parser across each
10224	* link (e.g. nexthdr/eat parameters from 'tc'). This way we can map
10225	* the u32 graph onto the hardware parse graph denoted in ixgbe_model.h
10226	* To add support for new nodes update ixgbe_model.h parse structures
10227	* this function _should_ be generic try not to hardcode values here.
10228	*/
10229	if (uhtid == 0x800) {
10230	field_ptr = (adapter->jump_tables[0])->mat;
10231	} else {
10232	if (uhtid >= IXGBE_MAX_LINK_HANDLE)
10233	return err;
10234	if (!adapter->jump_tables[uhtid])
10235	return err;
10236	field_ptr = (adapter->jump_tables[uhtid])->mat;
10237	}
10238
10239	if (!field_ptr)
10240	return err;
10241
10242	/* At this point we know the field_ptr is valid and need to either
10243	* build cls_u32 link or attach filter. Because adding a link to
10244	* a handle that does not exist is invalid and the same for adding
10245	* rules to handles that don't exist.
10246	*/
10247
10248	if (link_uhtid) {
10249	struct ixgbe_nexthdr *nexthdr = ixgbe_ipv4_jumps;
10250
10251	if (link_uhtid >= IXGBE_MAX_LINK_HANDLE)
10252	return err;
10253
10254	if (!test_bit(link_uhtid - 1, &adapter->tables))
10255	return err;
10256
10257	/* Multiple filters as links to the same hash table are not
10258	* supported. To add a new filter with the same next header
10259	* but different match/jump conditions, create a new hash table
10260	* and link to it.
10261	*/
10262	if (adapter->jump_tables[link_uhtid] &&
10263	(adapter->jump_tables[link_uhtid])->link_hdl) {
10264	e_err(drv, "Link filter exists for link: %x\n",
10265	link_uhtid);
10266	return err;
10267	}
10268
10269	for (i = 0; nexthdr[i].jump; i++) {
10270	if (nexthdr[i].o != cls->knode.sel->offoff ||
10271	nexthdr[i].s != cls->knode.sel->offshift ||
10272	nexthdr[i].m !=
10273	(__force u32)cls->knode.sel->offmask)
10274	return err;
10275
10276	jump = kzalloc(sizeof(*jump), GFP_KERNEL);
10277	if (!jump)
10278	return -ENOMEM;
10279	input = kzalloc(sizeof(*input), GFP_KERNEL);
10280	if (!input) {
10281	err = -ENOMEM;
10282	goto free_jump;
10283	}
10284	mask = kzalloc(sizeof(*mask), GFP_KERNEL);
10285	if (!mask) {
10286	err = -ENOMEM;
10287	goto free_input;
10288	}
10289	jump->input = input;
10290	jump->mask = mask;
10291	jump->link_hdl = cls->knode.handle;
10292
10293	err = ixgbe_clsu32_build_input(input, mask, cls,
10294	field_ptr, nexthdr: &nexthdr[i]);
10295	if (!err) {
10296	jump->mat = nexthdr[i].jump;
10297	adapter->jump_tables[link_uhtid] = jump;
10298	break;
10299	} else {
10300	kfree(objp: mask);
10301	kfree(objp: input);
10302	kfree(objp: jump);
10303	}
10304	}
10305	return 0;
10306	}
10307
10308	input = kzalloc(sizeof(*input), GFP_KERNEL);
10309	if (!input)
10310	return -ENOMEM;
10311	mask = kzalloc(sizeof(*mask), GFP_KERNEL);
10312	if (!mask) {
10313	err = -ENOMEM;
10314	goto free_input;
10315	}
10316
10317	if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) {
10318	if ((adapter->jump_tables[uhtid])->input)
10319	memcpy(input, (adapter->jump_tables[uhtid])->input,
10320	sizeof(*input));
10321	if ((adapter->jump_tables[uhtid])->mask)
10322	memcpy(mask, (adapter->jump_tables[uhtid])->mask,
10323	sizeof(*mask));
10324
10325	/* Lookup in all child hash tables if this location is already
10326	* filled with a filter
10327	*/
10328	for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) {
10329	struct ixgbe_jump_table *link = adapter->jump_tables[i];
10330
10331	if (link && (test_bit(loc - 1, link->child_loc_map))) {
10332	e_err(drv, "Filter exists in location: %x\n",
10333	loc);
10334	err = -EINVAL;
10335	goto err_out;
10336	}
10337	}
10338	}
10339	err = ixgbe_clsu32_build_input(input, mask, cls, field_ptr, NULL);
10340	if (err)
10341	goto err_out;
10342
10343	err = parse_tc_actions(adapter, exts: cls->knode.exts, action: &input->action,
10344	queue: &queue);
10345	if (err < 0)
10346	goto err_out;
10347
10348	input->sw_idx = loc;
10349
10350	spin_lock(lock: &adapter->fdir_perfect_lock);
10351
10352	if (hlist_empty(h: &adapter->fdir_filter_list)) {
10353	memcpy(&adapter->fdir_mask, mask, sizeof(*mask));
10354	err = ixgbe_fdir_set_input_mask_82599(hw, input_mask: mask);
10355	if (err)
10356	goto err_out_w_lock;
10357	} else if (memcmp(p: &adapter->fdir_mask, q: mask, size: sizeof(*mask))) {
10358	err = -EINVAL;
10359	goto err_out_w_lock;
10360	}
10361
10362	ixgbe_atr_compute_perfect_hash_82599(input: &input->filter, mask);
10363	err = ixgbe_fdir_write_perfect_filter_82599(hw, input: &input->filter,
10364	soft_id: input->sw_idx, queue);
10365	if (err)
10366	goto err_out_w_lock;
10367
10368	ixgbe_update_ethtool_fdir_entry(adapter, input, sw_idx: input->sw_idx);
10369	spin_unlock(lock: &adapter->fdir_perfect_lock);
10370
10371	if ((uhtid != 0x800) && (adapter->jump_tables[uhtid]))
10372	set_bit(nr: loc - 1, addr: (adapter->jump_tables[uhtid])->child_loc_map);
10373
10374	kfree(objp: mask);
10375	return err;
10376	err_out_w_lock:
10377	spin_unlock(lock: &adapter->fdir_perfect_lock);
10378	err_out:
10379	kfree(objp: mask);
10380	free_input:
10381	kfree(objp: input);
10382	free_jump:
10383	kfree(objp: jump);
10384	return err;
10385	}
10386
10387	static int ixgbe_setup_tc_cls_u32(struct ixgbe_adapter *adapter,
10388	struct tc_cls_u32_offload *cls_u32)
10389	{
10390	switch (cls_u32->command) {
10391	case TC_CLSU32_NEW_KNODE:
10392	case TC_CLSU32_REPLACE_KNODE:
10393	return ixgbe_configure_clsu32(adapter, cls: cls_u32);
10394	case TC_CLSU32_DELETE_KNODE:
10395	return ixgbe_delete_clsu32(adapter, cls: cls_u32);
10396	case TC_CLSU32_NEW_HNODE:
10397	case TC_CLSU32_REPLACE_HNODE:
10398	return ixgbe_configure_clsu32_add_hnode(adapter, cls: cls_u32);
10399	case TC_CLSU32_DELETE_HNODE:
10400	return ixgbe_configure_clsu32_del_hnode(adapter, cls: cls_u32);
10401	default:
10402	return -EOPNOTSUPP;
10403	}
10404	}
10405
10406	static int ixgbe_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
10407	void *cb_priv)
10408	{
10409	struct ixgbe_adapter *adapter = cb_priv;
10410
10411	if (!tc_cls_can_offload_and_chain0(dev: adapter->netdev, common: type_data))
10412	return -EOPNOTSUPP;
10413
10414	switch (type) {
10415	case TC_SETUP_CLSU32:
10416	return ixgbe_setup_tc_cls_u32(adapter, cls_u32: type_data);
10417	default:
10418	return -EOPNOTSUPP;
10419	}
10420	}
10421
10422	static int ixgbe_setup_tc_mqprio(struct net_device *dev,
10423	struct tc_mqprio_qopt *mqprio)
10424	{
10425	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
10426	return ixgbe_setup_tc(dev, tc: mqprio->num_tc);
10427	}
10428
10429	static LIST_HEAD(ixgbe_block_cb_list);
10430
10431	static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type,
10432	void *type_data)
10433	{
10434	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10435
10436	switch (type) {
10437	case TC_SETUP_BLOCK:
10438	return flow_block_cb_setup_simple(f: type_data,
10439	driver_list: &ixgbe_block_cb_list,
10440	cb: ixgbe_setup_tc_block_cb,
10441	cb_ident: adapter, cb_priv: adapter, ingress_only: true);
10442	case TC_SETUP_QDISC_MQPRIO:
10443	return ixgbe_setup_tc_mqprio(dev, mqprio: type_data);
10444	default:
10445	return -EOPNOTSUPP;
10446	}
10447	}
10448
10449	#ifdef CONFIG_PCI_IOV
10450	void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter)
10451	{
10452	struct net_device *netdev = adapter->netdev;
10453
10454	rtnl_lock();
10455	ixgbe_setup_tc(dev: netdev, tc: adapter->hw_tcs);
10456	rtnl_unlock();
10457	}
10458
10459	#endif
10460	void ixgbe_do_reset(struct net_device *netdev)
10461	{
10462	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
10463
10464	if (netif_running(dev: netdev))
10465	ixgbe_reinit_locked(adapter);
10466	else
10467	ixgbe_reset(adapter);
10468	}
10469
10470	static netdev_features_t ixgbe_fix_features(struct net_device *netdev,
10471	netdev_features_t features)
10472	{
10473	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
10474
10475	/* If Rx checksum is disabled, then RSC/LRO should also be disabled */
10476	if (!(features & NETIF_F_RXCSUM))
10477	features &= ~NETIF_F_LRO;
10478
10479	/* Turn off LRO if not RSC capable */
10480	if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE))
10481	features &= ~NETIF_F_LRO;
10482
10483	if (adapter->xdp_prog && (features & NETIF_F_LRO)) {
10484	e_dev_err("LRO is not supported with XDP\n");
10485	features &= ~NETIF_F_LRO;
10486	}
10487
10488	return features;
10489	}
10490
10491	static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter)
10492	{
10493	int rss = min_t(int, ixgbe_max_rss_indices(adapter),
10494	num_online_cpus());
10495
10496	/* go back to full RSS if we're not running SR-IOV */
10497	if (!adapter->ring_feature[RING_F_VMDQ].offset)
10498	adapter->flags &= ~(IXGBE_FLAG_VMDQ_ENABLED |
10499	IXGBE_FLAG_SRIOV_ENABLED);
10500
10501	adapter->ring_feature[RING_F_RSS].limit = rss;
10502	adapter->ring_feature[RING_F_VMDQ].limit = 1;
10503
10504	ixgbe_setup_tc(dev: adapter->netdev, tc: adapter->hw_tcs);
10505	}
10506
10507	static int ixgbe_set_features(struct net_device *netdev,
10508	netdev_features_t features)
10509	{
10510	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
10511	netdev_features_t changed = netdev->features ^ features;
10512	bool need_reset = false;
10513
10514	/* Make sure RSC matches LRO, reset if change */
10515	if (!(features & NETIF_F_LRO)) {
10516	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
10517	need_reset = true;
10518	adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED;
10519	} else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) &&
10520	!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {
10521	if (adapter->rx_itr_setting == 1 ||
10522	adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) {
10523	adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
10524	need_reset = true;
10525	} else if ((changed ^ features) & NETIF_F_LRO) {
10526	e_info(probe, "rx-usecs set too low, "
10527	"disabling RSC\n");
10528	}
10529	}
10530
10531	/*
10532	* Check if Flow Director n-tuple support or hw_tc support was
10533	* enabled or disabled. If the state changed, we need to reset.
10534	*/
10535	if ((features & NETIF_F_NTUPLE) || (features & NETIF_F_HW_TC)) {
10536	/* turn off ATR, enable perfect filters and reset */
10537	if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE))
10538	need_reset = true;
10539
10540	adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
10541	adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
10542	} else {
10543	/* turn off perfect filters, enable ATR and reset */
10544	if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
10545	need_reset = true;
10546
10547	adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
10548
10549	/* We cannot enable ATR if SR-IOV is enabled */
10550	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED ||
10551	/* We cannot enable ATR if we have 2 or more tcs */
10552	(adapter->hw_tcs > 1) ||
10553	/* We cannot enable ATR if RSS is disabled */
10554	(adapter->ring_feature[RING_F_RSS].limit <= 1) ||
10555	/* A sample rate of 0 indicates ATR disabled */
10556	(!adapter->atr_sample_rate))
10557	; /* do nothing not supported */
10558	else /* otherwise supported and set the flag */
10559	adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
10560	}
10561
10562	if (changed & NETIF_F_RXALL)
10563	need_reset = true;
10564
10565	netdev->features = features;
10566
10567	if ((changed & NETIF_F_HW_L2FW_DOFFLOAD) && adapter->num_rx_pools > 1)
10568	ixgbe_reset_l2fw_offload(adapter);
10569	else if (need_reset)
10570	ixgbe_do_reset(netdev);
10571	else if (changed & (NETIF_F_HW_VLAN_CTAG_RX |
10572	NETIF_F_HW_VLAN_CTAG_FILTER))
10573	ixgbe_set_rx_mode(netdev);
10574
10575	return 1;
10576	}
10577
10578	static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
10579	struct net_device *dev,
10580	const unsigned char *addr, u16 vid,
10581	u16 flags, bool *notified,
10582	struct netlink_ext_ack *extack)
10583	{
10584	/* guarantee we can provide a unique filter for the unicast address */
10585	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
10586	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10587	u16 pool = VMDQ_P(0);
10588
10589	if (netdev_uc_count(dev) >= ixgbe_available_rars(adapter, pool))
10590	return -ENOMEM;
10591	}
10592
10593	return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
10594	}
10595
10596	/**
10597	* ixgbe_configure_bridge_mode - set various bridge modes
10598	* @adapter: the private structure
10599	* @mode: requested bridge mode
10600	*
10601	* Configure some settings require for various bridge modes.
10602	**/
10603	static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter,
10604	__u16 mode)
10605	{
10606	struct ixgbe_hw *hw = &adapter->hw;
10607	unsigned int p, num_pools;
10608	u32 vmdctl;
10609
10610	switch (mode) {
10611	case BRIDGE_MODE_VEPA:
10612	/* disable Tx loopback, rely on switch hairpin mode */
10613	IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC, 0);
10614
10615	/* must enable Rx switching replication to allow multicast
10616	* packet reception on all VFs, and to enable source address
10617	* pruning.
10618	*/
10619	vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);
10620	vmdctl |= IXGBE_VT_CTL_REPLEN;
10621	IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
10622
10623	/* enable Rx source address pruning. Note, this requires
10624	* replication to be enabled or else it does nothing.
10625	*/
10626	num_pools = adapter->num_vfs + adapter->num_rx_pools;
10627	for (p = 0; p < num_pools; p++) {
10628	if (hw->mac.ops.set_source_address_pruning)
10629	hw->mac.ops.set_source_address_pruning(hw,
10630	true,
10631	p);
10632	}
10633	break;
10634	case BRIDGE_MODE_VEB:
10635	/* enable Tx loopback for internal VF/PF communication */
10636	IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC,
10637	IXGBE_PFDTXGSWC_VT_LBEN);
10638
10639	/* disable Rx switching replication unless we have SR-IOV
10640	* virtual functions
10641	*/
10642	vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);
10643	if (!adapter->num_vfs)
10644	vmdctl &= ~IXGBE_VT_CTL_REPLEN;
10645	IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
10646
10647	/* disable Rx source address pruning, since we don't expect to
10648	* be receiving external loopback of our transmitted frames.
10649	*/
10650	num_pools = adapter->num_vfs + adapter->num_rx_pools;
10651	for (p = 0; p < num_pools; p++) {
10652	if (hw->mac.ops.set_source_address_pruning)
10653	hw->mac.ops.set_source_address_pruning(hw,
10654	false,
10655	p);
10656	}
10657	break;
10658	default:
10659	return -EINVAL;
10660	}
10661
10662	adapter->bridge_mode = mode;
10663
10664	e_info(drv, "enabling bridge mode: %s\n",
10665	mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10666
10667	return 0;
10668	}
10669
10670	static int ixgbe_ndo_bridge_setlink(struct net_device *dev,
10671	struct nlmsghdr *nlh, u16 flags,
10672	struct netlink_ext_ack *extack)
10673	{
10674	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10675	struct nlattr *attr, *br_spec;
10676	int rem;
10677
10678	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
10679	return -EOPNOTSUPP;
10680
10681	br_spec = nlmsg_find_attr(nlh, hdrlen: sizeof(struct ifinfomsg), attrtype: IFLA_AF_SPEC);
10682	if (!br_spec)
10683	return -EINVAL;
10684
10685	nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {
10686	__u16 mode = nla_get_u16(nla: attr);
10687	int status = ixgbe_configure_bridge_mode(adapter, mode);
10688
10689	if (status)
10690	return status;
10691
10692	break;
10693	}
10694
10695	return 0;
10696	}
10697
10698	static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
10699	struct net_device *dev,
10700	u32 filter_mask, int nlflags)
10701	{
10702	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10703
10704	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
10705	return 0;
10706
10707	return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
10708	mode: adapter->bridge_mode, flags: 0, mask: 0, nlflags,
10709	filter_mask, NULL);
10710	}
10711
10712	static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev)
10713	{
10714	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: pdev);
10715	struct ixgbe_fwd_adapter *accel;
10716	int tcs = adapter->hw_tcs ? : 1;
10717	int pool, err;
10718
10719	if (adapter->xdp_prog) {
10720	e_warn(probe, "L2FW offload is not supported with XDP\n");
10721	return ERR_PTR(error: -EINVAL);
10722	}
10723
10724	/* The hardware supported by ixgbe only filters on the destination MAC
10725	* address. In order to avoid issues we only support offloading modes
10726	* where the hardware can actually provide the functionality.
10727	*/
10728	if (!macvlan_supports_dest_filter(dev: vdev))
10729	return ERR_PTR(error: -EMEDIUMTYPE);
10730
10731	/* We need to lock down the macvlan to be a single queue device so that
10732	* we can reuse the tc_to_txq field in the macvlan netdev to represent
10733	* the queue mapping to our netdev.
10734	*/
10735	if (netif_is_multiqueue(dev: vdev))
10736	return ERR_PTR(error: -ERANGE);
10737
10738	pool = find_first_zero_bit(addr: adapter->fwd_bitmask, size: adapter->num_rx_pools);
10739	if (pool == adapter->num_rx_pools) {
10740	u16 used_pools = adapter->num_vfs + adapter->num_rx_pools;
10741	u16 reserved_pools;
10742
10743	if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
10744	adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) ||
10745	adapter->num_rx_pools > IXGBE_MAX_MACVLANS)
10746	return ERR_PTR(error: -EBUSY);
10747
10748	/* Hardware has a limited number of available pools. Each VF,
10749	* and the PF require a pool. Check to ensure we don't
10750	* attempt to use more then the available number of pools.
10751	*/
10752	if (used_pools >= IXGBE_MAX_VF_FUNCTIONS)
10753	return ERR_PTR(error: -EBUSY);
10754
10755	/* Enable VMDq flag so device will be set in VM mode */
10756	adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED |
10757	IXGBE_FLAG_SRIOV_ENABLED;
10758
10759	/* Try to reserve as many queues per pool as possible,
10760	* we start with the configurations that support 4 queues
10761	* per pools, followed by 2, and then by just 1 per pool.
10762	*/
10763	if (used_pools < 32 && adapter->num_rx_pools < 16)
10764	reserved_pools = min_t(u16,
10765	32 - used_pools,
10766	16 - adapter->num_rx_pools);
10767	else if (adapter->num_rx_pools < 32)
10768	reserved_pools = min_t(u16,
10769	64 - used_pools,
10770	32 - adapter->num_rx_pools);
10771	else
10772	reserved_pools = 64 - used_pools;
10773
10774
10775	if (!reserved_pools)
10776	return ERR_PTR(error: -EBUSY);
10777
10778	adapter->ring_feature[RING_F_VMDQ].limit += reserved_pools;
10779
10780	/* Force reinit of ring allocation with VMDQ enabled */
10781	err = ixgbe_setup_tc(dev: pdev, tc: adapter->hw_tcs);
10782	if (err)
10783	return ERR_PTR(error: err);
10784
10785	if (pool >= adapter->num_rx_pools)
10786	return ERR_PTR(error: -ENOMEM);
10787	}
10788
10789	accel = kzalloc(sizeof(*accel), GFP_KERNEL);
10790	if (!accel)
10791	return ERR_PTR(error: -ENOMEM);
10792
10793	set_bit(nr: pool, addr: adapter->fwd_bitmask);
10794	netdev_set_sb_channel(dev: vdev, channel: pool);
10795	accel->pool = pool;
10796	accel->netdev = vdev;
10797
10798	if (!netif_running(dev: pdev))
10799	return accel;
10800
10801	err = ixgbe_fwd_ring_up(adapter, accel);
10802	if (err)
10803	return ERR_PTR(error: err);
10804
10805	return accel;
10806	}
10807
10808	static void ixgbe_fwd_del(struct net_device *pdev, void *priv)
10809	{
10810	struct ixgbe_fwd_adapter *accel = priv;
10811	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: pdev);
10812	unsigned int rxbase = accel->rx_base_queue;
10813	unsigned int i;
10814
10815	/* delete unicast filter associated with offloaded interface */
10816	ixgbe_del_mac_filter(adapter, addr: accel->netdev->dev_addr,
10817	VMDQ_P(accel->pool));
10818
10819	/* Allow remaining Rx packets to get flushed out of the
10820	* Rx FIFO before we drop the netdev for the ring.
10821	*/
10822	usleep_range(min: 10000, max: 20000);
10823
10824	for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
10825	struct ixgbe_ring *ring = adapter->rx_ring[rxbase + i];
10826	struct ixgbe_q_vector *qv = ring->q_vector;
10827
10828	/* Make sure we aren't processing any packets and clear
10829	* netdev to shut down the ring.
10830	*/
10831	if (netif_running(dev: adapter->netdev))
10832	napi_synchronize(n: &qv->napi);
10833	ring->netdev = NULL;
10834	}
10835
10836	/* unbind the queues and drop the subordinate channel config */
10837	netdev_unbind_sb_channel(dev: pdev, sb_dev: accel->netdev);
10838	netdev_set_sb_channel(dev: accel->netdev, channel: 0);
10839
10840	clear_bit(nr: accel->pool, addr: adapter->fwd_bitmask);
10841	kfree(objp: accel);
10842	}
10843
10844	#define IXGBE_MAX_MAC_HDR_LEN 127
10845	#define IXGBE_MAX_NETWORK_HDR_LEN 511
10846
10847	static netdev_features_t
10848	ixgbe_features_check(struct sk_buff *skb, struct net_device *dev,
10849	netdev_features_t features)
10850	{
10851	unsigned int network_hdr_len, mac_hdr_len;
10852
10853	/* Make certain the headers can be described by a context descriptor */
10854	mac_hdr_len = skb_network_offset(skb);
10855	if (unlikely(mac_hdr_len > IXGBE_MAX_MAC_HDR_LEN))
10856	return features & ~(NETIF_F_HW_CSUM |
10857	NETIF_F_SCTP_CRC |
10858	NETIF_F_GSO_UDP_L4 |
10859	NETIF_F_HW_VLAN_CTAG_TX |
10860	NETIF_F_TSO |
10861	NETIF_F_TSO6);
10862
10863	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
10864	if (unlikely(network_hdr_len > IXGBE_MAX_NETWORK_HDR_LEN))
10865	return features & ~(NETIF_F_HW_CSUM |
10866	NETIF_F_SCTP_CRC |
10867	NETIF_F_GSO_UDP_L4 |
10868	NETIF_F_TSO |
10869	NETIF_F_TSO6);
10870
10871	/* We can only support IPV4 TSO in tunnels if we can mangle the
10872	* inner IP ID field, so strip TSO if MANGLEID is not supported.
10873	* IPsec offoad sets skb->encapsulation but still can handle
10874	* the TSO, so it's the exception.
10875	*/
10876	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) {
10877	#ifdef CONFIG_IXGBE_IPSEC
10878	if (!secpath_exists(skb))
10879	#endif
10880	features &= ~NETIF_F_TSO;
10881	}
10882
10883	return features;
10884	}
10885
10886	static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog)
10887	{
10888	int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
10889	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10890	struct bpf_prog *old_prog;
10891	bool need_reset;
10892	int num_queues;
10893
10894	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
10895	return -EINVAL;
10896
10897	if (adapter->flags & IXGBE_FLAG_DCB_ENABLED)
10898	return -EINVAL;
10899
10900	/* verify ixgbe ring attributes are sufficient for XDP */
10901	for (i = 0; i < adapter->num_rx_queues; i++) {
10902	struct ixgbe_ring *ring = adapter->rx_ring[i];
10903
10904	if (ring_is_rsc_enabled(ring))
10905	return -EINVAL;
10906
10907	if (frame_size > ixgbe_rx_bufsz(ring))
10908	return -EINVAL;
10909	}
10910
10911	/* if the number of cpus is much larger than the maximum of queues,
10912	* we should stop it and then return with ENOMEM like before.
10913	*/
10914	if (nr_cpu_ids > IXGBE_MAX_XDP_QS * 2)
10915	return -ENOMEM;
10916
10917	old_prog = xchg(&adapter->xdp_prog, prog);
10918	need_reset = (!!prog != !!old_prog);
10919
10920	/* If transitioning XDP modes reconfigure rings */
10921	if (need_reset) {
10922	int err;
10923
10924	if (!prog)
10925	/* Wait until ndo_xsk_wakeup completes. */
10926	synchronize_rcu();
10927	err = ixgbe_setup_tc(dev, tc: adapter->hw_tcs);
10928
10929	if (err)
10930	return -EINVAL;
10931	if (!prog)
10932	xdp_features_clear_redirect_target(dev);
10933	} else {
10934	for (i = 0; i < adapter->num_rx_queues; i++) {
10935	WRITE_ONCE(adapter->rx_ring[i]->xdp_prog,
10936	adapter->xdp_prog);
10937	}
10938	}
10939
10940	if (old_prog)
10941	bpf_prog_put(prog: old_prog);
10942
10943	/* Kick start the NAPI context if there is an AF_XDP socket open
10944	* on that queue id. This so that receiving will start.
10945	*/
10946	if (need_reset && prog) {
10947	num_queues = min_t(int, adapter->num_rx_queues,
10948	adapter->num_xdp_queues);
10949	for (i = 0; i < num_queues; i++)
10950	if (adapter->xdp_ring[i]->xsk_pool)
10951	(void)ixgbe_xsk_wakeup(dev: adapter->netdev, queue_id: i,
10952	XDP_WAKEUP_RX);
10953	xdp_features_set_redirect_target(dev, support_sg: true);
10954	}
10955
10956	return 0;
10957	}
10958
10959	static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp)
10960	{
10961	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10962
10963	switch (xdp->command) {
10964	case XDP_SETUP_PROG:
10965	return ixgbe_xdp_setup(dev, prog: xdp->prog);
10966	case XDP_SETUP_XSK_POOL:
10967	return ixgbe_xsk_pool_setup(adapter, pool: xdp->xsk.pool,
10968	qid: xdp->xsk.queue_id);
10969
10970	default:
10971	return -EINVAL;
10972	}
10973	}
10974
10975	void ixgbe_xdp_ring_update_tail(struct ixgbe_ring *ring)
10976	{
10977	/* Force memory writes to complete before letting h/w know there
10978	* are new descriptors to fetch.
10979	*/
10980	wmb();
10981	writel(val: ring->next_to_use, addr: ring->tail);
10982	}
10983
10984	void ixgbe_xdp_ring_update_tail_locked(struct ixgbe_ring *ring)
10985	{
10986	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
10987	spin_lock(lock: &ring->tx_lock);
10988	ixgbe_xdp_ring_update_tail(ring);
10989	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
10990	spin_unlock(lock: &ring->tx_lock);
10991	}
10992
10993	static int ixgbe_xdp_xmit(struct net_device *dev, int n,
10994	struct xdp_frame **frames, u32 flags)
10995	{
10996	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev: dev);
10997	struct ixgbe_ring *ring;
10998	int nxmit = 0;
10999	int i;
11000
11001	if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state)))
11002	return -ENETDOWN;
11003
11004	if (!netif_carrier_ok(dev: adapter->netdev) ||
11005	!netif_running(dev: adapter->netdev))
11006	return -ENETDOWN;
11007
11008	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
11009	return -EINVAL;
11010
11011	/* During program transitions its possible adapter->xdp_prog is assigned
11012	* but ring has not been configured yet. In this case simply abort xmit.
11013	*/
11014	ring = adapter->xdp_prog ? ixgbe_determine_xdp_ring(adapter) : NULL;
11015	if (unlikely(!ring))
11016	return -ENXIO;
11017
11018	if (unlikely(test_bit(__IXGBE_TX_DISABLED, &ring->state)))
11019	return -ENXIO;
11020
11021	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
11022	spin_lock(lock: &ring->tx_lock);
11023
11024	for (i = 0; i < n; i++) {
11025	struct xdp_frame *xdpf = frames[i];
11026	int err;
11027
11028	err = ixgbe_xmit_xdp_ring(ring, xdpf);
11029	if (err != IXGBE_XDP_TX)
11030	break;
11031	nxmit++;
11032	}
11033
11034	if (unlikely(flags & XDP_XMIT_FLUSH))
11035	ixgbe_xdp_ring_update_tail(ring);
11036
11037	if (static_branch_unlikely(&ixgbe_xdp_locking_key))
11038	spin_unlock(lock: &ring->tx_lock);
11039
11040	return nxmit;
11041	}
11042
11043	static const struct net_device_ops ixgbe_netdev_ops = {
11044	.ndo_open = ixgbe_open,
11045	.ndo_stop = ixgbe_close,
11046	.ndo_start_xmit = ixgbe_xmit_frame,
11047	.ndo_set_rx_mode = ixgbe_set_rx_mode,
11048	.ndo_validate_addr = eth_validate_addr,
11049	.ndo_set_mac_address = ixgbe_set_mac,
11050	.ndo_change_mtu = ixgbe_change_mtu,
11051	.ndo_tx_timeout = ixgbe_tx_timeout,
11052	.ndo_set_tx_maxrate = ixgbe_tx_maxrate,
11053	.ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid,
11054	.ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid,
11055	.ndo_eth_ioctl = ixgbe_ioctl,
11056	.ndo_set_vf_mac = ixgbe_ndo_set_vf_mac,
11057	.ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan,
11058	.ndo_set_vf_rate = ixgbe_ndo_set_vf_bw,
11059	.ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk,
11060	.ndo_set_vf_link_state = ixgbe_ndo_set_vf_link_state,
11061	.ndo_set_vf_rss_query_en = ixgbe_ndo_set_vf_rss_query_en,
11062	.ndo_set_vf_trust = ixgbe_ndo_set_vf_trust,
11063	.ndo_get_vf_config = ixgbe_ndo_get_vf_config,
11064	.ndo_get_vf_stats = ixgbe_ndo_get_vf_stats,
11065	.ndo_get_stats64 = ixgbe_get_stats64,
11066	.ndo_setup_tc = __ixgbe_setup_tc,
11067	#ifdef IXGBE_FCOE
11068	.ndo_select_queue = ixgbe_select_queue,
11069	.ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
11070	.ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
11071	.ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
11072	.ndo_fcoe_enable = ixgbe_fcoe_enable,
11073	.ndo_fcoe_disable = ixgbe_fcoe_disable,
11074	.ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
11075	.ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo,
11076	#endif /* IXGBE_FCOE */
11077	.ndo_set_features = ixgbe_set_features,
11078	.ndo_fix_features = ixgbe_fix_features,
11079	.ndo_fdb_add = ixgbe_ndo_fdb_add,
11080	.ndo_bridge_setlink = ixgbe_ndo_bridge_setlink,
11081	.ndo_bridge_getlink = ixgbe_ndo_bridge_getlink,
11082	.ndo_dfwd_add_station = ixgbe_fwd_add,
11083	.ndo_dfwd_del_station = ixgbe_fwd_del,
11084	.ndo_features_check = ixgbe_features_check,
11085	.ndo_bpf = ixgbe_xdp,
11086	.ndo_xdp_xmit = ixgbe_xdp_xmit,
11087	.ndo_xsk_wakeup = ixgbe_xsk_wakeup,
11088	.ndo_hwtstamp_get = ixgbe_ptp_hwtstamp_get,
11089	.ndo_hwtstamp_set = ixgbe_ptp_hwtstamp_set,
11090	};
11091
11092	static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter,
11093	struct ixgbe_ring *tx_ring)
11094	{
11095	unsigned long wait_delay, delay_interval;
11096	struct ixgbe_hw *hw = &adapter->hw;
11097	u8 reg_idx = tx_ring->reg_idx;
11098	int wait_loop;
11099	u32 txdctl;
11100
11101	IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
11102
11103	/* delay mechanism from ixgbe_disable_tx */
11104	delay_interval = ixgbe_get_completion_timeout(adapter) / 100;
11105
11106	wait_loop = IXGBE_MAX_RX_DESC_POLL;
11107	wait_delay = delay_interval;
11108
11109	while (wait_loop--) {
11110	usleep_range(min: wait_delay, max: wait_delay + 10);
11111	wait_delay += delay_interval * 2;
11112	txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
11113
11114	if (!(txdctl & IXGBE_TXDCTL_ENABLE))
11115	return;
11116	}
11117
11118	e_err(drv, "TXDCTL.ENABLE not cleared within the polling period\n");
11119	}
11120
11121	static void ixgbe_disable_txr(struct ixgbe_adapter *adapter,
11122	struct ixgbe_ring *tx_ring)
11123	{
11124	set_bit(nr: __IXGBE_TX_DISABLED, addr: &tx_ring->state);
11125	ixgbe_disable_txr_hw(adapter, tx_ring);
11126	}
11127
11128	static void ixgbe_disable_rxr_hw(struct ixgbe_adapter *adapter,
11129	struct ixgbe_ring *rx_ring)
11130	{
11131	unsigned long wait_delay, delay_interval;
11132	struct ixgbe_hw *hw = &adapter->hw;
11133	u8 reg_idx = rx_ring->reg_idx;
11134	int wait_loop;
11135	u32 rxdctl;
11136
11137	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
11138	rxdctl &= ~IXGBE_RXDCTL_ENABLE;
11139	rxdctl |= IXGBE_RXDCTL_SWFLSH;
11140
11141	/* write value back with RXDCTL.ENABLE bit cleared */
11142	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
11143
11144	/* RXDCTL.EN may not change on 82598 if link is down, so skip it */
11145	if (hw->mac.type == ixgbe_mac_82598EB &&
11146	!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
11147	return;
11148
11149	/* delay mechanism from ixgbe_disable_rx */
11150	delay_interval = ixgbe_get_completion_timeout(adapter) / 100;
11151
11152	wait_loop = IXGBE_MAX_RX_DESC_POLL;
11153	wait_delay = delay_interval;
11154
11155	while (wait_loop--) {
11156	usleep_range(min: wait_delay, max: wait_delay + 10);
11157	wait_delay += delay_interval * 2;
11158	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
11159
11160	if (!(rxdctl & IXGBE_RXDCTL_ENABLE))
11161	return;
11162	}
11163
11164	e_err(drv, "RXDCTL.ENABLE not cleared within the polling period\n");
11165	}
11166
11167	static void ixgbe_reset_txr_stats(struct ixgbe_ring *tx_ring)
11168	{
11169	memset(&tx_ring->stats, 0, sizeof(tx_ring->stats));
11170	memset(&tx_ring->tx_stats, 0, sizeof(tx_ring->tx_stats));
11171	}
11172
11173	static void ixgbe_reset_rxr_stats(struct ixgbe_ring *rx_ring)
11174	{
11175	memset(&rx_ring->stats, 0, sizeof(rx_ring->stats));
11176	memset(&rx_ring->rx_stats, 0, sizeof(rx_ring->rx_stats));
11177	}
11178
11179	/**
11180	* ixgbe_irq_disable_single - Disable single IRQ vector
11181	* @adapter: adapter structure
11182	* @ring: ring index
11183	**/
11184	static void ixgbe_irq_disable_single(struct ixgbe_adapter *adapter, u32 ring)
11185	{
11186	struct ixgbe_hw *hw = &adapter->hw;
11187	u64 qmask = BIT_ULL(ring);
11188	u32 mask;
11189
11190	switch (adapter->hw.mac.type) {
11191	case ixgbe_mac_82598EB:
11192	mask = qmask & IXGBE_EIMC_RTX_QUEUE;
11193	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, mask);
11194	break;
11195	case ixgbe_mac_82599EB:
11196	case ixgbe_mac_X540:
11197	case ixgbe_mac_X550:
11198	case ixgbe_mac_X550EM_x:
11199	case ixgbe_mac_x550em_a:
11200	mask = (qmask & 0xFFFFFFFF);
11201	if (mask)
11202	IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
11203	mask = (qmask >> 32);
11204	if (mask)
11205	IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
11206	break;
11207	default:
11208	break;
11209	}
11210	IXGBE_WRITE_FLUSH(&adapter->hw);
11211	if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
11212	synchronize_irq(irq: adapter->msix_entries[ring].vector);
11213	else
11214	synchronize_irq(irq: adapter->pdev->irq);
11215	}
11216
11217	/**
11218	* ixgbe_txrx_ring_disable - Disable Rx/Tx/XDP Tx rings
11219	* @adapter: adapter structure
11220	* @ring: ring index
11221	*
11222	* This function disables a certain Rx/Tx/XDP Tx ring. The function
11223	* assumes that the netdev is running.
11224	**/
11225	void ixgbe_txrx_ring_disable(struct ixgbe_adapter *adapter, int ring)
11226	{
11227	struct ixgbe_ring *rx_ring, *tx_ring, *xdp_ring;
11228
11229	rx_ring = adapter->rx_ring[ring];
11230	tx_ring = adapter->tx_ring[ring];
11231	xdp_ring = adapter->xdp_ring[ring];
11232
11233	ixgbe_irq_disable_single(adapter, ring);
11234
11235	/* Rx/Tx/XDP Tx share the same napi context. */
11236	napi_disable(n: &rx_ring->q_vector->napi);
11237
11238	ixgbe_disable_txr(adapter, tx_ring);
11239	if (xdp_ring)
11240	ixgbe_disable_txr(adapter, tx_ring: xdp_ring);
11241	ixgbe_disable_rxr_hw(adapter, rx_ring);
11242
11243	if (xdp_ring)
11244	synchronize_rcu();
11245
11246	ixgbe_clean_tx_ring(tx_ring);
11247	if (xdp_ring)
11248	ixgbe_clean_tx_ring(tx_ring: xdp_ring);
11249	ixgbe_clean_rx_ring(rx_ring);
11250
11251	ixgbe_reset_txr_stats(tx_ring);
11252	if (xdp_ring)
11253	ixgbe_reset_txr_stats(tx_ring: xdp_ring);
11254	ixgbe_reset_rxr_stats(rx_ring);
11255	}
11256
11257	/**
11258	* ixgbe_txrx_ring_enable - Enable Rx/Tx/XDP Tx rings
11259	* @adapter: adapter structure
11260	* @ring: ring index
11261	*
11262	* This function enables a certain Rx/Tx/XDP Tx ring. The function
11263	* assumes that the netdev is running.
11264	**/
11265	void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring)
11266	{
11267	struct ixgbe_ring *rx_ring, *tx_ring, *xdp_ring;
11268
11269	rx_ring = adapter->rx_ring[ring];
11270	tx_ring = adapter->tx_ring[ring];
11271	xdp_ring = adapter->xdp_ring[ring];
11272
11273	ixgbe_configure_tx_ring(adapter, ring: tx_ring);
11274	if (xdp_ring)
11275	ixgbe_configure_tx_ring(adapter, ring: xdp_ring);
11276	ixgbe_configure_rx_ring(adapter, ring: rx_ring);
11277
11278	clear_bit(nr: __IXGBE_TX_DISABLED, addr: &tx_ring->state);
11279	if (xdp_ring)
11280	clear_bit(nr: __IXGBE_TX_DISABLED, addr: &xdp_ring->state);
11281
11282	/* Rx/Tx/XDP Tx share the same napi context. */
11283	napi_enable(n: &rx_ring->q_vector->napi);
11284	ixgbe_irq_enable_queues(adapter, BIT_ULL(ring));
11285	IXGBE_WRITE_FLUSH(&adapter->hw);
11286	}
11287
11288	/**
11289	* ixgbe_enumerate_functions - Get the number of ports this device has
11290	* @adapter: adapter structure
11291	*
11292	* This function enumerates the physical functions co-located on a single slot,
11293	* in order to determine how many ports a device has. This is most useful in
11294	* determining the required GT/s of PCIe bandwidth necessary for optimal
11295	* performance.
11296	**/
11297	static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter)
11298	{
11299	struct pci_dev *entry, *pdev = adapter->pdev;
11300	int physfns = 0;
11301
11302	/* Some cards can not use the generic count PCIe functions method,
11303	* because they are behind a parent switch, so we hardcode these with
11304	* the correct number of functions.
11305	*/
11306	if (ixgbe_pcie_from_parent(hw: &adapter->hw))
11307	physfns = 4;
11308
11309	list_for_each_entry(entry, &adapter->pdev->bus->devices, bus_list) {
11310	/* don't count virtual functions */
11311	if (entry->is_virtfn)
11312	continue;
11313
11314	/* When the devices on the bus don't all match our device ID,
11315	* we can't reliably determine the correct number of
11316	* functions. This can occur if a function has been direct
11317	* attached to a virtual machine using VT-d, for example. In
11318	* this case, simply return -1 to indicate this.
11319	*/
11320	if ((entry->vendor != pdev->vendor) ||
11321	(entry->device != pdev->device))
11322	return -1;
11323
11324	physfns++;
11325	}
11326
11327	return physfns;
11328	}
11329
11330	/**
11331	* ixgbe_wol_supported - Check whether device supports WoL
11332	* @adapter: the adapter private structure
11333	* @device_id: the device ID
11334	* @subdevice_id: the subsystem device ID
11335	*
11336	* This function is used by probe and ethtool to determine
11337	* which devices have WoL support
11338	*
11339	**/
11340	bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id,
11341	u16 subdevice_id)
11342	{
11343	struct ixgbe_hw *hw = &adapter->hw;
11344	u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK;
11345
11346	/* WOL not supported on 82598 */
11347	if (hw->mac.type == ixgbe_mac_82598EB)
11348	return false;
11349
11350	/* check eeprom to see if WOL is enabled for X540 and newer */
11351	if (hw->mac.type >= ixgbe_mac_X540) {
11352	if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
11353	((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) &&
11354	(hw->bus.func == 0)))
11355	return true;
11356	}
11357
11358	/* WOL is determined based on device IDs for 82599 MACs */
11359	switch (device_id) {
11360	case IXGBE_DEV_ID_82599_SFP:
11361	/* Only these subdevices could supports WOL */
11362	switch (subdevice_id) {
11363	case IXGBE_SUBDEV_ID_82599_560FLR:
11364	case IXGBE_SUBDEV_ID_82599_LOM_SNAP6:
11365	case IXGBE_SUBDEV_ID_82599_SFP_WOL0:
11366	case IXGBE_SUBDEV_ID_82599_SFP_2OCP:
11367	/* only support first port */
11368	if (hw->bus.func != 0)
11369	break;
11370	fallthrough;
11371	case IXGBE_SUBDEV_ID_82599_SP_560FLR:
11372	case IXGBE_SUBDEV_ID_82599_SFP:
11373	case IXGBE_SUBDEV_ID_82599_RNDC:
11374	case IXGBE_SUBDEV_ID_82599_ECNA_DP:
11375	case IXGBE_SUBDEV_ID_82599_SFP_1OCP:
11376	case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM1:
11377	case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM2:
11378	return true;
11379	}
11380	break;
11381	case IXGBE_DEV_ID_82599EN_SFP:
11382	/* Only these subdevices support WOL */
11383	switch (subdevice_id) {
11384	case IXGBE_SUBDEV_ID_82599EN_SFP_OCP1:
11385	return true;
11386	}
11387	break;
11388	case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
11389	/* All except this subdevice support WOL */
11390	if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
11391	return true;
11392	break;
11393	case IXGBE_DEV_ID_82599_KX4:
11394	return true;
11395	default:
11396	break;
11397	}
11398
11399	return false;
11400	}
11401
11402	/**
11403	* ixgbe_set_fw_version_e610 - Set FW version specifically on E610 adapters
11404	* @adapter: the adapter private structure
11405	*
11406	* This function is used by probe and ethtool to determine the FW version to
11407	* format to display. The FW version is taken from the EEPROM/NVM.
11408	*
11409	*/
11410	void ixgbe_set_fw_version_e610(struct ixgbe_adapter *adapter)
11411	{
11412	struct ixgbe_orom_info *orom = &adapter->hw.flash.orom;
11413	struct ixgbe_nvm_info *nvm = &adapter->hw.flash.nvm;
11414
11415	snprintf(buf: adapter->eeprom_id, size: sizeof(adapter->eeprom_id),
11416	fmt: "%x.%02x 0x%x %d.%d.%d", nvm->major, nvm->minor,
11417	nvm->eetrack, orom->major, orom->build, orom->patch);
11418	}
11419
11420	/**
11421	* ixgbe_set_fw_version - Set FW version
11422	* @adapter: the adapter private structure
11423	*
11424	* This function is used by probe and ethtool to determine the FW version to
11425	* format to display. The FW version is taken from the EEPROM/NVM.
11426	*/
11427	static void ixgbe_set_fw_version(struct ixgbe_adapter *adapter)
11428	{
11429	struct ixgbe_hw *hw = &adapter->hw;
11430	struct ixgbe_nvm_version nvm_ver;
11431
11432	if (adapter->hw.mac.type == ixgbe_mac_e610) {
11433	ixgbe_set_fw_version_e610(adapter);
11434	return;
11435	}
11436
11437	ixgbe_get_oem_prod_version(hw, nvm_ver: &nvm_ver);
11438	if (nvm_ver.oem_valid) {
11439	snprintf(buf: adapter->eeprom_id, size: sizeof(adapter->eeprom_id),
11440	fmt: "%x.%x.%x", nvm_ver.oem_major, nvm_ver.oem_minor,
11441	nvm_ver.oem_release);
11442	return;
11443	}
11444
11445	ixgbe_get_etk_id(hw, nvm_ver: &nvm_ver);
11446	ixgbe_get_orom_version(hw, nvm_ver: &nvm_ver);
11447
11448	if (nvm_ver.or_valid) {
11449	snprintf(buf: adapter->eeprom_id, size: sizeof(adapter->eeprom_id),
11450	fmt: "0x%08x, %d.%d.%d", nvm_ver.etk_id, nvm_ver.or_major,
11451	nvm_ver.or_build, nvm_ver.or_patch);
11452	return;
11453	}
11454
11455	/* Set ETrack ID format */
11456	snprintf(buf: adapter->eeprom_id, size: sizeof(adapter->eeprom_id),
11457	fmt: "0x%08x", nvm_ver.etk_id);
11458	}
11459
11460	/**
11461	* ixgbe_recovery_probe - Handle FW recovery mode during probe
11462	* @adapter: the adapter private structure
11463	*
11464	* Perform limited driver initialization when FW error is detected.
11465	*
11466	* Return: 0 on successful probe for E610, -EIO if recovery mode is detected
11467	* for non-E610 adapter, error status code on any other case.
11468	*/
11469	static int ixgbe_recovery_probe(struct ixgbe_adapter *adapter)
11470	{
11471	struct pci_dev *pdev = adapter->pdev;
11472	struct ixgbe_hw *hw = &adapter->hw;
11473	int err = -EIO;
11474
11475	if (hw->mac.type != ixgbe_mac_e610)
11476	return err;
11477
11478	ixgbe_get_hw_control(adapter);
11479	err = ixgbe_get_flash_data(hw: &adapter->hw);
11480	if (err)
11481	goto err_release_hw_control;
11482
11483	timer_setup(&adapter->service_timer, ixgbe_service_timer, 0);
11484	INIT_WORK(&adapter->service_task, ixgbe_recovery_service_task);
11485	set_bit(nr: __IXGBE_SERVICE_INITED, addr: &adapter->state);
11486	clear_bit(nr: __IXGBE_SERVICE_SCHED, addr: &adapter->state);
11487
11488	if (hw->mac.ops.get_bus_info)
11489	hw->mac.ops.get_bus_info(hw);
11490
11491	pci_set_drvdata(pdev, data: adapter);
11492	/* We are creating devlink interface so NIC can be managed,
11493	* e.g. new NVM image loaded
11494	*/
11495	devl_lock(devlink: adapter->devlink);
11496	ixgbe_devlink_register_port(adapter);
11497	SET_NETDEV_DEVLINK_PORT(adapter->netdev,
11498	&adapter->devlink_port);
11499	ixgbe_devlink_init_regions(adapter);
11500	devl_register(devlink: adapter->devlink);
11501	devl_unlock(devlink: adapter->devlink);
11502
11503	return 0;
11504	err_release_hw_control:
11505	ixgbe_release_hw_control(adapter);
11506	return err;
11507	}
11508
11509	/**
11510	* ixgbe_probe - Device Initialization Routine
11511	* @pdev: PCI device information struct
11512	* @ent: entry in ixgbe_pci_tbl
11513	*
11514	* Returns 0 on success, negative on failure
11515	*
11516	* ixgbe_probe initializes an adapter identified by a pci_dev structure.
11517	* The OS initialization, configuring of the adapter private structure,
11518	* and a hardware reset occur.
11519	**/
11520	static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
11521	{
11522	struct net_device *netdev;
11523	struct ixgbe_netdevice_priv *netdev_priv_wrapper;
11524	struct ixgbe_adapter *adapter = NULL;
11525	struct ixgbe_hw *hw;
11526	const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
11527	unsigned int indices = MAX_TX_QUEUES;
11528	u8 part_str[IXGBE_PBANUM_LENGTH];
11529	int i, err, expected_gts;
11530	bool disable_dev = false;
11531	#ifdef IXGBE_FCOE
11532	u16 device_caps;
11533	#endif
11534	u32 eec;
11535
11536	/* Catch broken hardware that put the wrong VF device ID in
11537	* the PCIe SR-IOV capability.
11538	*/
11539	if (pdev->is_virtfn) {
11540	WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
11541	pci_name(pdev), pdev->vendor, pdev->device);
11542	return -EINVAL;
11543	}
11544
11545	err = pci_enable_device_mem(dev: pdev);
11546	if (err)
11547	return err;
11548
11549	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
11550	if (err) {
11551	dev_err(&pdev->dev,
11552	"No usable DMA configuration, aborting\n");
11553	goto err_dma;
11554	}
11555
11556	err = pci_request_mem_regions(pdev, name: ixgbe_driver_name);
11557	if (err) {
11558	dev_err(&pdev->dev,
11559	"pci_request_selected_regions failed 0x%x\n", err);
11560	goto err_pci_reg;
11561	}
11562
11563	pci_set_master(dev: pdev);
11564	pci_save_state(dev: pdev);
11565
11566	if (ii->mac == ixgbe_mac_82598EB) {
11567	#ifdef CONFIG_IXGBE_DCB
11568	/* 8 TC w/ 4 queues per TC */
11569	indices = 4 * MAX_TRAFFIC_CLASS;
11570	#else
11571	indices = IXGBE_MAX_RSS_INDICES;
11572	#endif
11573	} else if (ii->mac == ixgbe_mac_e610) {
11574	indices = IXGBE_MAX_RSS_INDICES_X550;
11575	}
11576
11577	adapter = ixgbe_allocate_devlink(dev: &pdev->dev);
11578	if (IS_ERR(ptr: adapter)) {
11579	err = PTR_ERR(ptr: adapter);
11580	goto err_devlink;
11581	}
11582
11583	netdev = alloc_etherdev_mq(sizeof(*netdev_priv_wrapper), indices);
11584	if (!netdev) {
11585	err = -ENOMEM;
11586	goto err_alloc_etherdev;
11587	}
11588
11589	SET_NETDEV_DEV(netdev, &pdev->dev);
11590
11591	netdev_priv_wrapper = netdev_priv(dev: netdev);
11592	netdev_priv_wrapper->adapter = adapter;
11593
11594	adapter->netdev = netdev;
11595	adapter->pdev = pdev;
11596	hw = &adapter->hw;
11597	hw->back = adapter;
11598	adapter->msg_enable = netif_msg_init(debug_value: debug, DEFAULT_MSG_ENABLE);
11599
11600	hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
11601	pci_resource_len(pdev, 0));
11602	adapter->io_addr = hw->hw_addr;
11603	if (!hw->hw_addr) {
11604	err = -EIO;
11605	goto err_ioremap;
11606	}
11607
11608	/* Setup hw api */
11609	hw->mac.ops = *ii->mac_ops;
11610	hw->mac.type = ii->mac;
11611	hw->mvals = ii->mvals;
11612	if (ii->link_ops)
11613	hw->link.ops = *ii->link_ops;
11614
11615	/* EEPROM */
11616	hw->eeprom.ops = *ii->eeprom_ops;
11617	eec = IXGBE_READ_REG(hw, IXGBE_EEC(hw));
11618	if (ixgbe_removed(addr: hw->hw_addr)) {
11619	err = -EIO;
11620	goto err_ioremap;
11621	}
11622	/* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
11623	if (!(eec & BIT(8)))
11624	hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;
11625
11626	/* PHY */
11627	hw->phy.ops = *ii->phy_ops;
11628	hw->phy.sfp_type = ixgbe_sfp_type_unknown;
11629	/* ixgbe_identify_phy_generic will set prtad and mmds properly */
11630	hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
11631	hw->phy.mdio.mmds = 0;
11632	hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
11633	hw->phy.mdio.dev = netdev;
11634	hw->phy.mdio.mdio_read = ixgbe_mdio_read;
11635	hw->phy.mdio.mdio_write = ixgbe_mdio_write;
11636
11637	netdev->netdev_ops = &ixgbe_netdev_ops;
11638	ixgbe_set_ethtool_ops(netdev);
11639	netdev->watchdog_timeo = 5 * HZ;
11640	strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
11641
11642	/* setup the private structure */
11643	err = ixgbe_sw_init(adapter, ii);
11644	if (err)
11645	goto err_sw_init;
11646
11647	if (ixgbe_check_fw_error(adapter)) {
11648	err = ixgbe_recovery_probe(adapter);
11649	if (err)
11650	goto err_sw_init;
11651
11652	return 0;
11653	}
11654
11655	if (adapter->hw.mac.type == ixgbe_mac_e610) {
11656	err = ixgbe_get_caps(hw: &adapter->hw);
11657	if (err)
11658	dev_err(&pdev->dev, "ixgbe_get_caps failed %d\n", err);
11659
11660	err = ixgbe_get_flash_data(hw: &adapter->hw);
11661	if (err)
11662	goto err_sw_init;
11663	}
11664
11665	if (adapter->hw.mac.type == ixgbe_mac_82599EB)
11666	adapter->flags2 |= IXGBE_FLAG2_AUTO_DISABLE_VF;
11667
11668	switch (adapter->hw.mac.type) {
11669	case ixgbe_mac_X550:
11670	case ixgbe_mac_X550EM_x:
11671	case ixgbe_mac_e610:
11672	netdev->udp_tunnel_nic_info = &ixgbe_udp_tunnels_x550;
11673	break;
11674	case ixgbe_mac_x550em_a:
11675	netdev->udp_tunnel_nic_info = &ixgbe_udp_tunnels_x550em_a;
11676	break;
11677	default:
11678	break;
11679	}
11680
11681	/* Make it possible the adapter to be woken up via WOL */
11682	switch (adapter->hw.mac.type) {
11683	case ixgbe_mac_82599EB:
11684	case ixgbe_mac_X540:
11685	case ixgbe_mac_X550:
11686	case ixgbe_mac_X550EM_x:
11687	case ixgbe_mac_x550em_a:
11688	case ixgbe_mac_e610:
11689	IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
11690	break;
11691	default:
11692	break;
11693	}
11694
11695	/*
11696	* If there is a fan on this device and it has failed log the
11697	* failure.
11698	*/
11699	if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
11700	u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
11701	if (esdp & IXGBE_ESDP_SDP1)
11702	e_crit(probe, "Fan has stopped, replace the adapter\n");
11703	}
11704
11705	if (allow_unsupported_sfp)
11706	hw->allow_unsupported_sfp = allow_unsupported_sfp;
11707
11708	/* reset_hw fills in the perm_addr as well */
11709	hw->phy.reset_if_overtemp = true;
11710	err = hw->mac.ops.reset_hw(hw);
11711	hw->phy.reset_if_overtemp = false;
11712	ixgbe_set_eee_capable(adapter);
11713	if (err == -ENOENT) {
11714	err = 0;
11715	} else if (err == -EOPNOTSUPP) {
11716	e_dev_err("failed to load because an unsupported SFP+ or QSFP module type was detected.\n");
11717	e_dev_err("Reload the driver after installing a supported module.\n");
11718	goto err_sw_init;
11719	} else if (err) {
11720	e_dev_err("HW Init failed: %d\n", err);
11721	goto err_sw_init;
11722	}
11723
11724	#ifdef CONFIG_PCI_IOV
11725	/* SR-IOV not supported on the 82598 */
11726	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
11727	goto skip_sriov;
11728	/* Mailbox */
11729	ixgbe_init_mbx_params_pf(hw);
11730	hw->mbx.ops = ii->mbx_ops;
11731	pci_sriov_set_totalvfs(dev: pdev, IXGBE_MAX_VFS_DRV_LIMIT);
11732	ixgbe_enable_sriov(adapter, max_vfs);
11733	skip_sriov:
11734
11735	#endif
11736	netdev->features = NETIF_F_SG |
11737	NETIF_F_TSO |
11738	NETIF_F_TSO6 |
11739	NETIF_F_RXHASH |
11740	NETIF_F_RXCSUM |
11741	NETIF_F_HW_CSUM;
11742
11743	#define IXGBE_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
11744	NETIF_F_GSO_GRE_CSUM | \
11745	NETIF_F_GSO_IPXIP4 | \
11746	NETIF_F_GSO_IPXIP6 | \
11747	NETIF_F_GSO_UDP_TUNNEL | \
11748	NETIF_F_GSO_UDP_TUNNEL_CSUM)
11749
11750	netdev->gso_partial_features = IXGBE_GSO_PARTIAL_FEATURES;
11751	netdev->features |= NETIF_F_GSO_PARTIAL |
11752	IXGBE_GSO_PARTIAL_FEATURES;
11753
11754	if (hw->mac.type >= ixgbe_mac_82599EB)
11755	netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4;
11756
11757	#ifdef CONFIG_IXGBE_IPSEC
11758	#define IXGBE_ESP_FEATURES (NETIF_F_HW_ESP | \
11759	NETIF_F_HW_ESP_TX_CSUM | \
11760	NETIF_F_GSO_ESP)
11761
11762	if (adapter->ipsec)
11763	netdev->features |= IXGBE_ESP_FEATURES;
11764	#endif
11765	/* copy netdev features into list of user selectable features */
11766	netdev->hw_features |= netdev->features |
11767	NETIF_F_HW_VLAN_CTAG_FILTER |
11768	NETIF_F_HW_VLAN_CTAG_RX |
11769	NETIF_F_HW_VLAN_CTAG_TX |
11770	NETIF_F_RXALL |
11771	NETIF_F_HW_L2FW_DOFFLOAD;
11772
11773	if (hw->mac.type >= ixgbe_mac_82599EB)
11774	netdev->hw_features |= NETIF_F_NTUPLE |
11775	NETIF_F_HW_TC;
11776
11777	netdev->features |= NETIF_F_HIGHDMA;
11778
11779	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
11780	netdev->hw_enc_features |= netdev->vlan_features;
11781	netdev->mpls_features |= NETIF_F_SG |
11782	NETIF_F_TSO |
11783	NETIF_F_TSO6 |
11784	NETIF_F_HW_CSUM;
11785	netdev->mpls_features |= IXGBE_GSO_PARTIAL_FEATURES;
11786
11787	/* set this bit last since it cannot be part of vlan_features */
11788	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
11789	NETIF_F_HW_VLAN_CTAG_RX |
11790	NETIF_F_HW_VLAN_CTAG_TX;
11791
11792	netdev->priv_flags |= IFF_UNICAST_FLT;
11793	netdev->priv_flags |= IFF_SUPP_NOFCS;
11794
11795	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
11796	NETDEV_XDP_ACT_XSK_ZEROCOPY;
11797
11798	/* MTU range: 68 - 9710 */
11799	netdev->min_mtu = ETH_MIN_MTU;
11800	netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
11801
11802	#ifdef CONFIG_IXGBE_DCB
11803	if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE)
11804	netdev->dcbnl_ops = &ixgbe_dcbnl_ops;
11805	#endif
11806
11807	#ifdef IXGBE_FCOE
11808	if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
11809	unsigned int fcoe_l;
11810
11811	if (hw->mac.ops.get_device_caps) {
11812	hw->mac.ops.get_device_caps(hw, &device_caps);
11813	if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
11814	adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
11815	}
11816
11817
11818	fcoe_l = min_t(int, IXGBE_FCRETA_SIZE, num_online_cpus());
11819	adapter->ring_feature[RING_F_FCOE].limit = fcoe_l;
11820
11821	netdev->features |= NETIF_F_FSO |
11822	NETIF_F_FCOE_CRC;
11823
11824	netdev->vlan_features |= NETIF_F_FSO |
11825	NETIF_F_FCOE_CRC;
11826	}
11827	#endif /* IXGBE_FCOE */
11828	if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
11829	netdev->hw_features |= NETIF_F_LRO;
11830	if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
11831	netdev->features |= NETIF_F_LRO;
11832
11833	/* make sure the EEPROM is good */
11834	if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
11835	e_dev_err("The EEPROM Checksum Is Not Valid\n");
11836	err = -EIO;
11837	goto err_sw_init;
11838	}
11839
11840	eth_platform_get_mac_address(dev: &adapter->pdev->dev,
11841	mac_addr: adapter->hw.mac.perm_addr);
11842
11843	eth_hw_addr_set(dev: netdev, addr: hw->mac.perm_addr);
11844
11845	if (!is_valid_ether_addr(addr: netdev->dev_addr)) {
11846	e_dev_err("invalid MAC address\n");
11847	err = -EIO;
11848	goto err_sw_init;
11849	}
11850
11851	/* Set hw->mac.addr to permanent MAC address */
11852	ether_addr_copy(dst: hw->mac.addr, src: hw->mac.perm_addr);
11853	ixgbe_mac_set_default_filter(adapter);
11854
11855	timer_setup(&adapter->service_timer, ixgbe_service_timer, 0);
11856
11857	if (ixgbe_removed(addr: hw->hw_addr)) {
11858	err = -EIO;
11859	goto err_sw_init;
11860	}
11861	INIT_WORK(&adapter->service_task, ixgbe_service_task);
11862	set_bit(nr: __IXGBE_SERVICE_INITED, addr: &adapter->state);
11863	clear_bit(nr: __IXGBE_SERVICE_SCHED, addr: &adapter->state);
11864
11865	err = ixgbe_init_interrupt_scheme(adapter);
11866	if (err)
11867	goto err_sw_init;
11868
11869	for (i = 0; i < adapter->num_rx_queues; i++)
11870	u64_stats_init(syncp: &adapter->rx_ring[i]->syncp);
11871	for (i = 0; i < adapter->num_tx_queues; i++)
11872	u64_stats_init(syncp: &adapter->tx_ring[i]->syncp);
11873	for (i = 0; i < adapter->num_xdp_queues; i++)
11874	u64_stats_init(syncp: &adapter->xdp_ring[i]->syncp);
11875
11876	/* WOL not supported for all devices */
11877	adapter->wol = 0;
11878	hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap);
11879	hw->wol_enabled = ixgbe_wol_supported(adapter, device_id: pdev->device,
11880	subdevice_id: pdev->subsystem_device);
11881	if (hw->wol_enabled)
11882	adapter->wol = IXGBE_WUFC_MAG;
11883
11884	device_set_wakeup_enable(dev: &adapter->pdev->dev, enable: adapter->wol);
11885
11886	/* save off EEPROM version number */
11887	ixgbe_set_fw_version(adapter);
11888
11889	/* pick up the PCI bus settings for reporting later */
11890	if (ixgbe_pcie_from_parent(hw))
11891	ixgbe_get_parent_bus_info(adapter);
11892	else
11893	hw->mac.ops.get_bus_info(hw);
11894
11895	/* calculate the expected PCIe bandwidth required for optimal
11896	* performance. Note that some older parts will never have enough
11897	* bandwidth due to being older generation PCIe parts. We clamp these
11898	* parts to ensure no warning is displayed if it can't be fixed.
11899	*/
11900	switch (hw->mac.type) {
11901	case ixgbe_mac_82598EB:
11902	expected_gts = min(ixgbe_enumerate_functions(adapter) * 10, 16);
11903	break;
11904	default:
11905	expected_gts = ixgbe_enumerate_functions(adapter) * 10;
11906	break;
11907	}
11908
11909	/* don't check link if we failed to enumerate functions */
11910	if (expected_gts > 0)
11911	ixgbe_check_minimum_link(adapter, expected_gts);
11912
11913	err = hw->eeprom.ops.read_pba_string(hw, part_str, sizeof(part_str));
11914	if (err)
11915	strscpy(part_str, "Unknown", sizeof(part_str));
11916	if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
11917	e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
11918	hw->mac.type, hw->phy.type, hw->phy.sfp_type,
11919	part_str);
11920	else
11921	e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
11922	hw->mac.type, hw->phy.type, part_str);
11923
11924	e_dev_info("%pM\n", netdev->dev_addr);
11925
11926	/* reset the hardware with the new settings */
11927	err = hw->mac.ops.start_hw(hw);
11928	if (err == -EACCES) {
11929	/* We are running on a pre-production device, log a warning */
11930	e_dev_warn("This device is a pre-production adapter/LOM. "
11931	"Please be aware there may be issues associated "
11932	"with your hardware. If you are experiencing "
11933	"problems please contact your Intel or hardware "
11934	"representative who provided you with this "
11935	"hardware.\n");
11936	}
11937	strcpy(p: netdev->name, q: "eth%d");
11938	pci_set_drvdata(pdev, data: adapter);
11939
11940	devl_lock(devlink: adapter->devlink);
11941	ixgbe_devlink_register_port(adapter);
11942	SET_NETDEV_DEVLINK_PORT(adapter->netdev, &adapter->devlink_port);
11943
11944	err = register_netdev(dev: netdev);
11945	if (err)
11946	goto err_register;
11947
11948
11949	/* power down the optics for 82599 SFP+ fiber */
11950	if (hw->mac.ops.disable_tx_laser)
11951	hw->mac.ops.disable_tx_laser(hw);
11952
11953	/* carrier off reporting is important to ethtool even BEFORE open */
11954	netif_carrier_off(dev: netdev);
11955
11956	#ifdef CONFIG_IXGBE_DCA
11957	if (dca_add_requester(dev: &pdev->dev) == 0) {
11958	adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
11959	ixgbe_setup_dca(adapter);
11960	}
11961	#endif
11962	if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
11963	e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
11964	for (i = 0; i < adapter->num_vfs; i++)
11965	ixgbe_vf_configuration(pdev, event_mask: (i | 0x10000000));
11966	}
11967
11968	/* firmware requires driver version to be 0xFFFFFFFF
11969	* since os does not support feature
11970	*/
11971	if (hw->mac.ops.set_fw_drv_ver)
11972	hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF, 0xFF,
11973	sizeof(UTS_RELEASE) - 1,
11974	UTS_RELEASE);
11975
11976	/* add san mac addr to netdev */
11977	ixgbe_add_sanmac_netdev(dev: netdev);
11978
11979	e_dev_info("%s\n", ixgbe_default_device_descr);
11980
11981	#ifdef CONFIG_IXGBE_HWMON
11982	if (ixgbe_sysfs_init(adapter))
11983	e_err(probe, "failed to allocate sysfs resources\n");
11984	#endif /* CONFIG_IXGBE_HWMON */
11985
11986	ixgbe_dbg_adapter_init(adapter);
11987
11988	/* setup link for SFP devices with MNG FW, else wait for IXGBE_UP */
11989	if (ixgbe_mng_enabled(hw) && ixgbe_is_sfp(hw) && hw->mac.ops.setup_link)
11990	hw->mac.ops.setup_link(hw,
11991	IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL,
11992	true);
11993
11994	err = ixgbe_mii_bus_init(hw);
11995	if (err)
11996	goto err_netdev;
11997
11998	ixgbe_devlink_init_regions(adapter);
11999	devl_register(devlink: adapter->devlink);
12000	devl_unlock(devlink: adapter->devlink);
12001
12002	if (ixgbe_fwlog_init(hw))
12003	e_dev_info("Firmware logging not supported\n");
12004
12005	return 0;
12006
12007	err_netdev:
12008	unregister_netdev(dev: netdev);
12009	err_register:
12010	devl_port_unregister(devlink_port: &adapter->devlink_port);
12011	devl_unlock(devlink: adapter->devlink);
12012	ixgbe_release_hw_control(adapter);
12013	ixgbe_clear_interrupt_scheme(adapter);
12014	err_sw_init:
12015	if (hw->mac.type == ixgbe_mac_e610)
12016	mutex_destroy(lock: &adapter->hw.aci.lock);
12017	ixgbe_disable_sriov(adapter);
12018	adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
12019	iounmap(addr: adapter->io_addr);
12020	kfree(objp: adapter->jump_tables[0]);
12021	kfree(objp: adapter->mac_table);
12022	kfree(objp: adapter->rss_key);
12023	bitmap_free(bitmap: adapter->af_xdp_zc_qps);
12024	err_ioremap:
12025	disable_dev = !test_and_set_bit(nr: __IXGBE_DISABLED, addr: &adapter->state);
12026	free_netdev(dev: netdev);
12027	err_alloc_etherdev:
12028	devlink_free(devlink: adapter->devlink);
12029	pci_release_mem_regions(pdev);
12030	err_devlink:
12031	err_pci_reg:
12032	err_dma:
12033	if (!adapter || disable_dev)
12034	pci_disable_device(dev: pdev);
12035	return err;
12036	}
12037
12038	/**
12039	* ixgbe_remove - Device Removal Routine
12040	* @pdev: PCI device information struct
12041	*
12042	* ixgbe_remove is called by the PCI subsystem to alert the driver
12043	* that it should release a PCI device. This could be caused by a
12044	* Hot-Plug event, or because the driver is going to be removed from
12045	* memory.
12046	**/
12047	static void ixgbe_remove(struct pci_dev *pdev)
12048	{
12049	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
12050	struct net_device *netdev;
12051	bool disable_dev;
12052	int i;
12053
12054	/* if !adapter then we already cleaned up in probe */
12055	if (!adapter)
12056	return;
12057
12058	netdev = adapter->netdev;
12059	devl_lock(devlink: adapter->devlink);
12060	devl_unregister(devlink: adapter->devlink);
12061	ixgbe_devlink_destroy_regions(adapter);
12062	ixgbe_fwlog_deinit(hw: &adapter->hw);
12063	ixgbe_dbg_adapter_exit(adapter);
12064
12065	set_bit(nr: __IXGBE_REMOVING, addr: &adapter->state);
12066	cancel_work_sync(work: &adapter->service_task);
12067
12068	if (adapter->hw.mac.type == ixgbe_mac_e610)
12069	ixgbe_disable_link_status_events(adapter);
12070
12071	if (adapter->mii_bus)
12072	mdiobus_unregister(bus: adapter->mii_bus);
12073
12074	#ifdef CONFIG_IXGBE_DCA
12075	if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
12076	adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
12077	dca_remove_requester(dev: &pdev->dev);
12078	IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,
12079	IXGBE_DCA_CTRL_DCA_DISABLE);
12080	}
12081
12082	#endif
12083	#ifdef CONFIG_IXGBE_HWMON
12084	ixgbe_sysfs_exit(adapter);
12085	#endif /* CONFIG_IXGBE_HWMON */
12086
12087	/* remove the added san mac */
12088	ixgbe_del_sanmac_netdev(dev: netdev);
12089
12090	#ifdef CONFIG_PCI_IOV
12091	ixgbe_disable_sriov(adapter);
12092	#endif
12093	if (netdev->reg_state == NETREG_REGISTERED)
12094	unregister_netdev(dev: netdev);
12095
12096	devl_port_unregister(devlink_port: &adapter->devlink_port);
12097	devl_unlock(devlink: adapter->devlink);
12098
12099	ixgbe_stop_ipsec_offload(adapter);
12100	ixgbe_clear_interrupt_scheme(adapter);
12101
12102	ixgbe_release_hw_control(adapter);
12103
12104	#ifdef CONFIG_DCB
12105	kfree(objp: adapter->ixgbe_ieee_pfc);
12106	kfree(objp: adapter->ixgbe_ieee_ets);
12107
12108	#endif
12109	iounmap(addr: adapter->io_addr);
12110	pci_release_mem_regions(pdev);
12111
12112	e_dev_info("complete\n");
12113
12114	for (i = 0; i < IXGBE_MAX_LINK_HANDLE; i++) {
12115	if (adapter->jump_tables[i]) {
12116	kfree(objp: adapter->jump_tables[i]->input);
12117	kfree(objp: adapter->jump_tables[i]->mask);
12118	}
12119	kfree(objp: adapter->jump_tables[i]);
12120	}
12121
12122	kfree(objp: adapter->mac_table);
12123	kfree(objp: adapter->rss_key);
12124	bitmap_free(bitmap: adapter->af_xdp_zc_qps);
12125	disable_dev = !test_and_set_bit(nr: __IXGBE_DISABLED, addr: &adapter->state);
12126	free_netdev(dev: netdev);
12127
12128	if (adapter->hw.mac.type == ixgbe_mac_e610)
12129	mutex_destroy(lock: &adapter->hw.aci.lock);
12130
12131	if (disable_dev)
12132	pci_disable_device(dev: pdev);
12133
12134	devlink_free(devlink: adapter->devlink);
12135	}
12136
12137	/**
12138	* ixgbe_io_error_detected - called when PCI error is detected
12139	* @pdev: Pointer to PCI device
12140	* @state: The current pci connection state
12141	*
12142	* This function is called after a PCI bus error affecting
12143	* this device has been detected.
12144	*/
12145	static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
12146	pci_channel_state_t state)
12147	{
12148	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
12149	struct net_device *netdev = adapter->netdev;
12150
12151	#ifdef CONFIG_PCI_IOV
12152	struct ixgbe_hw *hw = &adapter->hw;
12153	struct pci_dev *bdev, *vfdev;
12154	u32 dw0, dw1, dw2, dw3;
12155	int vf, pos;
12156	u16 req_id, pf_func;
12157
12158	if (adapter->hw.mac.type == ixgbe_mac_82598EB ||
12159	adapter->num_vfs == 0)
12160	goto skip_bad_vf_detection;
12161
12162	bdev = pdev->bus->self;
12163	while (bdev && (pci_pcie_type(dev: bdev) != PCI_EXP_TYPE_ROOT_PORT))
12164	bdev = bdev->bus->self;
12165
12166	if (!bdev)
12167	goto skip_bad_vf_detection;
12168
12169	pos = pci_find_ext_capability(dev: bdev, PCI_EXT_CAP_ID_ERR);
12170	if (!pos)
12171	goto skip_bad_vf_detection;
12172
12173	dw0 = ixgbe_read_pci_cfg_dword(hw, reg: pos + PCI_ERR_HEADER_LOG);
12174	dw1 = ixgbe_read_pci_cfg_dword(hw, reg: pos + PCI_ERR_HEADER_LOG + 4);
12175	dw2 = ixgbe_read_pci_cfg_dword(hw, reg: pos + PCI_ERR_HEADER_LOG + 8);
12176	dw3 = ixgbe_read_pci_cfg_dword(hw, reg: pos + PCI_ERR_HEADER_LOG + 12);
12177	if (ixgbe_removed(addr: hw->hw_addr))
12178	goto skip_bad_vf_detection;
12179
12180	req_id = dw1 >> 16;
12181	/* On the 82599 if bit 7 of the requestor ID is set then it's a VF */
12182	if (!(req_id & 0x0080))
12183	goto skip_bad_vf_detection;
12184
12185	pf_func = req_id & 0x01;
12186	if ((pf_func & 1) == (pdev->devfn & 1)) {
12187	unsigned int device_id;
12188
12189	vf = FIELD_GET(0x7F, req_id);
12190	e_dev_err("VF %d has caused a PCIe error\n", vf);
12191	e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
12192	"%8.8x\tdw3: %8.8x\n",
12193	dw0, dw1, dw2, dw3);
12194	switch (adapter->hw.mac.type) {
12195	case ixgbe_mac_82599EB:
12196	device_id = IXGBE_82599_VF_DEVICE_ID;
12197	break;
12198	case ixgbe_mac_X540:
12199	device_id = IXGBE_X540_VF_DEVICE_ID;
12200	break;
12201	case ixgbe_mac_X550:
12202	device_id = IXGBE_DEV_ID_X550_VF;
12203	break;
12204	case ixgbe_mac_X550EM_x:
12205	device_id = IXGBE_DEV_ID_X550EM_X_VF;
12206	break;
12207	case ixgbe_mac_x550em_a:
12208	device_id = IXGBE_DEV_ID_X550EM_A_VF;
12209	break;
12210	case ixgbe_mac_e610:
12211	device_id = IXGBE_DEV_ID_E610_VF;
12212	break;
12213	default:
12214	device_id = 0;
12215	break;
12216	}
12217
12218	/* Find the pci device of the offending VF */
12219	vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, device: device_id, NULL);
12220	while (vfdev) {
12221	if (vfdev->devfn == (req_id & 0xFF))
12222	break;
12223	vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
12224	device: device_id, from: vfdev);
12225	}
12226	/*
12227	* There's a slim chance the VF could have been hot plugged,
12228	* so if it is no longer present we don't need to issue the
12229	* VFLR. Just clean up the AER in that case.
12230	*/
12231	if (vfdev) {
12232	pcie_flr(dev: vfdev);
12233	/* Free device reference count */
12234	pci_dev_put(dev: vfdev);
12235	}
12236	}
12237
12238	/*
12239	* Even though the error may have occurred on the other port
12240	* we still need to increment the vf error reference count for
12241	* both ports because the I/O resume function will be called
12242	* for both of them.
12243	*/
12244	adapter->vferr_refcount++;
12245
12246	return PCI_ERS_RESULT_RECOVERED;
12247
12248	skip_bad_vf_detection:
12249	#endif /* CONFIG_PCI_IOV */
12250	if (!test_bit(__IXGBE_SERVICE_INITED, &adapter->state))
12251	return PCI_ERS_RESULT_DISCONNECT;
12252
12253	if (!netif_device_present(dev: netdev))
12254	return PCI_ERS_RESULT_DISCONNECT;
12255
12256	rtnl_lock();
12257	netif_device_detach(dev: netdev);
12258
12259	if (netif_running(dev: netdev))
12260	ixgbe_close_suspend(adapter);
12261
12262	if (state == pci_channel_io_perm_failure) {
12263	rtnl_unlock();
12264	return PCI_ERS_RESULT_DISCONNECT;
12265	}
12266
12267	if (!test_and_set_bit(nr: __IXGBE_DISABLED, addr: &adapter->state))
12268	pci_disable_device(dev: pdev);
12269	rtnl_unlock();
12270
12271	/* Request a slot reset. */
12272	return PCI_ERS_RESULT_NEED_RESET;
12273	}
12274
12275	/**
12276	* ixgbe_io_slot_reset - called after the pci bus has been reset.
12277	* @pdev: Pointer to PCI device
12278	*
12279	* Restart the card from scratch, as if from a cold-boot.
12280	*/
12281	static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
12282	{
12283	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
12284	pci_ers_result_t result;
12285
12286	if (pci_enable_device_mem(dev: pdev)) {
12287	e_err(probe, "Cannot re-enable PCI device after reset.\n");
12288	result = PCI_ERS_RESULT_DISCONNECT;
12289	} else {
12290	smp_mb__before_atomic();
12291	clear_bit(nr: __IXGBE_DISABLED, addr: &adapter->state);
12292	adapter->hw.hw_addr = adapter->io_addr;
12293	pci_set_master(dev: pdev);
12294	pci_restore_state(dev: pdev);
12295
12296	pci_wake_from_d3(dev: pdev, enable: false);
12297
12298	ixgbe_reset(adapter);
12299	IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
12300	result = PCI_ERS_RESULT_RECOVERED;
12301	}
12302
12303	return result;
12304	}
12305
12306	/**
12307	* ixgbe_io_resume - called when traffic can start flowing again.
12308	* @pdev: Pointer to PCI device
12309	*
12310	* This callback is called when the error recovery driver tells us that
12311	* its OK to resume normal operation.
12312	*/
12313	static void ixgbe_io_resume(struct pci_dev *pdev)
12314	{
12315	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
12316	struct net_device *netdev = adapter->netdev;
12317
12318	#ifdef CONFIG_PCI_IOV
12319	if (adapter->vferr_refcount) {
12320	e_info(drv, "Resuming after VF err\n");
12321	adapter->vferr_refcount--;
12322	return;
12323	}
12324
12325	#endif
12326	rtnl_lock();
12327	if (netif_running(dev: netdev))
12328	ixgbe_open(netdev);
12329
12330	netif_device_attach(dev: netdev);
12331	rtnl_unlock();
12332	}
12333
12334	static const struct pci_error_handlers ixgbe_err_handler = {
12335	.error_detected = ixgbe_io_error_detected,
12336	.slot_reset = ixgbe_io_slot_reset,
12337	.resume = ixgbe_io_resume,
12338	};
12339
12340	static DEFINE_SIMPLE_DEV_PM_OPS(ixgbe_pm_ops, ixgbe_suspend, ixgbe_resume);
12341
12342	static struct pci_driver ixgbe_driver = {
12343	.name = ixgbe_driver_name,
12344	.id_table = ixgbe_pci_tbl,
12345	.probe = ixgbe_probe,
12346	.remove = ixgbe_remove,
12347	.driver.pm = pm_sleep_ptr(&ixgbe_pm_ops),
12348	.shutdown = ixgbe_shutdown,
12349	.sriov_configure = ixgbe_pci_sriov_configure,
12350	.err_handler = &ixgbe_err_handler
12351	};
12352
12353	/**
12354	* ixgbe_init_module - Driver Registration Routine
12355	*
12356	* ixgbe_init_module is the first routine called when the driver is
12357	* loaded. All it does is register with the PCI subsystem.
12358	**/
12359	static int __init ixgbe_init_module(void)
12360	{
12361	int ret;
12362	pr_info("%s\n", ixgbe_driver_string);
12363	pr_info("%s\n", ixgbe_copyright);
12364
12365	ixgbe_wq = create_singlethread_workqueue(ixgbe_driver_name);
12366	if (!ixgbe_wq) {
12367	pr_err("%s: Failed to create workqueue\n", ixgbe_driver_name);
12368	return -ENOMEM;
12369	}
12370
12371	ixgbe_dbg_init();
12372
12373	ret = pci_register_driver(&ixgbe_driver);
12374	if (ret) {
12375	destroy_workqueue(wq: ixgbe_wq);
12376	ixgbe_dbg_exit();
12377	return ret;
12378	}
12379
12380	#ifdef CONFIG_IXGBE_DCA
12381	dca_register_notify(nb: &dca_notifier);
12382	#endif
12383
12384	return 0;
12385	}
12386
12387	module_init(ixgbe_init_module);
12388
12389	/**
12390	* ixgbe_exit_module - Driver Exit Cleanup Routine
12391	*
12392	* ixgbe_exit_module is called just before the driver is removed
12393	* from memory.
12394	**/
12395	static void __exit ixgbe_exit_module(void)
12396	{
12397	#ifdef CONFIG_IXGBE_DCA
12398	dca_unregister_notify(nb: &dca_notifier);
12399	#endif
12400	pci_unregister_driver(dev: &ixgbe_driver);
12401
12402	ixgbe_dbg_exit();
12403	if (ixgbe_wq) {
12404	destroy_workqueue(wq: ixgbe_wq);
12405	ixgbe_wq = NULL;
12406	}
12407	}
12408
12409	#ifdef CONFIG_IXGBE_DCA
12410	static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
12411	void *p)
12412	{
12413	int ret_val;
12414
12415	ret_val = driver_for_each_device(drv: &ixgbe_driver.driver, NULL, data: &event,
12416	fn: __ixgbe_notify_dca);
12417
12418	return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
12419	}
12420
12421	#endif /* CONFIG_IXGBE_DCA */
12422
12423	module_exit(ixgbe_exit_module);
12424
12425	/* ixgbe_main.c */
12426