1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright(c) 1999 - 2018 Intel Corporation. */
3
4	/******************************************************************************
5	Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
6	******************************************************************************/
7
8	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10	#include <linux/types.h>
11	#include <linux/bitops.h>
12	#include <linux/module.h>
13	#include <linux/pci.h>
14	#include <linux/netdevice.h>
15	#include <linux/vmalloc.h>
16	#include <linux/string.h>
17	#include <linux/in.h>
18	#include <linux/ip.h>
19	#include <linux/tcp.h>
20	#include <linux/sctp.h>
21	#include <linux/ipv6.h>
22	#include <linux/slab.h>
23	#include <net/checksum.h>
24	#include <net/ip6_checksum.h>
25	#include <linux/ethtool.h>
26	#include <linux/if.h>
27	#include <linux/if_vlan.h>
28	#include <linux/prefetch.h>
29	#include <net/mpls.h>
30	#include <linux/bpf.h>
31	#include <linux/bpf_trace.h>
32	#include <linux/atomic.h>
33	#include <net/xfrm.h>
34
35	#include "ixgbevf.h"
36
37	const char ixgbevf_driver_name[] = "ixgbevf";
38	static const char ixgbevf_driver_string[] =
39	"Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
40
41	static char ixgbevf_copyright[] =
42	"Copyright (c) 2009 - 2018 Intel Corporation.";
43
44	static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
45	[board_82599_vf] = &ixgbevf_82599_vf_info,
46	[board_82599_vf_hv] = &ixgbevf_82599_vf_hv_info,
47	[board_X540_vf] = &ixgbevf_X540_vf_info,
48	[board_X540_vf_hv] = &ixgbevf_X540_vf_hv_info,
49	[board_X550_vf] = &ixgbevf_X550_vf_info,
50	[board_X550_vf_hv] = &ixgbevf_X550_vf_hv_info,
51	[board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info,
52	[board_X550EM_x_vf_hv] = &ixgbevf_X550EM_x_vf_hv_info,
53	[board_x550em_a_vf] = &ixgbevf_x550em_a_vf_info,
54	};
55
56	/* ixgbevf_pci_tbl - PCI Device ID Table
57	*
58	* Wildcard entries (PCI_ANY_ID) should come last
59	* Last entry must be all 0s
60	*
61	* { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
62	* Class, Class Mask, private data (not used) }
63	*/
64	static const struct pci_device_id ixgbevf_pci_tbl[] = {
65	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
66	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF_HV), board_82599_vf_hv },
67	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
68	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF_HV), board_X540_vf_hv },
69	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf },
70	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF_HV), board_X550_vf_hv },
71	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf },
72	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV), board_X550EM_x_vf_hv},
73	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_VF), board_x550em_a_vf },
74	/* required last entry */
75	{0, }
76	};
77	MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
78
79	MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
80	MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver");
81	MODULE_LICENSE("GPL v2");
82
83	#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
84	static int debug = -1;
85	module_param(debug, int, 0);
86	MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
87
88	static struct workqueue_struct *ixgbevf_wq;
89
90	static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter)
91	{
92	if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
93	!test_bit(__IXGBEVF_REMOVING, &adapter->state) &&
94	!test_and_set_bit(nr: __IXGBEVF_SERVICE_SCHED, addr: &adapter->state))
95	queue_work(wq: ixgbevf_wq, work: &adapter->service_task);
96	}
97
98	static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter)
99	{
100	BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state));
101
102	/* flush memory to make sure state is correct before next watchdog */
103	smp_mb__before_atomic();
104	clear_bit(nr: __IXGBEVF_SERVICE_SCHED, addr: &adapter->state);
105	}
106
107	/* forward decls */
108	static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
109	static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
110	static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
111	static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer);
112	static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
113	struct ixgbevf_rx_buffer *old_buff);
114
115	static void ixgbevf_remove_adapter(struct ixgbe_hw *hw)
116	{
117	struct ixgbevf_adapter *adapter = hw->back;
118
119	if (!hw->hw_addr)
120	return;
121	hw->hw_addr = NULL;
122	dev_err(&adapter->pdev->dev, "Adapter removed\n");
123	if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state))
124	ixgbevf_service_event_schedule(adapter);
125	}
126
127	static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg)
128	{
129	u32 value;
130
131	/* The following check not only optimizes a bit by not
132	* performing a read on the status register when the
133	* register just read was a status register read that
134	* returned IXGBE_FAILED_READ_REG. It also blocks any
135	* potential recursion.
136	*/
137	if (reg == IXGBE_VFSTATUS) {
138	ixgbevf_remove_adapter(hw);
139	return;
140	}
141	value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS);
142	if (value == IXGBE_FAILED_READ_REG)
143	ixgbevf_remove_adapter(hw);
144	}
145
146	u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg)
147	{
148	u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr);
149	u32 value;
150
151	if (IXGBE_REMOVED(reg_addr))
152	return IXGBE_FAILED_READ_REG;
153	value = readl(addr: reg_addr + reg);
154	if (unlikely(value == IXGBE_FAILED_READ_REG))
155	ixgbevf_check_remove(hw, reg);
156	return value;
157	}
158
159	/**
160	* ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
161	* @adapter: pointer to adapter struct
162	* @direction: 0 for Rx, 1 for Tx, -1 for other causes
163	* @queue: queue to map the corresponding interrupt to
164	* @msix_vector: the vector to map to the corresponding queue
165	**/
166	static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
167	u8 queue, u8 msix_vector)
168	{
169	u32 ivar, index;
170	struct ixgbe_hw *hw = &adapter->hw;
171
172	if (direction == -1) {
173	/* other causes */
174	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
175	ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
176	ivar &= ~0xFF;
177	ivar |= msix_vector;
178	IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
179	} else {
180	/* Tx or Rx causes */
181	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
182	index = ((16 * (queue & 1)) + (8 * direction));
183	ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
184	ivar &= ~(0xFF << index);
185	ivar |= (msix_vector << index);
186	IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
187	}
188	}
189
190	static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring)
191	{
192	return ring->stats.packets;
193	}
194
195	static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring)
196	{
197	struct ixgbevf_adapter *adapter = netdev_priv(dev: ring->netdev);
198	struct ixgbe_hw *hw = &adapter->hw;
199
200	u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx));
201	u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx));
202
203	if (head != tail)
204	return (head < tail) ?
205	tail - head : (tail + ring->count - head);
206
207	return 0;
208	}
209
210	static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring)
211	{
212	u32 tx_done = ixgbevf_get_tx_completed(ring: tx_ring);
213	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
214	u32 tx_pending = ixgbevf_get_tx_pending(ring: tx_ring);
215
216	clear_check_for_tx_hang(tx_ring);
217
218	/* Check for a hung queue, but be thorough. This verifies
219	* that a transmit has been completed since the previous
220	* check AND there is at least one packet pending. The
221	* ARMED bit is set to indicate a potential hang.
222	*/
223	if ((tx_done_old == tx_done) && tx_pending) {
224	/* make sure it is true for two checks in a row */
225	return test_and_set_bit(nr: __IXGBEVF_HANG_CHECK_ARMED,
226	addr: &tx_ring->state);
227	}
228	/* reset the countdown */
229	clear_bit(nr: __IXGBEVF_HANG_CHECK_ARMED, addr: &tx_ring->state);
230
231	/* update completed stats and continue */
232	tx_ring->tx_stats.tx_done_old = tx_done;
233
234	return false;
235	}
236
237	static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter)
238	{
239	/* Do the reset outside of interrupt context */
240	if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
241	set_bit(nr: __IXGBEVF_RESET_REQUESTED, addr: &adapter->state);
242	ixgbevf_service_event_schedule(adapter);
243	}
244	}
245
246	/**
247	* ixgbevf_tx_timeout - Respond to a Tx Hang
248	* @netdev: network interface device structure
249	* @txqueue: transmit queue hanging (unused)
250	**/
251	static void ixgbevf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
252	{
253	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
254
255	ixgbevf_tx_timeout_reset(adapter);
256	}
257
258	/**
259	* ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
260	* @q_vector: board private structure
261	* @tx_ring: tx ring to clean
262	* @napi_budget: Used to determine if we are in netpoll
263	**/
264	static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
265	struct ixgbevf_ring *tx_ring, int napi_budget)
266	{
267	struct ixgbevf_adapter *adapter = q_vector->adapter;
268	struct ixgbevf_tx_buffer *tx_buffer;
269	union ixgbe_adv_tx_desc *tx_desc;
270	unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
271	unsigned int budget = tx_ring->count / 2;
272	unsigned int i = tx_ring->next_to_clean;
273
274	if (test_bit(__IXGBEVF_DOWN, &adapter->state))
275	return true;
276
277	tx_buffer = &tx_ring->tx_buffer_info[i];
278	tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
279	i -= tx_ring->count;
280
281	do {
282	union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
283
284	/* if next_to_watch is not set then there is no work pending */
285	if (!eop_desc)
286	break;
287
288	/* prevent any other reads prior to eop_desc */
289	smp_rmb();
290
291	/* if DD is not set pending work has not been completed */
292	if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
293	break;
294
295	/* clear next_to_watch to prevent false hangs */
296	tx_buffer->next_to_watch = NULL;
297
298	/* update the statistics for this packet */
299	total_bytes += tx_buffer->bytecount;
300	total_packets += tx_buffer->gso_segs;
301	if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
302	total_ipsec++;
303
304	/* free the skb */
305	if (ring_is_xdp(tx_ring))
306	page_frag_free(addr: tx_buffer->data);
307	else
308	napi_consume_skb(skb: tx_buffer->skb, budget: napi_budget);
309
310	/* unmap skb header data */
311	dma_unmap_single(tx_ring->dev,
312	dma_unmap_addr(tx_buffer, dma),
313	dma_unmap_len(tx_buffer, len),
314	DMA_TO_DEVICE);
315
316	/* clear tx_buffer data */
317	dma_unmap_len_set(tx_buffer, len, 0);
318
319	/* unmap remaining buffers */
320	while (tx_desc != eop_desc) {
321	tx_buffer++;
322	tx_desc++;
323	i++;
324	if (unlikely(!i)) {
325	i -= tx_ring->count;
326	tx_buffer = tx_ring->tx_buffer_info;
327	tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
328	}
329
330	/* unmap any remaining paged data */
331	if (dma_unmap_len(tx_buffer, len)) {
332	dma_unmap_page(tx_ring->dev,
333	dma_unmap_addr(tx_buffer, dma),
334	dma_unmap_len(tx_buffer, len),
335	DMA_TO_DEVICE);
336	dma_unmap_len_set(tx_buffer, len, 0);
337	}
338	}
339
340	/* move us one more past the eop_desc for start of next pkt */
341	tx_buffer++;
342	tx_desc++;
343	i++;
344	if (unlikely(!i)) {
345	i -= tx_ring->count;
346	tx_buffer = tx_ring->tx_buffer_info;
347	tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
348	}
349
350	/* issue prefetch for next Tx descriptor */
351	prefetch(tx_desc);
352
353	/* update budget accounting */
354	budget--;
355	} while (likely(budget));
356
357	i += tx_ring->count;
358	tx_ring->next_to_clean = i;
359	u64_stats_update_begin(syncp: &tx_ring->syncp);
360	tx_ring->stats.bytes += total_bytes;
361	tx_ring->stats.packets += total_packets;
362	u64_stats_update_end(syncp: &tx_ring->syncp);
363	q_vector->tx.total_bytes += total_bytes;
364	q_vector->tx.total_packets += total_packets;
365	adapter->tx_ipsec += total_ipsec;
366
367	if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) {
368	struct ixgbe_hw *hw = &adapter->hw;
369	union ixgbe_adv_tx_desc *eop_desc;
370
371	eop_desc = tx_ring->tx_buffer_info[i].next_to_watch;
372
373	pr_err("Detected Tx Unit Hang%s\n"
374	" Tx Queue <%d>\n"
375	" TDH, TDT <%x>, <%x>\n"
376	" next_to_use <%x>\n"
377	" next_to_clean <%x>\n"
378	"tx_buffer_info[next_to_clean]\n"
379	" next_to_watch <%p>\n"
380	" eop_desc->wb.status <%x>\n"
381	" time_stamp <%lx>\n"
382	" jiffies <%lx>\n",
383	ring_is_xdp(tx_ring) ? " XDP" : "",
384	tx_ring->queue_index,
385	IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)),
386	IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)),
387	tx_ring->next_to_use, i,
388	eop_desc, (eop_desc ? eop_desc->wb.status : 0),
389	tx_ring->tx_buffer_info[i].time_stamp, jiffies);
390
391	if (!ring_is_xdp(tx_ring))
392	netif_stop_subqueue(dev: tx_ring->netdev,
393	queue_index: tx_ring->queue_index);
394
395	/* schedule immediate reset if we believe we hung */
396	ixgbevf_tx_timeout_reset(adapter);
397
398	return true;
399	}
400
401	if (ring_is_xdp(tx_ring))
402	return !!budget;
403
404	#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
405	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
406	(ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
407	/* Make sure that anybody stopping the queue after this
408	* sees the new next_to_clean.
409	*/
410	smp_mb();
411
412	if (__netif_subqueue_stopped(dev: tx_ring->netdev,
413	queue_index: tx_ring->queue_index) &&
414	!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
415	netif_wake_subqueue(dev: tx_ring->netdev,
416	queue_index: tx_ring->queue_index);
417	++tx_ring->tx_stats.restart_queue;
418	}
419	}
420
421	return !!budget;
422	}
423
424	/**
425	* ixgbevf_rx_skb - Helper function to determine proper Rx method
426	* @q_vector: structure containing interrupt and ring information
427	* @skb: packet to send up
428	**/
429	static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
430	struct sk_buff *skb)
431	{
432	napi_gro_receive(napi: &q_vector->napi, skb);
433	}
434
435	#define IXGBE_RSS_L4_TYPES_MASK \
436	((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \
437	(1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \
438	(1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \
439	(1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))
440
441	static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring,
442	union ixgbe_adv_rx_desc *rx_desc,
443	struct sk_buff *skb)
444	{
445	u16 rss_type;
446
447	if (!(ring->netdev->features & NETIF_F_RXHASH))
448	return;
449
450	rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
451	IXGBE_RXDADV_RSSTYPE_MASK;
452
453	if (!rss_type)
454	return;
455
456	skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
457	type: (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
458	PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
459	}
460
461	/**
462	* ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
463	* @ring: structure containig ring specific data
464	* @rx_desc: current Rx descriptor being processed
465	* @skb: skb currently being received and modified
466	**/
467	static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
468	union ixgbe_adv_rx_desc *rx_desc,
469	struct sk_buff *skb)
470	{
471	skb_checksum_none_assert(skb);
472
473	/* Rx csum disabled */
474	if (!(ring->netdev->features & NETIF_F_RXCSUM))
475	return;
476
477	/* if IP and error */
478	if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
479	ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
480	ring->rx_stats.csum_err++;
481	return;
482	}
483
484	if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
485	return;
486
487	if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
488	ring->rx_stats.csum_err++;
489	return;
490	}
491
492	/* It must be a TCP or UDP packet with a valid checksum */
493	skb->ip_summed = CHECKSUM_UNNECESSARY;
494	}
495
496	/**
497	* ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor
498	* @rx_ring: rx descriptor ring packet is being transacted on
499	* @rx_desc: pointer to the EOP Rx descriptor
500	* @skb: pointer to current skb being populated
501	*
502	* This function checks the ring, descriptor, and packet information in
503	* order to populate the checksum, VLAN, protocol, and other fields within
504	* the skb.
505	**/
506	static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring,
507	union ixgbe_adv_rx_desc *rx_desc,
508	struct sk_buff *skb)
509	{
510	ixgbevf_rx_hash(ring: rx_ring, rx_desc, skb);
511	ixgbevf_rx_checksum(ring: rx_ring, rx_desc, skb);
512
513	if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
514	u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
515	unsigned long *active_vlans = netdev_priv(dev: rx_ring->netdev);
516
517	if (test_bit(vid & VLAN_VID_MASK, active_vlans))
518	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci: vid);
519	}
520
521	if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))
522	ixgbevf_ipsec_rx(rx_ring, rx_desc, skb);
523
524	skb->protocol = eth_type_trans(skb, dev: rx_ring->netdev);
525	}
526
527	static
528	struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring,
529	const unsigned int size)
530	{
531	struct ixgbevf_rx_buffer *rx_buffer;
532
533	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
534	prefetchw(x: rx_buffer->page);
535
536	/* we are reusing so sync this buffer for CPU use */
537	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
538	addr: rx_buffer->dma,
539	offset: rx_buffer->page_offset,
540	size,
541	dir: DMA_FROM_DEVICE);
542
543	rx_buffer->pagecnt_bias--;
544
545	return rx_buffer;
546	}
547
548	static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring,
549	struct ixgbevf_rx_buffer *rx_buffer,
550	struct sk_buff *skb)
551	{
552	if (ixgbevf_can_reuse_rx_page(rx_buffer)) {
553	/* hand second half of page back to the ring */
554	ixgbevf_reuse_rx_page(rx_ring, old_buff: rx_buffer);
555	} else {
556	if (IS_ERR(ptr: skb))
557	/* We are not reusing the buffer so unmap it and free
558	* any references we are holding to it
559	*/
560	dma_unmap_page_attrs(dev: rx_ring->dev, addr: rx_buffer->dma,
561	ixgbevf_rx_pg_size(rx_ring),
562	dir: DMA_FROM_DEVICE,
563	IXGBEVF_RX_DMA_ATTR);
564	__page_frag_cache_drain(page: rx_buffer->page,
565	count: rx_buffer->pagecnt_bias);
566	}
567
568	/* clear contents of rx_buffer */
569	rx_buffer->page = NULL;
570	}
571
572	/**
573	* ixgbevf_is_non_eop - process handling of non-EOP buffers
574	* @rx_ring: Rx ring being processed
575	* @rx_desc: Rx descriptor for current buffer
576	*
577	* This function updates next to clean. If the buffer is an EOP buffer
578	* this function exits returning false, otherwise it will place the
579	* sk_buff in the next buffer to be chained and return true indicating
580	* that this is in fact a non-EOP buffer.
581	**/
582	static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring,
583	union ixgbe_adv_rx_desc *rx_desc)
584	{
585	u32 ntc = rx_ring->next_to_clean + 1;
586
587	/* fetch, update, and store next to clean */
588	ntc = (ntc < rx_ring->count) ? ntc : 0;
589	rx_ring->next_to_clean = ntc;
590
591	prefetch(IXGBEVF_RX_DESC(rx_ring, ntc));
592
593	if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
594	return false;
595
596	return true;
597	}
598
599	static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring)
600	{
601	return ring_uses_build_skb(rx_ring) ? IXGBEVF_SKB_PAD : 0;
602	}
603
604	static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring,
605	struct ixgbevf_rx_buffer *bi)
606	{
607	struct page *page = bi->page;
608	dma_addr_t dma;
609
610	/* since we are recycling buffers we should seldom need to alloc */
611	if (likely(page))
612	return true;
613
614	/* alloc new page for storage */
615	page = dev_alloc_pages(order: ixgbevf_rx_pg_order(ring: rx_ring));
616	if (unlikely(!page)) {
617	rx_ring->rx_stats.alloc_rx_page_failed++;
618	return false;
619	}
620
621	/* map page for use */
622	dma = dma_map_page_attrs(dev: rx_ring->dev, page, offset: 0,
623	ixgbevf_rx_pg_size(rx_ring),
624	dir: DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR);
625
626	/* if mapping failed free memory back to system since
627	* there isn't much point in holding memory we can't use
628	*/
629	if (dma_mapping_error(dev: rx_ring->dev, dma_addr: dma)) {
630	__free_pages(page, order: ixgbevf_rx_pg_order(ring: rx_ring));
631
632	rx_ring->rx_stats.alloc_rx_page_failed++;
633	return false;
634	}
635
636	bi->dma = dma;
637	bi->page = page;
638	bi->page_offset = ixgbevf_rx_offset(rx_ring);
639	bi->pagecnt_bias = 1;
640	rx_ring->rx_stats.alloc_rx_page++;
641
642	return true;
643	}
644
645	/**
646	* ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
647	* @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
648	* @cleaned_count: number of buffers to replace
649	**/
650	static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
651	u16 cleaned_count)
652	{
653	union ixgbe_adv_rx_desc *rx_desc;
654	struct ixgbevf_rx_buffer *bi;
655	unsigned int i = rx_ring->next_to_use;
656
657	/* nothing to do or no valid netdev defined */
658	if (!cleaned_count || !rx_ring->netdev)
659	return;
660
661	rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
662	bi = &rx_ring->rx_buffer_info[i];
663	i -= rx_ring->count;
664
665	do {
666	if (!ixgbevf_alloc_mapped_page(rx_ring, bi))
667	break;
668
669	/* sync the buffer for use by the device */
670	dma_sync_single_range_for_device(dev: rx_ring->dev, addr: bi->dma,
671	offset: bi->page_offset,
672	size: ixgbevf_rx_bufsz(ring: rx_ring),
673	dir: DMA_FROM_DEVICE);
674
675	/* Refresh the desc even if pkt_addr didn't change
676	* because each write-back erases this info.
677	*/
678	rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
679
680	rx_desc++;
681	bi++;
682	i++;
683	if (unlikely(!i)) {
684	rx_desc = IXGBEVF_RX_DESC(rx_ring, 0);
685	bi = rx_ring->rx_buffer_info;
686	i -= rx_ring->count;
687	}
688
689	/* clear the length for the next_to_use descriptor */
690	rx_desc->wb.upper.length = 0;
691
692	cleaned_count--;
693	} while (cleaned_count);
694
695	i += rx_ring->count;
696
697	if (rx_ring->next_to_use != i) {
698	/* record the next descriptor to use */
699	rx_ring->next_to_use = i;
700
701	/* update next to alloc since we have filled the ring */
702	rx_ring->next_to_alloc = i;
703
704	/* Force memory writes to complete before letting h/w
705	* know there are new descriptors to fetch. (Only
706	* applicable for weak-ordered memory model archs,
707	* such as IA-64).
708	*/
709	wmb();
710	ixgbevf_write_tail(ring: rx_ring, value: i);
711	}
712	}
713
714	/**
715	* ixgbevf_cleanup_headers - Correct corrupted or empty headers
716	* @rx_ring: rx descriptor ring packet is being transacted on
717	* @rx_desc: pointer to the EOP Rx descriptor
718	* @skb: pointer to current skb being fixed
719	*
720	* Check for corrupted packet headers caused by senders on the local L2
721	* embedded NIC switch not setting up their Tx Descriptors right. These
722	* should be very rare.
723	*
724	* Also address the case where we are pulling data in on pages only
725	* and as such no data is present in the skb header.
726	*
727	* In addition if skb is not at least 60 bytes we need to pad it so that
728	* it is large enough to qualify as a valid Ethernet frame.
729	*
730	* Returns true if an error was encountered and skb was freed.
731	**/
732	static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring,
733	union ixgbe_adv_rx_desc *rx_desc,
734	struct sk_buff *skb)
735	{
736	/* XDP packets use error pointer so abort at this point */
737	if (IS_ERR(ptr: skb))
738	return true;
739
740	/* verify that the packet does not have any known errors */
741	if (unlikely(ixgbevf_test_staterr(rx_desc,
742	IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) {
743	struct net_device *netdev = rx_ring->netdev;
744
745	if (!(netdev->features & NETIF_F_RXALL)) {
746	dev_kfree_skb_any(skb);
747	return true;
748	}
749	}
750
751	/* if eth_skb_pad returns an error the skb was freed */
752	if (eth_skb_pad(skb))
753	return true;
754
755	return false;
756	}
757
758	/**
759	* ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring
760	* @rx_ring: rx descriptor ring to store buffers on
761	* @old_buff: donor buffer to have page reused
762	*
763	* Synchronizes page for reuse by the adapter
764	**/
765	static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
766	struct ixgbevf_rx_buffer *old_buff)
767	{
768	struct ixgbevf_rx_buffer *new_buff;
769	u16 nta = rx_ring->next_to_alloc;
770
771	new_buff = &rx_ring->rx_buffer_info[nta];
772
773	/* update, and store next to alloc */
774	nta++;
775	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
776
777	/* transfer page from old buffer to new buffer */
778	new_buff->page = old_buff->page;
779	new_buff->dma = old_buff->dma;
780	new_buff->page_offset = old_buff->page_offset;
781	new_buff->pagecnt_bias = old_buff->pagecnt_bias;
782	}
783
784	static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer)
785	{
786	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
787	struct page *page = rx_buffer->page;
788
789	/* avoid re-using remote and pfmemalloc pages */
790	if (!dev_page_is_reusable(page))
791	return false;
792
793	#if (PAGE_SIZE < 8192)
794	/* if we are only owner of page we can reuse it */
795	if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
796	return false;
797	#else
798	#define IXGBEVF_LAST_OFFSET \
799	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048)
800
801	if (rx_buffer->page_offset > IXGBEVF_LAST_OFFSET)
802	return false;
803
804	#endif
805
806	/* If we have drained the page fragment pool we need to update
807	* the pagecnt_bias and page count so that we fully restock the
808	* number of references the driver holds.
809	*/
810	if (unlikely(!pagecnt_bias)) {
811	page_ref_add(page, USHRT_MAX);
812	rx_buffer->pagecnt_bias = USHRT_MAX;
813	}
814
815	return true;
816	}
817
818	/**
819	* ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff
820	* @rx_ring: rx descriptor ring to transact packets on
821	* @rx_buffer: buffer containing page to add
822	* @skb: sk_buff to place the data into
823	* @size: size of buffer to be added
824	*
825	* This function will add the data contained in rx_buffer->page to the skb.
826	**/
827	static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
828	struct ixgbevf_rx_buffer *rx_buffer,
829	struct sk_buff *skb,
830	unsigned int size)
831	{
832	#if (PAGE_SIZE < 8192)
833	unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
834	#else
835	unsigned int truesize = ring_uses_build_skb(rx_ring) ?
836	SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) :
837	SKB_DATA_ALIGN(size);
838	#endif
839	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page: rx_buffer->page,
840	off: rx_buffer->page_offset, size, truesize);
841	#if (PAGE_SIZE < 8192)
842	rx_buffer->page_offset ^= truesize;
843	#else
844	rx_buffer->page_offset += truesize;
845	#endif
846	}
847
848	static
849	struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring,
850	struct ixgbevf_rx_buffer *rx_buffer,
851	struct xdp_buff *xdp,
852	union ixgbe_adv_rx_desc *rx_desc)
853	{
854	unsigned int size = xdp->data_end - xdp->data;
855	#if (PAGE_SIZE < 8192)
856	unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
857	#else
858	unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
859	xdp->data_hard_start);
860	#endif
861	unsigned int headlen;
862	struct sk_buff *skb;
863
864	/* prefetch first cache line of first page */
865	net_prefetch(p: xdp->data);
866
867	/* Note, we get here by enabling legacy-rx via:
868	*
869	* ethtool --set-priv-flags <dev> legacy-rx on
870	*
871	* In this mode, we currently get 0 extra XDP headroom as
872	* opposed to having legacy-rx off, where we process XDP
873	* packets going to stack via ixgbevf_build_skb().
874	*
875	* For ixgbevf_construct_skb() mode it means that the
876	* xdp->data_meta will always point to xdp->data, since
877	* the helper cannot expand the head. Should this ever
878	* changed in future for legacy-rx mode on, then lets also
879	* add xdp->data_meta handling here.
880	*/
881
882	/* allocate a skb to store the frags */
883	skb = napi_alloc_skb(napi: &rx_ring->q_vector->napi, IXGBEVF_RX_HDR_SIZE);
884	if (unlikely(!skb))
885	return NULL;
886
887	/* Determine available headroom for copy */
888	headlen = size;
889	if (headlen > IXGBEVF_RX_HDR_SIZE)
890	headlen = eth_get_headlen(dev: skb->dev, data: xdp->data,
891	IXGBEVF_RX_HDR_SIZE);
892
893	/* align pull length to size of long to optimize memcpy performance */
894	memcpy(__skb_put(skb, headlen), xdp->data,
895	ALIGN(headlen, sizeof(long)));
896
897	/* update all of the pointers */
898	size -= headlen;
899	if (size) {
900	skb_add_rx_frag(skb, i: 0, page: rx_buffer->page,
901	off: (xdp->data + headlen) -
902	page_address(rx_buffer->page),
903	size, truesize);
904	#if (PAGE_SIZE < 8192)
905	rx_buffer->page_offset ^= truesize;
906	#else
907	rx_buffer->page_offset += truesize;
908	#endif
909	} else {
910	rx_buffer->pagecnt_bias++;
911	}
912
913	return skb;
914	}
915
916	static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
917	u32 qmask)
918	{
919	struct ixgbe_hw *hw = &adapter->hw;
920
921	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
922	}
923
924	static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring,
925	struct ixgbevf_rx_buffer *rx_buffer,
926	struct xdp_buff *xdp,
927	union ixgbe_adv_rx_desc *rx_desc)
928	{
929	unsigned int metasize = xdp->data - xdp->data_meta;
930	#if (PAGE_SIZE < 8192)
931	unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
932	#else
933	unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
934	SKB_DATA_ALIGN(xdp->data_end -
935	xdp->data_hard_start);
936	#endif
937	struct sk_buff *skb;
938
939	/* Prefetch first cache line of first page. If xdp->data_meta
940	* is unused, this points to xdp->data, otherwise, we likely
941	* have a consumer accessing first few bytes of meta data,
942	* and then actual data.
943	*/
944	net_prefetch(p: xdp->data_meta);
945
946	/* build an skb around the page buffer */
947	skb = napi_build_skb(data: xdp->data_hard_start, frag_size: truesize);
948	if (unlikely(!skb))
949	return NULL;
950
951	/* update pointers within the skb to store the data */
952	skb_reserve(skb, len: xdp->data - xdp->data_hard_start);
953	__skb_put(skb, len: xdp->data_end - xdp->data);
954	if (metasize)
955	skb_metadata_set(skb, meta_len: metasize);
956
957	/* update buffer offset */
958	#if (PAGE_SIZE < 8192)
959	rx_buffer->page_offset ^= truesize;
960	#else
961	rx_buffer->page_offset += truesize;
962	#endif
963
964	return skb;
965	}
966
967	#define IXGBEVF_XDP_PASS 0
968	#define IXGBEVF_XDP_CONSUMED 1
969	#define IXGBEVF_XDP_TX 2
970
971	static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring,
972	struct xdp_buff *xdp)
973	{
974	struct ixgbevf_tx_buffer *tx_buffer;
975	union ixgbe_adv_tx_desc *tx_desc;
976	u32 len, cmd_type;
977	dma_addr_t dma;
978	u16 i;
979
980	len = xdp->data_end - xdp->data;
981
982	if (unlikely(!ixgbevf_desc_unused(ring)))
983	return IXGBEVF_XDP_CONSUMED;
984
985	dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE);
986	if (dma_mapping_error(dev: ring->dev, dma_addr: dma))
987	return IXGBEVF_XDP_CONSUMED;
988
989	/* record the location of the first descriptor for this packet */
990	i = ring->next_to_use;
991	tx_buffer = &ring->tx_buffer_info[i];
992
993	dma_unmap_len_set(tx_buffer, len, len);
994	dma_unmap_addr_set(tx_buffer, dma, dma);
995	tx_buffer->data = xdp->data;
996	tx_buffer->bytecount = len;
997	tx_buffer->gso_segs = 1;
998	tx_buffer->protocol = 0;
999
1000	/* Populate minimal context descriptor that will provide for the
1001	* fact that we are expected to process Ethernet frames.
1002	*/
1003	if (!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state)) {
1004	struct ixgbe_adv_tx_context_desc *context_desc;
1005
1006	set_bit(nr: __IXGBEVF_TX_XDP_RING_PRIMED, addr: &ring->state);
1007
1008	context_desc = IXGBEVF_TX_CTXTDESC(ring, 0);
1009	context_desc->vlan_macip_lens =
1010	cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT);
1011	context_desc->fceof_saidx = 0;
1012	context_desc->type_tucmd_mlhl =
1013	cpu_to_le32(IXGBE_TXD_CMD_DEXT |
1014	IXGBE_ADVTXD_DTYP_CTXT);
1015	context_desc->mss_l4len_idx = 0;
1016
1017	i = 1;
1018	}
1019
1020	/* put descriptor type bits */
1021	cmd_type = IXGBE_ADVTXD_DTYP_DATA |
1022	IXGBE_ADVTXD_DCMD_DEXT |
1023	IXGBE_ADVTXD_DCMD_IFCS;
1024	cmd_type |= len | IXGBE_TXD_CMD;
1025
1026	tx_desc = IXGBEVF_TX_DESC(ring, i);
1027	tx_desc->read.buffer_addr = cpu_to_le64(dma);
1028
1029	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1030	tx_desc->read.olinfo_status =
1031	cpu_to_le32((len << IXGBE_ADVTXD_PAYLEN_SHIFT) |
1032	IXGBE_ADVTXD_CC);
1033
1034	/* Avoid any potential race with cleanup */
1035	smp_wmb();
1036
1037	/* set next_to_watch value indicating a packet is present */
1038	i++;
1039	if (i == ring->count)
1040	i = 0;
1041
1042	tx_buffer->next_to_watch = tx_desc;
1043	ring->next_to_use = i;
1044
1045	return IXGBEVF_XDP_TX;
1046	}
1047
1048	static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter,
1049	struct ixgbevf_ring *rx_ring,
1050	struct xdp_buff *xdp)
1051	{
1052	int result = IXGBEVF_XDP_PASS;
1053	struct ixgbevf_ring *xdp_ring;
1054	struct bpf_prog *xdp_prog;
1055	u32 act;
1056
1057	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
1058
1059	if (!xdp_prog)
1060	goto xdp_out;
1061
1062	act = bpf_prog_run_xdp(prog: xdp_prog, xdp);
1063	switch (act) {
1064	case XDP_PASS:
1065	break;
1066	case XDP_TX:
1067	xdp_ring = adapter->xdp_ring[rx_ring->queue_index];
1068	result = ixgbevf_xmit_xdp_ring(ring: xdp_ring, xdp);
1069	if (result == IXGBEVF_XDP_CONSUMED)
1070	goto out_failure;
1071	break;
1072	default:
1073	bpf_warn_invalid_xdp_action(dev: rx_ring->netdev, prog: xdp_prog, act);
1074	fallthrough;
1075	case XDP_ABORTED:
1076	out_failure:
1077	trace_xdp_exception(dev: rx_ring->netdev, xdp: xdp_prog, act);
1078	fallthrough; /* handle aborts by dropping packet */
1079	case XDP_DROP:
1080	result = IXGBEVF_XDP_CONSUMED;
1081	break;
1082	}
1083	xdp_out:
1084	return ERR_PTR(error: -result);
1085	}
1086
1087	static unsigned int ixgbevf_rx_frame_truesize(struct ixgbevf_ring *rx_ring,
1088	unsigned int size)
1089	{
1090	unsigned int truesize;
1091
1092	#if (PAGE_SIZE < 8192)
1093	truesize = ixgbevf_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
1094	#else
1095	truesize = ring_uses_build_skb(rx_ring) ?
1096	SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) +
1097	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
1098	SKB_DATA_ALIGN(size);
1099	#endif
1100	return truesize;
1101	}
1102
1103	static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring,
1104	struct ixgbevf_rx_buffer *rx_buffer,
1105	unsigned int size)
1106	{
1107	unsigned int truesize = ixgbevf_rx_frame_truesize(rx_ring, size);
1108
1109	#if (PAGE_SIZE < 8192)
1110	rx_buffer->page_offset ^= truesize;
1111	#else
1112	rx_buffer->page_offset += truesize;
1113	#endif
1114	}
1115
1116	static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
1117	struct ixgbevf_ring *rx_ring,
1118	int budget)
1119	{
1120	unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0;
1121	struct ixgbevf_adapter *adapter = q_vector->adapter;
1122	u16 cleaned_count = ixgbevf_desc_unused(ring: rx_ring);
1123	struct sk_buff *skb = rx_ring->skb;
1124	bool xdp_xmit = false;
1125	struct xdp_buff xdp;
1126
1127	/* Frame size depend on rx_ring setup when PAGE_SIZE=4K */
1128	#if (PAGE_SIZE < 8192)
1129	frame_sz = ixgbevf_rx_frame_truesize(rx_ring, size: 0);
1130	#endif
1131	xdp_init_buff(xdp: &xdp, frame_sz, rxq: &rx_ring->xdp_rxq);
1132
1133	while (likely(total_rx_packets < budget)) {
1134	struct ixgbevf_rx_buffer *rx_buffer;
1135	union ixgbe_adv_rx_desc *rx_desc;
1136	unsigned int size;
1137
1138	/* return some buffers to hardware, one at a time is too slow */
1139	if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
1140	ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
1141	cleaned_count = 0;
1142	}
1143
1144	rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
1145	size = le16_to_cpu(rx_desc->wb.upper.length);
1146	if (!size)
1147	break;
1148
1149	/* This memory barrier is needed to keep us from reading
1150	* any other fields out of the rx_desc until we know the
1151	* RXD_STAT_DD bit is set
1152	*/
1153	rmb();
1154
1155	rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size);
1156
1157	/* retrieve a buffer from the ring */
1158	if (!skb) {
1159	unsigned int offset = ixgbevf_rx_offset(rx_ring);
1160	unsigned char *hard_start;
1161
1162	hard_start = page_address(rx_buffer->page) +
1163	rx_buffer->page_offset - offset;
1164	xdp_prepare_buff(xdp: &xdp, hard_start, headroom: offset, data_len: size, meta_valid: true);
1165	#if (PAGE_SIZE > 4096)
1166	/* At larger PAGE_SIZE, frame_sz depend on len size */
1167	xdp.frame_sz = ixgbevf_rx_frame_truesize(rx_ring, size);
1168	#endif
1169	skb = ixgbevf_run_xdp(adapter, rx_ring, xdp: &xdp);
1170	}
1171
1172	if (IS_ERR(ptr: skb)) {
1173	if (PTR_ERR(ptr: skb) == -IXGBEVF_XDP_TX) {
1174	xdp_xmit = true;
1175	ixgbevf_rx_buffer_flip(rx_ring, rx_buffer,
1176	size);
1177	} else {
1178	rx_buffer->pagecnt_bias++;
1179	}
1180	total_rx_packets++;
1181	total_rx_bytes += size;
1182	} else if (skb) {
1183	ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size);
1184	} else if (ring_uses_build_skb(rx_ring)) {
1185	skb = ixgbevf_build_skb(rx_ring, rx_buffer,
1186	xdp: &xdp, rx_desc);
1187	} else {
1188	skb = ixgbevf_construct_skb(rx_ring, rx_buffer,
1189	xdp: &xdp, rx_desc);
1190	}
1191
1192	/* exit if we failed to retrieve a buffer */
1193	if (!skb) {
1194	rx_ring->rx_stats.alloc_rx_buff_failed++;
1195	rx_buffer->pagecnt_bias++;
1196	break;
1197	}
1198
1199	ixgbevf_put_rx_buffer(rx_ring, rx_buffer, skb);
1200	cleaned_count++;
1201
1202	/* fetch next buffer in frame if non-eop */
1203	if (ixgbevf_is_non_eop(rx_ring, rx_desc))
1204	continue;
1205
1206	/* verify the packet layout is correct */
1207	if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) {
1208	skb = NULL;
1209	continue;
1210	}
1211
1212	/* probably a little skewed due to removing CRC */
1213	total_rx_bytes += skb->len;
1214
1215	/* Workaround hardware that can't do proper VEPA multicast
1216	* source pruning.
1217	*/
1218	if ((skb->pkt_type == PACKET_BROADCAST ||
1219	skb->pkt_type == PACKET_MULTICAST) &&
1220	ether_addr_equal(addr1: rx_ring->netdev->dev_addr,
1221	addr2: eth_hdr(skb)->h_source)) {
1222	dev_kfree_skb_irq(skb);
1223	continue;
1224	}
1225
1226	/* populate checksum, VLAN, and protocol */
1227	ixgbevf_process_skb_fields(rx_ring, rx_desc, skb);
1228
1229	ixgbevf_rx_skb(q_vector, skb);
1230
1231	/* reset skb pointer */
1232	skb = NULL;
1233
1234	/* update budget accounting */
1235	total_rx_packets++;
1236	}
1237
1238	/* place incomplete frames back on ring for completion */
1239	rx_ring->skb = skb;
1240
1241	if (xdp_xmit) {
1242	struct ixgbevf_ring *xdp_ring =
1243	adapter->xdp_ring[rx_ring->queue_index];
1244
1245	/* Force memory writes to complete before letting h/w
1246	* know there are new descriptors to fetch.
1247	*/
1248	wmb();
1249	ixgbevf_write_tail(ring: xdp_ring, value: xdp_ring->next_to_use);
1250	}
1251
1252	u64_stats_update_begin(syncp: &rx_ring->syncp);
1253	rx_ring->stats.packets += total_rx_packets;
1254	rx_ring->stats.bytes += total_rx_bytes;
1255	u64_stats_update_end(syncp: &rx_ring->syncp);
1256	q_vector->rx.total_packets += total_rx_packets;
1257	q_vector->rx.total_bytes += total_rx_bytes;
1258
1259	return total_rx_packets;
1260	}
1261
1262	/**
1263	* ixgbevf_poll - NAPI polling calback
1264	* @napi: napi struct with our devices info in it
1265	* @budget: amount of work driver is allowed to do this pass, in packets
1266	*
1267	* This function will clean more than one or more rings associated with a
1268	* q_vector.
1269	**/
1270	static int ixgbevf_poll(struct napi_struct *napi, int budget)
1271	{
1272	struct ixgbevf_q_vector *q_vector =
1273	container_of(napi, struct ixgbevf_q_vector, napi);
1274	struct ixgbevf_adapter *adapter = q_vector->adapter;
1275	struct ixgbevf_ring *ring;
1276	int per_ring_budget, work_done = 0;
1277	bool clean_complete = true;
1278
1279	ixgbevf_for_each_ring(ring, q_vector->tx) {
1280	if (!ixgbevf_clean_tx_irq(q_vector, tx_ring: ring, napi_budget: budget))
1281	clean_complete = false;
1282	}
1283
1284	if (budget <= 0)
1285	return budget;
1286
1287	/* attempt to distribute budget to each queue fairly, but don't allow
1288	* the budget to go below 1 because we'll exit polling
1289	*/
1290	if (q_vector->rx.count > 1)
1291	per_ring_budget = max(budget/q_vector->rx.count, 1);
1292	else
1293	per_ring_budget = budget;
1294
1295	ixgbevf_for_each_ring(ring, q_vector->rx) {
1296	int cleaned = ixgbevf_clean_rx_irq(q_vector, rx_ring: ring,
1297	budget: per_ring_budget);
1298	work_done += cleaned;
1299	if (cleaned >= per_ring_budget)
1300	clean_complete = false;
1301	}
1302
1303	/* If all work not completed, return budget and keep polling */
1304	if (!clean_complete)
1305	return budget;
1306
1307	/* Exit the polling mode, but don't re-enable interrupts if stack might
1308	* poll us due to busy-polling
1309	*/
1310	if (likely(napi_complete_done(napi, work_done))) {
1311	if (adapter->rx_itr_setting == 1)
1312	ixgbevf_set_itr(q_vector);
1313	if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
1314	!test_bit(__IXGBEVF_REMOVING, &adapter->state))
1315	ixgbevf_irq_enable_queues(adapter,
1316	BIT(q_vector->v_idx));
1317	}
1318
1319	return min(work_done, budget - 1);
1320	}
1321
1322	/**
1323	* ixgbevf_write_eitr - write VTEITR register in hardware specific way
1324	* @q_vector: structure containing interrupt and ring information
1325	**/
1326	void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
1327	{
1328	struct ixgbevf_adapter *adapter = q_vector->adapter;
1329	struct ixgbe_hw *hw = &adapter->hw;
1330	int v_idx = q_vector->v_idx;
1331	u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
1332
1333	/* set the WDIS bit to not clear the timer bits and cause an
1334	* immediate assertion of the interrupt
1335	*/
1336	itr_reg |= IXGBE_EITR_CNT_WDIS;
1337
1338	IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
1339	}
1340
1341	/**
1342	* ixgbevf_configure_msix - Configure MSI-X hardware
1343	* @adapter: board private structure
1344	*
1345	* ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
1346	* interrupts.
1347	**/
1348	static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
1349	{
1350	struct ixgbevf_q_vector *q_vector;
1351	int q_vectors, v_idx;
1352
1353	q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1354	adapter->eims_enable_mask = 0;
1355
1356	/* Populate the IVAR table and set the ITR values to the
1357	* corresponding register.
1358	*/
1359	for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1360	struct ixgbevf_ring *ring;
1361
1362	q_vector = adapter->q_vector[v_idx];
1363
1364	ixgbevf_for_each_ring(ring, q_vector->rx)
1365	ixgbevf_set_ivar(adapter, direction: 0, queue: ring->reg_idx, msix_vector: v_idx);
1366
1367	ixgbevf_for_each_ring(ring, q_vector->tx)
1368	ixgbevf_set_ivar(adapter, direction: 1, queue: ring->reg_idx, msix_vector: v_idx);
1369
1370	if (q_vector->tx.ring && !q_vector->rx.ring) {
1371	/* Tx only vector */
1372	if (adapter->tx_itr_setting == 1)
1373	q_vector->itr = IXGBE_12K_ITR;
1374	else
1375	q_vector->itr = adapter->tx_itr_setting;
1376	} else {
1377	/* Rx or Rx/Tx vector */
1378	if (adapter->rx_itr_setting == 1)
1379	q_vector->itr = IXGBE_20K_ITR;
1380	else
1381	q_vector->itr = adapter->rx_itr_setting;
1382	}
1383
1384	/* add q_vector eims value to global eims_enable_mask */
1385	adapter->eims_enable_mask |= BIT(v_idx);
1386
1387	ixgbevf_write_eitr(q_vector);
1388	}
1389
1390	ixgbevf_set_ivar(adapter, direction: -1, queue: 1, msix_vector: v_idx);
1391	/* setup eims_other and add value to global eims_enable_mask */
1392	adapter->eims_other = BIT(v_idx);
1393	adapter->eims_enable_mask |= adapter->eims_other;
1394	}
1395
1396	enum latency_range {
1397	lowest_latency = 0,
1398	low_latency = 1,
1399	bulk_latency = 2,
1400	latency_invalid = 255
1401	};
1402
1403	/**
1404	* ixgbevf_update_itr - update the dynamic ITR value based on statistics
1405	* @q_vector: structure containing interrupt and ring information
1406	* @ring_container: structure containing ring performance data
1407	*
1408	* Stores a new ITR value based on packets and byte
1409	* counts during the last interrupt. The advantage of per interrupt
1410	* computation is faster updates and more accurate ITR for the current
1411	* traffic pattern. Constants in this function were computed
1412	* based on theoretical maximum wire speed and thresholds were set based
1413	* on testing data as well as attempting to minimize response time
1414	* while increasing bulk throughput.
1415	**/
1416	static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
1417	struct ixgbevf_ring_container *ring_container)
1418	{
1419	int bytes = ring_container->total_bytes;
1420	int packets = ring_container->total_packets;
1421	u32 timepassed_us;
1422	u64 bytes_perint;
1423	u8 itr_setting = ring_container->itr;
1424
1425	if (packets == 0)
1426	return;
1427
1428	/* simple throttle rate management
1429	* 0-20MB/s lowest (100000 ints/s)
1430	* 20-100MB/s low (20000 ints/s)
1431	* 100-1249MB/s bulk (12000 ints/s)
1432	*/
1433	/* what was last interrupt timeslice? */
1434	timepassed_us = q_vector->itr >> 2;
1435	if (timepassed_us == 0)
1436	return;
1437
1438	bytes_perint = bytes / timepassed_us; /* bytes/usec */
1439
1440	switch (itr_setting) {
1441	case lowest_latency:
1442	if (bytes_perint > 10)
1443	itr_setting = low_latency;
1444	break;
1445	case low_latency:
1446	if (bytes_perint > 20)
1447	itr_setting = bulk_latency;
1448	else if (bytes_perint <= 10)
1449	itr_setting = lowest_latency;
1450	break;
1451	case bulk_latency:
1452	if (bytes_perint <= 20)
1453	itr_setting = low_latency;
1454	break;
1455	}
1456
1457	/* clear work counters since we have the values we need */
1458	ring_container->total_bytes = 0;
1459	ring_container->total_packets = 0;
1460
1461	/* write updated itr to ring container */
1462	ring_container->itr = itr_setting;
1463	}
1464
1465	static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
1466	{
1467	u32 new_itr = q_vector->itr;
1468	u8 current_itr;
1469
1470	ixgbevf_update_itr(q_vector, ring_container: &q_vector->tx);
1471	ixgbevf_update_itr(q_vector, ring_container: &q_vector->rx);
1472
1473	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1474
1475	switch (current_itr) {
1476	/* counts and packets in update_itr are dependent on these numbers */
1477	case lowest_latency:
1478	new_itr = IXGBE_100K_ITR;
1479	break;
1480	case low_latency:
1481	new_itr = IXGBE_20K_ITR;
1482	break;
1483	case bulk_latency:
1484	new_itr = IXGBE_12K_ITR;
1485	break;
1486	default:
1487	break;
1488	}
1489
1490	if (new_itr != q_vector->itr) {
1491	/* do an exponential smoothing */
1492	new_itr = (10 * new_itr * q_vector->itr) /
1493	((9 * new_itr) + q_vector->itr);
1494
1495	/* save the algorithm value here */
1496	q_vector->itr = new_itr;
1497
1498	ixgbevf_write_eitr(q_vector);
1499	}
1500	}
1501
1502	static irqreturn_t ixgbevf_msix_other(int irq, void *data)
1503	{
1504	struct ixgbevf_adapter *adapter = data;
1505	struct ixgbe_hw *hw = &adapter->hw;
1506
1507	hw->mac.get_link_status = 1;
1508
1509	ixgbevf_service_event_schedule(adapter);
1510
1511	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
1512
1513	return IRQ_HANDLED;
1514	}
1515
1516	/**
1517	* ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
1518	* @irq: unused
1519	* @data: pointer to our q_vector struct for this interrupt vector
1520	**/
1521	static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
1522	{
1523	struct ixgbevf_q_vector *q_vector = data;
1524
1525	/* EIAM disabled interrupts (on this vector) for us */
1526	if (q_vector->rx.ring || q_vector->tx.ring)
1527	napi_schedule_irqoff(n: &q_vector->napi);
1528
1529	return IRQ_HANDLED;
1530	}
1531
1532	/**
1533	* ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
1534	* @adapter: board private structure
1535	*
1536	* ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
1537	* interrupts from the kernel.
1538	**/
1539	static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
1540	{
1541	struct net_device *netdev = adapter->netdev;
1542	int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1543	unsigned int ri = 0, ti = 0;
1544	int vector, err;
1545
1546	for (vector = 0; vector < q_vectors; vector++) {
1547	struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
1548	struct msix_entry *entry = &adapter->msix_entries[vector];
1549
1550	if (q_vector->tx.ring && q_vector->rx.ring) {
1551	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
1552	fmt: "%s-TxRx-%u", netdev->name, ri++);
1553	ti++;
1554	} else if (q_vector->rx.ring) {
1555	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
1556	fmt: "%s-rx-%u", netdev->name, ri++);
1557	} else if (q_vector->tx.ring) {
1558	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
1559	fmt: "%s-tx-%u", netdev->name, ti++);
1560	} else {
1561	/* skip this unused q_vector */
1562	continue;
1563	}
1564	err = request_irq(irq: entry->vector, handler: &ixgbevf_msix_clean_rings, flags: 0,
1565	name: q_vector->name, dev: q_vector);
1566	if (err) {
1567	hw_dbg(&adapter->hw,
1568	"request_irq failed for MSIX interrupt Error: %d\n",
1569	err);
1570	goto free_queue_irqs;
1571	}
1572	}
1573
1574	err = request_irq(irq: adapter->msix_entries[vector].vector,
1575	handler: &ixgbevf_msix_other, flags: 0, name: netdev->name, dev: adapter);
1576	if (err) {
1577	hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n",
1578	err);
1579	goto free_queue_irqs;
1580	}
1581
1582	return 0;
1583
1584	free_queue_irqs:
1585	while (vector) {
1586	vector--;
1587	free_irq(adapter->msix_entries[vector].vector,
1588	adapter->q_vector[vector]);
1589	}
1590	/* This failure is non-recoverable - it indicates the system is
1591	* out of MSIX vector resources and the VF driver cannot run
1592	* without them. Set the number of msix vectors to zero
1593	* indicating that not enough can be allocated. The error
1594	* will be returned to the user indicating device open failed.
1595	* Any further attempts to force the driver to open will also
1596	* fail. The only way to recover is to unload the driver and
1597	* reload it again. If the system has recovered some MSIX
1598	* vectors then it may succeed.
1599	*/
1600	adapter->num_msix_vectors = 0;
1601	return err;
1602	}
1603
1604	/**
1605	* ixgbevf_request_irq - initialize interrupts
1606	* @adapter: board private structure
1607	*
1608	* Attempts to configure interrupts using the best available
1609	* capabilities of the hardware and kernel.
1610	**/
1611	static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1612	{
1613	int err = ixgbevf_request_msix_irqs(adapter);
1614
1615	if (err)
1616	hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err);
1617
1618	return err;
1619	}
1620
1621	static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1622	{
1623	int i, q_vectors;
1624
1625	if (!adapter->msix_entries)
1626	return;
1627
1628	q_vectors = adapter->num_msix_vectors;
1629	i = q_vectors - 1;
1630
1631	free_irq(adapter->msix_entries[i].vector, adapter);
1632	i--;
1633
1634	for (; i >= 0; i--) {
1635	/* free only the irqs that were actually requested */
1636	if (!adapter->q_vector[i]->rx.ring &&
1637	!adapter->q_vector[i]->tx.ring)
1638	continue;
1639
1640	free_irq(adapter->msix_entries[i].vector,
1641	adapter->q_vector[i]);
1642	}
1643	}
1644
1645	/**
1646	* ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1647	* @adapter: board private structure
1648	**/
1649	static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1650	{
1651	struct ixgbe_hw *hw = &adapter->hw;
1652	int i;
1653
1654	IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
1655	IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
1656	IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
1657
1658	IXGBE_WRITE_FLUSH(hw);
1659
1660	for (i = 0; i < adapter->num_msix_vectors; i++)
1661	synchronize_irq(irq: adapter->msix_entries[i].vector);
1662	}
1663
1664	/**
1665	* ixgbevf_irq_enable - Enable default interrupt generation settings
1666	* @adapter: board private structure
1667	**/
1668	static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
1669	{
1670	struct ixgbe_hw *hw = &adapter->hw;
1671
1672	IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1673	IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1674	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
1675	}
1676
1677	/**
1678	* ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1679	* @adapter: board private structure
1680	* @ring: structure containing ring specific data
1681	*
1682	* Configure the Tx descriptor ring after a reset.
1683	**/
1684	static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1685	struct ixgbevf_ring *ring)
1686	{
1687	struct ixgbe_hw *hw = &adapter->hw;
1688	u64 tdba = ring->dma;
1689	int wait_loop = 10;
1690	u32 txdctl = IXGBE_TXDCTL_ENABLE;
1691	u8 reg_idx = ring->reg_idx;
1692
1693	/* disable queue to avoid issues while updating state */
1694	IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1695	IXGBE_WRITE_FLUSH(hw);
1696
1697	IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1698	IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1699	IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1700	ring->count * sizeof(union ixgbe_adv_tx_desc));
1701
1702	/* disable head writeback */
1703	IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1704	IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1705
1706	/* enable relaxed ordering */
1707	IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1708	(IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1709	IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1710
1711	/* reset head and tail pointers */
1712	IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1713	IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
1714	ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx);
1715
1716	/* reset ntu and ntc to place SW in sync with hardwdare */
1717	ring->next_to_clean = 0;
1718	ring->next_to_use = 0;
1719
1720	/* In order to avoid issues WTHRESH + PTHRESH should always be equal
1721	* to or less than the number of on chip descriptors, which is
1722	* currently 40.
1723	*/
1724	txdctl |= (8 << 16); /* WTHRESH = 8 */
1725
1726	/* Setting PTHRESH to 32 both improves performance */
1727	txdctl |= (1u << 8) | /* HTHRESH = 1 */
1728	32; /* PTHRESH = 32 */
1729
1730	/* reinitialize tx_buffer_info */
1731	memset(ring->tx_buffer_info, 0,
1732	sizeof(struct ixgbevf_tx_buffer) * ring->count);
1733
1734	clear_bit(nr: __IXGBEVF_HANG_CHECK_ARMED, addr: &ring->state);
1735	clear_bit(nr: __IXGBEVF_TX_XDP_RING_PRIMED, addr: &ring->state);
1736
1737	IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1738
1739	/* poll to verify queue is enabled */
1740	do {
1741	usleep_range(min: 1000, max: 2000);
1742	txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1743	} while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1744	if (!wait_loop)
1745	hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
1746	}
1747
1748	/**
1749	* ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1750	* @adapter: board private structure
1751	*
1752	* Configure the Tx unit of the MAC after a reset.
1753	**/
1754	static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1755	{
1756	u32 i;
1757
1758	/* Setup the HW Tx Head and Tail descriptor pointers */
1759	for (i = 0; i < adapter->num_tx_queues; i++)
1760	ixgbevf_configure_tx_ring(adapter, ring: adapter->tx_ring[i]);
1761	for (i = 0; i < adapter->num_xdp_queues; i++)
1762	ixgbevf_configure_tx_ring(adapter, ring: adapter->xdp_ring[i]);
1763	}
1764
1765	#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1766
1767	static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter,
1768	struct ixgbevf_ring *ring, int index)
1769	{
1770	struct ixgbe_hw *hw = &adapter->hw;
1771	u32 srrctl;
1772
1773	srrctl = IXGBE_SRRCTL_DROP_EN;
1774
1775	srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
1776	if (ring_uses_large_buffer(ring))
1777	srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1778	else
1779	srrctl |= IXGBEVF_RXBUFFER_2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1780	srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1781
1782	IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1783	}
1784
1785	static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1786	{
1787	struct ixgbe_hw *hw = &adapter->hw;
1788
1789	/* PSRTYPE must be initialized in 82599 */
1790	u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1791	IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1792	IXGBE_PSRTYPE_L2HDR;
1793
1794	if (adapter->num_rx_queues > 1)
1795	psrtype |= BIT(29);
1796
1797	IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1798	}
1799
1800	#define IXGBEVF_MAX_RX_DESC_POLL 10
1801	static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1802	struct ixgbevf_ring *ring)
1803	{
1804	struct ixgbe_hw *hw = &adapter->hw;
1805	int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1806	u32 rxdctl;
1807	u8 reg_idx = ring->reg_idx;
1808
1809	if (IXGBE_REMOVED(hw->hw_addr))
1810	return;
1811	rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1812	rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1813
1814	/* write value back with RXDCTL.ENABLE bit cleared */
1815	IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1816
1817	/* the hardware may take up to 100us to really disable the Rx queue */
1818	do {
1819	udelay(10);
1820	rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1821	} while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1822
1823	if (!wait_loop)
1824	pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1825	reg_idx);
1826	}
1827
1828	static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1829	struct ixgbevf_ring *ring)
1830	{
1831	struct ixgbe_hw *hw = &adapter->hw;
1832	int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1833	u32 rxdctl;
1834	u8 reg_idx = ring->reg_idx;
1835
1836	if (IXGBE_REMOVED(hw->hw_addr))
1837	return;
1838	do {
1839	usleep_range(min: 1000, max: 2000);
1840	rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1841	} while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1842
1843	if (!wait_loop)
1844	pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1845	reg_idx);
1846	}
1847
1848	/**
1849	* ixgbevf_init_rss_key - Initialize adapter RSS key
1850	* @adapter: device handle
1851	*
1852	* Allocates and initializes the RSS key if it is not allocated.
1853	**/
1854	static inline int ixgbevf_init_rss_key(struct ixgbevf_adapter *adapter)
1855	{
1856	u32 *rss_key;
1857
1858	if (!adapter->rss_key) {
1859	rss_key = kzalloc(IXGBEVF_RSS_HASH_KEY_SIZE, GFP_KERNEL);
1860	if (unlikely(!rss_key))
1861	return -ENOMEM;
1862
1863	netdev_rss_key_fill(buffer: rss_key, IXGBEVF_RSS_HASH_KEY_SIZE);
1864	adapter->rss_key = rss_key;
1865	}
1866
1867	return 0;
1868	}
1869
1870	static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter)
1871	{
1872	struct ixgbe_hw *hw = &adapter->hw;
1873	u32 vfmrqc = 0, vfreta = 0;
1874	u16 rss_i = adapter->num_rx_queues;
1875	u8 i, j;
1876
1877	/* Fill out hash function seeds */
1878	for (i = 0; i < IXGBEVF_VFRSSRK_REGS; i++)
1879	IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), *(adapter->rss_key + i));
1880
1881	for (i = 0, j = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++, j++) {
1882	if (j == rss_i)
1883	j = 0;
1884
1885	adapter->rss_indir_tbl[i] = j;
1886
1887	vfreta |= j << (i & 0x3) * 8;
1888	if ((i & 3) == 3) {
1889	IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta);
1890	vfreta = 0;
1891	}
1892	}
1893
1894	/* Perform hash on these packet types */
1895	vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 |
1896	IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP |
1897	IXGBE_VFMRQC_RSS_FIELD_IPV6 |
1898	IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP;
1899
1900	vfmrqc |= IXGBE_VFMRQC_RSSEN;
1901
1902	IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc);
1903	}
1904
1905	static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1906	struct ixgbevf_ring *ring)
1907	{
1908	struct ixgbe_hw *hw = &adapter->hw;
1909	union ixgbe_adv_rx_desc *rx_desc;
1910	u64 rdba = ring->dma;
1911	u32 rxdctl;
1912	u8 reg_idx = ring->reg_idx;
1913
1914	/* disable queue to avoid issues while updating state */
1915	rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1916	ixgbevf_disable_rx_queue(adapter, ring);
1917
1918	IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1919	IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1920	IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1921	ring->count * sizeof(union ixgbe_adv_rx_desc));
1922
1923	#ifndef CONFIG_SPARC
1924	/* enable relaxed ordering */
1925	IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1926	IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1927	#else
1928	IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1929	IXGBE_DCA_RXCTRL_DESC_RRO_EN |
1930	IXGBE_DCA_RXCTRL_DATA_WRO_EN);
1931	#endif
1932
1933	/* reset head and tail pointers */
1934	IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1935	IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
1936	ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx);
1937
1938	/* initialize rx_buffer_info */
1939	memset(ring->rx_buffer_info, 0,
1940	sizeof(struct ixgbevf_rx_buffer) * ring->count);
1941
1942	/* initialize Rx descriptor 0 */
1943	rx_desc = IXGBEVF_RX_DESC(ring, 0);
1944	rx_desc->wb.upper.length = 0;
1945
1946	/* reset ntu and ntc to place SW in sync with hardwdare */
1947	ring->next_to_clean = 0;
1948	ring->next_to_use = 0;
1949	ring->next_to_alloc = 0;
1950
1951	ixgbevf_configure_srrctl(adapter, ring, index: reg_idx);
1952
1953	/* RXDCTL.RLPML does not work on 82599 */
1954	if (adapter->hw.mac.type != ixgbe_mac_82599_vf) {
1955	rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
1956	IXGBE_RXDCTL_RLPML_EN);
1957
1958	#if (PAGE_SIZE < 8192)
1959	/* Limit the maximum frame size so we don't overrun the skb */
1960	if (ring_uses_build_skb(ring) &&
1961	!ring_uses_large_buffer(ring))
1962	rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB |
1963	IXGBE_RXDCTL_RLPML_EN;
1964	#endif
1965	}
1966
1967	rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1968	IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1969
1970	ixgbevf_rx_desc_queue_enable(adapter, ring);
1971	ixgbevf_alloc_rx_buffers(rx_ring: ring, cleaned_count: ixgbevf_desc_unused(ring));
1972	}
1973
1974	static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter,
1975	struct ixgbevf_ring *rx_ring)
1976	{
1977	struct net_device *netdev = adapter->netdev;
1978	unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1979
1980	/* set build_skb and buffer size flags */
1981	clear_ring_build_skb_enabled(rx_ring);
1982	clear_ring_uses_large_buffer(rx_ring);
1983
1984	if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX)
1985	return;
1986
1987	if (PAGE_SIZE < 8192)
1988	if (max_frame > IXGBEVF_MAX_FRAME_BUILD_SKB)
1989	set_ring_uses_large_buffer(rx_ring);
1990
1991	/* 82599 can't rely on RXDCTL.RLPML to restrict the size of the frame */
1992	if (adapter->hw.mac.type == ixgbe_mac_82599_vf && !ring_uses_large_buffer(rx_ring))
1993	return;
1994
1995	set_ring_build_skb_enabled(rx_ring);
1996	}
1997
1998	/**
1999	* ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
2000	* @adapter: board private structure
2001	*
2002	* Configure the Rx unit of the MAC after a reset.
2003	**/
2004	static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
2005	{
2006	struct ixgbe_hw *hw = &adapter->hw;
2007	struct net_device *netdev = adapter->netdev;
2008	int i, ret;
2009
2010	ixgbevf_setup_psrtype(adapter);
2011	if (hw->mac.type >= ixgbe_mac_X550_vf)
2012	ixgbevf_setup_vfmrqc(adapter);
2013
2014	spin_lock_bh(lock: &adapter->mbx_lock);
2015	/* notify the PF of our intent to use this size of frame */
2016	ret = hw->mac.ops.set_rlpml(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN);
2017	spin_unlock_bh(lock: &adapter->mbx_lock);
2018	if (ret)
2019	dev_err(&adapter->pdev->dev,
2020	"Failed to set MTU at %d\n", netdev->mtu);
2021
2022	/* Setup the HW Rx Head and Tail Descriptor Pointers and
2023	* the Base and Length of the Rx Descriptor Ring
2024	*/
2025	for (i = 0; i < adapter->num_rx_queues; i++) {
2026	struct ixgbevf_ring *rx_ring = adapter->rx_ring[i];
2027
2028	ixgbevf_set_rx_buffer_len(adapter, rx_ring);
2029	ixgbevf_configure_rx_ring(adapter, ring: rx_ring);
2030	}
2031	}
2032
2033	static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
2034	__be16 proto, u16 vid)
2035	{
2036	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
2037	struct ixgbe_hw *hw = &adapter->hw;
2038	int err;
2039
2040	spin_lock_bh(lock: &adapter->mbx_lock);
2041
2042	/* add VID to filter table */
2043	err = hw->mac.ops.set_vfta(hw, vid, 0, true);
2044
2045	spin_unlock_bh(lock: &adapter->mbx_lock);
2046
2047	if (err) {
2048	netdev_err(dev: netdev, format: "VF could not set VLAN %d\n", vid);
2049
2050	/* translate error return types so error makes sense */
2051	if (err == IXGBE_ERR_MBX)
2052	return -EIO;
2053
2054	if (err == IXGBE_ERR_INVALID_ARGUMENT)
2055	return -EACCES;
2056	}
2057
2058	set_bit(nr: vid, addr: adapter->active_vlans);
2059
2060	return err;
2061	}
2062
2063	static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
2064	__be16 proto, u16 vid)
2065	{
2066	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
2067	struct ixgbe_hw *hw = &adapter->hw;
2068	int err;
2069
2070	spin_lock_bh(lock: &adapter->mbx_lock);
2071
2072	/* remove VID from filter table */
2073	err = hw->mac.ops.set_vfta(hw, vid, 0, false);
2074
2075	spin_unlock_bh(lock: &adapter->mbx_lock);
2076
2077	if (err)
2078	netdev_err(dev: netdev, format: "Could not remove VLAN %d\n", vid);
2079
2080	clear_bit(nr: vid, addr: adapter->active_vlans);
2081
2082	return err;
2083	}
2084
2085	static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
2086	{
2087	u16 vid;
2088
2089	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2090	ixgbevf_vlan_rx_add_vid(netdev: adapter->netdev,
2091	htons(ETH_P_8021Q), vid);
2092	}
2093
2094	static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
2095	{
2096	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
2097	struct ixgbe_hw *hw = &adapter->hw;
2098	int count = 0;
2099
2100	if (!netdev_uc_empty(netdev)) {
2101	struct netdev_hw_addr *ha;
2102
2103	netdev_for_each_uc_addr(ha, netdev) {
2104	hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
2105	udelay(200);
2106	}
2107	} else {
2108	/* If the list is empty then send message to PF driver to
2109	* clear all MAC VLANs on this VF.
2110	*/
2111	hw->mac.ops.set_uc_addr(hw, 0, NULL);
2112	}
2113
2114	return count;
2115	}
2116
2117	/**
2118	* ixgbevf_set_rx_mode - Multicast and unicast set
2119	* @netdev: network interface device structure
2120	*
2121	* The set_rx_method entry point is called whenever the multicast address
2122	* list, unicast address list or the network interface flags are updated.
2123	* This routine is responsible for configuring the hardware for proper
2124	* multicast mode and configuring requested unicast filters.
2125	**/
2126	static void ixgbevf_set_rx_mode(struct net_device *netdev)
2127	{
2128	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
2129	struct ixgbe_hw *hw = &adapter->hw;
2130	unsigned int flags = netdev->flags;
2131	int xcast_mode;
2132
2133	/* request the most inclusive mode we need */
2134	if (flags & IFF_PROMISC)
2135	xcast_mode = IXGBEVF_XCAST_MODE_PROMISC;
2136	else if (flags & IFF_ALLMULTI)
2137	xcast_mode = IXGBEVF_XCAST_MODE_ALLMULTI;
2138	else if (flags & (IFF_BROADCAST | IFF_MULTICAST))
2139	xcast_mode = IXGBEVF_XCAST_MODE_MULTI;
2140	else
2141	xcast_mode = IXGBEVF_XCAST_MODE_NONE;
2142
2143	spin_lock_bh(lock: &adapter->mbx_lock);
2144
2145	hw->mac.ops.update_xcast_mode(hw, xcast_mode);
2146
2147	/* reprogram multicast list */
2148	hw->mac.ops.update_mc_addr_list(hw, netdev);
2149
2150	ixgbevf_write_uc_addr_list(netdev);
2151
2152	spin_unlock_bh(lock: &adapter->mbx_lock);
2153	}
2154
2155	static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
2156	{
2157	int q_idx;
2158	struct ixgbevf_q_vector *q_vector;
2159	int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2160
2161	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
2162	q_vector = adapter->q_vector[q_idx];
2163	napi_enable(n: &q_vector->napi);
2164	}
2165	}
2166
2167	static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
2168	{
2169	int q_idx;
2170	struct ixgbevf_q_vector *q_vector;
2171	int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2172
2173	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
2174	q_vector = adapter->q_vector[q_idx];
2175	napi_disable(n: &q_vector->napi);
2176	}
2177	}
2178
2179	static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
2180	{
2181	struct ixgbe_hw *hw = &adapter->hw;
2182	unsigned int def_q = 0;
2183	unsigned int num_tcs = 0;
2184	unsigned int num_rx_queues = adapter->num_rx_queues;
2185	unsigned int num_tx_queues = adapter->num_tx_queues;
2186	int err;
2187
2188	spin_lock_bh(lock: &adapter->mbx_lock);
2189
2190	/* fetch queue configuration from the PF */
2191	err = ixgbevf_get_queues(hw, num_tcs: &num_tcs, default_tc: &def_q);
2192
2193	spin_unlock_bh(lock: &adapter->mbx_lock);
2194
2195	if (err)
2196	return err;
2197
2198	if (num_tcs > 1) {
2199	/* we need only one Tx queue */
2200	num_tx_queues = 1;
2201
2202	/* update default Tx ring register index */
2203	adapter->tx_ring[0]->reg_idx = def_q;
2204
2205	/* we need as many queues as traffic classes */
2206	num_rx_queues = num_tcs;
2207	}
2208
2209	/* if we have a bad config abort request queue reset */
2210	if ((adapter->num_rx_queues != num_rx_queues) ||
2211	(adapter->num_tx_queues != num_tx_queues)) {
2212	/* force mailbox timeout to prevent further messages */
2213	hw->mbx.timeout = 0;
2214
2215	/* wait for watchdog to come around and bail us out */
2216	set_bit(nr: __IXGBEVF_QUEUE_RESET_REQUESTED, addr: &adapter->state);
2217	}
2218
2219	return 0;
2220	}
2221
2222	static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
2223	{
2224	ixgbevf_configure_dcb(adapter);
2225
2226	ixgbevf_set_rx_mode(netdev: adapter->netdev);
2227
2228	ixgbevf_restore_vlan(adapter);
2229	ixgbevf_ipsec_restore(adapter);
2230
2231	ixgbevf_configure_tx(adapter);
2232	ixgbevf_configure_rx(adapter);
2233	}
2234
2235	static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
2236	{
2237	/* Only save pre-reset stats if there are some */
2238	if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
2239	adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
2240	adapter->stats.base_vfgprc;
2241	adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
2242	adapter->stats.base_vfgptc;
2243	adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
2244	adapter->stats.base_vfgorc;
2245	adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
2246	adapter->stats.base_vfgotc;
2247	adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
2248	adapter->stats.base_vfmprc;
2249	}
2250	}
2251
2252	static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
2253	{
2254	struct ixgbe_hw *hw = &adapter->hw;
2255
2256	adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
2257	adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
2258	adapter->stats.last_vfgorc |=
2259	(((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
2260	adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
2261	adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
2262	adapter->stats.last_vfgotc |=
2263	(((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
2264	adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
2265
2266	adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
2267	adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
2268	adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
2269	adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
2270	adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
2271	}
2272
2273	static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
2274	{
2275	struct ixgbe_hw *hw = &adapter->hw;
2276	static const int api[] = {
2277	ixgbe_mbox_api_15,
2278	ixgbe_mbox_api_14,
2279	ixgbe_mbox_api_13,
2280	ixgbe_mbox_api_12,
2281	ixgbe_mbox_api_11,
2282	ixgbe_mbox_api_10,
2283	ixgbe_mbox_api_unknown
2284	};
2285	int err, idx = 0;
2286
2287	spin_lock_bh(lock: &adapter->mbx_lock);
2288
2289	while (api[idx] != ixgbe_mbox_api_unknown) {
2290	err = hw->mac.ops.negotiate_api_version(hw, api[idx]);
2291	if (!err)
2292	break;
2293	idx++;
2294	}
2295
2296	if (hw->api_version >= ixgbe_mbox_api_15) {
2297	hw->mbx.ops.init_params(hw);
2298	memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
2299	sizeof(struct ixgbe_mbx_operations));
2300	}
2301
2302	spin_unlock_bh(lock: &adapter->mbx_lock);
2303	}
2304
2305	static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
2306	{
2307	struct net_device *netdev = adapter->netdev;
2308	struct pci_dev *pdev = adapter->pdev;
2309	struct ixgbe_hw *hw = &adapter->hw;
2310	bool state;
2311
2312	ixgbevf_configure_msix(adapter);
2313
2314	spin_lock_bh(lock: &adapter->mbx_lock);
2315
2316	if (is_valid_ether_addr(addr: hw->mac.addr))
2317	hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
2318	else
2319	hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
2320
2321	spin_unlock_bh(lock: &adapter->mbx_lock);
2322
2323	state = adapter->link_state;
2324	hw->mac.ops.get_link_state(hw, &adapter->link_state);
2325	if (state && state != adapter->link_state)
2326	dev_info(&pdev->dev, "VF is administratively disabled\n");
2327
2328	smp_mb__before_atomic();
2329	clear_bit(nr: __IXGBEVF_DOWN, addr: &adapter->state);
2330	ixgbevf_napi_enable_all(adapter);
2331
2332	/* clear any pending interrupts, may auto mask */
2333	IXGBE_READ_REG(hw, IXGBE_VTEICR);
2334	ixgbevf_irq_enable(adapter);
2335
2336	/* enable transmits */
2337	netif_tx_start_all_queues(dev: netdev);
2338
2339	ixgbevf_save_reset_stats(adapter);
2340	ixgbevf_init_last_counter_stats(adapter);
2341
2342	hw->mac.get_link_status = 1;
2343	mod_timer(timer: &adapter->service_timer, expires: jiffies);
2344	}
2345
2346	void ixgbevf_up(struct ixgbevf_adapter *adapter)
2347	{
2348	ixgbevf_configure(adapter);
2349
2350	ixgbevf_up_complete(adapter);
2351	}
2352
2353	/**
2354	* ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
2355	* @rx_ring: ring to free buffers from
2356	**/
2357	static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
2358	{
2359	u16 i = rx_ring->next_to_clean;
2360
2361	/* Free Rx ring sk_buff */
2362	if (rx_ring->skb) {
2363	dev_kfree_skb(rx_ring->skb);
2364	rx_ring->skb = NULL;
2365	}
2366
2367	/* Free all the Rx ring pages */
2368	while (i != rx_ring->next_to_alloc) {
2369	struct ixgbevf_rx_buffer *rx_buffer;
2370
2371	rx_buffer = &rx_ring->rx_buffer_info[i];
2372
2373	/* Invalidate cache lines that may have been written to by
2374	* device so that we avoid corrupting memory.
2375	*/
2376	dma_sync_single_range_for_cpu(dev: rx_ring->dev,
2377	addr: rx_buffer->dma,
2378	offset: rx_buffer->page_offset,
2379	size: ixgbevf_rx_bufsz(ring: rx_ring),
2380	dir: DMA_FROM_DEVICE);
2381
2382	/* free resources associated with mapping */
2383	dma_unmap_page_attrs(dev: rx_ring->dev,
2384	addr: rx_buffer->dma,
2385	ixgbevf_rx_pg_size(rx_ring),
2386	dir: DMA_FROM_DEVICE,
2387	IXGBEVF_RX_DMA_ATTR);
2388
2389	__page_frag_cache_drain(page: rx_buffer->page,
2390	count: rx_buffer->pagecnt_bias);
2391
2392	i++;
2393	if (i == rx_ring->count)
2394	i = 0;
2395	}
2396
2397	rx_ring->next_to_alloc = 0;
2398	rx_ring->next_to_clean = 0;
2399	rx_ring->next_to_use = 0;
2400	}
2401
2402	/**
2403	* ixgbevf_clean_tx_ring - Free Tx Buffers
2404	* @tx_ring: ring to be cleaned
2405	**/
2406	static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
2407	{
2408	u16 i = tx_ring->next_to_clean;
2409	struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
2410
2411	while (i != tx_ring->next_to_use) {
2412	union ixgbe_adv_tx_desc *eop_desc, *tx_desc;
2413
2414	/* Free all the Tx ring sk_buffs */
2415	if (ring_is_xdp(tx_ring))
2416	page_frag_free(addr: tx_buffer->data);
2417	else
2418	dev_kfree_skb_any(skb: tx_buffer->skb);
2419
2420	/* unmap skb header data */
2421	dma_unmap_single(tx_ring->dev,
2422	dma_unmap_addr(tx_buffer, dma),
2423	dma_unmap_len(tx_buffer, len),
2424	DMA_TO_DEVICE);
2425
2426	/* check for eop_desc to determine the end of the packet */
2427	eop_desc = tx_buffer->next_to_watch;
2428	tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
2429
2430	/* unmap remaining buffers */
2431	while (tx_desc != eop_desc) {
2432	tx_buffer++;
2433	tx_desc++;
2434	i++;
2435	if (unlikely(i == tx_ring->count)) {
2436	i = 0;
2437	tx_buffer = tx_ring->tx_buffer_info;
2438	tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
2439	}
2440
2441	/* unmap any remaining paged data */
2442	if (dma_unmap_len(tx_buffer, len))
2443	dma_unmap_page(tx_ring->dev,
2444	dma_unmap_addr(tx_buffer, dma),
2445	dma_unmap_len(tx_buffer, len),
2446	DMA_TO_DEVICE);
2447	}
2448
2449	/* move us one more past the eop_desc for start of next pkt */
2450	tx_buffer++;
2451	i++;
2452	if (unlikely(i == tx_ring->count)) {
2453	i = 0;
2454	tx_buffer = tx_ring->tx_buffer_info;
2455	}
2456	}
2457
2458	/* reset next_to_use and next_to_clean */
2459	tx_ring->next_to_use = 0;
2460	tx_ring->next_to_clean = 0;
2461
2462	}
2463
2464	/**
2465	* ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
2466	* @adapter: board private structure
2467	**/
2468	static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
2469	{
2470	int i;
2471
2472	for (i = 0; i < adapter->num_rx_queues; i++)
2473	ixgbevf_clean_rx_ring(rx_ring: adapter->rx_ring[i]);
2474	}
2475
2476	/**
2477	* ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
2478	* @adapter: board private structure
2479	**/
2480	static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
2481	{
2482	int i;
2483
2484	for (i = 0; i < adapter->num_tx_queues; i++)
2485	ixgbevf_clean_tx_ring(tx_ring: adapter->tx_ring[i]);
2486	for (i = 0; i < adapter->num_xdp_queues; i++)
2487	ixgbevf_clean_tx_ring(tx_ring: adapter->xdp_ring[i]);
2488	}
2489
2490	void ixgbevf_down(struct ixgbevf_adapter *adapter)
2491	{
2492	struct net_device *netdev = adapter->netdev;
2493	struct ixgbe_hw *hw = &adapter->hw;
2494	int i;
2495
2496	/* signal that we are down to the interrupt handler */
2497	if (test_and_set_bit(nr: __IXGBEVF_DOWN, addr: &adapter->state))
2498	return; /* do nothing if already down */
2499
2500	/* disable all enabled Rx queues */
2501	for (i = 0; i < adapter->num_rx_queues; i++)
2502	ixgbevf_disable_rx_queue(adapter, ring: adapter->rx_ring[i]);
2503
2504	usleep_range(min: 10000, max: 20000);
2505
2506	netif_tx_stop_all_queues(dev: netdev);
2507
2508	/* call carrier off first to avoid false dev_watchdog timeouts */
2509	netif_carrier_off(dev: netdev);
2510	netif_tx_disable(dev: netdev);
2511
2512	ixgbevf_irq_disable(adapter);
2513
2514	ixgbevf_napi_disable_all(adapter);
2515
2516	del_timer_sync(timer: &adapter->service_timer);
2517
2518	/* disable transmits in the hardware now that interrupts are off */
2519	for (i = 0; i < adapter->num_tx_queues; i++) {
2520	u8 reg_idx = adapter->tx_ring[i]->reg_idx;
2521
2522	IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
2523	IXGBE_TXDCTL_SWFLSH);
2524	}
2525
2526	for (i = 0; i < adapter->num_xdp_queues; i++) {
2527	u8 reg_idx = adapter->xdp_ring[i]->reg_idx;
2528
2529	IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
2530	IXGBE_TXDCTL_SWFLSH);
2531	}
2532
2533	if (!pci_channel_offline(pdev: adapter->pdev))
2534	ixgbevf_reset(adapter);
2535
2536	ixgbevf_clean_all_tx_rings(adapter);
2537	ixgbevf_clean_all_rx_rings(adapter);
2538	}
2539
2540	void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
2541	{
2542	while (test_and_set_bit(nr: __IXGBEVF_RESETTING, addr: &adapter->state))
2543	msleep(msecs: 1);
2544
2545	ixgbevf_down(adapter);
2546	pci_set_master(dev: adapter->pdev);
2547	ixgbevf_up(adapter);
2548
2549	clear_bit(nr: __IXGBEVF_RESETTING, addr: &adapter->state);
2550	}
2551
2552	void ixgbevf_reset(struct ixgbevf_adapter *adapter)
2553	{
2554	struct ixgbe_hw *hw = &adapter->hw;
2555	struct net_device *netdev = adapter->netdev;
2556
2557	if (hw->mac.ops.reset_hw(hw)) {
2558	hw_dbg(hw, "PF still resetting\n");
2559	} else {
2560	hw->mac.ops.init_hw(hw);
2561	ixgbevf_negotiate_api(adapter);
2562	}
2563
2564	if (is_valid_ether_addr(addr: adapter->hw.mac.addr)) {
2565	eth_hw_addr_set(dev: netdev, addr: adapter->hw.mac.addr);
2566	ether_addr_copy(dst: netdev->perm_addr, src: adapter->hw.mac.addr);
2567	}
2568
2569	adapter->last_reset = jiffies;
2570	}
2571
2572	static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
2573	int vectors)
2574	{
2575	int vector_threshold;
2576
2577	/* We'll want at least 2 (vector_threshold):
2578	* 1) TxQ[0] + RxQ[0] handler
2579	* 2) Other (Link Status Change, etc.)
2580	*/
2581	vector_threshold = MIN_MSIX_COUNT;
2582
2583	/* The more we get, the more we will assign to Tx/Rx Cleanup
2584	* for the separate queues...where Rx Cleanup >= Tx Cleanup.
2585	* Right now, we simply care about how many we'll get; we'll
2586	* set them up later while requesting irq's.
2587	*/
2588	vectors = pci_enable_msix_range(dev: adapter->pdev, entries: adapter->msix_entries,
2589	minvec: vector_threshold, maxvec: vectors);
2590
2591	if (vectors < 0) {
2592	dev_err(&adapter->pdev->dev,
2593	"Unable to allocate MSI-X interrupts\n");
2594	kfree(objp: adapter->msix_entries);
2595	adapter->msix_entries = NULL;
2596	return vectors;
2597	}
2598
2599	/* Adjust for only the vectors we'll use, which is minimum
2600	* of max_msix_q_vectors + NON_Q_VECTORS, or the number of
2601	* vectors we were allocated.
2602	*/
2603	adapter->num_msix_vectors = vectors;
2604
2605	return 0;
2606	}
2607
2608	/**
2609	* ixgbevf_set_num_queues - Allocate queues for device, feature dependent
2610	* @adapter: board private structure to initialize
2611	*
2612	* This is the top level queue allocation routine. The order here is very
2613	* important, starting with the "most" number of features turned on at once,
2614	* and ending with the smallest set of features. This way large combinations
2615	* can be allocated if they're turned on, and smaller combinations are the
2616	* fall through conditions.
2617	*
2618	**/
2619	static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
2620	{
2621	struct ixgbe_hw *hw = &adapter->hw;
2622	unsigned int def_q = 0;
2623	unsigned int num_tcs = 0;
2624	int err;
2625
2626	/* Start with base case */
2627	adapter->num_rx_queues = 1;
2628	adapter->num_tx_queues = 1;
2629	adapter->num_xdp_queues = 0;
2630
2631	spin_lock_bh(lock: &adapter->mbx_lock);
2632
2633	/* fetch queue configuration from the PF */
2634	err = ixgbevf_get_queues(hw, num_tcs: &num_tcs, default_tc: &def_q);
2635
2636	spin_unlock_bh(lock: &adapter->mbx_lock);
2637
2638	if (err)
2639	return;
2640
2641	/* we need as many queues as traffic classes */
2642	if (num_tcs > 1) {
2643	adapter->num_rx_queues = num_tcs;
2644	} else {
2645	u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES);
2646
2647	switch (hw->api_version) {
2648	case ixgbe_mbox_api_11:
2649	case ixgbe_mbox_api_12:
2650	case ixgbe_mbox_api_13:
2651	case ixgbe_mbox_api_14:
2652	case ixgbe_mbox_api_15:
2653	if (adapter->xdp_prog &&
2654	hw->mac.max_tx_queues == rss)
2655	rss = rss > 3 ? 2 : 1;
2656
2657	adapter->num_rx_queues = rss;
2658	adapter->num_tx_queues = rss;
2659	adapter->num_xdp_queues = adapter->xdp_prog ? rss : 0;
2660	break;
2661	default:
2662	break;
2663	}
2664	}
2665	}
2666
2667	/**
2668	* ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
2669	* @adapter: board private structure to initialize
2670	*
2671	* Attempt to configure the interrupts using the best available
2672	* capabilities of the hardware and the kernel.
2673	**/
2674	static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
2675	{
2676	int vector, v_budget;
2677
2678	/* It's easy to be greedy for MSI-X vectors, but it really
2679	* doesn't do us much good if we have a lot more vectors
2680	* than CPU's. So let's be conservative and only ask for
2681	* (roughly) the same number of vectors as there are CPU's.
2682	* The default is to use pairs of vectors.
2683	*/
2684	v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
2685	v_budget = min_t(int, v_budget, num_online_cpus());
2686	v_budget += NON_Q_VECTORS;
2687
2688	adapter->msix_entries = kcalloc(n: v_budget,
2689	size: sizeof(struct msix_entry), GFP_KERNEL);
2690	if (!adapter->msix_entries)
2691	return -ENOMEM;
2692
2693	for (vector = 0; vector < v_budget; vector++)
2694	adapter->msix_entries[vector].entry = vector;
2695
2696	/* A failure in MSI-X entry allocation isn't fatal, but the VF driver
2697	* does not support any other modes, so we will simply fail here. Note
2698	* that we clean up the msix_entries pointer else-where.
2699	*/
2700	return ixgbevf_acquire_msix_vectors(adapter, vectors: v_budget);
2701	}
2702
2703	static void ixgbevf_add_ring(struct ixgbevf_ring *ring,
2704	struct ixgbevf_ring_container *head)
2705	{
2706	ring->next = head->ring;
2707	head->ring = ring;
2708	head->count++;
2709	}
2710
2711	/**
2712	* ixgbevf_alloc_q_vector - Allocate memory for a single interrupt vector
2713	* @adapter: board private structure to initialize
2714	* @v_idx: index of vector in adapter struct
2715	* @txr_count: number of Tx rings for q vector
2716	* @txr_idx: index of first Tx ring to assign
2717	* @xdp_count: total number of XDP rings to allocate
2718	* @xdp_idx: index of first XDP ring to allocate
2719	* @rxr_count: number of Rx rings for q vector
2720	* @rxr_idx: index of first Rx ring to assign
2721	*
2722	* We allocate one q_vector. If allocation fails we return -ENOMEM.
2723	**/
2724	static int ixgbevf_alloc_q_vector(struct ixgbevf_adapter *adapter, int v_idx,
2725	int txr_count, int txr_idx,
2726	int xdp_count, int xdp_idx,
2727	int rxr_count, int rxr_idx)
2728	{
2729	struct ixgbevf_q_vector *q_vector;
2730	int reg_idx = txr_idx + xdp_idx;
2731	struct ixgbevf_ring *ring;
2732	int ring_count, size;
2733
2734	ring_count = txr_count + xdp_count + rxr_count;
2735	size = sizeof(*q_vector) + (sizeof(*ring) * ring_count);
2736
2737	/* allocate q_vector and rings */
2738	q_vector = kzalloc(size, GFP_KERNEL);
2739	if (!q_vector)
2740	return -ENOMEM;
2741
2742	/* initialize NAPI */
2743	netif_napi_add(dev: adapter->netdev, napi: &q_vector->napi, poll: ixgbevf_poll);
2744
2745	/* tie q_vector and adapter together */
2746	adapter->q_vector[v_idx] = q_vector;
2747	q_vector->adapter = adapter;
2748	q_vector->v_idx = v_idx;
2749
2750	/* initialize pointer to rings */
2751	ring = q_vector->ring;
2752
2753	while (txr_count) {
2754	/* assign generic ring traits */
2755	ring->dev = &adapter->pdev->dev;
2756	ring->netdev = adapter->netdev;
2757
2758	/* configure backlink on ring */
2759	ring->q_vector = q_vector;
2760
2761	/* update q_vector Tx values */
2762	ixgbevf_add_ring(ring, head: &q_vector->tx);
2763
2764	/* apply Tx specific ring traits */
2765	ring->count = adapter->tx_ring_count;
2766	ring->queue_index = txr_idx;
2767	ring->reg_idx = reg_idx;
2768
2769	/* assign ring to adapter */
2770	adapter->tx_ring[txr_idx] = ring;
2771
2772	/* update count and index */
2773	txr_count--;
2774	txr_idx++;
2775	reg_idx++;
2776
2777	/* push pointer to next ring */
2778	ring++;
2779	}
2780
2781	while (xdp_count) {
2782	/* assign generic ring traits */
2783	ring->dev = &adapter->pdev->dev;
2784	ring->netdev = adapter->netdev;
2785
2786	/* configure backlink on ring */
2787	ring->q_vector = q_vector;
2788
2789	/* update q_vector Tx values */
2790	ixgbevf_add_ring(ring, head: &q_vector->tx);
2791
2792	/* apply Tx specific ring traits */
2793	ring->count = adapter->tx_ring_count;
2794	ring->queue_index = xdp_idx;
2795	ring->reg_idx = reg_idx;
2796	set_ring_xdp(ring);
2797
2798	/* assign ring to adapter */
2799	adapter->xdp_ring[xdp_idx] = ring;
2800
2801	/* update count and index */
2802	xdp_count--;
2803	xdp_idx++;
2804	reg_idx++;
2805
2806	/* push pointer to next ring */
2807	ring++;
2808	}
2809
2810	while (rxr_count) {
2811	/* assign generic ring traits */
2812	ring->dev = &adapter->pdev->dev;
2813	ring->netdev = adapter->netdev;
2814
2815	/* configure backlink on ring */
2816	ring->q_vector = q_vector;
2817
2818	/* update q_vector Rx values */
2819	ixgbevf_add_ring(ring, head: &q_vector->rx);
2820
2821	/* apply Rx specific ring traits */
2822	ring->count = adapter->rx_ring_count;
2823	ring->queue_index = rxr_idx;
2824	ring->reg_idx = rxr_idx;
2825
2826	/* assign ring to adapter */
2827	adapter->rx_ring[rxr_idx] = ring;
2828
2829	/* update count and index */
2830	rxr_count--;
2831	rxr_idx++;
2832
2833	/* push pointer to next ring */
2834	ring++;
2835	}
2836
2837	return 0;
2838	}
2839
2840	/**
2841	* ixgbevf_free_q_vector - Free memory allocated for specific interrupt vector
2842	* @adapter: board private structure to initialize
2843	* @v_idx: index of vector in adapter struct
2844	*
2845	* This function frees the memory allocated to the q_vector. In addition if
2846	* NAPI is enabled it will delete any references to the NAPI struct prior
2847	* to freeing the q_vector.
2848	**/
2849	static void ixgbevf_free_q_vector(struct ixgbevf_adapter *adapter, int v_idx)
2850	{
2851	struct ixgbevf_q_vector *q_vector = adapter->q_vector[v_idx];
2852	struct ixgbevf_ring *ring;
2853
2854	ixgbevf_for_each_ring(ring, q_vector->tx) {
2855	if (ring_is_xdp(ring))
2856	adapter->xdp_ring[ring->queue_index] = NULL;
2857	else
2858	adapter->tx_ring[ring->queue_index] = NULL;
2859	}
2860
2861	ixgbevf_for_each_ring(ring, q_vector->rx)
2862	adapter->rx_ring[ring->queue_index] = NULL;
2863
2864	adapter->q_vector[v_idx] = NULL;
2865	netif_napi_del(napi: &q_vector->napi);
2866
2867	/* ixgbevf_get_stats() might access the rings on this vector,
2868	* we must wait a grace period before freeing it.
2869	*/
2870	kfree_rcu(q_vector, rcu);
2871	}
2872
2873	/**
2874	* ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2875	* @adapter: board private structure to initialize
2876	*
2877	* We allocate one q_vector per queue interrupt. If allocation fails we
2878	* return -ENOMEM.
2879	**/
2880	static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2881	{
2882	int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2883	int rxr_remaining = adapter->num_rx_queues;
2884	int txr_remaining = adapter->num_tx_queues;
2885	int xdp_remaining = adapter->num_xdp_queues;
2886	int rxr_idx = 0, txr_idx = 0, xdp_idx = 0, v_idx = 0;
2887	int err;
2888
2889	if (q_vectors >= (rxr_remaining + txr_remaining + xdp_remaining)) {
2890	for (; rxr_remaining; v_idx++, q_vectors--) {
2891	int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors);
2892
2893	err = ixgbevf_alloc_q_vector(adapter, v_idx,
2894	txr_count: 0, txr_idx: 0, xdp_count: 0, xdp_idx: 0, rxr_count: rqpv, rxr_idx);
2895	if (err)
2896	goto err_out;
2897
2898	/* update counts and index */
2899	rxr_remaining -= rqpv;
2900	rxr_idx += rqpv;
2901	}
2902	}
2903
2904	for (; q_vectors; v_idx++, q_vectors--) {
2905	int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors);
2906	int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors);
2907	int xqpv = DIV_ROUND_UP(xdp_remaining, q_vectors);
2908
2909	err = ixgbevf_alloc_q_vector(adapter, v_idx,
2910	txr_count: tqpv, txr_idx,
2911	xdp_count: xqpv, xdp_idx,
2912	rxr_count: rqpv, rxr_idx);
2913
2914	if (err)
2915	goto err_out;
2916
2917	/* update counts and index */
2918	rxr_remaining -= rqpv;
2919	rxr_idx += rqpv;
2920	txr_remaining -= tqpv;
2921	txr_idx += tqpv;
2922	xdp_remaining -= xqpv;
2923	xdp_idx += xqpv;
2924	}
2925
2926	return 0;
2927
2928	err_out:
2929	while (v_idx) {
2930	v_idx--;
2931	ixgbevf_free_q_vector(adapter, v_idx);
2932	}
2933
2934	return -ENOMEM;
2935	}
2936
2937	/**
2938	* ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2939	* @adapter: board private structure to initialize
2940	*
2941	* This function frees the memory allocated to the q_vectors. In addition if
2942	* NAPI is enabled it will delete any references to the NAPI struct prior
2943	* to freeing the q_vector.
2944	**/
2945	static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2946	{
2947	int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2948
2949	while (q_vectors) {
2950	q_vectors--;
2951	ixgbevf_free_q_vector(adapter, v_idx: q_vectors);
2952	}
2953	}
2954
2955	/**
2956	* ixgbevf_reset_interrupt_capability - Reset MSIX setup
2957	* @adapter: board private structure
2958	*
2959	**/
2960	static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2961	{
2962	if (!adapter->msix_entries)
2963	return;
2964
2965	pci_disable_msix(dev: adapter->pdev);
2966	kfree(objp: adapter->msix_entries);
2967	adapter->msix_entries = NULL;
2968	}
2969
2970	/**
2971	* ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2972	* @adapter: board private structure to initialize
2973	*
2974	**/
2975	static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2976	{
2977	int err;
2978
2979	/* Number of supported queues */
2980	ixgbevf_set_num_queues(adapter);
2981
2982	err = ixgbevf_set_interrupt_capability(adapter);
2983	if (err) {
2984	hw_dbg(&adapter->hw,
2985	"Unable to setup interrupt capabilities\n");
2986	goto err_set_interrupt;
2987	}
2988
2989	err = ixgbevf_alloc_q_vectors(adapter);
2990	if (err) {
2991	hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n");
2992	goto err_alloc_q_vectors;
2993	}
2994
2995	hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u XDP Queue count %u\n",
2996	(adapter->num_rx_queues > 1) ? "Enabled" : "Disabled",
2997	adapter->num_rx_queues, adapter->num_tx_queues,
2998	adapter->num_xdp_queues);
2999
3000	set_bit(nr: __IXGBEVF_DOWN, addr: &adapter->state);
3001
3002	return 0;
3003	err_alloc_q_vectors:
3004	ixgbevf_reset_interrupt_capability(adapter);
3005	err_set_interrupt:
3006	return err;
3007	}
3008
3009	/**
3010	* ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
3011	* @adapter: board private structure to clear interrupt scheme on
3012	*
3013	* We go through and clear interrupt specific resources and reset the structure
3014	* to pre-load conditions
3015	**/
3016	static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
3017	{
3018	adapter->num_tx_queues = 0;
3019	adapter->num_xdp_queues = 0;
3020	adapter->num_rx_queues = 0;
3021
3022	ixgbevf_free_q_vectors(adapter);
3023	ixgbevf_reset_interrupt_capability(adapter);
3024	}
3025
3026	/**
3027	* ixgbevf_sw_init - Initialize general software structures
3028	* @adapter: board private structure to initialize
3029	*
3030	* ixgbevf_sw_init initializes the Adapter private data structure.
3031	* Fields are initialized based on PCI device information and
3032	* OS network device settings (MTU size).
3033	**/
3034	static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
3035	{
3036	struct ixgbe_hw *hw = &adapter->hw;
3037	struct pci_dev *pdev = adapter->pdev;
3038	struct net_device *netdev = adapter->netdev;
3039	int err;
3040
3041	/* PCI config space info */
3042	hw->vendor_id = pdev->vendor;
3043	hw->device_id = pdev->device;
3044	hw->revision_id = pdev->revision;
3045	hw->subsystem_vendor_id = pdev->subsystem_vendor;
3046	hw->subsystem_device_id = pdev->subsystem_device;
3047
3048	hw->mbx.ops.init_params(hw);
3049
3050	if (hw->mac.type >= ixgbe_mac_X550_vf) {
3051	err = ixgbevf_init_rss_key(adapter);
3052	if (err)
3053	goto out;
3054	}
3055
3056	/* assume legacy case in which PF would only give VF 2 queues */
3057	hw->mac.max_tx_queues = 2;
3058	hw->mac.max_rx_queues = 2;
3059
3060	/* lock to protect mailbox accesses */
3061	spin_lock_init(&adapter->mbx_lock);
3062
3063	err = hw->mac.ops.reset_hw(hw);
3064	if (err) {
3065	dev_info(&pdev->dev,
3066	"PF still in reset state. Is the PF interface up?\n");
3067	} else {
3068	err = hw->mac.ops.init_hw(hw);
3069	if (err) {
3070	pr_err("init_shared_code failed: %d\n", err);
3071	goto out;
3072	}
3073	ixgbevf_negotiate_api(adapter);
3074	err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
3075	if (err)
3076	dev_info(&pdev->dev, "Error reading MAC address\n");
3077	else if (is_zero_ether_addr(addr: adapter->hw.mac.addr))
3078	dev_info(&pdev->dev,
3079	"MAC address not assigned by administrator.\n");
3080	eth_hw_addr_set(dev: netdev, addr: hw->mac.addr);
3081	}
3082
3083	if (!is_valid_ether_addr(addr: netdev->dev_addr)) {
3084	dev_info(&pdev->dev, "Assigning random MAC address\n");
3085	eth_hw_addr_random(dev: netdev);
3086	ether_addr_copy(dst: hw->mac.addr, src: netdev->dev_addr);
3087	ether_addr_copy(dst: hw->mac.perm_addr, src: netdev->dev_addr);
3088	}
3089
3090	/* Enable dynamic interrupt throttling rates */
3091	adapter->rx_itr_setting = 1;
3092	adapter->tx_itr_setting = 1;
3093
3094	/* set default ring sizes */
3095	adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
3096	adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
3097
3098	adapter->link_state = true;
3099
3100	set_bit(nr: __IXGBEVF_DOWN, addr: &adapter->state);
3101	return 0;
3102
3103	out:
3104	return err;
3105	}
3106
3107	#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \
3108	{ \
3109	u32 current_counter = IXGBE_READ_REG(hw, reg); \
3110	if (current_counter < last_counter) \
3111	counter += 0x100000000LL; \
3112	last_counter = current_counter; \
3113	counter &= 0xFFFFFFFF00000000LL; \
3114	counter |= current_counter; \
3115	}
3116
3117	#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
3118	{ \
3119	u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \
3120	u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \
3121	u64 current_counter = (current_counter_msb << 32) | \
3122	current_counter_lsb; \
3123	if (current_counter < last_counter) \
3124	counter += 0x1000000000LL; \
3125	last_counter = current_counter; \
3126	counter &= 0xFFFFFFF000000000LL; \
3127	counter |= current_counter; \
3128	}
3129	/**
3130	* ixgbevf_update_stats - Update the board statistics counters.
3131	* @adapter: board private structure
3132	**/
3133	void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
3134	{
3135	struct ixgbe_hw *hw = &adapter->hw;
3136	u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
3137	u64 alloc_rx_page = 0, hw_csum_rx_error = 0;
3138	int i;
3139
3140	if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3141	test_bit(__IXGBEVF_RESETTING, &adapter->state))
3142	return;
3143
3144	UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
3145	adapter->stats.vfgprc);
3146	UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
3147	adapter->stats.vfgptc);
3148	UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
3149	adapter->stats.last_vfgorc,
3150	adapter->stats.vfgorc);
3151	UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
3152	adapter->stats.last_vfgotc,
3153	adapter->stats.vfgotc);
3154	UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
3155	adapter->stats.vfmprc);
3156
3157	for (i = 0; i < adapter->num_rx_queues; i++) {
3158	struct ixgbevf_ring *rx_ring = adapter->rx_ring[i];
3159
3160	hw_csum_rx_error += rx_ring->rx_stats.csum_err;
3161	alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
3162	alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
3163	alloc_rx_page += rx_ring->rx_stats.alloc_rx_page;
3164	}
3165
3166	adapter->hw_csum_rx_error = hw_csum_rx_error;
3167	adapter->alloc_rx_page_failed = alloc_rx_page_failed;
3168	adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
3169	adapter->alloc_rx_page = alloc_rx_page;
3170	}
3171
3172	/**
3173	* ixgbevf_service_timer - Timer Call-back
3174	* @t: pointer to timer_list struct
3175	**/
3176	static void ixgbevf_service_timer(struct timer_list *t)
3177	{
3178	struct ixgbevf_adapter *adapter = from_timer(adapter, t,
3179	service_timer);
3180
3181	/* Reset the timer */
3182	mod_timer(timer: &adapter->service_timer, expires: (HZ * 2) + jiffies);
3183
3184	ixgbevf_service_event_schedule(adapter);
3185	}
3186
3187	static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter)
3188	{
3189	if (!test_and_clear_bit(nr: __IXGBEVF_RESET_REQUESTED, addr: &adapter->state))
3190	return;
3191
3192	rtnl_lock();
3193	/* If we're already down or resetting, just bail */
3194	if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3195	test_bit(__IXGBEVF_REMOVING, &adapter->state) ||
3196	test_bit(__IXGBEVF_RESETTING, &adapter->state)) {
3197	rtnl_unlock();
3198	return;
3199	}
3200
3201	adapter->tx_timeout_count++;
3202
3203	ixgbevf_reinit_locked(adapter);
3204	rtnl_unlock();
3205	}
3206
3207	/**
3208	* ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts
3209	* @adapter: pointer to the device adapter structure
3210	*
3211	* This function serves two purposes. First it strobes the interrupt lines
3212	* in order to make certain interrupts are occurring. Secondly it sets the
3213	* bits needed to check for TX hangs. As a result we should immediately
3214	* determine if a hang has occurred.
3215	**/
3216	static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter)
3217	{
3218	struct ixgbe_hw *hw = &adapter->hw;
3219	u32 eics = 0;
3220	int i;
3221
3222	/* If we're down or resetting, just bail */
3223	if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3224	test_bit(__IXGBEVF_RESETTING, &adapter->state))
3225	return;
3226
3227	/* Force detection of hung controller */
3228	if (netif_carrier_ok(dev: adapter->netdev)) {
3229	for (i = 0; i < adapter->num_tx_queues; i++)
3230	set_check_for_tx_hang(adapter->tx_ring[i]);
3231	for (i = 0; i < adapter->num_xdp_queues; i++)
3232	set_check_for_tx_hang(adapter->xdp_ring[i]);
3233	}
3234
3235	/* get one bit for every active Tx/Rx interrupt vector */
3236	for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
3237	struct ixgbevf_q_vector *qv = adapter->q_vector[i];
3238
3239	if (qv->rx.ring || qv->tx.ring)
3240	eics |= BIT(i);
3241	}
3242
3243	/* Cause software interrupt to ensure rings are cleaned */
3244	IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
3245	}
3246
3247	/**
3248	* ixgbevf_watchdog_update_link - update the link status
3249	* @adapter: pointer to the device adapter structure
3250	**/
3251	static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter)
3252	{
3253	struct ixgbe_hw *hw = &adapter->hw;
3254	u32 link_speed = adapter->link_speed;
3255	bool link_up = adapter->link_up;
3256	s32 err;
3257
3258	spin_lock_bh(lock: &adapter->mbx_lock);
3259
3260	err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
3261
3262	spin_unlock_bh(lock: &adapter->mbx_lock);
3263
3264	/* if check for link returns error we will need to reset */
3265	if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) {
3266	set_bit(nr: __IXGBEVF_RESET_REQUESTED, addr: &adapter->state);
3267	link_up = false;
3268	}
3269
3270	adapter->link_up = link_up;
3271	adapter->link_speed = link_speed;
3272	}
3273
3274	/**
3275	* ixgbevf_watchdog_link_is_up - update netif_carrier status and
3276	* print link up message
3277	* @adapter: pointer to the device adapter structure
3278	**/
3279	static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter)
3280	{
3281	struct net_device *netdev = adapter->netdev;
3282
3283	/* only continue if link was previously down */
3284	if (netif_carrier_ok(dev: netdev))
3285	return;
3286
3287	dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n",
3288	(adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ?
3289	"10 Gbps" :
3290	(adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ?
3291	"1 Gbps" :
3292	(adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ?
3293	"100 Mbps" :
3294	"unknown speed");
3295
3296	netif_carrier_on(dev: netdev);
3297	}
3298
3299	/**
3300	* ixgbevf_watchdog_link_is_down - update netif_carrier status and
3301	* print link down message
3302	* @adapter: pointer to the adapter structure
3303	**/
3304	static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter)
3305	{
3306	struct net_device *netdev = adapter->netdev;
3307
3308	adapter->link_speed = 0;
3309
3310	/* only continue if link was up previously */
3311	if (!netif_carrier_ok(dev: netdev))
3312	return;
3313
3314	dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
3315
3316	netif_carrier_off(dev: netdev);
3317	}
3318
3319	/**
3320	* ixgbevf_watchdog_subtask - worker thread to bring link up
3321	* @adapter: board private structure
3322	**/
3323	static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter)
3324	{
3325	/* if interface is down do nothing */
3326	if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3327	test_bit(__IXGBEVF_RESETTING, &adapter->state))
3328	return;
3329
3330	ixgbevf_watchdog_update_link(adapter);
3331
3332	if (adapter->link_up && adapter->link_state)
3333	ixgbevf_watchdog_link_is_up(adapter);
3334	else
3335	ixgbevf_watchdog_link_is_down(adapter);
3336
3337	ixgbevf_update_stats(adapter);
3338	}
3339
3340	/**
3341	* ixgbevf_service_task - manages and runs subtasks
3342	* @work: pointer to work_struct containing our data
3343	**/
3344	static void ixgbevf_service_task(struct work_struct *work)
3345	{
3346	struct ixgbevf_adapter *adapter = container_of(work,
3347	struct ixgbevf_adapter,
3348	service_task);
3349	struct ixgbe_hw *hw = &adapter->hw;
3350
3351	if (IXGBE_REMOVED(hw->hw_addr)) {
3352	if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
3353	rtnl_lock();
3354	ixgbevf_down(adapter);
3355	rtnl_unlock();
3356	}
3357	return;
3358	}
3359
3360	ixgbevf_queue_reset_subtask(adapter);
3361	ixgbevf_reset_subtask(adapter);
3362	ixgbevf_watchdog_subtask(adapter);
3363	ixgbevf_check_hang_subtask(adapter);
3364
3365	ixgbevf_service_event_complete(adapter);
3366	}
3367
3368	/**
3369	* ixgbevf_free_tx_resources - Free Tx Resources per Queue
3370	* @tx_ring: Tx descriptor ring for a specific queue
3371	*
3372	* Free all transmit software resources
3373	**/
3374	void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
3375	{
3376	ixgbevf_clean_tx_ring(tx_ring);
3377
3378	vfree(addr: tx_ring->tx_buffer_info);
3379	tx_ring->tx_buffer_info = NULL;
3380
3381	/* if not set, then don't free */
3382	if (!tx_ring->desc)
3383	return;
3384
3385	dma_free_coherent(dev: tx_ring->dev, size: tx_ring->size, cpu_addr: tx_ring->desc,
3386	dma_handle: tx_ring->dma);
3387
3388	tx_ring->desc = NULL;
3389	}
3390
3391	/**
3392	* ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
3393	* @adapter: board private structure
3394	*
3395	* Free all transmit software resources
3396	**/
3397	static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
3398	{
3399	int i;
3400
3401	for (i = 0; i < adapter->num_tx_queues; i++)
3402	if (adapter->tx_ring[i]->desc)
3403	ixgbevf_free_tx_resources(tx_ring: adapter->tx_ring[i]);
3404	for (i = 0; i < adapter->num_xdp_queues; i++)
3405	if (adapter->xdp_ring[i]->desc)
3406	ixgbevf_free_tx_resources(tx_ring: adapter->xdp_ring[i]);
3407	}
3408
3409	/**
3410	* ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
3411	* @tx_ring: Tx descriptor ring (for a specific queue) to setup
3412	*
3413	* Return 0 on success, negative on failure
3414	**/
3415	int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
3416	{
3417	struct ixgbevf_adapter *adapter = netdev_priv(dev: tx_ring->netdev);
3418	int size;
3419
3420	size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
3421	tx_ring->tx_buffer_info = vmalloc(size);
3422	if (!tx_ring->tx_buffer_info)
3423	goto err;
3424
3425	u64_stats_init(syncp: &tx_ring->syncp);
3426
3427	/* round up to nearest 4K */
3428	tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
3429	tx_ring->size = ALIGN(tx_ring->size, 4096);
3430
3431	tx_ring->desc = dma_alloc_coherent(dev: tx_ring->dev, size: tx_ring->size,
3432	dma_handle: &tx_ring->dma, GFP_KERNEL);
3433	if (!tx_ring->desc)
3434	goto err;
3435
3436	return 0;
3437
3438	err:
3439	vfree(addr: tx_ring->tx_buffer_info);
3440	tx_ring->tx_buffer_info = NULL;
3441	hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n");
3442	return -ENOMEM;
3443	}
3444
3445	/**
3446	* ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
3447	* @adapter: board private structure
3448	*
3449	* If this function returns with an error, then it's possible one or
3450	* more of the rings is populated (while the rest are not). It is the
3451	* callers duty to clean those orphaned rings.
3452	*
3453	* Return 0 on success, negative on failure
3454	**/
3455	static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
3456	{
3457	int i, j = 0, err = 0;
3458
3459	for (i = 0; i < adapter->num_tx_queues; i++) {
3460	err = ixgbevf_setup_tx_resources(tx_ring: adapter->tx_ring[i]);
3461	if (!err)
3462	continue;
3463	hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", i);
3464	goto err_setup_tx;
3465	}
3466
3467	for (j = 0; j < adapter->num_xdp_queues; j++) {
3468	err = ixgbevf_setup_tx_resources(tx_ring: adapter->xdp_ring[j]);
3469	if (!err)
3470	continue;
3471	hw_dbg(&adapter->hw, "Allocation for XDP Queue %u failed\n", j);
3472	goto err_setup_tx;
3473	}
3474
3475	return 0;
3476	err_setup_tx:
3477	/* rewind the index freeing the rings as we go */
3478	while (j--)
3479	ixgbevf_free_tx_resources(tx_ring: adapter->xdp_ring[j]);
3480	while (i--)
3481	ixgbevf_free_tx_resources(tx_ring: adapter->tx_ring[i]);
3482
3483	return err;
3484	}
3485
3486	/**
3487	* ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
3488	* @adapter: board private structure
3489	* @rx_ring: Rx descriptor ring (for a specific queue) to setup
3490	*
3491	* Returns 0 on success, negative on failure
3492	**/
3493	int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
3494	struct ixgbevf_ring *rx_ring)
3495	{
3496	int size;
3497
3498	size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
3499	rx_ring->rx_buffer_info = vmalloc(size);
3500	if (!rx_ring->rx_buffer_info)
3501	goto err;
3502
3503	u64_stats_init(syncp: &rx_ring->syncp);
3504
3505	/* Round up to nearest 4K */
3506	rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
3507	rx_ring->size = ALIGN(rx_ring->size, 4096);
3508
3509	rx_ring->desc = dma_alloc_coherent(dev: rx_ring->dev, size: rx_ring->size,
3510	dma_handle: &rx_ring->dma, GFP_KERNEL);
3511
3512	if (!rx_ring->desc)
3513	goto err;
3514
3515	/* XDP RX-queue info */
3516	if (xdp_rxq_info_reg(xdp_rxq: &rx_ring->xdp_rxq, dev: adapter->netdev,
3517	queue_index: rx_ring->queue_index, napi_id: 0) < 0)
3518	goto err;
3519
3520	rx_ring->xdp_prog = adapter->xdp_prog;
3521
3522	return 0;
3523	err:
3524	vfree(addr: rx_ring->rx_buffer_info);
3525	rx_ring->rx_buffer_info = NULL;
3526	dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
3527	return -ENOMEM;
3528	}
3529
3530	/**
3531	* ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
3532	* @adapter: board private structure
3533	*
3534	* If this function returns with an error, then it's possible one or
3535	* more of the rings is populated (while the rest are not). It is the
3536	* callers duty to clean those orphaned rings.
3537	*
3538	* Return 0 on success, negative on failure
3539	**/
3540	static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
3541	{
3542	int i, err = 0;
3543
3544	for (i = 0; i < adapter->num_rx_queues; i++) {
3545	err = ixgbevf_setup_rx_resources(adapter, rx_ring: adapter->rx_ring[i]);
3546	if (!err)
3547	continue;
3548	hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", i);
3549	goto err_setup_rx;
3550	}
3551
3552	return 0;
3553	err_setup_rx:
3554	/* rewind the index freeing the rings as we go */
3555	while (i--)
3556	ixgbevf_free_rx_resources(adapter->rx_ring[i]);
3557	return err;
3558	}
3559
3560	/**
3561	* ixgbevf_free_rx_resources - Free Rx Resources
3562	* @rx_ring: ring to clean the resources from
3563	*
3564	* Free all receive software resources
3565	**/
3566	void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
3567	{
3568	ixgbevf_clean_rx_ring(rx_ring);
3569
3570	rx_ring->xdp_prog = NULL;
3571	xdp_rxq_info_unreg(xdp_rxq: &rx_ring->xdp_rxq);
3572	vfree(addr: rx_ring->rx_buffer_info);
3573	rx_ring->rx_buffer_info = NULL;
3574
3575	dma_free_coherent(dev: rx_ring->dev, size: rx_ring->size, cpu_addr: rx_ring->desc,
3576	dma_handle: rx_ring->dma);
3577
3578	rx_ring->desc = NULL;
3579	}
3580
3581	/**
3582	* ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
3583	* @adapter: board private structure
3584	*
3585	* Free all receive software resources
3586	**/
3587	static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
3588	{
3589	int i;
3590
3591	for (i = 0; i < adapter->num_rx_queues; i++)
3592	if (adapter->rx_ring[i]->desc)
3593	ixgbevf_free_rx_resources(rx_ring: adapter->rx_ring[i]);
3594	}
3595
3596	/**
3597	* ixgbevf_open - Called when a network interface is made active
3598	* @netdev: network interface device structure
3599	*
3600	* Returns 0 on success, negative value on failure
3601	*
3602	* The open entry point is called when a network interface is made
3603	* active by the system (IFF_UP). At this point all resources needed
3604	* for transmit and receive operations are allocated, the interrupt
3605	* handler is registered with the OS, the watchdog timer is started,
3606	* and the stack is notified that the interface is ready.
3607	**/
3608	int ixgbevf_open(struct net_device *netdev)
3609	{
3610	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
3611	struct ixgbe_hw *hw = &adapter->hw;
3612	int err;
3613
3614	/* A previous failure to open the device because of a lack of
3615	* available MSIX vector resources may have reset the number
3616	* of msix vectors variable to zero. The only way to recover
3617	* is to unload/reload the driver and hope that the system has
3618	* been able to recover some MSIX vector resources.
3619	*/
3620	if (!adapter->num_msix_vectors)
3621	return -ENOMEM;
3622
3623	if (hw->adapter_stopped) {
3624	ixgbevf_reset(adapter);
3625	/* if adapter is still stopped then PF isn't up and
3626	* the VF can't start.
3627	*/
3628	if (hw->adapter_stopped) {
3629	err = IXGBE_ERR_MBX;
3630	pr_err("Unable to start - perhaps the PF Driver isn't up yet\n");
3631	goto err_setup_reset;
3632	}
3633	}
3634
3635	/* disallow open during test */
3636	if (test_bit(__IXGBEVF_TESTING, &adapter->state))
3637	return -EBUSY;
3638
3639	netif_carrier_off(dev: netdev);
3640
3641	/* allocate transmit descriptors */
3642	err = ixgbevf_setup_all_tx_resources(adapter);
3643	if (err)
3644	goto err_setup_tx;
3645
3646	/* allocate receive descriptors */
3647	err = ixgbevf_setup_all_rx_resources(adapter);
3648	if (err)
3649	goto err_setup_rx;
3650
3651	ixgbevf_configure(adapter);
3652
3653	err = ixgbevf_request_irq(adapter);
3654	if (err)
3655	goto err_req_irq;
3656
3657	/* Notify the stack of the actual queue counts. */
3658	err = netif_set_real_num_tx_queues(dev: netdev, txq: adapter->num_tx_queues);
3659	if (err)
3660	goto err_set_queues;
3661
3662	err = netif_set_real_num_rx_queues(dev: netdev, rxq: adapter->num_rx_queues);
3663	if (err)
3664	goto err_set_queues;
3665
3666	ixgbevf_up_complete(adapter);
3667
3668	return 0;
3669
3670	err_set_queues:
3671	ixgbevf_free_irq(adapter);
3672	err_req_irq:
3673	ixgbevf_free_all_rx_resources(adapter);
3674	err_setup_rx:
3675	ixgbevf_free_all_tx_resources(adapter);
3676	err_setup_tx:
3677	ixgbevf_reset(adapter);
3678	err_setup_reset:
3679
3680	return err;
3681	}
3682
3683	/**
3684	* ixgbevf_close_suspend - actions necessary to both suspend and close flows
3685	* @adapter: the private adapter struct
3686	*
3687	* This function should contain the necessary work common to both suspending
3688	* and closing of the device.
3689	*/
3690	static void ixgbevf_close_suspend(struct ixgbevf_adapter *adapter)
3691	{
3692	ixgbevf_down(adapter);
3693	ixgbevf_free_irq(adapter);
3694	ixgbevf_free_all_tx_resources(adapter);
3695	ixgbevf_free_all_rx_resources(adapter);
3696	}
3697
3698	/**
3699	* ixgbevf_close - Disables a network interface
3700	* @netdev: network interface device structure
3701	*
3702	* Returns 0, this is not allowed to fail
3703	*
3704	* The close entry point is called when an interface is de-activated
3705	* by the OS. The hardware is still under the drivers control, but
3706	* needs to be disabled. A global MAC reset is issued to stop the
3707	* hardware, and all transmit and receive resources are freed.
3708	**/
3709	int ixgbevf_close(struct net_device *netdev)
3710	{
3711	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
3712
3713	if (netif_device_present(dev: netdev))
3714	ixgbevf_close_suspend(adapter);
3715
3716	return 0;
3717	}
3718
3719	static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter)
3720	{
3721	struct net_device *dev = adapter->netdev;
3722
3723	if (!test_and_clear_bit(nr: __IXGBEVF_QUEUE_RESET_REQUESTED,
3724	addr: &adapter->state))
3725	return;
3726
3727	/* if interface is down do nothing */
3728	if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3729	test_bit(__IXGBEVF_RESETTING, &adapter->state))
3730	return;
3731
3732	/* Hardware has to reinitialize queues and interrupts to
3733	* match packet buffer alignment. Unfortunately, the
3734	* hardware is not flexible enough to do this dynamically.
3735	*/
3736	rtnl_lock();
3737
3738	if (netif_running(dev))
3739	ixgbevf_close(netdev: dev);
3740
3741	ixgbevf_clear_interrupt_scheme(adapter);
3742	ixgbevf_init_interrupt_scheme(adapter);
3743
3744	if (netif_running(dev))
3745	ixgbevf_open(netdev: dev);
3746
3747	rtnl_unlock();
3748	}
3749
3750	static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
3751	u32 vlan_macip_lens, u32 fceof_saidx,
3752	u32 type_tucmd, u32 mss_l4len_idx)
3753	{
3754	struct ixgbe_adv_tx_context_desc *context_desc;
3755	u16 i = tx_ring->next_to_use;
3756
3757	context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
3758
3759	i++;
3760	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
3761
3762	/* set bits to identify this as an advanced context descriptor */
3763	type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3764
3765	context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
3766	context_desc->fceof_saidx = cpu_to_le32(fceof_saidx);
3767	context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
3768	context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3769	}
3770
3771	static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
3772	struct ixgbevf_tx_buffer *first,
3773	u8 *hdr_len,
3774	struct ixgbevf_ipsec_tx_data *itd)
3775	{
3776	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
3777	struct sk_buff *skb = first->skb;
3778	union {
3779	struct iphdr *v4;
3780	struct ipv6hdr *v6;
3781	unsigned char *hdr;
3782	} ip;
3783	union {
3784	struct tcphdr *tcp;
3785	unsigned char *hdr;
3786	} l4;
3787	u32 paylen, l4_offset;
3788	u32 fceof_saidx = 0;
3789	int err;
3790
3791	if (skb->ip_summed != CHECKSUM_PARTIAL)
3792	return 0;
3793
3794	if (!skb_is_gso(skb))
3795	return 0;
3796
3797	err = skb_cow_head(skb, headroom: 0);
3798	if (err < 0)
3799	return err;
3800
3801	if (eth_p_mpls(eth_type: first->protocol))
3802	ip.hdr = skb_inner_network_header(skb);
3803	else
3804	ip.hdr = skb_network_header(skb);
3805	l4.hdr = skb_checksum_start(skb);
3806
3807	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3808	type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
3809
3810	/* initialize outer IP header fields */
3811	if (ip.v4->version == 4) {
3812	unsigned char *csum_start = skb_checksum_start(skb);
3813	unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
3814	int len = csum_start - trans_start;
3815
3816	/* IP header will have to cancel out any data that
3817	* is not a part of the outer IP header, so set to
3818	* a reverse csum if needed, else init check to 0.
3819	*/
3820	ip.v4->check = (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) ?
3821	csum_fold(sum: csum_partial(buff: trans_start,
3822	len, sum: 0)) : 0;
3823	type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
3824
3825	ip.v4->tot_len = 0;
3826	first->tx_flags |= IXGBE_TX_FLAGS_TSO |
3827	IXGBE_TX_FLAGS_CSUM |
3828	IXGBE_TX_FLAGS_IPV4;
3829	} else {
3830	ip.v6->payload_len = 0;
3831	first->tx_flags |= IXGBE_TX_FLAGS_TSO |
3832	IXGBE_TX_FLAGS_CSUM;
3833	}
3834
3835	/* determine offset of inner transport header */
3836	l4_offset = l4.hdr - skb->data;
3837
3838	/* compute length of segmentation header */
3839	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
3840
3841	/* remove payload length from inner checksum */
3842	paylen = skb->len - l4_offset;
3843	csum_replace_by_diff(sum: &l4.tcp->check, diff: (__force __wsum)htonl(paylen));
3844
3845	/* update gso size and bytecount with header size */
3846	first->gso_segs = skb_shinfo(skb)->gso_segs;
3847	first->bytecount += (first->gso_segs - 1) * *hdr_len;
3848
3849	/* mss_l4len_id: use 1 as index for TSO */
3850	mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT;
3851	mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
3852	mss_l4len_idx |= (1u << IXGBE_ADVTXD_IDX_SHIFT);
3853
3854	fceof_saidx |= itd->pfsa;
3855	type_tucmd |= itd->flags | itd->trailer_len;
3856
3857	/* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
3858	vlan_macip_lens = l4.hdr - ip.hdr;
3859	vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT;
3860	vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
3861
3862	ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd,
3863	mss_l4len_idx);
3864
3865	return 1;
3866	}
3867
3868	static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
3869	struct ixgbevf_tx_buffer *first,
3870	struct ixgbevf_ipsec_tx_data *itd)
3871	{
3872	struct sk_buff *skb = first->skb;
3873	u32 vlan_macip_lens = 0;
3874	u32 fceof_saidx = 0;
3875	u32 type_tucmd = 0;
3876
3877	if (skb->ip_summed != CHECKSUM_PARTIAL)
3878	goto no_csum;
3879
3880	switch (skb->csum_offset) {
3881	case offsetof(struct tcphdr, check):
3882	type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
3883	fallthrough;
3884	case offsetof(struct udphdr, check):
3885	break;
3886	case offsetof(struct sctphdr, checksum):
3887	/* validate that this is actually an SCTP request */
3888	if (skb_csum_is_sctp(skb)) {
3889	type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3890	break;
3891	}
3892	fallthrough;
3893	default:
3894	skb_checksum_help(skb);
3895	goto no_csum;
3896	}
3897
3898	if (first->protocol == htons(ETH_P_IP))
3899	type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
3900
3901	/* update TX checksum flag */
3902	first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
3903	vlan_macip_lens = skb_checksum_start_offset(skb) -
3904	skb_network_offset(skb);
3905	no_csum:
3906	/* vlan_macip_lens: MACLEN, VLAN tag */
3907	vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
3908	vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
3909
3910	fceof_saidx |= itd->pfsa;
3911	type_tucmd |= itd->flags | itd->trailer_len;
3912
3913	ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
3914	fceof_saidx, type_tucmd, mss_l4len_idx: 0);
3915	}
3916
3917	static __le32 ixgbevf_tx_cmd_type(u32 tx_flags)
3918	{
3919	/* set type for advanced descriptor with frame checksum insertion */
3920	__le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
3921	IXGBE_ADVTXD_DCMD_IFCS |
3922	IXGBE_ADVTXD_DCMD_DEXT);
3923
3924	/* set HW VLAN bit if VLAN is present */
3925	if (tx_flags & IXGBE_TX_FLAGS_VLAN)
3926	cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
3927
3928	/* set segmentation enable bits for TSO/FSO */
3929	if (tx_flags & IXGBE_TX_FLAGS_TSO)
3930	cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
3931
3932	return cmd_type;
3933	}
3934
3935	static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
3936	u32 tx_flags, unsigned int paylen)
3937	{
3938	__le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
3939
3940	/* enable L4 checksum for TSO and TX checksum offload */
3941	if (tx_flags & IXGBE_TX_FLAGS_CSUM)
3942	olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
3943
3944	/* enble IPv4 checksum for TSO */
3945	if (tx_flags & IXGBE_TX_FLAGS_IPV4)
3946	olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
3947
3948	/* enable IPsec */
3949	if (tx_flags & IXGBE_TX_FLAGS_IPSEC)
3950	olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IPSEC);
3951
3952	/* use index 1 context for TSO/FSO/FCOE/IPSEC */
3953	if (tx_flags & (IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_IPSEC))
3954	olinfo_status |= cpu_to_le32(1u << IXGBE_ADVTXD_IDX_SHIFT);
3955
3956	/* Check Context must be set if Tx switch is enabled, which it
3957	* always is for case where virtual functions are running
3958	*/
3959	olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
3960
3961	tx_desc->read.olinfo_status = olinfo_status;
3962	}
3963
3964	static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
3965	struct ixgbevf_tx_buffer *first,
3966	const u8 hdr_len)
3967	{
3968	struct sk_buff *skb = first->skb;
3969	struct ixgbevf_tx_buffer *tx_buffer;
3970	union ixgbe_adv_tx_desc *tx_desc;
3971	skb_frag_t *frag;
3972	dma_addr_t dma;
3973	unsigned int data_len, size;
3974	u32 tx_flags = first->tx_flags;
3975	__le32 cmd_type = ixgbevf_tx_cmd_type(tx_flags);
3976	u16 i = tx_ring->next_to_use;
3977
3978	tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
3979
3980	ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen: skb->len - hdr_len);
3981
3982	size = skb_headlen(skb);
3983	data_len = skb->data_len;
3984
3985	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
3986
3987	tx_buffer = first;
3988
3989	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
3990	if (dma_mapping_error(dev: tx_ring->dev, dma_addr: dma))
3991	goto dma_error;
3992
3993	/* record length, and DMA address */
3994	dma_unmap_len_set(tx_buffer, len, size);
3995	dma_unmap_addr_set(tx_buffer, dma, dma);
3996
3997	tx_desc->read.buffer_addr = cpu_to_le64(dma);
3998
3999	while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
4000	tx_desc->read.cmd_type_len =
4001	cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
4002
4003	i++;
4004	tx_desc++;
4005	if (i == tx_ring->count) {
4006	tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
4007	i = 0;
4008	}
4009	tx_desc->read.olinfo_status = 0;
4010
4011	dma += IXGBE_MAX_DATA_PER_TXD;
4012	size -= IXGBE_MAX_DATA_PER_TXD;
4013
4014	tx_desc->read.buffer_addr = cpu_to_le64(dma);
4015	}
4016
4017	if (likely(!data_len))
4018	break;
4019
4020	tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4021
4022	i++;
4023	tx_desc++;
4024	if (i == tx_ring->count) {
4025	tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
4026	i = 0;
4027	}
4028	tx_desc->read.olinfo_status = 0;
4029
4030	size = skb_frag_size(frag);
4031	data_len -= size;
4032
4033	dma = skb_frag_dma_map(dev: tx_ring->dev, frag, offset: 0, size,
4034	dir: DMA_TO_DEVICE);
4035
4036	tx_buffer = &tx_ring->tx_buffer_info[i];
4037	}
4038
4039	/* write last descriptor with RS and EOP bits */
4040	cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
4041	tx_desc->read.cmd_type_len = cmd_type;
4042
4043	/* set the timestamp */
4044	first->time_stamp = jiffies;
4045
4046	skb_tx_timestamp(skb);
4047
4048	/* Force memory writes to complete before letting h/w know there
4049	* are new descriptors to fetch. (Only applicable for weak-ordered
4050	* memory model archs, such as IA-64).
4051	*
4052	* We also need this memory barrier (wmb) to make certain all of the
4053	* status bits have been updated before next_to_watch is written.
4054	*/
4055	wmb();
4056
4057	/* set next_to_watch value indicating a packet is present */
4058	first->next_to_watch = tx_desc;
4059
4060	i++;
4061	if (i == tx_ring->count)
4062	i = 0;
4063
4064	tx_ring->next_to_use = i;
4065
4066	/* notify HW of packet */
4067	ixgbevf_write_tail(ring: tx_ring, value: i);
4068
4069	return;
4070	dma_error:
4071	dev_err(tx_ring->dev, "TX DMA map failed\n");
4072	tx_buffer = &tx_ring->tx_buffer_info[i];
4073
4074	/* clear dma mappings for failed tx_buffer_info map */
4075	while (tx_buffer != first) {
4076	if (dma_unmap_len(tx_buffer, len))
4077	dma_unmap_page(tx_ring->dev,
4078	dma_unmap_addr(tx_buffer, dma),
4079	dma_unmap_len(tx_buffer, len),
4080	DMA_TO_DEVICE);
4081	dma_unmap_len_set(tx_buffer, len, 0);
4082
4083	if (i-- == 0)
4084	i += tx_ring->count;
4085	tx_buffer = &tx_ring->tx_buffer_info[i];
4086	}
4087
4088	if (dma_unmap_len(tx_buffer, len))
4089	dma_unmap_single(tx_ring->dev,
4090	dma_unmap_addr(tx_buffer, dma),
4091	dma_unmap_len(tx_buffer, len),
4092	DMA_TO_DEVICE);
4093	dma_unmap_len_set(tx_buffer, len, 0);
4094
4095	dev_kfree_skb_any(skb: tx_buffer->skb);
4096	tx_buffer->skb = NULL;
4097
4098	tx_ring->next_to_use = i;
4099	}
4100
4101	static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
4102	{
4103	netif_stop_subqueue(dev: tx_ring->netdev, queue_index: tx_ring->queue_index);
4104	/* Herbert's original patch had:
4105	* smp_mb__after_netif_stop_queue();
4106	* but since that doesn't exist yet, just open code it.
4107	*/
4108	smp_mb();
4109
4110	/* We need to check again in a case another CPU has just
4111	* made room available.
4112	*/
4113	if (likely(ixgbevf_desc_unused(tx_ring) < size))
4114	return -EBUSY;
4115
4116	/* A reprieve! - use start_queue because it doesn't call schedule */
4117	netif_start_subqueue(dev: tx_ring->netdev, queue_index: tx_ring->queue_index);
4118	++tx_ring->tx_stats.restart_queue;
4119
4120	return 0;
4121	}
4122
4123	static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
4124	{
4125	if (likely(ixgbevf_desc_unused(tx_ring) >= size))
4126	return 0;
4127	return __ixgbevf_maybe_stop_tx(tx_ring, size);
4128	}
4129
4130	static int ixgbevf_xmit_frame_ring(struct sk_buff *skb,
4131	struct ixgbevf_ring *tx_ring)
4132	{
4133	struct ixgbevf_tx_buffer *first;
4134	int tso;
4135	u32 tx_flags = 0;
4136	u16 count = TXD_USE_COUNT(skb_headlen(skb));
4137	struct ixgbevf_ipsec_tx_data ipsec_tx = { 0 };
4138	#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
4139	unsigned short f;
4140	#endif
4141	u8 hdr_len = 0;
4142	u8 *dst_mac = skb_header_pointer(skb, offset: 0, len: 0, NULL);
4143
4144	if (!dst_mac || is_link_local_ether_addr(addr: dst_mac)) {
4145	dev_kfree_skb_any(skb);
4146	return NETDEV_TX_OK;
4147	}
4148
4149	/* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
4150	* + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
4151	* + 2 desc gap to keep tail from touching head,
4152	* + 1 desc for context descriptor,
4153	* otherwise try next time
4154	*/
4155	#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
4156	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
4157	skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
4158
4159	count += TXD_USE_COUNT(skb_frag_size(frag));
4160	}
4161	#else
4162	count += skb_shinfo(skb)->nr_frags;
4163	#endif
4164	if (ixgbevf_maybe_stop_tx(tx_ring, size: count + 3)) {
4165	tx_ring->tx_stats.tx_busy++;
4166	return NETDEV_TX_BUSY;
4167	}
4168
4169	/* record the location of the first descriptor for this packet */
4170	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4171	first->skb = skb;
4172	first->bytecount = skb->len;
4173	first->gso_segs = 1;
4174
4175	if (skb_vlan_tag_present(skb)) {
4176	tx_flags |= skb_vlan_tag_get(skb);
4177	tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
4178	tx_flags |= IXGBE_TX_FLAGS_VLAN;
4179	}
4180
4181	/* record initial flags and protocol */
4182	first->tx_flags = tx_flags;
4183	first->protocol = vlan_get_protocol(skb);
4184
4185	#ifdef CONFIG_IXGBEVF_IPSEC
4186	if (xfrm_offload(skb) && !ixgbevf_ipsec_tx(tx_ring, first, itd: &ipsec_tx))
4187	goto out_drop;
4188	#endif
4189	tso = ixgbevf_tso(tx_ring, first, hdr_len: &hdr_len, itd: &ipsec_tx);
4190	if (tso < 0)
4191	goto out_drop;
4192	else if (!tso)
4193	ixgbevf_tx_csum(tx_ring, first, itd: &ipsec_tx);
4194
4195	ixgbevf_tx_map(tx_ring, first, hdr_len);
4196
4197	ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
4198
4199	return NETDEV_TX_OK;
4200
4201	out_drop:
4202	dev_kfree_skb_any(skb: first->skb);
4203	first->skb = NULL;
4204
4205	return NETDEV_TX_OK;
4206	}
4207
4208	static netdev_tx_t ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
4209	{
4210	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
4211	struct ixgbevf_ring *tx_ring;
4212
4213	if (skb->len <= 0) {
4214	dev_kfree_skb_any(skb);
4215	return NETDEV_TX_OK;
4216	}
4217
4218	/* The minimum packet size for olinfo paylen is 17 so pad the skb
4219	* in order to meet this minimum size requirement.
4220	*/
4221	if (skb->len < 17) {
4222	if (skb_padto(skb, len: 17))
4223	return NETDEV_TX_OK;
4224	skb->len = 17;
4225	}
4226
4227	tx_ring = adapter->tx_ring[skb->queue_mapping];
4228	return ixgbevf_xmit_frame_ring(skb, tx_ring);
4229	}
4230
4231	/**
4232	* ixgbevf_set_mac - Change the Ethernet Address of the NIC
4233	* @netdev: network interface device structure
4234	* @p: pointer to an address structure
4235	*
4236	* Returns 0 on success, negative on failure
4237	**/
4238	static int ixgbevf_set_mac(struct net_device *netdev, void *p)
4239	{
4240	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
4241	struct ixgbe_hw *hw = &adapter->hw;
4242	struct sockaddr *addr = p;
4243	int err;
4244
4245	if (!is_valid_ether_addr(addr: addr->sa_data))
4246	return -EADDRNOTAVAIL;
4247
4248	spin_lock_bh(lock: &adapter->mbx_lock);
4249
4250	err = hw->mac.ops.set_rar(hw, 0, addr->sa_data, 0);
4251
4252	spin_unlock_bh(lock: &adapter->mbx_lock);
4253
4254	if (err)
4255	return -EPERM;
4256
4257	ether_addr_copy(dst: hw->mac.addr, src: addr->sa_data);
4258	ether_addr_copy(dst: hw->mac.perm_addr, src: addr->sa_data);
4259	eth_hw_addr_set(dev: netdev, addr: addr->sa_data);
4260
4261	return 0;
4262	}
4263
4264	/**
4265	* ixgbevf_change_mtu - Change the Maximum Transfer Unit
4266	* @netdev: network interface device structure
4267	* @new_mtu: new value for maximum frame size
4268	*
4269	* Returns 0 on success, negative on failure
4270	**/
4271	static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
4272	{
4273	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
4274	struct ixgbe_hw *hw = &adapter->hw;
4275	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4276	int ret;
4277
4278	/* prevent MTU being changed to a size unsupported by XDP */
4279	if (adapter->xdp_prog) {
4280	dev_warn(&adapter->pdev->dev, "MTU cannot be changed while XDP program is loaded\n");
4281	return -EPERM;
4282	}
4283
4284	spin_lock_bh(lock: &adapter->mbx_lock);
4285	/* notify the PF of our intent to use this size of frame */
4286	ret = hw->mac.ops.set_rlpml(hw, max_frame);
4287	spin_unlock_bh(lock: &adapter->mbx_lock);
4288	if (ret)
4289	return -EINVAL;
4290
4291	hw_dbg(hw, "changing MTU from %d to %d\n",
4292	netdev->mtu, new_mtu);
4293
4294	/* must set new MTU before calling down or up */
4295	netdev->mtu = new_mtu;
4296
4297	if (netif_running(dev: netdev))
4298	ixgbevf_reinit_locked(adapter);
4299
4300	return 0;
4301	}
4302
4303	static int __maybe_unused ixgbevf_suspend(struct device *dev_d)
4304	{
4305	struct net_device *netdev = dev_get_drvdata(dev: dev_d);
4306	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
4307
4308	rtnl_lock();
4309	netif_device_detach(dev: netdev);
4310
4311	if (netif_running(dev: netdev))
4312	ixgbevf_close_suspend(adapter);
4313
4314	ixgbevf_clear_interrupt_scheme(adapter);
4315	rtnl_unlock();
4316
4317	return 0;
4318	}
4319
4320	static int __maybe_unused ixgbevf_resume(struct device *dev_d)
4321	{
4322	struct pci_dev *pdev = to_pci_dev(dev_d);
4323	struct net_device *netdev = pci_get_drvdata(pdev);
4324	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
4325	u32 err;
4326
4327	adapter->hw.hw_addr = adapter->io_addr;
4328	smp_mb__before_atomic();
4329	clear_bit(nr: __IXGBEVF_DISABLED, addr: &adapter->state);
4330	pci_set_master(dev: pdev);
4331
4332	ixgbevf_reset(adapter);
4333
4334	rtnl_lock();
4335	err = ixgbevf_init_interrupt_scheme(adapter);
4336	if (!err && netif_running(dev: netdev))
4337	err = ixgbevf_open(netdev);
4338	rtnl_unlock();
4339	if (err)
4340	return err;
4341
4342	netif_device_attach(dev: netdev);
4343
4344	return err;
4345	}
4346
4347	static void ixgbevf_shutdown(struct pci_dev *pdev)
4348	{
4349	ixgbevf_suspend(dev_d: &pdev->dev);
4350	}
4351
4352	static void ixgbevf_get_tx_ring_stats(struct rtnl_link_stats64 *stats,
4353	const struct ixgbevf_ring *ring)
4354	{
4355	u64 bytes, packets;
4356	unsigned int start;
4357
4358	if (ring) {
4359	do {
4360	start = u64_stats_fetch_begin(syncp: &ring->syncp);
4361	bytes = ring->stats.bytes;
4362	packets = ring->stats.packets;
4363	} while (u64_stats_fetch_retry(syncp: &ring->syncp, start));
4364	stats->tx_bytes += bytes;
4365	stats->tx_packets += packets;
4366	}
4367	}
4368
4369	static void ixgbevf_get_stats(struct net_device *netdev,
4370	struct rtnl_link_stats64 *stats)
4371	{
4372	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
4373	unsigned int start;
4374	u64 bytes, packets;
4375	const struct ixgbevf_ring *ring;
4376	int i;
4377
4378	ixgbevf_update_stats(adapter);
4379
4380	stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
4381
4382	rcu_read_lock();
4383	for (i = 0; i < adapter->num_rx_queues; i++) {
4384	ring = adapter->rx_ring[i];
4385	do {
4386	start = u64_stats_fetch_begin(syncp: &ring->syncp);
4387	bytes = ring->stats.bytes;
4388	packets = ring->stats.packets;
4389	} while (u64_stats_fetch_retry(syncp: &ring->syncp, start));
4390	stats->rx_bytes += bytes;
4391	stats->rx_packets += packets;
4392	}
4393
4394	for (i = 0; i < adapter->num_tx_queues; i++) {
4395	ring = adapter->tx_ring[i];
4396	ixgbevf_get_tx_ring_stats(stats, ring);
4397	}
4398
4399	for (i = 0; i < adapter->num_xdp_queues; i++) {
4400	ring = adapter->xdp_ring[i];
4401	ixgbevf_get_tx_ring_stats(stats, ring);
4402	}
4403	rcu_read_unlock();
4404	}
4405
4406	#define IXGBEVF_MAX_MAC_HDR_LEN 127
4407	#define IXGBEVF_MAX_NETWORK_HDR_LEN 511
4408
4409	static netdev_features_t
4410	ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev,
4411	netdev_features_t features)
4412	{
4413	unsigned int network_hdr_len, mac_hdr_len;
4414
4415	/* Make certain the headers can be described by a context descriptor */
4416	mac_hdr_len = skb_network_header(skb) - skb->data;
4417	if (unlikely(mac_hdr_len > IXGBEVF_MAX_MAC_HDR_LEN))
4418	return features & ~(NETIF_F_HW_CSUM |
4419	NETIF_F_SCTP_CRC |
4420	NETIF_F_HW_VLAN_CTAG_TX |
4421	NETIF_F_TSO |
4422	NETIF_F_TSO6);
4423
4424	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
4425	if (unlikely(network_hdr_len > IXGBEVF_MAX_NETWORK_HDR_LEN))
4426	return features & ~(NETIF_F_HW_CSUM |
4427	NETIF_F_SCTP_CRC |
4428	NETIF_F_TSO |
4429	NETIF_F_TSO6);
4430
4431	/* We can only support IPV4 TSO in tunnels if we can mangle the
4432	* inner IP ID field, so strip TSO if MANGLEID is not supported.
4433	*/
4434	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
4435	features &= ~NETIF_F_TSO;
4436
4437	return features;
4438	}
4439
4440	static int ixgbevf_xdp_setup(struct net_device *dev, struct bpf_prog *prog)
4441	{
4442	int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4443	struct ixgbevf_adapter *adapter = netdev_priv(dev);
4444	struct bpf_prog *old_prog;
4445
4446	/* verify ixgbevf ring attributes are sufficient for XDP */
4447	for (i = 0; i < adapter->num_rx_queues; i++) {
4448	struct ixgbevf_ring *ring = adapter->rx_ring[i];
4449
4450	if (frame_size > ixgbevf_rx_bufsz(ring))
4451	return -EINVAL;
4452	}
4453
4454	old_prog = xchg(&adapter->xdp_prog, prog);
4455
4456	/* If transitioning XDP modes reconfigure rings */
4457	if (!!prog != !!old_prog) {
4458	/* Hardware has to reinitialize queues and interrupts to
4459	* match packet buffer alignment. Unfortunately, the
4460	* hardware is not flexible enough to do this dynamically.
4461	*/
4462	if (netif_running(dev))
4463	ixgbevf_close(netdev: dev);
4464
4465	ixgbevf_clear_interrupt_scheme(adapter);
4466	ixgbevf_init_interrupt_scheme(adapter);
4467
4468	if (netif_running(dev))
4469	ixgbevf_open(netdev: dev);
4470	} else {
4471	for (i = 0; i < adapter->num_rx_queues; i++)
4472	xchg(&adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog);
4473	}
4474
4475	if (old_prog)
4476	bpf_prog_put(prog: old_prog);
4477
4478	return 0;
4479	}
4480
4481	static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp)
4482	{
4483	switch (xdp->command) {
4484	case XDP_SETUP_PROG:
4485	return ixgbevf_xdp_setup(dev, prog: xdp->prog);
4486	default:
4487	return -EINVAL;
4488	}
4489	}
4490
4491	static const struct net_device_ops ixgbevf_netdev_ops = {
4492	.ndo_open = ixgbevf_open,
4493	.ndo_stop = ixgbevf_close,
4494	.ndo_start_xmit = ixgbevf_xmit_frame,
4495	.ndo_set_rx_mode = ixgbevf_set_rx_mode,
4496	.ndo_get_stats64 = ixgbevf_get_stats,
4497	.ndo_validate_addr = eth_validate_addr,
4498	.ndo_set_mac_address = ixgbevf_set_mac,
4499	.ndo_change_mtu = ixgbevf_change_mtu,
4500	.ndo_tx_timeout = ixgbevf_tx_timeout,
4501	.ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid,
4502	.ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid,
4503	.ndo_features_check = ixgbevf_features_check,
4504	.ndo_bpf = ixgbevf_xdp,
4505	};
4506
4507	static void ixgbevf_assign_netdev_ops(struct net_device *dev)
4508	{
4509	dev->netdev_ops = &ixgbevf_netdev_ops;
4510	ixgbevf_set_ethtool_ops(netdev: dev);
4511	dev->watchdog_timeo = 5 * HZ;
4512	}
4513
4514	/**
4515	* ixgbevf_probe - Device Initialization Routine
4516	* @pdev: PCI device information struct
4517	* @ent: entry in ixgbevf_pci_tbl
4518	*
4519	* Returns 0 on success, negative on failure
4520	*
4521	* ixgbevf_probe initializes an adapter identified by a pci_dev structure.
4522	* The OS initialization, configuring of the adapter private structure,
4523	* and a hardware reset occur.
4524	**/
4525	static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4526	{
4527	struct net_device *netdev;
4528	struct ixgbevf_adapter *adapter = NULL;
4529	struct ixgbe_hw *hw = NULL;
4530	const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
4531	bool disable_dev = false;
4532	int err;
4533
4534	err = pci_enable_device(dev: pdev);
4535	if (err)
4536	return err;
4537
4538	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
4539	if (err) {
4540	dev_err(&pdev->dev, "No usable DMA configuration, aborting\n");
4541	goto err_dma;
4542	}
4543
4544	err = pci_request_regions(pdev, ixgbevf_driver_name);
4545	if (err) {
4546	dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
4547	goto err_pci_reg;
4548	}
4549
4550	pci_set_master(dev: pdev);
4551
4552	netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
4553	MAX_TX_QUEUES);
4554	if (!netdev) {
4555	err = -ENOMEM;
4556	goto err_alloc_etherdev;
4557	}
4558
4559	SET_NETDEV_DEV(netdev, &pdev->dev);
4560
4561	adapter = netdev_priv(dev: netdev);
4562
4563	adapter->netdev = netdev;
4564	adapter->pdev = pdev;
4565	hw = &adapter->hw;
4566	hw->back = adapter;
4567	adapter->msg_enable = netif_msg_init(debug_value: debug, DEFAULT_MSG_ENABLE);
4568
4569	/* call save state here in standalone driver because it relies on
4570	* adapter struct to exist, and needs to call netdev_priv
4571	*/
4572	pci_save_state(dev: pdev);
4573
4574	hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
4575	pci_resource_len(pdev, 0));
4576	adapter->io_addr = hw->hw_addr;
4577	if (!hw->hw_addr) {
4578	err = -EIO;
4579	goto err_ioremap;
4580	}
4581
4582	ixgbevf_assign_netdev_ops(dev: netdev);
4583
4584	/* Setup HW API */
4585	memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
4586	hw->mac.type = ii->mac;
4587
4588	memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops_legacy,
4589	sizeof(struct ixgbe_mbx_operations));
4590
4591	/* setup the private structure */
4592	err = ixgbevf_sw_init(adapter);
4593	if (err)
4594	goto err_sw_init;
4595
4596	/* The HW MAC address was set and/or determined in sw_init */
4597	if (!is_valid_ether_addr(addr: netdev->dev_addr)) {
4598	pr_err("invalid MAC address\n");
4599	err = -EIO;
4600	goto err_sw_init;
4601	}
4602
4603	netdev->hw_features = NETIF_F_SG |
4604	NETIF_F_TSO |
4605	NETIF_F_TSO6 |
4606	NETIF_F_RXCSUM |
4607	NETIF_F_HW_CSUM |
4608	NETIF_F_SCTP_CRC;
4609
4610	#define IXGBEVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
4611	NETIF_F_GSO_GRE_CSUM | \
4612	NETIF_F_GSO_IPXIP4 | \
4613	NETIF_F_GSO_IPXIP6 | \
4614	NETIF_F_GSO_UDP_TUNNEL | \
4615	NETIF_F_GSO_UDP_TUNNEL_CSUM)
4616
4617	netdev->gso_partial_features = IXGBEVF_GSO_PARTIAL_FEATURES;
4618	netdev->hw_features |= NETIF_F_GSO_PARTIAL |
4619	IXGBEVF_GSO_PARTIAL_FEATURES;
4620
4621	netdev->features = netdev->hw_features | NETIF_F_HIGHDMA;
4622
4623	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
4624	netdev->mpls_features |= NETIF_F_SG |
4625	NETIF_F_TSO |
4626	NETIF_F_TSO6 |
4627	NETIF_F_HW_CSUM;
4628	netdev->mpls_features |= IXGBEVF_GSO_PARTIAL_FEATURES;
4629	netdev->hw_enc_features |= netdev->vlan_features;
4630
4631	/* set this bit last since it cannot be part of vlan_features */
4632	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
4633	NETIF_F_HW_VLAN_CTAG_RX |
4634	NETIF_F_HW_VLAN_CTAG_TX;
4635
4636	netdev->priv_flags |= IFF_UNICAST_FLT;
4637	netdev->xdp_features = NETDEV_XDP_ACT_BASIC;
4638
4639	/* MTU range: 68 - 1504 or 9710 */
4640	netdev->min_mtu = ETH_MIN_MTU;
4641	switch (adapter->hw.api_version) {
4642	case ixgbe_mbox_api_11:
4643	case ixgbe_mbox_api_12:
4644	case ixgbe_mbox_api_13:
4645	case ixgbe_mbox_api_14:
4646	case ixgbe_mbox_api_15:
4647	netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE -
4648	(ETH_HLEN + ETH_FCS_LEN);
4649	break;
4650	default:
4651	if (adapter->hw.mac.type != ixgbe_mac_82599_vf)
4652	netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE -
4653	(ETH_HLEN + ETH_FCS_LEN);
4654	else
4655	netdev->max_mtu = ETH_DATA_LEN + ETH_FCS_LEN;
4656	break;
4657	}
4658
4659	if (IXGBE_REMOVED(hw->hw_addr)) {
4660	err = -EIO;
4661	goto err_sw_init;
4662	}
4663
4664	timer_setup(&adapter->service_timer, ixgbevf_service_timer, 0);
4665
4666	INIT_WORK(&adapter->service_task, ixgbevf_service_task);
4667	set_bit(nr: __IXGBEVF_SERVICE_INITED, addr: &adapter->state);
4668	clear_bit(nr: __IXGBEVF_SERVICE_SCHED, addr: &adapter->state);
4669
4670	err = ixgbevf_init_interrupt_scheme(adapter);
4671	if (err)
4672	goto err_sw_init;
4673
4674	strcpy(p: netdev->name, q: "eth%d");
4675
4676	err = register_netdev(dev: netdev);
4677	if (err)
4678	goto err_register;
4679
4680	pci_set_drvdata(pdev, data: netdev);
4681	netif_carrier_off(dev: netdev);
4682	ixgbevf_init_ipsec_offload(adapter);
4683
4684	ixgbevf_init_last_counter_stats(adapter);
4685
4686	/* print the VF info */
4687	dev_info(&pdev->dev, "%pM\n", netdev->dev_addr);
4688	dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type);
4689
4690	switch (hw->mac.type) {
4691	case ixgbe_mac_X550_vf:
4692	dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n");
4693	break;
4694	case ixgbe_mac_X540_vf:
4695	dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n");
4696	break;
4697	case ixgbe_mac_82599_vf:
4698	default:
4699	dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n");
4700	break;
4701	}
4702
4703	return 0;
4704
4705	err_register:
4706	ixgbevf_clear_interrupt_scheme(adapter);
4707	err_sw_init:
4708	ixgbevf_reset_interrupt_capability(adapter);
4709	iounmap(addr: adapter->io_addr);
4710	kfree(objp: adapter->rss_key);
4711	err_ioremap:
4712	disable_dev = !test_and_set_bit(nr: __IXGBEVF_DISABLED, addr: &adapter->state);
4713	free_netdev(dev: netdev);
4714	err_alloc_etherdev:
4715	pci_release_regions(pdev);
4716	err_pci_reg:
4717	err_dma:
4718	if (!adapter || disable_dev)
4719	pci_disable_device(dev: pdev);
4720	return err;
4721	}
4722
4723	/**
4724	* ixgbevf_remove - Device Removal Routine
4725	* @pdev: PCI device information struct
4726	*
4727	* ixgbevf_remove is called by the PCI subsystem to alert the driver
4728	* that it should release a PCI device. The could be caused by a
4729	* Hot-Plug event, or because the driver is going to be removed from
4730	* memory.
4731	**/
4732	static void ixgbevf_remove(struct pci_dev *pdev)
4733	{
4734	struct net_device *netdev = pci_get_drvdata(pdev);
4735	struct ixgbevf_adapter *adapter;
4736	bool disable_dev;
4737
4738	if (!netdev)
4739	return;
4740
4741	adapter = netdev_priv(dev: netdev);
4742
4743	set_bit(nr: __IXGBEVF_REMOVING, addr: &adapter->state);
4744	cancel_work_sync(work: &adapter->service_task);
4745
4746	if (netdev->reg_state == NETREG_REGISTERED)
4747	unregister_netdev(dev: netdev);
4748
4749	ixgbevf_stop_ipsec_offload(adapter);
4750	ixgbevf_clear_interrupt_scheme(adapter);
4751	ixgbevf_reset_interrupt_capability(adapter);
4752
4753	iounmap(addr: adapter->io_addr);
4754	pci_release_regions(pdev);
4755
4756	hw_dbg(&adapter->hw, "Remove complete\n");
4757
4758	kfree(objp: adapter->rss_key);
4759	disable_dev = !test_and_set_bit(nr: __IXGBEVF_DISABLED, addr: &adapter->state);
4760	free_netdev(dev: netdev);
4761
4762	if (disable_dev)
4763	pci_disable_device(dev: pdev);
4764	}
4765
4766	/**
4767	* ixgbevf_io_error_detected - called when PCI error is detected
4768	* @pdev: Pointer to PCI device
4769	* @state: The current pci connection state
4770	*
4771	* This function is called after a PCI bus error affecting
4772	* this device has been detected.
4773	**/
4774	static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
4775	pci_channel_state_t state)
4776	{
4777	struct net_device *netdev = pci_get_drvdata(pdev);
4778	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
4779
4780	if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state))
4781	return PCI_ERS_RESULT_DISCONNECT;
4782
4783	rtnl_lock();
4784	netif_device_detach(dev: netdev);
4785
4786	if (netif_running(dev: netdev))
4787	ixgbevf_close_suspend(adapter);
4788
4789	if (state == pci_channel_io_perm_failure) {
4790	rtnl_unlock();
4791	return PCI_ERS_RESULT_DISCONNECT;
4792	}
4793
4794	if (!test_and_set_bit(nr: __IXGBEVF_DISABLED, addr: &adapter->state))
4795	pci_disable_device(dev: pdev);
4796	rtnl_unlock();
4797
4798	/* Request a slot reset. */
4799	return PCI_ERS_RESULT_NEED_RESET;
4800	}
4801
4802	/**
4803	* ixgbevf_io_slot_reset - called after the pci bus has been reset.
4804	* @pdev: Pointer to PCI device
4805	*
4806	* Restart the card from scratch, as if from a cold-boot. Implementation
4807	* resembles the first-half of the ixgbevf_resume routine.
4808	**/
4809	static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
4810	{
4811	struct net_device *netdev = pci_get_drvdata(pdev);
4812	struct ixgbevf_adapter *adapter = netdev_priv(dev: netdev);
4813
4814	if (pci_enable_device_mem(dev: pdev)) {
4815	dev_err(&pdev->dev,
4816	"Cannot re-enable PCI device after reset.\n");
4817	return PCI_ERS_RESULT_DISCONNECT;
4818	}
4819
4820	adapter->hw.hw_addr = adapter->io_addr;
4821	smp_mb__before_atomic();
4822	clear_bit(nr: __IXGBEVF_DISABLED, addr: &adapter->state);
4823	pci_set_master(dev: pdev);
4824
4825	ixgbevf_reset(adapter);
4826
4827	return PCI_ERS_RESULT_RECOVERED;
4828	}
4829
4830	/**
4831	* ixgbevf_io_resume - called when traffic can start flowing again.
4832	* @pdev: Pointer to PCI device
4833	*
4834	* This callback is called when the error recovery driver tells us that
4835	* its OK to resume normal operation. Implementation resembles the
4836	* second-half of the ixgbevf_resume routine.
4837	**/
4838	static void ixgbevf_io_resume(struct pci_dev *pdev)
4839	{
4840	struct net_device *netdev = pci_get_drvdata(pdev);
4841
4842	rtnl_lock();
4843	if (netif_running(dev: netdev))
4844	ixgbevf_open(netdev);
4845
4846	netif_device_attach(dev: netdev);
4847	rtnl_unlock();
4848	}
4849
4850	/* PCI Error Recovery (ERS) */
4851	static const struct pci_error_handlers ixgbevf_err_handler = {
4852	.error_detected = ixgbevf_io_error_detected,
4853	.slot_reset = ixgbevf_io_slot_reset,
4854	.resume = ixgbevf_io_resume,
4855	};
4856
4857	static SIMPLE_DEV_PM_OPS(ixgbevf_pm_ops, ixgbevf_suspend, ixgbevf_resume);
4858
4859	static struct pci_driver ixgbevf_driver = {
4860	.name = ixgbevf_driver_name,
4861	.id_table = ixgbevf_pci_tbl,
4862	.probe = ixgbevf_probe,
4863	.remove = ixgbevf_remove,
4864
4865	/* Power Management Hooks */
4866	.driver.pm = &ixgbevf_pm_ops,
4867
4868	.shutdown = ixgbevf_shutdown,
4869	.err_handler = &ixgbevf_err_handler
4870	};
4871
4872	/**
4873	* ixgbevf_init_module - Driver Registration Routine
4874	*
4875	* ixgbevf_init_module is the first routine called when the driver is
4876	* loaded. All it does is register with the PCI subsystem.
4877	**/
4878	static int __init ixgbevf_init_module(void)
4879	{
4880	int err;
4881
4882	pr_info("%s\n", ixgbevf_driver_string);
4883	pr_info("%s\n", ixgbevf_copyright);
4884	ixgbevf_wq = create_singlethread_workqueue(ixgbevf_driver_name);
4885	if (!ixgbevf_wq) {
4886	pr_err("%s: Failed to create workqueue\n", ixgbevf_driver_name);
4887	return -ENOMEM;
4888	}
4889
4890	err = pci_register_driver(&ixgbevf_driver);
4891	if (err) {
4892	destroy_workqueue(wq: ixgbevf_wq);
4893	return err;
4894	}
4895
4896	return 0;
4897	}
4898
4899	module_init(ixgbevf_init_module);
4900
4901	/**
4902	* ixgbevf_exit_module - Driver Exit Cleanup Routine
4903	*
4904	* ixgbevf_exit_module is called just before the driver is removed
4905	* from memory.
4906	**/
4907	static void __exit ixgbevf_exit_module(void)
4908	{
4909	pci_unregister_driver(dev: &ixgbevf_driver);
4910	if (ixgbevf_wq) {
4911	destroy_workqueue(wq: ixgbevf_wq);
4912	ixgbevf_wq = NULL;
4913	}
4914	}
4915
4916	#ifdef DEBUG
4917	/**
4918	* ixgbevf_get_hw_dev_name - return device name string
4919	* used by hardware layer to print debugging information
4920	* @hw: pointer to private hardware struct
4921	**/
4922	char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
4923	{
4924	struct ixgbevf_adapter *adapter = hw->back;
4925
4926	return adapter->netdev->name;
4927	}
4928
4929	#endif
4930	module_exit(ixgbevf_exit_module);
4931
4932	/* ixgbevf_main.c */
4933