1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright(c) 2013 - 2018 Intel Corporation. */
3
4	#include "iavf.h"
5	#include "iavf_prototype.h"
6	/* All iavf tracepoints are defined by the include below, which must
7	* be included exactly once across the whole kernel with
8	* CREATE_TRACE_POINTS defined
9	*/
10	#define CREATE_TRACE_POINTS
11	#include "iavf_trace.h"
12
13	static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
14	static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
15	static int iavf_close(struct net_device *netdev);
16	static void iavf_init_get_resources(struct iavf_adapter *adapter);
17	static int iavf_check_reset_complete(struct iavf_hw *hw);
18
19	char iavf_driver_name[] = "iavf";
20	static const char iavf_driver_string[] =
21	"Intel(R) Ethernet Adaptive Virtual Function Network Driver";
22
23	static const char iavf_copyright[] =
24	"Copyright (c) 2013 - 2018 Intel Corporation.";
25
26	/* iavf_pci_tbl - PCI Device ID Table
27	*
28	* Wildcard entries (PCI_ANY_ID) should come last
29	* Last entry must be all 0s
30	*
31	* { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
32	* Class, Class Mask, private data (not used) }
33	*/
34	static const struct pci_device_id iavf_pci_tbl[] = {
35	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
36	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
37	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
38	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
39	/* required last entry */
40	{0, }
41	};
42
43	MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
44
45	MODULE_ALIAS("i40evf");
46	MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
47	MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");
48	MODULE_LICENSE("GPL v2");
49
50	static const struct net_device_ops iavf_netdev_ops;
51
52	int iavf_status_to_errno(enum iavf_status status)
53	{
54	switch (status) {
55	case IAVF_SUCCESS:
56	return 0;
57	case IAVF_ERR_PARAM:
58	case IAVF_ERR_MAC_TYPE:
59	case IAVF_ERR_INVALID_MAC_ADDR:
60	case IAVF_ERR_INVALID_LINK_SETTINGS:
61	case IAVF_ERR_INVALID_PD_ID:
62	case IAVF_ERR_INVALID_QP_ID:
63	case IAVF_ERR_INVALID_CQ_ID:
64	case IAVF_ERR_INVALID_CEQ_ID:
65	case IAVF_ERR_INVALID_AEQ_ID:
66	case IAVF_ERR_INVALID_SIZE:
67	case IAVF_ERR_INVALID_ARP_INDEX:
68	case IAVF_ERR_INVALID_FPM_FUNC_ID:
69	case IAVF_ERR_QP_INVALID_MSG_SIZE:
70	case IAVF_ERR_INVALID_FRAG_COUNT:
71	case IAVF_ERR_INVALID_ALIGNMENT:
72	case IAVF_ERR_INVALID_PUSH_PAGE_INDEX:
73	case IAVF_ERR_INVALID_IMM_DATA_SIZE:
74	case IAVF_ERR_INVALID_VF_ID:
75	case IAVF_ERR_INVALID_HMCFN_ID:
76	case IAVF_ERR_INVALID_PBLE_INDEX:
77	case IAVF_ERR_INVALID_SD_INDEX:
78	case IAVF_ERR_INVALID_PAGE_DESC_INDEX:
79	case IAVF_ERR_INVALID_SD_TYPE:
80	case IAVF_ERR_INVALID_HMC_OBJ_INDEX:
81	case IAVF_ERR_INVALID_HMC_OBJ_COUNT:
82	case IAVF_ERR_INVALID_SRQ_ARM_LIMIT:
83	return -EINVAL;
84	case IAVF_ERR_NVM:
85	case IAVF_ERR_NVM_CHECKSUM:
86	case IAVF_ERR_PHY:
87	case IAVF_ERR_CONFIG:
88	case IAVF_ERR_UNKNOWN_PHY:
89	case IAVF_ERR_LINK_SETUP:
90	case IAVF_ERR_ADAPTER_STOPPED:
91	case IAVF_ERR_PRIMARY_REQUESTS_PENDING:
92	case IAVF_ERR_AUTONEG_NOT_COMPLETE:
93	case IAVF_ERR_RESET_FAILED:
94	case IAVF_ERR_BAD_PTR:
95	case IAVF_ERR_SWFW_SYNC:
96	case IAVF_ERR_QP_TOOMANY_WRS_POSTED:
97	case IAVF_ERR_QUEUE_EMPTY:
98	case IAVF_ERR_FLUSHED_QUEUE:
99	case IAVF_ERR_OPCODE_MISMATCH:
100	case IAVF_ERR_CQP_COMPL_ERROR:
101	case IAVF_ERR_BACKING_PAGE_ERROR:
102	case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE:
103	case IAVF_ERR_MEMCPY_FAILED:
104	case IAVF_ERR_SRQ_ENABLED:
105	case IAVF_ERR_ADMIN_QUEUE_ERROR:
106	case IAVF_ERR_ADMIN_QUEUE_FULL:
107	case IAVF_ERR_BAD_RDMA_CQE:
108	case IAVF_ERR_NVM_BLANK_MODE:
109	case IAVF_ERR_PE_DOORBELL_NOT_ENABLED:
110	case IAVF_ERR_DIAG_TEST_FAILED:
111	case IAVF_ERR_FIRMWARE_API_VERSION:
112	case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
113	return -EIO;
114	case IAVF_ERR_DEVICE_NOT_SUPPORTED:
115	return -ENODEV;
116	case IAVF_ERR_NO_AVAILABLE_VSI:
117	case IAVF_ERR_RING_FULL:
118	return -ENOSPC;
119	case IAVF_ERR_NO_MEMORY:
120	return -ENOMEM;
121	case IAVF_ERR_TIMEOUT:
122	case IAVF_ERR_ADMIN_QUEUE_TIMEOUT:
123	return -ETIMEDOUT;
124	case IAVF_ERR_NOT_IMPLEMENTED:
125	case IAVF_NOT_SUPPORTED:
126	return -EOPNOTSUPP;
127	case IAVF_ERR_ADMIN_QUEUE_NO_WORK:
128	return -EALREADY;
129	case IAVF_ERR_NOT_READY:
130	return -EBUSY;
131	case IAVF_ERR_BUF_TOO_SHORT:
132	return -EMSGSIZE;
133	}
134
135	return -EIO;
136	}
137
138	int virtchnl_status_to_errno(enum virtchnl_status_code v_status)
139	{
140	switch (v_status) {
141	case VIRTCHNL_STATUS_SUCCESS:
142	return 0;
143	case VIRTCHNL_STATUS_ERR_PARAM:
144	case VIRTCHNL_STATUS_ERR_INVALID_VF_ID:
145	return -EINVAL;
146	case VIRTCHNL_STATUS_ERR_NO_MEMORY:
147	return -ENOMEM;
148	case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH:
149	case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR:
150	case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR:
151	return -EIO;
152	case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED:
153	return -EOPNOTSUPP;
154	}
155
156	return -EIO;
157	}
158
159	/**
160	* iavf_pdev_to_adapter - go from pci_dev to adapter
161	* @pdev: pci_dev pointer
162	*/
163	static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev)
164	{
165	return netdev_priv(dev: pci_get_drvdata(pdev));
166	}
167
168	/**
169	* iavf_is_reset_in_progress - Check if a reset is in progress
170	* @adapter: board private structure
171	*/
172	static bool iavf_is_reset_in_progress(struct iavf_adapter *adapter)
173	{
174	if (adapter->state == __IAVF_RESETTING ||
175	adapter->flags & (IAVF_FLAG_RESET_PENDING |
176	IAVF_FLAG_RESET_NEEDED))
177	return true;
178
179	return false;
180	}
181
182	/**
183	* iavf_wait_for_reset - Wait for reset to finish.
184	* @adapter: board private structure
185	*
186	* Returns 0 if reset finished successfully, negative on timeout or interrupt.
187	*/
188	int iavf_wait_for_reset(struct iavf_adapter *adapter)
189	{
190	int ret = wait_event_interruptible_timeout(adapter->reset_waitqueue,
191	!iavf_is_reset_in_progress(adapter),
192	msecs_to_jiffies(5000));
193
194	/* If ret < 0 then it means wait was interrupted.
195	* If ret == 0 then it means we got a timeout while waiting
196	* for reset to finish.
197	* If ret > 0 it means reset has finished.
198	*/
199	if (ret > 0)
200	return 0;
201	else if (ret < 0)
202	return -EINTR;
203	else
204	return -EBUSY;
205	}
206
207	/**
208	* iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
209	* @hw: pointer to the HW structure
210	* @mem: ptr to mem struct to fill out
211	* @size: size of memory requested
212	* @alignment: what to align the allocation to
213	**/
214	enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,
215	struct iavf_dma_mem *mem,
216	u64 size, u32 alignment)
217	{
218	struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
219
220	if (!mem)
221	return IAVF_ERR_PARAM;
222
223	mem->size = ALIGN(size, alignment);
224	mem->va = dma_alloc_coherent(dev: &adapter->pdev->dev, size: mem->size,
225	dma_handle: (dma_addr_t *)&mem->pa, GFP_KERNEL);
226	if (mem->va)
227	return 0;
228	else
229	return IAVF_ERR_NO_MEMORY;
230	}
231
232	/**
233	* iavf_free_dma_mem - wrapper for DMA memory freeing
234	* @hw: pointer to the HW structure
235	* @mem: ptr to mem struct to free
236	**/
237	enum iavf_status iavf_free_dma_mem(struct iavf_hw *hw, struct iavf_dma_mem *mem)
238	{
239	struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
240
241	if (!mem || !mem->va)
242	return IAVF_ERR_PARAM;
243	dma_free_coherent(dev: &adapter->pdev->dev, size: mem->size,
244	cpu_addr: mem->va, dma_handle: (dma_addr_t)mem->pa);
245	return 0;
246	}
247
248	/**
249	* iavf_allocate_virt_mem - virt memory alloc wrapper
250	* @hw: pointer to the HW structure
251	* @mem: ptr to mem struct to fill out
252	* @size: size of memory requested
253	**/
254	enum iavf_status iavf_allocate_virt_mem(struct iavf_hw *hw,
255	struct iavf_virt_mem *mem, u32 size)
256	{
257	if (!mem)
258	return IAVF_ERR_PARAM;
259
260	mem->size = size;
261	mem->va = kzalloc(size, GFP_KERNEL);
262
263	if (mem->va)
264	return 0;
265	else
266	return IAVF_ERR_NO_MEMORY;
267	}
268
269	/**
270	* iavf_free_virt_mem - virt memory free wrapper
271	* @hw: pointer to the HW structure
272	* @mem: ptr to mem struct to free
273	**/
274	void iavf_free_virt_mem(struct iavf_hw *hw, struct iavf_virt_mem *mem)
275	{
276	kfree(objp: mem->va);
277	}
278
279	/**
280	* iavf_lock_timeout - try to lock mutex but give up after timeout
281	* @lock: mutex that should be locked
282	* @msecs: timeout in msecs
283	*
284	* Returns 0 on success, negative on failure
285	**/
286	static int iavf_lock_timeout(struct mutex *lock, unsigned int msecs)
287	{
288	unsigned int wait, delay = 10;
289
290	for (wait = 0; wait < msecs; wait += delay) {
291	if (mutex_trylock(lock))
292	return 0;
293
294	msleep(msecs: delay);
295	}
296
297	return -1;
298	}
299
300	/**
301	* iavf_schedule_reset - Set the flags and schedule a reset event
302	* @adapter: board private structure
303	* @flags: IAVF_FLAG_RESET_PENDING or IAVF_FLAG_RESET_NEEDED
304	**/
305	void iavf_schedule_reset(struct iavf_adapter *adapter, u64 flags)
306	{
307	if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section) &&
308	!(adapter->flags &
309	(IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
310	adapter->flags |= flags;
311	queue_work(wq: adapter->wq, work: &adapter->reset_task);
312	}
313	}
314
315	/**
316	* iavf_schedule_aq_request - Set the flags and schedule aq request
317	* @adapter: board private structure
318	* @flags: requested aq flags
319	**/
320	void iavf_schedule_aq_request(struct iavf_adapter *adapter, u64 flags)
321	{
322	adapter->aq_required |= flags;
323	mod_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task, delay: 0);
324	}
325
326	/**
327	* iavf_tx_timeout - Respond to a Tx Hang
328	* @netdev: network interface device structure
329	* @txqueue: queue number that is timing out
330	**/
331	static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
332	{
333	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
334
335	adapter->tx_timeout_count++;
336	iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
337	}
338
339	/**
340	* iavf_misc_irq_disable - Mask off interrupt generation on the NIC
341	* @adapter: board private structure
342	**/
343	static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
344	{
345	struct iavf_hw *hw = &adapter->hw;
346
347	if (!adapter->msix_entries)
348	return;
349
350	wr32(hw, IAVF_VFINT_DYN_CTL01, 0);
351
352	iavf_flush(hw);
353
354	synchronize_irq(irq: adapter->msix_entries[0].vector);
355	}
356
357	/**
358	* iavf_misc_irq_enable - Enable default interrupt generation settings
359	* @adapter: board private structure
360	**/
361	static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
362	{
363	struct iavf_hw *hw = &adapter->hw;
364
365	wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |
366	IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
367	wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
368
369	iavf_flush(hw);
370	}
371
372	/**
373	* iavf_irq_disable - Mask off interrupt generation on the NIC
374	* @adapter: board private structure
375	**/
376	static void iavf_irq_disable(struct iavf_adapter *adapter)
377	{
378	int i;
379	struct iavf_hw *hw = &adapter->hw;
380
381	if (!adapter->msix_entries)
382	return;
383
384	for (i = 1; i < adapter->num_msix_vectors; i++) {
385	wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);
386	synchronize_irq(irq: adapter->msix_entries[i].vector);
387	}
388	iavf_flush(hw);
389	}
390
391	/**
392	* iavf_irq_enable_queues - Enable interrupt for all queues
393	* @adapter: board private structure
394	**/
395	static void iavf_irq_enable_queues(struct iavf_adapter *adapter)
396	{
397	struct iavf_hw *hw = &adapter->hw;
398	int i;
399
400	for (i = 1; i < adapter->num_msix_vectors; i++) {
401	wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),
402	IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
403	IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);
404	}
405	}
406
407	/**
408	* iavf_irq_enable - Enable default interrupt generation settings
409	* @adapter: board private structure
410	* @flush: boolean value whether to run rd32()
411	**/
412	void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
413	{
414	struct iavf_hw *hw = &adapter->hw;
415
416	iavf_misc_irq_enable(adapter);
417	iavf_irq_enable_queues(adapter);
418
419	if (flush)
420	iavf_flush(hw);
421	}
422
423	/**
424	* iavf_msix_aq - Interrupt handler for vector 0
425	* @irq: interrupt number
426	* @data: pointer to netdev
427	**/
428	static irqreturn_t iavf_msix_aq(int irq, void *data)
429	{
430	struct net_device *netdev = data;
431	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
432	struct iavf_hw *hw = &adapter->hw;
433
434	/* handle non-queue interrupts, these reads clear the registers */
435	rd32(hw, IAVF_VFINT_ICR01);
436	rd32(hw, IAVF_VFINT_ICR0_ENA1);
437
438	if (adapter->state != __IAVF_REMOVE)
439	/* schedule work on the private workqueue */
440	queue_work(wq: adapter->wq, work: &adapter->adminq_task);
441
442	return IRQ_HANDLED;
443	}
444
445	/**
446	* iavf_msix_clean_rings - MSIX mode Interrupt Handler
447	* @irq: interrupt number
448	* @data: pointer to a q_vector
449	**/
450	static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
451	{
452	struct iavf_q_vector *q_vector = data;
453
454	if (!q_vector->tx.ring && !q_vector->rx.ring)
455	return IRQ_HANDLED;
456
457	napi_schedule_irqoff(n: &q_vector->napi);
458
459	return IRQ_HANDLED;
460	}
461
462	/**
463	* iavf_map_vector_to_rxq - associate irqs with rx queues
464	* @adapter: board private structure
465	* @v_idx: interrupt number
466	* @r_idx: queue number
467	**/
468	static void
469	iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
470	{
471	struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
472	struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];
473	struct iavf_hw *hw = &adapter->hw;
474
475	rx_ring->q_vector = q_vector;
476	rx_ring->next = q_vector->rx.ring;
477	rx_ring->vsi = &adapter->vsi;
478	q_vector->rx.ring = rx_ring;
479	q_vector->rx.count++;
480	q_vector->rx.next_update = jiffies + 1;
481	q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
482	q_vector->ring_mask |= BIT(r_idx);
483	wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),
484	q_vector->rx.current_itr >> 1);
485	q_vector->rx.current_itr = q_vector->rx.target_itr;
486	}
487
488	/**
489	* iavf_map_vector_to_txq - associate irqs with tx queues
490	* @adapter: board private structure
491	* @v_idx: interrupt number
492	* @t_idx: queue number
493	**/
494	static void
495	iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
496	{
497	struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
498	struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];
499	struct iavf_hw *hw = &adapter->hw;
500
501	tx_ring->q_vector = q_vector;
502	tx_ring->next = q_vector->tx.ring;
503	tx_ring->vsi = &adapter->vsi;
504	q_vector->tx.ring = tx_ring;
505	q_vector->tx.count++;
506	q_vector->tx.next_update = jiffies + 1;
507	q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
508	q_vector->num_ringpairs++;
509	wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),
510	q_vector->tx.target_itr >> 1);
511	q_vector->tx.current_itr = q_vector->tx.target_itr;
512	}
513
514	/**
515	* iavf_map_rings_to_vectors - Maps descriptor rings to vectors
516	* @adapter: board private structure to initialize
517	*
518	* This function maps descriptor rings to the queue-specific vectors
519	* we were allotted through the MSI-X enabling code. Ideally, we'd have
520	* one vector per ring/queue, but on a constrained vector budget, we
521	* group the rings as "efficiently" as possible. You would add new
522	* mapping configurations in here.
523	**/
524	static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
525	{
526	int rings_remaining = adapter->num_active_queues;
527	int ridx = 0, vidx = 0;
528	int q_vectors;
529
530	q_vectors = adapter->num_msix_vectors - NONQ_VECS;
531
532	for (; ridx < rings_remaining; ridx++) {
533	iavf_map_vector_to_rxq(adapter, v_idx: vidx, r_idx: ridx);
534	iavf_map_vector_to_txq(adapter, v_idx: vidx, t_idx: ridx);
535
536	/* In the case where we have more queues than vectors, continue
537	* round-robin on vectors until all queues are mapped.
538	*/
539	if (++vidx >= q_vectors)
540	vidx = 0;
541	}
542
543	adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
544	}
545
546	/**
547	* iavf_irq_affinity_notify - Callback for affinity changes
548	* @notify: context as to what irq was changed
549	* @mask: the new affinity mask
550	*
551	* This is a callback function used by the irq_set_affinity_notifier function
552	* so that we may register to receive changes to the irq affinity masks.
553	**/
554	static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
555	const cpumask_t *mask)
556	{
557	struct iavf_q_vector *q_vector =
558	container_of(notify, struct iavf_q_vector, affinity_notify);
559
560	cpumask_copy(dstp: &q_vector->affinity_mask, srcp: mask);
561	}
562
563	/**
564	* iavf_irq_affinity_release - Callback for affinity notifier release
565	* @ref: internal core kernel usage
566	*
567	* This is a callback function used by the irq_set_affinity_notifier function
568	* to inform the current notification subscriber that they will no longer
569	* receive notifications.
570	**/
571	static void iavf_irq_affinity_release(struct kref *ref) {}
572
573	/**
574	* iavf_request_traffic_irqs - Initialize MSI-X interrupts
575	* @adapter: board private structure
576	* @basename: device basename
577	*
578	* Allocates MSI-X vectors for tx and rx handling, and requests
579	* interrupts from the kernel.
580	**/
581	static int
582	iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
583	{
584	unsigned int vector, q_vectors;
585	unsigned int rx_int_idx = 0, tx_int_idx = 0;
586	int irq_num, err;
587	int cpu;
588
589	iavf_irq_disable(adapter);
590	/* Decrement for Other and TCP Timer vectors */
591	q_vectors = adapter->num_msix_vectors - NONQ_VECS;
592
593	for (vector = 0; vector < q_vectors; vector++) {
594	struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];
595
596	irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
597
598	if (q_vector->tx.ring && q_vector->rx.ring) {
599	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
600	fmt: "iavf-%s-TxRx-%u", basename, rx_int_idx++);
601	tx_int_idx++;
602	} else if (q_vector->rx.ring) {
603	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
604	fmt: "iavf-%s-rx-%u", basename, rx_int_idx++);
605	} else if (q_vector->tx.ring) {
606	snprintf(buf: q_vector->name, size: sizeof(q_vector->name),
607	fmt: "iavf-%s-tx-%u", basename, tx_int_idx++);
608	} else {
609	/* skip this unused q_vector */
610	continue;
611	}
612	err = request_irq(irq: irq_num,
613	handler: iavf_msix_clean_rings,
614	flags: 0,
615	name: q_vector->name,
616	dev: q_vector);
617	if (err) {
618	dev_info(&adapter->pdev->dev,
619	"Request_irq failed, error: %d\n", err);
620	goto free_queue_irqs;
621	}
622	/* register for affinity change notifications */
623	q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
624	q_vector->affinity_notify.release =
625	iavf_irq_affinity_release;
626	irq_set_affinity_notifier(irq: irq_num, notify: &q_vector->affinity_notify);
627	/* Spread the IRQ affinity hints across online CPUs. Note that
628	* get_cpu_mask returns a mask with a permanent lifetime so
629	* it's safe to use as a hint for irq_update_affinity_hint.
630	*/
631	cpu = cpumask_local_spread(i: q_vector->v_idx, node: -1);
632	irq_update_affinity_hint(irq: irq_num, m: get_cpu_mask(cpu));
633	}
634
635	return 0;
636
637	free_queue_irqs:
638	while (vector) {
639	vector--;
640	irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
641	irq_set_affinity_notifier(irq: irq_num, NULL);
642	irq_update_affinity_hint(irq: irq_num, NULL);
643	free_irq(irq_num, &adapter->q_vectors[vector]);
644	}
645	return err;
646	}
647
648	/**
649	* iavf_request_misc_irq - Initialize MSI-X interrupts
650	* @adapter: board private structure
651	*
652	* Allocates MSI-X vector 0 and requests interrupts from the kernel. This
653	* vector is only for the admin queue, and stays active even when the netdev
654	* is closed.
655	**/
656	static int iavf_request_misc_irq(struct iavf_adapter *adapter)
657	{
658	struct net_device *netdev = adapter->netdev;
659	int err;
660
661	snprintf(buf: adapter->misc_vector_name,
662	size: sizeof(adapter->misc_vector_name) - 1, fmt: "iavf-%s:mbx",
663	dev_name(dev: &adapter->pdev->dev));
664	err = request_irq(irq: adapter->msix_entries[0].vector,
665	handler: &iavf_msix_aq, flags: 0,
666	name: adapter->misc_vector_name, dev: netdev);
667	if (err) {
668	dev_err(&adapter->pdev->dev,
669	"request_irq for %s failed: %d\n",
670	adapter->misc_vector_name, err);
671	free_irq(adapter->msix_entries[0].vector, netdev);
672	}
673	return err;
674	}
675
676	/**
677	* iavf_free_traffic_irqs - Free MSI-X interrupts
678	* @adapter: board private structure
679	*
680	* Frees all MSI-X vectors other than 0.
681	**/
682	static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
683	{
684	int vector, irq_num, q_vectors;
685
686	if (!adapter->msix_entries)
687	return;
688
689	q_vectors = adapter->num_msix_vectors - NONQ_VECS;
690
691	for (vector = 0; vector < q_vectors; vector++) {
692	irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
693	irq_set_affinity_notifier(irq: irq_num, NULL);
694	irq_update_affinity_hint(irq: irq_num, NULL);
695	free_irq(irq_num, &adapter->q_vectors[vector]);
696	}
697	}
698
699	/**
700	* iavf_free_misc_irq - Free MSI-X miscellaneous vector
701	* @adapter: board private structure
702	*
703	* Frees MSI-X vector 0.
704	**/
705	static void iavf_free_misc_irq(struct iavf_adapter *adapter)
706	{
707	struct net_device *netdev = adapter->netdev;
708
709	if (!adapter->msix_entries)
710	return;
711
712	free_irq(adapter->msix_entries[0].vector, netdev);
713	}
714
715	/**
716	* iavf_configure_tx - Configure Transmit Unit after Reset
717	* @adapter: board private structure
718	*
719	* Configure the Tx unit of the MAC after a reset.
720	**/
721	static void iavf_configure_tx(struct iavf_adapter *adapter)
722	{
723	struct iavf_hw *hw = &adapter->hw;
724	int i;
725
726	for (i = 0; i < adapter->num_active_queues; i++)
727	adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);
728	}
729
730	/**
731	* iavf_configure_rx - Configure Receive Unit after Reset
732	* @adapter: board private structure
733	*
734	* Configure the Rx unit of the MAC after a reset.
735	**/
736	static void iavf_configure_rx(struct iavf_adapter *adapter)
737	{
738	unsigned int rx_buf_len = IAVF_RXBUFFER_2048;
739	struct iavf_hw *hw = &adapter->hw;
740	int i;
741
742	/* Legacy Rx will always default to a 2048 buffer size. */
743	#if (PAGE_SIZE < 8192)
744	if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) {
745	struct net_device *netdev = adapter->netdev;
746
747	/* For jumbo frames on systems with 4K pages we have to use
748	* an order 1 page, so we might as well increase the size
749	* of our Rx buffer to make better use of the available space
750	*/
751	rx_buf_len = IAVF_RXBUFFER_3072;
752
753	/* We use a 1536 buffer size for configurations with
754	* standard Ethernet mtu. On x86 this gives us enough room
755	* for shared info and 192 bytes of padding.
756	*/
757	if (!IAVF_2K_TOO_SMALL_WITH_PADDING &&
758	(netdev->mtu <= ETH_DATA_LEN))
759	rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN;
760	}
761	#endif
762
763	for (i = 0; i < adapter->num_active_queues; i++) {
764	adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);
765	adapter->rx_rings[i].rx_buf_len = rx_buf_len;
766
767	if (adapter->flags & IAVF_FLAG_LEGACY_RX)
768	clear_ring_build_skb_enabled(ring: &adapter->rx_rings[i]);
769	else
770	set_ring_build_skb_enabled(&adapter->rx_rings[i]);
771	}
772	}
773
774	/**
775	* iavf_find_vlan - Search filter list for specific vlan filter
776	* @adapter: board private structure
777	* @vlan: vlan tag
778	*
779	* Returns ptr to the filter object or NULL. Must be called while holding the
780	* mac_vlan_list_lock.
781	**/
782	static struct
783	iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter,
784	struct iavf_vlan vlan)
785	{
786	struct iavf_vlan_filter *f;
787
788	list_for_each_entry(f, &adapter->vlan_filter_list, list) {
789	if (f->vlan.vid == vlan.vid &&
790	f->vlan.tpid == vlan.tpid)
791	return f;
792	}
793
794	return NULL;
795	}
796
797	/**
798	* iavf_add_vlan - Add a vlan filter to the list
799	* @adapter: board private structure
800	* @vlan: VLAN tag
801	*
802	* Returns ptr to the filter object or NULL when no memory available.
803	**/
804	static struct
805	iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,
806	struct iavf_vlan vlan)
807	{
808	struct iavf_vlan_filter *f = NULL;
809
810	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
811
812	f = iavf_find_vlan(adapter, vlan);
813	if (!f) {
814	f = kzalloc(size: sizeof(*f), GFP_ATOMIC);
815	if (!f)
816	goto clearout;
817
818	f->vlan = vlan;
819
820	list_add_tail(new: &f->list, head: &adapter->vlan_filter_list);
821	f->state = IAVF_VLAN_ADD;
822	adapter->num_vlan_filters++;
823	iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER);
824	}
825
826	clearout:
827	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
828	return f;
829	}
830
831	/**
832	* iavf_del_vlan - Remove a vlan filter from the list
833	* @adapter: board private structure
834	* @vlan: VLAN tag
835	**/
836	static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan)
837	{
838	struct iavf_vlan_filter *f;
839
840	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
841
842	f = iavf_find_vlan(adapter, vlan);
843	if (f) {
844	f->state = IAVF_VLAN_REMOVE;
845	iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DEL_VLAN_FILTER);
846	}
847
848	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
849	}
850
851	/**
852	* iavf_restore_filters
853	* @adapter: board private structure
854	*
855	* Restore existing non MAC filters when VF netdev comes back up
856	**/
857	static void iavf_restore_filters(struct iavf_adapter *adapter)
858	{
859	struct iavf_vlan_filter *f;
860
861	/* re-add all VLAN filters */
862	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
863
864	list_for_each_entry(f, &adapter->vlan_filter_list, list) {
865	if (f->state == IAVF_VLAN_INACTIVE)
866	f->state = IAVF_VLAN_ADD;
867	}
868
869	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
870	adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
871	}
872
873	/**
874	* iavf_get_num_vlans_added - get number of VLANs added
875	* @adapter: board private structure
876	*/
877	u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter)
878	{
879	return adapter->num_vlan_filters;
880	}
881
882	/**
883	* iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF
884	* @adapter: board private structure
885	*
886	* This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN,
887	* do not impose a limit as that maintains current behavior and for
888	* VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF.
889	**/
890	static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter)
891	{
892	/* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has
893	* never been a limit on the VF driver side
894	*/
895	if (VLAN_ALLOWED(adapter))
896	return VLAN_N_VID;
897	else if (VLAN_V2_ALLOWED(adapter))
898	return adapter->vlan_v2_caps.filtering.max_filters;
899
900	return 0;
901	}
902
903	/**
904	* iavf_max_vlans_added - check if maximum VLANs allowed already exist
905	* @adapter: board private structure
906	**/
907	static bool iavf_max_vlans_added(struct iavf_adapter *adapter)
908	{
909	if (iavf_get_num_vlans_added(adapter) <
910	iavf_get_max_vlans_allowed(adapter))
911	return false;
912
913	return true;
914	}
915
916	/**
917	* iavf_vlan_rx_add_vid - Add a VLAN filter to a device
918	* @netdev: network device struct
919	* @proto: unused protocol data
920	* @vid: VLAN tag
921	**/
922	static int iavf_vlan_rx_add_vid(struct net_device *netdev,
923	__always_unused __be16 proto, u16 vid)
924	{
925	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
926
927	/* Do not track VLAN 0 filter, always added by the PF on VF init */
928	if (!vid)
929	return 0;
930
931	if (!VLAN_FILTERING_ALLOWED(adapter))
932	return -EIO;
933
934	if (iavf_max_vlans_added(adapter)) {
935	netdev_err(dev: netdev, format: "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n",
936	iavf_get_max_vlans_allowed(adapter));
937	return -EIO;
938	}
939
940	if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))))
941	return -ENOMEM;
942
943	return 0;
944	}
945
946	/**
947	* iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
948	* @netdev: network device struct
949	* @proto: unused protocol data
950	* @vid: VLAN tag
951	**/
952	static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
953	__always_unused __be16 proto, u16 vid)
954	{
955	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
956
957	/* We do not track VLAN 0 filter */
958	if (!vid)
959	return 0;
960
961	iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)));
962	return 0;
963	}
964
965	/**
966	* iavf_find_filter - Search filter list for specific mac filter
967	* @adapter: board private structure
968	* @macaddr: the MAC address
969	*
970	* Returns ptr to the filter object or NULL. Must be called while holding the
971	* mac_vlan_list_lock.
972	**/
973	static struct
974	iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
975	const u8 *macaddr)
976	{
977	struct iavf_mac_filter *f;
978
979	if (!macaddr)
980	return NULL;
981
982	list_for_each_entry(f, &adapter->mac_filter_list, list) {
983	if (ether_addr_equal(addr1: macaddr, addr2: f->macaddr))
984	return f;
985	}
986	return NULL;
987	}
988
989	/**
990	* iavf_add_filter - Add a mac filter to the filter list
991	* @adapter: board private structure
992	* @macaddr: the MAC address
993	*
994	* Returns ptr to the filter object or NULL when no memory available.
995	**/
996	struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
997	const u8 *macaddr)
998	{
999	struct iavf_mac_filter *f;
1000
1001	if (!macaddr)
1002	return NULL;
1003
1004	f = iavf_find_filter(adapter, macaddr);
1005	if (!f) {
1006	f = kzalloc(size: sizeof(*f), GFP_ATOMIC);
1007	if (!f)
1008	return f;
1009
1010	ether_addr_copy(dst: f->macaddr, src: macaddr);
1011
1012	list_add_tail(new: &f->list, head: &adapter->mac_filter_list);
1013	f->add = true;
1014	f->add_handled = false;
1015	f->is_new_mac = true;
1016	f->is_primary = ether_addr_equal(addr1: macaddr, addr2: adapter->hw.mac.addr);
1017	adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
1018	} else {
1019	f->remove = false;
1020	}
1021
1022	return f;
1023	}
1024
1025	/**
1026	* iavf_replace_primary_mac - Replace current primary address
1027	* @adapter: board private structure
1028	* @new_mac: new MAC address to be applied
1029	*
1030	* Replace current dev_addr and send request to PF for removal of previous
1031	* primary MAC address filter and addition of new primary MAC filter.
1032	* Return 0 for success, -ENOMEM for failure.
1033	*
1034	* Do not call this with mac_vlan_list_lock!
1035	**/
1036	static int iavf_replace_primary_mac(struct iavf_adapter *adapter,
1037	const u8 *new_mac)
1038	{
1039	struct iavf_hw *hw = &adapter->hw;
1040	struct iavf_mac_filter *new_f;
1041	struct iavf_mac_filter *old_f;
1042
1043	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
1044
1045	new_f = iavf_add_filter(adapter, macaddr: new_mac);
1046	if (!new_f) {
1047	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
1048	return -ENOMEM;
1049	}
1050
1051	old_f = iavf_find_filter(adapter, macaddr: hw->mac.addr);
1052	if (old_f) {
1053	old_f->is_primary = false;
1054	old_f->remove = true;
1055	adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1056	}
1057	/* Always send the request to add if changing primary MAC,
1058	* even if filter is already present on the list
1059	*/
1060	new_f->is_primary = true;
1061	new_f->add = true;
1062	adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
1063	ether_addr_copy(dst: hw->mac.addr, src: new_mac);
1064
1065	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
1066
1067	/* schedule the watchdog task to immediately process the request */
1068	mod_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task, delay: 0);
1069	return 0;
1070	}
1071
1072	/**
1073	* iavf_is_mac_set_handled - wait for a response to set MAC from PF
1074	* @netdev: network interface device structure
1075	* @macaddr: MAC address to set
1076	*
1077	* Returns true on success, false on failure
1078	*/
1079	static bool iavf_is_mac_set_handled(struct net_device *netdev,
1080	const u8 *macaddr)
1081	{
1082	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
1083	struct iavf_mac_filter *f;
1084	bool ret = false;
1085
1086	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
1087
1088	f = iavf_find_filter(adapter, macaddr);
1089
1090	if (!f || (!f->add && f->add_handled))
1091	ret = true;
1092
1093	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
1094
1095	return ret;
1096	}
1097
1098	/**
1099	* iavf_set_mac - NDO callback to set port MAC address
1100	* @netdev: network interface device structure
1101	* @p: pointer to an address structure
1102	*
1103	* Returns 0 on success, negative on failure
1104	*/
1105	static int iavf_set_mac(struct net_device *netdev, void *p)
1106	{
1107	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
1108	struct sockaddr *addr = p;
1109	int ret;
1110
1111	if (!is_valid_ether_addr(addr: addr->sa_data))
1112	return -EADDRNOTAVAIL;
1113
1114	ret = iavf_replace_primary_mac(adapter, new_mac: addr->sa_data);
1115
1116	if (ret)
1117	return ret;
1118
1119	ret = wait_event_interruptible_timeout(adapter->vc_waitqueue,
1120	iavf_is_mac_set_handled(netdev, addr->sa_data),
1121	msecs_to_jiffies(2500));
1122
1123	/* If ret < 0 then it means wait was interrupted.
1124	* If ret == 0 then it means we got a timeout.
1125	* else it means we got response for set MAC from PF,
1126	* check if netdev MAC was updated to requested MAC,
1127	* if yes then set MAC succeeded otherwise it failed return -EACCES
1128	*/
1129	if (ret < 0)
1130	return ret;
1131
1132	if (!ret)
1133	return -EAGAIN;
1134
1135	if (!ether_addr_equal(addr1: netdev->dev_addr, addr2: addr->sa_data))
1136	return -EACCES;
1137
1138	return 0;
1139	}
1140
1141	/**
1142	* iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
1143	* @netdev: the netdevice
1144	* @addr: address to add
1145	*
1146	* Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1147	* __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1148	*/
1149	static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
1150	{
1151	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
1152
1153	if (iavf_add_filter(adapter, macaddr: addr))
1154	return 0;
1155	else
1156	return -ENOMEM;
1157	}
1158
1159	/**
1160	* iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1161	* @netdev: the netdevice
1162	* @addr: address to add
1163	*
1164	* Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1165	* __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1166	*/
1167	static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
1168	{
1169	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
1170	struct iavf_mac_filter *f;
1171
1172	/* Under some circumstances, we might receive a request to delete
1173	* our own device address from our uc list. Because we store the
1174	* device address in the VSI's MAC/VLAN filter list, we need to ignore
1175	* such requests and not delete our device address from this list.
1176	*/
1177	if (ether_addr_equal(addr1: addr, addr2: netdev->dev_addr))
1178	return 0;
1179
1180	f = iavf_find_filter(adapter, macaddr: addr);
1181	if (f) {
1182	f->remove = true;
1183	adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1184	}
1185	return 0;
1186	}
1187
1188	/**
1189	* iavf_promiscuous_mode_changed - check if promiscuous mode bits changed
1190	* @adapter: device specific adapter
1191	*/
1192	bool iavf_promiscuous_mode_changed(struct iavf_adapter *adapter)
1193	{
1194	return (adapter->current_netdev_promisc_flags ^ adapter->netdev->flags) &
1195	(IFF_PROMISC | IFF_ALLMULTI);
1196	}
1197
1198	/**
1199	* iavf_set_rx_mode - NDO callback to set the netdev filters
1200	* @netdev: network interface device structure
1201	**/
1202	static void iavf_set_rx_mode(struct net_device *netdev)
1203	{
1204	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
1205
1206	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
1207	__dev_uc_sync(dev: netdev, sync: iavf_addr_sync, unsync: iavf_addr_unsync);
1208	__dev_mc_sync(dev: netdev, sync: iavf_addr_sync, unsync: iavf_addr_unsync);
1209	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
1210
1211	spin_lock_bh(lock: &adapter->current_netdev_promisc_flags_lock);
1212	if (iavf_promiscuous_mode_changed(adapter))
1213	adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE;
1214	spin_unlock_bh(lock: &adapter->current_netdev_promisc_flags_lock);
1215	}
1216
1217	/**
1218	* iavf_napi_enable_all - enable NAPI on all queue vectors
1219	* @adapter: board private structure
1220	**/
1221	static void iavf_napi_enable_all(struct iavf_adapter *adapter)
1222	{
1223	int q_idx;
1224	struct iavf_q_vector *q_vector;
1225	int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1226
1227	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1228	struct napi_struct *napi;
1229
1230	q_vector = &adapter->q_vectors[q_idx];
1231	napi = &q_vector->napi;
1232	napi_enable(n: napi);
1233	}
1234	}
1235
1236	/**
1237	* iavf_napi_disable_all - disable NAPI on all queue vectors
1238	* @adapter: board private structure
1239	**/
1240	static void iavf_napi_disable_all(struct iavf_adapter *adapter)
1241	{
1242	int q_idx;
1243	struct iavf_q_vector *q_vector;
1244	int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1245
1246	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1247	q_vector = &adapter->q_vectors[q_idx];
1248	napi_disable(n: &q_vector->napi);
1249	}
1250	}
1251
1252	/**
1253	* iavf_configure - set up transmit and receive data structures
1254	* @adapter: board private structure
1255	**/
1256	static void iavf_configure(struct iavf_adapter *adapter)
1257	{
1258	struct net_device *netdev = adapter->netdev;
1259	int i;
1260
1261	iavf_set_rx_mode(netdev);
1262
1263	iavf_configure_tx(adapter);
1264	iavf_configure_rx(adapter);
1265	adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
1266
1267	for (i = 0; i < adapter->num_active_queues; i++) {
1268	struct iavf_ring *ring = &adapter->rx_rings[i];
1269
1270	iavf_alloc_rx_buffers(rxr: ring, IAVF_DESC_UNUSED(ring));
1271	}
1272	}
1273
1274	/**
1275	* iavf_up_complete - Finish the last steps of bringing up a connection
1276	* @adapter: board private structure
1277	*
1278	* Expects to be called while holding crit_lock.
1279	**/
1280	static void iavf_up_complete(struct iavf_adapter *adapter)
1281	{
1282	iavf_change_state(adapter, state: __IAVF_RUNNING);
1283	clear_bit(nr: __IAVF_VSI_DOWN, addr: adapter->vsi.state);
1284
1285	iavf_napi_enable_all(adapter);
1286
1287	adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES;
1288	mod_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task, delay: 0);
1289	}
1290
1291	/**
1292	* iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF
1293	* yet and mark other to be removed.
1294	* @adapter: board private structure
1295	**/
1296	static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)
1297	{
1298	struct iavf_vlan_filter *vlf, *vlftmp;
1299	struct iavf_mac_filter *f, *ftmp;
1300
1301	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
1302	/* clear the sync flag on all filters */
1303	__dev_uc_unsync(dev: adapter->netdev, NULL);
1304	__dev_mc_unsync(dev: adapter->netdev, NULL);
1305
1306	/* remove all MAC filters */
1307	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list,
1308	list) {
1309	if (f->add) {
1310	list_del(entry: &f->list);
1311	kfree(objp: f);
1312	} else {
1313	f->remove = true;
1314	}
1315	}
1316
1317	/* disable all VLAN filters */
1318	list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
1319	list)
1320	vlf->state = IAVF_VLAN_DISABLE;
1321
1322	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
1323	}
1324
1325	/**
1326	* iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and
1327	* mark other to be removed.
1328	* @adapter: board private structure
1329	**/
1330	static void iavf_clear_cloud_filters(struct iavf_adapter *adapter)
1331	{
1332	struct iavf_cloud_filter *cf, *cftmp;
1333
1334	/* remove all cloud filters */
1335	spin_lock_bh(lock: &adapter->cloud_filter_list_lock);
1336	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
1337	list) {
1338	if (cf->add) {
1339	list_del(entry: &cf->list);
1340	kfree(objp: cf);
1341	adapter->num_cloud_filters--;
1342	} else {
1343	cf->del = true;
1344	}
1345	}
1346	spin_unlock_bh(lock: &adapter->cloud_filter_list_lock);
1347	}
1348
1349	/**
1350	* iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark
1351	* other to be removed.
1352	* @adapter: board private structure
1353	**/
1354	static void iavf_clear_fdir_filters(struct iavf_adapter *adapter)
1355	{
1356	struct iavf_fdir_fltr *fdir, *fdirtmp;
1357
1358	/* remove all Flow Director filters */
1359	spin_lock_bh(lock: &adapter->fdir_fltr_lock);
1360	list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head,
1361	list) {
1362	if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) {
1363	list_del(entry: &fdir->list);
1364	kfree(objp: fdir);
1365	adapter->fdir_active_fltr--;
1366	} else {
1367	fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
1368	}
1369	}
1370	spin_unlock_bh(lock: &adapter->fdir_fltr_lock);
1371	}
1372
1373	/**
1374	* iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark
1375	* other to be removed.
1376	* @adapter: board private structure
1377	**/
1378	static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter)
1379	{
1380	struct iavf_adv_rss *rss, *rsstmp;
1381
1382	/* remove all advance RSS configuration */
1383	spin_lock_bh(lock: &adapter->adv_rss_lock);
1384	list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
1385	list) {
1386	if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) {
1387	list_del(entry: &rss->list);
1388	kfree(objp: rss);
1389	} else {
1390	rss->state = IAVF_ADV_RSS_DEL_REQUEST;
1391	}
1392	}
1393	spin_unlock_bh(lock: &adapter->adv_rss_lock);
1394	}
1395
1396	/**
1397	* iavf_down - Shutdown the connection processing
1398	* @adapter: board private structure
1399	*
1400	* Expects to be called while holding crit_lock.
1401	**/
1402	void iavf_down(struct iavf_adapter *adapter)
1403	{
1404	struct net_device *netdev = adapter->netdev;
1405
1406	if (adapter->state <= __IAVF_DOWN_PENDING)
1407	return;
1408
1409	netif_carrier_off(dev: netdev);
1410	netif_tx_disable(dev: netdev);
1411	adapter->link_up = false;
1412	iavf_napi_disable_all(adapter);
1413	iavf_irq_disable(adapter);
1414
1415	iavf_clear_mac_vlan_filters(adapter);
1416	iavf_clear_cloud_filters(adapter);
1417	iavf_clear_fdir_filters(adapter);
1418	iavf_clear_adv_rss_conf(adapter);
1419
1420	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
1421	return;
1422
1423	if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {
1424	/* cancel any current operation */
1425	adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1426	/* Schedule operations to close down the HW. Don't wait
1427	* here for this to complete. The watchdog is still running
1428	* and it will take care of this.
1429	*/
1430	if (!list_empty(head: &adapter->mac_filter_list))
1431	adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1432	if (!list_empty(head: &adapter->vlan_filter_list))
1433	adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
1434	if (!list_empty(head: &adapter->cloud_filter_list))
1435	adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
1436	if (!list_empty(head: &adapter->fdir_list_head))
1437	adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
1438	if (!list_empty(head: &adapter->adv_rss_list_head))
1439	adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
1440	}
1441
1442	adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
1443	mod_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task, delay: 0);
1444	}
1445
1446	/**
1447	* iavf_acquire_msix_vectors - Setup the MSIX capability
1448	* @adapter: board private structure
1449	* @vectors: number of vectors to request
1450	*
1451	* Work with the OS to set up the MSIX vectors needed.
1452	*
1453	* Returns 0 on success, negative on failure
1454	**/
1455	static int
1456	iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
1457	{
1458	int err, vector_threshold;
1459
1460	/* We'll want at least 3 (vector_threshold):
1461	* 0) Other (Admin Queue and link, mostly)
1462	* 1) TxQ[0] Cleanup
1463	* 2) RxQ[0] Cleanup
1464	*/
1465	vector_threshold = MIN_MSIX_COUNT;
1466
1467	/* The more we get, the more we will assign to Tx/Rx Cleanup
1468	* for the separate queues...where Rx Cleanup >= Tx Cleanup.
1469	* Right now, we simply care about how many we'll get; we'll
1470	* set them up later while requesting irq's.
1471	*/
1472	err = pci_enable_msix_range(dev: adapter->pdev, entries: adapter->msix_entries,
1473	minvec: vector_threshold, maxvec: vectors);
1474	if (err < 0) {
1475	dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1476	kfree(objp: adapter->msix_entries);
1477	adapter->msix_entries = NULL;
1478	return err;
1479	}
1480
1481	/* Adjust for only the vectors we'll use, which is minimum
1482	* of max_msix_q_vectors + NONQ_VECS, or the number of
1483	* vectors we were allocated.
1484	*/
1485	adapter->num_msix_vectors = err;
1486	return 0;
1487	}
1488
1489	/**
1490	* iavf_free_queues - Free memory for all rings
1491	* @adapter: board private structure to initialize
1492	*
1493	* Free all of the memory associated with queue pairs.
1494	**/
1495	static void iavf_free_queues(struct iavf_adapter *adapter)
1496	{
1497	if (!adapter->vsi_res)
1498	return;
1499	adapter->num_active_queues = 0;
1500	kfree(objp: adapter->tx_rings);
1501	adapter->tx_rings = NULL;
1502	kfree(objp: adapter->rx_rings);
1503	adapter->rx_rings = NULL;
1504	}
1505
1506	/**
1507	* iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload
1508	* @adapter: board private structure
1509	*
1510	* Based on negotiated capabilities, the VLAN tag needs to be inserted and/or
1511	* stripped in certain descriptor fields. Instead of checking the offload
1512	* capability bits in the hot path, cache the location the ring specific
1513	* flags.
1514	*/
1515	void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter)
1516	{
1517	int i;
1518
1519	for (i = 0; i < adapter->num_active_queues; i++) {
1520	struct iavf_ring *tx_ring = &adapter->tx_rings[i];
1521	struct iavf_ring *rx_ring = &adapter->rx_rings[i];
1522
1523	/* prevent multiple L2TAG bits being set after VFR */
1524	tx_ring->flags &=
1525	~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1526	IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2);
1527	rx_ring->flags &=
1528	~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1529	IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2);
1530
1531	if (VLAN_ALLOWED(adapter)) {
1532	tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1533	rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1534	} else if (VLAN_V2_ALLOWED(adapter)) {
1535	struct virtchnl_vlan_supported_caps *stripping_support;
1536	struct virtchnl_vlan_supported_caps *insertion_support;
1537
1538	stripping_support =
1539	&adapter->vlan_v2_caps.offloads.stripping_support;
1540	insertion_support =
1541	&adapter->vlan_v2_caps.offloads.insertion_support;
1542
1543	if (stripping_support->outer) {
1544	if (stripping_support->outer &
1545	VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1546	rx_ring->flags |=
1547	IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1548	else if (stripping_support->outer &
1549	VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1550	rx_ring->flags |=
1551	IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1552	} else if (stripping_support->inner) {
1553	if (stripping_support->inner &
1554	VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1555	rx_ring->flags |=
1556	IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1557	else if (stripping_support->inner &
1558	VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1559	rx_ring->flags |=
1560	IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1561	}
1562
1563	if (insertion_support->outer) {
1564	if (insertion_support->outer &
1565	VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1566	tx_ring->flags |=
1567	IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1568	else if (insertion_support->outer &
1569	VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1570	tx_ring->flags |=
1571	IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1572	} else if (insertion_support->inner) {
1573	if (insertion_support->inner &
1574	VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1575	tx_ring->flags |=
1576	IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1577	else if (insertion_support->inner &
1578	VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1579	tx_ring->flags |=
1580	IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1581	}
1582	}
1583	}
1584	}
1585
1586	/**
1587	* iavf_alloc_queues - Allocate memory for all rings
1588	* @adapter: board private structure to initialize
1589	*
1590	* We allocate one ring per queue at run-time since we don't know the
1591	* number of queues at compile-time. The polling_netdev array is
1592	* intended for Multiqueue, but should work fine with a single queue.
1593	**/
1594	static int iavf_alloc_queues(struct iavf_adapter *adapter)
1595	{
1596	int i, num_active_queues;
1597
1598	/* If we're in reset reallocating queues we don't actually know yet for
1599	* certain the PF gave us the number of queues we asked for but we'll
1600	* assume it did. Once basic reset is finished we'll confirm once we
1601	* start negotiating config with PF.
1602	*/
1603	if (adapter->num_req_queues)
1604	num_active_queues = adapter->num_req_queues;
1605	else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1606	adapter->num_tc)
1607	num_active_queues = adapter->ch_config.total_qps;
1608	else
1609	num_active_queues = min_t(int,
1610	adapter->vsi_res->num_queue_pairs,
1611	(int)(num_online_cpus()));
1612
1613
1614	adapter->tx_rings = kcalloc(n: num_active_queues,
1615	size: sizeof(struct iavf_ring), GFP_KERNEL);
1616	if (!adapter->tx_rings)
1617	goto err_out;
1618	adapter->rx_rings = kcalloc(n: num_active_queues,
1619	size: sizeof(struct iavf_ring), GFP_KERNEL);
1620	if (!adapter->rx_rings)
1621	goto err_out;
1622
1623	for (i = 0; i < num_active_queues; i++) {
1624	struct iavf_ring *tx_ring;
1625	struct iavf_ring *rx_ring;
1626
1627	tx_ring = &adapter->tx_rings[i];
1628
1629	tx_ring->queue_index = i;
1630	tx_ring->netdev = adapter->netdev;
1631	tx_ring->dev = &adapter->pdev->dev;
1632	tx_ring->count = adapter->tx_desc_count;
1633	tx_ring->itr_setting = IAVF_ITR_TX_DEF;
1634	if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
1635	tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;
1636
1637	rx_ring = &adapter->rx_rings[i];
1638	rx_ring->queue_index = i;
1639	rx_ring->netdev = adapter->netdev;
1640	rx_ring->dev = &adapter->pdev->dev;
1641	rx_ring->count = adapter->rx_desc_count;
1642	rx_ring->itr_setting = IAVF_ITR_RX_DEF;
1643	}
1644
1645	adapter->num_active_queues = num_active_queues;
1646
1647	iavf_set_queue_vlan_tag_loc(adapter);
1648
1649	return 0;
1650
1651	err_out:
1652	iavf_free_queues(adapter);
1653	return -ENOMEM;
1654	}
1655
1656	/**
1657	* iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
1658	* @adapter: board private structure to initialize
1659	*
1660	* Attempt to configure the interrupts using the best available
1661	* capabilities of the hardware and the kernel.
1662	**/
1663	static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
1664	{
1665	int vector, v_budget;
1666	int pairs = 0;
1667	int err = 0;
1668
1669	if (!adapter->vsi_res) {
1670	err = -EIO;
1671	goto out;
1672	}
1673	pairs = adapter->num_active_queues;
1674
1675	/* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1676	* us much good if we have more vectors than CPUs. However, we already
1677	* limit the total number of queues by the number of CPUs so we do not
1678	* need any further limiting here.
1679	*/
1680	v_budget = min_t(int, pairs + NONQ_VECS,
1681	(int)adapter->vf_res->max_vectors);
1682
1683	adapter->msix_entries = kcalloc(n: v_budget,
1684	size: sizeof(struct msix_entry), GFP_KERNEL);
1685	if (!adapter->msix_entries) {
1686	err = -ENOMEM;
1687	goto out;
1688	}
1689
1690	for (vector = 0; vector < v_budget; vector++)
1691	adapter->msix_entries[vector].entry = vector;
1692
1693	err = iavf_acquire_msix_vectors(adapter, vectors: v_budget);
1694	if (!err)
1695	iavf_schedule_finish_config(adapter);
1696
1697	out:
1698	return err;
1699	}
1700
1701	/**
1702	* iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands
1703	* @adapter: board private structure
1704	*
1705	* Return 0 on success, negative on failure
1706	**/
1707	static int iavf_config_rss_aq(struct iavf_adapter *adapter)
1708	{
1709	struct iavf_aqc_get_set_rss_key_data *rss_key =
1710	(struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;
1711	struct iavf_hw *hw = &adapter->hw;
1712	enum iavf_status status;
1713
1714	if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1715	/* bail because we already have a command pending */
1716	dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1717	adapter->current_op);
1718	return -EBUSY;
1719	}
1720
1721	status = iavf_aq_set_rss_key(hw, seid: adapter->vsi.id, key: rss_key);
1722	if (status) {
1723	dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1724	iavf_stat_str(hw, status),
1725	iavf_aq_str(hw, hw->aq.asq_last_status));
1726	return iavf_status_to_errno(status);
1727
1728	}
1729
1730	status = iavf_aq_set_rss_lut(hw, seid: adapter->vsi.id, pf_lut: false,
1731	lut: adapter->rss_lut, lut_size: adapter->rss_lut_size);
1732	if (status) {
1733	dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1734	iavf_stat_str(hw, status),
1735	iavf_aq_str(hw, hw->aq.asq_last_status));
1736	return iavf_status_to_errno(status);
1737	}
1738
1739	return 0;
1740
1741	}
1742
1743	/**
1744	* iavf_config_rss_reg - Configure RSS keys and lut by writing registers
1745	* @adapter: board private structure
1746	*
1747	* Returns 0 on success, negative on failure
1748	**/
1749	static int iavf_config_rss_reg(struct iavf_adapter *adapter)
1750	{
1751	struct iavf_hw *hw = &adapter->hw;
1752	u32 *dw;
1753	u16 i;
1754
1755	dw = (u32 *)adapter->rss_key;
1756	for (i = 0; i <= adapter->rss_key_size / 4; i++)
1757	wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);
1758
1759	dw = (u32 *)adapter->rss_lut;
1760	for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1761	wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);
1762
1763	iavf_flush(hw);
1764
1765	return 0;
1766	}
1767
1768	/**
1769	* iavf_config_rss - Configure RSS keys and lut
1770	* @adapter: board private structure
1771	*
1772	* Returns 0 on success, negative on failure
1773	**/
1774	int iavf_config_rss(struct iavf_adapter *adapter)
1775	{
1776
1777	if (RSS_PF(adapter)) {
1778	adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
1779	IAVF_FLAG_AQ_SET_RSS_KEY;
1780	return 0;
1781	} else if (RSS_AQ(adapter)) {
1782	return iavf_config_rss_aq(adapter);
1783	} else {
1784	return iavf_config_rss_reg(adapter);
1785	}
1786	}
1787
1788	/**
1789	* iavf_fill_rss_lut - Fill the lut with default values
1790	* @adapter: board private structure
1791	**/
1792	static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
1793	{
1794	u16 i;
1795
1796	for (i = 0; i < adapter->rss_lut_size; i++)
1797	adapter->rss_lut[i] = i % adapter->num_active_queues;
1798	}
1799
1800	/**
1801	* iavf_init_rss - Prepare for RSS
1802	* @adapter: board private structure
1803	*
1804	* Return 0 on success, negative on failure
1805	**/
1806	static int iavf_init_rss(struct iavf_adapter *adapter)
1807	{
1808	struct iavf_hw *hw = &adapter->hw;
1809
1810	if (!RSS_PF(adapter)) {
1811	/* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1812	if (adapter->vf_res->vf_cap_flags &
1813	VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1814	adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;
1815	else
1816	adapter->hena = IAVF_DEFAULT_RSS_HENA;
1817
1818	wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);
1819	wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1820	}
1821
1822	iavf_fill_rss_lut(adapter);
1823	netdev_rss_key_fill(buffer: (void *)adapter->rss_key, len: adapter->rss_key_size);
1824
1825	return iavf_config_rss(adapter);
1826	}
1827
1828	/**
1829	* iavf_alloc_q_vectors - Allocate memory for interrupt vectors
1830	* @adapter: board private structure to initialize
1831	*
1832	* We allocate one q_vector per queue interrupt. If allocation fails we
1833	* return -ENOMEM.
1834	**/
1835	static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
1836	{
1837	int q_idx = 0, num_q_vectors;
1838	struct iavf_q_vector *q_vector;
1839
1840	num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1841	adapter->q_vectors = kcalloc(n: num_q_vectors, size: sizeof(*q_vector),
1842	GFP_KERNEL);
1843	if (!adapter->q_vectors)
1844	return -ENOMEM;
1845
1846	for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1847	q_vector = &adapter->q_vectors[q_idx];
1848	q_vector->adapter = adapter;
1849	q_vector->vsi = &adapter->vsi;
1850	q_vector->v_idx = q_idx;
1851	q_vector->reg_idx = q_idx;
1852	cpumask_copy(dstp: &q_vector->affinity_mask, cpu_possible_mask);
1853	netif_napi_add(dev: adapter->netdev, napi: &q_vector->napi,
1854	poll: iavf_napi_poll);
1855	}
1856
1857	return 0;
1858	}
1859
1860	/**
1861	* iavf_free_q_vectors - Free memory allocated for interrupt vectors
1862	* @adapter: board private structure to initialize
1863	*
1864	* This function frees the memory allocated to the q_vectors. In addition if
1865	* NAPI is enabled it will delete any references to the NAPI struct prior
1866	* to freeing the q_vector.
1867	**/
1868	static void iavf_free_q_vectors(struct iavf_adapter *adapter)
1869	{
1870	int q_idx, num_q_vectors;
1871
1872	if (!adapter->q_vectors)
1873	return;
1874
1875	num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1876
1877	for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1878	struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];
1879
1880	netif_napi_del(napi: &q_vector->napi);
1881	}
1882	kfree(objp: adapter->q_vectors);
1883	adapter->q_vectors = NULL;
1884	}
1885
1886	/**
1887	* iavf_reset_interrupt_capability - Reset MSIX setup
1888	* @adapter: board private structure
1889	*
1890	**/
1891	static void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
1892	{
1893	if (!adapter->msix_entries)
1894	return;
1895
1896	pci_disable_msix(dev: adapter->pdev);
1897	kfree(objp: adapter->msix_entries);
1898	adapter->msix_entries = NULL;
1899	}
1900
1901	/**
1902	* iavf_init_interrupt_scheme - Determine if MSIX is supported and init
1903	* @adapter: board private structure to initialize
1904	*
1905	**/
1906	static int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
1907	{
1908	int err;
1909
1910	err = iavf_alloc_queues(adapter);
1911	if (err) {
1912	dev_err(&adapter->pdev->dev,
1913	"Unable to allocate memory for queues\n");
1914	goto err_alloc_queues;
1915	}
1916
1917	err = iavf_set_interrupt_capability(adapter);
1918	if (err) {
1919	dev_err(&adapter->pdev->dev,
1920	"Unable to setup interrupt capabilities\n");
1921	goto err_set_interrupt;
1922	}
1923
1924	err = iavf_alloc_q_vectors(adapter);
1925	if (err) {
1926	dev_err(&adapter->pdev->dev,
1927	"Unable to allocate memory for queue vectors\n");
1928	goto err_alloc_q_vectors;
1929	}
1930
1931	/* If we've made it so far while ADq flag being ON, then we haven't
1932	* bailed out anywhere in middle. And ADq isn't just enabled but actual
1933	* resources have been allocated in the reset path.
1934	* Now we can truly claim that ADq is enabled.
1935	*/
1936	if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1937	adapter->num_tc)
1938	dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1939	adapter->num_tc);
1940
1941	dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1942	(adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1943	adapter->num_active_queues);
1944
1945	return 0;
1946	err_alloc_q_vectors:
1947	iavf_reset_interrupt_capability(adapter);
1948	err_set_interrupt:
1949	iavf_free_queues(adapter);
1950	err_alloc_queues:
1951	return err;
1952	}
1953
1954	/**
1955	* iavf_free_interrupt_scheme - Undo what iavf_init_interrupt_scheme does
1956	* @adapter: board private structure
1957	**/
1958	static void iavf_free_interrupt_scheme(struct iavf_adapter *adapter)
1959	{
1960	iavf_free_q_vectors(adapter);
1961	iavf_reset_interrupt_capability(adapter);
1962	iavf_free_queues(adapter);
1963	}
1964
1965	/**
1966	* iavf_free_rss - Free memory used by RSS structs
1967	* @adapter: board private structure
1968	**/
1969	static void iavf_free_rss(struct iavf_adapter *adapter)
1970	{
1971	kfree(objp: adapter->rss_key);
1972	adapter->rss_key = NULL;
1973
1974	kfree(objp: adapter->rss_lut);
1975	adapter->rss_lut = NULL;
1976	}
1977
1978	/**
1979	* iavf_reinit_interrupt_scheme - Reallocate queues and vectors
1980	* @adapter: board private structure
1981	* @running: true if adapter->state == __IAVF_RUNNING
1982	*
1983	* Returns 0 on success, negative on failure
1984	**/
1985	static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter, bool running)
1986	{
1987	struct net_device *netdev = adapter->netdev;
1988	int err;
1989
1990	if (running)
1991	iavf_free_traffic_irqs(adapter);
1992	iavf_free_misc_irq(adapter);
1993	iavf_free_interrupt_scheme(adapter);
1994
1995	err = iavf_init_interrupt_scheme(adapter);
1996	if (err)
1997	goto err;
1998
1999	netif_tx_stop_all_queues(dev: netdev);
2000
2001	err = iavf_request_misc_irq(adapter);
2002	if (err)
2003	goto err;
2004
2005	set_bit(nr: __IAVF_VSI_DOWN, addr: adapter->vsi.state);
2006
2007	iavf_map_rings_to_vectors(adapter);
2008	err:
2009	return err;
2010	}
2011
2012	/**
2013	* iavf_finish_config - do all netdev work that needs RTNL
2014	* @work: our work_struct
2015	*
2016	* Do work that needs both RTNL and crit_lock.
2017	**/
2018	static void iavf_finish_config(struct work_struct *work)
2019	{
2020	struct iavf_adapter *adapter;
2021	int pairs, err;
2022
2023	adapter = container_of(work, struct iavf_adapter, finish_config);
2024
2025	/* Always take RTNL first to prevent circular lock dependency */
2026	rtnl_lock();
2027	mutex_lock(&adapter->crit_lock);
2028
2029	if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES) &&
2030	adapter->netdev->reg_state == NETREG_REGISTERED &&
2031	!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {
2032	netdev_update_features(dev: adapter->netdev);
2033	adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES;
2034	}
2035
2036	switch (adapter->state) {
2037	case __IAVF_DOWN:
2038	if (adapter->netdev->reg_state != NETREG_REGISTERED) {
2039	err = register_netdevice(dev: adapter->netdev);
2040	if (err) {
2041	dev_err(&adapter->pdev->dev, "Unable to register netdev (%d)\n",
2042	err);
2043
2044	/* go back and try again.*/
2045	iavf_free_rss(adapter);
2046	iavf_free_misc_irq(adapter);
2047	iavf_reset_interrupt_capability(adapter);
2048	iavf_change_state(adapter,
2049	state: __IAVF_INIT_CONFIG_ADAPTER);
2050	goto out;
2051	}
2052	}
2053
2054	/* Set the real number of queues when reset occurs while
2055	* state == __IAVF_DOWN
2056	*/
2057	fallthrough;
2058	case __IAVF_RUNNING:
2059	pairs = adapter->num_active_queues;
2060	netif_set_real_num_rx_queues(dev: adapter->netdev, rxq: pairs);
2061	netif_set_real_num_tx_queues(dev: adapter->netdev, txq: pairs);
2062	break;
2063
2064	default:
2065	break;
2066	}
2067
2068	out:
2069	mutex_unlock(lock: &adapter->crit_lock);
2070	rtnl_unlock();
2071	}
2072
2073	/**
2074	* iavf_schedule_finish_config - Set the flags and schedule a reset event
2075	* @adapter: board private structure
2076	**/
2077	void iavf_schedule_finish_config(struct iavf_adapter *adapter)
2078	{
2079	if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
2080	queue_work(wq: adapter->wq, work: &adapter->finish_config);
2081	}
2082
2083	/**
2084	* iavf_process_aq_command - process aq_required flags
2085	* and sends aq command
2086	* @adapter: pointer to iavf adapter structure
2087	*
2088	* Returns 0 on success
2089	* Returns error code if no command was sent
2090	* or error code if the command failed.
2091	**/
2092	static int iavf_process_aq_command(struct iavf_adapter *adapter)
2093	{
2094	if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)
2095	return iavf_send_vf_config_msg(adapter);
2096	if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS)
2097	return iavf_send_vf_offload_vlan_v2_msg(adapter);
2098	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
2099	iavf_disable_queues(adapter);
2100	return 0;
2101	}
2102
2103	if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
2104	iavf_map_queues(adapter);
2105	return 0;
2106	}
2107
2108	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
2109	iavf_add_ether_addrs(adapter);
2110	return 0;
2111	}
2112
2113	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
2114	iavf_add_vlans(adapter);
2115	return 0;
2116	}
2117
2118	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
2119	iavf_del_ether_addrs(adapter);
2120	return 0;
2121	}
2122
2123	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
2124	iavf_del_vlans(adapter);
2125	return 0;
2126	}
2127
2128	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
2129	iavf_enable_vlan_stripping(adapter);
2130	return 0;
2131	}
2132
2133	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
2134	iavf_disable_vlan_stripping(adapter);
2135	return 0;
2136	}
2137
2138	if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
2139	iavf_configure_queues(adapter);
2140	return 0;
2141	}
2142
2143	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
2144	iavf_enable_queues(adapter);
2145	return 0;
2146	}
2147
2148	if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
2149	/* This message goes straight to the firmware, not the
2150	* PF, so we don't have to set current_op as we will
2151	* not get a response through the ARQ.
2152	*/
2153	adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
2154	return 0;
2155	}
2156	if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
2157	iavf_get_hena(adapter);
2158	return 0;
2159	}
2160	if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
2161	iavf_set_hena(adapter);
2162	return 0;
2163	}
2164	if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
2165	iavf_set_rss_key(adapter);
2166	return 0;
2167	}
2168	if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
2169	iavf_set_rss_lut(adapter);
2170	return 0;
2171	}
2172
2173	if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE) {
2174	iavf_set_promiscuous(adapter);
2175	return 0;
2176	}
2177
2178	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
2179	iavf_enable_channels(adapter);
2180	return 0;
2181	}
2182
2183	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
2184	iavf_disable_channels(adapter);
2185	return 0;
2186	}
2187	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
2188	iavf_add_cloud_filter(adapter);
2189	return 0;
2190	}
2191
2192	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
2193	iavf_del_cloud_filter(adapter);
2194	return 0;
2195	}
2196	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
2197	iavf_del_cloud_filter(adapter);
2198	return 0;
2199	}
2200	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
2201	iavf_add_cloud_filter(adapter);
2202	return 0;
2203	}
2204	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {
2205	iavf_add_fdir_filter(adapter);
2206	return IAVF_SUCCESS;
2207	}
2208	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {
2209	iavf_del_fdir_filter(adapter);
2210	return IAVF_SUCCESS;
2211	}
2212	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {
2213	iavf_add_adv_rss_cfg(adapter);
2214	return 0;
2215	}
2216	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {
2217	iavf_del_adv_rss_cfg(adapter);
2218	return 0;
2219	}
2220	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) {
2221	iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2222	return 0;
2223	}
2224	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) {
2225	iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2226	return 0;
2227	}
2228	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) {
2229	iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2230	return 0;
2231	}
2232	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) {
2233	iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2234	return 0;
2235	}
2236	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) {
2237	iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2238	return 0;
2239	}
2240	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) {
2241	iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2242	return 0;
2243	}
2244	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) {
2245	iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2246	return 0;
2247	}
2248	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) {
2249	iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2250	return 0;
2251	}
2252
2253	if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) {
2254	iavf_request_stats(adapter);
2255	return 0;
2256	}
2257
2258	return -EAGAIN;
2259	}
2260
2261	/**
2262	* iavf_set_vlan_offload_features - set VLAN offload configuration
2263	* @adapter: board private structure
2264	* @prev_features: previous features used for comparison
2265	* @features: updated features used for configuration
2266	*
2267	* Set the aq_required bit(s) based on the requested features passed in to
2268	* configure VLAN stripping and/or VLAN insertion if supported. Also, schedule
2269	* the watchdog if any changes are requested to expedite the request via
2270	* virtchnl.
2271	**/
2272	static void
2273	iavf_set_vlan_offload_features(struct iavf_adapter *adapter,
2274	netdev_features_t prev_features,
2275	netdev_features_t features)
2276	{
2277	bool enable_stripping = true, enable_insertion = true;
2278	u16 vlan_ethertype = 0;
2279	u64 aq_required = 0;
2280
2281	/* keep cases separate because one ethertype for offloads can be
2282	* disabled at the same time as another is disabled, so check for an
2283	* enabled ethertype first, then check for disabled. Default to
2284	* ETH_P_8021Q so an ethertype is specified if disabling insertion and
2285	* stripping.
2286	*/
2287	if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2288	vlan_ethertype = ETH_P_8021AD;
2289	else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2290	vlan_ethertype = ETH_P_8021Q;
2291	else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2292	vlan_ethertype = ETH_P_8021AD;
2293	else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2294	vlan_ethertype = ETH_P_8021Q;
2295	else
2296	vlan_ethertype = ETH_P_8021Q;
2297
2298	if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
2299	enable_stripping = false;
2300	if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
2301	enable_insertion = false;
2302
2303	if (VLAN_ALLOWED(adapter)) {
2304	/* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN
2305	* stripping via virtchnl. VLAN insertion can be toggled on the
2306	* netdev, but it doesn't require a virtchnl message
2307	*/
2308	if (enable_stripping)
2309	aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
2310	else
2311	aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
2312
2313	} else if (VLAN_V2_ALLOWED(adapter)) {
2314	switch (vlan_ethertype) {
2315	case ETH_P_8021Q:
2316	if (enable_stripping)
2317	aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING;
2318	else
2319	aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING;
2320
2321	if (enable_insertion)
2322	aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION;
2323	else
2324	aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION;
2325	break;
2326	case ETH_P_8021AD:
2327	if (enable_stripping)
2328	aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING;
2329	else
2330	aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING;
2331
2332	if (enable_insertion)
2333	aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION;
2334	else
2335	aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION;
2336	break;
2337	}
2338	}
2339
2340	if (aq_required) {
2341	adapter->aq_required |= aq_required;
2342	mod_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task, delay: 0);
2343	}
2344	}
2345
2346	/**
2347	* iavf_startup - first step of driver startup
2348	* @adapter: board private structure
2349	*
2350	* Function process __IAVF_STARTUP driver state.
2351	* When success the state is changed to __IAVF_INIT_VERSION_CHECK
2352	* when fails the state is changed to __IAVF_INIT_FAILED
2353	**/
2354	static void iavf_startup(struct iavf_adapter *adapter)
2355	{
2356	struct pci_dev *pdev = adapter->pdev;
2357	struct iavf_hw *hw = &adapter->hw;
2358	enum iavf_status status;
2359	int ret;
2360
2361	WARN_ON(adapter->state != __IAVF_STARTUP);
2362
2363	/* driver loaded, probe complete */
2364	adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2365	adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2366
2367	ret = iavf_check_reset_complete(hw);
2368	if (ret) {
2369	dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
2370	ret);
2371	goto err;
2372	}
2373	hw->aq.num_arq_entries = IAVF_AQ_LEN;
2374	hw->aq.num_asq_entries = IAVF_AQ_LEN;
2375	hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2376	hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2377
2378	status = iavf_init_adminq(hw);
2379	if (status) {
2380	dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
2381	status);
2382	goto err;
2383	}
2384	ret = iavf_send_api_ver(adapter);
2385	if (ret) {
2386	dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret);
2387	iavf_shutdown_adminq(hw);
2388	goto err;
2389	}
2390	iavf_change_state(adapter, state: __IAVF_INIT_VERSION_CHECK);
2391	return;
2392	err:
2393	iavf_change_state(adapter, state: __IAVF_INIT_FAILED);
2394	}
2395
2396	/**
2397	* iavf_init_version_check - second step of driver startup
2398	* @adapter: board private structure
2399	*
2400	* Function process __IAVF_INIT_VERSION_CHECK driver state.
2401	* When success the state is changed to __IAVF_INIT_GET_RESOURCES
2402	* when fails the state is changed to __IAVF_INIT_FAILED
2403	**/
2404	static void iavf_init_version_check(struct iavf_adapter *adapter)
2405	{
2406	struct pci_dev *pdev = adapter->pdev;
2407	struct iavf_hw *hw = &adapter->hw;
2408	int err = -EAGAIN;
2409
2410	WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);
2411
2412	if (!iavf_asq_done(hw)) {
2413	dev_err(&pdev->dev, "Admin queue command never completed\n");
2414	iavf_shutdown_adminq(hw);
2415	iavf_change_state(adapter, state: __IAVF_STARTUP);
2416	goto err;
2417	}
2418
2419	/* aq msg sent, awaiting reply */
2420	err = iavf_verify_api_ver(adapter);
2421	if (err) {
2422	if (err == -EALREADY)
2423	err = iavf_send_api_ver(adapter);
2424	else
2425	dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
2426	adapter->pf_version.major,
2427	adapter->pf_version.minor,
2428	VIRTCHNL_VERSION_MAJOR,
2429	VIRTCHNL_VERSION_MINOR);
2430	goto err;
2431	}
2432	err = iavf_send_vf_config_msg(adapter);
2433	if (err) {
2434	dev_err(&pdev->dev, "Unable to send config request (%d)\n",
2435	err);
2436	goto err;
2437	}
2438	iavf_change_state(adapter, state: __IAVF_INIT_GET_RESOURCES);
2439	return;
2440	err:
2441	iavf_change_state(adapter, state: __IAVF_INIT_FAILED);
2442	}
2443
2444	/**
2445	* iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES
2446	* @adapter: board private structure
2447	*/
2448	int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter)
2449	{
2450	int i, num_req_queues = adapter->num_req_queues;
2451	struct iavf_vsi *vsi = &adapter->vsi;
2452
2453	for (i = 0; i < adapter->vf_res->num_vsis; i++) {
2454	if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
2455	adapter->vsi_res = &adapter->vf_res->vsi_res[i];
2456	}
2457	if (!adapter->vsi_res) {
2458	dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
2459	return -ENODEV;
2460	}
2461
2462	if (num_req_queues &&
2463	num_req_queues > adapter->vsi_res->num_queue_pairs) {
2464	/* Problem. The PF gave us fewer queues than what we had
2465	* negotiated in our request. Need a reset to see if we can't
2466	* get back to a working state.
2467	*/
2468	dev_err(&adapter->pdev->dev,
2469	"Requested %d queues, but PF only gave us %d.\n",
2470	num_req_queues,
2471	adapter->vsi_res->num_queue_pairs);
2472	adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED;
2473	adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
2474	iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
2475
2476	return -EAGAIN;
2477	}
2478	adapter->num_req_queues = 0;
2479	adapter->vsi.id = adapter->vsi_res->vsi_id;
2480
2481	adapter->vsi.back = adapter;
2482	adapter->vsi.base_vector = 1;
2483	vsi->netdev = adapter->netdev;
2484	vsi->qs_handle = adapter->vsi_res->qset_handle;
2485	if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
2486	adapter->rss_key_size = adapter->vf_res->rss_key_size;
2487	adapter->rss_lut_size = adapter->vf_res->rss_lut_size;
2488	} else {
2489	adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
2490	adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
2491	}
2492
2493	return 0;
2494	}
2495
2496	/**
2497	* iavf_init_get_resources - third step of driver startup
2498	* @adapter: board private structure
2499	*
2500	* Function process __IAVF_INIT_GET_RESOURCES driver state and
2501	* finishes driver initialization procedure.
2502	* When success the state is changed to __IAVF_DOWN
2503	* when fails the state is changed to __IAVF_INIT_FAILED
2504	**/
2505	static void iavf_init_get_resources(struct iavf_adapter *adapter)
2506	{
2507	struct pci_dev *pdev = adapter->pdev;
2508	struct iavf_hw *hw = &adapter->hw;
2509	int err;
2510
2511	WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
2512	/* aq msg sent, awaiting reply */
2513	if (!adapter->vf_res) {
2514	adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
2515	GFP_KERNEL);
2516	if (!adapter->vf_res) {
2517	err = -ENOMEM;
2518	goto err;
2519	}
2520	}
2521	err = iavf_get_vf_config(adapter);
2522	if (err == -EALREADY) {
2523	err = iavf_send_vf_config_msg(adapter);
2524	goto err;
2525	} else if (err == -EINVAL) {
2526	/* We only get -EINVAL if the device is in a very bad
2527	* state or if we've been disabled for previous bad
2528	* behavior. Either way, we're done now.
2529	*/
2530	iavf_shutdown_adminq(hw);
2531	dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
2532	return;
2533	}
2534	if (err) {
2535	dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
2536	goto err_alloc;
2537	}
2538
2539	err = iavf_parse_vf_resource_msg(adapter);
2540	if (err) {
2541	dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n",
2542	err);
2543	goto err_alloc;
2544	}
2545	/* Some features require additional messages to negotiate extended
2546	* capabilities. These are processed in sequence by the
2547	* __IAVF_INIT_EXTENDED_CAPS driver state.
2548	*/
2549	adapter->extended_caps = IAVF_EXTENDED_CAPS;
2550
2551	iavf_change_state(adapter, state: __IAVF_INIT_EXTENDED_CAPS);
2552	return;
2553
2554	err_alloc:
2555	kfree(objp: adapter->vf_res);
2556	adapter->vf_res = NULL;
2557	err:
2558	iavf_change_state(adapter, state: __IAVF_INIT_FAILED);
2559	}
2560
2561	/**
2562	* iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2563	* @adapter: board private structure
2564	*
2565	* Function processes send of the extended VLAN V2 capability message to the
2566	* PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent,
2567	* e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2.
2568	*/
2569	static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2570	{
2571	int ret;
2572
2573	WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2));
2574
2575	ret = iavf_send_vf_offload_vlan_v2_msg(adapter);
2576	if (ret && ret == -EOPNOTSUPP) {
2577	/* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case,
2578	* we did not send the capability exchange message and do not
2579	* expect a response.
2580	*/
2581	adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2582	}
2583
2584	/* We sent the message, so move on to the next step */
2585	adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2586	}
2587
2588	/**
2589	* iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2590	* @adapter: board private structure
2591	*
2592	* Function processes receipt of the extended VLAN V2 capability message from
2593	* the PF.
2594	**/
2595	static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2596	{
2597	int ret;
2598
2599	WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2));
2600
2601	memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps));
2602
2603	ret = iavf_get_vf_vlan_v2_caps(adapter);
2604	if (ret)
2605	goto err;
2606
2607	/* We've processed receipt of the VLAN V2 caps message */
2608	adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2609	return;
2610	err:
2611	/* We didn't receive a reply. Make sure we try sending again when
2612	* __IAVF_INIT_FAILED attempts to recover.
2613	*/
2614	adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2615	iavf_change_state(adapter, state: __IAVF_INIT_FAILED);
2616	}
2617
2618	/**
2619	* iavf_init_process_extended_caps - Part of driver startup
2620	* @adapter: board private structure
2621	*
2622	* Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state
2623	* handles negotiating capabilities for features which require an additional
2624	* message.
2625	*
2626	* Once all extended capabilities exchanges are finished, the driver will
2627	* transition into __IAVF_INIT_CONFIG_ADAPTER.
2628	*/
2629	static void iavf_init_process_extended_caps(struct iavf_adapter *adapter)
2630	{
2631	WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS);
2632
2633	/* Process capability exchange for VLAN V2 */
2634	if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) {
2635	iavf_init_send_offload_vlan_v2_caps(adapter);
2636	return;
2637	} else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) {
2638	iavf_init_recv_offload_vlan_v2_caps(adapter);
2639	return;
2640	}
2641
2642	/* When we reach here, no further extended capabilities exchanges are
2643	* necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER
2644	*/
2645	iavf_change_state(adapter, state: __IAVF_INIT_CONFIG_ADAPTER);
2646	}
2647
2648	/**
2649	* iavf_init_config_adapter - last part of driver startup
2650	* @adapter: board private structure
2651	*
2652	* After all the supported capabilities are negotiated, then the
2653	* __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization.
2654	*/
2655	static void iavf_init_config_adapter(struct iavf_adapter *adapter)
2656	{
2657	struct net_device *netdev = adapter->netdev;
2658	struct pci_dev *pdev = adapter->pdev;
2659	int err;
2660
2661	WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER);
2662
2663	if (iavf_process_config(adapter))
2664	goto err;
2665
2666	adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2667
2668	adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
2669
2670	netdev->netdev_ops = &iavf_netdev_ops;
2671	iavf_set_ethtool_ops(netdev);
2672	netdev->watchdog_timeo = 5 * HZ;
2673
2674	/* MTU range: 68 - 9710 */
2675	netdev->min_mtu = ETH_MIN_MTU;
2676	netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD;
2677
2678	if (!is_valid_ether_addr(addr: adapter->hw.mac.addr)) {
2679	dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
2680	adapter->hw.mac.addr);
2681	eth_hw_addr_random(dev: netdev);
2682	ether_addr_copy(dst: adapter->hw.mac.addr, src: netdev->dev_addr);
2683	} else {
2684	eth_hw_addr_set(dev: netdev, addr: adapter->hw.mac.addr);
2685	ether_addr_copy(dst: netdev->perm_addr, src: adapter->hw.mac.addr);
2686	}
2687
2688	adapter->tx_desc_count = IAVF_DEFAULT_TXD;
2689	adapter->rx_desc_count = IAVF_DEFAULT_RXD;
2690	err = iavf_init_interrupt_scheme(adapter);
2691	if (err)
2692	goto err_sw_init;
2693	iavf_map_rings_to_vectors(adapter);
2694	if (adapter->vf_res->vf_cap_flags &
2695	VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
2696	adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
2697
2698	err = iavf_request_misc_irq(adapter);
2699	if (err)
2700	goto err_sw_init;
2701
2702	netif_carrier_off(dev: netdev);
2703	adapter->link_up = false;
2704	netif_tx_stop_all_queues(dev: netdev);
2705
2706	dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
2707	if (netdev->features & NETIF_F_GRO)
2708	dev_info(&pdev->dev, "GRO is enabled\n");
2709
2710	iavf_change_state(adapter, state: __IAVF_DOWN);
2711	set_bit(nr: __IAVF_VSI_DOWN, addr: adapter->vsi.state);
2712
2713	iavf_misc_irq_enable(adapter);
2714	wake_up(&adapter->down_waitqueue);
2715
2716	adapter->rss_key = kzalloc(size: adapter->rss_key_size, GFP_KERNEL);
2717	adapter->rss_lut = kzalloc(size: adapter->rss_lut_size, GFP_KERNEL);
2718	if (!adapter->rss_key || !adapter->rss_lut) {
2719	err = -ENOMEM;
2720	goto err_mem;
2721	}
2722	if (RSS_AQ(adapter))
2723	adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
2724	else
2725	iavf_init_rss(adapter);
2726
2727	if (VLAN_V2_ALLOWED(adapter))
2728	/* request initial VLAN offload settings */
2729	iavf_set_vlan_offload_features(adapter, prev_features: 0, features: netdev->features);
2730
2731	iavf_schedule_finish_config(adapter);
2732	return;
2733
2734	err_mem:
2735	iavf_free_rss(adapter);
2736	iavf_free_misc_irq(adapter);
2737	err_sw_init:
2738	iavf_reset_interrupt_capability(adapter);
2739	err:
2740	iavf_change_state(adapter, state: __IAVF_INIT_FAILED);
2741	}
2742
2743	/**
2744	* iavf_watchdog_task - Periodic call-back task
2745	* @work: pointer to work_struct
2746	**/
2747	static void iavf_watchdog_task(struct work_struct *work)
2748	{
2749	struct iavf_adapter *adapter = container_of(work,
2750	struct iavf_adapter,
2751	watchdog_task.work);
2752	struct iavf_hw *hw = &adapter->hw;
2753	u32 reg_val;
2754
2755	if (!mutex_trylock(lock: &adapter->crit_lock)) {
2756	if (adapter->state == __IAVF_REMOVE)
2757	return;
2758
2759	goto restart_watchdog;
2760	}
2761
2762	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
2763	iavf_change_state(adapter, state: __IAVF_COMM_FAILED);
2764
2765	switch (adapter->state) {
2766	case __IAVF_STARTUP:
2767	iavf_startup(adapter);
2768	mutex_unlock(lock: &adapter->crit_lock);
2769	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task,
2770	delay: msecs_to_jiffies(m: 30));
2771	return;
2772	case __IAVF_INIT_VERSION_CHECK:
2773	iavf_init_version_check(adapter);
2774	mutex_unlock(lock: &adapter->crit_lock);
2775	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task,
2776	delay: msecs_to_jiffies(m: 30));
2777	return;
2778	case __IAVF_INIT_GET_RESOURCES:
2779	iavf_init_get_resources(adapter);
2780	mutex_unlock(lock: &adapter->crit_lock);
2781	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task,
2782	delay: msecs_to_jiffies(m: 1));
2783	return;
2784	case __IAVF_INIT_EXTENDED_CAPS:
2785	iavf_init_process_extended_caps(adapter);
2786	mutex_unlock(lock: &adapter->crit_lock);
2787	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task,
2788	delay: msecs_to_jiffies(m: 1));
2789	return;
2790	case __IAVF_INIT_CONFIG_ADAPTER:
2791	iavf_init_config_adapter(adapter);
2792	mutex_unlock(lock: &adapter->crit_lock);
2793	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task,
2794	delay: msecs_to_jiffies(m: 1));
2795	return;
2796	case __IAVF_INIT_FAILED:
2797	if (test_bit(__IAVF_IN_REMOVE_TASK,
2798	&adapter->crit_section)) {
2799	/* Do not update the state and do not reschedule
2800	* watchdog task, iavf_remove should handle this state
2801	* as it can loop forever
2802	*/
2803	mutex_unlock(lock: &adapter->crit_lock);
2804	return;
2805	}
2806	if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
2807	dev_err(&adapter->pdev->dev,
2808	"Failed to communicate with PF; waiting before retry\n");
2809	adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2810	iavf_shutdown_adminq(hw);
2811	mutex_unlock(lock: &adapter->crit_lock);
2812	queue_delayed_work(wq: adapter->wq,
2813	dwork: &adapter->watchdog_task, delay: (5 * HZ));
2814	return;
2815	}
2816	/* Try again from failed step*/
2817	iavf_change_state(adapter, state: adapter->last_state);
2818	mutex_unlock(lock: &adapter->crit_lock);
2819	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task, HZ);
2820	return;
2821	case __IAVF_COMM_FAILED:
2822	if (test_bit(__IAVF_IN_REMOVE_TASK,
2823	&adapter->crit_section)) {
2824	/* Set state to __IAVF_INIT_FAILED and perform remove
2825	* steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task
2826	* doesn't bring the state back to __IAVF_COMM_FAILED.
2827	*/
2828	iavf_change_state(adapter, state: __IAVF_INIT_FAILED);
2829	adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2830	mutex_unlock(lock: &adapter->crit_lock);
2831	return;
2832	}
2833	reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
2834	IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
2835	if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
2836	reg_val == VIRTCHNL_VFR_COMPLETED) {
2837	/* A chance for redemption! */
2838	dev_err(&adapter->pdev->dev,
2839	"Hardware came out of reset. Attempting reinit.\n");
2840	/* When init task contacts the PF and
2841	* gets everything set up again, it'll restart the
2842	* watchdog for us. Down, boy. Sit. Stay. Woof.
2843	*/
2844	iavf_change_state(adapter, state: __IAVF_STARTUP);
2845	adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2846	}
2847	adapter->aq_required = 0;
2848	adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2849	mutex_unlock(lock: &adapter->crit_lock);
2850	queue_delayed_work(wq: adapter->wq,
2851	dwork: &adapter->watchdog_task,
2852	delay: msecs_to_jiffies(m: 10));
2853	return;
2854	case __IAVF_RESETTING:
2855	mutex_unlock(lock: &adapter->crit_lock);
2856	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task,
2857	HZ * 2);
2858	return;
2859	case __IAVF_DOWN:
2860	case __IAVF_DOWN_PENDING:
2861	case __IAVF_TESTING:
2862	case __IAVF_RUNNING:
2863	if (adapter->current_op) {
2864	if (!iavf_asq_done(hw)) {
2865	dev_dbg(&adapter->pdev->dev,
2866	"Admin queue timeout\n");
2867	iavf_send_api_ver(adapter);
2868	}
2869	} else {
2870	int ret = iavf_process_aq_command(adapter);
2871
2872	/* An error will be returned if no commands were
2873	* processed; use this opportunity to update stats
2874	* if the error isn't -ENOTSUPP
2875	*/
2876	if (ret && ret != -EOPNOTSUPP &&
2877	adapter->state == __IAVF_RUNNING)
2878	iavf_request_stats(adapter);
2879	}
2880	if (adapter->state == __IAVF_RUNNING)
2881	iavf_detect_recover_hung(vsi: &adapter->vsi);
2882	break;
2883	case __IAVF_REMOVE:
2884	default:
2885	mutex_unlock(lock: &adapter->crit_lock);
2886	return;
2887	}
2888
2889	/* check for hw reset */
2890	reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2891	if (!reg_val) {
2892	adapter->aq_required = 0;
2893	adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2894	dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
2895	iavf_schedule_reset(adapter, IAVF_FLAG_RESET_PENDING);
2896	mutex_unlock(lock: &adapter->crit_lock);
2897	queue_delayed_work(wq: adapter->wq,
2898	dwork: &adapter->watchdog_task, HZ * 2);
2899	return;
2900	}
2901
2902	mutex_unlock(lock: &adapter->crit_lock);
2903	restart_watchdog:
2904	if (adapter->state >= __IAVF_DOWN)
2905	queue_work(wq: adapter->wq, work: &adapter->adminq_task);
2906	if (adapter->aq_required)
2907	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task,
2908	delay: msecs_to_jiffies(m: 20));
2909	else
2910	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task,
2911	HZ * 2);
2912	}
2913
2914	/**
2915	* iavf_disable_vf - disable VF
2916	* @adapter: board private structure
2917	*
2918	* Set communication failed flag and free all resources.
2919	* NOTE: This function is expected to be called with crit_lock being held.
2920	**/
2921	static void iavf_disable_vf(struct iavf_adapter *adapter)
2922	{
2923	struct iavf_mac_filter *f, *ftmp;
2924	struct iavf_vlan_filter *fv, *fvtmp;
2925	struct iavf_cloud_filter *cf, *cftmp;
2926
2927	adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2928
2929	/* We don't use netif_running() because it may be true prior to
2930	* ndo_open() returning, so we can't assume it means all our open
2931	* tasks have finished, since we're not holding the rtnl_lock here.
2932	*/
2933	if (adapter->state == __IAVF_RUNNING) {
2934	set_bit(nr: __IAVF_VSI_DOWN, addr: adapter->vsi.state);
2935	netif_carrier_off(dev: adapter->netdev);
2936	netif_tx_disable(dev: adapter->netdev);
2937	adapter->link_up = false;
2938	iavf_napi_disable_all(adapter);
2939	iavf_irq_disable(adapter);
2940	iavf_free_traffic_irqs(adapter);
2941	iavf_free_all_tx_resources(adapter);
2942	iavf_free_all_rx_resources(adapter);
2943	}
2944
2945	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
2946
2947	/* Delete all of the filters */
2948	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2949	list_del(entry: &f->list);
2950	kfree(objp: f);
2951	}
2952
2953	list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
2954	list_del(entry: &fv->list);
2955	kfree(objp: fv);
2956	}
2957	adapter->num_vlan_filters = 0;
2958
2959	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
2960
2961	spin_lock_bh(lock: &adapter->cloud_filter_list_lock);
2962	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
2963	list_del(entry: &cf->list);
2964	kfree(objp: cf);
2965	adapter->num_cloud_filters--;
2966	}
2967	spin_unlock_bh(lock: &adapter->cloud_filter_list_lock);
2968
2969	iavf_free_misc_irq(adapter);
2970	iavf_free_interrupt_scheme(adapter);
2971	memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
2972	iavf_shutdown_adminq(hw: &adapter->hw);
2973	adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2974	iavf_change_state(adapter, state: __IAVF_DOWN);
2975	wake_up(&adapter->down_waitqueue);
2976	dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
2977	}
2978
2979	/**
2980	* iavf_reset_task - Call-back task to handle hardware reset
2981	* @work: pointer to work_struct
2982	*
2983	* During reset we need to shut down and reinitialize the admin queue
2984	* before we can use it to communicate with the PF again. We also clear
2985	* and reinit the rings because that context is lost as well.
2986	**/
2987	static void iavf_reset_task(struct work_struct *work)
2988	{
2989	struct iavf_adapter *adapter = container_of(work,
2990	struct iavf_adapter,
2991	reset_task);
2992	struct virtchnl_vf_resource *vfres = adapter->vf_res;
2993	struct net_device *netdev = adapter->netdev;
2994	struct iavf_hw *hw = &adapter->hw;
2995	struct iavf_mac_filter *f, *ftmp;
2996	struct iavf_cloud_filter *cf;
2997	enum iavf_status status;
2998	u32 reg_val;
2999	int i = 0, err;
3000	bool running;
3001
3002	/* When device is being removed it doesn't make sense to run the reset
3003	* task, just return in such a case.
3004	*/
3005	if (!mutex_trylock(lock: &adapter->crit_lock)) {
3006	if (adapter->state != __IAVF_REMOVE)
3007	queue_work(wq: adapter->wq, work: &adapter->reset_task);
3008
3009	return;
3010	}
3011
3012	iavf_misc_irq_disable(adapter);
3013	if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
3014	adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
3015	/* Restart the AQ here. If we have been reset but didn't
3016	* detect it, or if the PF had to reinit, our AQ will be hosed.
3017	*/
3018	iavf_shutdown_adminq(hw);
3019	iavf_init_adminq(hw);
3020	iavf_request_reset(adapter);
3021	}
3022	adapter->flags |= IAVF_FLAG_RESET_PENDING;
3023
3024	/* poll until we see the reset actually happen */
3025	for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
3026	reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
3027	IAVF_VF_ARQLEN1_ARQENABLE_MASK;
3028	if (!reg_val)
3029	break;
3030	usleep_range(min: 5000, max: 10000);
3031	}
3032	if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
3033	dev_info(&adapter->pdev->dev, "Never saw reset\n");
3034	goto continue_reset; /* act like the reset happened */
3035	}
3036
3037	/* wait until the reset is complete and the PF is responding to us */
3038	for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
3039	/* sleep first to make sure a minimum wait time is met */
3040	msleep(IAVF_RESET_WAIT_MS);
3041
3042	reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
3043	IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
3044	if (reg_val == VIRTCHNL_VFR_VFACTIVE)
3045	break;
3046	}
3047
3048	pci_set_master(dev: adapter->pdev);
3049	pci_restore_msi_state(dev: adapter->pdev);
3050
3051	if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
3052	dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
3053	reg_val);
3054	iavf_disable_vf(adapter);
3055	mutex_unlock(lock: &adapter->crit_lock);
3056	return; /* Do not attempt to reinit. It's dead, Jim. */
3057	}
3058
3059	continue_reset:
3060	/* We don't use netif_running() because it may be true prior to
3061	* ndo_open() returning, so we can't assume it means all our open
3062	* tasks have finished, since we're not holding the rtnl_lock here.
3063	*/
3064	running = adapter->state == __IAVF_RUNNING;
3065
3066	if (running) {
3067	netif_carrier_off(dev: netdev);
3068	netif_tx_stop_all_queues(dev: netdev);
3069	adapter->link_up = false;
3070	iavf_napi_disable_all(adapter);
3071	}
3072	iavf_irq_disable(adapter);
3073
3074	iavf_change_state(adapter, state: __IAVF_RESETTING);
3075	adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
3076
3077	/* free the Tx/Rx rings and descriptors, might be better to just
3078	* re-use them sometime in the future
3079	*/
3080	iavf_free_all_rx_resources(adapter);
3081	iavf_free_all_tx_resources(adapter);
3082
3083	adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
3084	/* kill and reinit the admin queue */
3085	iavf_shutdown_adminq(hw);
3086	adapter->current_op = VIRTCHNL_OP_UNKNOWN;
3087	status = iavf_init_adminq(hw);
3088	if (status) {
3089	dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
3090	status);
3091	goto reset_err;
3092	}
3093	adapter->aq_required = 0;
3094
3095	if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3096	(adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3097	err = iavf_reinit_interrupt_scheme(adapter, running);
3098	if (err)
3099	goto reset_err;
3100	}
3101
3102	if (RSS_AQ(adapter)) {
3103	adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
3104	} else {
3105	err = iavf_init_rss(adapter);
3106	if (err)
3107	goto reset_err;
3108	}
3109
3110	adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
3111	/* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been
3112	* sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here,
3113	* however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until
3114	* VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have
3115	* been successfully sent and negotiated
3116	*/
3117	adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS;
3118	adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
3119
3120	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
3121
3122	/* Delete filter for the current MAC address, it could have
3123	* been changed by the PF via administratively set MAC.
3124	* Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.
3125	*/
3126	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
3127	if (ether_addr_equal(addr1: f->macaddr, addr2: adapter->hw.mac.addr)) {
3128	list_del(entry: &f->list);
3129	kfree(objp: f);
3130	}
3131	}
3132	/* re-add all MAC filters */
3133	list_for_each_entry(f, &adapter->mac_filter_list, list) {
3134	f->add = true;
3135	}
3136	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
3137
3138	/* check if TCs are running and re-add all cloud filters */
3139	spin_lock_bh(lock: &adapter->cloud_filter_list_lock);
3140	if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
3141	adapter->num_tc) {
3142	list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
3143	cf->add = true;
3144	}
3145	}
3146	spin_unlock_bh(lock: &adapter->cloud_filter_list_lock);
3147
3148	adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
3149	adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
3150	iavf_misc_irq_enable(adapter);
3151
3152	mod_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task, delay: 2);
3153
3154	/* We were running when the reset started, so we need to restore some
3155	* state here.
3156	*/
3157	if (running) {
3158	/* allocate transmit descriptors */
3159	err = iavf_setup_all_tx_resources(adapter);
3160	if (err)
3161	goto reset_err;
3162
3163	/* allocate receive descriptors */
3164	err = iavf_setup_all_rx_resources(adapter);
3165	if (err)
3166	goto reset_err;
3167
3168	if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3169	(adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3170	err = iavf_request_traffic_irqs(adapter, basename: netdev->name);
3171	if (err)
3172	goto reset_err;
3173
3174	adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED;
3175	}
3176
3177	iavf_configure(adapter);
3178
3179	/* iavf_up_complete() will switch device back
3180	* to __IAVF_RUNNING
3181	*/
3182	iavf_up_complete(adapter);
3183
3184	iavf_irq_enable(adapter, flush: true);
3185	} else {
3186	iavf_change_state(adapter, state: __IAVF_DOWN);
3187	wake_up(&adapter->down_waitqueue);
3188	}
3189
3190	adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
3191
3192	wake_up(&adapter->reset_waitqueue);
3193	mutex_unlock(lock: &adapter->crit_lock);
3194
3195	return;
3196	reset_err:
3197	if (running) {
3198	set_bit(nr: __IAVF_VSI_DOWN, addr: adapter->vsi.state);
3199	iavf_free_traffic_irqs(adapter);
3200	}
3201	iavf_disable_vf(adapter);
3202
3203	mutex_unlock(lock: &adapter->crit_lock);
3204	dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
3205	}
3206
3207	/**
3208	* iavf_adminq_task - worker thread to clean the admin queue
3209	* @work: pointer to work_struct containing our data
3210	**/
3211	static void iavf_adminq_task(struct work_struct *work)
3212	{
3213	struct iavf_adapter *adapter =
3214	container_of(work, struct iavf_adapter, adminq_task);
3215	struct iavf_hw *hw = &adapter->hw;
3216	struct iavf_arq_event_info event;
3217	enum virtchnl_ops v_op;
3218	enum iavf_status ret, v_ret;
3219	u32 val, oldval;
3220	u16 pending;
3221
3222	if (!mutex_trylock(lock: &adapter->crit_lock)) {
3223	if (adapter->state == __IAVF_REMOVE)
3224	return;
3225
3226	queue_work(wq: adapter->wq, work: &adapter->adminq_task);
3227	goto out;
3228	}
3229
3230	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
3231	goto unlock;
3232
3233	event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
3234	event.msg_buf = kzalloc(size: event.buf_len, GFP_KERNEL);
3235	if (!event.msg_buf)
3236	goto unlock;
3237
3238	do {
3239	ret = iavf_clean_arq_element(hw, e: &event, events_pending: &pending);
3240	v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
3241	v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);
3242
3243	if (ret || !v_op)
3244	break; /* No event to process or error cleaning ARQ */
3245
3246	iavf_virtchnl_completion(adapter, v_opcode: v_op, v_retval: v_ret, msg: event.msg_buf,
3247	msglen: event.msg_len);
3248	if (pending != 0)
3249	memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
3250	} while (pending);
3251
3252	if (iavf_is_reset_in_progress(adapter))
3253	goto freedom;
3254
3255	/* check for error indications */
3256	val = rd32(hw, hw->aq.arq.len);
3257	if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */
3258	goto freedom;
3259	oldval = val;
3260	if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
3261	dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
3262	val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;
3263	}
3264	if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {
3265	dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
3266	val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;
3267	}
3268	if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {
3269	dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
3270	val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;
3271	}
3272	if (oldval != val)
3273	wr32(hw, hw->aq.arq.len, val);
3274
3275	val = rd32(hw, hw->aq.asq.len);
3276	oldval = val;
3277	if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {
3278	dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
3279	val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;
3280	}
3281	if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {
3282	dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
3283	val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;
3284	}
3285	if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
3286	dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
3287	val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;
3288	}
3289	if (oldval != val)
3290	wr32(hw, hw->aq.asq.len, val);
3291
3292	freedom:
3293	kfree(objp: event.msg_buf);
3294	unlock:
3295	mutex_unlock(lock: &adapter->crit_lock);
3296	out:
3297	/* re-enable Admin queue interrupt cause */
3298	iavf_misc_irq_enable(adapter);
3299	}
3300
3301	/**
3302	* iavf_free_all_tx_resources - Free Tx Resources for All Queues
3303	* @adapter: board private structure
3304	*
3305	* Free all transmit software resources
3306	**/
3307	void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
3308	{
3309	int i;
3310
3311	if (!adapter->tx_rings)
3312	return;
3313
3314	for (i = 0; i < adapter->num_active_queues; i++)
3315	if (adapter->tx_rings[i].desc)
3316	iavf_free_tx_resources(tx_ring: &adapter->tx_rings[i]);
3317	}
3318
3319	/**
3320	* iavf_setup_all_tx_resources - allocate all queues Tx resources
3321	* @adapter: board private structure
3322	*
3323	* If this function returns with an error, then it's possible one or
3324	* more of the rings is populated (while the rest are not). It is the
3325	* callers duty to clean those orphaned rings.
3326	*
3327	* Return 0 on success, negative on failure
3328	**/
3329	static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
3330	{
3331	int i, err = 0;
3332
3333	for (i = 0; i < adapter->num_active_queues; i++) {
3334	adapter->tx_rings[i].count = adapter->tx_desc_count;
3335	err = iavf_setup_tx_descriptors(tx_ring: &adapter->tx_rings[i]);
3336	if (!err)
3337	continue;
3338	dev_err(&adapter->pdev->dev,
3339	"Allocation for Tx Queue %u failed\n", i);
3340	break;
3341	}
3342
3343	return err;
3344	}
3345
3346	/**
3347	* iavf_setup_all_rx_resources - allocate all queues Rx resources
3348	* @adapter: board private structure
3349	*
3350	* If this function returns with an error, then it's possible one or
3351	* more of the rings is populated (while the rest are not). It is the
3352	* callers duty to clean those orphaned rings.
3353	*
3354	* Return 0 on success, negative on failure
3355	**/
3356	static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
3357	{
3358	int i, err = 0;
3359
3360	for (i = 0; i < adapter->num_active_queues; i++) {
3361	adapter->rx_rings[i].count = adapter->rx_desc_count;
3362	err = iavf_setup_rx_descriptors(rx_ring: &adapter->rx_rings[i]);
3363	if (!err)
3364	continue;
3365	dev_err(&adapter->pdev->dev,
3366	"Allocation for Rx Queue %u failed\n", i);
3367	break;
3368	}
3369	return err;
3370	}
3371
3372	/**
3373	* iavf_free_all_rx_resources - Free Rx Resources for All Queues
3374	* @adapter: board private structure
3375	*
3376	* Free all receive software resources
3377	**/
3378	void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
3379	{
3380	int i;
3381
3382	if (!adapter->rx_rings)
3383	return;
3384
3385	for (i = 0; i < adapter->num_active_queues; i++)
3386	if (adapter->rx_rings[i].desc)
3387	iavf_free_rx_resources(rx_ring: &adapter->rx_rings[i]);
3388	}
3389
3390	/**
3391	* iavf_validate_tx_bandwidth - validate the max Tx bandwidth
3392	* @adapter: board private structure
3393	* @max_tx_rate: max Tx bw for a tc
3394	**/
3395	static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
3396	u64 max_tx_rate)
3397	{
3398	int speed = 0, ret = 0;
3399
3400	if (ADV_LINK_SUPPORT(adapter)) {
3401	if (adapter->link_speed_mbps < U32_MAX) {
3402	speed = adapter->link_speed_mbps;
3403	goto validate_bw;
3404	} else {
3405	dev_err(&adapter->pdev->dev, "Unknown link speed\n");
3406	return -EINVAL;
3407	}
3408	}
3409
3410	switch (adapter->link_speed) {
3411	case VIRTCHNL_LINK_SPEED_40GB:
3412	speed = SPEED_40000;
3413	break;
3414	case VIRTCHNL_LINK_SPEED_25GB:
3415	speed = SPEED_25000;
3416	break;
3417	case VIRTCHNL_LINK_SPEED_20GB:
3418	speed = SPEED_20000;
3419	break;
3420	case VIRTCHNL_LINK_SPEED_10GB:
3421	speed = SPEED_10000;
3422	break;
3423	case VIRTCHNL_LINK_SPEED_5GB:
3424	speed = SPEED_5000;
3425	break;
3426	case VIRTCHNL_LINK_SPEED_2_5GB:
3427	speed = SPEED_2500;
3428	break;
3429	case VIRTCHNL_LINK_SPEED_1GB:
3430	speed = SPEED_1000;
3431	break;
3432	case VIRTCHNL_LINK_SPEED_100MB:
3433	speed = SPEED_100;
3434	break;
3435	default:
3436	break;
3437	}
3438
3439	validate_bw:
3440	if (max_tx_rate > speed) {
3441	dev_err(&adapter->pdev->dev,
3442	"Invalid tx rate specified\n");
3443	ret = -EINVAL;
3444	}
3445
3446	return ret;
3447	}
3448
3449	/**
3450	* iavf_validate_ch_config - validate queue mapping info
3451	* @adapter: board private structure
3452	* @mqprio_qopt: queue parameters
3453	*
3454	* This function validates if the config provided by the user to
3455	* configure queue channels is valid or not. Returns 0 on a valid
3456	* config.
3457	**/
3458	static int iavf_validate_ch_config(struct iavf_adapter *adapter,
3459	struct tc_mqprio_qopt_offload *mqprio_qopt)
3460	{
3461	u64 total_max_rate = 0;
3462	u32 tx_rate_rem = 0;
3463	int i, num_qps = 0;
3464	u64 tx_rate = 0;
3465	int ret = 0;
3466
3467	if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
3468	mqprio_qopt->qopt.num_tc < 1)
3469	return -EINVAL;
3470
3471	for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
3472	if (!mqprio_qopt->qopt.count[i] ||
3473	mqprio_qopt->qopt.offset[i] != num_qps)
3474	return -EINVAL;
3475	if (mqprio_qopt->min_rate[i]) {
3476	dev_err(&adapter->pdev->dev,
3477	"Invalid min tx rate (greater than 0) specified for TC%d\n",
3478	i);
3479	return -EINVAL;
3480	}
3481
3482	/* convert to Mbps */
3483	tx_rate = div_u64(dividend: mqprio_qopt->max_rate[i],
3484	IAVF_MBPS_DIVISOR);
3485
3486	if (mqprio_qopt->max_rate[i] &&
3487	tx_rate < IAVF_MBPS_QUANTA) {
3488	dev_err(&adapter->pdev->dev,
3489	"Invalid max tx rate for TC%d, minimum %dMbps\n",
3490	i, IAVF_MBPS_QUANTA);
3491	return -EINVAL;
3492	}
3493
3494	(void)div_u64_rem(dividend: tx_rate, IAVF_MBPS_QUANTA, remainder: &tx_rate_rem);
3495
3496	if (tx_rate_rem != 0) {
3497	dev_err(&adapter->pdev->dev,
3498	"Invalid max tx rate for TC%d, not divisible by %d\n",
3499	i, IAVF_MBPS_QUANTA);
3500	return -EINVAL;
3501	}
3502
3503	total_max_rate += tx_rate;
3504	num_qps += mqprio_qopt->qopt.count[i];
3505	}
3506	if (num_qps > adapter->num_active_queues) {
3507	dev_err(&adapter->pdev->dev,
3508	"Cannot support requested number of queues\n");
3509	return -EINVAL;
3510	}
3511
3512	ret = iavf_validate_tx_bandwidth(adapter, max_tx_rate: total_max_rate);
3513	return ret;
3514	}
3515
3516	/**
3517	* iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes
3518	* @adapter: board private structure
3519	**/
3520	static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
3521	{
3522	struct iavf_cloud_filter *cf, *cftmp;
3523
3524	spin_lock_bh(lock: &adapter->cloud_filter_list_lock);
3525	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
3526	list) {
3527	list_del(entry: &cf->list);
3528	kfree(objp: cf);
3529	adapter->num_cloud_filters--;
3530	}
3531	spin_unlock_bh(lock: &adapter->cloud_filter_list_lock);
3532	}
3533
3534	/**
3535	* __iavf_setup_tc - configure multiple traffic classes
3536	* @netdev: network interface device structure
3537	* @type_data: tc offload data
3538	*
3539	* This function processes the config information provided by the
3540	* user to configure traffic classes/queue channels and packages the
3541	* information to request the PF to setup traffic classes.
3542	*
3543	* Returns 0 on success.
3544	**/
3545	static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
3546	{
3547	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
3548	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
3549	struct virtchnl_vf_resource *vfres = adapter->vf_res;
3550	u8 num_tc = 0, total_qps = 0;
3551	int ret = 0, netdev_tc = 0;
3552	u64 max_tx_rate;
3553	u16 mode;
3554	int i;
3555
3556	num_tc = mqprio_qopt->qopt.num_tc;
3557	mode = mqprio_qopt->mode;
3558
3559	/* delete queue_channel */
3560	if (!mqprio_qopt->qopt.hw) {
3561	if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
3562	/* reset the tc configuration */
3563	netdev_reset_tc(dev: netdev);
3564	adapter->num_tc = 0;
3565	netif_tx_stop_all_queues(dev: netdev);
3566	netif_tx_disable(dev: netdev);
3567	iavf_del_all_cloud_filters(adapter);
3568	adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
3569	total_qps = adapter->orig_num_active_queues;
3570	goto exit;
3571	} else {
3572	return -EINVAL;
3573	}
3574	}
3575
3576	/* add queue channel */
3577	if (mode == TC_MQPRIO_MODE_CHANNEL) {
3578	if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
3579	dev_err(&adapter->pdev->dev, "ADq not supported\n");
3580	return -EOPNOTSUPP;
3581	}
3582	if (adapter->ch_config.state != __IAVF_TC_INVALID) {
3583	dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
3584	return -EINVAL;
3585	}
3586
3587	ret = iavf_validate_ch_config(adapter, mqprio_qopt);
3588	if (ret)
3589	return ret;
3590	/* Return if same TC config is requested */
3591	if (adapter->num_tc == num_tc)
3592	return 0;
3593	adapter->num_tc = num_tc;
3594
3595	for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3596	if (i < num_tc) {
3597	adapter->ch_config.ch_info[i].count =
3598	mqprio_qopt->qopt.count[i];
3599	adapter->ch_config.ch_info[i].offset =
3600	mqprio_qopt->qopt.offset[i];
3601	total_qps += mqprio_qopt->qopt.count[i];
3602	max_tx_rate = mqprio_qopt->max_rate[i];
3603	/* convert to Mbps */
3604	max_tx_rate = div_u64(dividend: max_tx_rate,
3605	IAVF_MBPS_DIVISOR);
3606	adapter->ch_config.ch_info[i].max_tx_rate =
3607	max_tx_rate;
3608	} else {
3609	adapter->ch_config.ch_info[i].count = 1;
3610	adapter->ch_config.ch_info[i].offset = 0;
3611	}
3612	}
3613
3614	/* Take snapshot of original config such as "num_active_queues"
3615	* It is used later when delete ADQ flow is exercised, so that
3616	* once delete ADQ flow completes, VF shall go back to its
3617	* original queue configuration
3618	*/
3619
3620	adapter->orig_num_active_queues = adapter->num_active_queues;
3621
3622	/* Store queue info based on TC so that VF gets configured
3623	* with correct number of queues when VF completes ADQ config
3624	* flow
3625	*/
3626	adapter->ch_config.total_qps = total_qps;
3627
3628	netif_tx_stop_all_queues(dev: netdev);
3629	netif_tx_disable(dev: netdev);
3630	adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
3631	netdev_reset_tc(dev: netdev);
3632	/* Report the tc mapping up the stack */
3633	netdev_set_num_tc(dev: adapter->netdev, num_tc);
3634	for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3635	u16 qcount = mqprio_qopt->qopt.count[i];
3636	u16 qoffset = mqprio_qopt->qopt.offset[i];
3637
3638	if (i < num_tc)
3639	netdev_set_tc_queue(dev: netdev, tc: netdev_tc++, count: qcount,
3640	offset: qoffset);
3641	}
3642	}
3643	exit:
3644	if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
3645	return 0;
3646
3647	netif_set_real_num_rx_queues(dev: netdev, rxq: total_qps);
3648	netif_set_real_num_tx_queues(dev: netdev, txq: total_qps);
3649
3650	return ret;
3651	}
3652
3653	/**
3654	* iavf_parse_cls_flower - Parse tc flower filters provided by kernel
3655	* @adapter: board private structure
3656	* @f: pointer to struct flow_cls_offload
3657	* @filter: pointer to cloud filter structure
3658	*/
3659	static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
3660	struct flow_cls_offload *f,
3661	struct iavf_cloud_filter *filter)
3662	{
3663	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f);
3664	struct flow_dissector *dissector = rule->match.dissector;
3665	u16 n_proto_mask = 0;
3666	u16 n_proto_key = 0;
3667	u8 field_flags = 0;
3668	u16 addr_type = 0;
3669	u16 n_proto = 0;
3670	int i = 0;
3671	struct virtchnl_filter *vf = &filter->f;
3672
3673	if (dissector->used_keys &
3674	~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
3675	BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
3676	BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
3677	BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
3678	BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
3679	BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
3680	BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
3681	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
3682	dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%llx\n",
3683	dissector->used_keys);
3684	return -EOPNOTSUPP;
3685	}
3686
3687	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ENC_KEYID)) {
3688	struct flow_match_enc_keyid match;
3689
3690	flow_rule_match_enc_keyid(rule, out: &match);
3691	if (match.mask->keyid != 0)
3692	field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
3693	}
3694
3695	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_BASIC)) {
3696	struct flow_match_basic match;
3697
3698	flow_rule_match_basic(rule, out: &match);
3699	n_proto_key = ntohs(match.key->n_proto);
3700	n_proto_mask = ntohs(match.mask->n_proto);
3701
3702	if (n_proto_key == ETH_P_ALL) {
3703	n_proto_key = 0;
3704	n_proto_mask = 0;
3705	}
3706	n_proto = n_proto_key & n_proto_mask;
3707	if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
3708	return -EINVAL;
3709	if (n_proto == ETH_P_IPV6) {
3710	/* specify flow type as TCP IPv6 */
3711	vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
3712	}
3713
3714	if (match.key->ip_proto != IPPROTO_TCP) {
3715	dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
3716	return -EINVAL;
3717	}
3718	}
3719
3720	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
3721	struct flow_match_eth_addrs match;
3722
3723	flow_rule_match_eth_addrs(rule, out: &match);
3724
3725	/* use is_broadcast and is_zero to check for all 0xf or 0 */
3726	if (!is_zero_ether_addr(addr: match.mask->dst)) {
3727	if (is_broadcast_ether_addr(addr: match.mask->dst)) {
3728	field_flags |= IAVF_CLOUD_FIELD_OMAC;
3729	} else {
3730	dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
3731	match.mask->dst);
3732	return -EINVAL;
3733	}
3734	}
3735
3736	if (!is_zero_ether_addr(addr: match.mask->src)) {
3737	if (is_broadcast_ether_addr(addr: match.mask->src)) {
3738	field_flags |= IAVF_CLOUD_FIELD_IMAC;
3739	} else {
3740	dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
3741	match.mask->src);
3742	return -EINVAL;
3743	}
3744	}
3745
3746	if (!is_zero_ether_addr(addr: match.key->dst))
3747	if (is_valid_ether_addr(addr: match.key->dst) ||
3748	is_multicast_ether_addr(addr: match.key->dst)) {
3749	/* set the mask if a valid dst_mac address */
3750	for (i = 0; i < ETH_ALEN; i++)
3751	vf->mask.tcp_spec.dst_mac[i] |= 0xff;
3752	ether_addr_copy(dst: vf->data.tcp_spec.dst_mac,
3753	src: match.key->dst);
3754	}
3755
3756	if (!is_zero_ether_addr(addr: match.key->src))
3757	if (is_valid_ether_addr(addr: match.key->src) ||
3758	is_multicast_ether_addr(addr: match.key->src)) {
3759	/* set the mask if a valid dst_mac address */
3760	for (i = 0; i < ETH_ALEN; i++)
3761	vf->mask.tcp_spec.src_mac[i] |= 0xff;
3762	ether_addr_copy(dst: vf->data.tcp_spec.src_mac,
3763	src: match.key->src);
3764	}
3765	}
3766
3767	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_VLAN)) {
3768	struct flow_match_vlan match;
3769
3770	flow_rule_match_vlan(rule, out: &match);
3771	if (match.mask->vlan_id) {
3772	if (match.mask->vlan_id == VLAN_VID_MASK) {
3773	field_flags |= IAVF_CLOUD_FIELD_IVLAN;
3774	} else {
3775	dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
3776	match.mask->vlan_id);
3777	return -EINVAL;
3778	}
3779	}
3780	vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
3781	vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);
3782	}
3783
3784	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_CONTROL)) {
3785	struct flow_match_control match;
3786
3787	flow_rule_match_control(rule, out: &match);
3788	addr_type = match.key->addr_type;
3789	}
3790
3791	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
3792	struct flow_match_ipv4_addrs match;
3793
3794	flow_rule_match_ipv4_addrs(rule, out: &match);
3795	if (match.mask->dst) {
3796	if (match.mask->dst == cpu_to_be32(0xffffffff)) {
3797	field_flags |= IAVF_CLOUD_FIELD_IIP;
3798	} else {
3799	dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
3800	be32_to_cpu(match.mask->dst));
3801	return -EINVAL;
3802	}
3803	}
3804
3805	if (match.mask->src) {
3806	if (match.mask->src == cpu_to_be32(0xffffffff)) {
3807	field_flags |= IAVF_CLOUD_FIELD_IIP;
3808	} else {
3809	dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
3810	be32_to_cpu(match.mask->src));
3811	return -EINVAL;
3812	}
3813	}
3814
3815	if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
3816	dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
3817	return -EINVAL;
3818	}
3819	if (match.key->dst) {
3820	vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
3821	vf->data.tcp_spec.dst_ip[0] = match.key->dst;
3822	}
3823	if (match.key->src) {
3824	vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
3825	vf->data.tcp_spec.src_ip[0] = match.key->src;
3826	}
3827	}
3828
3829	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
3830	struct flow_match_ipv6_addrs match;
3831
3832	flow_rule_match_ipv6_addrs(rule, out: &match);
3833
3834	/* validate mask, make sure it is not IPV6_ADDR_ANY */
3835	if (ipv6_addr_any(a: &match.mask->dst)) {
3836	dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
3837	IPV6_ADDR_ANY);
3838	return -EINVAL;
3839	}
3840
3841	/* src and dest IPv6 address should not be LOOPBACK
3842	* (0:0:0:0:0:0:0:1) which can be represented as ::1
3843	*/
3844	if (ipv6_addr_loopback(a: &match.key->dst) ||
3845	ipv6_addr_loopback(a: &match.key->src)) {
3846	dev_err(&adapter->pdev->dev,
3847	"ipv6 addr should not be loopback\n");
3848	return -EINVAL;
3849	}
3850	if (!ipv6_addr_any(a: &match.mask->dst) ||
3851	!ipv6_addr_any(a: &match.mask->src))
3852	field_flags |= IAVF_CLOUD_FIELD_IIP;
3853
3854	for (i = 0; i < 4; i++)
3855	vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
3856	memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,
3857	sizeof(vf->data.tcp_spec.dst_ip));
3858	for (i = 0; i < 4; i++)
3859	vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
3860	memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,
3861	sizeof(vf->data.tcp_spec.src_ip));
3862	}
3863	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_PORTS)) {
3864	struct flow_match_ports match;
3865
3866	flow_rule_match_ports(rule, out: &match);
3867	if (match.mask->src) {
3868	if (match.mask->src == cpu_to_be16(0xffff)) {
3869	field_flags |= IAVF_CLOUD_FIELD_IIP;
3870	} else {
3871	dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
3872	be16_to_cpu(match.mask->src));
3873	return -EINVAL;
3874	}
3875	}
3876
3877	if (match.mask->dst) {
3878	if (match.mask->dst == cpu_to_be16(0xffff)) {
3879	field_flags |= IAVF_CLOUD_FIELD_IIP;
3880	} else {
3881	dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
3882	be16_to_cpu(match.mask->dst));
3883	return -EINVAL;
3884	}
3885	}
3886	if (match.key->dst) {
3887	vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
3888	vf->data.tcp_spec.dst_port = match.key->dst;
3889	}
3890
3891	if (match.key->src) {
3892	vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
3893	vf->data.tcp_spec.src_port = match.key->src;
3894	}
3895	}
3896	vf->field_flags = field_flags;
3897
3898	return 0;
3899	}
3900
3901	/**
3902	* iavf_handle_tclass - Forward to a traffic class on the device
3903	* @adapter: board private structure
3904	* @tc: traffic class index on the device
3905	* @filter: pointer to cloud filter structure
3906	*/
3907	static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
3908	struct iavf_cloud_filter *filter)
3909	{
3910	if (tc == 0)
3911	return 0;
3912	if (tc < adapter->num_tc) {
3913	if (!filter->f.data.tcp_spec.dst_port) {
3914	dev_err(&adapter->pdev->dev,
3915	"Specify destination port to redirect to traffic class other than TC0\n");
3916	return -EINVAL;
3917	}
3918	}
3919	/* redirect to a traffic class on the same device */
3920	filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
3921	filter->f.action_meta = tc;
3922	return 0;
3923	}
3924
3925	/**
3926	* iavf_find_cf - Find the cloud filter in the list
3927	* @adapter: Board private structure
3928	* @cookie: filter specific cookie
3929	*
3930	* Returns ptr to the filter object or NULL. Must be called while holding the
3931	* cloud_filter_list_lock.
3932	*/
3933	static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
3934	unsigned long *cookie)
3935	{
3936	struct iavf_cloud_filter *filter = NULL;
3937
3938	if (!cookie)
3939	return NULL;
3940
3941	list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
3942	if (!memcmp(p: cookie, q: &filter->cookie, size: sizeof(filter->cookie)))
3943	return filter;
3944	}
3945	return NULL;
3946	}
3947
3948	/**
3949	* iavf_configure_clsflower - Add tc flower filters
3950	* @adapter: board private structure
3951	* @cls_flower: Pointer to struct flow_cls_offload
3952	*/
3953	static int iavf_configure_clsflower(struct iavf_adapter *adapter,
3954	struct flow_cls_offload *cls_flower)
3955	{
3956	int tc = tc_classid_to_hwtc(dev: adapter->netdev, classid: cls_flower->classid);
3957	struct iavf_cloud_filter *filter = NULL;
3958	int err = -EINVAL, count = 50;
3959
3960	if (tc < 0) {
3961	dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
3962	return -EINVAL;
3963	}
3964
3965	filter = kzalloc(size: sizeof(*filter), GFP_KERNEL);
3966	if (!filter)
3967	return -ENOMEM;
3968
3969	while (!mutex_trylock(lock: &adapter->crit_lock)) {
3970	if (--count == 0) {
3971	kfree(objp: filter);
3972	return err;
3973	}
3974	udelay(1);
3975	}
3976
3977	filter->cookie = cls_flower->cookie;
3978
3979	/* bail out here if filter already exists */
3980	spin_lock_bh(lock: &adapter->cloud_filter_list_lock);
3981	if (iavf_find_cf(adapter, cookie: &cls_flower->cookie)) {
3982	dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n");
3983	err = -EEXIST;
3984	goto spin_unlock;
3985	}
3986	spin_unlock_bh(lock: &adapter->cloud_filter_list_lock);
3987
3988	/* set the mask to all zeroes to begin with */
3989	memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
3990	/* start out with flow type and eth type IPv4 to begin with */
3991	filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
3992	err = iavf_parse_cls_flower(adapter, f: cls_flower, filter);
3993	if (err)
3994	goto err;
3995
3996	err = iavf_handle_tclass(adapter, tc, filter);
3997	if (err)
3998	goto err;
3999
4000	/* add filter to the list */
4001	spin_lock_bh(lock: &adapter->cloud_filter_list_lock);
4002	list_add_tail(new: &filter->list, head: &adapter->cloud_filter_list);
4003	adapter->num_cloud_filters++;
4004	filter->add = true;
4005	adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
4006	spin_unlock:
4007	spin_unlock_bh(lock: &adapter->cloud_filter_list_lock);
4008	err:
4009	if (err)
4010	kfree(objp: filter);
4011
4012	mutex_unlock(lock: &adapter->crit_lock);
4013	return err;
4014	}
4015
4016	/**
4017	* iavf_delete_clsflower - Remove tc flower filters
4018	* @adapter: board private structure
4019	* @cls_flower: Pointer to struct flow_cls_offload
4020	*/
4021	static int iavf_delete_clsflower(struct iavf_adapter *adapter,
4022	struct flow_cls_offload *cls_flower)
4023	{
4024	struct iavf_cloud_filter *filter = NULL;
4025	int err = 0;
4026
4027	spin_lock_bh(lock: &adapter->cloud_filter_list_lock);
4028	filter = iavf_find_cf(adapter, cookie: &cls_flower->cookie);
4029	if (filter) {
4030	filter->del = true;
4031	adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
4032	} else {
4033	err = -EINVAL;
4034	}
4035	spin_unlock_bh(lock: &adapter->cloud_filter_list_lock);
4036
4037	return err;
4038	}
4039
4040	/**
4041	* iavf_setup_tc_cls_flower - flower classifier offloads
4042	* @adapter: board private structure
4043	* @cls_flower: pointer to flow_cls_offload struct with flow info
4044	*/
4045	static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
4046	struct flow_cls_offload *cls_flower)
4047	{
4048	switch (cls_flower->command) {
4049	case FLOW_CLS_REPLACE:
4050	return iavf_configure_clsflower(adapter, cls_flower);
4051	case FLOW_CLS_DESTROY:
4052	return iavf_delete_clsflower(adapter, cls_flower);
4053	case FLOW_CLS_STATS:
4054	return -EOPNOTSUPP;
4055	default:
4056	return -EOPNOTSUPP;
4057	}
4058	}
4059
4060	/**
4061	* iavf_setup_tc_block_cb - block callback for tc
4062	* @type: type of offload
4063	* @type_data: offload data
4064	* @cb_priv:
4065	*
4066	* This function is the block callback for traffic classes
4067	**/
4068	static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4069	void *cb_priv)
4070	{
4071	struct iavf_adapter *adapter = cb_priv;
4072
4073	if (!tc_cls_can_offload_and_chain0(dev: adapter->netdev, common: type_data))
4074	return -EOPNOTSUPP;
4075
4076	switch (type) {
4077	case TC_SETUP_CLSFLOWER:
4078	return iavf_setup_tc_cls_flower(adapter: cb_priv, cls_flower: type_data);
4079	default:
4080	return -EOPNOTSUPP;
4081	}
4082	}
4083
4084	static LIST_HEAD(iavf_block_cb_list);
4085
4086	/**
4087	* iavf_setup_tc - configure multiple traffic classes
4088	* @netdev: network interface device structure
4089	* @type: type of offload
4090	* @type_data: tc offload data
4091	*
4092	* This function is the callback to ndo_setup_tc in the
4093	* netdev_ops.
4094	*
4095	* Returns 0 on success
4096	**/
4097	static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
4098	void *type_data)
4099	{
4100	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
4101
4102	switch (type) {
4103	case TC_SETUP_QDISC_MQPRIO:
4104	return __iavf_setup_tc(netdev, type_data);
4105	case TC_SETUP_BLOCK:
4106	return flow_block_cb_setup_simple(f: type_data,
4107	driver_list: &iavf_block_cb_list,
4108	cb: iavf_setup_tc_block_cb,
4109	cb_ident: adapter, cb_priv: adapter, ingress_only: true);
4110	default:
4111	return -EOPNOTSUPP;
4112	}
4113	}
4114
4115	/**
4116	* iavf_open - Called when a network interface is made active
4117	* @netdev: network interface device structure
4118	*
4119	* Returns 0 on success, negative value on failure
4120	*
4121	* The open entry point is called when a network interface is made
4122	* active by the system (IFF_UP). At this point all resources needed
4123	* for transmit and receive operations are allocated, the interrupt
4124	* handler is registered with the OS, the watchdog is started,
4125	* and the stack is notified that the interface is ready.
4126	**/
4127	static int iavf_open(struct net_device *netdev)
4128	{
4129	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
4130	int err;
4131
4132	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
4133	dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
4134	return -EIO;
4135	}
4136
4137	while (!mutex_trylock(lock: &adapter->crit_lock)) {
4138	/* If we are in __IAVF_INIT_CONFIG_ADAPTER state the crit_lock
4139	* is already taken and iavf_open is called from an upper
4140	* device's notifier reacting on NETDEV_REGISTER event.
4141	* We have to leave here to avoid dead lock.
4142	*/
4143	if (adapter->state == __IAVF_INIT_CONFIG_ADAPTER)
4144	return -EBUSY;
4145
4146	usleep_range(min: 500, max: 1000);
4147	}
4148
4149	if (adapter->state != __IAVF_DOWN) {
4150	err = -EBUSY;
4151	goto err_unlock;
4152	}
4153
4154	if (adapter->state == __IAVF_RUNNING &&
4155	!test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) {
4156	dev_dbg(&adapter->pdev->dev, "VF is already open.\n");
4157	err = 0;
4158	goto err_unlock;
4159	}
4160
4161	/* allocate transmit descriptors */
4162	err = iavf_setup_all_tx_resources(adapter);
4163	if (err)
4164	goto err_setup_tx;
4165
4166	/* allocate receive descriptors */
4167	err = iavf_setup_all_rx_resources(adapter);
4168	if (err)
4169	goto err_setup_rx;
4170
4171	/* clear any pending interrupts, may auto mask */
4172	err = iavf_request_traffic_irqs(adapter, basename: netdev->name);
4173	if (err)
4174	goto err_req_irq;
4175
4176	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
4177
4178	iavf_add_filter(adapter, macaddr: adapter->hw.mac.addr);
4179
4180	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
4181
4182	/* Restore VLAN filters that were removed with IFF_DOWN */
4183	iavf_restore_filters(adapter);
4184
4185	iavf_configure(adapter);
4186
4187	iavf_up_complete(adapter);
4188
4189	iavf_irq_enable(adapter, flush: true);
4190
4191	mutex_unlock(lock: &adapter->crit_lock);
4192
4193	return 0;
4194
4195	err_req_irq:
4196	iavf_down(adapter);
4197	iavf_free_traffic_irqs(adapter);
4198	err_setup_rx:
4199	iavf_free_all_rx_resources(adapter);
4200	err_setup_tx:
4201	iavf_free_all_tx_resources(adapter);
4202	err_unlock:
4203	mutex_unlock(lock: &adapter->crit_lock);
4204
4205	return err;
4206	}
4207
4208	/**
4209	* iavf_close - Disables a network interface
4210	* @netdev: network interface device structure
4211	*
4212	* Returns 0, this is not allowed to fail
4213	*
4214	* The close entry point is called when an interface is de-activated
4215	* by the OS. The hardware is still under the drivers control, but
4216	* needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
4217	* are freed, along with all transmit and receive resources.
4218	**/
4219	static int iavf_close(struct net_device *netdev)
4220	{
4221	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
4222	u64 aq_to_restore;
4223	int status;
4224
4225	mutex_lock(&adapter->crit_lock);
4226
4227	if (adapter->state <= __IAVF_DOWN_PENDING) {
4228	mutex_unlock(lock: &adapter->crit_lock);
4229	return 0;
4230	}
4231
4232	set_bit(nr: __IAVF_VSI_DOWN, addr: adapter->vsi.state);
4233	/* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before
4234	* IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl
4235	* deadlock with adminq_task() until iavf_close timeouts. We must send
4236	* IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make
4237	* disable queues possible for vf. Give only necessary flags to
4238	* iavf_down and save other to set them right before iavf_close()
4239	* returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and
4240	* iavf will be in DOWN state.
4241	*/
4242	aq_to_restore = adapter->aq_required;
4243	adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG;
4244
4245	/* Remove flags which we do not want to send after close or we want to
4246	* send before disable queues.
4247	*/
4248	aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG |
4249	IAVF_FLAG_AQ_ENABLE_QUEUES |
4250	IAVF_FLAG_AQ_CONFIGURE_QUEUES |
4251	IAVF_FLAG_AQ_ADD_VLAN_FILTER |
4252	IAVF_FLAG_AQ_ADD_MAC_FILTER |
4253	IAVF_FLAG_AQ_ADD_CLOUD_FILTER |
4254	IAVF_FLAG_AQ_ADD_FDIR_FILTER |
4255	IAVF_FLAG_AQ_ADD_ADV_RSS_CFG);
4256
4257	iavf_down(adapter);
4258	iavf_change_state(adapter, state: __IAVF_DOWN_PENDING);
4259	iavf_free_traffic_irqs(adapter);
4260
4261	mutex_unlock(lock: &adapter->crit_lock);
4262
4263	/* We explicitly don't free resources here because the hardware is
4264	* still active and can DMA into memory. Resources are cleared in
4265	* iavf_virtchnl_completion() after we get confirmation from the PF
4266	* driver that the rings have been stopped.
4267	*
4268	* Also, we wait for state to transition to __IAVF_DOWN before
4269	* returning. State change occurs in iavf_virtchnl_completion() after
4270	* VF resources are released (which occurs after PF driver processes and
4271	* responds to admin queue commands).
4272	*/
4273
4274	status = wait_event_timeout(adapter->down_waitqueue,
4275	adapter->state == __IAVF_DOWN,
4276	msecs_to_jiffies(500));
4277	if (!status)
4278	netdev_warn(dev: netdev, format: "Device resources not yet released\n");
4279
4280	mutex_lock(&adapter->crit_lock);
4281	adapter->aq_required |= aq_to_restore;
4282	mutex_unlock(lock: &adapter->crit_lock);
4283	return 0;
4284	}
4285
4286	/**
4287	* iavf_change_mtu - Change the Maximum Transfer Unit
4288	* @netdev: network interface device structure
4289	* @new_mtu: new value for maximum frame size
4290	*
4291	* Returns 0 on success, negative on failure
4292	**/
4293	static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
4294	{
4295	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
4296	int ret = 0;
4297
4298	netdev_dbg(netdev, "changing MTU from %d to %d\n",
4299	netdev->mtu, new_mtu);
4300	netdev->mtu = new_mtu;
4301
4302	if (netif_running(dev: netdev)) {
4303	iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
4304	ret = iavf_wait_for_reset(adapter);
4305	if (ret < 0)
4306	netdev_warn(dev: netdev, format: "MTU change interrupted waiting for reset");
4307	else if (ret)
4308	netdev_warn(dev: netdev, format: "MTU change timed out waiting for reset");
4309	}
4310
4311	return ret;
4312	}
4313
4314	#define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \
4315	NETIF_F_HW_VLAN_CTAG_TX | \
4316	NETIF_F_HW_VLAN_STAG_RX | \
4317	NETIF_F_HW_VLAN_STAG_TX)
4318
4319	/**
4320	* iavf_set_features - set the netdev feature flags
4321	* @netdev: ptr to the netdev being adjusted
4322	* @features: the feature set that the stack is suggesting
4323	* Note: expects to be called while under rtnl_lock()
4324	**/
4325	static int iavf_set_features(struct net_device *netdev,
4326	netdev_features_t features)
4327	{
4328	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
4329
4330	/* trigger update on any VLAN feature change */
4331	if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^
4332	(features & NETIF_VLAN_OFFLOAD_FEATURES))
4333	iavf_set_vlan_offload_features(adapter, prev_features: netdev->features,
4334	features);
4335	if (CRC_OFFLOAD_ALLOWED(adapter) &&
4336	((netdev->features & NETIF_F_RXFCS) ^ (features & NETIF_F_RXFCS)))
4337	iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
4338
4339	return 0;
4340	}
4341
4342	/**
4343	* iavf_features_check - Validate encapsulated packet conforms to limits
4344	* @skb: skb buff
4345	* @dev: This physical port's netdev
4346	* @features: Offload features that the stack believes apply
4347	**/
4348	static netdev_features_t iavf_features_check(struct sk_buff *skb,
4349	struct net_device *dev,
4350	netdev_features_t features)
4351	{
4352	size_t len;
4353
4354	/* No point in doing any of this if neither checksum nor GSO are
4355	* being requested for this frame. We can rule out both by just
4356	* checking for CHECKSUM_PARTIAL
4357	*/
4358	if (skb->ip_summed != CHECKSUM_PARTIAL)
4359	return features;
4360
4361	/* We cannot support GSO if the MSS is going to be less than
4362	* 64 bytes. If it is then we need to drop support for GSO.
4363	*/
4364	if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
4365	features &= ~NETIF_F_GSO_MASK;
4366
4367	/* MACLEN can support at most 63 words */
4368	len = skb_network_header(skb) - skb->data;
4369	if (len & ~(63 * 2))
4370	goto out_err;
4371
4372	/* IPLEN and EIPLEN can support at most 127 dwords */
4373	len = skb_transport_header(skb) - skb_network_header(skb);
4374	if (len & ~(127 * 4))
4375	goto out_err;
4376
4377	if (skb->encapsulation) {
4378	/* L4TUNLEN can support 127 words */
4379	len = skb_inner_network_header(skb) - skb_transport_header(skb);
4380	if (len & ~(127 * 2))
4381	goto out_err;
4382
4383	/* IPLEN can support at most 127 dwords */
4384	len = skb_inner_transport_header(skb) -
4385	skb_inner_network_header(skb);
4386	if (len & ~(127 * 4))
4387	goto out_err;
4388	}
4389
4390	/* No need to validate L4LEN as TCP is the only protocol with a
4391	* flexible value and we support all possible values supported
4392	* by TCP, which is at most 15 dwords
4393	*/
4394
4395	return features;
4396	out_err:
4397	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4398	}
4399
4400	/**
4401	* iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off
4402	* @adapter: board private structure
4403	*
4404	* Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4405	* were negotiated determine the VLAN features that can be toggled on and off.
4406	**/
4407	static netdev_features_t
4408	iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter)
4409	{
4410	netdev_features_t hw_features = 0;
4411
4412	if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4413	return hw_features;
4414
4415	/* Enable VLAN features if supported */
4416	if (VLAN_ALLOWED(adapter)) {
4417	hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
4418	NETIF_F_HW_VLAN_CTAG_RX);
4419	} else if (VLAN_V2_ALLOWED(adapter)) {
4420	struct virtchnl_vlan_caps *vlan_v2_caps =
4421	&adapter->vlan_v2_caps;
4422	struct virtchnl_vlan_supported_caps *stripping_support =
4423	&vlan_v2_caps->offloads.stripping_support;
4424	struct virtchnl_vlan_supported_caps *insertion_support =
4425	&vlan_v2_caps->offloads.insertion_support;
4426
4427	if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4428	stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4429	if (stripping_support->outer &
4430	VIRTCHNL_VLAN_ETHERTYPE_8100)
4431	hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4432	if (stripping_support->outer &
4433	VIRTCHNL_VLAN_ETHERTYPE_88A8)
4434	hw_features |= NETIF_F_HW_VLAN_STAG_RX;
4435	} else if (stripping_support->inner !=
4436	VIRTCHNL_VLAN_UNSUPPORTED &&
4437	stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4438	if (stripping_support->inner &
4439	VIRTCHNL_VLAN_ETHERTYPE_8100)
4440	hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4441	}
4442
4443	if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4444	insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4445	if (insertion_support->outer &
4446	VIRTCHNL_VLAN_ETHERTYPE_8100)
4447	hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4448	if (insertion_support->outer &
4449	VIRTCHNL_VLAN_ETHERTYPE_88A8)
4450	hw_features |= NETIF_F_HW_VLAN_STAG_TX;
4451	} else if (insertion_support->inner &&
4452	insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4453	if (insertion_support->inner &
4454	VIRTCHNL_VLAN_ETHERTYPE_8100)
4455	hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4456	}
4457	}
4458
4459	if (CRC_OFFLOAD_ALLOWED(adapter))
4460	hw_features |= NETIF_F_RXFCS;
4461
4462	return hw_features;
4463	}
4464
4465	/**
4466	* iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures
4467	* @adapter: board private structure
4468	*
4469	* Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4470	* were negotiated determine the VLAN features that are enabled by default.
4471	**/
4472	static netdev_features_t
4473	iavf_get_netdev_vlan_features(struct iavf_adapter *adapter)
4474	{
4475	netdev_features_t features = 0;
4476
4477	if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4478	return features;
4479
4480	if (VLAN_ALLOWED(adapter)) {
4481	features |= NETIF_F_HW_VLAN_CTAG_FILTER |
4482	NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX;
4483	} else if (VLAN_V2_ALLOWED(adapter)) {
4484	struct virtchnl_vlan_caps *vlan_v2_caps =
4485	&adapter->vlan_v2_caps;
4486	struct virtchnl_vlan_supported_caps *filtering_support =
4487	&vlan_v2_caps->filtering.filtering_support;
4488	struct virtchnl_vlan_supported_caps *stripping_support =
4489	&vlan_v2_caps->offloads.stripping_support;
4490	struct virtchnl_vlan_supported_caps *insertion_support =
4491	&vlan_v2_caps->offloads.insertion_support;
4492	u32 ethertype_init;
4493
4494	/* give priority to outer stripping and don't support both outer
4495	* and inner stripping
4496	*/
4497	ethertype_init = vlan_v2_caps->offloads.ethertype_init;
4498	if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4499	if (stripping_support->outer &
4500	VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4501	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4502	features |= NETIF_F_HW_VLAN_CTAG_RX;
4503	else if (stripping_support->outer &
4504	VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4505	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4506	features |= NETIF_F_HW_VLAN_STAG_RX;
4507	} else if (stripping_support->inner !=
4508	VIRTCHNL_VLAN_UNSUPPORTED) {
4509	if (stripping_support->inner &
4510	VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4511	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4512	features |= NETIF_F_HW_VLAN_CTAG_RX;
4513	}
4514
4515	/* give priority to outer insertion and don't support both outer
4516	* and inner insertion
4517	*/
4518	if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4519	if (insertion_support->outer &
4520	VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4521	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4522	features |= NETIF_F_HW_VLAN_CTAG_TX;
4523	else if (insertion_support->outer &
4524	VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4525	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4526	features |= NETIF_F_HW_VLAN_STAG_TX;
4527	} else if (insertion_support->inner !=
4528	VIRTCHNL_VLAN_UNSUPPORTED) {
4529	if (insertion_support->inner &
4530	VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4531	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4532	features |= NETIF_F_HW_VLAN_CTAG_TX;
4533	}
4534
4535	/* give priority to outer filtering and don't bother if both
4536	* outer and inner filtering are enabled
4537	*/
4538	ethertype_init = vlan_v2_caps->filtering.ethertype_init;
4539	if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4540	if (filtering_support->outer &
4541	VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4542	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4543	features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4544	if (filtering_support->outer &
4545	VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4546	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4547	features |= NETIF_F_HW_VLAN_STAG_FILTER;
4548	} else if (filtering_support->inner !=
4549	VIRTCHNL_VLAN_UNSUPPORTED) {
4550	if (filtering_support->inner &
4551	VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4552	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4553	features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4554	if (filtering_support->inner &
4555	VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4556	ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4557	features |= NETIF_F_HW_VLAN_STAG_FILTER;
4558	}
4559	}
4560
4561	return features;
4562	}
4563
4564	#define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \
4565	(!(((requested) & (feature_bit)) && \
4566	!((allowed) & (feature_bit))))
4567
4568	/**
4569	* iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support
4570	* @adapter: board private structure
4571	* @requested_features: stack requested NETDEV features
4572	**/
4573	static netdev_features_t
4574	iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter,
4575	netdev_features_t requested_features)
4576	{
4577	netdev_features_t allowed_features;
4578
4579	allowed_features = iavf_get_netdev_vlan_hw_features(adapter) |
4580	iavf_get_netdev_vlan_features(adapter);
4581
4582	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4583	allowed_features,
4584	NETIF_F_HW_VLAN_CTAG_TX))
4585	requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX;
4586
4587	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4588	allowed_features,
4589	NETIF_F_HW_VLAN_CTAG_RX))
4590	requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
4591
4592	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4593	allowed_features,
4594	NETIF_F_HW_VLAN_STAG_TX))
4595	requested_features &= ~NETIF_F_HW_VLAN_STAG_TX;
4596	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4597	allowed_features,
4598	NETIF_F_HW_VLAN_STAG_RX))
4599	requested_features &= ~NETIF_F_HW_VLAN_STAG_RX;
4600
4601	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4602	allowed_features,
4603	NETIF_F_HW_VLAN_CTAG_FILTER))
4604	requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
4605
4606	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4607	allowed_features,
4608	NETIF_F_HW_VLAN_STAG_FILTER))
4609	requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER;
4610
4611	if ((requested_features &
4612	(NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
4613	(requested_features &
4614	(NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) &&
4615	adapter->vlan_v2_caps.offloads.ethertype_match ==
4616	VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) {
4617	netdev_warn(dev: adapter->netdev, format: "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
4618	requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX |
4619	NETIF_F_HW_VLAN_STAG_TX);
4620	}
4621
4622	return requested_features;
4623	}
4624
4625	/**
4626	* iavf_fix_strip_features - fix NETDEV CRC and VLAN strip features
4627	* @adapter: board private structure
4628	* @requested_features: stack requested NETDEV features
4629	*
4630	* Returns fixed-up features bits
4631	**/
4632	static netdev_features_t
4633	iavf_fix_strip_features(struct iavf_adapter *adapter,
4634	netdev_features_t requested_features)
4635	{
4636	struct net_device *netdev = adapter->netdev;
4637	bool crc_offload_req, is_vlan_strip;
4638	netdev_features_t vlan_strip;
4639	int num_non_zero_vlan;
4640
4641	crc_offload_req = CRC_OFFLOAD_ALLOWED(adapter) &&
4642	(requested_features & NETIF_F_RXFCS);
4643	num_non_zero_vlan = iavf_get_num_vlans_added(adapter);
4644	vlan_strip = (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX);
4645	is_vlan_strip = requested_features & vlan_strip;
4646
4647	if (!crc_offload_req)
4648	return requested_features;
4649
4650	if (!num_non_zero_vlan && (netdev->features & vlan_strip) &&
4651	!(netdev->features & NETIF_F_RXFCS) && is_vlan_strip) {
4652	requested_features &= ~vlan_strip;
4653	netdev_info(dev: netdev, format: "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n");
4654	return requested_features;
4655	}
4656
4657	if ((netdev->features & NETIF_F_RXFCS) && is_vlan_strip) {
4658	requested_features &= ~vlan_strip;
4659	if (!(netdev->features & vlan_strip))
4660	netdev_info(dev: netdev, format: "To enable VLAN stripping, first need to enable FCS/CRC stripping");
4661
4662	return requested_features;
4663	}
4664
4665	if (num_non_zero_vlan && is_vlan_strip &&
4666	!(netdev->features & NETIF_F_RXFCS)) {
4667	requested_features &= ~NETIF_F_RXFCS;
4668	netdev_info(dev: netdev, format: "To disable FCS/CRC stripping, first need to disable VLAN stripping");
4669	}
4670
4671	return requested_features;
4672	}
4673
4674	/**
4675	* iavf_fix_features - fix up the netdev feature bits
4676	* @netdev: our net device
4677	* @features: desired feature bits
4678	*
4679	* Returns fixed-up features bits
4680	**/
4681	static netdev_features_t iavf_fix_features(struct net_device *netdev,
4682	netdev_features_t features)
4683	{
4684	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
4685
4686	features = iavf_fix_netdev_vlan_features(adapter, requested_features: features);
4687
4688	return iavf_fix_strip_features(adapter, requested_features: features);
4689	}
4690
4691	static const struct net_device_ops iavf_netdev_ops = {
4692	.ndo_open = iavf_open,
4693	.ndo_stop = iavf_close,
4694	.ndo_start_xmit = iavf_xmit_frame,
4695	.ndo_set_rx_mode = iavf_set_rx_mode,
4696	.ndo_validate_addr = eth_validate_addr,
4697	.ndo_set_mac_address = iavf_set_mac,
4698	.ndo_change_mtu = iavf_change_mtu,
4699	.ndo_tx_timeout = iavf_tx_timeout,
4700	.ndo_vlan_rx_add_vid = iavf_vlan_rx_add_vid,
4701	.ndo_vlan_rx_kill_vid = iavf_vlan_rx_kill_vid,
4702	.ndo_features_check = iavf_features_check,
4703	.ndo_fix_features = iavf_fix_features,
4704	.ndo_set_features = iavf_set_features,
4705	.ndo_setup_tc = iavf_setup_tc,
4706	};
4707
4708	/**
4709	* iavf_check_reset_complete - check that VF reset is complete
4710	* @hw: pointer to hw struct
4711	*
4712	* Returns 0 if device is ready to use, or -EBUSY if it's in reset.
4713	**/
4714	static int iavf_check_reset_complete(struct iavf_hw *hw)
4715	{
4716	u32 rstat;
4717	int i;
4718
4719	for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
4720	rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
4721	IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
4722	if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
4723	(rstat == VIRTCHNL_VFR_COMPLETED))
4724	return 0;
4725	msleep(IAVF_RESET_WAIT_MS);
4726	}
4727	return -EBUSY;
4728	}
4729
4730	/**
4731	* iavf_process_config - Process the config information we got from the PF
4732	* @adapter: board private structure
4733	*
4734	* Verify that we have a valid config struct, and set up our netdev features
4735	* and our VSI struct.
4736	**/
4737	int iavf_process_config(struct iavf_adapter *adapter)
4738	{
4739	struct virtchnl_vf_resource *vfres = adapter->vf_res;
4740	netdev_features_t hw_vlan_features, vlan_features;
4741	struct net_device *netdev = adapter->netdev;
4742	netdev_features_t hw_enc_features;
4743	netdev_features_t hw_features;
4744
4745	hw_enc_features = NETIF_F_SG |
4746	NETIF_F_IP_CSUM |
4747	NETIF_F_IPV6_CSUM |
4748	NETIF_F_HIGHDMA |
4749	NETIF_F_SOFT_FEATURES |
4750	NETIF_F_TSO |
4751	NETIF_F_TSO_ECN |
4752	NETIF_F_TSO6 |
4753	NETIF_F_SCTP_CRC |
4754	NETIF_F_RXHASH |
4755	NETIF_F_RXCSUM |
4756	0;
4757
4758	/* advertise to stack only if offloads for encapsulated packets is
4759	* supported
4760	*/
4761	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
4762	hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
4763	NETIF_F_GSO_GRE |
4764	NETIF_F_GSO_GRE_CSUM |
4765	NETIF_F_GSO_IPXIP4 |
4766	NETIF_F_GSO_IPXIP6 |
4767	NETIF_F_GSO_UDP_TUNNEL_CSUM |
4768	NETIF_F_GSO_PARTIAL |
4769	0;
4770
4771	if (!(vfres->vf_cap_flags &
4772	VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
4773	netdev->gso_partial_features |=
4774	NETIF_F_GSO_UDP_TUNNEL_CSUM;
4775
4776	netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
4777	netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
4778	netdev->hw_enc_features |= hw_enc_features;
4779	}
4780	/* record features VLANs can make use of */
4781	netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
4782
4783	/* Write features and hw_features separately to avoid polluting
4784	* with, or dropping, features that are set when we registered.
4785	*/
4786	hw_features = hw_enc_features;
4787
4788	/* get HW VLAN features that can be toggled */
4789	hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter);
4790
4791	/* Enable cloud filter if ADQ is supported */
4792	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
4793	hw_features |= NETIF_F_HW_TC;
4794	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)
4795	hw_features |= NETIF_F_GSO_UDP_L4;
4796
4797	netdev->hw_features |= hw_features | hw_vlan_features;
4798	vlan_features = iavf_get_netdev_vlan_features(adapter);
4799
4800	netdev->features |= hw_features | vlan_features;
4801
4802	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
4803	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4804
4805	netdev->priv_flags |= IFF_UNICAST_FLT;
4806
4807	/* Do not turn on offloads when they are requested to be turned off.
4808	* TSO needs minimum 576 bytes to work correctly.
4809	*/
4810	if (netdev->wanted_features) {
4811	if (!(netdev->wanted_features & NETIF_F_TSO) ||
4812	netdev->mtu < 576)
4813	netdev->features &= ~NETIF_F_TSO;
4814	if (!(netdev->wanted_features & NETIF_F_TSO6) ||
4815	netdev->mtu < 576)
4816	netdev->features &= ~NETIF_F_TSO6;
4817	if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
4818	netdev->features &= ~NETIF_F_TSO_ECN;
4819	if (!(netdev->wanted_features & NETIF_F_GRO))
4820	netdev->features &= ~NETIF_F_GRO;
4821	if (!(netdev->wanted_features & NETIF_F_GSO))
4822	netdev->features &= ~NETIF_F_GSO;
4823	}
4824
4825	return 0;
4826	}
4827
4828	/**
4829	* iavf_shutdown - Shutdown the device in preparation for a reboot
4830	* @pdev: pci device structure
4831	**/
4832	static void iavf_shutdown(struct pci_dev *pdev)
4833	{
4834	struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
4835	struct net_device *netdev = adapter->netdev;
4836
4837	netif_device_detach(dev: netdev);
4838
4839	if (netif_running(dev: netdev))
4840	iavf_close(netdev);
4841
4842	if (iavf_lock_timeout(lock: &adapter->crit_lock, msecs: 5000))
4843	dev_warn(&adapter->pdev->dev, "%s: failed to acquire crit_lock\n", __func__);
4844	/* Prevent the watchdog from running. */
4845	iavf_change_state(adapter, state: __IAVF_REMOVE);
4846	adapter->aq_required = 0;
4847	mutex_unlock(lock: &adapter->crit_lock);
4848
4849	#ifdef CONFIG_PM
4850	pci_save_state(dev: pdev);
4851
4852	#endif
4853	pci_disable_device(dev: pdev);
4854	}
4855
4856	/**
4857	* iavf_probe - Device Initialization Routine
4858	* @pdev: PCI device information struct
4859	* @ent: entry in iavf_pci_tbl
4860	*
4861	* Returns 0 on success, negative on failure
4862	*
4863	* iavf_probe initializes an adapter identified by a pci_dev structure.
4864	* The OS initialization, configuring of the adapter private structure,
4865	* and a hardware reset occur.
4866	**/
4867	static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4868	{
4869	struct net_device *netdev;
4870	struct iavf_adapter *adapter = NULL;
4871	struct iavf_hw *hw = NULL;
4872	int err;
4873
4874	err = pci_enable_device(dev: pdev);
4875	if (err)
4876	return err;
4877
4878	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
4879	if (err) {
4880	dev_err(&pdev->dev,
4881	"DMA configuration failed: 0x%x\n", err);
4882	goto err_dma;
4883	}
4884
4885	err = pci_request_regions(pdev, iavf_driver_name);
4886	if (err) {
4887	dev_err(&pdev->dev,
4888	"pci_request_regions failed 0x%x\n", err);
4889	goto err_pci_reg;
4890	}
4891
4892	pci_set_master(dev: pdev);
4893
4894	netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
4895	IAVF_MAX_REQ_QUEUES);
4896	if (!netdev) {
4897	err = -ENOMEM;
4898	goto err_alloc_etherdev;
4899	}
4900
4901	SET_NETDEV_DEV(netdev, &pdev->dev);
4902
4903	pci_set_drvdata(pdev, data: netdev);
4904	adapter = netdev_priv(dev: netdev);
4905
4906	adapter->netdev = netdev;
4907	adapter->pdev = pdev;
4908
4909	hw = &adapter->hw;
4910	hw->back = adapter;
4911
4912	adapter->wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM,
4913	iavf_driver_name);
4914	if (!adapter->wq) {
4915	err = -ENOMEM;
4916	goto err_alloc_wq;
4917	}
4918
4919	adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
4920	iavf_change_state(adapter, state: __IAVF_STARTUP);
4921
4922	/* Call save state here because it relies on the adapter struct. */
4923	pci_save_state(dev: pdev);
4924
4925	hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
4926	pci_resource_len(pdev, 0));
4927	if (!hw->hw_addr) {
4928	err = -EIO;
4929	goto err_ioremap;
4930	}
4931	hw->vendor_id = pdev->vendor;
4932	hw->device_id = pdev->device;
4933	pci_read_config_byte(dev: pdev, PCI_REVISION_ID, val: &hw->revision_id);
4934	hw->subsystem_vendor_id = pdev->subsystem_vendor;
4935	hw->subsystem_device_id = pdev->subsystem_device;
4936	hw->bus.device = PCI_SLOT(pdev->devfn);
4937	hw->bus.func = PCI_FUNC(pdev->devfn);
4938	hw->bus.bus_id = pdev->bus->number;
4939
4940	/* set up the locks for the AQ, do this only once in probe
4941	* and destroy them only once in remove
4942	*/
4943	mutex_init(&adapter->crit_lock);
4944	mutex_init(&hw->aq.asq_mutex);
4945	mutex_init(&hw->aq.arq_mutex);
4946
4947	spin_lock_init(&adapter->mac_vlan_list_lock);
4948	spin_lock_init(&adapter->cloud_filter_list_lock);
4949	spin_lock_init(&adapter->fdir_fltr_lock);
4950	spin_lock_init(&adapter->adv_rss_lock);
4951	spin_lock_init(&adapter->current_netdev_promisc_flags_lock);
4952
4953	INIT_LIST_HEAD(list: &adapter->mac_filter_list);
4954	INIT_LIST_HEAD(list: &adapter->vlan_filter_list);
4955	INIT_LIST_HEAD(list: &adapter->cloud_filter_list);
4956	INIT_LIST_HEAD(list: &adapter->fdir_list_head);
4957	INIT_LIST_HEAD(list: &adapter->adv_rss_list_head);
4958
4959	INIT_WORK(&adapter->reset_task, iavf_reset_task);
4960	INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
4961	INIT_WORK(&adapter->finish_config, iavf_finish_config);
4962	INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
4963
4964	/* Setup the wait queue for indicating transition to down status */
4965	init_waitqueue_head(&adapter->down_waitqueue);
4966
4967	/* Setup the wait queue for indicating transition to running state */
4968	init_waitqueue_head(&adapter->reset_waitqueue);
4969
4970	/* Setup the wait queue for indicating virtchannel events */
4971	init_waitqueue_head(&adapter->vc_waitqueue);
4972
4973	queue_delayed_work(wq: adapter->wq, dwork: &adapter->watchdog_task,
4974	delay: msecs_to_jiffies(m: 5 * (pdev->devfn & 0x07)));
4975	/* Initialization goes on in the work. Do not add more of it below. */
4976	return 0;
4977
4978	err_ioremap:
4979	destroy_workqueue(wq: adapter->wq);
4980	err_alloc_wq:
4981	free_netdev(dev: netdev);
4982	err_alloc_etherdev:
4983	pci_release_regions(pdev);
4984	err_pci_reg:
4985	err_dma:
4986	pci_disable_device(dev: pdev);
4987	return err;
4988	}
4989
4990	/**
4991	* iavf_suspend - Power management suspend routine
4992	* @dev_d: device info pointer
4993	*
4994	* Called when the system (VM) is entering sleep/suspend.
4995	**/
4996	static int __maybe_unused iavf_suspend(struct device *dev_d)
4997	{
4998	struct net_device *netdev = dev_get_drvdata(dev: dev_d);
4999	struct iavf_adapter *adapter = netdev_priv(dev: netdev);
5000
5001	netif_device_detach(dev: netdev);
5002
5003	mutex_lock(&adapter->crit_lock);
5004
5005	if (netif_running(dev: netdev)) {
5006	rtnl_lock();
5007	iavf_down(adapter);
5008	rtnl_unlock();
5009	}
5010	iavf_free_misc_irq(adapter);
5011	iavf_reset_interrupt_capability(adapter);
5012
5013	mutex_unlock(lock: &adapter->crit_lock);
5014
5015	return 0;
5016	}
5017
5018	/**
5019	* iavf_resume - Power management resume routine
5020	* @dev_d: device info pointer
5021	*
5022	* Called when the system (VM) is resumed from sleep/suspend.
5023	**/
5024	static int __maybe_unused iavf_resume(struct device *dev_d)
5025	{
5026	struct pci_dev *pdev = to_pci_dev(dev_d);
5027	struct iavf_adapter *adapter;
5028	u32 err;
5029
5030	adapter = iavf_pdev_to_adapter(pdev);
5031
5032	pci_set_master(dev: pdev);
5033
5034	rtnl_lock();
5035	err = iavf_set_interrupt_capability(adapter);
5036	if (err) {
5037	rtnl_unlock();
5038	dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
5039	return err;
5040	}
5041	err = iavf_request_misc_irq(adapter);
5042	rtnl_unlock();
5043	if (err) {
5044	dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
5045	return err;
5046	}
5047
5048	queue_work(wq: adapter->wq, work: &adapter->reset_task);
5049
5050	netif_device_attach(dev: adapter->netdev);
5051
5052	return err;
5053	}
5054
5055	/**
5056	* iavf_remove - Device Removal Routine
5057	* @pdev: PCI device information struct
5058	*
5059	* iavf_remove is called by the PCI subsystem to alert the driver
5060	* that it should release a PCI device. The could be caused by a
5061	* Hot-Plug event, or because the driver is going to be removed from
5062	* memory.
5063	**/
5064	static void iavf_remove(struct pci_dev *pdev)
5065	{
5066	struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
5067	struct iavf_fdir_fltr *fdir, *fdirtmp;
5068	struct iavf_vlan_filter *vlf, *vlftmp;
5069	struct iavf_cloud_filter *cf, *cftmp;
5070	struct iavf_adv_rss *rss, *rsstmp;
5071	struct iavf_mac_filter *f, *ftmp;
5072	struct net_device *netdev;
5073	struct iavf_hw *hw;
5074
5075	netdev = adapter->netdev;
5076	hw = &adapter->hw;
5077
5078	if (test_and_set_bit(nr: __IAVF_IN_REMOVE_TASK, addr: &adapter->crit_section))
5079	return;
5080
5081	/* Wait until port initialization is complete.
5082	* There are flows where register/unregister netdev may race.
5083	*/
5084	while (1) {
5085	mutex_lock(&adapter->crit_lock);
5086	if (adapter->state == __IAVF_RUNNING ||
5087	adapter->state == __IAVF_DOWN ||
5088	adapter->state == __IAVF_INIT_FAILED) {
5089	mutex_unlock(lock: &adapter->crit_lock);
5090	break;
5091	}
5092	/* Simply return if we already went through iavf_shutdown */
5093	if (adapter->state == __IAVF_REMOVE) {
5094	mutex_unlock(lock: &adapter->crit_lock);
5095	return;
5096	}
5097
5098	mutex_unlock(lock: &adapter->crit_lock);
5099	usleep_range(min: 500, max: 1000);
5100	}
5101	cancel_delayed_work_sync(dwork: &adapter->watchdog_task);
5102	cancel_work_sync(work: &adapter->finish_config);
5103
5104	if (netdev->reg_state == NETREG_REGISTERED)
5105	unregister_netdev(dev: netdev);
5106
5107	mutex_lock(&adapter->crit_lock);
5108	dev_info(&adapter->pdev->dev, "Removing device\n");
5109	iavf_change_state(adapter, state: __IAVF_REMOVE);
5110
5111	iavf_request_reset(adapter);
5112	msleep(msecs: 50);
5113	/* If the FW isn't responding, kick it once, but only once. */
5114	if (!iavf_asq_done(hw)) {
5115	iavf_request_reset(adapter);
5116	msleep(msecs: 50);
5117	}
5118
5119	iavf_misc_irq_disable(adapter);
5120	/* Shut down all the garbage mashers on the detention level */
5121	cancel_work_sync(work: &adapter->reset_task);
5122	cancel_delayed_work_sync(dwork: &adapter->watchdog_task);
5123	cancel_work_sync(work: &adapter->adminq_task);
5124
5125	adapter->aq_required = 0;
5126	adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
5127
5128	iavf_free_all_tx_resources(adapter);
5129	iavf_free_all_rx_resources(adapter);
5130	iavf_free_misc_irq(adapter);
5131	iavf_free_interrupt_scheme(adapter);
5132
5133	iavf_free_rss(adapter);
5134
5135	if (hw->aq.asq.count)
5136	iavf_shutdown_adminq(hw);
5137
5138	/* destroy the locks only once, here */
5139	mutex_destroy(lock: &hw->aq.arq_mutex);
5140	mutex_destroy(lock: &hw->aq.asq_mutex);
5141	mutex_unlock(lock: &adapter->crit_lock);
5142	mutex_destroy(lock: &adapter->crit_lock);
5143
5144	iounmap(addr: hw->hw_addr);
5145	pci_release_regions(pdev);
5146	kfree(objp: adapter->vf_res);
5147	spin_lock_bh(lock: &adapter->mac_vlan_list_lock);
5148	/* If we got removed before an up/down sequence, we've got a filter
5149	* hanging out there that we need to get rid of.
5150	*/
5151	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
5152	list_del(entry: &f->list);
5153	kfree(objp: f);
5154	}
5155	list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
5156	list) {
5157	list_del(entry: &vlf->list);
5158	kfree(objp: vlf);
5159	}
5160
5161	spin_unlock_bh(lock: &adapter->mac_vlan_list_lock);
5162
5163	spin_lock_bh(lock: &adapter->cloud_filter_list_lock);
5164	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
5165	list_del(entry: &cf->list);
5166	kfree(objp: cf);
5167	}
5168	spin_unlock_bh(lock: &adapter->cloud_filter_list_lock);
5169
5170	spin_lock_bh(lock: &adapter->fdir_fltr_lock);
5171	list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {
5172	list_del(entry: &fdir->list);
5173	kfree(objp: fdir);
5174	}
5175	spin_unlock_bh(lock: &adapter->fdir_fltr_lock);
5176
5177	spin_lock_bh(lock: &adapter->adv_rss_lock);
5178	list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
5179	list) {
5180	list_del(entry: &rss->list);
5181	kfree(objp: rss);
5182	}
5183	spin_unlock_bh(lock: &adapter->adv_rss_lock);
5184
5185	destroy_workqueue(wq: adapter->wq);
5186
5187	free_netdev(dev: netdev);
5188
5189	pci_disable_device(dev: pdev);
5190	}
5191
5192	static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);
5193
5194	static struct pci_driver iavf_driver = {
5195	.name = iavf_driver_name,
5196	.id_table = iavf_pci_tbl,
5197	.probe = iavf_probe,
5198	.remove = iavf_remove,
5199	.driver.pm = &iavf_pm_ops,
5200	.shutdown = iavf_shutdown,
5201	};
5202
5203	/**
5204	* iavf_init_module - Driver Registration Routine
5205	*
5206	* iavf_init_module is the first routine called when the driver is
5207	* loaded. All it does is register with the PCI subsystem.
5208	**/
5209	static int __init iavf_init_module(void)
5210	{
5211	pr_info("iavf: %s\n", iavf_driver_string);
5212
5213	pr_info("%s\n", iavf_copyright);
5214
5215	return pci_register_driver(&iavf_driver);
5216	}
5217
5218	module_init(iavf_init_module);
5219
5220	/**
5221	* iavf_exit_module - Driver Exit Cleanup Routine
5222	*
5223	* iavf_exit_module is called just before the driver is removed
5224	* from memory.
5225	**/
5226	static void __exit iavf_exit_module(void)
5227	{
5228	pci_unregister_driver(dev: &iavf_driver);
5229	}
5230
5231	module_exit(iavf_exit_module);
5232
5233	/* iavf_main.c */
5234