1	// SPDX-License-Identifier: GPL-2.0
2	/* Marvell Octeon EP (EndPoint) Ethernet Driver
3	*
4	* Copyright (C) 2020 Marvell.
5	*
6	*/
7
8	#include <linux/types.h>
9	#include <linux/module.h>
10	#include <linux/pci.h>
11	#include <linux/netdevice.h>
12	#include <linux/etherdevice.h>
13	#include <linux/rtnetlink.h>
14	#include <linux/vmalloc.h>
15
16	#include "octep_config.h"
17	#include "octep_main.h"
18	#include "octep_ctrl_net.h"
19	#include "octep_pfvf_mbox.h"
20
21	#define OCTEP_INTR_POLL_TIME_MSECS 100
22	struct workqueue_struct *octep_wq;
23
24	/* Supported Devices */
25	static const struct pci_device_id octep_pci_id_tbl[] = {
26	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN98_PF)},
27	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)},
28	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_PF)},
29	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KA_PF)},
30	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KA_PF)},
31	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KB_PF)},
32	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KB_PF)},
33	{0, },
34	};
35	MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl);
36
37	MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
38	MODULE_DESCRIPTION(OCTEP_DRV_STRING);
39	MODULE_LICENSE("GPL");
40
41	/**
42	* octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
43	*
44	* @oct: Octeon device private data structure.
45	*
46	* Allocate resources to hold per Tx/Rx queue interrupt info.
47	* This is the information passed to interrupt handler, from which napi poll
48	* is scheduled and includes quick access to private data of Tx/Rx queue
49	* corresponding to the interrupt being handled.
50	*
51	* Return: 0, on successful allocation of resources for all queue interrupts.
52	* -1, if failed to allocate any resource.
53	*/
54	static int octep_alloc_ioq_vectors(struct octep_device *oct)
55	{
56	int i;
57	struct octep_ioq_vector *ioq_vector;
58
59	for (i = 0; i < oct->num_oqs; i++) {
60	oct->ioq_vector[i] = vzalloc(size: sizeof(*oct->ioq_vector[i]));
61	if (!oct->ioq_vector[i])
62	goto free_ioq_vector;
63
64	ioq_vector = oct->ioq_vector[i];
65	ioq_vector->iq = oct->iq[i];
66	ioq_vector->oq = oct->oq[i];
67	ioq_vector->octep_dev = oct;
68	}
69
70	dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
71	return 0;
72
73	free_ioq_vector:
74	while (i) {
75	i--;
76	vfree(addr: oct->ioq_vector[i]);
77	oct->ioq_vector[i] = NULL;
78	}
79	return -1;
80	}
81
82	/**
83	* octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
84	*
85	* @oct: Octeon device private data structure.
86	*/
87	static void octep_free_ioq_vectors(struct octep_device *oct)
88	{
89	int i;
90
91	for (i = 0; i < oct->num_oqs; i++) {
92	if (oct->ioq_vector[i]) {
93	vfree(addr: oct->ioq_vector[i]);
94	oct->ioq_vector[i] = NULL;
95	}
96	}
97	netdev_info(dev: oct->netdev, format: "Freed IOQ Vectors\n");
98	}
99
100	/**
101	* octep_enable_msix_range() - enable MSI-x interrupts.
102	*
103	* @oct: Octeon device private data structure.
104	*
105	* Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
106	* for the Octeon device.
107	*
108	* Return: 0, on successfully enabling all MSI-x interrupts.
109	* -1, if failed to enable any MSI-x interrupt.
110	*/
111	static int octep_enable_msix_range(struct octep_device *oct)
112	{
113	int num_msix, msix_allocated;
114	int i;
115
116	/* Generic interrupts apart from input/output queues */
117	num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
118	oct->msix_entries = kcalloc(n: num_msix,
119	size: sizeof(struct msix_entry), GFP_KERNEL);
120	if (!oct->msix_entries)
121	goto msix_alloc_err;
122
123	for (i = 0; i < num_msix; i++)
124	oct->msix_entries[i].entry = i;
125
126	msix_allocated = pci_enable_msix_range(dev: oct->pdev, entries: oct->msix_entries,
127	minvec: num_msix, maxvec: num_msix);
128	if (msix_allocated != num_msix) {
129	dev_err(&oct->pdev->dev,
130	"Failed to enable %d msix irqs; got only %d\n",
131	num_msix, msix_allocated);
132	goto enable_msix_err;
133	}
134	oct->num_irqs = msix_allocated;
135	dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
136
137	return 0;
138
139	enable_msix_err:
140	if (msix_allocated > 0)
141	pci_disable_msix(dev: oct->pdev);
142	kfree(objp: oct->msix_entries);
143	oct->msix_entries = NULL;
144	msix_alloc_err:
145	return -1;
146	}
147
148	/**
149	* octep_disable_msix() - disable MSI-x interrupts.
150	*
151	* @oct: Octeon device private data structure.
152	*
153	* Disable MSI-x on the Octeon device.
154	*/
155	static void octep_disable_msix(struct octep_device *oct)
156	{
157	pci_disable_msix(dev: oct->pdev);
158	kfree(objp: oct->msix_entries);
159	oct->msix_entries = NULL;
160	dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
161	}
162
163	/**
164	* octep_mbox_intr_handler() - common handler for pfvf mbox interrupts.
165	*
166	* @irq: Interrupt number.
167	* @data: interrupt data.
168	*
169	* this is common handler for pfvf mbox interrupts.
170	*/
171	static irqreturn_t octep_mbox_intr_handler(int irq, void *data)
172	{
173	struct octep_device *oct = data;
174
175	return oct->hw_ops.mbox_intr_handler(oct);
176	}
177
178	/**
179	* octep_oei_intr_handler() - common handler for output endpoint interrupts.
180	*
181	* @irq: Interrupt number.
182	* @data: interrupt data.
183	*
184	* this is common handler for all output endpoint interrupts.
185	*/
186	static irqreturn_t octep_oei_intr_handler(int irq, void *data)
187	{
188	struct octep_device *oct = data;
189
190	return oct->hw_ops.oei_intr_handler(oct);
191	}
192
193	/**
194	* octep_ire_intr_handler() - common handler for input ring error interrupts.
195	*
196	* @irq: Interrupt number.
197	* @data: interrupt data.
198	*
199	* this is common handler for input ring error interrupts.
200	*/
201	static irqreturn_t octep_ire_intr_handler(int irq, void *data)
202	{
203	struct octep_device *oct = data;
204
205	return oct->hw_ops.ire_intr_handler(oct);
206	}
207
208	/**
209	* octep_ore_intr_handler() - common handler for output ring error interrupts.
210	*
211	* @irq: Interrupt number.
212	* @data: interrupt data.
213	*
214	* this is common handler for output ring error interrupts.
215	*/
216	static irqreturn_t octep_ore_intr_handler(int irq, void *data)
217	{
218	struct octep_device *oct = data;
219
220	return oct->hw_ops.ore_intr_handler(oct);
221	}
222
223	/**
224	* octep_vfire_intr_handler() - common handler for vf input ring error interrupts.
225	*
226	* @irq: Interrupt number.
227	* @data: interrupt data.
228	*
229	* this is common handler for vf input ring error interrupts.
230	*/
231	static irqreturn_t octep_vfire_intr_handler(int irq, void *data)
232	{
233	struct octep_device *oct = data;
234
235	return oct->hw_ops.vfire_intr_handler(oct);
236	}
237
238	/**
239	* octep_vfore_intr_handler() - common handler for vf output ring error interrupts.
240	*
241	* @irq: Interrupt number.
242	* @data: interrupt data.
243	*
244	* this is common handler for vf output ring error interrupts.
245	*/
246	static irqreturn_t octep_vfore_intr_handler(int irq, void *data)
247	{
248	struct octep_device *oct = data;
249
250	return oct->hw_ops.vfore_intr_handler(oct);
251	}
252
253	/**
254	* octep_dma_intr_handler() - common handler for dpi dma related interrupts.
255	*
256	* @irq: Interrupt number.
257	* @data: interrupt data.
258	*
259	* this is common handler for dpi dma related interrupts.
260	*/
261	static irqreturn_t octep_dma_intr_handler(int irq, void *data)
262	{
263	struct octep_device *oct = data;
264
265	return oct->hw_ops.dma_intr_handler(oct);
266	}
267
268	/**
269	* octep_dma_vf_intr_handler() - common handler for dpi dma transaction error interrupts for VFs.
270	*
271	* @irq: Interrupt number.
272	* @data: interrupt data.
273	*
274	* this is common handler for dpi dma transaction error interrupts for VFs.
275	*/
276	static irqreturn_t octep_dma_vf_intr_handler(int irq, void *data)
277	{
278	struct octep_device *oct = data;
279
280	return oct->hw_ops.dma_vf_intr_handler(oct);
281	}
282
283	/**
284	* octep_pp_vf_intr_handler() - common handler for pp transaction error interrupts for VFs.
285	*
286	* @irq: Interrupt number.
287	* @data: interrupt data.
288	*
289	* this is common handler for pp transaction error interrupts for VFs.
290	*/
291	static irqreturn_t octep_pp_vf_intr_handler(int irq, void *data)
292	{
293	struct octep_device *oct = data;
294
295	return oct->hw_ops.pp_vf_intr_handler(oct);
296	}
297
298	/**
299	* octep_misc_intr_handler() - common handler for mac related interrupts.
300	*
301	* @irq: Interrupt number.
302	* @data: interrupt data.
303	*
304	* this is common handler for mac related interrupts.
305	*/
306	static irqreturn_t octep_misc_intr_handler(int irq, void *data)
307	{
308	struct octep_device *oct = data;
309
310	return oct->hw_ops.misc_intr_handler(oct);
311	}
312
313	/**
314	* octep_rsvd_intr_handler() - common handler for reserved interrupts (future use).
315	*
316	* @irq: Interrupt number.
317	* @data: interrupt data.
318	*
319	* this is common handler for all reserved interrupts.
320	*/
321	static irqreturn_t octep_rsvd_intr_handler(int irq, void *data)
322	{
323	struct octep_device *oct = data;
324
325	return oct->hw_ops.rsvd_intr_handler(oct);
326	}
327
328	/**
329	* octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
330	*
331	* @irq: Interrupt number.
332	* @data: interrupt data contains pointers to Tx/Rx queue private data
333	* and correspong NAPI context.
334	*
335	* this is common handler for all non-queue (generic) interrupts.
336	*/
337	static irqreturn_t octep_ioq_intr_handler(int irq, void *data)
338	{
339	struct octep_ioq_vector *ioq_vector = data;
340	struct octep_device *oct = ioq_vector->octep_dev;
341
342	return oct->hw_ops.ioq_intr_handler(ioq_vector);
343	}
344
345	/**
346	* octep_request_irqs() - Register interrupt handlers.
347	*
348	* @oct: Octeon device private data structure.
349	*
350	* Register handlers for all queue and non-queue interrupts.
351	*
352	* Return: 0, on successful registration of all interrupt handlers.
353	* -1, on any error.
354	*/
355	static int octep_request_irqs(struct octep_device *oct)
356	{
357	struct net_device *netdev = oct->netdev;
358	struct octep_ioq_vector *ioq_vector;
359	struct msix_entry *msix_entry;
360	char **non_ioq_msix_names;
361	int num_non_ioq_msix;
362	int ret, i, j;
363
364	num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf);
365	non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf);
366
367	oct->non_ioq_irq_names = kcalloc(n: num_non_ioq_msix,
368	OCTEP_MSIX_NAME_SIZE, GFP_KERNEL);
369	if (!oct->non_ioq_irq_names)
370	goto alloc_err;
371
372	/* First few MSI-X interrupts are non-queue interrupts */
373	for (i = 0; i < num_non_ioq_msix; i++) {
374	char *irq_name;
375
376	irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE];
377	msix_entry = &oct->msix_entries[i];
378
379	snprintf(buf: irq_name, OCTEP_MSIX_NAME_SIZE,
380	fmt: "%s-%s", netdev->name, non_ioq_msix_names[i]);
381	if (!strncmp(non_ioq_msix_names[i], "epf_mbox_rint", strlen("epf_mbox_rint"))) {
382	ret = request_irq(irq: msix_entry->vector,
383	handler: octep_mbox_intr_handler, flags: 0,
384	name: irq_name, dev: oct);
385	} else if (!strncmp(non_ioq_msix_names[i], "epf_oei_rint",
386	strlen("epf_oei_rint"))) {
387	ret = request_irq(irq: msix_entry->vector,
388	handler: octep_oei_intr_handler, flags: 0,
389	name: irq_name, dev: oct);
390	} else if (!strncmp(non_ioq_msix_names[i], "epf_ire_rint",
391	strlen("epf_ire_rint"))) {
392	ret = request_irq(irq: msix_entry->vector,
393	handler: octep_ire_intr_handler, flags: 0,
394	name: irq_name, dev: oct);
395	} else if (!strncmp(non_ioq_msix_names[i], "epf_ore_rint",
396	strlen("epf_ore_rint"))) {
397	ret = request_irq(irq: msix_entry->vector,
398	handler: octep_ore_intr_handler, flags: 0,
399	name: irq_name, dev: oct);
400	} else if (!strncmp(non_ioq_msix_names[i], "epf_vfire_rint",
401	strlen("epf_vfire_rint"))) {
402	ret = request_irq(irq: msix_entry->vector,
403	handler: octep_vfire_intr_handler, flags: 0,
404	name: irq_name, dev: oct);
405	} else if (!strncmp(non_ioq_msix_names[i], "epf_vfore_rint",
406	strlen("epf_vfore_rint"))) {
407	ret = request_irq(irq: msix_entry->vector,
408	handler: octep_vfore_intr_handler, flags: 0,
409	name: irq_name, dev: oct);
410	} else if (!strncmp(non_ioq_msix_names[i], "epf_dma_rint",
411	strlen("epf_dma_rint"))) {
412	ret = request_irq(irq: msix_entry->vector,
413	handler: octep_dma_intr_handler, flags: 0,
414	name: irq_name, dev: oct);
415	} else if (!strncmp(non_ioq_msix_names[i], "epf_dma_vf_rint",
416	strlen("epf_dma_vf_rint"))) {
417	ret = request_irq(irq: msix_entry->vector,
418	handler: octep_dma_vf_intr_handler, flags: 0,
419	name: irq_name, dev: oct);
420	} else if (!strncmp(non_ioq_msix_names[i], "epf_pp_vf_rint",
421	strlen("epf_pp_vf_rint"))) {
422	ret = request_irq(irq: msix_entry->vector,
423	handler: octep_pp_vf_intr_handler, flags: 0,
424	name: irq_name, dev: oct);
425	} else if (!strncmp(non_ioq_msix_names[i], "epf_misc_rint",
426	strlen("epf_misc_rint"))) {
427	ret = request_irq(irq: msix_entry->vector,
428	handler: octep_misc_intr_handler, flags: 0,
429	name: irq_name, dev: oct);
430	} else {
431	ret = request_irq(irq: msix_entry->vector,
432	handler: octep_rsvd_intr_handler, flags: 0,
433	name: irq_name, dev: oct);
434	}
435
436	if (ret) {
437	netdev_err(dev: netdev,
438	format: "request_irq failed for %s; err=%d",
439	irq_name, ret);
440	goto non_ioq_irq_err;
441	}
442	}
443
444	/* Request IRQs for Tx/Rx queues */
445	for (j = 0; j < oct->num_oqs; j++) {
446	ioq_vector = oct->ioq_vector[j];
447	msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
448
449	snprintf(buf: ioq_vector->name, size: sizeof(ioq_vector->name),
450	fmt: "%s-q%d", netdev->name, j);
451	ret = request_irq(irq: msix_entry->vector,
452	handler: octep_ioq_intr_handler, flags: 0,
453	name: ioq_vector->name, dev: ioq_vector);
454	if (ret) {
455	netdev_err(dev: netdev,
456	format: "request_irq failed for Q-%d; err=%d",
457	j, ret);
458	goto ioq_irq_err;
459	}
460
461	cpumask_set_cpu(cpu: j % num_online_cpus(),
462	dstp: &ioq_vector->affinity_mask);
463	irq_set_affinity_hint(irq: msix_entry->vector,
464	m: &ioq_vector->affinity_mask);
465	}
466
467	return 0;
468	ioq_irq_err:
469	while (j) {
470	--j;
471	ioq_vector = oct->ioq_vector[j];
472	msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
473
474	irq_set_affinity_hint(irq: msix_entry->vector, NULL);
475	free_irq(msix_entry->vector, ioq_vector);
476	}
477	non_ioq_irq_err:
478	while (i) {
479	--i;
480	free_irq(oct->msix_entries[i].vector, oct);
481	}
482	kfree(objp: oct->non_ioq_irq_names);
483	oct->non_ioq_irq_names = NULL;
484	alloc_err:
485	return -1;
486	}
487
488	/**
489	* octep_free_irqs() - free all registered interrupts.
490	*
491	* @oct: Octeon device private data structure.
492	*
493	* Free all queue and non-queue interrupts of the Octeon device.
494	*/
495	static void octep_free_irqs(struct octep_device *oct)
496	{
497	int i;
498
499	/* First few MSI-X interrupts are non queue interrupts; free them */
500	for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++)
501	free_irq(oct->msix_entries[i].vector, oct);
502	kfree(objp: oct->non_ioq_irq_names);
503
504	/* Free IRQs for Input/Output (Tx/Rx) queues */
505	for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) {
506	irq_set_affinity_hint(irq: oct->msix_entries[i].vector, NULL);
507	free_irq(oct->msix_entries[i].vector,
508	oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]);
509	}
510	netdev_info(dev: oct->netdev, format: "IRQs freed\n");
511	}
512
513	/**
514	* octep_setup_irqs() - setup interrupts for the Octeon device.
515	*
516	* @oct: Octeon device private data structure.
517	*
518	* Allocate data structures to hold per interrupt information, allocate/enable
519	* MSI-x interrupt and register interrupt handlers.
520	*
521	* Return: 0, on successful allocation and registration of all interrupts.
522	* -1, on any error.
523	*/
524	static int octep_setup_irqs(struct octep_device *oct)
525	{
526	if (octep_alloc_ioq_vectors(oct))
527	goto ioq_vector_err;
528
529	if (octep_enable_msix_range(oct))
530	goto enable_msix_err;
531
532	if (octep_request_irqs(oct))
533	goto request_irq_err;
534
535	return 0;
536
537	request_irq_err:
538	octep_disable_msix(oct);
539	enable_msix_err:
540	octep_free_ioq_vectors(oct);
541	ioq_vector_err:
542	return -1;
543	}
544
545	/**
546	* octep_clean_irqs() - free all interrupts and its resources.
547	*
548	* @oct: Octeon device private data structure.
549	*/
550	static void octep_clean_irqs(struct octep_device *oct)
551	{
552	octep_free_irqs(oct);
553	octep_disable_msix(oct);
554	octep_free_ioq_vectors(oct);
555	}
556
557	/**
558	* octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
559	*
560	* @iq: Octeon Tx queue data structure.
561	* @oq: Octeon Rx queue data structure.
562	*/
563	static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq)
564	{
565	u32 pkts_pend = oq->pkts_pending;
566
567	netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
568	if (iq->pkts_processed) {
569	writel(val: iq->pkts_processed, addr: iq->inst_cnt_reg);
570	iq->pkt_in_done -= iq->pkts_processed;
571	iq->pkts_processed = 0;
572	}
573	if (oq->last_pkt_count - pkts_pend) {
574	writel(val: oq->last_pkt_count - pkts_pend, addr: oq->pkts_sent_reg);
575	oq->last_pkt_count = pkts_pend;
576	}
577
578	/* Flush the previous wrties before writing to RESEND bit */
579	wmb();
580	writeq(val: 1UL << OCTEP_OQ_INTR_RESEND_BIT, addr: oq->pkts_sent_reg);
581	writeq(val: 1UL << OCTEP_IQ_INTR_RESEND_BIT, addr: iq->inst_cnt_reg);
582	}
583
584	/**
585	* octep_napi_poll() - NAPI poll function for Tx/Rx.
586	*
587	* @napi: pointer to napi context.
588	* @budget: max number of packets to be processed in single invocation.
589	*/
590	static int octep_napi_poll(struct napi_struct *napi, int budget)
591	{
592	struct octep_ioq_vector *ioq_vector =
593	container_of(napi, struct octep_ioq_vector, napi);
594	u32 tx_pending, rx_done;
595
596	tx_pending = octep_iq_process_completions(iq: ioq_vector->iq, budget);
597	rx_done = octep_oq_process_rx(oq: ioq_vector->oq, budget);
598
599	/* need more polling if tx completion processing is still pending or
600	* processed at least 'budget' number of rx packets.
601	*/
602	if (tx_pending || rx_done >= budget)
603	return budget;
604
605	napi_complete(n: napi);
606	octep_enable_ioq_irq(iq: ioq_vector->iq, oq: ioq_vector->oq);
607	return rx_done;
608	}
609
610	/**
611	* octep_napi_add() - Add NAPI poll for all Tx/Rx queues.
612	*
613	* @oct: Octeon device private data structure.
614	*/
615	static void octep_napi_add(struct octep_device *oct)
616	{
617	int i;
618
619	for (i = 0; i < oct->num_oqs; i++) {
620	netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
621	netif_napi_add(dev: oct->netdev, napi: &oct->ioq_vector[i]->napi,
622	poll: octep_napi_poll);
623	oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
624	}
625	}
626
627	/**
628	* octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
629	*
630	* @oct: Octeon device private data structure.
631	*/
632	static void octep_napi_delete(struct octep_device *oct)
633	{
634	int i;
635
636	for (i = 0; i < oct->num_oqs; i++) {
637	netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
638	netif_napi_del(napi: &oct->ioq_vector[i]->napi);
639	oct->oq[i]->napi = NULL;
640	}
641	}
642
643	/**
644	* octep_napi_enable() - enable NAPI for all Tx/Rx queues.
645	*
646	* @oct: Octeon device private data structure.
647	*/
648	static void octep_napi_enable(struct octep_device *oct)
649	{
650	int i;
651
652	for (i = 0; i < oct->num_oqs; i++) {
653	netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
654	napi_enable(n: &oct->ioq_vector[i]->napi);
655	}
656	}
657
658	/**
659	* octep_napi_disable() - disable NAPI for all Tx/Rx queues.
660	*
661	* @oct: Octeon device private data structure.
662	*/
663	static void octep_napi_disable(struct octep_device *oct)
664	{
665	int i;
666
667	for (i = 0; i < oct->num_oqs; i++) {
668	netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
669	napi_disable(n: &oct->ioq_vector[i]->napi);
670	}
671	}
672
673	static void octep_link_up(struct net_device *netdev)
674	{
675	netif_carrier_on(dev: netdev);
676	netif_tx_start_all_queues(dev: netdev);
677	}
678
679	/**
680	* octep_open() - start the octeon network device.
681	*
682	* @netdev: pointer to kernel network device.
683	*
684	* setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
685	* and interrupts..
686	*
687	* Return: 0, on successfully setting up device and bring it up.
688	* -1, on any error.
689	*/
690	static int octep_open(struct net_device *netdev)
691	{
692	struct octep_device *oct = netdev_priv(dev: netdev);
693	int err, ret;
694
695	netdev_info(dev: netdev, format: "Starting netdev ...\n");
696	netif_carrier_off(dev: netdev);
697
698	oct->hw_ops.reset_io_queues(oct);
699
700	if (octep_setup_iqs(oct))
701	goto setup_iq_err;
702	if (octep_setup_oqs(oct))
703	goto setup_oq_err;
704	if (octep_setup_irqs(oct))
705	goto setup_irq_err;
706
707	err = netif_set_real_num_tx_queues(dev: netdev, txq: oct->num_oqs);
708	if (err)
709	goto set_queues_err;
710	err = netif_set_real_num_rx_queues(dev: netdev, rxq: oct->num_iqs);
711	if (err)
712	goto set_queues_err;
713
714	octep_napi_add(oct);
715	octep_napi_enable(oct);
716
717	oct->link_info.admin_up = 1;
718	octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, up: true,
719	wait_for_response: false);
720	octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, up: true,
721	wait_for_response: false);
722	oct->poll_non_ioq_intr = false;
723
724	/* Enable the input and output queues for this Octeon device */
725	oct->hw_ops.enable_io_queues(oct);
726
727	/* Enable Octeon device interrupts */
728	oct->hw_ops.enable_interrupts(oct);
729
730	octep_oq_dbell_init(oct);
731
732	ret = octep_ctrl_net_get_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID);
733	if (ret > 0)
734	octep_link_up(netdev);
735
736	return 0;
737
738	set_queues_err:
739	octep_clean_irqs(oct);
740	setup_irq_err:
741	octep_free_oqs(oct);
742	setup_oq_err:
743	octep_free_iqs(oct);
744	setup_iq_err:
745	return -1;
746	}
747
748	/**
749	* octep_stop() - stop the octeon network device.
750	*
751	* @netdev: pointer to kernel network device.
752	*
753	* stop the device Tx/Rx operations, bring down the link and
754	* free up all resources allocated for Tx/Rx queues and interrupts.
755	*/
756	static int octep_stop(struct net_device *netdev)
757	{
758	struct octep_device *oct = netdev_priv(dev: netdev);
759
760	netdev_info(dev: netdev, format: "Stopping the device ...\n");
761
762	octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, up: false,
763	wait_for_response: false);
764	octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, up: false,
765	wait_for_response: false);
766
767	/* Stop Tx from stack */
768	netif_tx_stop_all_queues(dev: netdev);
769	netif_carrier_off(dev: netdev);
770	netif_tx_disable(dev: netdev);
771
772	oct->link_info.admin_up = 0;
773	oct->link_info.oper_up = 0;
774
775	oct->hw_ops.disable_interrupts(oct);
776	octep_napi_disable(oct);
777	octep_napi_delete(oct);
778
779	octep_clean_irqs(oct);
780	octep_clean_iqs(oct);
781
782	oct->hw_ops.disable_io_queues(oct);
783	oct->hw_ops.reset_io_queues(oct);
784	octep_free_oqs(oct);
785	octep_free_iqs(oct);
786
787	oct->poll_non_ioq_intr = true;
788	queue_delayed_work(wq: octep_wq, dwork: &oct->intr_poll_task,
789	delay: msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
790
791	netdev_info(dev: netdev, format: "Device stopped !!\n");
792	return 0;
793	}
794
795	/**
796	* octep_iq_full_check() - check if a Tx queue is full.
797	*
798	* @iq: Octeon Tx queue data structure.
799	*
800	* Return: 0, if the Tx queue is not full.
801	* 1, if the Tx queue is full.
802	*/
803	static inline int octep_iq_full_check(struct octep_iq *iq)
804	{
805	if (likely((IQ_INSTR_SPACE(iq)) >
806	OCTEP_WAKE_QUEUE_THRESHOLD))
807	return 0;
808
809	/* Stop the queue if unable to send */
810	netif_stop_subqueue(dev: iq->netdev, queue_index: iq->q_no);
811
812	/* Allow for pending updates in write index
813	* from iq_process_completion in other cpus
814	* to reflect, in case queue gets free
815	* entries.
816	*/
817	smp_mb();
818
819	/* check again and restart the queue, in case NAPI has just freed
820	* enough Tx ring entries.
821	*/
822	if (unlikely(IQ_INSTR_SPACE(iq) >
823	OCTEP_WAKE_QUEUE_THRESHOLD)) {
824	netif_start_subqueue(dev: iq->netdev, queue_index: iq->q_no);
825	iq->stats.restart_cnt++;
826	return 0;
827	}
828
829	return 1;
830	}
831
832	/**
833	* octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
834	*
835	* @skb: packet skbuff pointer.
836	* @netdev: kernel network device.
837	*
838	* Return: NETDEV_TX_BUSY, if Tx Queue is full.
839	* NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
840	*/
841	static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
842	struct net_device *netdev)
843	{
844	struct octep_device *oct = netdev_priv(dev: netdev);
845	netdev_features_t feat = netdev->features;
846	struct octep_tx_sglist_desc *sglist;
847	struct octep_tx_buffer *tx_buffer;
848	struct octep_tx_desc_hw *hw_desc;
849	struct skb_shared_info *shinfo;
850	struct octep_instr_hdr *ih;
851	struct octep_iq *iq;
852	skb_frag_t *frag;
853	u16 nr_frags, si;
854	int xmit_more;
855	u16 q_no, wi;
856
857	if (skb_put_padto(skb, ETH_ZLEN))
858	return NETDEV_TX_OK;
859
860	q_no = skb_get_queue_mapping(skb);
861	if (q_no >= oct->num_iqs) {
862	netdev_err(dev: netdev, format: "Invalid Tx skb->queue_mapping=%d\n", q_no);
863	q_no = q_no % oct->num_iqs;
864	}
865
866	iq = oct->iq[q_no];
867
868	shinfo = skb_shinfo(skb);
869	nr_frags = shinfo->nr_frags;
870
871	wi = iq->host_write_index;
872	hw_desc = &iq->desc_ring[wi];
873	hw_desc->ih64 = 0;
874
875	tx_buffer = iq->buff_info + wi;
876	tx_buffer->skb = skb;
877
878	ih = &hw_desc->ih;
879	ih->pkind = oct->conf->fw_info.pkind;
880	ih->fsz = oct->conf->fw_info.fsz;
881	ih->tlen = skb->len + ih->fsz;
882
883	if (!nr_frags) {
884	tx_buffer->gather = 0;
885	tx_buffer->dma = dma_map_single(iq->dev, skb->data,
886	skb->len, DMA_TO_DEVICE);
887	if (dma_mapping_error(dev: iq->dev, dma_addr: tx_buffer->dma))
888	goto dma_map_err;
889	hw_desc->dptr = tx_buffer->dma;
890	} else {
891	/* Scatter/Gather */
892	dma_addr_t dma;
893	u16 len;
894
895	sglist = tx_buffer->sglist;
896
897	ih->gsz = nr_frags + 1;
898	ih->gather = 1;
899	tx_buffer->gather = 1;
900
901	len = skb_headlen(skb);
902	dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
903	if (dma_mapping_error(dev: iq->dev, dma_addr: dma))
904	goto dma_map_err;
905
906	memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
907	sglist[0].len[3] = len;
908	sglist[0].dma_ptr[0] = dma;
909
910	si = 1; /* entry 0 is main skb, mapped above */
911	frag = &shinfo->frags[0];
912	while (nr_frags--) {
913	len = skb_frag_size(frag);
914	dma = skb_frag_dma_map(dev: iq->dev, frag, offset: 0,
915	size: len, dir: DMA_TO_DEVICE);
916	if (dma_mapping_error(dev: iq->dev, dma_addr: dma))
917	goto dma_map_sg_err;
918
919	sglist[si >> 2].len[3 - (si & 3)] = len;
920	sglist[si >> 2].dma_ptr[si & 3] = dma;
921
922	frag++;
923	si++;
924	}
925	hw_desc->dptr = tx_buffer->sglist_dma;
926	}
927
928	if (oct->conf->fw_info.tx_ol_flags) {
929	if ((feat & (NETIF_F_TSO)) && (skb_is_gso(skb))) {
930	hw_desc->txm.ol_flags = OCTEP_TX_OFFLOAD_CKSUM;
931	hw_desc->txm.ol_flags |= OCTEP_TX_OFFLOAD_TSO;
932	hw_desc->txm.gso_size = skb_shinfo(skb)->gso_size;
933	hw_desc->txm.gso_segs = skb_shinfo(skb)->gso_segs;
934	} else if (feat & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
935	hw_desc->txm.ol_flags = OCTEP_TX_OFFLOAD_CKSUM;
936	}
937	/* due to ESR txm will be swapped by hw */
938	hw_desc->txm64[0] = (__force u64)cpu_to_be64(hw_desc->txm64[0]);
939	}
940
941	xmit_more = netdev_xmit_more();
942
943	__netdev_tx_sent_queue(dev_queue: iq->netdev_q, bytes: skb->len, xmit_more);
944
945	skb_tx_timestamp(skb);
946	iq->fill_cnt++;
947	wi++;
948	iq->host_write_index = wi & iq->ring_size_mask;
949
950	/* octep_iq_full_check stops the queue and returns
951	* true if so, in case the queue has become full
952	* by inserting current packet. If so, we can
953	* go ahead and ring doorbell.
954	*/
955	if (!octep_iq_full_check(iq) && xmit_more &&
956	iq->fill_cnt < iq->fill_threshold)
957	return NETDEV_TX_OK;
958
959	/* Flush the hw descriptor before writing to doorbell */
960	wmb();
961	/* Ring Doorbell to notify the NIC of new packets */
962	writel(val: iq->fill_cnt, addr: iq->doorbell_reg);
963	iq->stats.instr_posted += iq->fill_cnt;
964	iq->fill_cnt = 0;
965	return NETDEV_TX_OK;
966
967	dma_map_sg_err:
968	if (si > 0) {
969	dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
970	sglist[0].len[3], DMA_TO_DEVICE);
971	sglist[0].len[3] = 0;
972	}
973	while (si > 1) {
974	dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
975	sglist[si >> 2].len[3 - (si & 3)], DMA_TO_DEVICE);
976	sglist[si >> 2].len[3 - (si & 3)] = 0;
977	si--;
978	}
979	tx_buffer->gather = 0;
980	dma_map_err:
981	dev_kfree_skb_any(skb);
982	return NETDEV_TX_OK;
983	}
984
985	/**
986	* octep_get_stats64() - Get Octeon network device statistics.
987	*
988	* @netdev: kernel network device.
989	* @stats: pointer to stats structure to be filled in.
990	*/
991	static void octep_get_stats64(struct net_device *netdev,
992	struct rtnl_link_stats64 *stats)
993	{
994	u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
995	struct octep_device *oct = netdev_priv(dev: netdev);
996	int q;
997
998	if (netif_running(dev: netdev))
999	octep_ctrl_net_get_if_stats(oct,
1000	OCTEP_CTRL_NET_INVALID_VFID,
1001	rx_stats: &oct->iface_rx_stats,
1002	tx_stats: &oct->iface_tx_stats);
1003
1004	tx_packets = 0;
1005	tx_bytes = 0;
1006	rx_packets = 0;
1007	rx_bytes = 0;
1008	for (q = 0; q < oct->num_oqs; q++) {
1009	struct octep_iq *iq = oct->iq[q];
1010	struct octep_oq *oq = oct->oq[q];
1011
1012	tx_packets += iq->stats.instr_completed;
1013	tx_bytes += iq->stats.bytes_sent;
1014	rx_packets += oq->stats.packets;
1015	rx_bytes += oq->stats.bytes;
1016	}
1017	stats->tx_packets = tx_packets;
1018	stats->tx_bytes = tx_bytes;
1019	stats->rx_packets = rx_packets;
1020	stats->rx_bytes = rx_bytes;
1021	stats->multicast = oct->iface_rx_stats.mcast_pkts;
1022	stats->rx_errors = oct->iface_rx_stats.err_pkts;
1023	stats->collisions = oct->iface_tx_stats.xscol;
1024	stats->tx_fifo_errors = oct->iface_tx_stats.undflw;
1025	}
1026
1027	/**
1028	* octep_tx_timeout_task - work queue task to Handle Tx queue timeout.
1029	*
1030	* @work: pointer to Tx queue timeout work_struct
1031	*
1032	* Stop and start the device so that it frees up all queue resources
1033	* and restarts the queues, that potentially clears a Tx queue timeout
1034	* condition.
1035	**/
1036	static void octep_tx_timeout_task(struct work_struct *work)
1037	{
1038	struct octep_device *oct = container_of(work, struct octep_device,
1039	tx_timeout_task);
1040	struct net_device *netdev = oct->netdev;
1041
1042	rtnl_lock();
1043	if (netif_running(dev: netdev)) {
1044	octep_stop(netdev);
1045	octep_open(netdev);
1046	}
1047	rtnl_unlock();
1048	}
1049
1050	/**
1051	* octep_tx_timeout() - Handle Tx Queue timeout.
1052	*
1053	* @netdev: pointer to kernel network device.
1054	* @txqueue: Timed out Tx queue number.
1055	*
1056	* Schedule a work to handle Tx queue timeout.
1057	*/
1058	static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1059	{
1060	struct octep_device *oct = netdev_priv(dev: netdev);
1061
1062	queue_work(wq: octep_wq, work: &oct->tx_timeout_task);
1063	}
1064
1065	static int octep_set_mac(struct net_device *netdev, void *p)
1066	{
1067	struct octep_device *oct = netdev_priv(dev: netdev);
1068	struct sockaddr *addr = (struct sockaddr *)p;
1069	int err;
1070
1071	if (!is_valid_ether_addr(addr: addr->sa_data))
1072	return -EADDRNOTAVAIL;
1073
1074	err = octep_ctrl_net_set_mac_addr(oct, OCTEP_CTRL_NET_INVALID_VFID,
1075	addr: addr->sa_data, wait_for_response: true);
1076	if (err)
1077	return err;
1078
1079	memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
1080	eth_hw_addr_set(dev: netdev, addr: addr->sa_data);
1081
1082	return 0;
1083	}
1084
1085	static int octep_change_mtu(struct net_device *netdev, int new_mtu)
1086	{
1087	struct octep_device *oct = netdev_priv(dev: netdev);
1088	struct octep_iface_link_info *link_info;
1089	int err = 0;
1090
1091	link_info = &oct->link_info;
1092	if (link_info->mtu == new_mtu)
1093	return 0;
1094
1095	err = octep_ctrl_net_set_mtu(oct, OCTEP_CTRL_NET_INVALID_VFID, mtu: new_mtu,
1096	wait_for_response: true);
1097	if (!err) {
1098	oct->link_info.mtu = new_mtu;
1099	netdev->mtu = new_mtu;
1100	}
1101
1102	return err;
1103	}
1104
1105	static int octep_set_features(struct net_device *dev, netdev_features_t features)
1106	{
1107	struct octep_ctrl_net_offloads offloads = { 0 };
1108	struct octep_device *oct = netdev_priv(dev);
1109	int err;
1110
1111	/* We only support features received from firmware */
1112	if ((features & dev->hw_features) != features)
1113	return -EINVAL;
1114
1115	if (features & NETIF_F_TSO)
1116	offloads.tx_offloads |= OCTEP_TX_OFFLOAD_TSO;
1117
1118	if (features & NETIF_F_TSO6)
1119	offloads.tx_offloads |= OCTEP_TX_OFFLOAD_TSO;
1120
1121	if (features & NETIF_F_IP_CSUM)
1122	offloads.tx_offloads |= OCTEP_TX_OFFLOAD_CKSUM;
1123
1124	if (features & NETIF_F_IPV6_CSUM)
1125	offloads.tx_offloads |= OCTEP_TX_OFFLOAD_CKSUM;
1126
1127	if (features & NETIF_F_RXCSUM)
1128	offloads.rx_offloads |= OCTEP_RX_OFFLOAD_CKSUM;
1129
1130	err = octep_ctrl_net_set_offloads(oct,
1131	OCTEP_CTRL_NET_INVALID_VFID,
1132	offloads: &offloads,
1133	wait_for_response: true);
1134	if (!err)
1135	dev->features = features;
1136
1137	return err;
1138	}
1139
1140	static const struct net_device_ops octep_netdev_ops = {
1141	.ndo_open = octep_open,
1142	.ndo_stop = octep_stop,
1143	.ndo_start_xmit = octep_start_xmit,
1144	.ndo_get_stats64 = octep_get_stats64,
1145	.ndo_tx_timeout = octep_tx_timeout,
1146	.ndo_set_mac_address = octep_set_mac,
1147	.ndo_change_mtu = octep_change_mtu,
1148	.ndo_set_features = octep_set_features,
1149	};
1150
1151	/**
1152	* octep_intr_poll_task - work queue task to process non-ioq interrupts.
1153	*
1154	* @work: pointer to mbox work_struct
1155	*
1156	* Process non-ioq interrupts to handle control mailbox, pfvf mailbox.
1157	**/
1158	static void octep_intr_poll_task(struct work_struct *work)
1159	{
1160	struct octep_device *oct = container_of(work, struct octep_device,
1161	intr_poll_task.work);
1162
1163	if (!oct->poll_non_ioq_intr) {
1164	dev_info(&oct->pdev->dev, "Interrupt poll task stopped.\n");
1165	return;
1166	}
1167
1168	oct->hw_ops.poll_non_ioq_interrupts(oct);
1169	queue_delayed_work(wq: octep_wq, dwork: &oct->intr_poll_task,
1170	delay: msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
1171	}
1172
1173	/**
1174	* octep_hb_timeout_task - work queue task to check firmware heartbeat.
1175	*
1176	* @work: pointer to hb work_struct
1177	*
1178	* Check for heartbeat miss count. Uninitialize oct device if miss count
1179	* exceeds configured max heartbeat miss count.
1180	*
1181	**/
1182	static void octep_hb_timeout_task(struct work_struct *work)
1183	{
1184	struct octep_device *oct = container_of(work, struct octep_device,
1185	hb_task.work);
1186
1187	int miss_cnt;
1188
1189	miss_cnt = atomic_inc_return(v: &oct->hb_miss_cnt);
1190	if (miss_cnt < oct->conf->fw_info.hb_miss_count) {
1191	queue_delayed_work(wq: octep_wq, dwork: &oct->hb_task,
1192	delay: msecs_to_jiffies(m: oct->conf->fw_info.hb_interval));
1193	return;
1194	}
1195
1196	dev_err(&oct->pdev->dev, "Missed %u heartbeats. Uninitializing\n",
1197	miss_cnt);
1198	rtnl_lock();
1199	if (netif_running(dev: oct->netdev))
1200	octep_stop(netdev: oct->netdev);
1201	rtnl_unlock();
1202	}
1203
1204	/**
1205	* octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages.
1206	*
1207	* @work: pointer to ctrl mbox work_struct
1208	*
1209	* Poll ctrl mbox message queue and handle control messages from firmware.
1210	**/
1211	static void octep_ctrl_mbox_task(struct work_struct *work)
1212	{
1213	struct octep_device *oct = container_of(work, struct octep_device,
1214	ctrl_mbox_task);
1215
1216	octep_ctrl_net_recv_fw_messages(oct);
1217	}
1218
1219	static const char *octep_devid_to_str(struct octep_device *oct)
1220	{
1221	switch (oct->chip_id) {
1222	case OCTEP_PCI_DEVICE_ID_CN98_PF:
1223	return "CN98XX";
1224	case OCTEP_PCI_DEVICE_ID_CN93_PF:
1225	return "CN93XX";
1226	case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
1227	return "CNF95N";
1228	case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
1229	return "CN10KA";
1230	case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
1231	return "CNF10KA";
1232	case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
1233	return "CNF10KB";
1234	case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
1235	return "CN10KB";
1236	default:
1237	return "Unsupported";
1238	}
1239	}
1240
1241	/**
1242	* octep_device_setup() - Setup Octeon Device.
1243	*
1244	* @oct: Octeon device private data structure.
1245	*
1246	* Setup Octeon device hardware operations, configuration, etc ...
1247	*/
1248	int octep_device_setup(struct octep_device *oct)
1249	{
1250	struct pci_dev *pdev = oct->pdev;
1251	int i, ret;
1252
1253	/* allocate memory for oct->conf */
1254	oct->conf = kzalloc(size: sizeof(*oct->conf), GFP_KERNEL);
1255	if (!oct->conf)
1256	return -ENOMEM;
1257
1258	/* Map BAR regions */
1259	for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1260	oct->mmio[i].hw_addr =
1261	ioremap(pci_resource_start(oct->pdev, i * 2),
1262	pci_resource_len(oct->pdev, i * 2));
1263	if (!oct->mmio[i].hw_addr)
1264	goto unmap_prev;
1265
1266	oct->mmio[i].mapped = 1;
1267	}
1268
1269	oct->chip_id = pdev->device;
1270	oct->rev_id = pdev->revision;
1271	dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
1272
1273	switch (oct->chip_id) {
1274	case OCTEP_PCI_DEVICE_ID_CN98_PF:
1275	case OCTEP_PCI_DEVICE_ID_CN93_PF:
1276	case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
1277	dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
1278	octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct),
1279	OCTEP_MINOR_REV(oct));
1280	octep_device_setup_cn93_pf(oct);
1281	break;
1282	case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
1283	case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
1284	case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
1285	case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
1286	dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
1287	octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct), OCTEP_MINOR_REV(oct));
1288	octep_device_setup_cnxk_pf(oct);
1289	break;
1290	default:
1291	dev_err(&pdev->dev,
1292	"%s: unsupported device\n", __func__);
1293	goto unsupported_dev;
1294	}
1295
1296
1297	ret = octep_ctrl_net_init(oct);
1298	if (ret)
1299	return ret;
1300
1301	INIT_WORK(&oct->tx_timeout_task, octep_tx_timeout_task);
1302	INIT_WORK(&oct->ctrl_mbox_task, octep_ctrl_mbox_task);
1303	INIT_DELAYED_WORK(&oct->intr_poll_task, octep_intr_poll_task);
1304	oct->poll_non_ioq_intr = true;
1305	queue_delayed_work(wq: octep_wq, dwork: &oct->intr_poll_task,
1306	delay: msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
1307
1308	atomic_set(v: &oct->hb_miss_cnt, i: 0);
1309	INIT_DELAYED_WORK(&oct->hb_task, octep_hb_timeout_task);
1310
1311	return 0;
1312
1313	unsupported_dev:
1314	i = OCTEP_MMIO_REGIONS;
1315	unmap_prev:
1316	while (i--)
1317	iounmap(addr: oct->mmio[i].hw_addr);
1318
1319	kfree(objp: oct->conf);
1320	return -1;
1321	}
1322
1323	/**
1324	* octep_device_cleanup() - Cleanup Octeon Device.
1325	*
1326	* @oct: Octeon device private data structure.
1327	*
1328	* Cleanup Octeon device allocated resources.
1329	*/
1330	static void octep_device_cleanup(struct octep_device *oct)
1331	{
1332	int i;
1333
1334	oct->poll_non_ioq_intr = false;
1335	cancel_delayed_work_sync(dwork: &oct->intr_poll_task);
1336	cancel_work_sync(work: &oct->ctrl_mbox_task);
1337
1338	dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
1339
1340	for (i = 0; i < OCTEP_MAX_VF; i++) {
1341	vfree(addr: oct->mbox[i]);
1342	oct->mbox[i] = NULL;
1343	}
1344
1345	octep_delete_pfvf_mbox(oct);
1346	octep_ctrl_net_uninit(oct);
1347	cancel_delayed_work_sync(dwork: &oct->hb_task);
1348
1349	oct->hw_ops.soft_reset(oct);
1350	for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1351	if (oct->mmio[i].mapped)
1352	iounmap(addr: oct->mmio[i].hw_addr);
1353	}
1354
1355	kfree(objp: oct->conf);
1356	oct->conf = NULL;
1357	}
1358
1359	static bool get_fw_ready_status(struct pci_dev *pdev)
1360	{
1361	u32 pos = 0;
1362	u16 vsec_id;
1363	u8 status;
1364
1365	while ((pos = pci_find_next_ext_capability(dev: pdev, pos,
1366	PCI_EXT_CAP_ID_VNDR))) {
1367	pci_read_config_word(dev: pdev, where: pos + 4, val: &vsec_id);
1368	#define FW_STATUS_VSEC_ID 0xA3
1369	if (vsec_id != FW_STATUS_VSEC_ID)
1370	continue;
1371
1372	pci_read_config_byte(dev: pdev, where: (pos + 8), val: &status);
1373	dev_info(&pdev->dev, "Firmware ready status = %u\n", status);
1374	#define FW_STATUS_READY 1ULL
1375	return status == FW_STATUS_READY;
1376	}
1377	return false;
1378	}
1379
1380	/**
1381	* octep_probe() - Octeon PCI device probe handler.
1382	*
1383	* @pdev: PCI device structure.
1384	* @ent: entry in Octeon PCI device ID table.
1385	*
1386	* Initializes and enables the Octeon PCI device for network operations.
1387	* Initializes Octeon private data structure and registers a network device.
1388	*/
1389	static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1390	{
1391	struct octep_device *octep_dev = NULL;
1392	struct net_device *netdev;
1393	int max_rx_pktlen;
1394	int err;
1395
1396	err = pci_enable_device(dev: pdev);
1397	if (err) {
1398	dev_err(&pdev->dev, "Failed to enable PCI device\n");
1399	return err;
1400	}
1401
1402	err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
1403	if (err) {
1404	dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1405	goto err_dma_mask;
1406	}
1407
1408	err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME);
1409	if (err) {
1410	dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1411	goto err_pci_regions;
1412	}
1413
1414	pci_set_master(dev: pdev);
1415
1416	if (!get_fw_ready_status(pdev)) {
1417	dev_notice(&pdev->dev, "Firmware not ready; defer probe.\n");
1418	err = -EPROBE_DEFER;
1419	goto err_alloc_netdev;
1420	}
1421
1422	netdev = alloc_etherdev_mq(sizeof(struct octep_device),
1423	OCTEP_MAX_QUEUES);
1424	if (!netdev) {
1425	dev_err(&pdev->dev, "Failed to allocate netdev\n");
1426	err = -ENOMEM;
1427	goto err_alloc_netdev;
1428	}
1429	SET_NETDEV_DEV(netdev, &pdev->dev);
1430
1431	octep_dev = netdev_priv(dev: netdev);
1432	octep_dev->netdev = netdev;
1433	octep_dev->pdev = pdev;
1434	octep_dev->dev = &pdev->dev;
1435	pci_set_drvdata(pdev, data: octep_dev);
1436
1437	err = octep_device_setup(oct: octep_dev);
1438	if (err) {
1439	dev_err(&pdev->dev, "Device setup failed\n");
1440	goto err_octep_config;
1441	}
1442
1443	err = octep_setup_pfvf_mbox(oct: octep_dev);
1444	if (err) {
1445	dev_err(&pdev->dev, "PF-VF mailbox setup failed\n");
1446	goto register_dev_err;
1447	}
1448
1449	err = octep_ctrl_net_get_info(oct: octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
1450	info: &octep_dev->conf->fw_info);
1451	if (err) {
1452	dev_err(&pdev->dev, "Failed to get firmware info\n");
1453	goto register_dev_err;
1454	}
1455	dev_info(&octep_dev->pdev->dev, "Heartbeat interval %u msecs Heartbeat miss count %u\n",
1456	octep_dev->conf->fw_info.hb_interval,
1457	octep_dev->conf->fw_info.hb_miss_count);
1458	queue_delayed_work(wq: octep_wq, dwork: &octep_dev->hb_task,
1459	delay: msecs_to_jiffies(m: octep_dev->conf->fw_info.hb_interval));
1460
1461	netdev->netdev_ops = &octep_netdev_ops;
1462	octep_set_ethtool_ops(netdev);
1463	netif_carrier_off(dev: netdev);
1464
1465	netdev->hw_features = NETIF_F_SG;
1466	if (OCTEP_TX_IP_CSUM(octep_dev->conf->fw_info.tx_ol_flags))
1467	netdev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
1468
1469	if (OCTEP_RX_IP_CSUM(octep_dev->conf->fw_info.rx_ol_flags))
1470	netdev->hw_features |= NETIF_F_RXCSUM;
1471
1472	max_rx_pktlen = octep_ctrl_net_get_mtu(oct: octep_dev, OCTEP_CTRL_NET_INVALID_VFID);
1473	if (max_rx_pktlen < 0) {
1474	dev_err(&octep_dev->pdev->dev,
1475	"Failed to get max receive packet size; err = %d\n", max_rx_pktlen);
1476	err = max_rx_pktlen;
1477	goto register_dev_err;
1478	}
1479	netdev->min_mtu = OCTEP_MIN_MTU;
1480	netdev->max_mtu = max_rx_pktlen - (ETH_HLEN + ETH_FCS_LEN);
1481	netdev->mtu = OCTEP_DEFAULT_MTU;
1482
1483	if (OCTEP_TX_TSO(octep_dev->conf->fw_info.tx_ol_flags)) {
1484	netdev->hw_features |= NETIF_F_TSO;
1485	netif_set_tso_max_size(dev: netdev, size: netdev->max_mtu);
1486	}
1487
1488	netdev->features |= netdev->hw_features;
1489	err = octep_ctrl_net_get_mac_addr(oct: octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
1490	addr: octep_dev->mac_addr);
1491	if (err) {
1492	dev_err(&pdev->dev, "Failed to get mac address\n");
1493	goto register_dev_err;
1494	}
1495	eth_hw_addr_set(dev: netdev, addr: octep_dev->mac_addr);
1496
1497	err = register_netdev(dev: netdev);
1498	if (err) {
1499	dev_err(&pdev->dev, "Failed to register netdev\n");
1500	goto register_dev_err;
1501	}
1502	dev_info(&pdev->dev, "Device probe successful\n");
1503	return 0;
1504
1505	register_dev_err:
1506	octep_device_cleanup(oct: octep_dev);
1507	err_octep_config:
1508	free_netdev(dev: netdev);
1509	err_alloc_netdev:
1510	pci_release_mem_regions(pdev);
1511	err_pci_regions:
1512	err_dma_mask:
1513	pci_disable_device(dev: pdev);
1514	return err;
1515	}
1516
1517	static int octep_sriov_disable(struct octep_device *oct)
1518	{
1519	struct pci_dev *pdev = oct->pdev;
1520
1521	if (pci_vfs_assigned(dev: oct->pdev)) {
1522	dev_warn(&pdev->dev, "Can't disable SRIOV while VFs are assigned\n");
1523	return -EPERM;
1524	}
1525
1526	pci_disable_sriov(dev: pdev);
1527	CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1528
1529	return 0;
1530	}
1531
1532	/**
1533	* octep_remove() - Remove Octeon PCI device from driver control.
1534	*
1535	* @pdev: PCI device structure of the Octeon device.
1536	*
1537	* Cleanup all resources allocated for the Octeon device.
1538	* Unregister from network device and disable the PCI device.
1539	*/
1540	static void octep_remove(struct pci_dev *pdev)
1541	{
1542	struct octep_device *oct = pci_get_drvdata(pdev);
1543	struct net_device *netdev;
1544
1545	if (!oct)
1546	return;
1547
1548	netdev = oct->netdev;
1549	octep_sriov_disable(oct);
1550	if (netdev->reg_state == NETREG_REGISTERED)
1551	unregister_netdev(dev: netdev);
1552
1553	cancel_work_sync(work: &oct->tx_timeout_task);
1554	octep_device_cleanup(oct);
1555	pci_release_mem_regions(pdev);
1556	free_netdev(dev: netdev);
1557	pci_disable_device(dev: pdev);
1558	}
1559
1560	static int octep_sriov_enable(struct octep_device *oct, int num_vfs)
1561	{
1562	struct pci_dev *pdev = oct->pdev;
1563	int err;
1564
1565	CFG_GET_ACTIVE_VFS(oct->conf) = num_vfs;
1566	err = pci_enable_sriov(dev: pdev, nr_virtfn: num_vfs);
1567	if (err) {
1568	dev_warn(&pdev->dev, "Failed to enable SRIOV err=%d\n", err);
1569	CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1570	return err;
1571	}
1572
1573	return num_vfs;
1574	}
1575
1576	static int octep_sriov_configure(struct pci_dev *pdev, int num_vfs)
1577	{
1578	struct octep_device *oct = pci_get_drvdata(pdev);
1579	int max_nvfs;
1580
1581	if (num_vfs == 0)
1582	return octep_sriov_disable(oct);
1583
1584	max_nvfs = CFG_GET_MAX_VFS(oct->conf);
1585
1586	if (num_vfs > max_nvfs) {
1587	dev_err(&pdev->dev, "Invalid VF count Max supported VFs = %d\n",
1588	max_nvfs);
1589	return -EINVAL;
1590	}
1591
1592	return octep_sriov_enable(oct, num_vfs);
1593	}
1594
1595	static struct pci_driver octep_driver = {
1596	.name = OCTEP_DRV_NAME,
1597	.id_table = octep_pci_id_tbl,
1598	.probe = octep_probe,
1599	.remove = octep_remove,
1600	.sriov_configure = octep_sriov_configure,
1601	};
1602
1603	/**
1604	* octep_init_module() - Module initialiation.
1605	*
1606	* create common resource for the driver and register PCI driver.
1607	*/
1608	static int __init octep_init_module(void)
1609	{
1610	int ret;
1611
1612	pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING);
1613
1614	/* work queue for all deferred tasks */
1615	octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME);
1616	if (!octep_wq) {
1617	pr_err("%s: Failed to create common workqueue\n",
1618	OCTEP_DRV_NAME);
1619	return -ENOMEM;
1620	}
1621
1622	ret = pci_register_driver(&octep_driver);
1623	if (ret < 0) {
1624	pr_err("%s: Failed to register PCI driver; err=%d\n",
1625	OCTEP_DRV_NAME, ret);
1626	destroy_workqueue(wq: octep_wq);
1627	return ret;
1628	}
1629
1630	pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME);
1631
1632	return ret;
1633	}
1634
1635	/**
1636	* octep_exit_module() - Module exit routine.
1637	*
1638	* unregister the driver with PCI subsystem and cleanup common resources.
1639	*/
1640	static void __exit octep_exit_module(void)
1641	{
1642	pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME);
1643
1644	pci_unregister_driver(dev: &octep_driver);
1645	destroy_workqueue(wq: octep_wq);
1646
1647	pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME);
1648	}
1649
1650	module_init(octep_init_module);
1651	module_exit(octep_exit_module);
1652