mana_en.c source code [linux/drivers/net/ethernet/microsoft/mana/mana_en.c]

1	// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2	/ Copyright (c) 2021, Microsoft Corporation. /
3
4	#include <uapi/linux/bpf.h>
5
6	#include <linux/inetdevice.h>
7	#include <linux/etherdevice.h>
8	#include <linux/ethtool.h>
9	#include <linux/filter.h>
10	#include <linux/mm.h>
11	#include <linux/pci.h>
12
13	#include <net/checksum.h>
14	#include <net/ip6_checksum.h>
15	#include <net/page_pool/helpers.h>
16	#include <net/xdp.h>
17
18	#include <net/mana/mana.h>
19	#include <net/mana/mana_auxiliary.h>
20
21	static DEFINE_IDA(mana_adev_ida);
22
23	static int mana_adev_idx_alloc(void)
24	{
25	return ida_alloc(ida: &mana_adev_ida, GFP_KERNEL);
26	}
27
28	static void mana_adev_idx_free(int idx)
29	{
30	ida_free(&mana_adev_ida, id: idx);
31	}
32
33	/ Microsoft Azure Network Adapter (MANA) functions /
34
35	static int mana_open(struct net_device *ndev)
36	{
37	struct mana_port_context *apc = netdev_priv(dev: ndev);
38	int err;
39
40	err = mana_alloc_queues(ndev);
41	if (err)
42	return err;
43
44	apc->port_is_up = true;
45
46	/ Ensure port state updated before txq state /
47	smp_wmb();
48
49	netif_carrier_on(dev: ndev);
50	netif_tx_wake_all_queues(dev: ndev);
51
52	return `0`;
53	}
54
55	static int mana_close(struct net_device *ndev)
56	{
57	struct mana_port_context *apc = netdev_priv(dev: ndev);
58
59	if (!apc->port_is_up)
60	return `0`;
61
62	return mana_detach(ndev, from_close: true);
63	}
64
65	static bool mana_can_tx(struct gdma_queue *wq)
66	{
67	return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE;
68	}
69
70	static unsigned int mana_checksum_info(struct sk_buff *skb)
71	{
72	if (skb->protocol == htons(ETH_P_IP)) {
73	struct iphdr *ip = ip_hdr(skb);
74
75	if (ip->protocol == IPPROTO_TCP)
76	return IPPROTO_TCP;
77
78	if (ip->protocol == IPPROTO_UDP)
79	return IPPROTO_UDP;
80	} else if (skb->protocol == htons(ETH_P_IPV6)) {
81	struct ipv6hdr *ip6 = ipv6_hdr(skb);
82
83	if (ip6->nexthdr == IPPROTO_TCP)
84	return IPPROTO_TCP;
85
86	if (ip6->nexthdr == IPPROTO_UDP)
87	return IPPROTO_UDP;
88	}
89
90	/ No csum offloading /
91	return `0`;
92	}
93
94	static void mana_add_sge(struct mana_tx_package tp, struct* mana_skb_head *ash,
95	int sg_i, dma_addr_t da, int sge_len, u32 gpa_mkey)
96	{
97	ash->dma_handle[sg_i] = da;
98	ash->size[sg_i] = sge_len;
99
100	tp->wqe_req.sgl[sg_i].address = da;
101	tp->wqe_req.sgl[sg_i].mem_key = gpa_mkey;
102	tp->wqe_req.sgl[sg_i].size = sge_len;
103	}
104
105	static int mana_map_skb(struct sk_buff skb, struct* mana_port_context *apc,
106	struct mana_tx_package tp, int* gso_hs)
107	{
108	struct mana_skb_head ash = (struct* mana_skb_head *)skb->head;
109	int hsg = `1`; / num of SGEs of linear part /
110	struct gdma_dev *gd = apc->ac->gdma_dev;
111	int skb_hlen = skb_headlen(skb);
112	int sge0_len, sge1_len = `0`;
113	struct gdma_context *gc;
114	struct device *dev;
115	skb_frag_t *frag;
116	dma_addr_t da;
117	int sg_i;
118	int i;
119
120	gc = gd->gdma_context;
121	dev = gc->dev;
122
123	if (gso_hs && gso_hs < skb_hlen) {
124	sge0_len = gso_hs;
125	sge1_len = skb_hlen - gso_hs;
126	} else {
127	sge0_len = skb_hlen;
128	}
129
130	da = dma_map_single(dev, skb->data, sge0_len, DMA_TO_DEVICE);
131	if (dma_mapping_error(dev, dma_addr: da))
132	return -ENOMEM;
133
134	mana_add_sge(tp, ash, sg_i: `0`, da, sge_len: sge0_len, gpa_mkey: gd->gpa_mkey);
135
136	if (sge1_len) {
137	sg_i = `1`;
138	da = dma_map_single(dev, skb->data + sge0_len, sge1_len,
139	DMA_TO_DEVICE);
140	if (dma_mapping_error(dev, dma_addr: da))
141	goto frag_err;
142
143	mana_add_sge(tp, ash, sg_i, da, sge_len: sge1_len, gpa_mkey: gd->gpa_mkey);
144	hsg = `2`;
145	}
146
147	for (i = `0`; i < skb_shinfo(skb)->nr_frags; i++) {
148	sg_i = hsg + i;
149
150	frag = &skb_shinfo(skb)->frags[i];
151	da = skb_frag_dma_map(dev, frag, offset: `0`, size: skb_frag_size(frag),
152	dir: DMA_TO_DEVICE);
153	if (dma_mapping_error(dev, dma_addr: da))
154	goto frag_err;
155
156	mana_add_sge(tp, ash, sg_i, da, sge_len: skb_frag_size(frag),
157	gpa_mkey: gd->gpa_mkey);
158	}
159
160	return `0`;
161
162	frag_err:
163	for (i = sg_i - `1`; i >= hsg; i--)
164	dma_unmap_page(dev, ash->dma_handle[i], ash->size[i],
165	DMA_TO_DEVICE);
166
167	for (i = hsg - `1`; i >= `0`; i--)
168	dma_unmap_single(dev, ash->dma_handle[i], ash->size[i],
169	DMA_TO_DEVICE);
170
171	return -ENOMEM;
172	}
173
174	/ Handle the case when GSO SKB linear length is too large.*
175	* MANA NIC requires GSO packets to put only the packet header to SGE0.
176	* So, we need 2 SGEs for the skb linear part which contains more than the
177	* header.
178	* Return a positive value for the number of SGEs, or a negative value
179	* for an error.
180	*/
181	static int mana_fix_skb_head(struct net_device ndev, struct* sk_buff *skb,
182	int gso_hs)
183	{
184	int num_sge = `1` + skb_shinfo(skb)->nr_frags;
185	int skb_hlen = skb_headlen(skb);
186
187	if (gso_hs < skb_hlen) {
188	num_sge++;
189	} else if (gso_hs > skb_hlen) {
190	if (net_ratelimit())
191	netdev_err(dev: ndev,
192	format: "TX nonlinear head: hs:%d, skb_hlen:%d\n",
193	gso_hs, skb_hlen);
194
195	return -EINVAL;
196	}
197
198	return num_sge;
199	}
200
201	/ Get the GSO packet's header size /
202	static int mana_get_gso_hs(struct sk_buff *skb)
203	{
204	int gso_hs;
205
206	if (skb->encapsulation) {
207	gso_hs = skb_inner_tcp_all_headers(skb);
208	} else {
209	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
210	gso_hs = skb_transport_offset(skb) +
211	sizeof(struct udphdr);
212	} else {
213	gso_hs = skb_tcp_all_headers(skb);
214	}
215	}
216
217	return gso_hs;
218	}
219
220	netdev_tx_t mana_start_xmit(struct sk_buff skb, struct* net_device *ndev)
221	{
222	enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT;
223	struct mana_port_context *apc = netdev_priv(dev: ndev);
224	int gso_hs = `0`; / zero for non-GSO pkts /
225	u16 txq_idx = skb_get_queue_mapping(skb);
226	struct gdma_dev *gd = apc->ac->gdma_dev;
227	bool ipv4 = false, ipv6 = false;
228	struct mana_tx_package pkg = {};
229	struct netdev_queue *net_txq;
230	struct mana_stats_tx *tx_stats;
231	struct gdma_queue *gdma_sq;
232	unsigned int csum_type;
233	struct mana_txq *txq;
234	struct mana_cq *cq;
235	int err, len;
236
237	if (unlikely(!apc->port_is_up))
238	goto tx_drop;
239
240	if (skb_cow_head(skb, MANA_HEADROOM))
241	goto tx_drop_count;
242
243	txq = &apc->tx_qp[txq_idx].txq;
244	gdma_sq = txq->gdma_sq;
245	cq = &apc->tx_qp[txq_idx].tx_cq;
246	tx_stats = &txq->stats;
247
248	pkg.tx_oob.s_oob.vcq_num = cq->gdma_id;
249	pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame;
250
251	if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) {
252	pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset;
253	pkt_fmt = MANA_LONG_PKT_FMT;
254	} else {
255	pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset;
256	}
257
258	if (skb_vlan_tag_present(skb)) {
259	pkt_fmt = MANA_LONG_PKT_FMT;
260	pkg.tx_oob.l_oob.inject_vlan_pri_tag = `1`;
261	pkg.tx_oob.l_oob.pcp = skb_vlan_tag_get_prio(skb);
262	pkg.tx_oob.l_oob.dei = skb_vlan_tag_get_cfi(skb);
263	pkg.tx_oob.l_oob.vlan_id = skb_vlan_tag_get_id(skb);
264	}
265
266	pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt;
267
268	if (pkt_fmt == MANA_SHORT_PKT_FMT) {
269	pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob);
270	u64_stats_update_begin(syncp: &tx_stats->syncp);
271	tx_stats->short_pkt_fmt++;
272	u64_stats_update_end(syncp: &tx_stats->syncp);
273	} else {
274	pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob);
275	u64_stats_update_begin(syncp: &tx_stats->syncp);
276	tx_stats->long_pkt_fmt++;
277	u64_stats_update_end(syncp: &tx_stats->syncp);
278	}
279
280	pkg.wqe_req.inline_oob_data = &pkg.tx_oob;
281	pkg.wqe_req.flags = `0`;
282	pkg.wqe_req.client_data_unit = `0`;
283
284	pkg.wqe_req.num_sge = `1` + skb_shinfo(skb)->nr_frags;
285
286	if (skb->protocol == htons(ETH_P_IP))
287	ipv4 = true;
288	else if (skb->protocol == htons(ETH_P_IPV6))
289	ipv6 = true;
290
291	if (skb_is_gso(skb)) {
292	int num_sge;
293
294	gso_hs = mana_get_gso_hs(skb);
295
296	num_sge = mana_fix_skb_head(ndev, skb, gso_hs);
297	if (num_sge > `0`)
298	pkg.wqe_req.num_sge = num_sge;
299	else
300	goto tx_drop_count;
301
302	u64_stats_update_begin(syncp: &tx_stats->syncp);
303	if (skb->encapsulation) {
304	tx_stats->tso_inner_packets++;
305	tx_stats->tso_inner_bytes += skb->len - gso_hs;
306	} else {
307	tx_stats->tso_packets++;
308	tx_stats->tso_bytes += skb->len - gso_hs;
309	}
310	u64_stats_update_end(syncp: &tx_stats->syncp);
311
312	pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
313	pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
314
315	pkg.tx_oob.s_oob.comp_iphdr_csum = `1`;
316	pkg.tx_oob.s_oob.comp_tcp_csum = `1`;
317	pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);
318
319	pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size;
320	pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL \| GDMA_WR_PAD_BY_SGE0;
321	if (ipv4) {
322	ip_hdr(skb)->tot_len = `0`;
323	ip_hdr(skb)->check = `0`;
324	tcp_hdr(skb)->check =
325	~csum_tcpudp_magic(saddr: ip_hdr(skb)->saddr,
326	daddr: ip_hdr(skb)->daddr, len: `0`,
327	IPPROTO_TCP, sum: `0`);
328	} else {
329	ipv6_hdr(skb)->payload_len = `0`;
330	tcp_hdr(skb)->check =
331	~csum_ipv6_magic(saddr: &ipv6_hdr(skb)->saddr,
332	daddr: &ipv6_hdr(skb)->daddr, len: `0`,
333	IPPROTO_TCP, sum: `0`);
334	}
335	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
336	csum_type = mana_checksum_info(skb);
337
338	u64_stats_update_begin(syncp: &tx_stats->syncp);
339	tx_stats->csum_partial++;
340	u64_stats_update_end(syncp: &tx_stats->syncp);
341
342	if (csum_type == IPPROTO_TCP) {
343	pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
344	pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
345
346	pkg.tx_oob.s_oob.comp_tcp_csum = `1`;
347	pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);
348
349	} else if (csum_type == IPPROTO_UDP) {
350	pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
351	pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
352
353	pkg.tx_oob.s_oob.comp_udp_csum = `1`;
354	} else {
355	/ Can't do offload of this type of checksum /
356	if (skb_checksum_help(skb))
357	goto tx_drop_count;
358	}
359	}
360
361	WARN_ON_ONCE(pkg.wqe_req.num_sge > MAX_TX_WQE_SGL_ENTRIES);
362
363	if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) {
364	pkg.wqe_req.sgl = pkg.sgl_array;
365	} else {
366	pkg.sgl_ptr = kmalloc_array(n: pkg.wqe_req.num_sge,
367	size: sizeof(struct gdma_sge),
368	GFP_ATOMIC);
369	if (!pkg.sgl_ptr)
370	goto tx_drop_count;
371
372	pkg.wqe_req.sgl = pkg.sgl_ptr;
373	}
374
375	if (mana_map_skb(skb, apc, tp: &pkg, gso_hs)) {
376	u64_stats_update_begin(syncp: &tx_stats->syncp);
377	tx_stats->mana_map_err++;
378	u64_stats_update_end(syncp: &tx_stats->syncp);
379	goto free_sgl_ptr;
380	}
381
382	skb_queue_tail(list: &txq->pending_skbs, newsk: skb);
383
384	len = skb->len;
385	net_txq = netdev_get_tx_queue(dev: ndev, index: txq_idx);
386
387	err = mana_gd_post_work_request(wq: gdma_sq, wqe_req: &pkg.wqe_req,
388	wqe_info: (struct gdma_posted_wqe_info *)skb->cb);
389	if (!mana_can_tx(wq: gdma_sq)) {
390	netif_tx_stop_queue(dev_queue: net_txq);
391	apc->eth_stats.stop_queue++;
392	}
393
394	if (err) {
395	(void)skb_dequeue_tail(list: &txq->pending_skbs);
396	netdev_warn(dev: ndev, format: "Failed to post TX OOB: %d\n", err);
397	err = NETDEV_TX_BUSY;
398	goto tx_busy;
399	}
400
401	err = NETDEV_TX_OK;
402	atomic_inc(v: &txq->pending_sends);
403
404	mana_gd_wq_ring_doorbell(gc: gd->gdma_context, queue: gdma_sq);
405
406	/ skb may be freed after mana_gd_post_work_request. Do not use it. /
407	skb = NULL;
408
409	tx_stats = &txq->stats;
410	u64_stats_update_begin(syncp: &tx_stats->syncp);
411	tx_stats->packets++;
412	tx_stats->bytes += len;
413	u64_stats_update_end(syncp: &tx_stats->syncp);
414
415	tx_busy:
416	if (netif_tx_queue_stopped(dev_queue: net_txq) && mana_can_tx(wq: gdma_sq)) {
417	netif_tx_wake_queue(dev_queue: net_txq);
418	apc->eth_stats.wake_queue++;
419	}
420
421	kfree(objp: pkg.sgl_ptr);
422	return err;
423
424	free_sgl_ptr:
425	kfree(objp: pkg.sgl_ptr);
426	tx_drop_count:
427	ndev->stats.tx_dropped++;
428	tx_drop:
429	dev_kfree_skb_any(skb);
430	return NETDEV_TX_OK;
431	}
432
433	static void mana_get_stats64(struct net_device *ndev,
434	struct rtnl_link_stats64 *st)
435	{
436	struct mana_port_context *apc = netdev_priv(dev: ndev);
437	unsigned int num_queues = apc->num_queues;
438	struct mana_stats_rx *rx_stats;
439	struct mana_stats_tx *tx_stats;
440	unsigned int start;
441	u64 packets, bytes;
442	int q;
443
444	if (!apc->port_is_up)
445	return;
446
447	netdev_stats_to_stats64(stats64: st, netdev_stats: &ndev->stats);
448
449	for (q = `0`; q < num_queues; q++) {
450	rx_stats = &apc->rxqs[q]->stats;
451
452	do {
453	start = u64_stats_fetch_begin(syncp: &rx_stats->syncp);
454	packets = rx_stats->packets;
455	bytes = rx_stats->bytes;
456	} while (u64_stats_fetch_retry(syncp: &rx_stats->syncp, start));
457
458	st->rx_packets += packets;
459	st->rx_bytes += bytes;
460	}
461
462	for (q = `0`; q < num_queues; q++) {
463	tx_stats = &apc->tx_qp[q].txq.stats;
464
465	do {
466	start = u64_stats_fetch_begin(syncp: &tx_stats->syncp);
467	packets = tx_stats->packets;
468	bytes = tx_stats->bytes;
469	} while (u64_stats_fetch_retry(syncp: &tx_stats->syncp, start));
470
471	st->tx_packets += packets;
472	st->tx_bytes += bytes;
473	}
474	}
475
476	static int mana_get_tx_queue(struct net_device ndev, struct* sk_buff *skb,
477	int old_q)
478	{
479	struct mana_port_context *apc = netdev_priv(dev: ndev);
480	u32 hash = skb_get_hash(skb);
481	struct sock *sk = skb->sk;
482	int txq;
483
484	txq = apc->indir_table[hash & MANA_INDIRECT_TABLE_MASK];
485
486	if (txq != old_q && sk && sk_fullsock(sk) &&
487	rcu_access_pointer(sk->sk_dst_cache))
488	sk_tx_queue_set(sk, tx_queue: txq);
489
490	return txq;
491	}
492
493	static u16 mana_select_queue(struct net_device ndev, struct* sk_buff *skb,
494	struct net_device *sb_dev)
495	{
496	int txq;
497
498	if (ndev->real_num_tx_queues == `1`)
499	return `0`;
500
501	txq = sk_tx_queue_get(sk: skb->sk);
502
503	if (txq < `0` \|\| skb->ooo_okay \|\| txq >= ndev->real_num_tx_queues) {
504	if (skb_rx_queue_recorded(skb))
505	txq = skb_get_rx_queue(skb);
506	else
507	txq = mana_get_tx_queue(ndev, skb, old_q: txq);
508	}
509
510	return txq;
511	}
512
513	/ Release pre-allocated RX buffers /
514	static void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc)
515	{
516	struct device *dev;
517	int i;
518
519	dev = mpc->ac->gdma_dev->gdma_context->dev;
520
521	if (!mpc->rxbufs_pre)
522	goto out1;
523
524	if (!mpc->das_pre)
525	goto out2;
526
527	while (mpc->rxbpre_total) {
528	i = --mpc->rxbpre_total;
529	dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize,
530	DMA_FROM_DEVICE);
531	put_page(page: virt_to_head_page(x: mpc->rxbufs_pre[i]));
532	}
533
534	kfree(objp: mpc->das_pre);
535	mpc->das_pre = NULL;
536
537	out2:
538	kfree(objp: mpc->rxbufs_pre);
539	mpc->rxbufs_pre = NULL;
540
541	out1:
542	mpc->rxbpre_datasize = `0`;
543	mpc->rxbpre_alloc_size = `0`;
544	mpc->rxbpre_headroom = `0`;
545	}
546
547	/ Get a buffer from the pre-allocated RX buffers /
548	static void mana_get_rxbuf_pre(struct* mana_rxq rxq, dma_addr_t da)
549	{
550	struct net_device *ndev = rxq->ndev;
551	struct mana_port_context *mpc;
552	void *va;
553
554	mpc = netdev_priv(dev: ndev);
555
556	if (!mpc->rxbufs_pre \|\| !mpc->das_pre \|\| !mpc->rxbpre_total) {
557	netdev_err(dev: ndev, format: "No RX pre-allocated bufs\n");
558	return NULL;
559	}
560
561	/ Check sizes to catch unexpected coding error /
562	if (mpc->rxbpre_datasize != rxq->datasize) {
563	netdev_err(dev: ndev, format: "rxbpre_datasize mismatch: %u: %u\n",
564	mpc->rxbpre_datasize, rxq->datasize);
565	return NULL;
566	}
567
568	if (mpc->rxbpre_alloc_size != rxq->alloc_size) {
569	netdev_err(dev: ndev, format: "rxbpre_alloc_size mismatch: %u: %u\n",
570	mpc->rxbpre_alloc_size, rxq->alloc_size);
571	return NULL;
572	}
573
574	if (mpc->rxbpre_headroom != rxq->headroom) {
575	netdev_err(dev: ndev, format: "rxbpre_headroom mismatch: %u: %u\n",
576	mpc->rxbpre_headroom, rxq->headroom);
577	return NULL;
578	}
579
580	mpc->rxbpre_total--;
581
582	*da = mpc->das_pre[mpc->rxbpre_total];
583	va = mpc->rxbufs_pre[mpc->rxbpre_total];
584	mpc->rxbufs_pre[mpc->rxbpre_total] = NULL;
585
586	/ Deallocate the array after all buffers are gone /
587	if (!mpc->rxbpre_total)
588	mana_pre_dealloc_rxbufs(mpc);
589
590	return va;
591	}
592
593	/ Get RX buffer's data size, alloc size, XDP headroom based on MTU /
594	static void mana_get_rxbuf_cfg(int mtu, u32 datasize, u32 alloc_size,
595	u32 *headroom)
596	{
597	if (mtu > MANA_XDP_MTU_MAX)
598	headroom = `0`; /* no support for XDP /
599	else
600	*headroom = XDP_PACKET_HEADROOM;
601
602	alloc_size = mtu + MANA_RXBUF_PAD + headroom;
603
604	*datasize = mtu + ETH_HLEN;
605	}
606
607	static int mana_pre_alloc_rxbufs(struct mana_port_context mpc, int* new_mtu)
608	{
609	struct device *dev;
610	struct page *page;
611	dma_addr_t da;
612	int num_rxb;
613	void *va;
614	int i;
615
616	mana_get_rxbuf_cfg(mtu: new_mtu, datasize: &mpc->rxbpre_datasize,
617	alloc_size: &mpc->rxbpre_alloc_size, headroom: &mpc->rxbpre_headroom);
618
619	dev = mpc->ac->gdma_dev->gdma_context->dev;
620
621	num_rxb = mpc->num_queues * RX_BUFFERS_PER_QUEUE;
622
623	WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n");
624	mpc->rxbufs_pre = kmalloc_array(n: num_rxb, size: sizeof(void *), GFP_KERNEL);
625	if (!mpc->rxbufs_pre)
626	goto error;
627
628	mpc->das_pre = kmalloc_array(n: num_rxb, size: sizeof(dma_addr_t), GFP_KERNEL);
629	if (!mpc->das_pre)
630	goto error;
631
632	mpc->rxbpre_total = `0`;
633
634	for (i = `0`; i < num_rxb; i++) {
635	if (mpc->rxbpre_alloc_size > PAGE_SIZE) {
636	va = netdev_alloc_frag(fragsz: mpc->rxbpre_alloc_size);
637	if (!va)
638	goto error;
639
640	page = virt_to_head_page(x: va);
641	/ Check if the frag falls back to single page /
642	if (compound_order(page) <
643	get_order(size: mpc->rxbpre_alloc_size)) {
644	put_page(page);
645	goto error;
646	}
647	} else {
648	page = dev_alloc_page();
649	if (!page)
650	goto error;
651
652	va = page_to_virt(page);
653	}
654
655	da = dma_map_single(dev, va + mpc->rxbpre_headroom,
656	mpc->rxbpre_datasize, DMA_FROM_DEVICE);
657	if (dma_mapping_error(dev, dma_addr: da)) {
658	put_page(page: virt_to_head_page(x: va));
659	goto error;
660	}
661
662	mpc->rxbufs_pre[i] = va;
663	mpc->das_pre[i] = da;
664	mpc->rxbpre_total = i + `1`;
665	}
666
667	return `0`;
668
669	error:
670	mana_pre_dealloc_rxbufs(mpc);
671	return -ENOMEM;
672	}
673
674	static int mana_change_mtu(struct net_device ndev, int* new_mtu)
675	{
676	struct mana_port_context *mpc = netdev_priv(dev: ndev);
677	unsigned int old_mtu = ndev->mtu;
678	int err;
679
680	/ Pre-allocate buffers to prevent failure in mana_attach later /
681	err = mana_pre_alloc_rxbufs(mpc, new_mtu);
682	if (err) {
683	netdev_err(dev: ndev, format: "Insufficient memory for new MTU\n");
684	return err;
685	}
686
687	err = mana_detach(ndev, from_close: false);
688	if (err) {
689	netdev_err(dev: ndev, format: "mana_detach failed: %d\n", err);
690	goto out;
691	}
692
693	ndev->mtu = new_mtu;
694
695	err = mana_attach(ndev);
696	if (err) {
697	netdev_err(dev: ndev, format: "mana_attach failed: %d\n", err);
698	ndev->mtu = old_mtu;
699	}
700
701	out:
702	mana_pre_dealloc_rxbufs(mpc);
703	return err;
704	}
705
706	static const struct net_device_ops mana_devops = {
707	.ndo_open = mana_open,
708	.ndo_stop = mana_close,
709	.ndo_select_queue = mana_select_queue,
710	.ndo_start_xmit = mana_start_xmit,
711	.ndo_validate_addr = eth_validate_addr,
712	.ndo_get_stats64 = mana_get_stats64,
713	.ndo_bpf = mana_bpf,
714	.ndo_xdp_xmit = mana_xdp_xmit,
715	.ndo_change_mtu = mana_change_mtu,
716	};
717
718	static void mana_cleanup_port_context(struct mana_port_context *apc)
719	{
720	kfree(objp: apc->rxqs);
721	apc->rxqs = NULL;
722	}
723
724	static int mana_init_port_context(struct mana_port_context *apc)
725	{
726	apc->rxqs = kcalloc(n: apc->num_queues, size: sizeof(struct mana_rxq *),
727	GFP_KERNEL);
728
729	return !apc->rxqs ? -ENOMEM : `0`;
730	}
731
732	static int mana_send_request(struct mana_context ac, void* *in_buf,
733	u32 in_len, void *out_buf, u32 out_len)
734	{
735	struct gdma_context *gc = ac->gdma_dev->gdma_context;
736	struct gdma_resp_hdr *resp = out_buf;
737	struct gdma_req_hdr *req = in_buf;
738	struct device *dev = gc->dev;
739	static atomic_t activity_id;
740	int err;
741
742	req->dev_id = gc->mana.dev_id;
743	req->activity_id = atomic_inc_return(v: &activity_id);
744
745	err = mana_gd_send_request(gc, req_len: in_len, req: in_buf, resp_len: out_len,
746	resp: out_buf);
747	if (err \|\| resp->status) {
748	dev_err(dev, "Failed to send mana message: %d, 0x%x\n",
749	err, resp->status);
750	return err ? err : -EPROTO;
751	}
752
753	if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 \|\|
754	req->activity_id != resp->activity_id) {
755	dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n",
756	req->dev_id.as_uint32, resp->dev_id.as_uint32,
757	req->activity_id, resp->activity_id);
758	return -EPROTO;
759	}
760
761	return `0`;
762	}
763
764	static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr,
765	const enum mana_command_code expected_code,
766	const u32 min_size)
767	{
768	if (resp_hdr->response.msg_type != expected_code)
769	return -EPROTO;
770
771	if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1)
772	return -EPROTO;
773
774	if (resp_hdr->response.msg_size < min_size)
775	return -EPROTO;
776
777	return `0`;
778	}
779
780	static int mana_pf_register_hw_vport(struct mana_port_context *apc)
781	{
782	struct mana_register_hw_vport_resp resp = {};
783	struct mana_register_hw_vport_req req = {};
784	int err;
785
786	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_REGISTER_HW_PORT,
787	req_size: sizeof(req), resp_size: sizeof(resp));
788	req.attached_gfid = `1`;
789	req.is_pf_default_vport = `1`;
790	req.allow_all_ether_types = `1`;
791
792	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
793	out_len: sizeof(resp));
794	if (err) {
795	netdev_err(dev: apc->ndev, format: "Failed to register hw vPort: %d\n", err);
796	return err;
797	}
798
799	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_REGISTER_HW_PORT,
800	min_size: sizeof(resp));
801	if (err \|\| resp.hdr.status) {
802	netdev_err(dev: apc->ndev, format: "Failed to register hw vPort: %d, 0x%x\n",
803	err, resp.hdr.status);
804	return err ? err : -EPROTO;
805	}
806
807	apc->port_handle = resp.hw_vport_handle;
808	return `0`;
809	}
810
811	static void mana_pf_deregister_hw_vport(struct mana_port_context *apc)
812	{
813	struct mana_deregister_hw_vport_resp resp = {};
814	struct mana_deregister_hw_vport_req req = {};
815	int err;
816
817	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_DEREGISTER_HW_PORT,
818	req_size: sizeof(req), resp_size: sizeof(resp));
819	req.hw_vport_handle = apc->port_handle;
820
821	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
822	out_len: sizeof(resp));
823	if (err) {
824	netdev_err(dev: apc->ndev, format: "Failed to unregister hw vPort: %d\n",
825	err);
826	return;
827	}
828
829	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_DEREGISTER_HW_PORT,
830	min_size: sizeof(resp));
831	if (err \|\| resp.hdr.status)
832	netdev_err(dev: apc->ndev,
833	format: "Failed to deregister hw vPort: %d, 0x%x\n",
834	err, resp.hdr.status);
835	}
836
837	static int mana_pf_register_filter(struct mana_port_context *apc)
838	{
839	struct mana_register_filter_resp resp = {};
840	struct mana_register_filter_req req = {};
841	int err;
842
843	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_REGISTER_FILTER,
844	req_size: sizeof(req), resp_size: sizeof(resp));
845	req.vport = apc->port_handle;
846	memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN);
847
848	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
849	out_len: sizeof(resp));
850	if (err) {
851	netdev_err(dev: apc->ndev, format: "Failed to register filter: %d\n", err);
852	return err;
853	}
854
855	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_REGISTER_FILTER,
856	min_size: sizeof(resp));
857	if (err \|\| resp.hdr.status) {
858	netdev_err(dev: apc->ndev, format: "Failed to register filter: %d, 0x%x\n",
859	err, resp.hdr.status);
860	return err ? err : -EPROTO;
861	}
862
863	apc->pf_filter_handle = resp.filter_handle;
864	return `0`;
865	}
866
867	static void mana_pf_deregister_filter(struct mana_port_context *apc)
868	{
869	struct mana_deregister_filter_resp resp = {};
870	struct mana_deregister_filter_req req = {};
871	int err;
872
873	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_DEREGISTER_FILTER,
874	req_size: sizeof(req), resp_size: sizeof(resp));
875	req.filter_handle = apc->pf_filter_handle;
876
877	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
878	out_len: sizeof(resp));
879	if (err) {
880	netdev_err(dev: apc->ndev, format: "Failed to unregister filter: %d\n",
881	err);
882	return;
883	}
884
885	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_DEREGISTER_FILTER,
886	min_size: sizeof(resp));
887	if (err \|\| resp.hdr.status)
888	netdev_err(dev: apc->ndev,
889	format: "Failed to deregister filter: %d, 0x%x\n",
890	err, resp.hdr.status);
891	}
892
893	static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver,
894	u32 proto_minor_ver, u32 proto_micro_ver,
895	u16 *max_num_vports)
896	{
897	struct gdma_context *gc = ac->gdma_dev->gdma_context;
898	struct mana_query_device_cfg_resp resp = {};
899	struct mana_query_device_cfg_req req = {};
900	struct device *dev = gc->dev;
901	int err = `0`;
902
903	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_QUERY_DEV_CONFIG,
904	req_size: sizeof(req), resp_size: sizeof(resp));
905
906	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;
907
908	req.proto_major_ver = proto_major_ver;
909	req.proto_minor_ver = proto_minor_ver;
910	req.proto_micro_ver = proto_micro_ver;
911
912	err = mana_send_request(ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp, out_len: sizeof(resp));
913	if (err) {
914	dev_err(dev, "Failed to query config: %d", err);
915	return err;
916	}
917
918	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_QUERY_DEV_CONFIG,
919	min_size: sizeof(resp));
920	if (err \|\| resp.hdr.status) {
921	dev_err(dev, "Invalid query result: %d, 0x%x\n", err,
922	resp.hdr.status);
923	if (!err)
924	err = -EPROTO;
925	return err;
926	}
927
928	*max_num_vports = resp.max_num_vports;
929
930	if (resp.hdr.response.msg_version == GDMA_MESSAGE_V2)
931	gc->adapter_mtu = resp.adapter_mtu;
932	else
933	gc->adapter_mtu = ETH_FRAME_LEN;
934
935	return `0`;
936	}
937
938	static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index,
939	u32 max_sq, u32 max_rq, u32 *num_indir_entry)
940	{
941	struct mana_query_vport_cfg_resp resp = {};
942	struct mana_query_vport_cfg_req req = {};
943	int err;
944
945	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_QUERY_VPORT_CONFIG,
946	req_size: sizeof(req), resp_size: sizeof(resp));
947
948	req.vport_index = vport_index;
949
950	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
951	out_len: sizeof(resp));
952	if (err)
953	return err;
954
955	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_QUERY_VPORT_CONFIG,
956	min_size: sizeof(resp));
957	if (err)
958	return err;
959
960	if (resp.hdr.status)
961	return -EPROTO;
962
963	*max_sq = resp.max_num_sq;
964	*max_rq = resp.max_num_rq;
965	*num_indir_entry = resp.num_indirection_ent;
966
967	apc->port_handle = resp.vport;
968	ether_addr_copy(dst: apc->mac_addr, src: resp.mac_addr);
969
970	return `0`;
971	}
972
973	void mana_uncfg_vport(struct mana_port_context *apc)
974	{
975	mutex_lock(&apc->vport_mutex);
976	apc->vport_use_count--;
977	WARN_ON(apc->vport_use_count < `0`);
978	mutex_unlock(lock: &apc->vport_mutex);
979	}
980	EXPORT_SYMBOL_NS(mana_uncfg_vport, NET_MANA);
981
982	int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id,
983	u32 doorbell_pg_id)
984	{
985	struct mana_config_vport_resp resp = {};
986	struct mana_config_vport_req req = {};
987	int err;
988
989	/ This function is used to program the Ethernet port in the hardware*
990	* table. It can be called from the Ethernet driver or the RDMA driver.
991	*
992	* For Ethernet usage, the hardware supports only one active user on a
993	* physical port. The driver checks on the port usage before programming
994	* the hardware when creating the RAW QP (RDMA driver) or exposing the
995	* device to kernel NET layer (Ethernet driver).
996	*
997	* Because the RDMA driver doesn't know in advance which QP type the
998	* user will create, it exposes the device with all its ports. The user
999	* may not be able to create RAW QP on a port if this port is already
1000	* in used by the Ethernet driver from the kernel.
1001	*
1002	* This physical port limitation only applies to the RAW QP. For RC QP,
1003	* the hardware doesn't have this limitation. The user can create RC
1004	* QPs on a physical port up to the hardware limits independent of the
1005	* Ethernet usage on the same port.
1006	*/
1007	mutex_lock(&apc->vport_mutex);
1008	if (apc->vport_use_count > `0`) {
1009	mutex_unlock(lock: &apc->vport_mutex);
1010	return -EBUSY;
1011	}
1012	apc->vport_use_count++;
1013	mutex_unlock(lock: &apc->vport_mutex);
1014
1015	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_CONFIG_VPORT_TX,
1016	req_size: sizeof(req), resp_size: sizeof(resp));
1017	req.vport = apc->port_handle;
1018	req.pdid = protection_dom_id;
1019	req.doorbell_pageid = doorbell_pg_id;
1020
1021	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
1022	out_len: sizeof(resp));
1023	if (err) {
1024	netdev_err(dev: apc->ndev, format: "Failed to configure vPort: %d\n", err);
1025	goto out;
1026	}
1027
1028	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_CONFIG_VPORT_TX,
1029	min_size: sizeof(resp));
1030	if (err \|\| resp.hdr.status) {
1031	netdev_err(dev: apc->ndev, format: "Failed to configure vPort: %d, 0x%x\n",
1032	err, resp.hdr.status);
1033	if (!err)
1034	err = -EPROTO;
1035
1036	goto out;
1037	}
1038
1039	apc->tx_shortform_allowed = resp.short_form_allowed;
1040	apc->tx_vp_offset = resp.tx_vport_offset;
1041
1042	netdev_info(dev: apc->ndev, format: "Configured vPort %llu PD %u DB %u\n",
1043	apc->port_handle, protection_dom_id, doorbell_pg_id);
1044	out:
1045	if (err)
1046	mana_uncfg_vport(apc);
1047
1048	return err;
1049	}
1050	EXPORT_SYMBOL_NS(mana_cfg_vport, NET_MANA);
1051
1052	static int mana_cfg_vport_steering(struct mana_port_context *apc,
1053	enum TRI_STATE rx,
1054	bool update_default_rxobj, bool update_key,
1055	bool update_tab)
1056	{
1057	u16 num_entries = MANA_INDIRECT_TABLE_SIZE;
1058	struct mana_cfg_rx_steer_req_v2 *req;
1059	struct mana_cfg_rx_steer_resp resp = {};
1060	struct net_device *ndev = apc->ndev;
1061	mana_handle_t *req_indir_tab;
1062	u32 req_buf_size;
1063	int err;
1064
1065	req_buf_size = sizeof(req) + sizeof(mana_handle_t) num_entries;
1066	req = kzalloc(size: req_buf_size, GFP_KERNEL);
1067	if (!req)
1068	return -ENOMEM;
1069
1070	mana_gd_init_req_hdr(hdr: &req->hdr, code: MANA_CONFIG_VPORT_RX, req_size: req_buf_size,
1071	resp_size: sizeof(resp));
1072
1073	req->hdr.req.msg_version = GDMA_MESSAGE_V2;
1074
1075	req->vport = apc->port_handle;
1076	req->num_indir_entries = num_entries;
1077	req->indir_tab_offset = sizeof(*req);
1078	req->rx_enable = rx;
1079	req->rss_enable = apc->rss_state;
1080	req->update_default_rxobj = update_default_rxobj;
1081	req->update_hashkey = update_key;
1082	req->update_indir_tab = update_tab;
1083	req->default_rxobj = apc->default_rxobj;
1084	req->cqe_coalescing_enable = `0`;
1085
1086	if (update_key)
1087	memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);
1088
1089	if (update_tab) {
1090	req_indir_tab = (mana_handle_t *)(req + `1`);
1091	memcpy(req_indir_tab, apc->rxobj_table,
1092	req->num_indir_entries * sizeof(mana_handle_t));
1093	}
1094
1095	err = mana_send_request(ac: apc->ac, in_buf: req, in_len: req_buf_size, out_buf: &resp,
1096	out_len: sizeof(resp));
1097	if (err) {
1098	netdev_err(dev: ndev, format: "Failed to configure vPort RX: %d\n", err);
1099	goto out;
1100	}
1101
1102	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_CONFIG_VPORT_RX,
1103	min_size: sizeof(resp));
1104	if (err) {
1105	netdev_err(dev: ndev, format: "vPort RX configuration failed: %d\n", err);
1106	goto out;
1107	}
1108
1109	if (resp.hdr.status) {
1110	netdev_err(dev: ndev, format: "vPort RX configuration failed: 0x%x\n",
1111	resp.hdr.status);
1112	err = -EPROTO;
1113	}
1114
1115	netdev_info(dev: ndev, format: "Configured steering vPort %llu entries %u\n",
1116	apc->port_handle, num_entries);
1117	out:
1118	kfree(objp: req);
1119	return err;
1120	}
1121
1122	int mana_create_wq_obj(struct mana_port_context *apc,
1123	mana_handle_t vport,
1124	u32 wq_type, struct mana_obj_spec *wq_spec,
1125	struct mana_obj_spec *cq_spec,
1126	mana_handle_t *wq_obj)
1127	{
1128	struct mana_create_wqobj_resp resp = {};
1129	struct mana_create_wqobj_req req = {};
1130	struct net_device *ndev = apc->ndev;
1131	int err;
1132
1133	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_CREATE_WQ_OBJ,
1134	req_size: sizeof(req), resp_size: sizeof(resp));
1135	req.vport = vport;
1136	req.wq_type = wq_type;
1137	req.wq_gdma_region = wq_spec->gdma_region;
1138	req.cq_gdma_region = cq_spec->gdma_region;
1139	req.wq_size = wq_spec->queue_size;
1140	req.cq_size = cq_spec->queue_size;
1141	req.cq_moderation_ctx_id = cq_spec->modr_ctx_id;
1142	req.cq_parent_qid = cq_spec->attached_eq;
1143
1144	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
1145	out_len: sizeof(resp));
1146	if (err) {
1147	netdev_err(dev: ndev, format: "Failed to create WQ object: %d\n", err);
1148	goto out;
1149	}
1150
1151	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_CREATE_WQ_OBJ,
1152	min_size: sizeof(resp));
1153	if (err \|\| resp.hdr.status) {
1154	netdev_err(dev: ndev, format: "Failed to create WQ object: %d, 0x%x\n", err,
1155	resp.hdr.status);
1156	if (!err)
1157	err = -EPROTO;
1158	goto out;
1159	}
1160
1161	if (resp.wq_obj == INVALID_MANA_HANDLE) {
1162	netdev_err(dev: ndev, format: "Got an invalid WQ object handle\n");
1163	err = -EPROTO;
1164	goto out;
1165	}
1166
1167	*wq_obj = resp.wq_obj;
1168	wq_spec->queue_index = resp.wq_id;
1169	cq_spec->queue_index = resp.cq_id;
1170
1171	return `0`;
1172	out:
1173	return err;
1174	}
1175	EXPORT_SYMBOL_NS(mana_create_wq_obj, NET_MANA);
1176
1177	void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type,
1178	mana_handle_t wq_obj)
1179	{
1180	struct mana_destroy_wqobj_resp resp = {};
1181	struct mana_destroy_wqobj_req req = {};
1182	struct net_device *ndev = apc->ndev;
1183	int err;
1184
1185	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_DESTROY_WQ_OBJ,
1186	req_size: sizeof(req), resp_size: sizeof(resp));
1187	req.wq_type = wq_type;
1188	req.wq_obj_handle = wq_obj;
1189
1190	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
1191	out_len: sizeof(resp));
1192	if (err) {
1193	netdev_err(dev: ndev, format: "Failed to destroy WQ object: %d\n", err);
1194	return;
1195	}
1196
1197	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_DESTROY_WQ_OBJ,
1198	min_size: sizeof(resp));
1199	if (err \|\| resp.hdr.status)
1200	netdev_err(dev: ndev, format: "Failed to destroy WQ object: %d, 0x%x\n", err,
1201	resp.hdr.status);
1202	}
1203	EXPORT_SYMBOL_NS(mana_destroy_wq_obj, NET_MANA);
1204
1205	static void mana_destroy_eq(struct mana_context *ac)
1206	{
1207	struct gdma_context *gc = ac->gdma_dev->gdma_context;
1208	struct gdma_queue *eq;
1209	int i;
1210
1211	if (!ac->eqs)
1212	return;
1213
1214	for (i = `0`; i < gc->max_num_queues; i++) {
1215	eq = ac->eqs[i].eq;
1216	if (!eq)
1217	continue;
1218
1219	mana_gd_destroy_queue(gc, queue: eq);
1220	}
1221
1222	kfree(objp: ac->eqs);
1223	ac->eqs = NULL;
1224	}
1225
1226	static int mana_create_eq(struct mana_context *ac)
1227	{
1228	struct gdma_dev *gd = ac->gdma_dev;
1229	struct gdma_context *gc = gd->gdma_context;
1230	struct gdma_queue_spec spec = {};
1231	int err;
1232	int i;
1233
1234	ac->eqs = kcalloc(n: gc->max_num_queues, size: sizeof(struct mana_eq),
1235	GFP_KERNEL);
1236	if (!ac->eqs)
1237	return -ENOMEM;
1238
1239	spec.type = GDMA_EQ;
1240	spec.monitor_avl_buf = false;
1241	spec.queue_size = EQ_SIZE;
1242	spec.eq.callback = NULL;
1243	spec.eq.context = ac->eqs;
1244	spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE;
1245
1246	for (i = `0`; i < gc->max_num_queues; i++) {
1247	spec.eq.msix_index = (i + `1`) % gc->num_msix_usable;
1248	err = mana_gd_create_mana_eq(gd, spec: &spec, queue_ptr: &ac->eqs[i].eq);
1249	if (err)
1250	goto out;
1251	}
1252
1253	return `0`;
1254	out:
1255	mana_destroy_eq(ac);
1256	return err;
1257	}
1258
1259	static int mana_fence_rq(struct mana_port_context apc, struct* mana_rxq *rxq)
1260	{
1261	struct mana_fence_rq_resp resp = {};
1262	struct mana_fence_rq_req req = {};
1263	int err;
1264
1265	init_completion(x: &rxq->fence_event);
1266
1267	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_FENCE_RQ,
1268	req_size: sizeof(req), resp_size: sizeof(resp));
1269	req.wq_obj_handle = rxq->rxobj;
1270
1271	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
1272	out_len: sizeof(resp));
1273	if (err) {
1274	netdev_err(dev: apc->ndev, format: "Failed to fence RQ %u: %d\n",
1275	rxq->rxq_idx, err);
1276	return err;
1277	}
1278
1279	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_FENCE_RQ, min_size: sizeof(resp));
1280	if (err \|\| resp.hdr.status) {
1281	netdev_err(dev: apc->ndev, format: "Failed to fence RQ %u: %d, 0x%x\n",
1282	rxq->rxq_idx, err, resp.hdr.status);
1283	if (!err)
1284	err = -EPROTO;
1285
1286	return err;
1287	}
1288
1289	if (wait_for_completion_timeout(x: &rxq->fence_event, timeout: `10` * HZ) == `0`) {
1290	netdev_err(dev: apc->ndev, format: "Failed to fence RQ %u: timed out\n",
1291	rxq->rxq_idx);
1292	return -ETIMEDOUT;
1293	}
1294
1295	return `0`;
1296	}
1297
1298	static void mana_fence_rqs(struct mana_port_context *apc)
1299	{
1300	unsigned int rxq_idx;
1301	struct mana_rxq *rxq;
1302	int err;
1303
1304	for (rxq_idx = `0`; rxq_idx < apc->num_queues; rxq_idx++) {
1305	rxq = apc->rxqs[rxq_idx];
1306	err = mana_fence_rq(apc, rxq);
1307
1308	/ In case of any error, use sleep instead. /
1309	if (err)
1310	msleep(msecs: `100`);
1311	}
1312	}
1313
1314	static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units)
1315	{
1316	u32 used_space_old;
1317	u32 used_space_new;
1318
1319	used_space_old = wq->head - wq->tail;
1320	used_space_new = wq->head - (wq->tail + num_units);
1321
1322	if (WARN_ON_ONCE(used_space_new > used_space_old))
1323	return -ERANGE;
1324
1325	wq->tail += num_units;
1326	return `0`;
1327	}
1328
1329	static void mana_unmap_skb(struct sk_buff skb, struct* mana_port_context *apc)
1330	{
1331	struct mana_skb_head ash = (struct* mana_skb_head *)skb->head;
1332	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
1333	struct device *dev = gc->dev;
1334	int hsg, i;
1335
1336	/ Number of SGEs of linear part /
1337	hsg = (skb_is_gso(skb) && skb_headlen(skb) > ash->size[`0`]) ? `2` : `1`;
1338
1339	for (i = `0`; i < hsg; i++)
1340	dma_unmap_single(dev, ash->dma_handle[i], ash->size[i],
1341	DMA_TO_DEVICE);
1342
1343	for (i = hsg; i < skb_shinfo(skb)->nr_frags + hsg; i++)
1344	dma_unmap_page(dev, ash->dma_handle[i], ash->size[i],
1345	DMA_TO_DEVICE);
1346	}
1347
1348	static void mana_poll_tx_cq(struct mana_cq *cq)
1349	{
1350	struct gdma_comp *completions = cq->gdma_comp_buf;
1351	struct gdma_posted_wqe_info *wqe_info;
1352	unsigned int pkt_transmitted = `0`;
1353	unsigned int wqe_unit_cnt = `0`;
1354	struct mana_txq *txq = cq->txq;
1355	struct mana_port_context *apc;
1356	struct netdev_queue *net_txq;
1357	struct gdma_queue *gdma_wq;
1358	unsigned int avail_space;
1359	struct net_device *ndev;
1360	struct sk_buff *skb;
1361	bool txq_stopped;
1362	int comp_read;
1363	int i;
1364
1365	ndev = txq->ndev;
1366	apc = netdev_priv(dev: ndev);
1367
1368	comp_read = mana_gd_poll_cq(cq: cq->gdma_cq, comp: completions,
1369	CQE_POLLING_BUFFER);
1370
1371	if (comp_read < `1`)
1372	return;
1373
1374	for (i = `0`; i < comp_read; i++) {
1375	struct mana_tx_comp_oob *cqe_oob;
1376
1377	if (WARN_ON_ONCE(!completions[i].is_sq))
1378	return;
1379
1380	cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data;
1381	if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type !=
1382	MANA_CQE_COMPLETION))
1383	return;
1384
1385	switch (cqe_oob->cqe_hdr.cqe_type) {
1386	case CQE_TX_OKAY:
1387	break;
1388
1389	case CQE_TX_SA_DROP:
1390	case CQE_TX_MTU_DROP:
1391	case CQE_TX_INVALID_OOB:
1392	case CQE_TX_INVALID_ETH_TYPE:
1393	case CQE_TX_HDR_PROCESSING_ERROR:
1394	case CQE_TX_VF_DISABLED:
1395	case CQE_TX_VPORT_IDX_OUT_OF_RANGE:
1396	case CQE_TX_VPORT_DISABLED:
1397	case CQE_TX_VLAN_TAGGING_VIOLATION:
1398	if (net_ratelimit())
1399	netdev_err(dev: ndev, format: "TX: CQE error %d\n",
1400	cqe_oob->cqe_hdr.cqe_type);
1401
1402	apc->eth_stats.tx_cqe_err++;
1403	break;
1404
1405	default:
1406	/ If the CQE type is unknown, log an error,*
1407	* and still free the SKB, update tail, etc.
1408	*/
1409	if (net_ratelimit())
1410	netdev_err(dev: ndev, format: "TX: unknown CQE type %d\n",
1411	cqe_oob->cqe_hdr.cqe_type);
1412
1413	apc->eth_stats.tx_cqe_unknown_type++;
1414	break;
1415	}
1416
1417	if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num))
1418	return;
1419
1420	skb = skb_dequeue(list: &txq->pending_skbs);
1421	if (WARN_ON_ONCE(!skb))
1422	return;
1423
1424	wqe_info = (struct gdma_posted_wqe_info *)skb->cb;
1425	wqe_unit_cnt += wqe_info->wqe_size_in_bu;
1426
1427	mana_unmap_skb(skb, apc);
1428
1429	napi_consume_skb(skb, budget: cq->budget);
1430
1431	pkt_transmitted++;
1432	}
1433
1434	if (WARN_ON_ONCE(wqe_unit_cnt == `0`))
1435	return;
1436
1437	mana_move_wq_tail(wq: txq->gdma_sq, num_units: wqe_unit_cnt);
1438
1439	gdma_wq = txq->gdma_sq;
1440	avail_space = mana_gd_wq_avail_space(wq: gdma_wq);
1441
1442	/ Ensure tail updated before checking q stop /
1443	smp_mb();
1444
1445	net_txq = txq->net_txq;
1446	txq_stopped = netif_tx_queue_stopped(dev_queue: net_txq);
1447
1448	/ Ensure checking txq_stopped before apc->port_is_up. /
1449	smp_rmb();
1450
1451	if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) {
1452	netif_tx_wake_queue(dev_queue: net_txq);
1453	apc->eth_stats.wake_queue++;
1454	}
1455
1456	if (atomic_sub_return(i: pkt_transmitted, v: &txq->pending_sends) < `0`)
1457	WARN_ON_ONCE(`1`);
1458
1459	cq->work_done = pkt_transmitted;
1460	}
1461
1462	static void mana_post_pkt_rxq(struct mana_rxq *rxq)
1463	{
1464	struct mana_recv_buf_oob *recv_buf_oob;
1465	u32 curr_index;
1466	int err;
1467
1468	curr_index = rxq->buf_index++;
1469	if (rxq->buf_index == rxq->num_rx_buf)
1470	rxq->buf_index = `0`;
1471
1472	recv_buf_oob = &rxq->rx_oobs[curr_index];
1473
1474	err = mana_gd_post_work_request(wq: rxq->gdma_rq, wqe_req: &recv_buf_oob->wqe_req,
1475	wqe_info: &recv_buf_oob->wqe_inf);
1476	if (WARN_ON_ONCE(err))
1477	return;
1478
1479	WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != `1`);
1480	}
1481
1482	static struct sk_buff mana_build_skb(struct* mana_rxq rxq, void* *buf_va,
1483	uint pkt_len, struct xdp_buff *xdp)
1484	{
1485	struct sk_buff *skb = napi_build_skb(data: buf_va, frag_size: rxq->alloc_size);
1486
1487	if (!skb)
1488	return NULL;
1489
1490	if (xdp->data_hard_start) {
1491	skb_reserve(skb, len: xdp->data - xdp->data_hard_start);
1492	skb_put(skb, len: xdp->data_end - xdp->data);
1493	return skb;
1494	}
1495
1496	skb_reserve(skb, len: rxq->headroom);
1497	skb_put(skb, len: pkt_len);
1498
1499	return skb;
1500	}
1501
1502	static void mana_rx_skb(void *buf_va, bool from_pool,
1503	struct mana_rxcomp_oob cqe, struct* mana_rxq *rxq)
1504	{
1505	struct mana_stats_rx *rx_stats = &rxq->stats;
1506	struct net_device *ndev = rxq->ndev;
1507	uint pkt_len = cqe->ppi[`0`].pkt_len;
1508	u16 rxq_idx = rxq->rxq_idx;
1509	struct napi_struct *napi;
1510	struct xdp_buff xdp = {};
1511	struct sk_buff *skb;
1512	u32 hash_value;
1513	u32 act;
1514
1515	rxq->rx_cq.work_done++;
1516	napi = &rxq->rx_cq.napi;
1517
1518	if (!buf_va) {
1519	++ndev->stats.rx_dropped;
1520	return;
1521	}
1522
1523	act = mana_run_xdp(ndev, rxq, xdp: &xdp, buf_va, pkt_len);
1524
1525	if (act == XDP_REDIRECT && !rxq->xdp_rc)
1526	return;
1527
1528	if (act != XDP_PASS && act != XDP_TX)
1529	goto drop_xdp;
1530
1531	skb = mana_build_skb(rxq, buf_va, pkt_len, xdp: &xdp);
1532
1533	if (!skb)
1534	goto drop;
1535
1536	if (from_pool)
1537	skb_mark_for_recycle(skb);
1538
1539	skb->dev = napi->dev;
1540
1541	skb->protocol = eth_type_trans(skb, dev: ndev);
1542	skb_checksum_none_assert(skb);
1543	skb_record_rx_queue(skb, rx_queue: rxq_idx);
1544
1545	if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) {
1546	if (cqe->rx_tcp_csum_succeed \|\| cqe->rx_udp_csum_succeed)
1547	skb->ip_summed = CHECKSUM_UNNECESSARY;
1548	}
1549
1550	if (cqe->rx_hashtype != `0` && (ndev->features & NETIF_F_RXHASH)) {
1551	hash_value = cqe->ppi[`0`].pkt_hash;
1552
1553	if (cqe->rx_hashtype & MANA_HASH_L4)
1554	skb_set_hash(skb, hash: hash_value, type: PKT_HASH_TYPE_L4);
1555	else
1556	skb_set_hash(skb, hash: hash_value, type: PKT_HASH_TYPE_L3);
1557	}
1558
1559	if (cqe->rx_vlantag_present) {
1560	u16 vlan_tci = cqe->rx_vlan_id;
1561
1562	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci);
1563	}
1564
1565	u64_stats_update_begin(syncp: &rx_stats->syncp);
1566	rx_stats->packets++;
1567	rx_stats->bytes += pkt_len;
1568
1569	if (act == XDP_TX)
1570	rx_stats->xdp_tx++;
1571	u64_stats_update_end(syncp: &rx_stats->syncp);
1572
1573	if (act == XDP_TX) {
1574	skb_set_queue_mapping(skb, queue_mapping: rxq_idx);
1575	mana_xdp_tx(skb, ndev);
1576	return;
1577	}
1578
1579	napi_gro_receive(napi, skb);
1580
1581	return;
1582
1583	drop_xdp:
1584	u64_stats_update_begin(syncp: &rx_stats->syncp);
1585	rx_stats->xdp_drop++;
1586	u64_stats_update_end(syncp: &rx_stats->syncp);
1587
1588	drop:
1589	if (from_pool) {
1590	page_pool_recycle_direct(pool: rxq->page_pool,
1591	page: virt_to_head_page(x: buf_va));
1592	} else {
1593	WARN_ON_ONCE(rxq->xdp_save_va);
1594	/ Save for reuse /
1595	rxq->xdp_save_va = buf_va;
1596	}
1597
1598	++ndev->stats.rx_dropped;
1599
1600	return;
1601	}
1602
1603	static void mana_get_rxfrag(struct* mana_rxq rxq, struct* device *dev,
1604	dma_addr_t da, bool from_pool, bool is_napi)
1605	{
1606	struct page *page;
1607	void *va;
1608
1609	*from_pool = false;
1610
1611	/ Reuse XDP dropped page if available /
1612	if (rxq->xdp_save_va) {
1613	va = rxq->xdp_save_va;
1614	rxq->xdp_save_va = NULL;
1615	} else if (rxq->alloc_size > PAGE_SIZE) {
1616	if (is_napi)
1617	va = napi_alloc_frag(fragsz: rxq->alloc_size);
1618	else
1619	va = netdev_alloc_frag(fragsz: rxq->alloc_size);
1620
1621	if (!va)
1622	return NULL;
1623
1624	page = virt_to_head_page(x: va);
1625	/ Check if the frag falls back to single page /
1626	if (compound_order(page) < get_order(size: rxq->alloc_size)) {
1627	put_page(page);
1628	return NULL;
1629	}
1630	} else {
1631	page = page_pool_dev_alloc_pages(pool: rxq->page_pool);
1632	if (!page)
1633	return NULL;
1634
1635	*from_pool = true;
1636	va = page_to_virt(page);
1637	}
1638
1639	*da = dma_map_single(dev, va + rxq->headroom, rxq->datasize,
1640	DMA_FROM_DEVICE);
1641	if (dma_mapping_error(dev, dma_addr: *da)) {
1642	if (*from_pool)
1643	page_pool_put_full_page(pool: rxq->page_pool, page, allow_direct: false);
1644	else
1645	put_page(page: virt_to_head_page(x: va));
1646
1647	return NULL;
1648	}
1649
1650	return va;
1651	}
1652
1653	/ Allocate frag for rx buffer, and save the old buf /
1654	static void mana_refill_rx_oob(struct device dev, struct* mana_rxq *rxq,
1655	struct mana_recv_buf_oob rxoob, void* **old_buf,
1656	bool *old_fp)
1657	{
1658	bool from_pool;
1659	dma_addr_t da;
1660	void *va;
1661
1662	va = mana_get_rxfrag(rxq, dev, da: &da, from_pool: &from_pool, is_napi: true);
1663	if (!va)
1664	return;
1665
1666	dma_unmap_single(dev, rxoob->sgl[`0`].address, rxq->datasize,
1667	DMA_FROM_DEVICE);
1668	*old_buf = rxoob->buf_va;
1669	*old_fp = rxoob->from_pool;
1670
1671	rxoob->buf_va = va;
1672	rxoob->sgl[`0`].address = da;
1673	rxoob->from_pool = from_pool;
1674	}
1675
1676	static void mana_process_rx_cqe(struct mana_rxq rxq, struct* mana_cq *cq,
1677	struct gdma_comp *cqe)
1678	{
1679	struct mana_rxcomp_oob oob = (struct* mana_rxcomp_oob *)cqe->cqe_data;
1680	struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context;
1681	struct net_device *ndev = rxq->ndev;
1682	struct mana_recv_buf_oob *rxbuf_oob;
1683	struct mana_port_context *apc;
1684	struct device *dev = gc->dev;
1685	void *old_buf = NULL;
1686	u32 curr, pktlen;
1687	bool old_fp;
1688
1689	apc = netdev_priv(dev: ndev);
1690
1691	switch (oob->cqe_hdr.cqe_type) {
1692	case CQE_RX_OKAY:
1693	break;
1694
1695	case CQE_RX_TRUNCATED:
1696	++ndev->stats.rx_dropped;
1697	rxbuf_oob = &rxq->rx_oobs[rxq->buf_index];
1698	netdev_warn_once(ndev, "Dropped a truncated packet\n");
1699	goto drop;
1700
1701	case CQE_RX_COALESCED_4:
1702	netdev_err(dev: ndev, format: "RX coalescing is unsupported\n");
1703	apc->eth_stats.rx_coalesced_err++;
1704	return;
1705
1706	case CQE_RX_OBJECT_FENCE:
1707	complete(&rxq->fence_event);
1708	return;
1709
1710	default:
1711	netdev_err(dev: ndev, format: "Unknown RX CQE type = %d\n",
1712	oob->cqe_hdr.cqe_type);
1713	apc->eth_stats.rx_cqe_unknown_type++;
1714	return;
1715	}
1716
1717	pktlen = oob->ppi[`0`].pkt_len;
1718
1719	if (pktlen == `0`) {
1720	/ data packets should never have packetlength of zero /
1721	netdev_err(dev: ndev, format: "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
1722	rxq->gdma_id, cq->gdma_id, rxq->rxobj);
1723	return;
1724	}
1725
1726	curr = rxq->buf_index;
1727	rxbuf_oob = &rxq->rx_oobs[curr];
1728	WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != `1`);
1729
1730	mana_refill_rx_oob(dev, rxq, rxoob: rxbuf_oob, old_buf: &old_buf, old_fp: &old_fp);
1731
1732	/ Unsuccessful refill will have old_buf == NULL.*
1733	* In this case, mana_rx_skb() will drop the packet.
1734	*/
1735	mana_rx_skb(buf_va: old_buf, from_pool: old_fp, cqe: oob, rxq);
1736
1737	drop:
1738	mana_move_wq_tail(wq: rxq->gdma_rq, num_units: rxbuf_oob->wqe_inf.wqe_size_in_bu);
1739
1740	mana_post_pkt_rxq(rxq);
1741	}
1742
1743	static void mana_poll_rx_cq(struct mana_cq *cq)
1744	{
1745	struct gdma_comp *comp = cq->gdma_comp_buf;
1746	struct mana_rxq *rxq = cq->rxq;
1747	int comp_read, i;
1748
1749	comp_read = mana_gd_poll_cq(cq: cq->gdma_cq, comp, CQE_POLLING_BUFFER);
1750	WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER);
1751
1752	rxq->xdp_flush = false;
1753
1754	for (i = `0`; i < comp_read; i++) {
1755	if (WARN_ON_ONCE(comp[i].is_sq))
1756	return;
1757
1758	/ verify recv cqe references the right rxq /
1759	if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id))
1760	return;
1761
1762	mana_process_rx_cqe(rxq, cq, cqe: &comp[i]);
1763	}
1764
1765	if (comp_read > `0`) {
1766	struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context;
1767
1768	mana_gd_wq_ring_doorbell(gc, queue: rxq->gdma_rq);
1769	}
1770
1771	if (rxq->xdp_flush)
1772	xdp_do_flush();
1773	}
1774
1775	static int mana_cq_handler(void context, struct* gdma_queue *gdma_queue)
1776	{
1777	struct mana_cq *cq = context;
1778	u8 arm_bit;
1779	int w;
1780
1781	WARN_ON_ONCE(cq->gdma_cq != gdma_queue);
1782
1783	if (cq->type == MANA_CQ_TYPE_RX)
1784	mana_poll_rx_cq(cq);
1785	else
1786	mana_poll_tx_cq(cq);
1787
1788	w = cq->work_done;
1789
1790	if (w < cq->budget &&
1791	napi_complete_done(n: &cq->napi, work_done: w)) {
1792	arm_bit = SET_ARM_BIT;
1793	} else {
1794	arm_bit = `0`;
1795	}
1796
1797	mana_gd_ring_cq(cq: gdma_queue, arm_bit);
1798
1799	return w;
1800	}
1801
1802	static int mana_poll(struct napi_struct napi, int* budget)
1803	{
1804	struct mana_cq cq = container_of(napi, struct* mana_cq, napi);
1805	int w;
1806
1807	cq->work_done = `0`;
1808	cq->budget = budget;
1809
1810	w = mana_cq_handler(context: cq, gdma_queue: cq->gdma_cq);
1811
1812	return min(w, budget);
1813	}
1814
1815	static void mana_schedule_napi(void context, struct* gdma_queue *gdma_queue)
1816	{
1817	struct mana_cq *cq = context;
1818
1819	napi_schedule_irqoff(n: &cq->napi);
1820	}
1821
1822	static void mana_deinit_cq(struct mana_port_context apc, struct* mana_cq *cq)
1823	{
1824	struct gdma_dev *gd = apc->ac->gdma_dev;
1825
1826	if (!cq->gdma_cq)
1827	return;
1828
1829	mana_gd_destroy_queue(gc: gd->gdma_context, queue: cq->gdma_cq);
1830	}
1831
1832	static void mana_deinit_txq(struct mana_port_context apc, struct* mana_txq *txq)
1833	{
1834	struct gdma_dev *gd = apc->ac->gdma_dev;
1835
1836	if (!txq->gdma_sq)
1837	return;
1838
1839	mana_gd_destroy_queue(gc: gd->gdma_context, queue: txq->gdma_sq);
1840	}
1841
1842	static void mana_destroy_txq(struct mana_port_context *apc)
1843	{
1844	struct napi_struct *napi;
1845	int i;
1846
1847	if (!apc->tx_qp)
1848	return;
1849
1850	for (i = `0`; i < apc->num_queues; i++) {
1851	napi = &apc->tx_qp[i].tx_cq.napi;
1852	napi_synchronize(n: napi);
1853	napi_disable(n: napi);
1854	netif_napi_del(napi);
1855
1856	mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object);
1857
1858	mana_deinit_cq(apc, cq: &apc->tx_qp[i].tx_cq);
1859
1860	mana_deinit_txq(apc, txq: &apc->tx_qp[i].txq);
1861	}
1862
1863	kfree(objp: apc->tx_qp);
1864	apc->tx_qp = NULL;
1865	}
1866
1867	static int mana_create_txq(struct mana_port_context *apc,
1868	struct net_device *net)
1869	{
1870	struct mana_context *ac = apc->ac;
1871	struct gdma_dev *gd = ac->gdma_dev;
1872	struct mana_obj_spec wq_spec;
1873	struct mana_obj_spec cq_spec;
1874	struct gdma_queue_spec spec;
1875	struct gdma_context *gc;
1876	struct mana_txq *txq;
1877	struct mana_cq *cq;
1878	u32 txq_size;
1879	u32 cq_size;
1880	int err;
1881	int i;
1882
1883	apc->tx_qp = kcalloc(n: apc->num_queues, size: sizeof(struct mana_tx_qp),
1884	GFP_KERNEL);
1885	if (!apc->tx_qp)
1886	return -ENOMEM;
1887
1888	/ The minimum size of the WQE is 32 bytes, hence*
1889	* MAX_SEND_BUFFERS_PER_QUEUE represents the maximum number of WQEs
1890	* the SQ can store. This value is then used to size other queues
1891	* to prevent overflow.
1892	*/
1893	txq_size = MAX_SEND_BUFFERS_PER_QUEUE * `32`;
1894	BUILD_BUG_ON(!PAGE_ALIGNED(txq_size));
1895
1896	cq_size = MAX_SEND_BUFFERS_PER_QUEUE * COMP_ENTRY_SIZE;
1897	cq_size = PAGE_ALIGN(cq_size);
1898
1899	gc = gd->gdma_context;
1900
1901	for (i = `0`; i < apc->num_queues; i++) {
1902	apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE;
1903
1904	/ Create SQ /
1905	txq = &apc->tx_qp[i].txq;
1906
1907	u64_stats_init(syncp: &txq->stats.syncp);
1908	txq->ndev = net;
1909	txq->net_txq = netdev_get_tx_queue(dev: net, index: i);
1910	txq->vp_offset = apc->tx_vp_offset;
1911	skb_queue_head_init(list: &txq->pending_skbs);
1912
1913	memset(&spec, `0`, sizeof(spec));
1914	spec.type = GDMA_SQ;
1915	spec.monitor_avl_buf = true;
1916	spec.queue_size = txq_size;
1917	err = mana_gd_create_mana_wq_cq(gd, spec: &spec, queue_ptr: &txq->gdma_sq);
1918	if (err)
1919	goto out;
1920
1921	/ Create SQ's CQ /
1922	cq = &apc->tx_qp[i].tx_cq;
1923	cq->type = MANA_CQ_TYPE_TX;
1924
1925	cq->txq = txq;
1926
1927	memset(&spec, `0`, sizeof(spec));
1928	spec.type = GDMA_CQ;
1929	spec.monitor_avl_buf = false;
1930	spec.queue_size = cq_size;
1931	spec.cq.callback = mana_schedule_napi;
1932	spec.cq.parent_eq = ac->eqs[i].eq;
1933	spec.cq.context = cq;
1934	err = mana_gd_create_mana_wq_cq(gd, spec: &spec, queue_ptr: &cq->gdma_cq);
1935	if (err)
1936	goto out;
1937
1938	memset(&wq_spec, `0`, sizeof(wq_spec));
1939	memset(&cq_spec, `0`, sizeof(cq_spec));
1940
1941	wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle;
1942	wq_spec.queue_size = txq->gdma_sq->queue_size;
1943
1944	cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle;
1945	cq_spec.queue_size = cq->gdma_cq->queue_size;
1946	cq_spec.modr_ctx_id = `0`;
1947	cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
1948
1949	err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ,
1950	&wq_spec, &cq_spec,
1951	&apc->tx_qp[i].tx_object);
1952
1953	if (err)
1954	goto out;
1955
1956	txq->gdma_sq->id = wq_spec.queue_index;
1957	cq->gdma_cq->id = cq_spec.queue_index;
1958
1959	txq->gdma_sq->mem_info.dma_region_handle =
1960	GDMA_INVALID_DMA_REGION;
1961	cq->gdma_cq->mem_info.dma_region_handle =
1962	GDMA_INVALID_DMA_REGION;
1963
1964	txq->gdma_txq_id = txq->gdma_sq->id;
1965
1966	cq->gdma_id = cq->gdma_cq->id;
1967
1968	if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
1969	err = -EINVAL;
1970	goto out;
1971	}
1972
1973	gc->cq_table[cq->gdma_id] = cq->gdma_cq;
1974
1975	netif_napi_add_tx(dev: net, napi: &cq->napi, poll: mana_poll);
1976	napi_enable(n: &cq->napi);
1977
1978	mana_gd_ring_cq(cq: cq->gdma_cq, SET_ARM_BIT);
1979	}
1980
1981	return `0`;
1982	out:
1983	mana_destroy_txq(apc);
1984	return err;
1985	}
1986
1987	static void mana_destroy_rxq(struct mana_port_context *apc,
1988	struct mana_rxq *rxq, bool validate_state)
1989
1990	{
1991	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
1992	struct mana_recv_buf_oob *rx_oob;
1993	struct device *dev = gc->dev;
1994	struct napi_struct *napi;
1995	struct page *page;
1996	int i;
1997
1998	if (!rxq)
1999	return;
2000
2001	napi = &rxq->rx_cq.napi;
2002
2003	if (validate_state)
2004	napi_synchronize(n: napi);
2005
2006	napi_disable(n: napi);
2007
2008	xdp_rxq_info_unreg(xdp_rxq: &rxq->xdp_rxq);
2009
2010	netif_napi_del(napi);
2011
2012	mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj);
2013
2014	mana_deinit_cq(apc, cq: &rxq->rx_cq);
2015
2016	if (rxq->xdp_save_va)
2017	put_page(page: virt_to_head_page(x: rxq->xdp_save_va));
2018
2019	for (i = `0`; i < rxq->num_rx_buf; i++) {
2020	rx_oob = &rxq->rx_oobs[i];
2021
2022	if (!rx_oob->buf_va)
2023	continue;
2024
2025	dma_unmap_single(dev, rx_oob->sgl[`0`].address,
2026	rx_oob->sgl[`0`].size, DMA_FROM_DEVICE);
2027
2028	page = virt_to_head_page(x: rx_oob->buf_va);
2029
2030	if (rx_oob->from_pool)
2031	page_pool_put_full_page(pool: rxq->page_pool, page, allow_direct: false);
2032	else
2033	put_page(page);
2034
2035	rx_oob->buf_va = NULL;
2036	}
2037
2038	page_pool_destroy(pool: rxq->page_pool);
2039
2040	if (rxq->gdma_rq)
2041	mana_gd_destroy_queue(gc, queue: rxq->gdma_rq);
2042
2043	kfree(objp: rxq);
2044	}
2045
2046	static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key,
2047	struct mana_rxq rxq, struct* device *dev)
2048	{
2049	struct mana_port_context *mpc = netdev_priv(dev: rxq->ndev);
2050	bool from_pool = false;
2051	dma_addr_t da;
2052	void *va;
2053
2054	if (mpc->rxbufs_pre)
2055	va = mana_get_rxbuf_pre(rxq, da: &da);
2056	else
2057	va = mana_get_rxfrag(rxq, dev, da: &da, from_pool: &from_pool, is_napi: false);
2058
2059	if (!va)
2060	return -ENOMEM;
2061
2062	rx_oob->buf_va = va;
2063	rx_oob->from_pool = from_pool;
2064
2065	rx_oob->sgl[`0`].address = da;
2066	rx_oob->sgl[`0`].size = rxq->datasize;
2067	rx_oob->sgl[`0`].mem_key = mem_key;
2068
2069	return `0`;
2070	}
2071
2072	#define MANA_WQE_HEADER_SIZE 16
2073	#define MANA_WQE_SGE_SIZE 16
2074
2075	static int mana_alloc_rx_wqe(struct mana_port_context *apc,
2076	struct mana_rxq rxq, u32 rxq_size, u32 *cq_size)
2077	{
2078	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
2079	struct mana_recv_buf_oob *rx_oob;
2080	struct device *dev = gc->dev;
2081	u32 buf_idx;
2082	int ret;
2083
2084	WARN_ON(rxq->datasize == `0`);
2085
2086	*rxq_size = `0`;
2087	*cq_size = `0`;
2088
2089	for (buf_idx = `0`; buf_idx < rxq->num_rx_buf; buf_idx++) {
2090	rx_oob = &rxq->rx_oobs[buf_idx];
2091	memset(rx_oob, `0`, sizeof(*rx_oob));
2092
2093	rx_oob->num_sge = `1`;
2094
2095	ret = mana_fill_rx_oob(rx_oob, mem_key: apc->ac->gdma_dev->gpa_mkey, rxq,
2096	dev);
2097	if (ret)
2098	return ret;
2099
2100	rx_oob->wqe_req.sgl = rx_oob->sgl;
2101	rx_oob->wqe_req.num_sge = rx_oob->num_sge;
2102	rx_oob->wqe_req.inline_oob_size = `0`;
2103	rx_oob->wqe_req.inline_oob_data = NULL;
2104	rx_oob->wqe_req.flags = `0`;
2105	rx_oob->wqe_req.client_data_unit = `0`;
2106
2107	*rxq_size += ALIGN(MANA_WQE_HEADER_SIZE +
2108	MANA_WQE_SGE_SIZE * rx_oob->num_sge, `32`);
2109	*cq_size += COMP_ENTRY_SIZE;
2110	}
2111
2112	return `0`;
2113	}
2114
2115	static int mana_push_wqe(struct mana_rxq *rxq)
2116	{
2117	struct mana_recv_buf_oob *rx_oob;
2118	u32 buf_idx;
2119	int err;
2120
2121	for (buf_idx = `0`; buf_idx < rxq->num_rx_buf; buf_idx++) {
2122	rx_oob = &rxq->rx_oobs[buf_idx];
2123
2124	err = mana_gd_post_and_ring(queue: rxq->gdma_rq, wqe: &rx_oob->wqe_req,
2125	wqe_info: &rx_oob->wqe_inf);
2126	if (err)
2127	return -ENOSPC;
2128	}
2129
2130	return `0`;
2131	}
2132
2133	static int mana_create_page_pool(struct mana_rxq rxq, struct* gdma_context *gc)
2134	{
2135	struct page_pool_params pprm = {};
2136	int ret;
2137
2138	pprm.pool_size = RX_BUFFERS_PER_QUEUE;
2139	pprm.nid = gc->numa_node;
2140	pprm.napi = &rxq->rx_cq.napi;
2141	pprm.netdev = rxq->ndev;
2142
2143	rxq->page_pool = page_pool_create(params: &pprm);
2144
2145	if (IS_ERR(ptr: rxq->page_pool)) {
2146	ret = PTR_ERR(ptr: rxq->page_pool);
2147	rxq->page_pool = NULL;
2148	return ret;
2149	}
2150
2151	return `0`;
2152	}
2153
2154	static struct mana_rxq mana_create_rxq(struct* mana_port_context *apc,
2155	u32 rxq_idx, struct mana_eq *eq,
2156	struct net_device *ndev)
2157	{
2158	struct gdma_dev *gd = apc->ac->gdma_dev;
2159	struct mana_obj_spec wq_spec;
2160	struct mana_obj_spec cq_spec;
2161	struct gdma_queue_spec spec;
2162	struct mana_cq *cq = NULL;
2163	struct gdma_context *gc;
2164	u32 cq_size, rq_size;
2165	struct mana_rxq *rxq;
2166	int err;
2167
2168	gc = gd->gdma_context;
2169
2170	rxq = kzalloc(struct_size(rxq, rx_oobs, RX_BUFFERS_PER_QUEUE),
2171	GFP_KERNEL);
2172	if (!rxq)
2173	return NULL;
2174
2175	rxq->ndev = ndev;
2176	rxq->num_rx_buf = RX_BUFFERS_PER_QUEUE;
2177	rxq->rxq_idx = rxq_idx;
2178	rxq->rxobj = INVALID_MANA_HANDLE;
2179
2180	mana_get_rxbuf_cfg(mtu: ndev->mtu, datasize: &rxq->datasize, alloc_size: &rxq->alloc_size,
2181	headroom: &rxq->headroom);
2182
2183	/ Create page pool for RX queue /
2184	err = mana_create_page_pool(rxq, gc);
2185	if (err) {
2186	netdev_err(dev: ndev, format: "Create page pool err:%d\n", err);
2187	goto out;
2188	}
2189
2190	err = mana_alloc_rx_wqe(apc, rxq, rxq_size: &rq_size, cq_size: &cq_size);
2191	if (err)
2192	goto out;
2193
2194	rq_size = PAGE_ALIGN(rq_size);
2195	cq_size = PAGE_ALIGN(cq_size);
2196
2197	/ Create RQ /
2198	memset(&spec, `0`, sizeof(spec));
2199	spec.type = GDMA_RQ;
2200	spec.monitor_avl_buf = true;
2201	spec.queue_size = rq_size;
2202	err = mana_gd_create_mana_wq_cq(gd, spec: &spec, queue_ptr: &rxq->gdma_rq);
2203	if (err)
2204	goto out;
2205
2206	/ Create RQ's CQ /
2207	cq = &rxq->rx_cq;
2208	cq->type = MANA_CQ_TYPE_RX;
2209	cq->rxq = rxq;
2210
2211	memset(&spec, `0`, sizeof(spec));
2212	spec.type = GDMA_CQ;
2213	spec.monitor_avl_buf = false;
2214	spec.queue_size = cq_size;
2215	spec.cq.callback = mana_schedule_napi;
2216	spec.cq.parent_eq = eq->eq;
2217	spec.cq.context = cq;
2218	err = mana_gd_create_mana_wq_cq(gd, spec: &spec, queue_ptr: &cq->gdma_cq);
2219	if (err)
2220	goto out;
2221
2222	memset(&wq_spec, `0`, sizeof(wq_spec));
2223	memset(&cq_spec, `0`, sizeof(cq_spec));
2224	wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle;
2225	wq_spec.queue_size = rxq->gdma_rq->queue_size;
2226
2227	cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle;
2228	cq_spec.queue_size = cq->gdma_cq->queue_size;
2229	cq_spec.modr_ctx_id = `0`;
2230	cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
2231
2232	err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ,
2233	&wq_spec, &cq_spec, &rxq->rxobj);
2234	if (err)
2235	goto out;
2236
2237	rxq->gdma_rq->id = wq_spec.queue_index;
2238	cq->gdma_cq->id = cq_spec.queue_index;
2239
2240	rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;
2241	cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;
2242
2243	rxq->gdma_id = rxq->gdma_rq->id;
2244	cq->gdma_id = cq->gdma_cq->id;
2245
2246	err = mana_push_wqe(rxq);
2247	if (err)
2248	goto out;
2249
2250	if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
2251	err = -EINVAL;
2252	goto out;
2253	}
2254
2255	gc->cq_table[cq->gdma_id] = cq->gdma_cq;
2256
2257	netif_napi_add_weight(dev: ndev, napi: &cq->napi, poll: mana_poll, weight: `1`);
2258
2259	WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx,
2260	cq->napi.napi_id));
2261	WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL,
2262	rxq->page_pool));
2263
2264	napi_enable(n: &cq->napi);
2265
2266	mana_gd_ring_cq(cq: cq->gdma_cq, SET_ARM_BIT);
2267	out:
2268	if (!err)
2269	return rxq;
2270
2271	netdev_err(dev: ndev, format: "Failed to create RXQ: err = %d\n", err);
2272
2273	mana_destroy_rxq(apc, rxq, validate_state: false);
2274
2275	if (cq)
2276	mana_deinit_cq(apc, cq);
2277
2278	return NULL;
2279	}
2280
2281	static int mana_add_rx_queues(struct mana_port_context *apc,
2282	struct net_device *ndev)
2283	{
2284	struct mana_context *ac = apc->ac;
2285	struct mana_rxq *rxq;
2286	int err = `0`;
2287	int i;
2288
2289	for (i = `0`; i < apc->num_queues; i++) {
2290	rxq = mana_create_rxq(apc, rxq_idx: i, eq: &ac->eqs[i], ndev);
2291	if (!rxq) {
2292	err = -ENOMEM;
2293	goto out;
2294	}
2295
2296	u64_stats_init(syncp: &rxq->stats.syncp);
2297
2298	apc->rxqs[i] = rxq;
2299	}
2300
2301	apc->default_rxobj = apc->rxqs[`0`]->rxobj;
2302	out:
2303	return err;
2304	}
2305
2306	static void mana_destroy_vport(struct mana_port_context *apc)
2307	{
2308	struct gdma_dev *gd = apc->ac->gdma_dev;
2309	struct mana_rxq *rxq;
2310	u32 rxq_idx;
2311
2312	for (rxq_idx = `0`; rxq_idx < apc->num_queues; rxq_idx++) {
2313	rxq = apc->rxqs[rxq_idx];
2314	if (!rxq)
2315	continue;
2316
2317	mana_destroy_rxq(apc, rxq, validate_state: true);
2318	apc->rxqs[rxq_idx] = NULL;
2319	}
2320
2321	mana_destroy_txq(apc);
2322	mana_uncfg_vport(apc);
2323
2324	if (gd->gdma_context->is_pf)
2325	mana_pf_deregister_hw_vport(apc);
2326	}
2327
2328	static int mana_create_vport(struct mana_port_context *apc,
2329	struct net_device *net)
2330	{
2331	struct gdma_dev *gd = apc->ac->gdma_dev;
2332	int err;
2333
2334	apc->default_rxobj = INVALID_MANA_HANDLE;
2335
2336	if (gd->gdma_context->is_pf) {
2337	err = mana_pf_register_hw_vport(apc);
2338	if (err)
2339	return err;
2340	}
2341
2342	err = mana_cfg_vport(apc, gd->pdid, gd->doorbell);
2343	if (err)
2344	return err;
2345
2346	return mana_create_txq(apc, net);
2347	}
2348
2349	static void mana_rss_table_init(struct mana_port_context *apc)
2350	{
2351	int i;
2352
2353	for (i = `0`; i < MANA_INDIRECT_TABLE_SIZE; i++)
2354	apc->indir_table[i] =
2355	ethtool_rxfh_indir_default(index: i, n_rx_rings: apc->num_queues);
2356	}
2357
2358	int mana_config_rss(struct mana_port_context apc, enum* TRI_STATE rx,
2359	bool update_hash, bool update_tab)
2360	{
2361	u32 queue_idx;
2362	int err;
2363	int i;
2364
2365	if (update_tab) {
2366	for (i = `0`; i < MANA_INDIRECT_TABLE_SIZE; i++) {
2367	queue_idx = apc->indir_table[i];
2368	apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj;
2369	}
2370	}
2371
2372	err = mana_cfg_vport_steering(apc, rx, update_default_rxobj: true, update_key: update_hash, update_tab);
2373	if (err)
2374	return err;
2375
2376	mana_fence_rqs(apc);
2377
2378	return `0`;
2379	}
2380
2381	void mana_query_gf_stats(struct mana_port_context *apc)
2382	{
2383	struct mana_query_gf_stat_resp resp = {};
2384	struct mana_query_gf_stat_req req = {};
2385	struct net_device *ndev = apc->ndev;
2386	int err;
2387
2388	mana_gd_init_req_hdr(hdr: &req.hdr, code: MANA_QUERY_GF_STAT,
2389	req_size: sizeof(req), resp_size: sizeof(resp));
2390	req.req_stats = STATISTICS_FLAGS_RX_DISCARDS_NO_WQE \|
2391	STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED \|
2392	STATISTICS_FLAGS_HC_RX_BYTES \|
2393	STATISTICS_FLAGS_HC_RX_UCAST_PACKETS \|
2394	STATISTICS_FLAGS_HC_RX_UCAST_BYTES \|
2395	STATISTICS_FLAGS_HC_RX_MCAST_PACKETS \|
2396	STATISTICS_FLAGS_HC_RX_MCAST_BYTES \|
2397	STATISTICS_FLAGS_HC_RX_BCAST_PACKETS \|
2398	STATISTICS_FLAGS_HC_RX_BCAST_BYTES \|
2399	STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED \|
2400	STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED \|
2401	STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS \|
2402	STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT \|
2403	STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT \|
2404	STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT \|
2405	STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT \|
2406	STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT \|
2407	STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION \|
2408	STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB \|
2409	STATISTICS_FLAGS_HC_TX_BYTES \|
2410	STATISTICS_FLAGS_HC_TX_UCAST_PACKETS \|
2411	STATISTICS_FLAGS_HC_TX_UCAST_BYTES \|
2412	STATISTICS_FLAGS_HC_TX_MCAST_PACKETS \|
2413	STATISTICS_FLAGS_HC_TX_MCAST_BYTES \|
2414	STATISTICS_FLAGS_HC_TX_BCAST_PACKETS \|
2415	STATISTICS_FLAGS_HC_TX_BCAST_BYTES \|
2416	STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR;
2417
2418	err = mana_send_request(ac: apc->ac, in_buf: &req, in_len: sizeof(req), out_buf: &resp,
2419	out_len: sizeof(resp));
2420	if (err) {
2421	netdev_err(dev: ndev, format: "Failed to query GF stats: %d\n", err);
2422	return;
2423	}
2424	err = mana_verify_resp_hdr(resp_hdr: &resp.hdr, expected_code: MANA_QUERY_GF_STAT,
2425	min_size: sizeof(resp));
2426	if (err \|\| resp.hdr.status) {
2427	netdev_err(dev: ndev, format: "Failed to query GF stats: %d, 0x%x\n", err,
2428	resp.hdr.status);
2429	return;
2430	}
2431
2432	apc->eth_stats.hc_rx_discards_no_wqe = resp.rx_discards_nowqe;
2433	apc->eth_stats.hc_rx_err_vport_disabled = resp.rx_err_vport_disabled;
2434	apc->eth_stats.hc_rx_bytes = resp.hc_rx_bytes;
2435	apc->eth_stats.hc_rx_ucast_pkts = resp.hc_rx_ucast_pkts;
2436	apc->eth_stats.hc_rx_ucast_bytes = resp.hc_rx_ucast_bytes;
2437	apc->eth_stats.hc_rx_bcast_pkts = resp.hc_rx_bcast_pkts;
2438	apc->eth_stats.hc_rx_bcast_bytes = resp.hc_rx_bcast_bytes;
2439	apc->eth_stats.hc_rx_mcast_pkts = resp.hc_rx_mcast_pkts;
2440	apc->eth_stats.hc_rx_mcast_bytes = resp.hc_rx_mcast_bytes;
2441	apc->eth_stats.hc_tx_err_gf_disabled = resp.tx_err_gf_disabled;
2442	apc->eth_stats.hc_tx_err_vport_disabled = resp.tx_err_vport_disabled;
2443	apc->eth_stats.hc_tx_err_inval_vportoffset_pkt =
2444	resp.tx_err_inval_vport_offset_pkt;
2445	apc->eth_stats.hc_tx_err_vlan_enforcement =
2446	resp.tx_err_vlan_enforcement;
2447	apc->eth_stats.hc_tx_err_eth_type_enforcement =
2448	resp.tx_err_ethtype_enforcement;
2449	apc->eth_stats.hc_tx_err_sa_enforcement = resp.tx_err_SA_enforcement;
2450	apc->eth_stats.hc_tx_err_sqpdid_enforcement =
2451	resp.tx_err_SQPDID_enforcement;
2452	apc->eth_stats.hc_tx_err_cqpdid_enforcement =
2453	resp.tx_err_CQPDID_enforcement;
2454	apc->eth_stats.hc_tx_err_mtu_violation = resp.tx_err_mtu_violation;
2455	apc->eth_stats.hc_tx_err_inval_oob = resp.tx_err_inval_oob;
2456	apc->eth_stats.hc_tx_bytes = resp.hc_tx_bytes;
2457	apc->eth_stats.hc_tx_ucast_pkts = resp.hc_tx_ucast_pkts;
2458	apc->eth_stats.hc_tx_ucast_bytes = resp.hc_tx_ucast_bytes;
2459	apc->eth_stats.hc_tx_bcast_pkts = resp.hc_tx_bcast_pkts;
2460	apc->eth_stats.hc_tx_bcast_bytes = resp.hc_tx_bcast_bytes;
2461	apc->eth_stats.hc_tx_mcast_pkts = resp.hc_tx_mcast_pkts;
2462	apc->eth_stats.hc_tx_mcast_bytes = resp.hc_tx_mcast_bytes;
2463	apc->eth_stats.hc_tx_err_gdma = resp.tx_err_gdma;
2464	}
2465
2466	static int mana_init_port(struct net_device *ndev)
2467	{
2468	struct mana_port_context *apc = netdev_priv(dev: ndev);
2469	u32 max_txq, max_rxq, max_queues;
2470	int port_idx = apc->port_idx;
2471	u32 num_indirect_entries;
2472	int err;
2473
2474	err = mana_init_port_context(apc);
2475	if (err)
2476	return err;
2477
2478	err = mana_query_vport_cfg(apc, vport_index: port_idx, max_sq: &max_txq, max_rq: &max_rxq,
2479	num_indir_entry: &num_indirect_entries);
2480	if (err) {
2481	netdev_err(dev: ndev, format: "Failed to query info for vPort %d\n",
2482	port_idx);
2483	goto reset_apc;
2484	}
2485
2486	max_queues = min_t(u32, max_txq, max_rxq);
2487	if (apc->max_queues > max_queues)
2488	apc->max_queues = max_queues;
2489
2490	if (apc->num_queues > apc->max_queues)
2491	apc->num_queues = apc->max_queues;
2492
2493	eth_hw_addr_set(dev: ndev, addr: apc->mac_addr);
2494
2495	return `0`;
2496
2497	reset_apc:
2498	kfree(objp: apc->rxqs);
2499	apc->rxqs = NULL;
2500	return err;
2501	}
2502
2503	int mana_alloc_queues(struct net_device *ndev)
2504	{
2505	struct mana_port_context *apc = netdev_priv(dev: ndev);
2506	struct gdma_dev *gd = apc->ac->gdma_dev;
2507	int err;
2508
2509	err = mana_create_vport(apc, net: ndev);
2510	if (err)
2511	return err;
2512
2513	err = netif_set_real_num_tx_queues(dev: ndev, txq: apc->num_queues);
2514	if (err)
2515	goto destroy_vport;
2516
2517	err = mana_add_rx_queues(apc, ndev);
2518	if (err)
2519	goto destroy_vport;
2520
2521	apc->rss_state = apc->num_queues > `1` ? TRI_STATE_TRUE : TRI_STATE_FALSE;
2522
2523	err = netif_set_real_num_rx_queues(dev: ndev, rxq: apc->num_queues);
2524	if (err)
2525	goto destroy_vport;
2526
2527	mana_rss_table_init(apc);
2528
2529	err = mana_config_rss(apc, rx: TRI_STATE_TRUE, update_hash: true, update_tab: true);
2530	if (err)
2531	goto destroy_vport;
2532
2533	if (gd->gdma_context->is_pf) {
2534	err = mana_pf_register_filter(apc);
2535	if (err)
2536	goto destroy_vport;
2537	}
2538
2539	mana_chn_setxdp(apc, prog: mana_xdp_get(apc));
2540
2541	return `0`;
2542
2543	destroy_vport:
2544	mana_destroy_vport(apc);
2545	return err;
2546	}
2547
2548	int mana_attach(struct net_device *ndev)
2549	{
2550	struct mana_port_context *apc = netdev_priv(dev: ndev);
2551	int err;
2552
2553	ASSERT_RTNL();
2554
2555	err = mana_init_port(ndev);
2556	if (err)
2557	return err;
2558
2559	if (apc->port_st_save) {
2560	err = mana_alloc_queues(ndev);
2561	if (err) {
2562	mana_cleanup_port_context(apc);
2563	return err;
2564	}
2565	}
2566
2567	apc->port_is_up = apc->port_st_save;
2568
2569	/ Ensure port state updated before txq state /
2570	smp_wmb();
2571
2572	if (apc->port_is_up)
2573	netif_carrier_on(dev: ndev);
2574
2575	netif_device_attach(dev: ndev);
2576
2577	return `0`;
2578	}
2579
2580	static int mana_dealloc_queues(struct net_device *ndev)
2581	{
2582	struct mana_port_context *apc = netdev_priv(dev: ndev);
2583	unsigned long timeout = jiffies + `120` * HZ;
2584	struct gdma_dev *gd = apc->ac->gdma_dev;
2585	struct mana_txq *txq;
2586	struct sk_buff *skb;
2587	int i, err;
2588	u32 tsleep;
2589
2590	if (apc->port_is_up)
2591	return -EINVAL;
2592
2593	mana_chn_setxdp(apc, NULL);
2594
2595	if (gd->gdma_context->is_pf)
2596	mana_pf_deregister_filter(apc);
2597
2598	/ No packet can be transmitted now since apc->port_is_up is false.*
2599	* There is still a tiny chance that mana_poll_tx_cq() can re-enable
2600	* a txq because it may not timely see apc->port_is_up being cleared
2601	* to false, but it doesn't matter since mana_start_xmit() drops any
2602	* new packets due to apc->port_is_up being false.
2603	*
2604	* Drain all the in-flight TX packets.
2605	* A timeout of 120 seconds for all the queues is used.
2606	* This will break the while loop when h/w is not responding.
2607	* This value of 120 has been decided here considering max
2608	* number of queues.
2609	*/
2610
2611	for (i = `0`; i < apc->num_queues; i++) {
2612	txq = &apc->tx_qp[i].txq;
2613	tsleep = `1000`;
2614	while (atomic_read(v: &txq->pending_sends) > `0` &&
2615	time_before(jiffies, timeout)) {
2616	usleep_range(min: tsleep, max: tsleep + `1000`);
2617	tsleep <<= `1`;
2618	}
2619	if (atomic_read(v: &txq->pending_sends)) {
2620	err = pcie_flr(to_pci_dev(gd->gdma_context->dev));
2621	if (err) {
2622	netdev_err(dev: ndev, format: "flr failed %d with %d pkts pending in txq %u\n",
2623	err, atomic_read(v: &txq->pending_sends),
2624	txq->gdma_txq_id);
2625	}
2626	break;
2627	}
2628	}
2629
2630	for (i = `0`; i < apc->num_queues; i++) {
2631	txq = &apc->tx_qp[i].txq;
2632	while ((skb = skb_dequeue(list: &txq->pending_skbs))) {
2633	mana_unmap_skb(skb, apc);
2634	dev_kfree_skb_any(skb);
2635	}
2636	atomic_set(v: &txq->pending_sends, i: `0`);
2637	}
2638	/ We're 100% sure the queues can no longer be woken up, because*
2639	* we're sure now mana_poll_tx_cq() can't be running.
2640	*/
2641
2642	apc->rss_state = TRI_STATE_FALSE;
2643	err = mana_config_rss(apc, rx: TRI_STATE_FALSE, update_hash: false, update_tab: false);
2644	if (err) {
2645	netdev_err(dev: ndev, format: "Failed to disable vPort: %d\n", err);
2646	return err;
2647	}
2648
2649	mana_destroy_vport(apc);
2650
2651	return `0`;
2652	}
2653
2654	int mana_detach(struct net_device *ndev, bool from_close)
2655	{
2656	struct mana_port_context *apc = netdev_priv(dev: ndev);
2657	int err;
2658
2659	ASSERT_RTNL();
2660
2661	apc->port_st_save = apc->port_is_up;
2662	apc->port_is_up = false;
2663
2664	/ Ensure port state updated before txq state /
2665	smp_wmb();
2666
2667	netif_tx_disable(dev: ndev);
2668	netif_carrier_off(dev: ndev);
2669
2670	if (apc->port_st_save) {
2671	err = mana_dealloc_queues(ndev);
2672	if (err)
2673	return err;
2674	}
2675
2676	if (!from_close) {
2677	netif_device_detach(dev: ndev);
2678	mana_cleanup_port_context(apc);
2679	}
2680
2681	return `0`;
2682	}
2683
2684	static int mana_probe_port(struct mana_context ac, int* port_idx,
2685	struct net_device **ndev_storage)
2686	{
2687	struct gdma_context *gc = ac->gdma_dev->gdma_context;
2688	struct mana_port_context *apc;
2689	struct net_device *ndev;
2690	int err;
2691
2692	ndev = alloc_etherdev_mq(sizeof(struct mana_port_context),
2693	gc->max_num_queues);
2694	if (!ndev)
2695	return -ENOMEM;
2696
2697	*ndev_storage = ndev;
2698
2699	apc = netdev_priv(dev: ndev);
2700	apc->ac = ac;
2701	apc->ndev = ndev;
2702	apc->max_queues = gc->max_num_queues;
2703	apc->num_queues = gc->max_num_queues;
2704	apc->port_handle = INVALID_MANA_HANDLE;
2705	apc->pf_filter_handle = INVALID_MANA_HANDLE;
2706	apc->port_idx = port_idx;
2707
2708	mutex_init(&apc->vport_mutex);
2709	apc->vport_use_count = `0`;
2710
2711	ndev->netdev_ops = &mana_devops;
2712	ndev->ethtool_ops = &mana_ethtool_ops;
2713	ndev->mtu = ETH_DATA_LEN;
2714	ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;
2715	ndev->min_mtu = ETH_MIN_MTU;
2716	ndev->needed_headroom = MANA_HEADROOM;
2717	ndev->dev_port = port_idx;
2718	SET_NETDEV_DEV(ndev, gc->dev);
2719
2720	netif_carrier_off(dev: ndev);
2721
2722	netdev_rss_key_fill(buffer: apc->hashkey, MANA_HASH_KEY_SIZE);
2723
2724	err = mana_init_port(ndev);
2725	if (err)
2726	goto free_net;
2727
2728	netdev_lockdep_set_classes(ndev);
2729
2730	ndev->hw_features = NETIF_F_SG \| NETIF_F_IP_CSUM \| NETIF_F_IPV6_CSUM;
2731	ndev->hw_features \|= NETIF_F_RXCSUM;
2732	ndev->hw_features \|= NETIF_F_TSO \| NETIF_F_TSO6;
2733	ndev->hw_features \|= NETIF_F_RXHASH;
2734	ndev->features = ndev->hw_features \| NETIF_F_HW_VLAN_CTAG_TX \|
2735	NETIF_F_HW_VLAN_CTAG_RX;
2736	ndev->vlan_features = ndev->features;
2737	xdp_set_features_flag(dev: ndev, val: NETDEV_XDP_ACT_BASIC \|
2738	NETDEV_XDP_ACT_REDIRECT \|
2739	NETDEV_XDP_ACT_NDO_XMIT);
2740
2741	err = register_netdev(dev: ndev);
2742	if (err) {
2743	netdev_err(dev: ndev, format: "Unable to register netdev.\n");
2744	goto reset_apc;
2745	}
2746
2747	return `0`;
2748
2749	reset_apc:
2750	kfree(objp: apc->rxqs);
2751	apc->rxqs = NULL;
2752	free_net:
2753	*ndev_storage = NULL;
2754	netdev_err(dev: ndev, format: "Failed to probe vPort %d: %d\n", port_idx, err);
2755	free_netdev(dev: ndev);
2756	return err;
2757	}
2758
2759	static void adev_release(struct device *dev)
2760	{
2761	struct mana_adev madev = container_of(dev, struct* mana_adev, adev.dev);
2762
2763	kfree(objp: madev);
2764	}
2765
2766	static void remove_adev(struct gdma_dev *gd)
2767	{
2768	struct auxiliary_device *adev = gd->adev;
2769	int id = adev->id;
2770
2771	auxiliary_device_delete(auxdev: adev);
2772	auxiliary_device_uninit(auxdev: adev);
2773
2774	mana_adev_idx_free(idx: id);
2775	gd->adev = NULL;
2776	}
2777
2778	static int add_adev(struct gdma_dev *gd)
2779	{
2780	struct auxiliary_device *adev;
2781	struct mana_adev *madev;
2782	int ret;
2783
2784	madev = kzalloc(size: sizeof(*madev), GFP_KERNEL);
2785	if (!madev)
2786	return -ENOMEM;
2787
2788	adev = &madev->adev;
2789	ret = mana_adev_idx_alloc();
2790	if (ret < `0`)
2791	goto idx_fail;
2792	adev->id = ret;
2793
2794	adev->name = "rdma";
2795	adev->dev.parent = gd->gdma_context->dev;
2796	adev->dev.release = adev_release;
2797	madev->mdev = gd;
2798
2799	ret = auxiliary_device_init(auxdev: adev);
2800	if (ret)
2801	goto init_fail;
2802
2803	ret = auxiliary_device_add(adev);
2804	if (ret)
2805	goto add_fail;
2806
2807	gd->adev = adev;
2808	return `0`;
2809
2810	add_fail:
2811	auxiliary_device_uninit(auxdev: adev);
2812
2813	init_fail:
2814	mana_adev_idx_free(idx: adev->id);
2815
2816	idx_fail:
2817	kfree(objp: madev);
2818
2819	return ret;
2820	}
2821
2822	int mana_probe(struct gdma_dev *gd, bool resuming)
2823	{
2824	struct gdma_context *gc = gd->gdma_context;
2825	struct mana_context *ac = gd->driver_data;
2826	struct device *dev = gc->dev;
2827	u16 num_ports = `0`;
2828	int err;
2829	int i;
2830
2831	dev_info(dev,
2832	"Microsoft Azure Network Adapter protocol version: %d.%d.%d\n",
2833	MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION);
2834
2835	err = mana_gd_register_device(gd);
2836	if (err)
2837	return err;
2838
2839	if (!resuming) {
2840	ac = kzalloc(size: sizeof(*ac), GFP_KERNEL);
2841	if (!ac)
2842	return -ENOMEM;
2843
2844	ac->gdma_dev = gd;
2845	gd->driver_data = ac;
2846	}
2847
2848	err = mana_create_eq(ac);
2849	if (err)
2850	goto out;
2851
2852	err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION,
2853	MANA_MICRO_VERSION, max_num_vports: &num_ports);
2854	if (err)
2855	goto out;
2856
2857	if (!resuming) {
2858	ac->num_ports = num_ports;
2859	} else {
2860	if (ac->num_ports != num_ports) {
2861	dev_err(dev, "The number of vPorts changed: %d->%d\n",
2862	ac->num_ports, num_ports);
2863	err = -EPROTO;
2864	goto out;
2865	}
2866	}
2867
2868	if (ac->num_ports == `0`)
2869	dev_err(dev, "Failed to detect any vPort\n");
2870
2871	if (ac->num_ports > MAX_PORTS_IN_MANA_DEV)
2872	ac->num_ports = MAX_PORTS_IN_MANA_DEV;
2873
2874	if (!resuming) {
2875	for (i = `0`; i < ac->num_ports; i++) {
2876	err = mana_probe_port(ac, port_idx: i, ndev_storage: &ac->ports[i]);
2877	if (err)
2878	break;
2879	}
2880	} else {
2881	for (i = `0`; i < ac->num_ports; i++) {
2882	rtnl_lock();
2883	err = mana_attach(ndev: ac->ports[i]);
2884	rtnl_unlock();
2885	if (err)
2886	break;
2887	}
2888	}
2889
2890	err = add_adev(gd);
2891	out:
2892	if (err)
2893	mana_remove(gd, suspending: false);
2894
2895	return err;
2896	}
2897
2898	void mana_remove(struct gdma_dev *gd, bool suspending)
2899	{
2900	struct gdma_context *gc = gd->gdma_context;
2901	struct mana_context *ac = gd->driver_data;
2902	struct device *dev = gc->dev;
2903	struct net_device *ndev;
2904	int err;
2905	int i;
2906
2907	/ adev currently doesn't support suspending, always remove it /
2908	if (gd->adev)
2909	remove_adev(gd);
2910
2911	for (i = `0`; i < ac->num_ports; i++) {
2912	ndev = ac->ports[i];
2913	if (!ndev) {
2914	if (i == `0`)
2915	dev_err(dev, "No net device to remove\n");
2916	goto out;
2917	}
2918
2919	/ All cleanup actions should stay after rtnl_lock(), otherwise*
2920	* other functions may access partially cleaned up data.
2921	*/
2922	rtnl_lock();
2923
2924	err = mana_detach(ndev, from_close: false);
2925	if (err)
2926	netdev_err(dev: ndev, format: "Failed to detach vPort %d: %d\n",
2927	i, err);
2928
2929	if (suspending) {
2930	/ No need to unregister the ndev. /
2931	rtnl_unlock();
2932	continue;
2933	}
2934
2935	unregister_netdevice(dev: ndev);
2936
2937	rtnl_unlock();
2938
2939	free_netdev(dev: ndev);
2940	}
2941
2942	mana_destroy_eq(ac);
2943	out:
2944	mana_gd_deregister_device(gd);
2945
2946	if (suspending)
2947	return;
2948
2949	gd->driver_data = NULL;
2950	gd->gdma_context = NULL;
2951	kfree(objp: ac);
2952	}
2953

source code of linux/drivers/net/ethernet/microsoft/mana/mana_en.c