1	/*
2	* Broadcom NetXtreme-E RoCE driver.
3	*
4	* Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5	* Broadcom refers to Broadcom Limited and/or its subsidiaries.
6	*
7	* This software is available to you under a choice of one of two
8	* licenses. You may choose to be licensed under the terms of the GNU
9	* General Public License (GPL) Version 2, available from the file
10	* COPYING in the main directory of this source tree, or the
11	* BSD license below:
12	*
13	* Redistribution and use in source and binary forms, with or without
14	* modification, are permitted provided that the following conditions
15	* are met:
16	*
17	* 1. Redistributions of source code must retain the above copyright
18	* notice, this list of conditions and the following disclaimer.
19	* 2. Redistributions in binary form must reproduce the above copyright
20	* notice, this list of conditions and the following disclaimer in
21	* the documentation and/or other materials provided with the
22	* distribution.
23	*
24	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26	* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28	* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31	* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32	* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33	* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34	* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35	*
36	* Description: IB Verbs interpreter
37	*/
38
39	#include <linux/interrupt.h>
40	#include <linux/types.h>
41	#include <linux/pci.h>
42	#include <linux/netdevice.h>
43	#include <linux/if_ether.h>
44	#include <net/addrconf.h>
45
46	#include <rdma/ib_verbs.h>
47	#include <rdma/ib_user_verbs.h>
48	#include <rdma/ib_umem.h>
49	#include <rdma/ib_addr.h>
50	#include <rdma/ib_mad.h>
51	#include <rdma/ib_cache.h>
52	#include <rdma/ib_pma.h>
53	#include <rdma/uverbs_ioctl.h>
54	#include <linux/hashtable.h>
55
56	#include "roce_hsi.h"
57	#include "qplib_res.h"
58	#include "qplib_sp.h"
59	#include "qplib_fp.h"
60	#include "qplib_rcfw.h"
61
62	#include "bnxt_re.h"
63	#include "ib_verbs.h"
64	#include "debugfs.h"
65
66	#include <rdma/uverbs_types.h>
67	#include <rdma/uverbs_std_types.h>
68
69	#include <rdma/ib_user_ioctl_cmds.h>
70
71	#define UVERBS_MODULE_NAME bnxt_re
72	#include <rdma/uverbs_named_ioctl.h>
73
74	#include <rdma/bnxt_re-abi.h>
75
76	static int __from_ib_access_flags(int iflags)
77	{
78	int qflags = 0;
79
80	if (iflags & IB_ACCESS_LOCAL_WRITE)
81	qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
82	if (iflags & IB_ACCESS_REMOTE_READ)
83	qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
84	if (iflags & IB_ACCESS_REMOTE_WRITE)
85	qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
86	if (iflags & IB_ACCESS_REMOTE_ATOMIC)
87	qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
88	if (iflags & IB_ACCESS_MW_BIND)
89	qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
90	if (iflags & IB_ZERO_BASED)
91	qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
92	if (iflags & IB_ACCESS_ON_DEMAND)
93	qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
94	return qflags;
95	};
96
97	static int __to_ib_access_flags(int qflags)
98	{
99	int iflags = 0;
100
101	if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
102	iflags |= IB_ACCESS_LOCAL_WRITE;
103	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
104	iflags |= IB_ACCESS_REMOTE_WRITE;
105	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
106	iflags |= IB_ACCESS_REMOTE_READ;
107	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
108	iflags |= IB_ACCESS_REMOTE_ATOMIC;
109	if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
110	iflags |= IB_ACCESS_MW_BIND;
111	if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
112	iflags |= IB_ZERO_BASED;
113	if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
114	iflags |= IB_ACCESS_ON_DEMAND;
115	return iflags;
116	}
117
118	static u8 __qp_access_flags_from_ib(struct bnxt_qplib_chip_ctx *cctx, int iflags)
119	{
120	u8 qflags = 0;
121
122	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))
123	/* For Wh+ */
124	return (u8)__from_ib_access_flags(iflags);
125
126	/* For P5, P7 and later chips */
127	if (iflags & IB_ACCESS_LOCAL_WRITE)
128	qflags |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
129	if (iflags & IB_ACCESS_REMOTE_WRITE)
130	qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
131	if (iflags & IB_ACCESS_REMOTE_READ)
132	qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
133	if (iflags & IB_ACCESS_REMOTE_ATOMIC)
134	qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC;
135
136	return qflags;
137	}
138
139	static int __qp_access_flags_to_ib(struct bnxt_qplib_chip_ctx *cctx, u8 qflags)
140	{
141	int iflags = 0;
142
143	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))
144	/* For Wh+ */
145	return __to_ib_access_flags(qflags);
146
147	/* For P5, P7 and later chips */
148	if (qflags & CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE)
149	iflags |= IB_ACCESS_LOCAL_WRITE;
150	if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE)
151	iflags |= IB_ACCESS_REMOTE_WRITE;
152	if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_READ)
153	iflags |= IB_ACCESS_REMOTE_READ;
154	if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC)
155	iflags |= IB_ACCESS_REMOTE_ATOMIC;
156
157	return iflags;
158	}
159
160	static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev,
161	struct bnxt_qplib_mrw *qplib_mr)
162	{
163	if (_is_relaxed_ordering_supported(dev_cap_ext_flags2: rdev->dev_attr->dev_cap_flags2) &&
164	pcie_relaxed_ordering_enabled(dev: rdev->en_dev->pdev))
165	qplib_mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;
166	}
167
168	static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
169	struct bnxt_qplib_sge *sg_list, int num)
170	{
171	int i, total = 0;
172
173	for (i = 0; i < num; i++) {
174	sg_list[i].addr = ib_sg_list[i].addr;
175	sg_list[i].lkey = ib_sg_list[i].lkey;
176	sg_list[i].size = ib_sg_list[i].length;
177	total += sg_list[i].size;
178	}
179	return total;
180	}
181
182	/* Device */
183	int bnxt_re_query_device(struct ib_device *ibdev,
184	struct ib_device_attr *ib_attr,
185	struct ib_udata *udata)
186	{
187	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
188	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
189
190	memset(ib_attr, 0, sizeof(*ib_attr));
191	memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
192	min(sizeof(dev_attr->fw_ver),
193	sizeof(ib_attr->fw_ver)));
194	addrconf_addr_eui48(eui: (u8 *)&ib_attr->sys_image_guid,
195	addr: rdev->netdev->dev_addr);
196	ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
197	ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED;
198
199	ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
200	ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
201	ib_attr->hw_ver = rdev->en_dev->pdev->revision;
202	ib_attr->max_qp = dev_attr->max_qp;
203	ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
204	ib_attr->device_cap_flags =
205	IB_DEVICE_CURR_QP_STATE_MOD
206	| IB_DEVICE_RC_RNR_NAK_GEN
207	| IB_DEVICE_SHUTDOWN_PORT
208	| IB_DEVICE_SYS_IMAGE_GUID
209	| IB_DEVICE_RESIZE_MAX_WR
210	| IB_DEVICE_PORT_ACTIVE_EVENT
211	| IB_DEVICE_N_NOTIFY_CQ
212	| IB_DEVICE_MEM_WINDOW
213	| IB_DEVICE_MEM_WINDOW_TYPE_2B
214	| IB_DEVICE_MEM_MGT_EXTENSIONS;
215	ib_attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
216	ib_attr->max_send_sge = dev_attr->max_qp_sges;
217	ib_attr->max_recv_sge = dev_attr->max_qp_sges;
218	ib_attr->max_sge_rd = dev_attr->max_qp_sges;
219	ib_attr->max_cq = dev_attr->max_cq;
220	ib_attr->max_cqe = dev_attr->max_cq_wqes;
221	ib_attr->max_mr = dev_attr->max_mr;
222	ib_attr->max_pd = dev_attr->max_pd;
223	ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
224	ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
225	ib_attr->atomic_cap = IB_ATOMIC_NONE;
226	ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
227	if (dev_attr->is_atomic) {
228	ib_attr->atomic_cap = IB_ATOMIC_GLOB;
229	ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
230	}
231
232	ib_attr->max_ee_rd_atom = 0;
233	ib_attr->max_res_rd_atom = 0;
234	ib_attr->max_ee_init_rd_atom = 0;
235	ib_attr->max_ee = 0;
236	ib_attr->max_rdd = 0;
237	ib_attr->max_mw = dev_attr->max_mw;
238	ib_attr->max_raw_ipv6_qp = 0;
239	ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
240	ib_attr->max_mcast_grp = 0;
241	ib_attr->max_mcast_qp_attach = 0;
242	ib_attr->max_total_mcast_qp_attach = 0;
243	ib_attr->max_ah = dev_attr->max_ah;
244
245	ib_attr->max_srq = dev_attr->max_srq;
246	ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
247	ib_attr->max_srq_sge = dev_attr->max_srq_sges;
248
249	ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
250
251	ib_attr->max_pkeys = 1;
252	ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
253	return 0;
254	}
255
256	int bnxt_re_modify_device(struct ib_device *ibdev,
257	int device_modify_mask,
258	struct ib_device_modify *device_modify)
259	{
260	ibdev_dbg(ibdev, "Modify device with mask 0x%x", device_modify_mask);
261
262	if (device_modify_mask & ~IB_DEVICE_MODIFY_NODE_DESC)
263	return -EOPNOTSUPP;
264
265	if (!(device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC))
266	return 0;
267
268	memcpy(ibdev->node_desc, device_modify->node_desc, IB_DEVICE_NODE_DESC_MAX);
269	return 0;
270	}
271
272	/* Port */
273	int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,
274	struct ib_port_attr *port_attr)
275	{
276	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
277	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
278	int rc;
279
280	memset(port_attr, 0, sizeof(*port_attr));
281
282	if (netif_running(dev: rdev->netdev) && netif_carrier_ok(dev: rdev->netdev)) {
283	port_attr->state = IB_PORT_ACTIVE;
284	port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
285	} else {
286	port_attr->state = IB_PORT_DOWN;
287	port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
288	}
289	port_attr->max_mtu = IB_MTU_4096;
290	port_attr->active_mtu = iboe_get_mtu(mtu: rdev->netdev->mtu);
291	/* One GID is reserved for RawEth QP. Report one less */
292	port_attr->gid_tbl_len = (rdev->rcfw.roce_mirror ? (dev_attr->max_sgid - 1) :
293	dev_attr->max_sgid);
294	port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
295	IB_PORT_DEVICE_MGMT_SUP |
296	IB_PORT_VENDOR_CLASS_SUP;
297	port_attr->ip_gids = true;
298
299	port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
300	port_attr->bad_pkey_cntr = 0;
301	port_attr->qkey_viol_cntr = 0;
302	port_attr->pkey_tbl_len = dev_attr->max_pkey;
303	port_attr->lid = 0;
304	port_attr->sm_lid = 0;
305	port_attr->lmc = 0;
306	port_attr->max_vl_num = 4;
307	port_attr->sm_sl = 0;
308	port_attr->subnet_timeout = 0;
309	port_attr->init_type_reply = 0;
310	rc = ib_get_eth_speed(dev: &rdev->ibdev, port_num, speed: &port_attr->active_speed,
311	width: &port_attr->active_width);
312
313	return rc;
314	}
315
316	int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
317	struct ib_port_immutable *immutable)
318	{
319	struct ib_port_attr port_attr;
320
321	if (bnxt_re_query_port(ibdev, port_num, port_attr: &port_attr))
322	return -EINVAL;
323
324	immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
325	immutable->gid_tbl_len = port_attr.gid_tbl_len;
326	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
327	immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
328	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
329	return 0;
330	}
331
332	void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
333	{
334	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
335
336	snprintf(buf: str, IB_FW_VERSION_NAME_MAX, fmt: "%d.%d.%d.%d",
337	rdev->dev_attr->fw_ver[0], rdev->dev_attr->fw_ver[1],
338	rdev->dev_attr->fw_ver[2], rdev->dev_attr->fw_ver[3]);
339	}
340
341	int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
342	u16 index, u16 *pkey)
343	{
344	if (index > 0)
345	return -EINVAL;
346
347	*pkey = IB_DEFAULT_PKEY_FULL;
348
349	return 0;
350	}
351
352	int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
353	int index, union ib_gid *gid)
354	{
355	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
356	int rc;
357
358	/* Ignore port_num */
359	memset(gid, 0, sizeof(*gid));
360	rc = bnxt_qplib_get_sgid(res: &rdev->qplib_res,
361	sgid_tbl: &rdev->qplib_res.sgid_tbl, index,
362	gid: (struct bnxt_qplib_gid *)gid);
363	return rc;
364	}
365
366	int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)
367	{
368	int rc = 0;
369	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
370	struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
371	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
372	struct bnxt_qplib_gid *gid_to_del;
373	u16 vlan_id = 0xFFFF;
374
375	/* Delete the entry from the hardware */
376	ctx = *context;
377	if (!ctx)
378	return -EINVAL;
379
380	if (sgid_tbl->active) {
381	if (ctx->idx >= sgid_tbl->max)
382	return -EINVAL;
383	gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
384	vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
385	/* DEL_GID is called in WQ context(netdevice_event_work_handler)
386	* or via the ib_unregister_device path. In the former case QP1
387	* may not be destroyed yet, in which case just return as FW
388	* needs that entry to be present and will fail it's deletion.
389	* We could get invoked again after QP1 is destroyed OR get an
390	* ADD_GID call with a different GID value for the same index
391	* where we issue MODIFY_GID cmd to update the GID entry -- TBD
392	*/
393	if (ctx->idx == 0 &&
394	rdma_link_local_addr(addr: (struct in6_addr *)gid_to_del) &&
395	ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) {
396	ibdev_dbg(&rdev->ibdev,
397	"Trying to delete GID0 while QP1 is alive\n");
398	return -EFAULT;
399	}
400	ctx->refcnt--;
401	if (!ctx->refcnt) {
402	rc = bnxt_qplib_del_sgid(sgid_tbl, gid: gid_to_del,
403	vlan_id, update: true);
404	if (rc) {
405	ibdev_err(ibdev: &rdev->ibdev,
406	format: "Failed to remove GID: %#x", rc);
407	} else {
408	ctx_tbl = sgid_tbl->ctx;
409	ctx_tbl[ctx->idx] = NULL;
410	kfree(objp: ctx);
411	}
412	}
413	} else {
414	return -EINVAL;
415	}
416	return rc;
417	}
418
419	int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
420	{
421	int rc;
422	u32 tbl_idx = 0;
423	u16 vlan_id = 0xFFFF;
424	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
425	struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
426	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
427
428	rc = rdma_read_gid_l2_fields(attr, vlan_id: &vlan_id, NULL);
429	if (rc)
430	return rc;
431
432	rc = bnxt_qplib_add_sgid(sgid_tbl, gid: (struct bnxt_qplib_gid *)&attr->gid,
433	mac: rdev->qplib_res.netdev->dev_addr,
434	vlan_id, update: true, index: &tbl_idx, is_ugid: false, stats_ctx_id: 0);
435	if (rc == -EALREADY) {
436	ctx_tbl = sgid_tbl->ctx;
437	ctx_tbl[tbl_idx]->refcnt++;
438	*context = ctx_tbl[tbl_idx];
439	return 0;
440	}
441
442	if (rc < 0) {
443	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to add GID: %#x", rc);
444	return rc;
445	}
446
447	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
448	if (!ctx)
449	return -ENOMEM;
450	ctx_tbl = sgid_tbl->ctx;
451	ctx->idx = tbl_idx;
452	ctx->refcnt = 1;
453	ctx_tbl[tbl_idx] = ctx;
454	*context = ctx;
455
456	return rc;
457	}
458
459	enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
460	u32 port_num)
461	{
462	return IB_LINK_LAYER_ETHERNET;
463	}
464
465	#define BNXT_RE_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
466
467	static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
468	{
469	struct bnxt_re_fence_data *fence = &pd->fence;
470	struct ib_mr *ib_mr = &fence->mr->ib_mr;
471	struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
472	struct bnxt_re_dev *rdev = pd->rdev;
473
474	if (bnxt_qplib_is_chip_gen_p5_p7(cctx: rdev->chip_ctx))
475	return;
476
477	memset(wqe, 0, sizeof(*wqe));
478	wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
479	wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
480	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
481	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
482	wqe->bind.zero_based = false;
483	wqe->bind.parent_l_key = ib_mr->lkey;
484	wqe->bind.va = (u64)(unsigned long)fence->va;
485	wqe->bind.length = fence->size;
486	wqe->bind.access_cntl = __from_ib_access_flags(iflags: IB_ACCESS_REMOTE_READ);
487	wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
488
489	/* Save the initial rkey in fence structure for now;
490	* wqe->bind.r_key will be set at (re)bind time.
491	*/
492	fence->bind_rkey = ib_inc_rkey(rkey: fence->mw->rkey);
493	}
494
495	static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
496	{
497	struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
498	qplib_qp);
499	struct ib_pd *ib_pd = qp->ib_qp.pd;
500	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
501	struct bnxt_re_fence_data *fence = &pd->fence;
502	struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
503	struct bnxt_qplib_swqe wqe;
504	int rc;
505
506	memcpy(&wqe, fence_wqe, sizeof(wqe));
507	wqe.bind.r_key = fence->bind_rkey;
508	fence->bind_rkey = ib_inc_rkey(rkey: fence->bind_rkey);
509
510	ibdev_dbg(&qp->rdev->ibdev,
511	"Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
512	wqe.bind.r_key, qp->qplib_qp.id, pd);
513	rc = bnxt_qplib_post_send(qp: &qp->qplib_qp, wqe: &wqe);
514	if (rc) {
515	ibdev_err(ibdev: &qp->rdev->ibdev, format: "Failed to bind fence-WQE\n");
516	return rc;
517	}
518	bnxt_qplib_post_send_db(qp: &qp->qplib_qp);
519
520	return rc;
521	}
522
523	static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
524	{
525	struct bnxt_re_fence_data *fence = &pd->fence;
526	struct bnxt_re_dev *rdev = pd->rdev;
527	struct device *dev = &rdev->en_dev->pdev->dev;
528	struct bnxt_re_mr *mr = fence->mr;
529
530	if (bnxt_qplib_is_chip_gen_p5_p7(cctx: rdev->chip_ctx))
531	return;
532
533	if (fence->mw) {
534	bnxt_re_dealloc_mw(mw: fence->mw);
535	fence->mw = NULL;
536	}
537	if (mr) {
538	if (mr->ib_mr.rkey)
539	bnxt_qplib_dereg_mrw(res: &rdev->qplib_res, mrw: &mr->qplib_mr,
540	block: true);
541	if (mr->ib_mr.lkey)
542	bnxt_qplib_free_mrw(res: &rdev->qplib_res, mr: &mr->qplib_mr);
543	kfree(objp: mr);
544	fence->mr = NULL;
545	}
546	if (fence->dma_addr) {
547	dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
548	DMA_BIDIRECTIONAL);
549	fence->dma_addr = 0;
550	}
551	}
552
553	static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
554	{
555	int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
556	struct bnxt_re_fence_data *fence = &pd->fence;
557	struct bnxt_re_dev *rdev = pd->rdev;
558	struct device *dev = &rdev->en_dev->pdev->dev;
559	struct bnxt_re_mr *mr = NULL;
560	dma_addr_t dma_addr = 0;
561	struct ib_mw *mw;
562	int rc;
563
564	if (bnxt_qplib_is_chip_gen_p5_p7(cctx: rdev->chip_ctx))
565	return 0;
566
567	dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
568	DMA_BIDIRECTIONAL);
569	rc = dma_mapping_error(dev, dma_addr);
570	if (rc) {
571	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to dma-map fence-MR-mem\n");
572	rc = -EIO;
573	fence->dma_addr = 0;
574	goto fail;
575	}
576	fence->dma_addr = dma_addr;
577
578	/* Allocate a MR */
579	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
580	if (!mr) {
581	rc = -ENOMEM;
582	goto fail;
583	}
584	fence->mr = mr;
585	mr->rdev = rdev;
586	mr->qplib_mr.pd = &pd->qplib_pd;
587	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
588	mr->qplib_mr.access_flags = __from_ib_access_flags(iflags: mr_access_flags);
589	if (!_is_alloc_mr_unified(dev_cap_flags: rdev->dev_attr->dev_cap_flags)) {
590	rc = bnxt_qplib_alloc_mrw(res: &rdev->qplib_res, mrw: &mr->qplib_mr);
591	if (rc) {
592	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to alloc fence-HW-MR\n");
593	goto fail;
594	}
595
596	/* Register MR */
597	mr->ib_mr.lkey = mr->qplib_mr.lkey;
598	} else {
599	mr->qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
600	}
601	mr->qplib_mr.va = (u64)(unsigned long)fence->va;
602	mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
603	rc = bnxt_qplib_reg_mr(res: &rdev->qplib_res, mr: &mr->qplib_mr, NULL,
604	BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE,
605	unified_mr: _is_alloc_mr_unified(dev_cap_flags: rdev->dev_attr->dev_cap_flags));
606	if (rc) {
607	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to register fence-MR\n");
608	goto fail;
609	}
610	mr->ib_mr.rkey = mr->qplib_mr.rkey;
611
612	/* Create a fence MW only for kernel consumers */
613	mw = bnxt_re_alloc_mw(ib_pd: &pd->ib_pd, type: IB_MW_TYPE_1, NULL);
614	if (IS_ERR(ptr: mw)) {
615	ibdev_err(ibdev: &rdev->ibdev,
616	format: "Failed to create fence-MW for PD: %p\n", pd);
617	rc = PTR_ERR(ptr: mw);
618	goto fail;
619	}
620	fence->mw = mw;
621
622	bnxt_re_create_fence_wqe(pd);
623	return 0;
624
625	fail:
626	bnxt_re_destroy_fence_mr(pd);
627	return rc;
628	}
629
630	static struct bnxt_re_user_mmap_entry*
631	bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset,
632	enum bnxt_re_mmap_flag mmap_flag, u64 *offset)
633	{
634	struct bnxt_re_user_mmap_entry *entry;
635	int ret;
636
637	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
638	if (!entry)
639	return NULL;
640
641	entry->mem_offset = mem_offset;
642	entry->mmap_flag = mmap_flag;
643	entry->uctx = uctx;
644
645	switch (mmap_flag) {
646	case BNXT_RE_MMAP_SH_PAGE:
647	ret = rdma_user_mmap_entry_insert_exact(ucontext: &uctx->ib_uctx,
648	entry: &entry->rdma_entry, PAGE_SIZE, pgoff: 0);
649	break;
650	case BNXT_RE_MMAP_UC_DB:
651	case BNXT_RE_MMAP_WC_DB:
652	case BNXT_RE_MMAP_DBR_BAR:
653	case BNXT_RE_MMAP_DBR_PAGE:
654	case BNXT_RE_MMAP_TOGGLE_PAGE:
655	ret = rdma_user_mmap_entry_insert(ucontext: &uctx->ib_uctx,
656	entry: &entry->rdma_entry, PAGE_SIZE);
657	break;
658	default:
659	ret = -EINVAL;
660	break;
661	}
662
663	if (ret) {
664	kfree(objp: entry);
665	return NULL;
666	}
667	if (offset)
668	*offset = rdma_user_mmap_get_offset(entry: &entry->rdma_entry);
669
670	return entry;
671	}
672
673	/* Protection Domains */
674	int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
675	{
676	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
677	struct bnxt_re_dev *rdev = pd->rdev;
678
679	if (udata) {
680	rdma_user_mmap_entry_remove(entry: pd->pd_db_mmap);
681	pd->pd_db_mmap = NULL;
682	}
683
684	bnxt_re_destroy_fence_mr(pd);
685
686	if (pd->qplib_pd.id) {
687	if (!bnxt_qplib_dealloc_pd(res: &rdev->qplib_res,
688	pd_tbl: &rdev->qplib_res.pd_tbl,
689	pd: &pd->qplib_pd))
690	atomic_dec(v: &rdev->stats.res.pd_count);
691	}
692	return 0;
693	}
694
695	int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
696	{
697	struct ib_device *ibdev = ibpd->device;
698	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
699	struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
700	udata, struct bnxt_re_ucontext, ib_uctx);
701	struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
702	struct bnxt_re_user_mmap_entry *entry = NULL;
703	u32 active_pds;
704	int rc = 0;
705
706	pd->rdev = rdev;
707	if (bnxt_qplib_alloc_pd(res: &rdev->qplib_res, pd: &pd->qplib_pd)) {
708	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to allocate HW PD");
709	rc = -ENOMEM;
710	goto fail;
711	}
712
713	if (udata) {
714	struct bnxt_re_pd_resp resp = {};
715
716	if (!ucntx->dpi.dbr) {
717	/* Allocate DPI in alloc_pd to avoid failing of
718	* ibv_devinfo and family of application when DPIs
719	* are depleted.
720	*/
721	if (bnxt_qplib_alloc_dpi(res: &rdev->qplib_res,
722	dpi: &ucntx->dpi, app: ucntx, type: BNXT_QPLIB_DPI_TYPE_UC)) {
723	rc = -ENOMEM;
724	goto dbfail;
725	}
726	}
727
728	resp.pdid = pd->qplib_pd.id;
729	/* Still allow mapping this DBR to the new user PD. */
730	resp.dpi = ucntx->dpi.dpi;
731
732	entry = bnxt_re_mmap_entry_insert(uctx: ucntx, mem_offset: (u64)ucntx->dpi.umdbr,
733	mmap_flag: BNXT_RE_MMAP_UC_DB, offset: &resp.dbr);
734
735	if (!entry) {
736	rc = -ENOMEM;
737	goto dbfail;
738	}
739
740	pd->pd_db_mmap = &entry->rdma_entry;
741
742	rc = ib_copy_to_udata(udata, src: &resp, min(sizeof(resp), udata->outlen));
743	if (rc) {
744	rdma_user_mmap_entry_remove(entry: pd->pd_db_mmap);
745	rc = -EFAULT;
746	goto dbfail;
747	}
748	}
749
750	if (!udata)
751	if (bnxt_re_create_fence_mr(pd))
752	ibdev_warn(ibdev: &rdev->ibdev,
753	format: "Failed to create Fence-MR\n");
754	active_pds = atomic_inc_return(v: &rdev->stats.res.pd_count);
755	if (active_pds > rdev->stats.res.pd_watermark)
756	rdev->stats.res.pd_watermark = active_pds;
757
758	return 0;
759	dbfail:
760	bnxt_qplib_dealloc_pd(res: &rdev->qplib_res, pd_tbl: &rdev->qplib_res.pd_tbl,
761	pd: &pd->qplib_pd);
762	fail:
763	return rc;
764	}
765
766	/* Address Handles */
767	int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
768	{
769	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
770	struct bnxt_re_dev *rdev = ah->rdev;
771	bool block = true;
772	int rc;
773
774	block = !(flags & RDMA_DESTROY_AH_SLEEPABLE);
775	rc = bnxt_qplib_destroy_ah(res: &rdev->qplib_res, ah: &ah->qplib_ah, block);
776	if (BNXT_RE_CHECK_RC(rc)) {
777	if (rc == -ETIMEDOUT)
778	rc = 0;
779	else
780	goto fail;
781	}
782	atomic_dec(v: &rdev->stats.res.ah_count);
783	fail:
784	return rc;
785	}
786
787	static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
788	{
789	u8 nw_type;
790
791	switch (ntype) {
792	case RDMA_NETWORK_IPV4:
793	nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
794	break;
795	case RDMA_NETWORK_IPV6:
796	nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
797	break;
798	default:
799	nw_type = CMDQ_CREATE_AH_TYPE_V1;
800	break;
801	}
802	return nw_type;
803	}
804
805	int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
806	struct ib_udata *udata)
807	{
808	struct ib_pd *ib_pd = ib_ah->pd;
809	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
810	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
811	const struct ib_global_route *grh = rdma_ah_read_grh(attr: ah_attr);
812	struct bnxt_re_dev *rdev = pd->rdev;
813	const struct ib_gid_attr *sgid_attr;
814	struct bnxt_re_gid_ctx *ctx;
815	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
816	u32 active_ahs;
817	u8 nw_type;
818	int rc;
819
820	if (!(rdma_ah_get_ah_flags(attr: ah_attr) & IB_AH_GRH)) {
821	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to alloc AH: GRH not set");
822	return -EINVAL;
823	}
824
825	ah->rdev = rdev;
826	ah->qplib_ah.pd = &pd->qplib_pd;
827
828	/* Supply the configuration for the HW */
829	memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
830	sizeof(union ib_gid));
831	sgid_attr = grh->sgid_attr;
832	/* Get the HW context of the GID. The reference
833	* of GID table entry is already taken by the caller.
834	*/
835	ctx = rdma_read_gid_hw_context(attr: sgid_attr);
836	ah->qplib_ah.sgid_index = ctx->idx;
837	ah->qplib_ah.host_sgid_index = grh->sgid_index;
838	ah->qplib_ah.traffic_class = grh->traffic_class;
839	ah->qplib_ah.flow_label = grh->flow_label;
840	ah->qplib_ah.hop_limit = grh->hop_limit;
841	ah->qplib_ah.sl = rdma_ah_get_sl(attr: ah_attr);
842
843	/* Get network header type for this GID */
844	nw_type = rdma_gid_attr_network_type(attr: sgid_attr);
845	ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(ntype: nw_type);
846
847	memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
848	rc = bnxt_qplib_create_ah(res: &rdev->qplib_res, ah: &ah->qplib_ah,
849	block: !(init_attr->flags &
850	RDMA_CREATE_AH_SLEEPABLE));
851	if (rc) {
852	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to allocate HW AH");
853	return rc;
854	}
855
856	/* Write AVID to shared page. */
857	if (udata) {
858	struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
859	udata, struct bnxt_re_ucontext, ib_uctx);
860	unsigned long flag;
861	u32 *wrptr;
862
863	spin_lock_irqsave(&uctx->sh_lock, flag);
864	wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
865	*wrptr = ah->qplib_ah.id;
866	wmb(); /* make sure cache is updated. */
867	spin_unlock_irqrestore(lock: &uctx->sh_lock, flags: flag);
868	}
869	active_ahs = atomic_inc_return(v: &rdev->stats.res.ah_count);
870	if (active_ahs > rdev->stats.res.ah_watermark)
871	rdev->stats.res.ah_watermark = active_ahs;
872
873	return 0;
874	}
875
876	int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
877	{
878	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
879
880	ah_attr->type = ib_ah->type;
881	rdma_ah_set_sl(attr: ah_attr, sl: ah->qplib_ah.sl);
882	memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
883	rdma_ah_set_grh(attr: ah_attr, NULL, flow_label: 0,
884	sgid_index: ah->qplib_ah.host_sgid_index,
885	hop_limit: 0, traffic_class: ah->qplib_ah.traffic_class);
886	rdma_ah_set_dgid_raw(attr: ah_attr, dgid: ah->qplib_ah.dgid.data);
887	rdma_ah_set_port_num(attr: ah_attr, port_num: 1);
888	rdma_ah_set_static_rate(attr: ah_attr, static_rate: 0);
889	return 0;
890	}
891
892	unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
893	__acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
894	{
895	unsigned long flags;
896
897	spin_lock_irqsave(&qp->scq->cq_lock, flags);
898	if (qp->rcq != qp->scq)
899	spin_lock(lock: &qp->rcq->cq_lock);
900	else
901	__acquire(&qp->rcq->cq_lock);
902
903	return flags;
904	}
905
906	void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
907	unsigned long flags)
908	__releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
909	{
910	if (qp->rcq != qp->scq)
911	spin_unlock(lock: &qp->rcq->cq_lock);
912	else
913	__release(&qp->rcq->cq_lock);
914	spin_unlock_irqrestore(lock: &qp->scq->cq_lock, flags);
915	}
916
917	static void bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
918	{
919	struct bnxt_re_qp *gsi_sqp;
920	struct bnxt_re_ah *gsi_sah;
921	struct bnxt_re_dev *rdev;
922	int rc;
923
924	rdev = qp->rdev;
925	gsi_sqp = rdev->gsi_ctx.gsi_sqp;
926	gsi_sah = rdev->gsi_ctx.gsi_sah;
927
928	ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n");
929	bnxt_qplib_destroy_ah(res: &rdev->qplib_res,
930	ah: &gsi_sah->qplib_ah,
931	block: true);
932	atomic_dec(v: &rdev->stats.res.ah_count);
933	bnxt_qplib_clean_qp(qp: &qp->qplib_qp);
934
935	ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
936	rc = bnxt_qplib_destroy_qp(res: &rdev->qplib_res, qp: &gsi_sqp->qplib_qp);
937	if (rc)
938	ibdev_err(ibdev: &rdev->ibdev, format: "Destroy Shadow QP failed");
939
940	bnxt_qplib_free_qp_res(res: &rdev->qplib_res, qp: &gsi_sqp->qplib_qp);
941
942	/* remove from active qp list */
943	mutex_lock(&rdev->qp_lock);
944	list_del(entry: &gsi_sqp->list);
945	mutex_unlock(lock: &rdev->qp_lock);
946	atomic_dec(v: &rdev->stats.res.qp_count);
947
948	kfree(objp: rdev->gsi_ctx.sqp_tbl);
949	kfree(objp: gsi_sah);
950	kfree(objp: gsi_sqp);
951	rdev->gsi_ctx.gsi_sqp = NULL;
952	rdev->gsi_ctx.gsi_sah = NULL;
953	rdev->gsi_ctx.sqp_tbl = NULL;
954	}
955
956	static void bnxt_re_del_unique_gid(struct bnxt_re_dev *rdev)
957	{
958	int rc;
959
960	if (!rdev->rcfw.roce_mirror)
961	return;
962
963	rc = bnxt_qplib_del_sgid(sgid_tbl: &rdev->qplib_res.sgid_tbl,
964	gid: (struct bnxt_qplib_gid *)&rdev->ugid,
965	vlan_id: 0xFFFF, update: true);
966	if (rc)
967	dev_err(rdev_to_dev(rdev), "Failed to delete unique GID, rc: %d\n", rc);
968	}
969
970	/* Queue Pairs */
971	int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
972	{
973	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
974	struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
975	struct bnxt_re_dev *rdev = qp->rdev;
976	struct bnxt_qplib_nq *scq_nq = NULL;
977	struct bnxt_qplib_nq *rcq_nq = NULL;
978	unsigned int flags;
979	int rc;
980
981	bnxt_re_debug_rem_qpinfo(rdev, qp);
982
983	bnxt_qplib_flush_cqn_wq(qp: &qp->qplib_qp);
984
985	rc = bnxt_qplib_destroy_qp(res: &rdev->qplib_res, qp: &qp->qplib_qp);
986	if (rc)
987	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to destroy HW QP");
988
989	if (rdma_is_kernel_res(res: &qp->ib_qp.res)) {
990	flags = bnxt_re_lock_cqs(qp);
991	bnxt_qplib_clean_qp(qp: &qp->qplib_qp);
992	bnxt_re_unlock_cqs(qp, flags);
993	}
994
995	bnxt_qplib_free_qp_res(res: &rdev->qplib_res, qp: &qp->qplib_qp);
996
997	if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
998	bnxt_re_destroy_gsi_sqp(qp);
999
1000	mutex_lock(&rdev->qp_lock);
1001	list_del(entry: &qp->list);
1002	mutex_unlock(lock: &rdev->qp_lock);
1003	atomic_dec(v: &rdev->stats.res.qp_count);
1004	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC)
1005	atomic_dec(v: &rdev->stats.res.rc_qp_count);
1006	else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD)
1007	atomic_dec(v: &rdev->stats.res.ud_qp_count);
1008
1009	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE)
1010	bnxt_re_del_unique_gid(rdev);
1011
1012	ib_umem_release(umem: qp->rumem);
1013	ib_umem_release(umem: qp->sumem);
1014
1015	/* Flush all the entries of notification queue associated with
1016	* given qp.
1017	*/
1018	scq_nq = qplib_qp->scq->nq;
1019	rcq_nq = qplib_qp->rcq->nq;
1020	bnxt_re_synchronize_nq(nq: scq_nq);
1021	if (scq_nq != rcq_nq)
1022	bnxt_re_synchronize_nq(nq: rcq_nq);
1023
1024	return 0;
1025	}
1026
1027	static u8 __from_ib_qp_type(enum ib_qp_type type)
1028	{
1029	switch (type) {
1030	case IB_QPT_GSI:
1031	return CMDQ_CREATE_QP1_TYPE_GSI;
1032	case IB_QPT_RC:
1033	return CMDQ_CREATE_QP_TYPE_RC;
1034	case IB_QPT_UD:
1035	return CMDQ_CREATE_QP_TYPE_UD;
1036	case IB_QPT_RAW_PACKET:
1037	return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE;
1038	default:
1039	return IB_QPT_MAX;
1040	}
1041	}
1042
1043	static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
1044	int rsge, int max)
1045	{
1046	if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
1047	rsge = max;
1048	return bnxt_re_get_rwqe_size(nsge: rsge);
1049	}
1050
1051	static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
1052	{
1053	u16 wqe_size, calc_ils;
1054
1055	wqe_size = bnxt_re_get_swqe_size(nsge);
1056	if (ilsize) {
1057	calc_ils = sizeof(struct sq_send_hdr) + ilsize;
1058	wqe_size = max_t(u16, calc_ils, wqe_size);
1059	wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
1060	}
1061	return wqe_size;
1062	}
1063
1064	static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
1065	struct ib_qp_init_attr *init_attr)
1066	{
1067	struct bnxt_qplib_dev_attr *dev_attr;
1068	struct bnxt_qplib_qp *qplqp;
1069	struct bnxt_re_dev *rdev;
1070	struct bnxt_qplib_q *sq;
1071	int align, ilsize;
1072
1073	rdev = qp->rdev;
1074	qplqp = &qp->qplib_qp;
1075	sq = &qplqp->sq;
1076	dev_attr = rdev->dev_attr;
1077
1078	align = sizeof(struct sq_send_hdr);
1079	ilsize = ALIGN(init_attr->cap.max_inline_data, align);
1080
1081	/* For gen p4 and gen p5 fixed wqe compatibility mode
1082	* wqe size is fixed to 128 bytes - ie 6 SGEs
1083	*/
1084	if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) {
1085	sq->wqe_size = bnxt_re_get_swqe_size(BNXT_STATIC_MAX_SGE);
1086	sq->max_sge = BNXT_STATIC_MAX_SGE;
1087	} else {
1088	sq->wqe_size = bnxt_re_get_wqe_size(ilsize, nsge: sq->max_sge);
1089	if (sq->wqe_size > bnxt_re_get_swqe_size(nsge: dev_attr->max_qp_sges))
1090	return -EINVAL;
1091	}
1092
1093	if (init_attr->cap.max_inline_data) {
1094	qplqp->max_inline_data = sq->wqe_size -
1095	sizeof(struct sq_send_hdr);
1096	init_attr->cap.max_inline_data = qplqp->max_inline_data;
1097	}
1098
1099	return 0;
1100	}
1101
1102	static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
1103	struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx,
1104	struct bnxt_re_qp_req *ureq)
1105	{
1106	struct bnxt_qplib_qp *qplib_qp;
1107	int bytes = 0, psn_sz;
1108	struct ib_umem *umem;
1109	int psn_nume;
1110
1111	qplib_qp = &qp->qplib_qp;
1112
1113	bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
1114	/* Consider mapping PSN search memory only for RC QPs. */
1115	if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
1116	psn_sz = bnxt_qplib_is_chip_gen_p5_p7(cctx: rdev->chip_ctx) ?
1117	sizeof(struct sq_psn_search_ext) :
1118	sizeof(struct sq_psn_search);
1119	if (cntx && bnxt_re_is_var_size_supported(rdev, uctx: cntx)) {
1120	psn_nume = ureq->sq_slots;
1121	} else {
1122	psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
1123	qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
1124	sizeof(struct bnxt_qplib_sge));
1125	}
1126	if (_is_host_msn_table(dev_cap_ext_flags2: rdev->qplib_res.dattr->dev_cap_flags2))
1127	psn_nume = roundup_pow_of_two(psn_nume);
1128	bytes += (psn_nume * psn_sz);
1129	}
1130
1131	bytes = PAGE_ALIGN(bytes);
1132	umem = ib_umem_get(device: &rdev->ibdev, addr: ureq->qpsva, size: bytes,
1133	access: IB_ACCESS_LOCAL_WRITE);
1134	if (IS_ERR(ptr: umem))
1135	return PTR_ERR(ptr: umem);
1136
1137	qp->sumem = umem;
1138	qplib_qp->sq.sg_info.umem = umem;
1139	qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
1140	qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
1141	qplib_qp->qp_handle = ureq->qp_handle;
1142
1143	if (!qp->qplib_qp.srq) {
1144	bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
1145	bytes = PAGE_ALIGN(bytes);
1146	umem = ib_umem_get(device: &rdev->ibdev, addr: ureq->qprva, size: bytes,
1147	access: IB_ACCESS_LOCAL_WRITE);
1148	if (IS_ERR(ptr: umem))
1149	goto rqfail;
1150	qp->rumem = umem;
1151	qplib_qp->rq.sg_info.umem = umem;
1152	qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
1153	qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
1154	}
1155
1156	qplib_qp->dpi = &cntx->dpi;
1157	return 0;
1158	rqfail:
1159	ib_umem_release(umem: qp->sumem);
1160	qp->sumem = NULL;
1161	memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
1162
1163	return PTR_ERR(ptr: umem);
1164	}
1165
1166	static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
1167	(struct bnxt_re_pd *pd,
1168	struct bnxt_qplib_res *qp1_res,
1169	struct bnxt_qplib_qp *qp1_qp)
1170	{
1171	struct bnxt_re_dev *rdev = pd->rdev;
1172	struct bnxt_re_ah *ah;
1173	union ib_gid sgid;
1174	int rc;
1175
1176	ah = kzalloc(sizeof(*ah), GFP_KERNEL);
1177	if (!ah)
1178	return NULL;
1179
1180	ah->rdev = rdev;
1181	ah->qplib_ah.pd = &pd->qplib_pd;
1182
1183	rc = bnxt_re_query_gid(ibdev: &rdev->ibdev, port_num: 1, index: 0, gid: &sgid);
1184	if (rc)
1185	goto fail;
1186
1187	/* supply the dgid data same as sgid */
1188	memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
1189	sizeof(union ib_gid));
1190	ah->qplib_ah.sgid_index = 0;
1191
1192	ah->qplib_ah.traffic_class = 0;
1193	ah->qplib_ah.flow_label = 0;
1194	ah->qplib_ah.hop_limit = 1;
1195	ah->qplib_ah.sl = 0;
1196	/* Have DMAC same as SMAC */
1197	ether_addr_copy(dst: ah->qplib_ah.dmac, src: rdev->netdev->dev_addr);
1198
1199	rc = bnxt_qplib_create_ah(res: &rdev->qplib_res, ah: &ah->qplib_ah, block: false);
1200	if (rc) {
1201	ibdev_err(ibdev: &rdev->ibdev,
1202	format: "Failed to allocate HW AH for Shadow QP");
1203	goto fail;
1204	}
1205	atomic_inc(v: &rdev->stats.res.ah_count);
1206
1207	return ah;
1208
1209	fail:
1210	kfree(objp: ah);
1211	return NULL;
1212	}
1213
1214	static struct bnxt_re_qp *bnxt_re_create_shadow_qp
1215	(struct bnxt_re_pd *pd,
1216	struct bnxt_qplib_res *qp1_res,
1217	struct bnxt_qplib_qp *qp1_qp)
1218	{
1219	struct bnxt_re_dev *rdev = pd->rdev;
1220	struct bnxt_re_qp *qp;
1221	int rc;
1222
1223	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1224	if (!qp)
1225	return NULL;
1226
1227	qp->rdev = rdev;
1228
1229	/* Initialize the shadow QP structure from the QP1 values */
1230	ether_addr_copy(dst: qp->qplib_qp.smac, src: rdev->netdev->dev_addr);
1231
1232	qp->qplib_qp.pd = &pd->qplib_pd;
1233	qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1234	qp->qplib_qp.type = IB_QPT_UD;
1235
1236	qp->qplib_qp.max_inline_data = 0;
1237	qp->qplib_qp.sig_type = true;
1238
1239	/* Shadow QP SQ depth should be same as QP1 RQ depth */
1240	qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(ilsize: 0, nsge: 6);
1241	qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1242	qp->qplib_qp.sq.max_sw_wqe = qp1_qp->rq.max_wqe;
1243	qp->qplib_qp.sq.max_sge = 2;
1244	/* Q full delta can be 1 since it is internal QP */
1245	qp->qplib_qp.sq.q_full_delta = 1;
1246	qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;
1247	qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;
1248
1249	qp->qplib_qp.scq = qp1_qp->scq;
1250	qp->qplib_qp.rcq = qp1_qp->rcq;
1251
1252	qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(nsge: 6);
1253	qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1254	qp->qplib_qp.rq.max_sw_wqe = qp1_qp->rq.max_wqe;
1255	qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
1256	/* Q full delta can be 1 since it is internal QP */
1257	qp->qplib_qp.rq.q_full_delta = 1;
1258	qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;
1259	qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;
1260
1261	qp->qplib_qp.mtu = qp1_qp->mtu;
1262
1263	qp->qplib_qp.sq_hdr_buf_size = 0;
1264	qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1265	qp->qplib_qp.dpi = &rdev->dpi_privileged;
1266
1267	rc = bnxt_qplib_create_qp(res: qp1_res, qp: &qp->qplib_qp);
1268	if (rc)
1269	goto fail;
1270
1271	spin_lock_init(&qp->sq_lock);
1272	INIT_LIST_HEAD(list: &qp->list);
1273	mutex_lock(&rdev->qp_lock);
1274	list_add_tail(new: &qp->list, head: &rdev->qp_list);
1275	atomic_inc(v: &rdev->stats.res.qp_count);
1276	mutex_unlock(lock: &rdev->qp_lock);
1277	return qp;
1278	fail:
1279	kfree(objp: qp);
1280	return NULL;
1281	}
1282
1283	static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1284	struct ib_qp_init_attr *init_attr,
1285	struct bnxt_re_ucontext *uctx)
1286	{
1287	struct bnxt_qplib_dev_attr *dev_attr;
1288	struct bnxt_qplib_qp *qplqp;
1289	struct bnxt_re_dev *rdev;
1290	struct bnxt_qplib_q *rq;
1291	int entries;
1292
1293	rdev = qp->rdev;
1294	qplqp = &qp->qplib_qp;
1295	rq = &qplqp->rq;
1296	dev_attr = rdev->dev_attr;
1297
1298	if (init_attr->srq) {
1299	struct bnxt_re_srq *srq;
1300
1301	srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq);
1302	qplqp->srq = &srq->qplib_srq;
1303	rq->max_wqe = 0;
1304	} else {
1305	rq->max_sge = init_attr->cap.max_recv_sge;
1306	if (rq->max_sge > dev_attr->max_qp_sges)
1307	rq->max_sge = dev_attr->max_qp_sges;
1308	init_attr->cap.max_recv_sge = rq->max_sge;
1309	rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rsge: rq->max_sge,
1310	max: dev_attr->max_qp_sges);
1311	/* Allocate 1 more than what's provided so posting max doesn't
1312	* mean empty.
1313	*/
1314	entries = bnxt_re_init_depth(ent: init_attr->cap.max_recv_wr + 1, uctx);
1315	rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1316	rq->max_sw_wqe = rq->max_wqe;
1317	rq->q_full_delta = 0;
1318	rq->sg_info.pgsize = PAGE_SIZE;
1319	rq->sg_info.pgshft = PAGE_SHIFT;
1320	}
1321
1322	return 0;
1323	}
1324
1325	static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
1326	{
1327	struct bnxt_qplib_dev_attr *dev_attr;
1328	struct bnxt_qplib_qp *qplqp;
1329	struct bnxt_re_dev *rdev;
1330
1331	rdev = qp->rdev;
1332	qplqp = &qp->qplib_qp;
1333	dev_attr = rdev->dev_attr;
1334
1335	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx: rdev->chip_ctx)) {
1336	qplqp->rq.max_sge = dev_attr->max_qp_sges;
1337	if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1338	qplqp->rq.max_sge = dev_attr->max_qp_sges;
1339	qplqp->rq.max_sge = 6;
1340	}
1341	}
1342
1343	static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1344	struct ib_qp_init_attr *init_attr,
1345	struct bnxt_re_ucontext *uctx,
1346	struct bnxt_re_qp_req *ureq)
1347	{
1348	struct bnxt_qplib_dev_attr *dev_attr;
1349	struct bnxt_qplib_qp *qplqp;
1350	struct bnxt_re_dev *rdev;
1351	struct bnxt_qplib_q *sq;
1352	int diff = 0;
1353	int entries;
1354	int rc;
1355
1356	rdev = qp->rdev;
1357	qplqp = &qp->qplib_qp;
1358	sq = &qplqp->sq;
1359	dev_attr = rdev->dev_attr;
1360
1361	sq->max_sge = init_attr->cap.max_send_sge;
1362	entries = init_attr->cap.max_send_wr;
1363	if (uctx && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {
1364	sq->max_wqe = ureq->sq_slots;
1365	sq->max_sw_wqe = ureq->sq_slots;
1366	sq->wqe_size = sizeof(struct sq_sge);
1367	} else {
1368	if (sq->max_sge > dev_attr->max_qp_sges) {
1369	sq->max_sge = dev_attr->max_qp_sges;
1370	init_attr->cap.max_send_sge = sq->max_sge;
1371	}
1372
1373	rc = bnxt_re_setup_swqe_size(qp, init_attr);
1374	if (rc)
1375	return rc;
1376
1377	/* Allocate 128 + 1 more than what's provided */
1378	diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
1379	0 : BNXT_QPLIB_RESERVED_QP_WRS;
1380	entries = bnxt_re_init_depth(ent: entries + diff + 1, uctx);
1381	sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
1382	if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
1383	sq->max_sw_wqe = bnxt_qplib_get_depth(que: sq, wqe_mode: qplqp->wqe_mode, is_sq: true);
1384	else
1385	sq->max_sw_wqe = sq->max_wqe;
1386
1387	}
1388	sq->q_full_delta = diff + 1;
1389	/*
1390	* Reserving one slot for Phantom WQE. Application can
1391	* post one extra entry in this case. But allowing this to avoid
1392	* unexpected Queue full condition
1393	*/
1394	qplqp->sq.q_full_delta -= 1;
1395	qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1396	qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
1397
1398	return 0;
1399	}
1400
1401	static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
1402	struct ib_qp_init_attr *init_attr,
1403	struct bnxt_re_ucontext *uctx)
1404	{
1405	struct bnxt_qplib_dev_attr *dev_attr;
1406	struct bnxt_qplib_qp *qplqp;
1407	struct bnxt_re_dev *rdev;
1408	int entries;
1409
1410	rdev = qp->rdev;
1411	qplqp = &qp->qplib_qp;
1412	dev_attr = rdev->dev_attr;
1413
1414	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx: rdev->chip_ctx)) {
1415	entries = bnxt_re_init_depth(ent: init_attr->cap.max_send_wr + 1, uctx);
1416	qplqp->sq.max_wqe = min_t(u32, entries,
1417	dev_attr->max_qp_wqes + 1);
1418	qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1419	init_attr->cap.max_send_wr;
1420	qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
1421	if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
1422	qplqp->sq.max_sge = dev_attr->max_qp_sges;
1423	}
1424	}
1425
1426	static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
1427	struct ib_qp_init_attr *init_attr)
1428	{
1429	struct bnxt_qplib_chip_ctx *chip_ctx;
1430	int qptype;
1431
1432	chip_ctx = rdev->chip_ctx;
1433
1434	qptype = __from_ib_qp_type(type: init_attr->qp_type);
1435	if (qptype == IB_QPT_MAX) {
1436	ibdev_err(ibdev: &rdev->ibdev, format: "QP type 0x%x not supported", qptype);
1437	qptype = -EOPNOTSUPP;
1438	goto out;
1439	}
1440
1441	if (bnxt_qplib_is_chip_gen_p5_p7(cctx: chip_ctx) &&
1442	init_attr->qp_type == IB_QPT_GSI)
1443	qptype = CMDQ_CREATE_QP_TYPE_GSI;
1444	out:
1445	return qptype;
1446	}
1447
1448	static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1449	struct ib_qp_init_attr *init_attr,
1450	struct bnxt_re_ucontext *uctx,
1451	struct bnxt_re_qp_req *ureq)
1452	{
1453	struct bnxt_qplib_dev_attr *dev_attr;
1454	struct bnxt_qplib_qp *qplqp;
1455	struct bnxt_re_dev *rdev;
1456	struct bnxt_re_cq *cq;
1457	int rc = 0, qptype;
1458
1459	rdev = qp->rdev;
1460	qplqp = &qp->qplib_qp;
1461	dev_attr = rdev->dev_attr;
1462
1463	/* Setup misc params */
1464	ether_addr_copy(dst: qplqp->smac, src: rdev->netdev->dev_addr);
1465	qplqp->pd = &pd->qplib_pd;
1466	qplqp->qp_handle = (u64)qplqp;
1467	qplqp->max_inline_data = init_attr->cap.max_inline_data;
1468	qplqp->sig_type = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1469	qptype = bnxt_re_init_qp_type(rdev, init_attr);
1470	if (qptype < 0) {
1471	rc = qptype;
1472	goto out;
1473	}
1474	qplqp->type = (u8)qptype;
1475	qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx);
1476	if (init_attr->qp_type == IB_QPT_RC) {
1477	qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
1478	qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1479	}
1480	qplqp->mtu = ib_mtu_enum_to_int(mtu: iboe_get_mtu(mtu: rdev->netdev->mtu));
1481	qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1482	if (init_attr->create_flags) {
1483	ibdev_dbg(&rdev->ibdev,
1484	"QP create flags 0x%x not supported",
1485	init_attr->create_flags);
1486	return -EOPNOTSUPP;
1487	}
1488
1489	/* Setup CQs */
1490	if (init_attr->send_cq) {
1491	cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq);
1492	qplqp->scq = &cq->qplib_cq;
1493	qp->scq = cq;
1494	}
1495
1496	if (init_attr->recv_cq) {
1497	cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq);
1498	qplqp->rcq = &cq->qplib_cq;
1499	qp->rcq = cq;
1500	}
1501
1502	/* Setup RQ/SRQ */
1503	rc = bnxt_re_init_rq_attr(qp, init_attr, uctx);
1504	if (rc)
1505	goto out;
1506	if (init_attr->qp_type == IB_QPT_GSI)
1507	bnxt_re_adjust_gsi_rq_attr(qp);
1508
1509	/* Setup SQ */
1510	rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq);
1511	if (rc)
1512	goto out;
1513	if (init_attr->qp_type == IB_QPT_GSI)
1514	bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx);
1515
1516	if (uctx) /* This will update DPI and qp_handle */
1517	rc = bnxt_re_init_user_qp(rdev, pd, qp, cntx: uctx, ureq);
1518	out:
1519	return rc;
1520	}
1521
1522	static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
1523	struct bnxt_re_pd *pd)
1524	{
1525	struct bnxt_re_sqp_entries *sqp_tbl;
1526	struct bnxt_re_dev *rdev;
1527	struct bnxt_re_qp *sqp;
1528	struct bnxt_re_ah *sah;
1529	int rc = 0;
1530
1531	rdev = qp->rdev;
1532	/* Create a shadow QP to handle the QP1 traffic */
1533	sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl),
1534	GFP_KERNEL);
1535	if (!sqp_tbl)
1536	return -ENOMEM;
1537	rdev->gsi_ctx.sqp_tbl = sqp_tbl;
1538
1539	sqp = bnxt_re_create_shadow_qp(pd, qp1_res: &rdev->qplib_res, qp1_qp: &qp->qplib_qp);
1540	if (!sqp) {
1541	rc = -ENODEV;
1542	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to create Shadow QP for QP1");
1543	goto out;
1544	}
1545	rdev->gsi_ctx.gsi_sqp = sqp;
1546
1547	sqp->rcq = qp->rcq;
1548	sqp->scq = qp->scq;
1549	sah = bnxt_re_create_shadow_qp_ah(pd, qp1_res: &rdev->qplib_res,
1550	qp1_qp: &qp->qplib_qp);
1551	if (!sah) {
1552	bnxt_qplib_destroy_qp(res: &rdev->qplib_res,
1553	qp: &sqp->qplib_qp);
1554	rc = -ENODEV;
1555	ibdev_err(ibdev: &rdev->ibdev,
1556	format: "Failed to create AH entry for ShadowQP");
1557	goto out;
1558	}
1559	rdev->gsi_ctx.gsi_sah = sah;
1560
1561	return 0;
1562	out:
1563	kfree(objp: sqp_tbl);
1564	return rc;
1565	}
1566
1567	static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1568	struct ib_qp_init_attr *init_attr)
1569	{
1570	struct bnxt_re_dev *rdev;
1571	struct bnxt_qplib_qp *qplqp;
1572	int rc;
1573
1574	rdev = qp->rdev;
1575	qplqp = &qp->qplib_qp;
1576
1577	qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1578	qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
1579
1580	rc = bnxt_qplib_create_qp1(res: &rdev->qplib_res, qp: qplqp);
1581	if (rc) {
1582	ibdev_err(ibdev: &rdev->ibdev, format: "create HW QP1 failed!");
1583	goto out;
1584	}
1585
1586	rc = bnxt_re_create_shadow_gsi(qp, pd);
1587	out:
1588	return rc;
1589	}
1590
1591	static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
1592	struct ib_qp_init_attr *init_attr,
1593	struct bnxt_qplib_dev_attr *dev_attr)
1594	{
1595	bool rc = true;
1596
1597	if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes ||
1598	init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes ||
1599	init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
1600	init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
1601	init_attr->cap.max_inline_data > dev_attr->max_inline_data) {
1602	ibdev_err(ibdev: &rdev->ibdev,
1603	format: "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
1604	init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
1605	init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
1606	init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
1607	init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
1608	init_attr->cap.max_inline_data,
1609	dev_attr->max_inline_data);
1610	rc = false;
1611	}
1612	return rc;
1613	}
1614
1615	static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev)
1616	{
1617	struct bnxt_qplib_ctx *hctx = &rdev->qplib_ctx;
1618	struct bnxt_qplib_res *res = &rdev->qplib_res;
1619	int rc;
1620
1621	if (!rdev->rcfw.roce_mirror)
1622	return 0;
1623
1624	rdev->ugid.global.subnet_prefix = cpu_to_be64(0xfe8000000000abcdLL);
1625	addrconf_ifid_eui48(eui: &rdev->ugid.raw[8], dev: rdev->netdev);
1626
1627	rc = bnxt_qplib_add_sgid(sgid_tbl: &res->sgid_tbl,
1628	gid: (struct bnxt_qplib_gid *)&rdev->ugid,
1629	mac: rdev->qplib_res.netdev->dev_addr,
1630	vlan_id: 0xFFFF, update: true, index: &rdev->ugid_index, is_ugid: true,
1631	stats_ctx_id: hctx->stats3.fw_id);
1632	if (rc)
1633	dev_err(rdev_to_dev(rdev), "Failed to add unique GID. rc = %d\n", rc);
1634
1635	return rc;
1636	}
1637
1638	int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1639	struct ib_udata *udata)
1640	{
1641	struct bnxt_qplib_dev_attr *dev_attr;
1642	struct bnxt_re_ucontext *uctx;
1643	struct bnxt_re_qp_req ureq;
1644	struct bnxt_re_dev *rdev;
1645	struct bnxt_re_pd *pd;
1646	struct bnxt_re_qp *qp;
1647	struct ib_pd *ib_pd;
1648	u32 active_qps;
1649	int rc;
1650
1651	ib_pd = ib_qp->pd;
1652	pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1653	rdev = pd->rdev;
1654	dev_attr = rdev->dev_attr;
1655	qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1656
1657	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
1658	if (udata)
1659	if (ib_copy_from_udata(dest: &ureq, udata, min(udata->inlen, sizeof(ureq))))
1660	return -EFAULT;
1661
1662	rc = bnxt_re_test_qp_limits(rdev, init_attr: qp_init_attr, dev_attr);
1663	if (!rc) {
1664	rc = -EINVAL;
1665	goto fail;
1666	}
1667
1668	qp->rdev = rdev;
1669	rc = bnxt_re_init_qp_attr(qp, pd, init_attr: qp_init_attr, uctx, ureq: &ureq);
1670	if (rc)
1671	goto fail;
1672
1673	if (qp_init_attr->qp_type == IB_QPT_GSI &&
1674	!(bnxt_qplib_is_chip_gen_p5_p7(cctx: rdev->chip_ctx))) {
1675	rc = bnxt_re_create_gsi_qp(qp, pd, init_attr: qp_init_attr);
1676	if (rc == -ENODEV)
1677	goto qp_destroy;
1678	if (rc)
1679	goto fail;
1680	} else {
1681	rc = bnxt_qplib_create_qp(res: &rdev->qplib_res, qp: &qp->qplib_qp);
1682	if (rc) {
1683	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to create HW QP");
1684	goto free_umem;
1685	}
1686	if (udata) {
1687	struct bnxt_re_qp_resp resp;
1688
1689	resp.qpid = qp->qplib_qp.id;
1690	resp.rsvd = 0;
1691	rc = ib_copy_to_udata(udata, src: &resp, len: sizeof(resp));
1692	if (rc) {
1693	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to copy QP udata");
1694	goto qp_destroy;
1695	}
1696	}
1697	}
1698
1699	/* Support for RawEth QP is added to capture TCP pkt dump.
1700	* So unique SGID is used to avoid incorrect statistics on per
1701	* function stats_ctx
1702	*/
1703	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE) {
1704	rc = bnxt_re_add_unique_gid(rdev);
1705	if (rc)
1706	goto qp_destroy;
1707	qp->qplib_qp.ugid_index = rdev->ugid_index;
1708	}
1709
1710	qp->ib_qp.qp_num = qp->qplib_qp.id;
1711	if (qp_init_attr->qp_type == IB_QPT_GSI)
1712	rdev->gsi_ctx.gsi_qp = qp;
1713	spin_lock_init(&qp->sq_lock);
1714	spin_lock_init(&qp->rq_lock);
1715	INIT_LIST_HEAD(list: &qp->list);
1716	mutex_lock(&rdev->qp_lock);
1717	list_add_tail(new: &qp->list, head: &rdev->qp_list);
1718	mutex_unlock(lock: &rdev->qp_lock);
1719	active_qps = atomic_inc_return(v: &rdev->stats.res.qp_count);
1720	if (active_qps > rdev->stats.res.qp_watermark)
1721	rdev->stats.res.qp_watermark = active_qps;
1722	if (qp_init_attr->qp_type == IB_QPT_RC) {
1723	active_qps = atomic_inc_return(v: &rdev->stats.res.rc_qp_count);
1724	if (active_qps > rdev->stats.res.rc_qp_watermark)
1725	rdev->stats.res.rc_qp_watermark = active_qps;
1726	} else if (qp_init_attr->qp_type == IB_QPT_UD) {
1727	active_qps = atomic_inc_return(v: &rdev->stats.res.ud_qp_count);
1728	if (active_qps > rdev->stats.res.ud_qp_watermark)
1729	rdev->stats.res.ud_qp_watermark = active_qps;
1730	}
1731	bnxt_re_debug_add_qpinfo(rdev, qp);
1732
1733	return 0;
1734	qp_destroy:
1735	bnxt_qplib_destroy_qp(res: &rdev->qplib_res, qp: &qp->qplib_qp);
1736	free_umem:
1737	ib_umem_release(umem: qp->rumem);
1738	ib_umem_release(umem: qp->sumem);
1739	fail:
1740	return rc;
1741	}
1742
1743	static u8 __from_ib_qp_state(enum ib_qp_state state)
1744	{
1745	switch (state) {
1746	case IB_QPS_RESET:
1747	return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1748	case IB_QPS_INIT:
1749	return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1750	case IB_QPS_RTR:
1751	return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1752	case IB_QPS_RTS:
1753	return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1754	case IB_QPS_SQD:
1755	return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1756	case IB_QPS_SQE:
1757	return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1758	case IB_QPS_ERR:
1759	default:
1760	return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1761	}
1762	}
1763
1764	static enum ib_qp_state __to_ib_qp_state(u8 state)
1765	{
1766	switch (state) {
1767	case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1768	return IB_QPS_RESET;
1769	case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1770	return IB_QPS_INIT;
1771	case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1772	return IB_QPS_RTR;
1773	case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1774	return IB_QPS_RTS;
1775	case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1776	return IB_QPS_SQD;
1777	case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1778	return IB_QPS_SQE;
1779	case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1780	default:
1781	return IB_QPS_ERR;
1782	}
1783	}
1784
1785	static u32 __from_ib_mtu(enum ib_mtu mtu)
1786	{
1787	switch (mtu) {
1788	case IB_MTU_256:
1789	return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1790	case IB_MTU_512:
1791	return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1792	case IB_MTU_1024:
1793	return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1794	case IB_MTU_2048:
1795	return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1796	case IB_MTU_4096:
1797	return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1798	default:
1799	return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1800	}
1801	}
1802
1803	static enum ib_mtu __to_ib_mtu(u32 mtu)
1804	{
1805	switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1806	case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1807	return IB_MTU_256;
1808	case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1809	return IB_MTU_512;
1810	case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1811	return IB_MTU_1024;
1812	case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1813	return IB_MTU_2048;
1814	case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1815	return IB_MTU_4096;
1816	default:
1817	return IB_MTU_2048;
1818	}
1819	}
1820
1821	/* Shared Receive Queues */
1822	int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
1823	{
1824	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1825	ib_srq);
1826	struct bnxt_re_dev *rdev = srq->rdev;
1827	struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1828
1829	if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
1830	free_page((unsigned long)srq->uctx_srq_page);
1831	hash_del(node: &srq->hash_entry);
1832	}
1833	bnxt_qplib_destroy_srq(res: &rdev->qplib_res, srq: qplib_srq);
1834	ib_umem_release(umem: srq->umem);
1835	atomic_dec(v: &rdev->stats.res.srq_count);
1836	return 0;
1837	}
1838
1839	static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
1840	struct bnxt_re_pd *pd,
1841	struct bnxt_re_srq *srq,
1842	struct ib_udata *udata)
1843	{
1844	struct bnxt_re_srq_req ureq;
1845	struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1846	struct ib_umem *umem;
1847	int bytes = 0;
1848	struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
1849	udata, struct bnxt_re_ucontext, ib_uctx);
1850
1851	if (ib_copy_from_udata(dest: &ureq, udata, len: sizeof(ureq)))
1852	return -EFAULT;
1853
1854	bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
1855	bytes = PAGE_ALIGN(bytes);
1856	umem = ib_umem_get(device: &rdev->ibdev, addr: ureq.srqva, size: bytes,
1857	access: IB_ACCESS_LOCAL_WRITE);
1858	if (IS_ERR(ptr: umem))
1859	return PTR_ERR(ptr: umem);
1860
1861	srq->umem = umem;
1862	qplib_srq->sg_info.umem = umem;
1863	qplib_srq->sg_info.pgsize = PAGE_SIZE;
1864	qplib_srq->sg_info.pgshft = PAGE_SHIFT;
1865	qplib_srq->srq_handle = ureq.srq_handle;
1866	qplib_srq->dpi = &cntx->dpi;
1867
1868	return 0;
1869	}
1870
1871	int bnxt_re_create_srq(struct ib_srq *ib_srq,
1872	struct ib_srq_init_attr *srq_init_attr,
1873	struct ib_udata *udata)
1874	{
1875	struct bnxt_qplib_dev_attr *dev_attr;
1876	struct bnxt_re_ucontext *uctx;
1877	struct bnxt_re_dev *rdev;
1878	struct bnxt_re_srq *srq;
1879	struct bnxt_re_pd *pd;
1880	struct ib_pd *ib_pd;
1881	u32 active_srqs;
1882	int rc, entries;
1883
1884	ib_pd = ib_srq->pd;
1885	pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1886	rdev = pd->rdev;
1887	dev_attr = rdev->dev_attr;
1888	srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
1889
1890	if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
1891	ibdev_err(ibdev: &rdev->ibdev, format: "Create CQ failed - max exceeded");
1892	rc = -EINVAL;
1893	goto exit;
1894	}
1895
1896	if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
1897	rc = -EOPNOTSUPP;
1898	goto exit;
1899	}
1900
1901	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
1902	srq->rdev = rdev;
1903	srq->qplib_srq.pd = &pd->qplib_pd;
1904	srq->qplib_srq.dpi = &rdev->dpi_privileged;
1905	/* Allocate 1 more than what's provided so posting max doesn't
1906	* mean empty
1907	*/
1908	entries = bnxt_re_init_depth(ent: srq_init_attr->attr.max_wr + 1, uctx);
1909	if (entries > dev_attr->max_srq_wqes + 1)
1910	entries = dev_attr->max_srq_wqes + 1;
1911	srq->qplib_srq.max_wqe = entries;
1912
1913	srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
1914	/* 128 byte wqe size for SRQ . So use max sges */
1915	srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(nsge: dev_attr->max_srq_sges);
1916	srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1917	srq->srq_limit = srq_init_attr->attr.srq_limit;
1918	srq->qplib_srq.eventq_hw_ring_id = rdev->nqr->nq[0].ring_id;
1919	srq->qplib_srq.sg_info.pgsize = PAGE_SIZE;
1920	srq->qplib_srq.sg_info.pgshft = PAGE_SHIFT;
1921
1922	if (udata) {
1923	rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
1924	if (rc)
1925	goto fail;
1926	}
1927
1928	rc = bnxt_qplib_create_srq(res: &rdev->qplib_res, srq: &srq->qplib_srq);
1929	if (rc) {
1930	ibdev_err(ibdev: &rdev->ibdev, format: "Create HW SRQ failed!");
1931	goto fail;
1932	}
1933
1934	if (udata) {
1935	struct bnxt_re_srq_resp resp = {};
1936
1937	resp.srqid = srq->qplib_srq.id;
1938	if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
1939	hash_add(rdev->srq_hash, &srq->hash_entry, srq->qplib_srq.id);
1940	srq->uctx_srq_page = (void *)get_zeroed_page(GFP_KERNEL);
1941	if (!srq->uctx_srq_page) {
1942	rc = -ENOMEM;
1943	goto fail;
1944	}
1945	resp.comp_mask |= BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT;
1946	}
1947	rc = ib_copy_to_udata(udata, src: &resp, len: sizeof(resp));
1948	if (rc) {
1949	ibdev_err(ibdev: &rdev->ibdev, format: "SRQ copy to udata failed!");
1950	bnxt_qplib_destroy_srq(res: &rdev->qplib_res,
1951	srq: &srq->qplib_srq);
1952	goto fail;
1953	}
1954	}
1955	active_srqs = atomic_inc_return(v: &rdev->stats.res.srq_count);
1956	if (active_srqs > rdev->stats.res.srq_watermark)
1957	rdev->stats.res.srq_watermark = active_srqs;
1958	spin_lock_init(&srq->lock);
1959
1960	return 0;
1961
1962	fail:
1963	ib_umem_release(umem: srq->umem);
1964	exit:
1965	return rc;
1966	}
1967
1968	int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
1969	enum ib_srq_attr_mask srq_attr_mask,
1970	struct ib_udata *udata)
1971	{
1972	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1973	ib_srq);
1974	struct bnxt_re_dev *rdev = srq->rdev;
1975
1976	switch (srq_attr_mask) {
1977	case IB_SRQ_MAX_WR:
1978	/* SRQ resize is not supported */
1979	return -EINVAL;
1980	case IB_SRQ_LIMIT:
1981	/* Change the SRQ threshold */
1982	if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
1983	return -EINVAL;
1984
1985	srq->qplib_srq.threshold = srq_attr->srq_limit;
1986	bnxt_qplib_srq_arm_db(info: &srq->qplib_srq.dbinfo, th: srq->qplib_srq.threshold);
1987
1988	/* On success, update the shadow */
1989	srq->srq_limit = srq_attr->srq_limit;
1990	/* No need to Build and send response back to udata */
1991	return 0;
1992	default:
1993	ibdev_err(ibdev: &rdev->ibdev,
1994	format: "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
1995	return -EINVAL;
1996	}
1997	}
1998
1999	int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
2000	{
2001	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
2002	ib_srq);
2003	struct bnxt_re_srq tsrq;
2004	struct bnxt_re_dev *rdev = srq->rdev;
2005	int rc;
2006
2007	/* Get live SRQ attr */
2008	tsrq.qplib_srq.id = srq->qplib_srq.id;
2009	rc = bnxt_qplib_query_srq(res: &rdev->qplib_res, srq: &tsrq.qplib_srq);
2010	if (rc) {
2011	ibdev_err(ibdev: &rdev->ibdev, format: "Query HW SRQ failed!");
2012	return rc;
2013	}
2014	srq_attr->max_wr = srq->qplib_srq.max_wqe;
2015	srq_attr->max_sge = srq->qplib_srq.max_sge;
2016	srq_attr->srq_limit = tsrq.qplib_srq.threshold;
2017
2018	return 0;
2019	}
2020
2021	int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
2022	const struct ib_recv_wr **bad_wr)
2023	{
2024	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
2025	ib_srq);
2026	struct bnxt_qplib_swqe wqe;
2027	unsigned long flags;
2028	int rc = 0;
2029
2030	spin_lock_irqsave(&srq->lock, flags);
2031	while (wr) {
2032	/* Transcribe each ib_recv_wr to qplib_swqe */
2033	wqe.num_sge = wr->num_sge;
2034	bnxt_re_build_sgl(ib_sg_list: wr->sg_list, sg_list: wqe.sg_list, num: wr->num_sge);
2035	wqe.wr_id = wr->wr_id;
2036	wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2037
2038	rc = bnxt_qplib_post_srq_recv(srq: &srq->qplib_srq, wqe: &wqe);
2039	if (rc) {
2040	*bad_wr = wr;
2041	break;
2042	}
2043	wr = wr->next;
2044	}
2045	spin_unlock_irqrestore(lock: &srq->lock, flags);
2046
2047	return rc;
2048	}
2049	static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
2050	struct bnxt_re_qp *qp1_qp,
2051	int qp_attr_mask)
2052	{
2053	struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
2054	int rc;
2055
2056	if (qp_attr_mask & IB_QP_STATE) {
2057	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2058	qp->qplib_qp.state = qp1_qp->qplib_qp.state;
2059	}
2060	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2061	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2062	qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
2063	}
2064
2065	if (qp_attr_mask & IB_QP_QKEY) {
2066	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2067	/* Using a Random QKEY */
2068	qp->qplib_qp.qkey = 0x81818181;
2069	}
2070	if (qp_attr_mask & IB_QP_SQ_PSN) {
2071	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2072	qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
2073	}
2074
2075	rc = bnxt_qplib_modify_qp(res: &rdev->qplib_res, qp: &qp->qplib_qp);
2076	if (rc)
2077	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to modify Shadow QP for QP1");
2078	return rc;
2079	}
2080
2081	int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2082	int qp_attr_mask, struct ib_udata *udata)
2083	{
2084	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2085	struct bnxt_re_dev *rdev = qp->rdev;
2086	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
2087	enum ib_qp_state curr_qp_state, new_qp_state;
2088	int rc, entries;
2089	unsigned int flags;
2090	u8 nw_type;
2091
2092	if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2093	return -EOPNOTSUPP;
2094
2095	qp->qplib_qp.modify_flags = 0;
2096	if (qp_attr_mask & IB_QP_STATE) {
2097	curr_qp_state = __to_ib_qp_state(state: qp->qplib_qp.cur_qp_state);
2098	new_qp_state = qp_attr->qp_state;
2099	if (!ib_modify_qp_is_ok(cur_state: curr_qp_state, next_state: new_qp_state,
2100	type: ib_qp->qp_type, mask: qp_attr_mask)) {
2101	ibdev_err(ibdev: &rdev->ibdev,
2102	format: "Invalid attribute mask: %#x specified ",
2103	qp_attr_mask);
2104	ibdev_err(ibdev: &rdev->ibdev,
2105	format: "for qpn: %#x type: %#x",
2106	ib_qp->qp_num, ib_qp->qp_type);
2107	ibdev_err(ibdev: &rdev->ibdev,
2108	format: "curr_qp_state=0x%x, new_qp_state=0x%x\n",
2109	curr_qp_state, new_qp_state);
2110	return -EINVAL;
2111	}
2112	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2113	qp->qplib_qp.state = __from_ib_qp_state(state: qp_attr->qp_state);
2114
2115	if (!qp->sumem &&
2116	qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
2117	ibdev_dbg(&rdev->ibdev,
2118	"Move QP = %p to flush list\n", qp);
2119	flags = bnxt_re_lock_cqs(qp);
2120	bnxt_qplib_add_flush_qp(qp: &qp->qplib_qp);
2121	bnxt_re_unlock_cqs(qp, flags);
2122	}
2123	if (!qp->sumem &&
2124	qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
2125	ibdev_dbg(&rdev->ibdev,
2126	"Move QP = %p out of flush list\n", qp);
2127	flags = bnxt_re_lock_cqs(qp);
2128	bnxt_qplib_clean_qp(qp: &qp->qplib_qp);
2129	bnxt_re_unlock_cqs(qp, flags);
2130	}
2131	}
2132	if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
2133	qp->qplib_qp.modify_flags |=
2134	CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
2135	qp->qplib_qp.en_sqd_async_notify = true;
2136	}
2137	if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
2138	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
2139	qp->qplib_qp.access =
2140	__qp_access_flags_from_ib(cctx: qp->qplib_qp.cctx,
2141	iflags: qp_attr->qp_access_flags);
2142	/* LOCAL_WRITE access must be set to allow RC receive */
2143	qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
2144	}
2145	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2146	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2147	qp->qplib_qp.pkey_index = qp_attr->pkey_index;
2148	}
2149	if (qp_attr_mask & IB_QP_QKEY) {
2150	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2151	qp->qplib_qp.qkey = qp_attr->qkey;
2152	}
2153	if (qp_attr_mask & IB_QP_AV) {
2154	const struct ib_global_route *grh =
2155	rdma_ah_read_grh(attr: &qp_attr->ah_attr);
2156	const struct ib_gid_attr *sgid_attr;
2157	struct bnxt_re_gid_ctx *ctx;
2158
2159	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
2160	CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
2161	CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
2162	CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
2163	CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
2164	CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
2165	CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
2166	memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
2167	sizeof(qp->qplib_qp.ah.dgid.data));
2168	qp->qplib_qp.ah.flow_label = grh->flow_label;
2169	sgid_attr = grh->sgid_attr;
2170	/* Get the HW context of the GID. The reference
2171	* of GID table entry is already taken by the caller.
2172	*/
2173	ctx = rdma_read_gid_hw_context(attr: sgid_attr);
2174	qp->qplib_qp.ah.sgid_index = ctx->idx;
2175	qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
2176	qp->qplib_qp.ah.hop_limit = grh->hop_limit;
2177	qp->qplib_qp.ah.traffic_class = grh->traffic_class >> 2;
2178	qp->qplib_qp.ah.sl = rdma_ah_get_sl(attr: &qp_attr->ah_attr);
2179	ether_addr_copy(dst: qp->qplib_qp.ah.dmac,
2180	src: qp_attr->ah_attr.roce.dmac);
2181
2182	rc = rdma_read_gid_l2_fields(attr: sgid_attr, NULL,
2183	smac: &qp->qplib_qp.smac[0]);
2184	if (rc)
2185	return rc;
2186
2187	nw_type = rdma_gid_attr_network_type(attr: sgid_attr);
2188	switch (nw_type) {
2189	case RDMA_NETWORK_IPV4:
2190	qp->qplib_qp.nw_type =
2191	CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
2192	break;
2193	case RDMA_NETWORK_IPV6:
2194	qp->qplib_qp.nw_type =
2195	CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
2196	break;
2197	default:
2198	qp->qplib_qp.nw_type =
2199	CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
2200	break;
2201	}
2202	}
2203
2204	if (qp_attr->qp_state == IB_QPS_RTR) {
2205	enum ib_mtu qpmtu;
2206
2207	qpmtu = iboe_get_mtu(mtu: rdev->netdev->mtu);
2208	if (qp_attr_mask & IB_QP_PATH_MTU) {
2209	if (ib_mtu_enum_to_int(mtu: qp_attr->path_mtu) >
2210	ib_mtu_enum_to_int(mtu: qpmtu))
2211	return -EINVAL;
2212	qpmtu = qp_attr->path_mtu;
2213	}
2214
2215	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
2216	qp->qplib_qp.path_mtu = __from_ib_mtu(mtu: qpmtu);
2217	qp->qplib_qp.mtu = ib_mtu_enum_to_int(mtu: qpmtu);
2218	}
2219
2220	if (qp_attr_mask & IB_QP_TIMEOUT) {
2221	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
2222	qp->qplib_qp.timeout = qp_attr->timeout;
2223	}
2224	if (qp_attr_mask & IB_QP_RETRY_CNT) {
2225	qp->qplib_qp.modify_flags |=
2226	CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
2227	qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
2228	}
2229	if (qp_attr_mask & IB_QP_RNR_RETRY) {
2230	qp->qplib_qp.modify_flags |=
2231	CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
2232	qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
2233	}
2234	if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
2235	qp->qplib_qp.modify_flags |=
2236	CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
2237	qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
2238	}
2239	if (qp_attr_mask & IB_QP_RQ_PSN) {
2240	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
2241	qp->qplib_qp.rq.psn = qp_attr->rq_psn;
2242	}
2243	if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2244	qp->qplib_qp.modify_flags |=
2245	CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
2246	/* Cap the max_rd_atomic to device max */
2247	qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
2248	dev_attr->max_qp_rd_atom);
2249	}
2250	if (qp_attr_mask & IB_QP_SQ_PSN) {
2251	qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2252	qp->qplib_qp.sq.psn = qp_attr->sq_psn;
2253	}
2254	if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2255	if (qp_attr->max_dest_rd_atomic >
2256	dev_attr->max_qp_init_rd_atom) {
2257	ibdev_err(ibdev: &rdev->ibdev,
2258	format: "max_dest_rd_atomic requested%d is > dev_max%d",
2259	qp_attr->max_dest_rd_atomic,
2260	dev_attr->max_qp_init_rd_atom);
2261	return -EINVAL;
2262	}
2263
2264	qp->qplib_qp.modify_flags |=
2265	CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
2266	qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
2267	}
2268	if (qp_attr_mask & IB_QP_CAP) {
2269	struct bnxt_re_ucontext *uctx =
2270	rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
2271
2272	qp->qplib_qp.modify_flags |=
2273	CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
2274	CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
2275	CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
2276	CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
2277	CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
2278	if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
2279	(qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
2280	(qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
2281	(qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
2282	(qp_attr->cap.max_inline_data >=
2283	dev_attr->max_inline_data)) {
2284	ibdev_err(ibdev: &rdev->ibdev,
2285	format: "Create QP failed - max exceeded");
2286	return -EINVAL;
2287	}
2288	entries = bnxt_re_init_depth(ent: qp_attr->cap.max_send_wr, uctx);
2289	qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
2290	dev_attr->max_qp_wqes + 1);
2291	qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2292	qp_attr->cap.max_send_wr;
2293	/*
2294	* Reserving one slot for Phantom WQE. Some application can
2295	* post one extra entry in this case. Allowing this to avoid
2296	* unexpected Queue full condition
2297	*/
2298	qp->qplib_qp.sq.q_full_delta -= 1;
2299	qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2300	if (qp->qplib_qp.rq.max_wqe) {
2301	entries = bnxt_re_init_depth(ent: qp_attr->cap.max_recv_wr, uctx);
2302	qp->qplib_qp.rq.max_wqe =
2303	min_t(u32, entries, dev_attr->max_qp_wqes + 1);
2304	qp->qplib_qp.rq.max_sw_wqe = qp->qplib_qp.rq.max_wqe;
2305	qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2306	qp_attr->cap.max_recv_wr;
2307	qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2308	} else {
2309	/* SRQ was used prior, just ignore the RQ caps */
2310	}
2311	}
2312	if (qp_attr_mask & IB_QP_DEST_QPN) {
2313	qp->qplib_qp.modify_flags |=
2314	CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2315	qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2316	}
2317	rc = bnxt_qplib_modify_qp(res: &rdev->qplib_res, qp: &qp->qplib_qp);
2318	if (rc) {
2319	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to modify HW QP");
2320	return rc;
2321	}
2322	if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
2323	rc = bnxt_re_modify_shadow_qp(rdev, qp1_qp: qp, qp_attr_mask);
2324	return rc;
2325	}
2326
2327	int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2328	int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2329	{
2330	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2331	struct bnxt_re_dev *rdev = qp->rdev;
2332	struct bnxt_qplib_qp *qplib_qp;
2333	int rc;
2334
2335	qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL);
2336	if (!qplib_qp)
2337	return -ENOMEM;
2338
2339	qplib_qp->id = qp->qplib_qp.id;
2340	qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
2341
2342	rc = bnxt_qplib_query_qp(res: &rdev->qplib_res, qp: qplib_qp);
2343	if (rc) {
2344	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to query HW QP");
2345	goto out;
2346	}
2347	qp_attr->qp_state = __to_ib_qp_state(state: qplib_qp->state);
2348	qp_attr->cur_qp_state = __to_ib_qp_state(state: qplib_qp->cur_qp_state);
2349	qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2350	qp_attr->qp_access_flags = __qp_access_flags_to_ib(cctx: qp->qplib_qp.cctx,
2351	qflags: qplib_qp->access);
2352	qp_attr->pkey_index = qplib_qp->pkey_index;
2353	qp_attr->qkey = qplib_qp->qkey;
2354	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2355	rdma_ah_set_grh(attr: &qp_attr->ah_attr, NULL, flow_label: qplib_qp->udp_sport,
2356	sgid_index: qplib_qp->ah.host_sgid_index,
2357	hop_limit: qplib_qp->ah.hop_limit,
2358	traffic_class: qplib_qp->ah.traffic_class);
2359	rdma_ah_set_dgid_raw(attr: &qp_attr->ah_attr, dgid: qplib_qp->ah.dgid.data);
2360	rdma_ah_set_sl(attr: &qp_attr->ah_attr, sl: qplib_qp->ah.sl);
2361	ether_addr_copy(dst: qp_attr->ah_attr.roce.dmac, src: qplib_qp->ah.dmac);
2362	qp_attr->path_mtu = __to_ib_mtu(mtu: qplib_qp->path_mtu);
2363	qp_attr->timeout = qplib_qp->timeout;
2364	qp_attr->retry_cnt = qplib_qp->retry_cnt;
2365	qp_attr->rnr_retry = qplib_qp->rnr_retry;
2366	qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2367	qp_attr->port_num = __to_ib_port_num(port_id: qplib_qp->port_id);
2368	qp_attr->rq_psn = qplib_qp->rq.psn;
2369	qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2370	qp_attr->sq_psn = qplib_qp->sq.psn;
2371	qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2372	qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2373	IB_SIGNAL_REQ_WR;
2374	qp_attr->dest_qp_num = qplib_qp->dest_qpn;
2375
2376	qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2377	qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2378	qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2379	qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2380	qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2381	qp_init_attr->cap = qp_attr->cap;
2382
2383	out:
2384	kfree(objp: qplib_qp);
2385	return rc;
2386	}
2387
2388	/* Routine for sending QP1 packets for RoCE V1 an V2
2389	*/
2390	static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
2391	const struct ib_send_wr *wr,
2392	struct bnxt_qplib_swqe *wqe,
2393	int payload_size)
2394	{
2395	struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
2396	ib_ah);
2397	struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
2398	const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr;
2399	struct bnxt_qplib_sge sge;
2400	u8 nw_type;
2401	u16 ether_type;
2402	union ib_gid dgid;
2403	bool is_eth = false;
2404	bool is_vlan = false;
2405	bool is_grh = false;
2406	bool is_udp = false;
2407	u8 ip_version = 0;
2408	u16 vlan_id = 0xFFFF;
2409	void *buf;
2410	int i, rc;
2411
2412	memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
2413
2414	rc = rdma_read_gid_l2_fields(attr: sgid_attr, vlan_id: &vlan_id, NULL);
2415	if (rc)
2416	return rc;
2417
2418	/* Get network header type for this GID */
2419	nw_type = rdma_gid_attr_network_type(attr: sgid_attr);
2420	switch (nw_type) {
2421	case RDMA_NETWORK_IPV4:
2422	nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
2423	break;
2424	case RDMA_NETWORK_IPV6:
2425	nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
2426	break;
2427	default:
2428	nw_type = BNXT_RE_ROCE_V1_PACKET;
2429	break;
2430	}
2431	memcpy(&dgid.raw, &qplib_ah->dgid, 16);
2432	is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2433	if (is_udp) {
2434	if (ipv6_addr_v4mapped(a: (struct in6_addr *)&sgid_attr->gid)) {
2435	ip_version = 4;
2436	ether_type = ETH_P_IP;
2437	} else {
2438	ip_version = 6;
2439	ether_type = ETH_P_IPV6;
2440	}
2441	is_grh = false;
2442	} else {
2443	ether_type = ETH_P_IBOE;
2444	is_grh = true;
2445	}
2446
2447	is_eth = true;
2448	is_vlan = vlan_id && (vlan_id < 0x1000);
2449
2450	ib_ud_header_init(payload_bytes: payload_size, lrh_present: !is_eth, eth_present: is_eth, vlan_present: is_vlan, grh_present: is_grh,
2451	ip_version, udp_present: is_udp, immediate_present: 0, header: &qp->qp1_hdr);
2452
2453	/* ETH */
2454	ether_addr_copy(dst: qp->qp1_hdr.eth.dmac_h, src: ah->qplib_ah.dmac);
2455	ether_addr_copy(dst: qp->qp1_hdr.eth.smac_h, src: qp->qplib_qp.smac);
2456
2457	/* For vlan, check the sgid for vlan existence */
2458
2459	if (!is_vlan) {
2460	qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
2461	} else {
2462	qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
2463	qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
2464	}
2465
2466	if (is_grh || (ip_version == 6)) {
2467	memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw,
2468	sizeof(sgid_attr->gid));
2469	memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
2470	sizeof(sgid_attr->gid));
2471	qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit;
2472	}
2473
2474	if (ip_version == 4) {
2475	qp->qp1_hdr.ip4.tos = 0;
2476	qp->qp1_hdr.ip4.id = 0;
2477	qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
2478	qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
2479
2480	memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4);
2481	memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
2482	qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(header: &qp->qp1_hdr);
2483	}
2484
2485	if (is_udp) {
2486	qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
2487	qp->qp1_hdr.udp.sport = htons(0x8CD1);
2488	qp->qp1_hdr.udp.csum = 0;
2489	}
2490
2491	/* BTH */
2492	if (wr->opcode == IB_WR_SEND_WITH_IMM) {
2493	qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2494	qp->qp1_hdr.immediate_present = 1;
2495	} else {
2496	qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2497	}
2498	if (wr->send_flags & IB_SEND_SOLICITED)
2499	qp->qp1_hdr.bth.solicited_event = 1;
2500	/* pad_count */
2501	qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
2502
2503	/* P_key for QP1 is for all members */
2504	qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
2505	qp->qp1_hdr.bth.destination_qpn = IB_QP1;
2506	qp->qp1_hdr.bth.ack_req = 0;
2507	qp->send_psn++;
2508	qp->send_psn &= BTH_PSN_MASK;
2509	qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
2510	/* DETH */
2511	/* Use the priviledged Q_Key for QP1 */
2512	qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
2513	qp->qp1_hdr.deth.source_qpn = IB_QP1;
2514
2515	/* Pack the QP1 to the transmit buffer */
2516	buf = bnxt_qplib_get_qp1_sq_buf(qp: &qp->qplib_qp, sge: &sge);
2517	if (buf) {
2518	ib_ud_header_pack(header: &qp->qp1_hdr, buf);
2519	for (i = wqe->num_sge; i; i--) {
2520	wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
2521	wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
2522	wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
2523	}
2524
2525	/*
2526	* Max Header buf size for IPV6 RoCE V2 is 86,
2527	* which is same as the QP1 SQ header buffer.
2528	* Header buf size for IPV4 RoCE V2 can be 66.
2529	* ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
2530	* Subtract 20 bytes from QP1 SQ header buf size
2531	*/
2532	if (is_udp && ip_version == 4)
2533	sge.size -= 20;
2534	/*
2535	* Max Header buf size for RoCE V1 is 78.
2536	* ETH(14) + VLAN(4) + GRH(40) + BTH(20).
2537	* Subtract 8 bytes from QP1 SQ header buf size
2538	*/
2539	if (!is_udp)
2540	sge.size -= 8;
2541
2542	/* Subtract 4 bytes for non vlan packets */
2543	if (!is_vlan)
2544	sge.size -= 4;
2545
2546	wqe->sg_list[0].addr = sge.addr;
2547	wqe->sg_list[0].lkey = sge.lkey;
2548	wqe->sg_list[0].size = sge.size;
2549	wqe->num_sge++;
2550
2551	} else {
2552	ibdev_err(ibdev: &qp->rdev->ibdev, format: "QP1 buffer is empty!");
2553	rc = -ENOMEM;
2554	}
2555	return rc;
2556	}
2557
2558	/* For the MAD layer, it only provides the recv SGE the size of
2559	* ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH,
2560	* nor RoCE iCRC. The Cu+ solution must provide buffer for the entire
2561	* receive packet (334 bytes) with no VLAN and then copy the GRH
2562	* and the MAD datagram out to the provided SGE.
2563	*/
2564	static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
2565	const struct ib_recv_wr *wr,
2566	struct bnxt_qplib_swqe *wqe,
2567	int payload_size)
2568	{
2569	struct bnxt_re_sqp_entries *sqp_entry;
2570	struct bnxt_qplib_sge ref, sge;
2571	struct bnxt_re_dev *rdev;
2572	u32 rq_prod_index;
2573
2574	rdev = qp->rdev;
2575
2576	rq_prod_index = bnxt_qplib_get_rq_prod_index(qp: &qp->qplib_qp);
2577
2578	if (!bnxt_qplib_get_qp1_rq_buf(qp: &qp->qplib_qp, sge: &sge))
2579	return -ENOMEM;
2580
2581	/* Create 1 SGE to receive the entire
2582	* ethernet packet
2583	*/
2584	/* Save the reference from ULP */
2585	ref.addr = wqe->sg_list[0].addr;
2586	ref.lkey = wqe->sg_list[0].lkey;
2587	ref.size = wqe->sg_list[0].size;
2588
2589	sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
2590
2591	/* SGE 1 */
2592	wqe->sg_list[0].addr = sge.addr;
2593	wqe->sg_list[0].lkey = sge.lkey;
2594	wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
2595	sge.size -= wqe->sg_list[0].size;
2596
2597	sqp_entry->sge.addr = ref.addr;
2598	sqp_entry->sge.lkey = ref.lkey;
2599	sqp_entry->sge.size = ref.size;
2600	/* Store the wrid for reporting completion */
2601	sqp_entry->wrid = wqe->wr_id;
2602	/* change the wqe->wrid to table index */
2603	wqe->wr_id = rq_prod_index;
2604	return 0;
2605	}
2606
2607	static int is_ud_qp(struct bnxt_re_qp *qp)
2608	{
2609	return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
2610	qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
2611	}
2612
2613	static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
2614	const struct ib_send_wr *wr,
2615	struct bnxt_qplib_swqe *wqe)
2616	{
2617	struct bnxt_re_ah *ah = NULL;
2618
2619	if (is_ud_qp(qp)) {
2620	ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
2621	wqe->send.q_key = ud_wr(wr)->remote_qkey;
2622	wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
2623	wqe->send.avid = ah->qplib_ah.id;
2624	}
2625	switch (wr->opcode) {
2626	case IB_WR_SEND:
2627	wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
2628	break;
2629	case IB_WR_SEND_WITH_IMM:
2630	wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
2631	wqe->send.imm_data = be32_to_cpu(wr->ex.imm_data);
2632	break;
2633	case IB_WR_SEND_WITH_INV:
2634	wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
2635	wqe->send.inv_key = wr->ex.invalidate_rkey;
2636	break;
2637	default:
2638	return -EINVAL;
2639	}
2640	if (wr->send_flags & IB_SEND_SIGNALED)
2641	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2642	if (wr->send_flags & IB_SEND_FENCE)
2643	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2644	if (wr->send_flags & IB_SEND_SOLICITED)
2645	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2646	if (wr->send_flags & IB_SEND_INLINE)
2647	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2648
2649	return 0;
2650	}
2651
2652	static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
2653	struct bnxt_qplib_swqe *wqe)
2654	{
2655	switch (wr->opcode) {
2656	case IB_WR_RDMA_WRITE:
2657	wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
2658	break;
2659	case IB_WR_RDMA_WRITE_WITH_IMM:
2660	wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
2661	wqe->rdma.imm_data = be32_to_cpu(wr->ex.imm_data);
2662	break;
2663	case IB_WR_RDMA_READ:
2664	wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
2665	wqe->rdma.inv_key = wr->ex.invalidate_rkey;
2666	break;
2667	default:
2668	return -EINVAL;
2669	}
2670	wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
2671	wqe->rdma.r_key = rdma_wr(wr)->rkey;
2672	if (wr->send_flags & IB_SEND_SIGNALED)
2673	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2674	if (wr->send_flags & IB_SEND_FENCE)
2675	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2676	if (wr->send_flags & IB_SEND_SOLICITED)
2677	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2678	if (wr->send_flags & IB_SEND_INLINE)
2679	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2680
2681	return 0;
2682	}
2683
2684	static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
2685	struct bnxt_qplib_swqe *wqe)
2686	{
2687	switch (wr->opcode) {
2688	case IB_WR_ATOMIC_CMP_AND_SWP:
2689	wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
2690	wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2691	wqe->atomic.swap_data = atomic_wr(wr)->swap;
2692	break;
2693	case IB_WR_ATOMIC_FETCH_AND_ADD:
2694	wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
2695	wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2696	break;
2697	default:
2698	return -EINVAL;
2699	}
2700	wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
2701	wqe->atomic.r_key = atomic_wr(wr)->rkey;
2702	if (wr->send_flags & IB_SEND_SIGNALED)
2703	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2704	if (wr->send_flags & IB_SEND_FENCE)
2705	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2706	if (wr->send_flags & IB_SEND_SOLICITED)
2707	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2708	return 0;
2709	}
2710
2711	static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
2712	struct bnxt_qplib_swqe *wqe)
2713	{
2714	wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
2715	wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
2716
2717	if (wr->send_flags & IB_SEND_SIGNALED)
2718	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2719	if (wr->send_flags & IB_SEND_SOLICITED)
2720	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2721
2722	return 0;
2723	}
2724
2725	static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
2726	struct bnxt_qplib_swqe *wqe)
2727	{
2728	struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
2729	struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
2730	int access = wr->access;
2731
2732	wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
2733	wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
2734	wqe->frmr.page_list = mr->pages;
2735	wqe->frmr.page_list_len = mr->npages;
2736	wqe->frmr.levels = qplib_frpl->hwq.level;
2737	wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
2738
2739	if (wr->wr.send_flags & IB_SEND_SIGNALED)
2740	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2741
2742	if (access & IB_ACCESS_LOCAL_WRITE)
2743	wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2744	if (access & IB_ACCESS_REMOTE_READ)
2745	wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2746	if (access & IB_ACCESS_REMOTE_WRITE)
2747	wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2748	if (access & IB_ACCESS_REMOTE_ATOMIC)
2749	wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2750	if (access & IB_ACCESS_MW_BIND)
2751	wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2752
2753	wqe->frmr.l_key = wr->key;
2754	wqe->frmr.length = wr->mr->length;
2755	wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K);
2756	wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K);
2757	wqe->frmr.va = wr->mr->iova;
2758	return 0;
2759	}
2760
2761	static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
2762	const struct ib_send_wr *wr,
2763	struct bnxt_qplib_swqe *wqe)
2764	{
2765	/* Copy the inline data to the data field */
2766	u8 *in_data;
2767	u32 i, sge_len;
2768	void *sge_addr;
2769
2770	in_data = wqe->inline_data;
2771	for (i = 0; i < wr->num_sge; i++) {
2772	sge_addr = (void *)(unsigned long)
2773	wr->sg_list[i].addr;
2774	sge_len = wr->sg_list[i].length;
2775
2776	if ((sge_len + wqe->inline_len) >
2777	BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
2778	ibdev_err(ibdev: &rdev->ibdev,
2779	format: "Inline data size requested > supported value");
2780	return -EINVAL;
2781	}
2782	sge_len = wr->sg_list[i].length;
2783
2784	memcpy(in_data, sge_addr, sge_len);
2785	in_data += wr->sg_list[i].length;
2786	wqe->inline_len += wr->sg_list[i].length;
2787	}
2788	return wqe->inline_len;
2789	}
2790
2791	static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
2792	const struct ib_send_wr *wr,
2793	struct bnxt_qplib_swqe *wqe)
2794	{
2795	int payload_sz = 0;
2796
2797	if (wr->send_flags & IB_SEND_INLINE)
2798	payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
2799	else
2800	payload_sz = bnxt_re_build_sgl(ib_sg_list: wr->sg_list, sg_list: wqe->sg_list,
2801	num: wqe->num_sge);
2802
2803	return payload_sz;
2804	}
2805
2806	static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
2807	{
2808	if ((qp->ib_qp.qp_type == IB_QPT_UD ||
2809	qp->ib_qp.qp_type == IB_QPT_GSI ||
2810	qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
2811	qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
2812	int qp_attr_mask;
2813	struct ib_qp_attr qp_attr;
2814
2815	qp_attr_mask = IB_QP_STATE;
2816	qp_attr.qp_state = IB_QPS_RTS;
2817	bnxt_re_modify_qp(ib_qp: &qp->ib_qp, qp_attr: &qp_attr, qp_attr_mask, NULL);
2818	qp->qplib_qp.wqe_cnt = 0;
2819	}
2820	}
2821
2822	static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
2823	struct bnxt_re_qp *qp,
2824	const struct ib_send_wr *wr)
2825	{
2826	int rc = 0, payload_sz = 0;
2827	unsigned long flags;
2828
2829	spin_lock_irqsave(&qp->sq_lock, flags);
2830	while (wr) {
2831	struct bnxt_qplib_swqe wqe = {};
2832
2833	/* Common */
2834	wqe.num_sge = wr->num_sge;
2835	if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2836	ibdev_err(ibdev: &rdev->ibdev,
2837	format: "Limit exceeded for Send SGEs");
2838	rc = -EINVAL;
2839	goto bad;
2840	}
2841
2842	payload_sz = bnxt_re_copy_wr_payload(rdev: qp->rdev, wr, wqe: &wqe);
2843	if (payload_sz < 0) {
2844	rc = -EINVAL;
2845	goto bad;
2846	}
2847	wqe.wr_id = wr->wr_id;
2848
2849	wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
2850
2851	rc = bnxt_re_build_send_wqe(qp, wr, wqe: &wqe);
2852	if (!rc)
2853	rc = bnxt_qplib_post_send(qp: &qp->qplib_qp, wqe: &wqe);
2854	bad:
2855	if (rc) {
2856	ibdev_err(ibdev: &rdev->ibdev,
2857	format: "Post send failed opcode = %#x rc = %d",
2858	wr->opcode, rc);
2859	break;
2860	}
2861	wr = wr->next;
2862	}
2863	bnxt_qplib_post_send_db(qp: &qp->qplib_qp);
2864	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx: qp->rdev->chip_ctx))
2865	bnxt_ud_qp_hw_stall_workaround(qp);
2866	spin_unlock_irqrestore(lock: &qp->sq_lock, flags);
2867	return rc;
2868	}
2869
2870	static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe)
2871	{
2872	/* Need unconditional fence for non-wire memory opcode
2873	* to work as expected.
2874	*/
2875	if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV ||
2876	wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR ||
2877	wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR ||
2878	wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW)
2879	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2880	}
2881
2882	int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
2883	const struct ib_send_wr **bad_wr)
2884	{
2885	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2886	struct bnxt_qplib_swqe wqe;
2887	int rc = 0, payload_sz = 0;
2888	unsigned long flags;
2889
2890	spin_lock_irqsave(&qp->sq_lock, flags);
2891	while (wr) {
2892	/* House keeping */
2893	memset(&wqe, 0, sizeof(wqe));
2894
2895	/* Common */
2896	wqe.num_sge = wr->num_sge;
2897	if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2898	ibdev_err(ibdev: &qp->rdev->ibdev,
2899	format: "Limit exceeded for Send SGEs");
2900	rc = -EINVAL;
2901	goto bad;
2902	}
2903
2904	payload_sz = bnxt_re_copy_wr_payload(rdev: qp->rdev, wr, wqe: &wqe);
2905	if (payload_sz < 0) {
2906	rc = -EINVAL;
2907	goto bad;
2908	}
2909	wqe.wr_id = wr->wr_id;
2910
2911	switch (wr->opcode) {
2912	case IB_WR_SEND:
2913	case IB_WR_SEND_WITH_IMM:
2914	if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) {
2915	rc = bnxt_re_build_qp1_send_v2(qp, wr, wqe: &wqe,
2916	payload_size: payload_sz);
2917	if (rc)
2918	goto bad;
2919	wqe.rawqp1.lflags |=
2920	SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2921	}
2922	if (wr->send_flags & IB_SEND_IP_CSUM)
2923	wqe.rawqp1.lflags |=
2924	SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2925	fallthrough;
2926	case IB_WR_SEND_WITH_INV:
2927	rc = bnxt_re_build_send_wqe(qp, wr, wqe: &wqe);
2928	break;
2929	case IB_WR_RDMA_WRITE:
2930	case IB_WR_RDMA_WRITE_WITH_IMM:
2931	case IB_WR_RDMA_READ:
2932	rc = bnxt_re_build_rdma_wqe(wr, wqe: &wqe);
2933	break;
2934	case IB_WR_ATOMIC_CMP_AND_SWP:
2935	case IB_WR_ATOMIC_FETCH_AND_ADD:
2936	rc = bnxt_re_build_atomic_wqe(wr, wqe: &wqe);
2937	break;
2938	case IB_WR_RDMA_READ_WITH_INV:
2939	ibdev_err(ibdev: &qp->rdev->ibdev,
2940	format: "RDMA Read with Invalidate is not supported");
2941	rc = -EINVAL;
2942	goto bad;
2943	case IB_WR_LOCAL_INV:
2944	rc = bnxt_re_build_inv_wqe(wr, wqe: &wqe);
2945	break;
2946	case IB_WR_REG_MR:
2947	rc = bnxt_re_build_reg_wqe(wr: reg_wr(wr), wqe: &wqe);
2948	break;
2949	default:
2950	/* Unsupported WRs */
2951	ibdev_err(ibdev: &qp->rdev->ibdev,
2952	format: "WR (%#x) is not supported", wr->opcode);
2953	rc = -EINVAL;
2954	goto bad;
2955	}
2956	if (!rc) {
2957	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx: qp->rdev->chip_ctx))
2958	bnxt_re_legacy_set_uc_fence(wqe: &wqe);
2959	rc = bnxt_qplib_post_send(qp: &qp->qplib_qp, wqe: &wqe);
2960	}
2961	bad:
2962	if (rc) {
2963	ibdev_err(ibdev: &qp->rdev->ibdev,
2964	format: "post_send failed op:%#x qps = %#x rc = %d\n",
2965	wr->opcode, qp->qplib_qp.state, rc);
2966	*bad_wr = wr;
2967	break;
2968	}
2969	wr = wr->next;
2970	}
2971	bnxt_qplib_post_send_db(qp: &qp->qplib_qp);
2972	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx: qp->rdev->chip_ctx))
2973	bnxt_ud_qp_hw_stall_workaround(qp);
2974	spin_unlock_irqrestore(lock: &qp->sq_lock, flags);
2975
2976	return rc;
2977	}
2978
2979	static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2980	struct bnxt_re_qp *qp,
2981	const struct ib_recv_wr *wr)
2982	{
2983	struct bnxt_qplib_swqe wqe;
2984	int rc = 0;
2985
2986	while (wr) {
2987	/* House keeping */
2988	memset(&wqe, 0, sizeof(wqe));
2989
2990	/* Common */
2991	wqe.num_sge = wr->num_sge;
2992	if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2993	ibdev_err(ibdev: &rdev->ibdev,
2994	format: "Limit exceeded for Receive SGEs");
2995	rc = -EINVAL;
2996	break;
2997	}
2998	bnxt_re_build_sgl(ib_sg_list: wr->sg_list, sg_list: wqe.sg_list, num: wr->num_sge);
2999	wqe.wr_id = wr->wr_id;
3000	wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
3001
3002	rc = bnxt_qplib_post_recv(qp: &qp->qplib_qp, wqe: &wqe);
3003	if (rc)
3004	break;
3005
3006	wr = wr->next;
3007	}
3008	if (!rc)
3009	bnxt_qplib_post_recv_db(qp: &qp->qplib_qp);
3010	return rc;
3011	}
3012
3013	int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
3014	const struct ib_recv_wr **bad_wr)
3015	{
3016	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
3017	struct bnxt_qplib_swqe wqe;
3018	int rc = 0, payload_sz = 0;
3019	unsigned long flags;
3020	u32 count = 0;
3021
3022	spin_lock_irqsave(&qp->rq_lock, flags);
3023	while (wr) {
3024	/* House keeping */
3025	memset(&wqe, 0, sizeof(wqe));
3026
3027	/* Common */
3028	wqe.num_sge = wr->num_sge;
3029	if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
3030	ibdev_err(ibdev: &qp->rdev->ibdev,
3031	format: "Limit exceeded for Receive SGEs");
3032	rc = -EINVAL;
3033	*bad_wr = wr;
3034	break;
3035	}
3036
3037	payload_sz = bnxt_re_build_sgl(ib_sg_list: wr->sg_list, sg_list: wqe.sg_list,
3038	num: wr->num_sge);
3039	wqe.wr_id = wr->wr_id;
3040	wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
3041
3042	if (ib_qp->qp_type == IB_QPT_GSI &&
3043	qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI)
3044	rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, wqe: &wqe,
3045	payload_size: payload_sz);
3046	if (!rc)
3047	rc = bnxt_qplib_post_recv(qp: &qp->qplib_qp, wqe: &wqe);
3048	if (rc) {
3049	*bad_wr = wr;
3050	break;
3051	}
3052
3053	/* Ring DB if the RQEs posted reaches a threshold value */
3054	if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
3055	bnxt_qplib_post_recv_db(qp: &qp->qplib_qp);
3056	count = 0;
3057	}
3058
3059	wr = wr->next;
3060	}
3061
3062	if (count)
3063	bnxt_qplib_post_recv_db(qp: &qp->qplib_qp);
3064
3065	spin_unlock_irqrestore(lock: &qp->rq_lock, flags);
3066
3067	return rc;
3068	}
3069
3070	static struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev)
3071	{
3072	int min, indx;
3073
3074	mutex_lock(&rdev->nqr->load_lock);
3075	for (indx = 0, min = 0; indx < (rdev->nqr->num_msix - 1); indx++) {
3076	if (rdev->nqr->nq[min].load > rdev->nqr->nq[indx].load)
3077	min = indx;
3078	}
3079	rdev->nqr->nq[min].load++;
3080	mutex_unlock(lock: &rdev->nqr->load_lock);
3081
3082	return &rdev->nqr->nq[min];
3083	}
3084
3085	static void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq)
3086	{
3087	mutex_lock(&rdev->nqr->load_lock);
3088	nq->load--;
3089	mutex_unlock(lock: &rdev->nqr->load_lock);
3090	}
3091
3092	/* Completion Queues */
3093	int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
3094	{
3095	struct bnxt_qplib_chip_ctx *cctx;
3096	struct bnxt_qplib_nq *nq;
3097	struct bnxt_re_dev *rdev;
3098	struct bnxt_re_cq *cq;
3099
3100	cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3101	rdev = cq->rdev;
3102	nq = cq->qplib_cq.nq;
3103	cctx = rdev->chip_ctx;
3104
3105	if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
3106	free_page((unsigned long)cq->uctx_cq_page);
3107	hash_del(node: &cq->hash_entry);
3108	}
3109	bnxt_qplib_destroy_cq(res: &rdev->qplib_res, cq: &cq->qplib_cq);
3110
3111	bnxt_re_put_nq(rdev, nq);
3112	ib_umem_release(umem: cq->umem);
3113
3114	atomic_dec(v: &rdev->stats.res.cq_count);
3115	kfree(objp: cq->cql);
3116	return 0;
3117	}
3118
3119	int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
3120	struct uverbs_attr_bundle *attrs)
3121	{
3122	struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
3123	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
3124	struct ib_udata *udata = &attrs->driver_udata;
3125	struct bnxt_re_ucontext *uctx =
3126	rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
3127	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
3128	struct bnxt_qplib_chip_ctx *cctx;
3129	int cqe = attr->cqe;
3130	int rc, entries;
3131	u32 active_cqs;
3132
3133	if (attr->flags)
3134	return -EOPNOTSUPP;
3135
3136	/* Validate CQ fields */
3137	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
3138	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to create CQ -max exceeded");
3139	return -EINVAL;
3140	}
3141
3142	cq->rdev = rdev;
3143	cctx = rdev->chip_ctx;
3144	cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
3145
3146	entries = bnxt_re_init_depth(ent: cqe + 1, uctx);
3147	if (entries > dev_attr->max_cq_wqes + 1)
3148	entries = dev_attr->max_cq_wqes + 1;
3149
3150	cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
3151	cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
3152	if (udata) {
3153	struct bnxt_re_cq_req req;
3154	if (ib_copy_from_udata(dest: &req, udata, len: sizeof(req))) {
3155	rc = -EFAULT;
3156	goto fail;
3157	}
3158
3159	cq->umem = ib_umem_get(device: &rdev->ibdev, addr: req.cq_va,
3160	size: entries * sizeof(struct cq_base),
3161	access: IB_ACCESS_LOCAL_WRITE);
3162	if (IS_ERR(ptr: cq->umem)) {
3163	rc = PTR_ERR(ptr: cq->umem);
3164	goto fail;
3165	}
3166	cq->qplib_cq.sg_info.umem = cq->umem;
3167	cq->qplib_cq.dpi = &uctx->dpi;
3168	} else {
3169	cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
3170	cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
3171	GFP_KERNEL);
3172	if (!cq->cql) {
3173	rc = -ENOMEM;
3174	goto fail;
3175	}
3176
3177	cq->qplib_cq.dpi = &rdev->dpi_privileged;
3178	}
3179	cq->qplib_cq.max_wqe = entries;
3180	cq->qplib_cq.coalescing = &rdev->cq_coalescing;
3181	cq->qplib_cq.nq = bnxt_re_get_nq(rdev);
3182	cq->qplib_cq.cnq_hw_ring_id = cq->qplib_cq.nq->ring_id;
3183
3184	rc = bnxt_qplib_create_cq(res: &rdev->qplib_res, cq: &cq->qplib_cq);
3185	if (rc) {
3186	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to create HW CQ");
3187	goto fail;
3188	}
3189
3190	cq->ib_cq.cqe = entries;
3191	cq->cq_period = cq->qplib_cq.period;
3192
3193	active_cqs = atomic_inc_return(v: &rdev->stats.res.cq_count);
3194	if (active_cqs > rdev->stats.res.cq_watermark)
3195	rdev->stats.res.cq_watermark = active_cqs;
3196	spin_lock_init(&cq->cq_lock);
3197
3198	if (udata) {
3199	struct bnxt_re_cq_resp resp = {};
3200
3201	if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
3202	hash_add(rdev->cq_hash, &cq->hash_entry, cq->qplib_cq.id);
3203	/* Allocate a page */
3204	cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL);
3205	if (!cq->uctx_cq_page) {
3206	rc = -ENOMEM;
3207	goto c2fail;
3208	}
3209	resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT;
3210	}
3211	resp.cqid = cq->qplib_cq.id;
3212	resp.tail = cq->qplib_cq.hwq.cons;
3213	resp.phase = cq->qplib_cq.period;
3214	resp.rsvd = 0;
3215	rc = ib_copy_to_udata(udata, src: &resp, min(sizeof(resp), udata->outlen));
3216	if (rc) {
3217	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to copy CQ udata");
3218	bnxt_qplib_destroy_cq(res: &rdev->qplib_res, cq: &cq->qplib_cq);
3219	goto free_mem;
3220	}
3221	}
3222
3223	return 0;
3224
3225	free_mem:
3226	free_page((unsigned long)cq->uctx_cq_page);
3227	c2fail:
3228	ib_umem_release(umem: cq->umem);
3229	fail:
3230	kfree(objp: cq->cql);
3231	return rc;
3232	}
3233
3234	static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
3235	{
3236	struct bnxt_re_dev *rdev = cq->rdev;
3237
3238	bnxt_qplib_resize_cq_complete(res: &rdev->qplib_res, cq: &cq->qplib_cq);
3239
3240	cq->qplib_cq.max_wqe = cq->resize_cqe;
3241	if (cq->resize_umem) {
3242	ib_umem_release(umem: cq->umem);
3243	cq->umem = cq->resize_umem;
3244	cq->resize_umem = NULL;
3245	cq->resize_cqe = 0;
3246	}
3247	}
3248
3249	int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
3250	{
3251	struct bnxt_qplib_sg_info sg_info = {};
3252	struct bnxt_qplib_dpi *orig_dpi = NULL;
3253	struct bnxt_qplib_dev_attr *dev_attr;
3254	struct bnxt_re_ucontext *uctx = NULL;
3255	struct bnxt_re_resize_cq_req req;
3256	struct bnxt_re_dev *rdev;
3257	struct bnxt_re_cq *cq;
3258	int rc, entries;
3259
3260	cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
3261	rdev = cq->rdev;
3262	dev_attr = rdev->dev_attr;
3263	if (!ibcq->uobject) {
3264	ibdev_err(ibdev: &rdev->ibdev, format: "Kernel CQ Resize not supported");
3265	return -EOPNOTSUPP;
3266	}
3267
3268	if (cq->resize_umem) {
3269	ibdev_err(ibdev: &rdev->ibdev, format: "Resize CQ %#x failed - Busy",
3270	cq->qplib_cq.id);
3271	return -EBUSY;
3272	}
3273
3274	/* Check the requested cq depth out of supported depth */
3275	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
3276	ibdev_err(ibdev: &rdev->ibdev, format: "Resize CQ %#x failed - out of range cqe %d",
3277	cq->qplib_cq.id, cqe);
3278	return -EINVAL;
3279	}
3280
3281	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
3282	entries = bnxt_re_init_depth(ent: cqe + 1, uctx);
3283	if (entries > dev_attr->max_cq_wqes + 1)
3284	entries = dev_attr->max_cq_wqes + 1;
3285
3286	/* uverbs consumer */
3287	if (ib_copy_from_udata(dest: &req, udata, len: sizeof(req))) {
3288	rc = -EFAULT;
3289	goto fail;
3290	}
3291
3292	cq->resize_umem = ib_umem_get(device: &rdev->ibdev, addr: req.cq_va,
3293	size: entries * sizeof(struct cq_base),
3294	access: IB_ACCESS_LOCAL_WRITE);
3295	if (IS_ERR(ptr: cq->resize_umem)) {
3296	rc = PTR_ERR(ptr: cq->resize_umem);
3297	ibdev_err(ibdev: &rdev->ibdev, format: "%s: ib_umem_get failed! rc = %pe\n",
3298	__func__, cq->resize_umem);
3299	cq->resize_umem = NULL;
3300	goto fail;
3301	}
3302	cq->resize_cqe = entries;
3303	memcpy(&sg_info, &cq->qplib_cq.sg_info, sizeof(sg_info));
3304	orig_dpi = cq->qplib_cq.dpi;
3305
3306	cq->qplib_cq.sg_info.umem = cq->resize_umem;
3307	cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
3308	cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
3309	cq->qplib_cq.dpi = &uctx->dpi;
3310
3311	rc = bnxt_qplib_resize_cq(res: &rdev->qplib_res, cq: &cq->qplib_cq, new_cqes: entries);
3312	if (rc) {
3313	ibdev_err(ibdev: &rdev->ibdev, format: "Resize HW CQ %#x failed!",
3314	cq->qplib_cq.id);
3315	goto fail;
3316	}
3317
3318	cq->ib_cq.cqe = cq->resize_cqe;
3319	atomic_inc(v: &rdev->stats.res.resize_count);
3320
3321	return 0;
3322
3323	fail:
3324	if (cq->resize_umem) {
3325	ib_umem_release(umem: cq->resize_umem);
3326	cq->resize_umem = NULL;
3327	cq->resize_cqe = 0;
3328	memcpy(&cq->qplib_cq.sg_info, &sg_info, sizeof(sg_info));
3329	cq->qplib_cq.dpi = orig_dpi;
3330	}
3331	return rc;
3332	}
3333
3334	static u8 __req_to_ib_wc_status(u8 qstatus)
3335	{
3336	switch (qstatus) {
3337	case CQ_REQ_STATUS_OK:
3338	return IB_WC_SUCCESS;
3339	case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
3340	return IB_WC_BAD_RESP_ERR;
3341	case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
3342	return IB_WC_LOC_LEN_ERR;
3343	case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
3344	return IB_WC_LOC_QP_OP_ERR;
3345	case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
3346	return IB_WC_LOC_PROT_ERR;
3347	case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
3348	return IB_WC_GENERAL_ERR;
3349	case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
3350	return IB_WC_REM_INV_REQ_ERR;
3351	case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
3352	return IB_WC_REM_ACCESS_ERR;
3353	case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
3354	return IB_WC_REM_OP_ERR;
3355	case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
3356	return IB_WC_RNR_RETRY_EXC_ERR;
3357	case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
3358	return IB_WC_RETRY_EXC_ERR;
3359	case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
3360	return IB_WC_WR_FLUSH_ERR;
3361	default:
3362	return IB_WC_GENERAL_ERR;
3363	}
3364	return 0;
3365	}
3366
3367	static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
3368	{
3369	switch (qstatus) {
3370	case CQ_RES_RAWETH_QP1_STATUS_OK:
3371	return IB_WC_SUCCESS;
3372	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
3373	return IB_WC_LOC_ACCESS_ERR;
3374	case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
3375	return IB_WC_LOC_LEN_ERR;
3376	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
3377	return IB_WC_LOC_PROT_ERR;
3378	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
3379	return IB_WC_LOC_QP_OP_ERR;
3380	case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
3381	return IB_WC_GENERAL_ERR;
3382	case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
3383	return IB_WC_WR_FLUSH_ERR;
3384	case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
3385	return IB_WC_WR_FLUSH_ERR;
3386	default:
3387	return IB_WC_GENERAL_ERR;
3388	}
3389	}
3390
3391	static u8 __rc_to_ib_wc_status(u8 qstatus)
3392	{
3393	switch (qstatus) {
3394	case CQ_RES_RC_STATUS_OK:
3395	return IB_WC_SUCCESS;
3396	case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
3397	return IB_WC_LOC_ACCESS_ERR;
3398	case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
3399	return IB_WC_LOC_LEN_ERR;
3400	case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
3401	return IB_WC_LOC_PROT_ERR;
3402	case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
3403	return IB_WC_LOC_QP_OP_ERR;
3404	case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
3405	return IB_WC_GENERAL_ERR;
3406	case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
3407	return IB_WC_REM_INV_REQ_ERR;
3408	case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
3409	return IB_WC_WR_FLUSH_ERR;
3410	case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
3411	return IB_WC_WR_FLUSH_ERR;
3412	default:
3413	return IB_WC_GENERAL_ERR;
3414	}
3415	}
3416
3417	static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
3418	{
3419	switch (cqe->type) {
3420	case BNXT_QPLIB_SWQE_TYPE_SEND:
3421	wc->opcode = IB_WC_SEND;
3422	break;
3423	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
3424	wc->opcode = IB_WC_SEND;
3425	wc->wc_flags |= IB_WC_WITH_IMM;
3426	break;
3427	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
3428	wc->opcode = IB_WC_SEND;
3429	wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3430	break;
3431	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
3432	wc->opcode = IB_WC_RDMA_WRITE;
3433	break;
3434	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
3435	wc->opcode = IB_WC_RDMA_WRITE;
3436	wc->wc_flags |= IB_WC_WITH_IMM;
3437	break;
3438	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
3439	wc->opcode = IB_WC_RDMA_READ;
3440	break;
3441	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
3442	wc->opcode = IB_WC_COMP_SWAP;
3443	break;
3444	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
3445	wc->opcode = IB_WC_FETCH_ADD;
3446	break;
3447	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
3448	wc->opcode = IB_WC_LOCAL_INV;
3449	break;
3450	case BNXT_QPLIB_SWQE_TYPE_REG_MR:
3451	wc->opcode = IB_WC_REG_MR;
3452	break;
3453	default:
3454	wc->opcode = IB_WC_SEND;
3455	break;
3456	}
3457
3458	wc->status = __req_to_ib_wc_status(qstatus: cqe->status);
3459	}
3460
3461	static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
3462	u16 raweth_qp1_flags2)
3463	{
3464	bool is_ipv6 = false, is_ipv4 = false;
3465
3466	/* raweth_qp1_flags Bit 9-6 indicates itype */
3467	if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3468	!= CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3469	return -1;
3470
3471	if (raweth_qp1_flags2 &
3472	CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
3473	raweth_qp1_flags2 &
3474	CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
3475	/* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
3476	(raweth_qp1_flags2 &
3477	CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
3478	(is_ipv6 = true) : (is_ipv4 = true);
3479	return ((is_ipv6) ?
3480	BNXT_RE_ROCEV2_IPV6_PACKET :
3481	BNXT_RE_ROCEV2_IPV4_PACKET);
3482	} else {
3483	return BNXT_RE_ROCE_V1_PACKET;
3484	}
3485	}
3486
3487	static int bnxt_re_to_ib_nw_type(int nw_type)
3488	{
3489	u8 nw_hdr_type = 0xFF;
3490
3491	switch (nw_type) {
3492	case BNXT_RE_ROCE_V1_PACKET:
3493	nw_hdr_type = RDMA_NETWORK_ROCE_V1;
3494	break;
3495	case BNXT_RE_ROCEV2_IPV4_PACKET:
3496	nw_hdr_type = RDMA_NETWORK_IPV4;
3497	break;
3498	case BNXT_RE_ROCEV2_IPV6_PACKET:
3499	nw_hdr_type = RDMA_NETWORK_IPV6;
3500	break;
3501	}
3502	return nw_hdr_type;
3503	}
3504
3505	static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
3506	void *rq_hdr_buf)
3507	{
3508	u8 *tmp_buf = NULL;
3509	struct ethhdr *eth_hdr;
3510	u16 eth_type;
3511	bool rc = false;
3512
3513	tmp_buf = (u8 *)rq_hdr_buf;
3514	/*
3515	* If dest mac is not same as I/F mac, this could be a
3516	* loopback address or multicast address, check whether
3517	* it is a loopback packet
3518	*/
3519	if (!ether_addr_equal(addr1: tmp_buf, addr2: rdev->netdev->dev_addr)) {
3520	tmp_buf += 4;
3521	/* Check the ether type */
3522	eth_hdr = (struct ethhdr *)tmp_buf;
3523	eth_type = ntohs(eth_hdr->h_proto);
3524	switch (eth_type) {
3525	case ETH_P_IBOE:
3526	rc = true;
3527	break;
3528	case ETH_P_IP:
3529	case ETH_P_IPV6: {
3530	u32 len;
3531	struct udphdr *udp_hdr;
3532
3533	len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
3534	sizeof(struct ipv6hdr));
3535	tmp_buf += sizeof(struct ethhdr) + len;
3536	udp_hdr = (struct udphdr *)tmp_buf;
3537	if (ntohs(udp_hdr->dest) ==
3538	ROCE_V2_UDP_DPORT)
3539	rc = true;
3540	break;
3541	}
3542	default:
3543	break;
3544	}
3545	}
3546
3547	return rc;
3548	}
3549
3550	static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp,
3551	struct bnxt_qplib_cqe *cqe)
3552	{
3553	struct bnxt_re_dev *rdev = gsi_qp->rdev;
3554	struct bnxt_re_sqp_entries *sqp_entry = NULL;
3555	struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp;
3556	dma_addr_t shrq_hdr_buf_map;
3557	struct ib_sge s_sge[2] = {};
3558	struct ib_sge r_sge[2] = {};
3559	struct bnxt_re_ah *gsi_sah;
3560	struct ib_recv_wr rwr = {};
3561	dma_addr_t rq_hdr_buf_map;
3562	struct ib_ud_wr udwr = {};
3563	struct ib_send_wr *swr;
3564	u32 skip_bytes = 0;
3565	int pkt_type = 0;
3566	void *rq_hdr_buf;
3567	u32 offset = 0;
3568	u32 tbl_idx;
3569	int rc;
3570
3571	swr = &udwr.wr;
3572	tbl_idx = cqe->wr_id;
3573
3574	rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf +
3575	(tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size);
3576	rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(qp: &gsi_qp->qplib_qp,
3577	index: tbl_idx);
3578
3579	/* Shadow QP header buffer */
3580	shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(qp: &gsi_qp->qplib_qp,
3581	index: tbl_idx);
3582	sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3583
3584	/* Store this cqe */
3585	memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
3586	sqp_entry->qp1_qp = gsi_qp;
3587
3588	/* Find packet type from the cqe */
3589
3590	pkt_type = bnxt_re_check_packet_type(raweth_qp1_flags: cqe->raweth_qp1_flags,
3591	raweth_qp1_flags2: cqe->raweth_qp1_flags2);
3592	if (pkt_type < 0) {
3593	ibdev_err(ibdev: &rdev->ibdev, format: "Invalid packet\n");
3594	return -EINVAL;
3595	}
3596
3597	/* Adjust the offset for the user buffer and post in the rq */
3598
3599	if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
3600	offset = 20;
3601
3602	/*
3603	* QP1 loopback packet has 4 bytes of internal header before
3604	* ether header. Skip these four bytes.
3605	*/
3606	if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
3607	skip_bytes = 4;
3608
3609	/* First send SGE . Skip the ether header*/
3610	s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
3611	+ skip_bytes;
3612	s_sge[0].lkey = 0xFFFFFFFF;
3613	s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
3614	BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
3615
3616	/* Second Send SGE */
3617	s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
3618	BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
3619	if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
3620	s_sge[1].addr += 8;
3621	s_sge[1].lkey = 0xFFFFFFFF;
3622	s_sge[1].length = 256;
3623
3624	/* First recv SGE */
3625
3626	r_sge[0].addr = shrq_hdr_buf_map;
3627	r_sge[0].lkey = 0xFFFFFFFF;
3628	r_sge[0].length = 40;
3629
3630	r_sge[1].addr = sqp_entry->sge.addr + offset;
3631	r_sge[1].lkey = sqp_entry->sge.lkey;
3632	r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
3633
3634	/* Create receive work request */
3635	rwr.num_sge = 2;
3636	rwr.sg_list = r_sge;
3637	rwr.wr_id = tbl_idx;
3638	rwr.next = NULL;
3639
3640	rc = bnxt_re_post_recv_shadow_qp(rdev, qp: gsi_sqp, wr: &rwr);
3641	if (rc) {
3642	ibdev_err(ibdev: &rdev->ibdev,
3643	format: "Failed to post Rx buffers to shadow QP");
3644	return -ENOMEM;
3645	}
3646
3647	swr->num_sge = 2;
3648	swr->sg_list = s_sge;
3649	swr->wr_id = tbl_idx;
3650	swr->opcode = IB_WR_SEND;
3651	swr->next = NULL;
3652	gsi_sah = rdev->gsi_ctx.gsi_sah;
3653	udwr.ah = &gsi_sah->ib_ah;
3654	udwr.remote_qpn = gsi_sqp->qplib_qp.id;
3655	udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
3656
3657	/* post data received in the send queue */
3658	return bnxt_re_post_send_shadow_qp(rdev, qp: gsi_sqp, wr: swr);
3659	}
3660
3661	static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
3662	struct bnxt_qplib_cqe *cqe)
3663	{
3664	wc->opcode = IB_WC_RECV;
3665	wc->status = __rawqp1_to_ib_wc_status(qstatus: cqe->status);
3666	wc->wc_flags |= IB_WC_GRH;
3667	}
3668
3669	static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
3670	u16 vlan_id)
3671	{
3672	/*
3673	* Check if the vlan is configured in the host. If not configured, it
3674	* can be a transparent VLAN. So dont report the vlan id.
3675	*/
3676	if (!__vlan_find_dev_deep_rcu(real_dev: rdev->netdev,
3677	htons(ETH_P_8021Q), vlan_id))
3678	return false;
3679	return true;
3680	}
3681
3682	static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe,
3683	u16 *vid, u8 *sl)
3684	{
3685	bool ret = false;
3686	u32 metadata;
3687	u16 tpid;
3688
3689	metadata = orig_cqe->raweth_qp1_metadata;
3690	if (orig_cqe->raweth_qp1_flags2 &
3691	CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
3692	tpid = ((metadata &
3693	CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
3694	CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
3695	if (tpid == ETH_P_8021Q) {
3696	*vid = metadata &
3697	CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
3698	*sl = (metadata &
3699	CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
3700	CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
3701	ret = true;
3702	}
3703	}
3704
3705	return ret;
3706	}
3707
3708	static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
3709	struct bnxt_qplib_cqe *cqe)
3710	{
3711	wc->opcode = IB_WC_RECV;
3712	wc->status = __rc_to_ib_wc_status(qstatus: cqe->status);
3713
3714	if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3715	wc->wc_flags |= IB_WC_WITH_IMM;
3716	if (cqe->flags & CQ_RES_RC_FLAGS_INV)
3717	wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3718	if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
3719	(CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
3720	wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3721	}
3722
3723	static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
3724	struct ib_wc *wc,
3725	struct bnxt_qplib_cqe *cqe)
3726	{
3727	struct bnxt_re_dev *rdev = gsi_sqp->rdev;
3728	struct bnxt_re_qp *gsi_qp = NULL;
3729	struct bnxt_qplib_cqe *orig_cqe = NULL;
3730	struct bnxt_re_sqp_entries *sqp_entry = NULL;
3731	int nw_type;
3732	u32 tbl_idx;
3733	u16 vlan_id;
3734	u8 sl;
3735
3736	tbl_idx = cqe->wr_id;
3737
3738	sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3739	gsi_qp = sqp_entry->qp1_qp;
3740	orig_cqe = &sqp_entry->cqe;
3741
3742	wc->wr_id = sqp_entry->wrid;
3743	wc->byte_len = orig_cqe->length;
3744	wc->qp = &gsi_qp->ib_qp;
3745
3746	wc->ex.imm_data = cpu_to_be32(orig_cqe->immdata);
3747	wc->src_qp = orig_cqe->src_qp;
3748	memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
3749	if (bnxt_re_is_vlan_pkt(orig_cqe, vid: &vlan_id, sl: &sl)) {
3750	if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3751	wc->vlan_id = vlan_id;
3752	wc->sl = sl;
3753	wc->wc_flags |= IB_WC_WITH_VLAN;
3754	}
3755	}
3756	wc->port_num = 1;
3757	wc->vendor_err = orig_cqe->status;
3758
3759	wc->opcode = IB_WC_RECV;
3760	wc->status = __rawqp1_to_ib_wc_status(qstatus: orig_cqe->status);
3761	wc->wc_flags |= IB_WC_GRH;
3762
3763	nw_type = bnxt_re_check_packet_type(raweth_qp1_flags: orig_cqe->raweth_qp1_flags,
3764	raweth_qp1_flags2: orig_cqe->raweth_qp1_flags2);
3765	if (nw_type >= 0) {
3766	wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3767	wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3768	}
3769	}
3770
3771	static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
3772	struct ib_wc *wc,
3773	struct bnxt_qplib_cqe *cqe)
3774	{
3775	struct bnxt_re_dev *rdev;
3776	u16 vlan_id = 0;
3777	u8 nw_type;
3778
3779	rdev = qp->rdev;
3780	wc->opcode = IB_WC_RECV;
3781	wc->status = __rc_to_ib_wc_status(qstatus: cqe->status);
3782
3783	if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
3784	wc->wc_flags |= IB_WC_WITH_IMM;
3785	/* report only on GSI QP for Thor */
3786	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) {
3787	wc->wc_flags |= IB_WC_GRH;
3788	memcpy(wc->smac, cqe->smac, ETH_ALEN);
3789	wc->wc_flags |= IB_WC_WITH_SMAC;
3790	if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
3791	vlan_id = (cqe->cfa_meta & 0xFFF);
3792	}
3793	/* Mark only if vlan_id is non zero */
3794	if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3795	wc->vlan_id = vlan_id;
3796	wc->wc_flags |= IB_WC_WITH_VLAN;
3797	}
3798	nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >>
3799	CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;
3800	wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3801	wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3802	}
3803
3804	}
3805
3806	static int send_phantom_wqe(struct bnxt_re_qp *qp)
3807	{
3808	struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
3809	unsigned long flags;
3810	int rc;
3811
3812	spin_lock_irqsave(&qp->sq_lock, flags);
3813
3814	rc = bnxt_re_bind_fence_mw(qplib_qp: lib_qp);
3815	if (!rc) {
3816	lib_qp->sq.phantom_wqe_cnt++;
3817	ibdev_dbg(&qp->rdev->ibdev,
3818	"qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
3819	lib_qp->id, lib_qp->sq.hwq.prod,
3820	HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
3821	lib_qp->sq.phantom_wqe_cnt);
3822	}
3823
3824	spin_unlock_irqrestore(lock: &qp->sq_lock, flags);
3825	return rc;
3826	}
3827
3828	int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
3829	{
3830	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3831	struct bnxt_re_qp *qp, *sh_qp;
3832	struct bnxt_qplib_cqe *cqe;
3833	int i, ncqe, budget;
3834	struct bnxt_qplib_q *sq;
3835	struct bnxt_qplib_qp *lib_qp;
3836	u32 tbl_idx;
3837	struct bnxt_re_sqp_entries *sqp_entry = NULL;
3838	unsigned long flags;
3839
3840	/* User CQ; the only processing we do is to
3841	* complete any pending CQ resize operation.
3842	*/
3843	if (cq->umem) {
3844	if (cq->resize_umem)
3845	bnxt_re_resize_cq_complete(cq);
3846	return 0;
3847	}
3848
3849	spin_lock_irqsave(&cq->cq_lock, flags);
3850	budget = min_t(u32, num_entries, cq->max_cql);
3851	num_entries = budget;
3852	if (!cq->cql) {
3853	ibdev_err(ibdev: &cq->rdev->ibdev, format: "POLL CQ : no CQL to use");
3854	goto exit;
3855	}
3856	cqe = &cq->cql[0];
3857	while (budget) {
3858	lib_qp = NULL;
3859	ncqe = bnxt_qplib_poll_cq(cq: &cq->qplib_cq, cqe, num: budget, qp: &lib_qp);
3860	if (lib_qp) {
3861	sq = &lib_qp->sq;
3862	if (sq->send_phantom) {
3863	qp = container_of(lib_qp,
3864	struct bnxt_re_qp, qplib_qp);
3865	if (send_phantom_wqe(qp) == -ENOMEM)
3866	ibdev_err(ibdev: &cq->rdev->ibdev,
3867	format: "Phantom failed! Scheduled to send again\n");
3868	else
3869	sq->send_phantom = false;
3870	}
3871	}
3872	if (ncqe < budget)
3873	ncqe += bnxt_qplib_process_flush_list(cq: &cq->qplib_cq,
3874	cqe: cqe + ncqe,
3875	num_cqes: budget - ncqe);
3876
3877	if (!ncqe)
3878	break;
3879
3880	for (i = 0; i < ncqe; i++, cqe++) {
3881	/* Transcribe each qplib_wqe back to ib_wc */
3882	memset(wc, 0, sizeof(*wc));
3883
3884	wc->wr_id = cqe->wr_id;
3885	wc->byte_len = cqe->length;
3886	qp = container_of
3887	((struct bnxt_qplib_qp *)
3888	(unsigned long)(cqe->qp_handle),
3889	struct bnxt_re_qp, qplib_qp);
3890	wc->qp = &qp->ib_qp;
3891	if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3892	wc->ex.imm_data = cpu_to_be32(cqe->immdata);
3893	else
3894	wc->ex.invalidate_rkey = cqe->invrkey;
3895	wc->src_qp = cqe->src_qp;
3896	memcpy(wc->smac, cqe->smac, ETH_ALEN);
3897	wc->port_num = 1;
3898	wc->vendor_err = cqe->status;
3899
3900	switch (cqe->opcode) {
3901	case CQ_BASE_CQE_TYPE_REQ:
3902	sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3903	if (sh_qp &&
3904	qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3905	/* Handle this completion with
3906	* the stored completion
3907	*/
3908	memset(wc, 0, sizeof(*wc));
3909	continue;
3910	}
3911	bnxt_re_process_req_wc(wc, cqe);
3912	break;
3913	case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
3914	if (!cqe->status) {
3915	int rc = 0;
3916
3917	rc = bnxt_re_process_raw_qp_pkt_rx
3918	(gsi_qp: qp, cqe);
3919	if (!rc) {
3920	memset(wc, 0, sizeof(*wc));
3921	continue;
3922	}
3923	cqe->status = -1;
3924	}
3925	/* Errors need not be looped back.
3926	* But change the wr_id to the one
3927	* stored in the table
3928	*/
3929	tbl_idx = cqe->wr_id;
3930	sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx];
3931	wc->wr_id = sqp_entry->wrid;
3932	bnxt_re_process_res_rawqp1_wc(wc, cqe);
3933	break;
3934	case CQ_BASE_CQE_TYPE_RES_RC:
3935	bnxt_re_process_res_rc_wc(wc, cqe);
3936	break;
3937	case CQ_BASE_CQE_TYPE_RES_UD:
3938	sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3939	if (sh_qp &&
3940	qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3941	/* Handle this completion with
3942	* the stored completion
3943	*/
3944	if (cqe->status) {
3945	continue;
3946	} else {
3947	bnxt_re_process_res_shadow_qp_wc
3948	(gsi_sqp: qp, wc, cqe);
3949	break;
3950	}
3951	}
3952	bnxt_re_process_res_ud_wc(qp, wc, cqe);
3953	break;
3954	default:
3955	ibdev_err(ibdev: &cq->rdev->ibdev,
3956	format: "POLL CQ : type 0x%x not handled",
3957	cqe->opcode);
3958	continue;
3959	}
3960	wc++;
3961	budget--;
3962	}
3963	}
3964	exit:
3965	spin_unlock_irqrestore(lock: &cq->cq_lock, flags);
3966	return num_entries - budget;
3967	}
3968
3969	int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
3970	enum ib_cq_notify_flags ib_cqn_flags)
3971	{
3972	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3973	int type = 0, rc = 0;
3974	unsigned long flags;
3975
3976	spin_lock_irqsave(&cq->cq_lock, flags);
3977	/* Trigger on the very next completion */
3978	if (ib_cqn_flags & IB_CQ_NEXT_COMP)
3979	type = DBC_DBC_TYPE_CQ_ARMALL;
3980	/* Trigger on the next solicited completion */
3981	else if (ib_cqn_flags & IB_CQ_SOLICITED)
3982	type = DBC_DBC_TYPE_CQ_ARMSE;
3983
3984	/* Poll to see if there are missed events */
3985	if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
3986	!(bnxt_qplib_is_cq_empty(cq: &cq->qplib_cq))) {
3987	rc = 1;
3988	goto exit;
3989	}
3990	bnxt_qplib_req_notify_cq(cq: &cq->qplib_cq, arm_type: type);
3991
3992	exit:
3993	spin_unlock_irqrestore(lock: &cq->cq_lock, flags);
3994	return rc;
3995	}
3996
3997	/* Memory Regions */
3998	struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
3999	{
4000	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4001	struct bnxt_re_dev *rdev = pd->rdev;
4002	struct bnxt_re_mr *mr;
4003	u32 active_mrs;
4004	int rc;
4005
4006	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4007	if (!mr)
4008	return ERR_PTR(error: -ENOMEM);
4009
4010	mr->rdev = rdev;
4011	mr->qplib_mr.pd = &pd->qplib_pd;
4012	mr->qplib_mr.access_flags = __from_ib_access_flags(iflags: mr_access_flags);
4013	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
4014
4015	if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
4016	bnxt_re_check_and_set_relaxed_ordering(rdev, qplib_mr: &mr->qplib_mr);
4017
4018	/* Allocate and register 0 as the address */
4019	rc = bnxt_qplib_alloc_mrw(res: &rdev->qplib_res, mrw: &mr->qplib_mr);
4020	if (rc)
4021	goto fail;
4022
4023	mr->qplib_mr.hwq.level = PBL_LVL_MAX;
4024	mr->qplib_mr.total_size = -1; /* Infinte length */
4025	rc = bnxt_qplib_reg_mr(res: &rdev->qplib_res, mr: &mr->qplib_mr, NULL, num_pbls: 0,
4026	PAGE_SIZE, unified_mr: false);
4027	if (rc)
4028	goto fail_mr;
4029
4030	mr->ib_mr.lkey = mr->qplib_mr.lkey;
4031	if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
4032	IB_ACCESS_REMOTE_ATOMIC))
4033	mr->ib_mr.rkey = mr->ib_mr.lkey;
4034	active_mrs = atomic_inc_return(v: &rdev->stats.res.mr_count);
4035	if (active_mrs > rdev->stats.res.mr_watermark)
4036	rdev->stats.res.mr_watermark = active_mrs;
4037
4038	return &mr->ib_mr;
4039
4040	fail_mr:
4041	bnxt_qplib_free_mrw(res: &rdev->qplib_res, mr: &mr->qplib_mr);
4042	fail:
4043	kfree(objp: mr);
4044	return ERR_PTR(error: rc);
4045	}
4046
4047	int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
4048	{
4049	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4050	struct bnxt_re_dev *rdev = mr->rdev;
4051	int rc;
4052
4053	rc = bnxt_qplib_free_mrw(res: &rdev->qplib_res, mr: &mr->qplib_mr);
4054	if (rc) {
4055	ibdev_err(ibdev: &rdev->ibdev, format: "Dereg MR failed: %#x\n", rc);
4056	return rc;
4057	}
4058
4059	if (mr->pages) {
4060	rc = bnxt_qplib_free_fast_reg_page_list(res: &rdev->qplib_res,
4061	frpl: &mr->qplib_frpl);
4062	kfree(objp: mr->pages);
4063	mr->npages = 0;
4064	mr->pages = NULL;
4065	}
4066	ib_umem_release(umem: mr->ib_umem);
4067
4068	kfree(objp: mr);
4069	atomic_dec(v: &rdev->stats.res.mr_count);
4070	return rc;
4071	}
4072
4073	static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
4074	{
4075	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4076
4077	if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
4078	return -ENOMEM;
4079
4080	mr->pages[mr->npages++] = addr;
4081	return 0;
4082	}
4083
4084	int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
4085	unsigned int *sg_offset)
4086	{
4087	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4088
4089	mr->npages = 0;
4090	return ib_sg_to_pages(mr: ib_mr, sgl: sg, sg_nents, sg_offset, set_page: bnxt_re_set_page);
4091	}
4092
4093	struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
4094	u32 max_num_sg)
4095	{
4096	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4097	struct bnxt_re_dev *rdev = pd->rdev;
4098	struct bnxt_re_mr *mr = NULL;
4099	u32 active_mrs;
4100	int rc;
4101
4102	if (type != IB_MR_TYPE_MEM_REG) {
4103	ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type);
4104	return ERR_PTR(error: -EINVAL);
4105	}
4106	if (max_num_sg > MAX_PBL_LVL_1_PGS)
4107	return ERR_PTR(error: -EINVAL);
4108
4109	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4110	if (!mr)
4111	return ERR_PTR(error: -ENOMEM);
4112
4113	mr->rdev = rdev;
4114	mr->qplib_mr.pd = &pd->qplib_pd;
4115	mr->qplib_mr.access_flags = BNXT_QPLIB_FR_PMR;
4116	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
4117
4118	rc = bnxt_qplib_alloc_mrw(res: &rdev->qplib_res, mrw: &mr->qplib_mr);
4119	if (rc)
4120	goto bail;
4121
4122	mr->ib_mr.lkey = mr->qplib_mr.lkey;
4123	mr->ib_mr.rkey = mr->ib_mr.lkey;
4124
4125	mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
4126	if (!mr->pages) {
4127	rc = -ENOMEM;
4128	goto fail;
4129	}
4130	rc = bnxt_qplib_alloc_fast_reg_page_list(res: &rdev->qplib_res,
4131	frpl: &mr->qplib_frpl, max: max_num_sg);
4132	if (rc) {
4133	ibdev_err(ibdev: &rdev->ibdev,
4134	format: "Failed to allocate HW FR page list");
4135	goto fail_mr;
4136	}
4137
4138	active_mrs = atomic_inc_return(v: &rdev->stats.res.mr_count);
4139	if (active_mrs > rdev->stats.res.mr_watermark)
4140	rdev->stats.res.mr_watermark = active_mrs;
4141	return &mr->ib_mr;
4142
4143	fail_mr:
4144	kfree(objp: mr->pages);
4145	fail:
4146	bnxt_qplib_free_mrw(res: &rdev->qplib_res, mr: &mr->qplib_mr);
4147	bail:
4148	kfree(objp: mr);
4149	return ERR_PTR(error: rc);
4150	}
4151
4152	struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
4153	struct ib_udata *udata)
4154	{
4155	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4156	struct bnxt_re_dev *rdev = pd->rdev;
4157	struct bnxt_re_mw *mw;
4158	u32 active_mws;
4159	int rc;
4160
4161	mw = kzalloc(sizeof(*mw), GFP_KERNEL);
4162	if (!mw)
4163	return ERR_PTR(error: -ENOMEM);
4164	mw->rdev = rdev;
4165	mw->qplib_mw.pd = &pd->qplib_pd;
4166
4167	mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
4168	CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
4169	CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
4170	rc = bnxt_qplib_alloc_mrw(res: &rdev->qplib_res, mrw: &mw->qplib_mw);
4171	if (rc) {
4172	ibdev_err(ibdev: &rdev->ibdev, format: "Allocate MW failed!");
4173	goto fail;
4174	}
4175	mw->ib_mw.rkey = mw->qplib_mw.rkey;
4176
4177	active_mws = atomic_inc_return(v: &rdev->stats.res.mw_count);
4178	if (active_mws > rdev->stats.res.mw_watermark)
4179	rdev->stats.res.mw_watermark = active_mws;
4180	return &mw->ib_mw;
4181
4182	fail:
4183	kfree(objp: mw);
4184	return ERR_PTR(error: rc);
4185	}
4186
4187	int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
4188	{
4189	struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw);
4190	struct bnxt_re_dev *rdev = mw->rdev;
4191	int rc;
4192
4193	rc = bnxt_qplib_free_mrw(res: &rdev->qplib_res, mr: &mw->qplib_mw);
4194	if (rc) {
4195	ibdev_err(ibdev: &rdev->ibdev, format: "Free MW failed: %#x\n", rc);
4196	return rc;
4197	}
4198
4199	kfree(objp: mw);
4200	atomic_dec(v: &rdev->stats.res.mw_count);
4201	return rc;
4202	}
4203
4204	static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64 virt_addr,
4205	int mr_access_flags, struct ib_umem *umem)
4206	{
4207	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4208	struct bnxt_re_dev *rdev = pd->rdev;
4209	unsigned long page_size;
4210	struct bnxt_re_mr *mr;
4211	int umem_pgs, rc;
4212	u32 active_mrs;
4213
4214	if (length > BNXT_RE_MAX_MR_SIZE) {
4215	ibdev_err(ibdev: &rdev->ibdev, format: "MR Size: %lld > Max supported:%lld\n",
4216	length, BNXT_RE_MAX_MR_SIZE);
4217	return ERR_PTR(error: -ENOMEM);
4218	}
4219
4220	page_size = ib_umem_find_best_pgsz(umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt: virt_addr);
4221	if (!page_size) {
4222	ibdev_err(ibdev: &rdev->ibdev, format: "umem page size unsupported!");
4223	return ERR_PTR(error: -EINVAL);
4224	}
4225
4226	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4227	if (!mr)
4228	return ERR_PTR(error: -ENOMEM);
4229
4230	mr->rdev = rdev;
4231	mr->qplib_mr.pd = &pd->qplib_pd;
4232	mr->qplib_mr.access_flags = __from_ib_access_flags(iflags: mr_access_flags);
4233	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
4234
4235	if (!_is_alloc_mr_unified(dev_cap_flags: rdev->dev_attr->dev_cap_flags)) {
4236	rc = bnxt_qplib_alloc_mrw(res: &rdev->qplib_res, mrw: &mr->qplib_mr);
4237	if (rc) {
4238	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to allocate MR rc = %d", rc);
4239	rc = -EIO;
4240	goto free_mr;
4241	}
4242	/* The fixed portion of the rkey is the same as the lkey */
4243	mr->ib_mr.rkey = mr->qplib_mr.rkey;
4244	} else {
4245	mr->qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
4246	}
4247	mr->ib_umem = umem;
4248	mr->qplib_mr.va = virt_addr;
4249	mr->qplib_mr.total_size = length;
4250
4251	if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
4252	bnxt_re_check_and_set_relaxed_ordering(rdev, qplib_mr: &mr->qplib_mr);
4253
4254	umem_pgs = ib_umem_num_dma_blocks(umem, pgsz: page_size);
4255	rc = bnxt_qplib_reg_mr(res: &rdev->qplib_res, mr: &mr->qplib_mr, umem,
4256	num_pbls: umem_pgs, buf_pg_size: page_size,
4257	unified_mr: _is_alloc_mr_unified(dev_cap_flags: rdev->dev_attr->dev_cap_flags));
4258	if (rc) {
4259	ibdev_err(ibdev: &rdev->ibdev, format: "Failed to register user MR - rc = %d\n", rc);
4260	rc = -EIO;
4261	goto free_mrw;
4262	}
4263
4264	mr->ib_mr.lkey = mr->qplib_mr.lkey;
4265	mr->ib_mr.rkey = mr->qplib_mr.lkey;
4266	active_mrs = atomic_inc_return(v: &rdev->stats.res.mr_count);
4267	if (active_mrs > rdev->stats.res.mr_watermark)
4268	rdev->stats.res.mr_watermark = active_mrs;
4269
4270	return &mr->ib_mr;
4271
4272	free_mrw:
4273	bnxt_qplib_free_mrw(res: &rdev->qplib_res, mr: &mr->qplib_mr);
4274	free_mr:
4275	kfree(objp: mr);
4276	return ERR_PTR(error: rc);
4277	}
4278
4279	struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
4280	u64 virt_addr, int mr_access_flags,
4281	struct ib_dmah *dmah,
4282	struct ib_udata *udata)
4283	{
4284	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4285	struct bnxt_re_dev *rdev = pd->rdev;
4286	struct ib_umem *umem;
4287	struct ib_mr *ib_mr;
4288
4289	if (dmah)
4290	return ERR_PTR(error: -EOPNOTSUPP);
4291
4292	umem = ib_umem_get(device: &rdev->ibdev, addr: start, size: length, access: mr_access_flags);
4293	if (IS_ERR(ptr: umem))
4294	return ERR_CAST(ptr: umem);
4295
4296	ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4297	if (IS_ERR(ptr: ib_mr))
4298	ib_umem_release(umem);
4299	return ib_mr;
4300	}
4301
4302	struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
4303	u64 length, u64 virt_addr, int fd,
4304	int mr_access_flags,
4305	struct ib_dmah *dmah,
4306	struct uverbs_attr_bundle *attrs)
4307	{
4308	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4309	struct bnxt_re_dev *rdev = pd->rdev;
4310	struct ib_umem_dmabuf *umem_dmabuf;
4311	struct ib_umem *umem;
4312	struct ib_mr *ib_mr;
4313
4314	if (dmah)
4315	return ERR_PTR(error: -EOPNOTSUPP);
4316
4317	umem_dmabuf = ib_umem_dmabuf_get_pinned(device: &rdev->ibdev, offset: start, size: length,
4318	fd, access: mr_access_flags);
4319	if (IS_ERR(ptr: umem_dmabuf))
4320	return ERR_CAST(ptr: umem_dmabuf);
4321
4322	umem = &umem_dmabuf->umem;
4323
4324	ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4325	if (IS_ERR(ptr: ib_mr))
4326	ib_umem_release(umem);
4327	return ib_mr;
4328	}
4329
4330	int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
4331	{
4332	struct ib_device *ibdev = ctx->device;
4333	struct bnxt_re_ucontext *uctx =
4334	container_of(ctx, struct bnxt_re_ucontext, ib_uctx);
4335	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
4336	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
4337	struct bnxt_re_user_mmap_entry *entry;
4338	struct bnxt_re_uctx_resp resp = {};
4339	struct bnxt_re_uctx_req ureq = {};
4340	u32 chip_met_rev_num = 0;
4341	int rc;
4342
4343	ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver);
4344
4345	if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
4346	ibdev_dbg(ibdev, " is different from the device %d ",
4347	BNXT_RE_ABI_VERSION);
4348	return -EPERM;
4349	}
4350
4351	uctx->rdev = rdev;
4352
4353	uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
4354	if (!uctx->shpg) {
4355	rc = -ENOMEM;
4356	goto fail;
4357	}
4358	spin_lock_init(&uctx->sh_lock);
4359
4360	resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX;
4361	chip_met_rev_num = rdev->chip_ctx->chip_num;
4362	chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) <<
4363	BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
4364	chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) <<
4365	BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
4366	resp.chip_id0 = chip_met_rev_num;
4367	/*Temp, Use xa_alloc instead */
4368	resp.dev_id = rdev->en_dev->pdev->devfn;
4369	resp.max_qp = rdev->qplib_ctx.qpc_count;
4370	resp.pg_size = PAGE_SIZE;
4371	resp.cqe_sz = sizeof(struct cq_base);
4372	resp.max_cqd = dev_attr->max_cq_wqes;
4373
4374	if (rdev->chip_ctx->modes.db_push)
4375	resp.comp_mask |= BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED;
4376
4377	entry = bnxt_re_mmap_entry_insert(uctx, mem_offset: 0, mmap_flag: BNXT_RE_MMAP_SH_PAGE, NULL);
4378	if (!entry) {
4379	rc = -ENOMEM;
4380	goto cfail;
4381	}
4382	uctx->shpage_mmap = &entry->rdma_entry;
4383	if (rdev->pacing.dbr_pacing)
4384	resp.comp_mask |= BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED;
4385
4386	if (_is_host_msn_table(dev_cap_ext_flags2: rdev->qplib_res.dattr->dev_cap_flags2))
4387	resp.comp_mask |= BNXT_RE_UCNTX_CMASK_MSN_TABLE_ENABLED;
4388
4389	if (udata->inlen >= sizeof(ureq)) {
4390	rc = ib_copy_from_udata(dest: &ureq, udata, min(udata->inlen, sizeof(ureq)));
4391	if (rc)
4392	goto cfail;
4393	if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT) {
4394	resp.comp_mask |= BNXT_RE_UCNTX_CMASK_POW2_DISABLED;
4395	uctx->cmask |= BNXT_RE_UCNTX_CAP_POW2_DISABLED;
4396	}
4397	if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT) {
4398	resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE;
4399	resp.mode = rdev->chip_ctx->modes.wqe_mode;
4400	if (resp.mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
4401	uctx->cmask |= BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;
4402	}
4403	}
4404
4405	rc = ib_copy_to_udata(udata, src: &resp, min(udata->outlen, sizeof(resp)));
4406	if (rc) {
4407	ibdev_err(ibdev, format: "Failed to copy user context");
4408	rc = -EFAULT;
4409	goto cfail;
4410	}
4411
4412	return 0;
4413	cfail:
4414	free_page((unsigned long)uctx->shpg);
4415	uctx->shpg = NULL;
4416	fail:
4417	return rc;
4418	}
4419
4420	void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
4421	{
4422	struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4423	struct bnxt_re_ucontext,
4424	ib_uctx);
4425
4426	struct bnxt_re_dev *rdev = uctx->rdev;
4427
4428	rdma_user_mmap_entry_remove(entry: uctx->shpage_mmap);
4429	uctx->shpage_mmap = NULL;
4430	if (uctx->shpg)
4431	free_page((unsigned long)uctx->shpg);
4432
4433	if (uctx->dpi.dbr) {
4434	/* Free DPI only if this is the first PD allocated by the
4435	* application and mark the context dpi as NULL
4436	*/
4437	bnxt_qplib_dealloc_dpi(res: &rdev->qplib_res, dpi: &uctx->dpi);
4438	uctx->dpi.dbr = NULL;
4439	}
4440	}
4441
4442	static int bnxt_re_setup_vnic(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp)
4443	{
4444	int rc;
4445
4446	rc = bnxt_re_hwrm_alloc_vnic(rdev);
4447	if (rc)
4448	return rc;
4449
4450	rc = bnxt_re_hwrm_cfg_vnic(rdev, qp_id: qp->qplib_qp.id);
4451	if (rc)
4452	goto out_free_vnic;
4453
4454	return 0;
4455	out_free_vnic:
4456	bnxt_re_hwrm_free_vnic(rdev);
4457	return rc;
4458	}
4459
4460	struct ib_flow *bnxt_re_create_flow(struct ib_qp *ib_qp,
4461	struct ib_flow_attr *attr,
4462	struct ib_udata *udata)
4463	{
4464	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
4465	struct bnxt_re_dev *rdev = qp->rdev;
4466	struct bnxt_re_flow *flow;
4467	int rc;
4468
4469	if (attr->type != IB_FLOW_ATTR_SNIFFER ||
4470	!rdev->rcfw.roce_mirror)
4471	return ERR_PTR(error: -EOPNOTSUPP);
4472
4473	mutex_lock(&rdev->qp_lock);
4474	if (rdev->sniffer_flow_created) {
4475	ibdev_err(ibdev: &rdev->ibdev, format: "RoCE Mirroring is already Configured\n");
4476	mutex_unlock(lock: &rdev->qp_lock);
4477	return ERR_PTR(error: -EBUSY);
4478	}
4479
4480	flow = kzalloc(sizeof(*flow), GFP_KERNEL);
4481	if (!flow) {
4482	mutex_unlock(lock: &rdev->qp_lock);
4483	return ERR_PTR(error: -ENOMEM);
4484	}
4485
4486	flow->rdev = rdev;
4487
4488	rc = bnxt_re_setup_vnic(rdev, qp);
4489	if (rc)
4490	goto out_free_flow;
4491
4492	rc = bnxt_qplib_create_flow(res: &rdev->qplib_res);
4493	if (rc)
4494	goto out_free_vnic;
4495
4496	rdev->sniffer_flow_created = 1;
4497	mutex_unlock(lock: &rdev->qp_lock);
4498
4499	return &flow->ib_flow;
4500
4501	out_free_vnic:
4502	bnxt_re_hwrm_free_vnic(rdev);
4503	out_free_flow:
4504	mutex_unlock(lock: &rdev->qp_lock);
4505	kfree(objp: flow);
4506	return ERR_PTR(error: rc);
4507	}
4508
4509	int bnxt_re_destroy_flow(struct ib_flow *flow_id)
4510	{
4511	struct bnxt_re_flow *flow =
4512	container_of(flow_id, struct bnxt_re_flow, ib_flow);
4513	struct bnxt_re_dev *rdev = flow->rdev;
4514	int rc;
4515
4516	mutex_lock(&rdev->qp_lock);
4517	rc = bnxt_qplib_destroy_flow(res: &rdev->qplib_res);
4518	if (rc)
4519	ibdev_dbg(&rdev->ibdev, "failed to destroy_flow rc = %d\n", rc);
4520	rdev->sniffer_flow_created = 0;
4521
4522	bnxt_re_hwrm_free_vnic(rdev);
4523	mutex_unlock(lock: &rdev->qp_lock);
4524	kfree(objp: flow);
4525
4526	return rc;
4527	}
4528
4529	static struct bnxt_re_cq *bnxt_re_search_for_cq(struct bnxt_re_dev *rdev, u32 cq_id)
4530	{
4531	struct bnxt_re_cq *cq = NULL, *tmp_cq;
4532
4533	hash_for_each_possible(rdev->cq_hash, tmp_cq, hash_entry, cq_id) {
4534	if (tmp_cq->qplib_cq.id == cq_id) {
4535	cq = tmp_cq;
4536	break;
4537	}
4538	}
4539	return cq;
4540	}
4541
4542	static struct bnxt_re_srq *bnxt_re_search_for_srq(struct bnxt_re_dev *rdev, u32 srq_id)
4543	{
4544	struct bnxt_re_srq *srq = NULL, *tmp_srq;
4545
4546	hash_for_each_possible(rdev->srq_hash, tmp_srq, hash_entry, srq_id) {
4547	if (tmp_srq->qplib_srq.id == srq_id) {
4548	srq = tmp_srq;
4549	break;
4550	}
4551	}
4552	return srq;
4553	}
4554
4555	/* Helper function to mmap the virtual memory from user app */
4556	int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
4557	{
4558	struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4559	struct bnxt_re_ucontext,
4560	ib_uctx);
4561	struct bnxt_re_user_mmap_entry *bnxt_entry;
4562	struct rdma_user_mmap_entry *rdma_entry;
4563	int ret = 0;
4564	u64 pfn;
4565
4566	rdma_entry = rdma_user_mmap_entry_get(ucontext: &uctx->ib_uctx, vma);
4567	if (!rdma_entry)
4568	return -EINVAL;
4569
4570	bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4571	rdma_entry);
4572
4573	switch (bnxt_entry->mmap_flag) {
4574	case BNXT_RE_MMAP_WC_DB:
4575	pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4576	ret = rdma_user_mmap_io(ucontext: ib_uctx, vma, pfn, PAGE_SIZE,
4577	pgprot_writecombine(prot: vma->vm_page_prot),
4578	entry: rdma_entry);
4579	break;
4580	case BNXT_RE_MMAP_UC_DB:
4581	pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4582	ret = rdma_user_mmap_io(ucontext: ib_uctx, vma, pfn, PAGE_SIZE,
4583	pgprot_noncached(vma->vm_page_prot),
4584	entry: rdma_entry);
4585	break;
4586	case BNXT_RE_MMAP_SH_PAGE:
4587	ret = vm_insert_page(vma, addr: vma->vm_start, virt_to_page(uctx->shpg));
4588	break;
4589	case BNXT_RE_MMAP_DBR_BAR:
4590	pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4591	ret = rdma_user_mmap_io(ucontext: ib_uctx, vma, pfn, PAGE_SIZE,
4592	pgprot_noncached(vma->vm_page_prot),
4593	entry: rdma_entry);
4594	break;
4595	case BNXT_RE_MMAP_DBR_PAGE:
4596	case BNXT_RE_MMAP_TOGGLE_PAGE:
4597	/* Driver doesn't expect write access for user space */
4598	if (vma->vm_flags & VM_WRITE)
4599	ret = -EFAULT;
4600	else
4601	ret = vm_insert_page(vma, addr: vma->vm_start,
4602	virt_to_page((void *)bnxt_entry->mem_offset));
4603	break;
4604	default:
4605	ret = -EINVAL;
4606	break;
4607	}
4608
4609	rdma_user_mmap_entry_put(entry: rdma_entry);
4610	return ret;
4611	}
4612
4613	void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
4614	{
4615	struct bnxt_re_user_mmap_entry *bnxt_entry;
4616
4617	bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4618	rdma_entry);
4619
4620	kfree(objp: bnxt_entry);
4621	}
4622
4623	int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags,
4624	u32 port_num, const struct ib_wc *in_wc,
4625	const struct ib_grh *in_grh,
4626	const struct ib_mad *in_mad, struct ib_mad *out_mad,
4627	size_t *out_mad_size, u16 *out_mad_pkey_index)
4628	{
4629	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
4630	struct ib_class_port_info cpi = {};
4631	int ret = IB_MAD_RESULT_SUCCESS;
4632	int rc = 0;
4633
4634	if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT)
4635	return ret;
4636
4637	switch (in_mad->mad_hdr.attr_id) {
4638	case IB_PMA_CLASS_PORT_INFO:
4639	cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;
4640	memcpy((out_mad->data + 40), &cpi, sizeof(cpi));
4641	break;
4642	case IB_PMA_PORT_COUNTERS_EXT:
4643	rc = bnxt_re_assign_pma_port_ext_counters(rdev, out_mad);
4644	break;
4645	case IB_PMA_PORT_COUNTERS:
4646	rc = bnxt_re_assign_pma_port_counters(rdev, out_mad);
4647	break;
4648	default:
4649	rc = -EINVAL;
4650	break;
4651	}
4652	if (rc)
4653	return IB_MAD_RESULT_FAILURE;
4654	ret |= IB_MAD_RESULT_REPLY;
4655	return ret;
4656	}
4657
4658	static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs)
4659	{
4660	struct bnxt_re_ucontext *uctx;
4661
4662	uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
4663	bnxt_re_pacing_alert(rdev: uctx->rdev);
4664	return 0;
4665	}
4666
4667	static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs)
4668	{
4669	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs_bundle: attrs, idx: BNXT_RE_ALLOC_PAGE_HANDLE);
4670	enum bnxt_re_alloc_page_type alloc_type;
4671	struct bnxt_re_user_mmap_entry *entry;
4672	enum bnxt_re_mmap_flag mmap_flag;
4673	struct bnxt_qplib_chip_ctx *cctx;
4674	struct bnxt_re_ucontext *uctx;
4675	struct bnxt_re_dev *rdev;
4676	u64 mmap_offset;
4677	u32 length;
4678	u32 dpi;
4679	u64 addr;
4680	int err;
4681
4682	uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
4683	if (IS_ERR(ptr: uctx))
4684	return PTR_ERR(ptr: uctx);
4685
4686	err = uverbs_get_const(&alloc_type, attrs, BNXT_RE_ALLOC_PAGE_TYPE);
4687	if (err)
4688	return err;
4689
4690	rdev = uctx->rdev;
4691	cctx = rdev->chip_ctx;
4692
4693	switch (alloc_type) {
4694	case BNXT_RE_ALLOC_WC_PAGE:
4695	if (cctx->modes.db_push) {
4696	if (bnxt_qplib_alloc_dpi(res: &rdev->qplib_res, dpi: &uctx->wcdpi,
4697	app: uctx, type: BNXT_QPLIB_DPI_TYPE_WC))
4698	return -ENOMEM;
4699	length = PAGE_SIZE;
4700	dpi = uctx->wcdpi.dpi;
4701	addr = (u64)uctx->wcdpi.umdbr;
4702	mmap_flag = BNXT_RE_MMAP_WC_DB;
4703	} else {
4704	return -EINVAL;
4705	}
4706
4707	break;
4708	case BNXT_RE_ALLOC_DBR_BAR_PAGE:
4709	length = PAGE_SIZE;
4710	addr = (u64)rdev->pacing.dbr_bar_addr;
4711	mmap_flag = BNXT_RE_MMAP_DBR_BAR;
4712	break;
4713
4714	case BNXT_RE_ALLOC_DBR_PAGE:
4715	length = PAGE_SIZE;
4716	addr = (u64)rdev->pacing.dbr_page;
4717	mmap_flag = BNXT_RE_MMAP_DBR_PAGE;
4718	break;
4719
4720	default:
4721	return -EOPNOTSUPP;
4722	}
4723
4724	entry = bnxt_re_mmap_entry_insert(uctx, mem_offset: addr, mmap_flag, offset: &mmap_offset);
4725	if (!entry)
4726	return -ENOMEM;
4727
4728	uobj->object = entry;
4729	uverbs_finalize_uobj_create(attrs_bundle: attrs, idx: BNXT_RE_ALLOC_PAGE_HANDLE);
4730	err = uverbs_copy_to(attrs_bundle: attrs, idx: BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
4731	from: &mmap_offset, size: sizeof(mmap_offset));
4732	if (err)
4733	return err;
4734
4735	err = uverbs_copy_to(attrs_bundle: attrs, idx: BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
4736	from: &length, size: sizeof(length));
4737	if (err)
4738	return err;
4739
4740	err = uverbs_copy_to(attrs_bundle: attrs, idx: BNXT_RE_ALLOC_PAGE_DPI,
4741	from: &dpi, size: sizeof(dpi));
4742	if (err)
4743	return err;
4744
4745	return 0;
4746	}
4747
4748	static int alloc_page_obj_cleanup(struct ib_uobject *uobject,
4749	enum rdma_remove_reason why,
4750	struct uverbs_attr_bundle *attrs)
4751	{
4752	struct bnxt_re_user_mmap_entry *entry = uobject->object;
4753	struct bnxt_re_ucontext *uctx = entry->uctx;
4754
4755	switch (entry->mmap_flag) {
4756	case BNXT_RE_MMAP_WC_DB:
4757	if (uctx && uctx->wcdpi.dbr) {
4758	struct bnxt_re_dev *rdev = uctx->rdev;
4759
4760	bnxt_qplib_dealloc_dpi(res: &rdev->qplib_res, dpi: &uctx->wcdpi);
4761	uctx->wcdpi.dbr = NULL;
4762	}
4763	break;
4764	case BNXT_RE_MMAP_DBR_BAR:
4765	case BNXT_RE_MMAP_DBR_PAGE:
4766	break;
4767	default:
4768	goto exit;
4769	}
4770	rdma_user_mmap_entry_remove(entry: &entry->rdma_entry);
4771	exit:
4772	return 0;
4773	}
4774
4775	DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_ALLOC_PAGE,
4776	UVERBS_ATTR_IDR(BNXT_RE_ALLOC_PAGE_HANDLE,
4777	BNXT_RE_OBJECT_ALLOC_PAGE,
4778	UVERBS_ACCESS_NEW,
4779	UA_MANDATORY),
4780	UVERBS_ATTR_CONST_IN(BNXT_RE_ALLOC_PAGE_TYPE,
4781	enum bnxt_re_alloc_page_type,
4782	UA_MANDATORY),
4783	UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
4784	UVERBS_ATTR_TYPE(u64),
4785	UA_MANDATORY),
4786	UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
4787	UVERBS_ATTR_TYPE(u32),
4788	UA_MANDATORY),
4789	UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_DPI,
4790	UVERBS_ATTR_TYPE(u32),
4791	UA_MANDATORY));
4792
4793	DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DESTROY_PAGE,
4794	UVERBS_ATTR_IDR(BNXT_RE_DESTROY_PAGE_HANDLE,
4795	BNXT_RE_OBJECT_ALLOC_PAGE,
4796	UVERBS_ACCESS_DESTROY,
4797	UA_MANDATORY));
4798
4799	DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE,
4800	UVERBS_TYPE_ALLOC_IDR(alloc_page_obj_cleanup),
4801	&UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE),
4802	&UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE));
4803
4804	DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV);
4805
4806	DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV,
4807	&UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV));
4808
4809	/* Toggle MEM */
4810	static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bundle *attrs)
4811	{
4812	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs_bundle: attrs, idx: BNXT_RE_TOGGLE_MEM_HANDLE);
4813	enum bnxt_re_mmap_flag mmap_flag = BNXT_RE_MMAP_TOGGLE_PAGE;
4814	enum bnxt_re_get_toggle_mem_type res_type;
4815	struct bnxt_re_user_mmap_entry *entry;
4816	struct bnxt_re_ucontext *uctx;
4817	struct ib_ucontext *ib_uctx;
4818	struct bnxt_re_dev *rdev;
4819	struct bnxt_re_srq *srq;
4820	u32 length = PAGE_SIZE;
4821	struct bnxt_re_cq *cq;
4822	u64 mem_offset;
4823	u32 offset = 0;
4824	u64 addr = 0;
4825	u32 res_id;
4826	int err;
4827
4828	ib_uctx = ib_uverbs_get_ucontext(attrs);
4829	if (IS_ERR(ptr: ib_uctx))
4830	return PTR_ERR(ptr: ib_uctx);
4831
4832	err = uverbs_get_const(&res_type, attrs, BNXT_RE_TOGGLE_MEM_TYPE);
4833	if (err)
4834	return err;
4835
4836	uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
4837	rdev = uctx->rdev;
4838	err = uverbs_copy_from(&res_id, attrs, BNXT_RE_TOGGLE_MEM_RES_ID);
4839	if (err)
4840	return err;
4841
4842	switch (res_type) {
4843	case BNXT_RE_CQ_TOGGLE_MEM:
4844	cq = bnxt_re_search_for_cq(rdev, cq_id: res_id);
4845	if (!cq)
4846	return -EINVAL;
4847
4848	addr = (u64)cq->uctx_cq_page;
4849	break;
4850	case BNXT_RE_SRQ_TOGGLE_MEM:
4851	srq = bnxt_re_search_for_srq(rdev, srq_id: res_id);
4852	if (!srq)
4853	return -EINVAL;
4854
4855	addr = (u64)srq->uctx_srq_page;
4856	break;
4857
4858	default:
4859	return -EOPNOTSUPP;
4860	}
4861
4862	entry = bnxt_re_mmap_entry_insert(uctx, mem_offset: addr, mmap_flag, offset: &mem_offset);
4863	if (!entry)
4864	return -ENOMEM;
4865
4866	uobj->object = entry;
4867	uverbs_finalize_uobj_create(attrs_bundle: attrs, idx: BNXT_RE_TOGGLE_MEM_HANDLE);
4868	err = uverbs_copy_to(attrs_bundle: attrs, idx: BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
4869	from: &mem_offset, size: sizeof(mem_offset));
4870	if (err)
4871	return err;
4872
4873	err = uverbs_copy_to(attrs_bundle: attrs, idx: BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
4874	from: &length, size: sizeof(length));
4875	if (err)
4876	return err;
4877
4878	err = uverbs_copy_to(attrs_bundle: attrs, idx: BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
4879	from: &offset, size: sizeof(offset));
4880	if (err)
4881	return err;
4882
4883	return 0;
4884	}
4885
4886	static int get_toggle_mem_obj_cleanup(struct ib_uobject *uobject,
4887	enum rdma_remove_reason why,
4888	struct uverbs_attr_bundle *attrs)
4889	{
4890	struct bnxt_re_user_mmap_entry *entry = uobject->object;
4891
4892	rdma_user_mmap_entry_remove(entry: &entry->rdma_entry);
4893	return 0;
4894	}
4895
4896	DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM,
4897	UVERBS_ATTR_IDR(BNXT_RE_TOGGLE_MEM_HANDLE,
4898	BNXT_RE_OBJECT_GET_TOGGLE_MEM,
4899	UVERBS_ACCESS_NEW,
4900	UA_MANDATORY),
4901	UVERBS_ATTR_CONST_IN(BNXT_RE_TOGGLE_MEM_TYPE,
4902	enum bnxt_re_get_toggle_mem_type,
4903	UA_MANDATORY),
4904	UVERBS_ATTR_PTR_IN(BNXT_RE_TOGGLE_MEM_RES_ID,
4905	UVERBS_ATTR_TYPE(u32),
4906	UA_MANDATORY),
4907	UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
4908	UVERBS_ATTR_TYPE(u64),
4909	UA_MANDATORY),
4910	UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
4911	UVERBS_ATTR_TYPE(u32),
4912	UA_MANDATORY),
4913	UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
4914	UVERBS_ATTR_TYPE(u32),
4915	UA_MANDATORY));
4916
4917	DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM,
4918	UVERBS_ATTR_IDR(BNXT_RE_RELEASE_TOGGLE_MEM_HANDLE,
4919	BNXT_RE_OBJECT_GET_TOGGLE_MEM,
4920	UVERBS_ACCESS_DESTROY,
4921	UA_MANDATORY));
4922
4923	DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_GET_TOGGLE_MEM,
4924	UVERBS_TYPE_ALLOC_IDR(get_toggle_mem_obj_cleanup),
4925	&UVERBS_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM),
4926	&UVERBS_METHOD(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM));
4927
4928	const struct uapi_definition bnxt_re_uapi_defs[] = {
4929	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE),
4930	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV),
4931	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM),
4932	{}
4933	};
4934