1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2	/* Copyright (c) 2015 - 2021 Intel Corporation */
3	#include "main.h"
4
5	/**
6	* irdma_query_device - get device attributes
7	* @ibdev: device pointer from stack
8	* @props: returning device attributes
9	* @udata: user data
10	*/
11	static int irdma_query_device(struct ib_device *ibdev,
12	struct ib_device_attr *props,
13	struct ib_udata *udata)
14	{
15	struct irdma_device *iwdev = to_iwdev(ibdev);
16	struct irdma_pci_f *rf = iwdev->rf;
17	struct pci_dev *pcidev = iwdev->rf->pcidev;
18	struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs;
19
20	if (udata->inlen || udata->outlen)
21	return -EINVAL;
22
23	memset(props, 0, sizeof(*props));
24	addrconf_addr_eui48(eui: (u8 *)&props->sys_image_guid,
25	addr: iwdev->netdev->dev_addr);
26	props->fw_ver = (u64)irdma_fw_major_ver(dev: &rf->sc_dev) << 32 |
27	irdma_fw_minor_ver(dev: &rf->sc_dev);
28	props->device_cap_flags = IB_DEVICE_MEM_WINDOW |
29	IB_DEVICE_MEM_MGT_EXTENSIONS;
30	props->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
31	props->vendor_id = pcidev->vendor;
32	props->vendor_part_id = pcidev->device;
33
34	props->hw_ver = rf->pcidev->revision;
35	props->page_size_cap = hw_attrs->page_size_cap;
36	props->max_mr_size = hw_attrs->max_mr_size;
37	props->max_qp = rf->max_qp - rf->used_qps;
38	props->max_qp_wr = hw_attrs->max_qp_wr;
39	props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags;
40	props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags;
41	props->max_cq = rf->max_cq - rf->used_cqs;
42	props->max_cqe = rf->max_cqe - 1;
43	props->max_mr = rf->max_mr - rf->used_mrs;
44	props->max_mw = props->max_mr;
45	props->max_pd = rf->max_pd - rf->used_pds;
46	props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges;
47	props->max_qp_rd_atom = hw_attrs->max_hw_ird;
48	props->max_qp_init_rd_atom = hw_attrs->max_hw_ord;
49	if (rdma_protocol_roce(device: ibdev, port_num: 1)) {
50	props->device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN;
51	props->max_pkeys = IRDMA_PKEY_TBL_SZ;
52	}
53
54	props->max_ah = rf->max_ah;
55	props->max_mcast_grp = rf->max_mcg;
56	props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX;
57	props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX;
58	props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR;
59	#define HCA_CLOCK_TIMESTAMP_MASK 0x1ffff
60	if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_2)
61	props->timestamp_mask = HCA_CLOCK_TIMESTAMP_MASK;
62
63	return 0;
64	}
65
66	/**
67	* irdma_query_port - get port attributes
68	* @ibdev: device pointer from stack
69	* @port: port number for query
70	* @props: returning device attributes
71	*/
72	static int irdma_query_port(struct ib_device *ibdev, u32 port,
73	struct ib_port_attr *props)
74	{
75	struct irdma_device *iwdev = to_iwdev(ibdev);
76	struct net_device *netdev = iwdev->netdev;
77
78	/* no need to zero out pros here. done by caller */
79
80	props->max_mtu = IB_MTU_4096;
81	props->active_mtu = ib_mtu_int_to_enum(mtu: netdev->mtu);
82	props->lid = 1;
83	props->lmc = 0;
84	props->sm_lid = 0;
85	props->sm_sl = 0;
86	if (netif_carrier_ok(dev: netdev) && netif_running(dev: netdev)) {
87	props->state = IB_PORT_ACTIVE;
88	props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
89	} else {
90	props->state = IB_PORT_DOWN;
91	props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
92	}
93
94	ib_get_eth_speed(dev: ibdev, port_num: port, speed: &props->active_speed,
95	width: &props->active_width);
96
97	if (rdma_protocol_roce(device: ibdev, port_num: 1)) {
98	props->gid_tbl_len = 32;
99	props->ip_gids = true;
100	props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ;
101	} else {
102	props->gid_tbl_len = 1;
103	}
104	props->qkey_viol_cntr = 0;
105	props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP;
106	props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size;
107
108	return 0;
109	}
110
111	/**
112	* irdma_disassociate_ucontext - Disassociate user context
113	* @context: ib user context
114	*/
115	static void irdma_disassociate_ucontext(struct ib_ucontext *context)
116	{
117	}
118
119	static int irdma_mmap_legacy(struct irdma_ucontext *ucontext,
120	struct vm_area_struct *vma)
121	{
122	u64 pfn;
123
124	if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE)
125	return -EINVAL;
126
127	vma->vm_private_data = ucontext;
128	pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] +
129	pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT;
130
131	return rdma_user_mmap_io(ucontext: &ucontext->ibucontext, vma, pfn, PAGE_SIZE,
132	pgprot_noncached(vma->vm_page_prot), NULL);
133	}
134
135	static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
136	{
137	struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry);
138
139	kfree(objp: entry);
140	}
141
142	static struct rdma_user_mmap_entry*
143	irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset,
144	enum irdma_mmap_flag mmap_flag, u64 *mmap_offset)
145	{
146	struct irdma_user_mmap_entry *entry = kzalloc(size: sizeof(*entry), GFP_KERNEL);
147	int ret;
148
149	if (!entry)
150	return NULL;
151
152	entry->bar_offset = bar_offset;
153	entry->mmap_flag = mmap_flag;
154
155	ret = rdma_user_mmap_entry_insert(ucontext: &ucontext->ibucontext,
156	entry: &entry->rdma_entry, PAGE_SIZE);
157	if (ret) {
158	kfree(objp: entry);
159	return NULL;
160	}
161	*mmap_offset = rdma_user_mmap_get_offset(entry: &entry->rdma_entry);
162
163	return &entry->rdma_entry;
164	}
165
166	/**
167	* irdma_mmap - user memory map
168	* @context: context created during alloc
169	* @vma: kernel info for user memory map
170	*/
171	static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
172	{
173	struct rdma_user_mmap_entry *rdma_entry;
174	struct irdma_user_mmap_entry *entry;
175	struct irdma_ucontext *ucontext;
176	u64 pfn;
177	int ret;
178
179	ucontext = to_ucontext(ibucontext: context);
180
181	/* Legacy support for libi40iw with hard-coded mmap key */
182	if (ucontext->legacy_mode)
183	return irdma_mmap_legacy(ucontext, vma);
184
185	rdma_entry = rdma_user_mmap_entry_get(ucontext: &ucontext->ibucontext, vma);
186	if (!rdma_entry) {
187	ibdev_dbg(&ucontext->iwdev->ibdev,
188	"VERBS: pgoff[0x%lx] does not have valid entry\n",
189	vma->vm_pgoff);
190	return -EINVAL;
191	}
192
193	entry = to_irdma_mmap_entry(rdma_entry);
194	ibdev_dbg(&ucontext->iwdev->ibdev,
195	"VERBS: bar_offset [0x%llx] mmap_flag [%d]\n",
196	entry->bar_offset, entry->mmap_flag);
197
198	pfn = (entry->bar_offset +
199	pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT;
200
201	switch (entry->mmap_flag) {
202	case IRDMA_MMAP_IO_NC:
203	ret = rdma_user_mmap_io(ucontext: context, vma, pfn, PAGE_SIZE,
204	pgprot_noncached(vma->vm_page_prot),
205	entry: rdma_entry);
206	break;
207	case IRDMA_MMAP_IO_WC:
208	ret = rdma_user_mmap_io(ucontext: context, vma, pfn, PAGE_SIZE,
209	pgprot_writecombine(prot: vma->vm_page_prot),
210	entry: rdma_entry);
211	break;
212	default:
213	ret = -EINVAL;
214	}
215
216	if (ret)
217	ibdev_dbg(&ucontext->iwdev->ibdev,
218	"VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n",
219	entry->bar_offset, entry->mmap_flag, ret);
220	rdma_user_mmap_entry_put(entry: rdma_entry);
221
222	return ret;
223	}
224
225	/**
226	* irdma_alloc_push_page - allocate a push page for qp
227	* @iwqp: qp pointer
228	*/
229	static void irdma_alloc_push_page(struct irdma_qp *iwqp)
230	{
231	struct irdma_cqp_request *cqp_request;
232	struct cqp_cmds_info *cqp_info;
233	struct irdma_device *iwdev = iwqp->iwdev;
234	struct irdma_sc_qp *qp = &iwqp->sc_qp;
235	int status;
236
237	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &iwdev->rf->cqp, wait: true);
238	if (!cqp_request)
239	return;
240
241	cqp_info = &cqp_request->info;
242	cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
243	cqp_info->post_sq = 1;
244	cqp_info->in.u.manage_push_page.info.push_idx = 0;
245	cqp_info->in.u.manage_push_page.info.qs_handle =
246	qp->vsi->qos[qp->user_pri].qs_handle;
247	cqp_info->in.u.manage_push_page.info.free_page = 0;
248	cqp_info->in.u.manage_push_page.info.push_page_type = 0;
249	cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp;
250	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
251
252	status = irdma_handle_cqp_op(rf: iwdev->rf, cqp_request);
253	if (!status && cqp_request->compl_info.op_ret_val <
254	iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) {
255	qp->push_idx = cqp_request->compl_info.op_ret_val;
256	qp->push_offset = 0;
257	}
258
259	irdma_put_cqp_request(cqp: &iwdev->rf->cqp, cqp_request);
260	}
261
262	/**
263	* irdma_alloc_ucontext - Allocate the user context data structure
264	* @uctx: uverbs context pointer
265	* @udata: user data
266	*
267	* This keeps track of all objects associated with a particular
268	* user-mode client.
269	*/
270	static int irdma_alloc_ucontext(struct ib_ucontext *uctx,
271	struct ib_udata *udata)
272	{
273	#define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8)
274	#define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd)
275	struct ib_device *ibdev = uctx->device;
276	struct irdma_device *iwdev = to_iwdev(ibdev);
277	struct irdma_alloc_ucontext_req req = {};
278	struct irdma_alloc_ucontext_resp uresp = {};
279	struct irdma_ucontext *ucontext = to_ucontext(ibucontext: uctx);
280	struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;
281
282	if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN ||
283	udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN)
284	return -EINVAL;
285
286	if (ib_copy_from_udata(dest: &req, udata, min(sizeof(req), udata->inlen)))
287	return -EINVAL;
288
289	if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER)
290	goto ver_error;
291
292	ucontext->iwdev = iwdev;
293	ucontext->abi_ver = req.userspace_ver;
294
295	if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR)
296	ucontext->use_raw_attrs = true;
297
298	/* GEN_1 legacy support with libi40iw */
299	if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) {
300	if (uk_attrs->hw_rev != IRDMA_GEN_1)
301	return -EOPNOTSUPP;
302
303	ucontext->legacy_mode = true;
304	uresp.max_qps = iwdev->rf->max_qp;
305	uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;
306	uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;
307	uresp.kernel_ver = req.userspace_ver;
308	if (ib_copy_to_udata(udata, src: &uresp,
309	min(sizeof(uresp), udata->outlen)))
310	return -EFAULT;
311	} else {
312	u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
313
314	ucontext->db_mmap_entry =
315	irdma_user_mmap_entry_insert(ucontext, bar_offset: bar_off,
316	mmap_flag: IRDMA_MMAP_IO_NC,
317	mmap_offset: &uresp.db_mmap_key);
318	if (!ucontext->db_mmap_entry)
319	return -ENOMEM;
320
321	uresp.kernel_ver = IRDMA_ABI_VER;
322	uresp.feature_flags = uk_attrs->feature_flags;
323	uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags;
324	uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges;
325	uresp.max_hw_inline = uk_attrs->max_hw_inline;
326	uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta;
327	uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta;
328	uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk;
329	uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size;
330	uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
331	uresp.hw_rev = uk_attrs->hw_rev;
332	uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
333	uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size;
334	uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
335	if (ib_copy_to_udata(udata, src: &uresp,
336	min(sizeof(uresp), udata->outlen))) {
337	rdma_user_mmap_entry_remove(entry: ucontext->db_mmap_entry);
338	return -EFAULT;
339	}
340	}
341
342	INIT_LIST_HEAD(list: &ucontext->cq_reg_mem_list);
343	spin_lock_init(&ucontext->cq_reg_mem_list_lock);
344	INIT_LIST_HEAD(list: &ucontext->qp_reg_mem_list);
345	spin_lock_init(&ucontext->qp_reg_mem_list_lock);
346
347	return 0;
348
349	ver_error:
350	ibdev_err(ibdev: &iwdev->ibdev,
351	format: "Invalid userspace driver version detected. Detected version %d, should be %d\n",
352	req.userspace_ver, IRDMA_ABI_VER);
353	return -EINVAL;
354	}
355
356	/**
357	* irdma_dealloc_ucontext - deallocate the user context data structure
358	* @context: user context created during alloc
359	*/
360	static void irdma_dealloc_ucontext(struct ib_ucontext *context)
361	{
362	struct irdma_ucontext *ucontext = to_ucontext(ibucontext: context);
363
364	rdma_user_mmap_entry_remove(entry: ucontext->db_mmap_entry);
365	}
366
367	/**
368	* irdma_alloc_pd - allocate protection domain
369	* @pd: PD pointer
370	* @udata: user data
371	*/
372	static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
373	{
374	#define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd)
375	struct irdma_pd *iwpd = to_iwpd(ibpd: pd);
376	struct irdma_device *iwdev = to_iwdev(ibdev: pd->device);
377	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
378	struct irdma_pci_f *rf = iwdev->rf;
379	struct irdma_alloc_pd_resp uresp = {};
380	struct irdma_sc_pd *sc_pd;
381	u32 pd_id = 0;
382	int err;
383
384	if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN)
385	return -EINVAL;
386
387	err = irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_pds, max_rsrc: rf->max_pd, req_rsrc_num: &pd_id,
388	next: &rf->next_pd);
389	if (err)
390	return err;
391
392	sc_pd = &iwpd->sc_pd;
393	if (udata) {
394	struct irdma_ucontext *ucontext =
395	rdma_udata_to_drv_context(udata, struct irdma_ucontext,
396	ibucontext);
397	irdma_sc_pd_init(dev, pd: sc_pd, pd_id, abi_ver: ucontext->abi_ver);
398	uresp.pd_id = pd_id;
399	if (ib_copy_to_udata(udata, src: &uresp,
400	min(sizeof(uresp), udata->outlen))) {
401	err = -EFAULT;
402	goto error;
403	}
404	} else {
405	irdma_sc_pd_init(dev, pd: sc_pd, pd_id, IRDMA_ABI_VER);
406	}
407
408	return 0;
409	error:
410	irdma_free_rsrc(rf, rsrc_array: rf->allocated_pds, rsrc_num: pd_id);
411
412	return err;
413	}
414
415	/**
416	* irdma_dealloc_pd - deallocate pd
417	* @ibpd: ptr of pd to be deallocated
418	* @udata: user data
419	*/
420	static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
421	{
422	struct irdma_pd *iwpd = to_iwpd(ibpd);
423	struct irdma_device *iwdev = to_iwdev(ibdev: ibpd->device);
424
425	irdma_free_rsrc(rf: iwdev->rf, rsrc_array: iwdev->rf->allocated_pds, rsrc_num: iwpd->sc_pd.pd_id);
426
427	return 0;
428	}
429
430	/**
431	* irdma_get_pbl - Retrieve pbl from a list given a virtual
432	* address
433	* @va: user virtual address
434	* @pbl_list: pbl list to search in (QP's or CQ's)
435	*/
436	static struct irdma_pbl *irdma_get_pbl(unsigned long va,
437	struct list_head *pbl_list)
438	{
439	struct irdma_pbl *iwpbl;
440
441	list_for_each_entry (iwpbl, pbl_list, list) {
442	if (iwpbl->user_base == va) {
443	list_del(entry: &iwpbl->list);
444	iwpbl->on_list = false;
445	return iwpbl;
446	}
447	}
448
449	return NULL;
450	}
451
452	/**
453	* irdma_clean_cqes - clean cq entries for qp
454	* @iwqp: qp ptr (user or kernel)
455	* @iwcq: cq ptr
456	*/
457	static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq)
458	{
459	struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk;
460	unsigned long flags;
461
462	spin_lock_irqsave(&iwcq->lock, flags);
463	irdma_uk_clean_cq(q: &iwqp->sc_qp.qp_uk, cq: ukcq);
464	spin_unlock_irqrestore(lock: &iwcq->lock, flags);
465	}
466
467	static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp)
468	{
469	if (iwqp->push_db_mmap_entry) {
470	rdma_user_mmap_entry_remove(entry: iwqp->push_db_mmap_entry);
471	iwqp->push_db_mmap_entry = NULL;
472	}
473	if (iwqp->push_wqe_mmap_entry) {
474	rdma_user_mmap_entry_remove(entry: iwqp->push_wqe_mmap_entry);
475	iwqp->push_wqe_mmap_entry = NULL;
476	}
477	}
478
479	static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext,
480	struct irdma_qp *iwqp,
481	u64 *push_wqe_mmap_key,
482	u64 *push_db_mmap_key)
483	{
484	struct irdma_device *iwdev = ucontext->iwdev;
485	u64 rsvd, bar_off;
486
487	rsvd = IRDMA_PF_BAR_RSVD;
488	bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
489	/* skip over db page */
490	bar_off += IRDMA_HW_PAGE_SIZE;
491	/* push wqe page */
492	bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE;
493	iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext,
494	bar_offset: bar_off, mmap_flag: IRDMA_MMAP_IO_WC,
495	mmap_offset: push_wqe_mmap_key);
496	if (!iwqp->push_wqe_mmap_entry)
497	return -ENOMEM;
498
499	/* push doorbell page */
500	bar_off += IRDMA_HW_PAGE_SIZE;
501	iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext,
502	bar_offset: bar_off, mmap_flag: IRDMA_MMAP_IO_NC,
503	mmap_offset: push_db_mmap_key);
504	if (!iwqp->push_db_mmap_entry) {
505	rdma_user_mmap_entry_remove(entry: iwqp->push_wqe_mmap_entry);
506	return -ENOMEM;
507	}
508
509	return 0;
510	}
511
512	/**
513	* irdma_destroy_qp - destroy qp
514	* @ibqp: qp's ib pointer also to get to device's qp address
515	* @udata: user data
516	*/
517	static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
518	{
519	struct irdma_qp *iwqp = to_iwqp(ibqp);
520	struct irdma_device *iwdev = iwqp->iwdev;
521
522	iwqp->sc_qp.qp_uk.destroy_pending = true;
523
524	if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS)
525	irdma_modify_qp_to_err(sc_qp: &iwqp->sc_qp);
526
527	if (!iwqp->user_mode)
528	cancel_delayed_work_sync(dwork: &iwqp->dwork_flush);
529
530	if (!iwqp->user_mode) {
531	if (iwqp->iwscq) {
532	irdma_clean_cqes(iwqp, iwcq: iwqp->iwscq);
533	if (iwqp->iwrcq != iwqp->iwscq)
534	irdma_clean_cqes(iwqp, iwcq: iwqp->iwrcq);
535	}
536	}
537
538	irdma_qp_rem_ref(ibqp: &iwqp->ibqp);
539	wait_for_completion(&iwqp->free_qp);
540	irdma_free_lsmm_rsrc(iwqp);
541	irdma_cqp_qp_destroy_cmd(dev: &iwdev->rf->sc_dev, qp: &iwqp->sc_qp);
542
543	irdma_remove_push_mmap_entries(iwqp);
544	irdma_free_qp_rsrc(iwqp);
545
546	return 0;
547	}
548
549	/**
550	* irdma_setup_virt_qp - setup for allocation of virtual qp
551	* @iwdev: irdma device
552	* @iwqp: qp ptr
553	* @init_info: initialize info to return
554	*/
555	static void irdma_setup_virt_qp(struct irdma_device *iwdev,
556	struct irdma_qp *iwqp,
557	struct irdma_qp_init_info *init_info)
558	{
559	struct irdma_pbl *iwpbl = iwqp->iwpbl;
560	struct irdma_qp_mr *qpmr = &iwpbl->qp_mr;
561
562	iwqp->page = qpmr->sq_page;
563	init_info->shadow_area_pa = qpmr->shadow;
564	if (iwpbl->pbl_allocated) {
565	init_info->virtual_map = true;
566	init_info->sq_pa = qpmr->sq_pbl.idx;
567	init_info->rq_pa = qpmr->rq_pbl.idx;
568	} else {
569	init_info->sq_pa = qpmr->sq_pbl.addr;
570	init_info->rq_pa = qpmr->rq_pbl.addr;
571	}
572	}
573
574	/**
575	* irdma_setup_umode_qp - setup sq and rq size in user mode qp
576	* @udata: udata
577	* @iwdev: iwarp device
578	* @iwqp: qp ptr (user or kernel)
579	* @info: initialize info to return
580	* @init_attr: Initial QP create attributes
581	*/
582	static int irdma_setup_umode_qp(struct ib_udata *udata,
583	struct irdma_device *iwdev,
584	struct irdma_qp *iwqp,
585	struct irdma_qp_init_info *info,
586	struct ib_qp_init_attr *init_attr)
587	{
588	struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata,
589	struct irdma_ucontext, ibucontext);
590	struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
591	struct irdma_create_qp_req req;
592	unsigned long flags;
593	int ret;
594
595	ret = ib_copy_from_udata(dest: &req, udata,
596	min(sizeof(req), udata->inlen));
597	if (ret) {
598	ibdev_dbg(&iwdev->ibdev, "VERBS: ib_copy_from_data fail\n");
599	return ret;
600	}
601
602	iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;
603	iwqp->user_mode = 1;
604	if (req.user_wqe_bufs) {
605	info->qp_uk_init_info.legacy_mode = ucontext->legacy_mode;
606	spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
607	iwqp->iwpbl = irdma_get_pbl(va: (unsigned long)req.user_wqe_bufs,
608	pbl_list: &ucontext->qp_reg_mem_list);
609	spin_unlock_irqrestore(lock: &ucontext->qp_reg_mem_list_lock, flags);
610
611	if (!iwqp->iwpbl) {
612	ret = -ENODATA;
613	ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n");
614	return ret;
615	}
616	}
617
618	if (!ucontext->use_raw_attrs) {
619	/**
620	* Maintain backward compat with older ABI which passes sq and
621	* rq depth in quanta in cap.max_send_wr and cap.max_recv_wr.
622	* There is no way to compute the correct value of
623	* iwqp->max_send_wr/max_recv_wr in the kernel.
624	*/
625	iwqp->max_send_wr = init_attr->cap.max_send_wr;
626	iwqp->max_recv_wr = init_attr->cap.max_recv_wr;
627	ukinfo->sq_size = init_attr->cap.max_send_wr;
628	ukinfo->rq_size = init_attr->cap.max_recv_wr;
629	irdma_uk_calc_shift_wq(ukinfo, sq_shift: &ukinfo->sq_shift,
630	rq_shift: &ukinfo->rq_shift);
631	} else {
632	ret = irdma_uk_calc_depth_shift_sq(ukinfo, sq_depth: &ukinfo->sq_depth,
633	sq_shift: &ukinfo->sq_shift);
634	if (ret)
635	return ret;
636
637	ret = irdma_uk_calc_depth_shift_rq(ukinfo, rq_depth: &ukinfo->rq_depth,
638	rq_shift: &ukinfo->rq_shift);
639	if (ret)
640	return ret;
641
642	iwqp->max_send_wr =
643	(ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift;
644	iwqp->max_recv_wr =
645	(ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift;
646	ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift;
647	ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift;
648	}
649
650	irdma_setup_virt_qp(iwdev, iwqp, init_info: info);
651
652	return 0;
653	}
654
655	/**
656	* irdma_setup_kmode_qp - setup initialization for kernel mode qp
657	* @iwdev: iwarp device
658	* @iwqp: qp ptr (user or kernel)
659	* @info: initialize info to return
660	* @init_attr: Initial QP create attributes
661	*/
662	static int irdma_setup_kmode_qp(struct irdma_device *iwdev,
663	struct irdma_qp *iwqp,
664	struct irdma_qp_init_info *info,
665	struct ib_qp_init_attr *init_attr)
666	{
667	struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem;
668	u32 size;
669	int status;
670	struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
671
672	status = irdma_uk_calc_depth_shift_sq(ukinfo, sq_depth: &ukinfo->sq_depth,
673	sq_shift: &ukinfo->sq_shift);
674	if (status)
675	return status;
676
677	status = irdma_uk_calc_depth_shift_rq(ukinfo, rq_depth: &ukinfo->rq_depth,
678	rq_shift: &ukinfo->rq_shift);
679	if (status)
680	return status;
681
682	iwqp->kqp.sq_wrid_mem =
683	kcalloc(n: ukinfo->sq_depth, size: sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL);
684	if (!iwqp->kqp.sq_wrid_mem)
685	return -ENOMEM;
686
687	iwqp->kqp.rq_wrid_mem =
688	kcalloc(n: ukinfo->rq_depth, size: sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL);
689
690	if (!iwqp->kqp.rq_wrid_mem) {
691	kfree(objp: iwqp->kqp.sq_wrid_mem);
692	iwqp->kqp.sq_wrid_mem = NULL;
693	return -ENOMEM;
694	}
695
696	ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem;
697	ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem;
698
699	size = (ukinfo->sq_depth + ukinfo->rq_depth) * IRDMA_QP_WQE_MIN_SIZE;
700	size += (IRDMA_SHADOW_AREA_SIZE << 3);
701
702	mem->size = ALIGN(size, 256);
703	mem->va = dma_alloc_coherent(dev: iwdev->rf->hw.device, size: mem->size,
704	dma_handle: &mem->pa, GFP_KERNEL);
705	if (!mem->va) {
706	kfree(objp: iwqp->kqp.sq_wrid_mem);
707	iwqp->kqp.sq_wrid_mem = NULL;
708	kfree(objp: iwqp->kqp.rq_wrid_mem);
709	iwqp->kqp.rq_wrid_mem = NULL;
710	return -ENOMEM;
711	}
712
713	ukinfo->sq = mem->va;
714	info->sq_pa = mem->pa;
715	ukinfo->rq = &ukinfo->sq[ukinfo->sq_depth];
716	info->rq_pa = info->sq_pa + (ukinfo->sq_depth * IRDMA_QP_WQE_MIN_SIZE);
717	ukinfo->shadow_area = ukinfo->rq[ukinfo->rq_depth].elem;
718	info->shadow_area_pa =
719	info->rq_pa + (ukinfo->rq_depth * IRDMA_QP_WQE_MIN_SIZE);
720	ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift;
721	ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift;
722
723	iwqp->max_send_wr = (ukinfo->sq_depth - IRDMA_SQ_RSVD) >> ukinfo->sq_shift;
724	iwqp->max_recv_wr = (ukinfo->rq_depth - IRDMA_RQ_RSVD) >> ukinfo->rq_shift;
725	init_attr->cap.max_send_wr = iwqp->max_send_wr;
726	init_attr->cap.max_recv_wr = iwqp->max_recv_wr;
727
728	return 0;
729	}
730
731	static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp)
732	{
733	struct irdma_pci_f *rf = iwqp->iwdev->rf;
734	struct irdma_cqp_request *cqp_request;
735	struct cqp_cmds_info *cqp_info;
736	struct irdma_create_qp_info *qp_info;
737	int status;
738
739	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true);
740	if (!cqp_request)
741	return -ENOMEM;
742
743	cqp_info = &cqp_request->info;
744	qp_info = &cqp_request->info.in.u.qp_create.info;
745	memset(qp_info, 0, sizeof(*qp_info));
746	qp_info->mac_valid = true;
747	qp_info->cq_num_valid = true;
748	qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE;
749
750	cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
751	cqp_info->post_sq = 1;
752	cqp_info->in.u.qp_create.qp = &iwqp->sc_qp;
753	cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
754	status = irdma_handle_cqp_op(rf, cqp_request);
755	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
756
757	return status;
758	}
759
760	static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
761	struct irdma_qp_host_ctx_info *ctx_info)
762	{
763	struct irdma_device *iwdev = iwqp->iwdev;
764	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
765	struct irdma_roce_offload_info *roce_info;
766	struct irdma_udp_offload_info *udp_info;
767
768	udp_info = &iwqp->udp_info;
769	udp_info->snd_mss = ib_mtu_enum_to_int(mtu: ib_mtu_int_to_enum(mtu: iwdev->vsi.mtu));
770	udp_info->cwnd = iwdev->roce_cwnd;
771	udp_info->rexmit_thresh = 2;
772	udp_info->rnr_nak_thresh = 2;
773	udp_info->src_port = 0xc000;
774	udp_info->dst_port = ROCE_V2_UDP_DPORT;
775	roce_info = &iwqp->roce_info;
776	ether_addr_copy(dst: roce_info->mac_addr, src: iwdev->netdev->dev_addr);
777
778	roce_info->rd_en = true;
779	roce_info->wr_rdresp_en = true;
780	roce_info->bind_en = true;
781	roce_info->dcqcn_en = false;
782	roce_info->rtomin = 5;
783
784	roce_info->ack_credits = iwdev->roce_ackcreds;
785	roce_info->ird_size = dev->hw_attrs.max_hw_ird;
786	roce_info->ord_size = dev->hw_attrs.max_hw_ord;
787
788	if (!iwqp->user_mode) {
789	roce_info->priv_mode_en = true;
790	roce_info->fast_reg_en = true;
791	roce_info->udprivcq_en = true;
792	}
793	roce_info->roce_tver = 0;
794
795	ctx_info->roce_info = &iwqp->roce_info;
796	ctx_info->udp_info = &iwqp->udp_info;
797	irdma_sc_qp_setctx_roce(qp: &iwqp->sc_qp, qp_ctx: iwqp->host_ctx.va, info: ctx_info);
798	}
799
800	static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
801	struct irdma_qp_host_ctx_info *ctx_info)
802	{
803	struct irdma_device *iwdev = iwqp->iwdev;
804	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
805	struct irdma_iwarp_offload_info *iwarp_info;
806
807	iwarp_info = &iwqp->iwarp_info;
808	ether_addr_copy(dst: iwarp_info->mac_addr, src: iwdev->netdev->dev_addr);
809	iwarp_info->rd_en = true;
810	iwarp_info->wr_rdresp_en = true;
811	iwarp_info->bind_en = true;
812	iwarp_info->ecn_en = true;
813	iwarp_info->rtomin = 5;
814
815	if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
816	iwarp_info->ib_rd_en = true;
817	if (!iwqp->user_mode) {
818	iwarp_info->priv_mode_en = true;
819	iwarp_info->fast_reg_en = true;
820	}
821	iwarp_info->ddp_ver = 1;
822	iwarp_info->rdmap_ver = 1;
823
824	ctx_info->iwarp_info = &iwqp->iwarp_info;
825	ctx_info->iwarp_info_valid = true;
826	irdma_sc_qp_setctx(qp: &iwqp->sc_qp, qp_ctx: iwqp->host_ctx.va, info: ctx_info);
827	ctx_info->iwarp_info_valid = false;
828	}
829
830	static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr,
831	struct irdma_device *iwdev)
832	{
833	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
834	struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs;
835
836	if (init_attr->create_flags)
837	return -EOPNOTSUPP;
838
839	if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline ||
840	init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags ||
841	init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags ||
842	init_attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta ||
843	init_attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta)
844	return -EINVAL;
845
846	if (rdma_protocol_roce(device: &iwdev->ibdev, port_num: 1)) {
847	if (init_attr->qp_type != IB_QPT_RC &&
848	init_attr->qp_type != IB_QPT_UD &&
849	init_attr->qp_type != IB_QPT_GSI)
850	return -EOPNOTSUPP;
851	} else {
852	if (init_attr->qp_type != IB_QPT_RC)
853	return -EOPNOTSUPP;
854	}
855
856	return 0;
857	}
858
859	static void irdma_flush_worker(struct work_struct *work)
860	{
861	struct delayed_work *dwork = to_delayed_work(work);
862	struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush);
863
864	irdma_generate_flush_completions(iwqp);
865	}
866
867	/**
868	* irdma_create_qp - create qp
869	* @ibqp: ptr of qp
870	* @init_attr: attributes for qp
871	* @udata: user data for create qp
872	*/
873	static int irdma_create_qp(struct ib_qp *ibqp,
874	struct ib_qp_init_attr *init_attr,
875	struct ib_udata *udata)
876	{
877	#define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx)
878	#define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd)
879	struct ib_pd *ibpd = ibqp->pd;
880	struct irdma_pd *iwpd = to_iwpd(ibpd);
881	struct irdma_device *iwdev = to_iwdev(ibdev: ibpd->device);
882	struct irdma_pci_f *rf = iwdev->rf;
883	struct irdma_qp *iwqp = to_iwqp(ibqp);
884	struct irdma_create_qp_resp uresp = {};
885	u32 qp_num = 0;
886	int err_code;
887	struct irdma_sc_qp *qp;
888	struct irdma_sc_dev *dev = &rf->sc_dev;
889	struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs;
890	struct irdma_qp_init_info init_info = {};
891	struct irdma_qp_host_ctx_info *ctx_info;
892
893	err_code = irdma_validate_qp_attrs(init_attr, iwdev);
894	if (err_code)
895	return err_code;
896
897	if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN ||
898	udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN))
899	return -EINVAL;
900
901	init_info.vsi = &iwdev->vsi;
902	init_info.qp_uk_init_info.uk_attrs = uk_attrs;
903	init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr;
904	init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr;
905	init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
906	init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge;
907	init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data;
908
909	qp = &iwqp->sc_qp;
910	qp->qp_uk.back_qp = iwqp;
911	qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
912
913	iwqp->iwdev = iwdev;
914	iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE,
915	256);
916	iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev: dev->hw->device,
917	size: iwqp->q2_ctx_mem.size,
918	dma_handle: &iwqp->q2_ctx_mem.pa,
919	GFP_KERNEL);
920	if (!iwqp->q2_ctx_mem.va)
921	return -ENOMEM;
922
923	init_info.q2 = iwqp->q2_ctx_mem.va;
924	init_info.q2_pa = iwqp->q2_ctx_mem.pa;
925	init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE);
926	init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE;
927
928	if (init_attr->qp_type == IB_QPT_GSI)
929	qp_num = 1;
930	else
931	err_code = irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_qps, max_rsrc: rf->max_qp,
932	req_rsrc_num: &qp_num, next: &rf->next_qp);
933	if (err_code)
934	goto error;
935
936	iwqp->iwpd = iwpd;
937	iwqp->ibqp.qp_num = qp_num;
938	qp = &iwqp->sc_qp;
939	iwqp->iwscq = to_iwcq(ibcq: init_attr->send_cq);
940	iwqp->iwrcq = to_iwcq(ibcq: init_attr->recv_cq);
941	iwqp->host_ctx.va = init_info.host_ctx;
942	iwqp->host_ctx.pa = init_info.host_ctx_pa;
943	iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE;
944
945	init_info.pd = &iwpd->sc_pd;
946	init_info.qp_uk_init_info.qp_id = qp_num;
947	if (!rdma_protocol_roce(device: &iwdev->ibdev, port_num: 1))
948	init_info.qp_uk_init_info.first_sq_wq = 1;
949	iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;
950	init_waitqueue_head(&iwqp->waitq);
951	init_waitqueue_head(&iwqp->mod_qp_waitq);
952
953	if (udata) {
954	init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver;
955	err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, info: &init_info,
956	init_attr);
957	} else {
958	INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker);
959	init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER;
960	err_code = irdma_setup_kmode_qp(iwdev, iwqp, info: &init_info, init_attr);
961	}
962
963	if (err_code) {
964	ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n");
965	goto error;
966	}
967
968	if (rdma_protocol_roce(device: &iwdev->ibdev, port_num: 1)) {
969	if (init_attr->qp_type == IB_QPT_RC) {
970	init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC;
971	init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |
972	IRDMA_WRITE_WITH_IMM |
973	IRDMA_ROCE;
974	} else {
975	init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD;
976	init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |
977	IRDMA_ROCE;
978	}
979	} else {
980	init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP;
981	init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM;
982	}
983
984	if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1)
985	init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE;
986
987	err_code = irdma_sc_qp_init(qp, info: &init_info);
988	if (err_code) {
989	ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n");
990	goto error;
991	}
992
993	ctx_info = &iwqp->ctx_info;
994	ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
995	ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
996
997	if (rdma_protocol_roce(device: &iwdev->ibdev, port_num: 1))
998	irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info);
999	else
1000	irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info);
1001
1002	err_code = irdma_cqp_create_qp_cmd(iwqp);
1003	if (err_code)
1004	goto error;
1005
1006	refcount_set(r: &iwqp->refcnt, n: 1);
1007	spin_lock_init(&iwqp->lock);
1008	spin_lock_init(&iwqp->sc_qp.pfpdu.lock);
1009	iwqp->sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1010	rf->qp_table[qp_num] = iwqp;
1011
1012	if (rdma_protocol_roce(device: &iwdev->ibdev, port_num: 1)) {
1013	if (dev->ws_add(&iwdev->vsi, 0)) {
1014	irdma_cqp_qp_destroy_cmd(dev: &rf->sc_dev, qp: &iwqp->sc_qp);
1015	err_code = -EINVAL;
1016	goto error;
1017	}
1018
1019	irdma_qp_add_qos(qp: &iwqp->sc_qp);
1020	}
1021
1022	if (udata) {
1023	/* GEN_1 legacy support with libi40iw does not have expanded uresp struct */
1024	if (udata->outlen < sizeof(uresp)) {
1025	uresp.lsmm = 1;
1026	uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1;
1027	} else {
1028	if (rdma_protocol_iwarp(device: &iwdev->ibdev, port_num: 1))
1029	uresp.lsmm = 1;
1030	}
1031	uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size;
1032	uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size;
1033	uresp.qp_id = qp_num;
1034	uresp.qp_caps = qp->qp_uk.qp_caps;
1035
1036	err_code = ib_copy_to_udata(udata, src: &uresp,
1037	min(sizeof(uresp), udata->outlen));
1038	if (err_code) {
1039	ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n");
1040	irdma_destroy_qp(ibqp: &iwqp->ibqp, udata);
1041	return err_code;
1042	}
1043	}
1044
1045	init_completion(x: &iwqp->free_qp);
1046	return 0;
1047
1048	error:
1049	irdma_free_qp_rsrc(iwqp);
1050	return err_code;
1051	}
1052
1053	static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp)
1054	{
1055	int acc_flags = 0;
1056
1057	if (rdma_protocol_roce(device: iwqp->ibqp.device, port_num: 1)) {
1058	if (iwqp->roce_info.wr_rdresp_en) {
1059	acc_flags |= IB_ACCESS_LOCAL_WRITE;
1060	acc_flags |= IB_ACCESS_REMOTE_WRITE;
1061	}
1062	if (iwqp->roce_info.rd_en)
1063	acc_flags |= IB_ACCESS_REMOTE_READ;
1064	if (iwqp->roce_info.bind_en)
1065	acc_flags |= IB_ACCESS_MW_BIND;
1066	} else {
1067	if (iwqp->iwarp_info.wr_rdresp_en) {
1068	acc_flags |= IB_ACCESS_LOCAL_WRITE;
1069	acc_flags |= IB_ACCESS_REMOTE_WRITE;
1070	}
1071	if (iwqp->iwarp_info.rd_en)
1072	acc_flags |= IB_ACCESS_REMOTE_READ;
1073	if (iwqp->iwarp_info.bind_en)
1074	acc_flags |= IB_ACCESS_MW_BIND;
1075	}
1076	return acc_flags;
1077	}
1078
1079	/**
1080	* irdma_query_qp - query qp attributes
1081	* @ibqp: qp pointer
1082	* @attr: attributes pointer
1083	* @attr_mask: Not used
1084	* @init_attr: qp attributes to return
1085	*/
1086	static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1087	int attr_mask, struct ib_qp_init_attr *init_attr)
1088	{
1089	struct irdma_qp *iwqp = to_iwqp(ibqp);
1090	struct irdma_sc_qp *qp = &iwqp->sc_qp;
1091
1092	memset(attr, 0, sizeof(*attr));
1093	memset(init_attr, 0, sizeof(*init_attr));
1094
1095	attr->qp_state = iwqp->ibqp_state;
1096	attr->cur_qp_state = iwqp->ibqp_state;
1097	attr->cap.max_send_wr = iwqp->max_send_wr;
1098	attr->cap.max_recv_wr = iwqp->max_recv_wr;
1099	attr->cap.max_inline_data = qp->qp_uk.max_inline_data;
1100	attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt;
1101	attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt;
1102	attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp);
1103	attr->port_num = 1;
1104	if (rdma_protocol_roce(device: ibqp->device, port_num: 1)) {
1105	attr->path_mtu = ib_mtu_int_to_enum(mtu: iwqp->udp_info.snd_mss);
1106	attr->qkey = iwqp->roce_info.qkey;
1107	attr->rq_psn = iwqp->udp_info.epsn;
1108	attr->sq_psn = iwqp->udp_info.psn_nxt;
1109	attr->dest_qp_num = iwqp->roce_info.dest_qp;
1110	attr->pkey_index = iwqp->roce_info.p_key;
1111	attr->retry_cnt = iwqp->udp_info.rexmit_thresh;
1112	attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh;
1113	attr->max_rd_atomic = iwqp->roce_info.ord_size;
1114	attr->max_dest_rd_atomic = iwqp->roce_info.ird_size;
1115	}
1116
1117	init_attr->event_handler = iwqp->ibqp.event_handler;
1118	init_attr->qp_context = iwqp->ibqp.qp_context;
1119	init_attr->send_cq = iwqp->ibqp.send_cq;
1120	init_attr->recv_cq = iwqp->ibqp.recv_cq;
1121	init_attr->cap = attr->cap;
1122
1123	return 0;
1124	}
1125
1126	/**
1127	* irdma_query_pkey - Query partition key
1128	* @ibdev: device pointer from stack
1129	* @port: port number
1130	* @index: index of pkey
1131	* @pkey: pointer to store the pkey
1132	*/
1133	static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1134	u16 *pkey)
1135	{
1136	if (index >= IRDMA_PKEY_TBL_SZ)
1137	return -EINVAL;
1138
1139	*pkey = IRDMA_DEFAULT_PKEY;
1140	return 0;
1141	}
1142
1143	static u8 irdma_roce_get_vlan_prio(const struct ib_gid_attr *attr, u8 prio)
1144	{
1145	struct net_device *ndev;
1146
1147	rcu_read_lock();
1148	ndev = rcu_dereference(attr->ndev);
1149	if (!ndev)
1150	goto exit;
1151	if (is_vlan_dev(dev: ndev)) {
1152	u16 vlan_qos = vlan_dev_get_egress_qos_mask(dev: ndev, skprio: prio);
1153
1154	prio = (vlan_qos & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1155	}
1156	exit:
1157	rcu_read_unlock();
1158	return prio;
1159	}
1160
1161	static int irdma_wait_for_suspend(struct irdma_qp *iwqp)
1162	{
1163	if (!wait_event_timeout(iwqp->iwdev->suspend_wq,
1164	!iwqp->suspend_pending,
1165	msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS))) {
1166	iwqp->suspend_pending = false;
1167	ibdev_warn(ibdev: &iwqp->iwdev->ibdev,
1168	format: "modify_qp timed out waiting for suspend. qp_id = %d, last_ae = 0x%x\n",
1169	iwqp->ibqp.qp_num, iwqp->last_aeq);
1170	return -EBUSY;
1171	}
1172
1173	return 0;
1174	}
1175
1176	/**
1177	* irdma_modify_qp_roce - modify qp request
1178	* @ibqp: qp's pointer for modify
1179	* @attr: access attributes
1180	* @attr_mask: state mask
1181	* @udata: user data
1182	*/
1183	int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1184	int attr_mask, struct ib_udata *udata)
1185	{
1186	#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush)
1187	#define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid)
1188	struct irdma_pd *iwpd = to_iwpd(ibpd: ibqp->pd);
1189	struct irdma_qp *iwqp = to_iwqp(ibqp);
1190	struct irdma_device *iwdev = iwqp->iwdev;
1191	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
1192	struct irdma_qp_host_ctx_info *ctx_info;
1193	struct irdma_roce_offload_info *roce_info;
1194	struct irdma_udp_offload_info *udp_info;
1195	struct irdma_modify_qp_info info = {};
1196	struct irdma_modify_qp_resp uresp = {};
1197	struct irdma_modify_qp_req ureq = {};
1198	unsigned long flags;
1199	u8 issue_modify_qp = 0;
1200	int ret = 0;
1201
1202	ctx_info = &iwqp->ctx_info;
1203	roce_info = &iwqp->roce_info;
1204	udp_info = &iwqp->udp_info;
1205
1206	if (udata) {
1207	/* udata inlen/outlen can be 0 when supporting legacy libi40iw */
1208	if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) ||
1209	(udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN))
1210	return -EINVAL;
1211	}
1212
1213	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1214	return -EOPNOTSUPP;
1215
1216	if (attr_mask & IB_QP_DEST_QPN)
1217	roce_info->dest_qp = attr->dest_qp_num;
1218
1219	if (attr_mask & IB_QP_PKEY_INDEX) {
1220	ret = irdma_query_pkey(ibdev: ibqp->device, port: 0, index: attr->pkey_index,
1221	pkey: &roce_info->p_key);
1222	if (ret)
1223	return ret;
1224	}
1225
1226	if (attr_mask & IB_QP_QKEY)
1227	roce_info->qkey = attr->qkey;
1228
1229	if (attr_mask & IB_QP_PATH_MTU)
1230	udp_info->snd_mss = ib_mtu_enum_to_int(mtu: attr->path_mtu);
1231
1232	if (attr_mask & IB_QP_SQ_PSN) {
1233	udp_info->psn_nxt = attr->sq_psn;
1234	udp_info->lsn = 0xffff;
1235	udp_info->psn_una = attr->sq_psn;
1236	udp_info->psn_max = attr->sq_psn;
1237	}
1238
1239	if (attr_mask & IB_QP_RQ_PSN)
1240	udp_info->epsn = attr->rq_psn;
1241
1242	if (attr_mask & IB_QP_RNR_RETRY)
1243	udp_info->rnr_nak_thresh = attr->rnr_retry;
1244
1245	if (attr_mask & IB_QP_RETRY_CNT)
1246	udp_info->rexmit_thresh = attr->retry_cnt;
1247
1248	ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id;
1249
1250	if (attr_mask & IB_QP_AV) {
1251	struct irdma_av *av = &iwqp->roce_ah.av;
1252	const struct ib_gid_attr *sgid_attr =
1253	attr->ah_attr.grh.sgid_attr;
1254	u16 vlan_id = VLAN_N_VID;
1255	u32 local_ip[4];
1256
1257	memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah));
1258	if (attr->ah_attr.ah_flags & IB_AH_GRH) {
1259	udp_info->ttl = attr->ah_attr.grh.hop_limit;
1260	udp_info->flow_label = attr->ah_attr.grh.flow_label;
1261	udp_info->tos = attr->ah_attr.grh.traffic_class;
1262	udp_info->src_port =
1263	rdma_get_udp_sport(fl: udp_info->flow_label,
1264	lqpn: ibqp->qp_num,
1265	rqpn: roce_info->dest_qp);
1266	irdma_qp_rem_qos(qp: &iwqp->sc_qp);
1267	dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri);
1268	if (iwqp->sc_qp.vsi->dscp_mode)
1269	ctx_info->user_pri =
1270	iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(tos: udp_info->tos)];
1271	else
1272	ctx_info->user_pri = rt_tos2priority(tos: udp_info->tos);
1273	}
1274	ret = rdma_read_gid_l2_fields(attr: sgid_attr, vlan_id: &vlan_id,
1275	smac: ctx_info->roce_info->mac_addr);
1276	if (ret)
1277	return ret;
1278	ctx_info->user_pri = irdma_roce_get_vlan_prio(attr: sgid_attr,
1279	prio: ctx_info->user_pri);
1280	if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri))
1281	return -ENOMEM;
1282	iwqp->sc_qp.user_pri = ctx_info->user_pri;
1283	irdma_qp_add_qos(qp: &iwqp->sc_qp);
1284
1285	if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
1286	vlan_id = 0;
1287	if (vlan_id < VLAN_N_VID) {
1288	udp_info->insert_vlan_tag = true;
1289	udp_info->vlan_tag = vlan_id |
1290	ctx_info->user_pri << VLAN_PRIO_SHIFT;
1291	} else {
1292	udp_info->insert_vlan_tag = false;
1293	}
1294
1295	av->attrs = attr->ah_attr;
1296	rdma_gid2ip(out: (struct sockaddr *)&av->sgid_addr, gid: &sgid_attr->gid);
1297	rdma_gid2ip(out: (struct sockaddr *)&av->dgid_addr, gid: &attr->ah_attr.grh.dgid);
1298	av->net_type = rdma_gid_attr_network_type(attr: sgid_attr);
1299	if (av->net_type == RDMA_NETWORK_IPV6) {
1300	__be32 *daddr =
1301	av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;
1302	__be32 *saddr =
1303	av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32;
1304
1305	irdma_copy_ip_ntohl(dst: &udp_info->dest_ip_addr[0], src: daddr);
1306	irdma_copy_ip_ntohl(dst: &udp_info->local_ipaddr[0], src: saddr);
1307
1308	udp_info->ipv4 = false;
1309	irdma_copy_ip_ntohl(dst: local_ip, src: daddr);
1310
1311	} else if (av->net_type == RDMA_NETWORK_IPV4) {
1312	__be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr;
1313	__be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr;
1314
1315	local_ip[0] = ntohl(daddr);
1316
1317	udp_info->ipv4 = true;
1318	udp_info->dest_ip_addr[0] = 0;
1319	udp_info->dest_ip_addr[1] = 0;
1320	udp_info->dest_ip_addr[2] = 0;
1321	udp_info->dest_ip_addr[3] = local_ip[0];
1322
1323	udp_info->local_ipaddr[0] = 0;
1324	udp_info->local_ipaddr[1] = 0;
1325	udp_info->local_ipaddr[2] = 0;
1326	udp_info->local_ipaddr[3] = ntohl(saddr);
1327	}
1328	udp_info->arp_idx =
1329	irdma_add_arp(rf: iwdev->rf, ip: local_ip, ipv4: udp_info->ipv4,
1330	mac: attr->ah_attr.roce.dmac);
1331	}
1332
1333	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1334	if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) {
1335	ibdev_err(ibdev: &iwdev->ibdev,
1336	format: "rd_atomic = %d, above max_hw_ord=%d\n",
1337	attr->max_rd_atomic,
1338	dev->hw_attrs.max_hw_ord);
1339	return -EINVAL;
1340	}
1341	if (attr->max_rd_atomic)
1342	roce_info->ord_size = attr->max_rd_atomic;
1343	info.ord_valid = true;
1344	}
1345
1346	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1347	if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) {
1348	ibdev_err(ibdev: &iwdev->ibdev,
1349	format: "rd_atomic = %d, above max_hw_ird=%d\n",
1350	attr->max_rd_atomic,
1351	dev->hw_attrs.max_hw_ird);
1352	return -EINVAL;
1353	}
1354	if (attr->max_dest_rd_atomic)
1355	roce_info->ird_size = attr->max_dest_rd_atomic;
1356	}
1357
1358	if (attr_mask & IB_QP_ACCESS_FLAGS) {
1359	if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE)
1360	roce_info->wr_rdresp_en = true;
1361	if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
1362	roce_info->wr_rdresp_en = true;
1363	if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
1364	roce_info->rd_en = true;
1365	}
1366
1367	wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend));
1368
1369	ibdev_dbg(&iwdev->ibdev,
1370	"VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n",
1371	__builtin_return_address(0), ibqp->qp_num, attr->qp_state,
1372	iwqp->ibqp_state, iwqp->iwarp_state, attr_mask);
1373
1374	spin_lock_irqsave(&iwqp->lock, flags);
1375	if (attr_mask & IB_QP_STATE) {
1376	if (!ib_modify_qp_is_ok(cur_state: iwqp->ibqp_state, next_state: attr->qp_state,
1377	type: iwqp->ibqp.qp_type, mask: attr_mask)) {
1378	ibdev_warn(ibdev: &iwdev->ibdev, format: "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n",
1379	iwqp->ibqp.qp_num, iwqp->ibqp_state,
1380	attr->qp_state);
1381	ret = -EINVAL;
1382	goto exit;
1383	}
1384	info.curr_iwarp_state = iwqp->iwarp_state;
1385
1386	switch (attr->qp_state) {
1387	case IB_QPS_INIT:
1388	if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) {
1389	ret = -EINVAL;
1390	goto exit;
1391	}
1392
1393	if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) {
1394	info.next_iwarp_state = IRDMA_QP_STATE_IDLE;
1395	issue_modify_qp = 1;
1396	}
1397	break;
1398	case IB_QPS_RTR:
1399	if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) {
1400	ret = -EINVAL;
1401	goto exit;
1402	}
1403	info.arp_cache_idx_valid = true;
1404	info.cq_num_valid = true;
1405	info.next_iwarp_state = IRDMA_QP_STATE_RTR;
1406	issue_modify_qp = 1;
1407	break;
1408	case IB_QPS_RTS:
1409	if (iwqp->ibqp_state < IB_QPS_RTR ||
1410	iwqp->ibqp_state == IB_QPS_ERR) {
1411	ret = -EINVAL;
1412	goto exit;
1413	}
1414
1415	info.arp_cache_idx_valid = true;
1416	info.cq_num_valid = true;
1417	info.ord_valid = true;
1418	info.next_iwarp_state = IRDMA_QP_STATE_RTS;
1419	issue_modify_qp = 1;
1420	if (iwdev->push_mode && udata &&
1421	iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX &&
1422	dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1423	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1424	irdma_alloc_push_page(iwqp);
1425	spin_lock_irqsave(&iwqp->lock, flags);
1426	}
1427	break;
1428	case IB_QPS_SQD:
1429	if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD)
1430	goto exit;
1431
1432	if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) {
1433	ret = -EINVAL;
1434	goto exit;
1435	}
1436
1437	info.next_iwarp_state = IRDMA_QP_STATE_SQD;
1438	issue_modify_qp = 1;
1439	iwqp->suspend_pending = true;
1440	break;
1441	case IB_QPS_SQE:
1442	case IB_QPS_ERR:
1443	case IB_QPS_RESET:
1444	if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {
1445	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1446	if (udata && udata->inlen) {
1447	if (ib_copy_from_udata(dest: &ureq, udata,
1448	min(sizeof(ureq), udata->inlen)))
1449	return -EINVAL;
1450
1451	irdma_flush_wqes(iwqp,
1452	flush_mask: (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) |
1453	(ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) |
1454	IRDMA_REFLUSH);
1455	}
1456	return 0;
1457	}
1458
1459	info.next_iwarp_state = IRDMA_QP_STATE_ERROR;
1460	issue_modify_qp = 1;
1461	break;
1462	default:
1463	ret = -EINVAL;
1464	goto exit;
1465	}
1466
1467	iwqp->ibqp_state = attr->qp_state;
1468	}
1469
1470	ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
1471	ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
1472	irdma_sc_qp_setctx_roce(qp: &iwqp->sc_qp, qp_ctx: iwqp->host_ctx.va, info: ctx_info);
1473	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1474
1475	if (attr_mask & IB_QP_STATE) {
1476	if (issue_modify_qp) {
1477	ctx_info->rem_endpoint_idx = udp_info->arp_idx;
1478	if (irdma_hw_modify_qp(iwdev, iwqp, info: &info, wait: true))
1479	return -EINVAL;
1480	if (info.next_iwarp_state == IRDMA_QP_STATE_SQD) {
1481	ret = irdma_wait_for_suspend(iwqp);
1482	if (ret)
1483	return ret;
1484	}
1485	spin_lock_irqsave(&iwqp->lock, flags);
1486	if (iwqp->iwarp_state == info.curr_iwarp_state) {
1487	iwqp->iwarp_state = info.next_iwarp_state;
1488	iwqp->ibqp_state = attr->qp_state;
1489	}
1490	if (iwqp->ibqp_state > IB_QPS_RTS &&
1491	!iwqp->flush_issued) {
1492	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1493	irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ |
1494	IRDMA_FLUSH_RQ |
1495	IRDMA_FLUSH_WAIT);
1496	iwqp->flush_issued = 1;
1497	} else {
1498	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1499	}
1500	} else {
1501	iwqp->ibqp_state = attr->qp_state;
1502	}
1503	if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1504	struct irdma_ucontext *ucontext;
1505
1506	ucontext = rdma_udata_to_drv_context(udata,
1507	struct irdma_ucontext, ibucontext);
1508	if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&
1509	!iwqp->push_wqe_mmap_entry &&
1510	!irdma_setup_push_mmap_entries(ucontext, iwqp,
1511	push_wqe_mmap_key: &uresp.push_wqe_mmap_key, push_db_mmap_key: &uresp.push_db_mmap_key)) {
1512	uresp.push_valid = 1;
1513	uresp.push_offset = iwqp->sc_qp.push_offset;
1514	}
1515	ret = ib_copy_to_udata(udata, src: &uresp, min(sizeof(uresp),
1516	udata->outlen));
1517	if (ret) {
1518	irdma_remove_push_mmap_entries(iwqp);
1519	ibdev_dbg(&iwdev->ibdev,
1520	"VERBS: copy_to_udata failed\n");
1521	return ret;
1522	}
1523	}
1524	}
1525
1526	return 0;
1527	exit:
1528	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1529
1530	return ret;
1531	}
1532
1533	/**
1534	* irdma_modify_qp - modify qp request
1535	* @ibqp: qp's pointer for modify
1536	* @attr: access attributes
1537	* @attr_mask: state mask
1538	* @udata: user data
1539	*/
1540	int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1541	struct ib_udata *udata)
1542	{
1543	#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush)
1544	#define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid)
1545	struct irdma_qp *iwqp = to_iwqp(ibqp);
1546	struct irdma_device *iwdev = iwqp->iwdev;
1547	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
1548	struct irdma_qp_host_ctx_info *ctx_info;
1549	struct irdma_tcp_offload_info *tcp_info;
1550	struct irdma_iwarp_offload_info *offload_info;
1551	struct irdma_modify_qp_info info = {};
1552	struct irdma_modify_qp_resp uresp = {};
1553	struct irdma_modify_qp_req ureq = {};
1554	u8 issue_modify_qp = 0;
1555	u8 dont_wait = 0;
1556	int err;
1557	unsigned long flags;
1558
1559	if (udata) {
1560	/* udata inlen/outlen can be 0 when supporting legacy libi40iw */
1561	if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) ||
1562	(udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN))
1563	return -EINVAL;
1564	}
1565
1566	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1567	return -EOPNOTSUPP;
1568
1569	ctx_info = &iwqp->ctx_info;
1570	offload_info = &iwqp->iwarp_info;
1571	tcp_info = &iwqp->tcp_info;
1572	wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend));
1573	ibdev_dbg(&iwdev->ibdev,
1574	"VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n",
1575	__builtin_return_address(0), ibqp->qp_num, attr->qp_state,
1576	iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq,
1577	iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask);
1578
1579	spin_lock_irqsave(&iwqp->lock, flags);
1580	if (attr_mask & IB_QP_STATE) {
1581	info.curr_iwarp_state = iwqp->iwarp_state;
1582	switch (attr->qp_state) {
1583	case IB_QPS_INIT:
1584	case IB_QPS_RTR:
1585	if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) {
1586	err = -EINVAL;
1587	goto exit;
1588	}
1589
1590	if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) {
1591	info.next_iwarp_state = IRDMA_QP_STATE_IDLE;
1592	issue_modify_qp = 1;
1593	}
1594	if (iwdev->push_mode && udata &&
1595	iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX &&
1596	dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1597	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1598	irdma_alloc_push_page(iwqp);
1599	spin_lock_irqsave(&iwqp->lock, flags);
1600	}
1601	break;
1602	case IB_QPS_RTS:
1603	if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS ||
1604	!iwqp->cm_id) {
1605	err = -EINVAL;
1606	goto exit;
1607	}
1608
1609	issue_modify_qp = 1;
1610	iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
1611	iwqp->hte_added = 1;
1612	info.next_iwarp_state = IRDMA_QP_STATE_RTS;
1613	info.tcp_ctx_valid = true;
1614	info.ord_valid = true;
1615	info.arp_cache_idx_valid = true;
1616	info.cq_num_valid = true;
1617	break;
1618	case IB_QPS_SQD:
1619	if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) {
1620	err = 0;
1621	goto exit;
1622	}
1623
1624	if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING ||
1625	iwqp->iwarp_state < IRDMA_QP_STATE_RTS) {
1626	err = 0;
1627	goto exit;
1628	}
1629
1630	if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) {
1631	err = -EINVAL;
1632	goto exit;
1633	}
1634
1635	info.next_iwarp_state = IRDMA_QP_STATE_CLOSING;
1636	issue_modify_qp = 1;
1637	break;
1638	case IB_QPS_SQE:
1639	if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) {
1640	err = -EINVAL;
1641	goto exit;
1642	}
1643
1644	info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE;
1645	issue_modify_qp = 1;
1646	break;
1647	case IB_QPS_ERR:
1648	case IB_QPS_RESET:
1649	if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {
1650	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1651	if (udata && udata->inlen) {
1652	if (ib_copy_from_udata(dest: &ureq, udata,
1653	min(sizeof(ureq), udata->inlen)))
1654	return -EINVAL;
1655
1656	irdma_flush_wqes(iwqp,
1657	flush_mask: (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) |
1658	(ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) |
1659	IRDMA_REFLUSH);
1660	}
1661	return 0;
1662	}
1663
1664	if (iwqp->sc_qp.term_flags) {
1665	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1666	irdma_terminate_del_timer(qp: &iwqp->sc_qp);
1667	spin_lock_irqsave(&iwqp->lock, flags);
1668	}
1669	info.next_iwarp_state = IRDMA_QP_STATE_ERROR;
1670	if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED &&
1671	iwdev->iw_status &&
1672	iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT)
1673	info.reset_tcp_conn = true;
1674	else
1675	dont_wait = 1;
1676
1677	issue_modify_qp = 1;
1678	info.next_iwarp_state = IRDMA_QP_STATE_ERROR;
1679	break;
1680	default:
1681	err = -EINVAL;
1682	goto exit;
1683	}
1684
1685	iwqp->ibqp_state = attr->qp_state;
1686	}
1687	if (attr_mask & IB_QP_ACCESS_FLAGS) {
1688	ctx_info->iwarp_info_valid = true;
1689	if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE)
1690	offload_info->wr_rdresp_en = true;
1691	if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
1692	offload_info->wr_rdresp_en = true;
1693	if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
1694	offload_info->rd_en = true;
1695	}
1696
1697	if (ctx_info->iwarp_info_valid) {
1698	ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
1699	ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
1700	irdma_sc_qp_setctx(qp: &iwqp->sc_qp, qp_ctx: iwqp->host_ctx.va, info: ctx_info);
1701	}
1702	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1703
1704	if (attr_mask & IB_QP_STATE) {
1705	if (issue_modify_qp) {
1706	ctx_info->rem_endpoint_idx = tcp_info->arp_idx;
1707	if (irdma_hw_modify_qp(iwdev, iwqp, info: &info, wait: true))
1708	return -EINVAL;
1709	}
1710
1711	spin_lock_irqsave(&iwqp->lock, flags);
1712	if (iwqp->iwarp_state == info.curr_iwarp_state) {
1713	iwqp->iwarp_state = info.next_iwarp_state;
1714	iwqp->ibqp_state = attr->qp_state;
1715	}
1716	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1717	}
1718
1719	if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) {
1720	if (dont_wait) {
1721	if (iwqp->hw_tcp_state) {
1722	spin_lock_irqsave(&iwqp->lock, flags);
1723	iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;
1724	iwqp->last_aeq = IRDMA_AE_RESET_SENT;
1725	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1726	}
1727	irdma_cm_disconn(qp: iwqp);
1728	} else {
1729	int close_timer_started;
1730
1731	spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags);
1732
1733	if (iwqp->cm_node) {
1734	refcount_inc(r: &iwqp->cm_node->refcnt);
1735	spin_unlock_irqrestore(lock: &iwdev->cm_core.ht_lock, flags);
1736	close_timer_started = atomic_inc_return(v: &iwqp->close_timer_started);
1737	if (iwqp->cm_id && close_timer_started == 1)
1738	irdma_schedule_cm_timer(cm_node: iwqp->cm_node,
1739	sqbuf: (struct irdma_puda_buf *)iwqp,
1740	type: IRDMA_TIMER_TYPE_CLOSE, send_retrans: 1, close_when_complete: 0);
1741
1742	irdma_rem_ref_cm_node(cm_node: iwqp->cm_node);
1743	} else {
1744	spin_unlock_irqrestore(lock: &iwdev->cm_core.ht_lock, flags);
1745	}
1746	}
1747	}
1748	if (attr_mask & IB_QP_STATE && udata && udata->outlen &&
1749	dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1750	struct irdma_ucontext *ucontext;
1751
1752	ucontext = rdma_udata_to_drv_context(udata,
1753	struct irdma_ucontext, ibucontext);
1754	if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&
1755	!iwqp->push_wqe_mmap_entry &&
1756	!irdma_setup_push_mmap_entries(ucontext, iwqp,
1757	push_wqe_mmap_key: &uresp.push_wqe_mmap_key, push_db_mmap_key: &uresp.push_db_mmap_key)) {
1758	uresp.push_valid = 1;
1759	uresp.push_offset = iwqp->sc_qp.push_offset;
1760	}
1761
1762	err = ib_copy_to_udata(udata, src: &uresp, min(sizeof(uresp),
1763	udata->outlen));
1764	if (err) {
1765	irdma_remove_push_mmap_entries(iwqp);
1766	ibdev_dbg(&iwdev->ibdev,
1767	"VERBS: copy_to_udata failed\n");
1768	return err;
1769	}
1770	}
1771
1772	return 0;
1773	exit:
1774	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
1775
1776	return err;
1777	}
1778
1779	/**
1780	* irdma_cq_free_rsrc - free up resources for cq
1781	* @rf: RDMA PCI function
1782	* @iwcq: cq ptr
1783	*/
1784	static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq)
1785	{
1786	struct irdma_sc_cq *cq = &iwcq->sc_cq;
1787
1788	if (!iwcq->user_mode) {
1789	dma_free_coherent(dev: rf->sc_dev.hw->device, size: iwcq->kmem.size,
1790	cpu_addr: iwcq->kmem.va, dma_handle: iwcq->kmem.pa);
1791	iwcq->kmem.va = NULL;
1792	dma_free_coherent(dev: rf->sc_dev.hw->device,
1793	size: iwcq->kmem_shadow.size,
1794	cpu_addr: iwcq->kmem_shadow.va, dma_handle: iwcq->kmem_shadow.pa);
1795	iwcq->kmem_shadow.va = NULL;
1796	}
1797
1798	irdma_free_rsrc(rf, rsrc_array: rf->allocated_cqs, rsrc_num: cq->cq_uk.cq_id);
1799	}
1800
1801	/**
1802	* irdma_free_cqbuf - worker to free a cq buffer
1803	* @work: provides access to the cq buffer to free
1804	*/
1805	static void irdma_free_cqbuf(struct work_struct *work)
1806	{
1807	struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work);
1808
1809	dma_free_coherent(dev: cq_buf->hw->device, size: cq_buf->kmem_buf.size,
1810	cpu_addr: cq_buf->kmem_buf.va, dma_handle: cq_buf->kmem_buf.pa);
1811	cq_buf->kmem_buf.va = NULL;
1812	kfree(objp: cq_buf);
1813	}
1814
1815	/**
1816	* irdma_process_resize_list - remove resized cq buffers from the resize_list
1817	* @iwcq: cq which owns the resize_list
1818	* @iwdev: irdma device
1819	* @lcqe_buf: the buffer where the last cqe is received
1820	*/
1821	static int irdma_process_resize_list(struct irdma_cq *iwcq,
1822	struct irdma_device *iwdev,
1823	struct irdma_cq_buf *lcqe_buf)
1824	{
1825	struct list_head *tmp_node, *list_node;
1826	struct irdma_cq_buf *cq_buf;
1827	int cnt = 0;
1828
1829	list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) {
1830	cq_buf = list_entry(list_node, struct irdma_cq_buf, list);
1831	if (cq_buf == lcqe_buf)
1832	return cnt;
1833
1834	list_del(entry: &cq_buf->list);
1835	queue_work(wq: iwdev->cleanup_wq, work: &cq_buf->work);
1836	cnt++;
1837	}
1838
1839	return cnt;
1840	}
1841
1842	/**
1843	* irdma_destroy_cq - destroy cq
1844	* @ib_cq: cq pointer
1845	* @udata: user data
1846	*/
1847	static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
1848	{
1849	struct irdma_device *iwdev = to_iwdev(ibdev: ib_cq->device);
1850	struct irdma_cq *iwcq = to_iwcq(ibcq: ib_cq);
1851	struct irdma_sc_cq *cq = &iwcq->sc_cq;
1852	struct irdma_sc_dev *dev = cq->dev;
1853	struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id];
1854	struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq);
1855	unsigned long flags;
1856
1857	spin_lock_irqsave(&iwcq->lock, flags);
1858	if (!list_empty(head: &iwcq->cmpl_generated))
1859	irdma_remove_cmpls_list(iwcq);
1860	if (!list_empty(head: &iwcq->resize_list))
1861	irdma_process_resize_list(iwcq, iwdev, NULL);
1862	spin_unlock_irqrestore(lock: &iwcq->lock, flags);
1863
1864	irdma_cq_rem_ref(ibcq: ib_cq);
1865	wait_for_completion(&iwcq->free_cq);
1866
1867	irdma_cq_wq_destroy(rf: iwdev->rf, cq);
1868
1869	spin_lock_irqsave(&iwceq->ce_lock, flags);
1870	irdma_sc_cleanup_ceqes(cq, ceq);
1871	spin_unlock_irqrestore(lock: &iwceq->ce_lock, flags);
1872	irdma_cq_free_rsrc(rf: iwdev->rf, iwcq);
1873
1874	return 0;
1875	}
1876
1877	/**
1878	* irdma_resize_cq - resize cq
1879	* @ibcq: cq to be resized
1880	* @entries: desired cq size
1881	* @udata: user data
1882	*/
1883	static int irdma_resize_cq(struct ib_cq *ibcq, int entries,
1884	struct ib_udata *udata)
1885	{
1886	#define IRDMA_RESIZE_CQ_MIN_REQ_LEN offsetofend(struct irdma_resize_cq_req, user_cq_buffer)
1887	struct irdma_cq *iwcq = to_iwcq(ibcq);
1888	struct irdma_sc_dev *dev = iwcq->sc_cq.dev;
1889	struct irdma_cqp_request *cqp_request;
1890	struct cqp_cmds_info *cqp_info;
1891	struct irdma_modify_cq_info *m_info;
1892	struct irdma_modify_cq_info info = {};
1893	struct irdma_dma_mem kmem_buf;
1894	struct irdma_cq_mr *cqmr_buf;
1895	struct irdma_pbl *iwpbl_buf;
1896	struct irdma_device *iwdev;
1897	struct irdma_pci_f *rf;
1898	struct irdma_cq_buf *cq_buf = NULL;
1899	unsigned long flags;
1900	int ret;
1901
1902	iwdev = to_iwdev(ibdev: ibcq->device);
1903	rf = iwdev->rf;
1904
1905	if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
1906	IRDMA_FEATURE_CQ_RESIZE))
1907	return -EOPNOTSUPP;
1908
1909	if (udata && udata->inlen < IRDMA_RESIZE_CQ_MIN_REQ_LEN)
1910	return -EINVAL;
1911
1912	if (entries > rf->max_cqe)
1913	return -EINVAL;
1914
1915	if (!iwcq->user_mode) {
1916	entries++;
1917	if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1918	entries *= 2;
1919	}
1920
1921	info.cq_size = max(entries, 4);
1922
1923	if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1)
1924	return 0;
1925
1926	if (udata) {
1927	struct irdma_resize_cq_req req = {};
1928	struct irdma_ucontext *ucontext =
1929	rdma_udata_to_drv_context(udata, struct irdma_ucontext,
1930	ibucontext);
1931
1932	/* CQ resize not supported with legacy GEN_1 libi40iw */
1933	if (ucontext->legacy_mode)
1934	return -EOPNOTSUPP;
1935
1936	if (ib_copy_from_udata(dest: &req, udata,
1937	min(sizeof(req), udata->inlen)))
1938	return -EINVAL;
1939
1940	spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1941	iwpbl_buf = irdma_get_pbl(va: (unsigned long)req.user_cq_buffer,
1942	pbl_list: &ucontext->cq_reg_mem_list);
1943	spin_unlock_irqrestore(lock: &ucontext->cq_reg_mem_list_lock, flags);
1944
1945	if (!iwpbl_buf)
1946	return -ENOMEM;
1947
1948	cqmr_buf = &iwpbl_buf->cq_mr;
1949	if (iwpbl_buf->pbl_allocated) {
1950	info.virtual_map = true;
1951	info.pbl_chunk_size = 1;
1952	info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx;
1953	} else {
1954	info.cq_pa = cqmr_buf->cq_pbl.addr;
1955	}
1956	} else {
1957	/* Kmode CQ resize */
1958	int rsize;
1959
1960	rsize = info.cq_size * sizeof(struct irdma_cqe);
1961	kmem_buf.size = ALIGN(round_up(rsize, 256), 256);
1962	kmem_buf.va = dma_alloc_coherent(dev: dev->hw->device,
1963	size: kmem_buf.size, dma_handle: &kmem_buf.pa,
1964	GFP_KERNEL);
1965	if (!kmem_buf.va)
1966	return -ENOMEM;
1967
1968	info.cq_base = kmem_buf.va;
1969	info.cq_pa = kmem_buf.pa;
1970	cq_buf = kzalloc(size: sizeof(*cq_buf), GFP_KERNEL);
1971	if (!cq_buf) {
1972	ret = -ENOMEM;
1973	goto error;
1974	}
1975	}
1976
1977	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true);
1978	if (!cqp_request) {
1979	ret = -ENOMEM;
1980	goto error;
1981	}
1982
1983	info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold;
1984	info.cq_resize = true;
1985
1986	cqp_info = &cqp_request->info;
1987	m_info = &cqp_info->in.u.cq_modify.info;
1988	memcpy(m_info, &info, sizeof(*m_info));
1989
1990	cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY;
1991	cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq;
1992	cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request;
1993	cqp_info->post_sq = 1;
1994	ret = irdma_handle_cqp_op(rf, cqp_request);
1995	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
1996	if (ret)
1997	goto error;
1998
1999	spin_lock_irqsave(&iwcq->lock, flags);
2000	if (cq_buf) {
2001	cq_buf->kmem_buf = iwcq->kmem;
2002	cq_buf->hw = dev->hw;
2003	memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk));
2004	INIT_WORK(&cq_buf->work, irdma_free_cqbuf);
2005	list_add_tail(new: &cq_buf->list, head: &iwcq->resize_list);
2006	iwcq->kmem = kmem_buf;
2007	}
2008
2009	irdma_sc_cq_resize(cq: &iwcq->sc_cq, info: &info);
2010	ibcq->cqe = info.cq_size - 1;
2011	spin_unlock_irqrestore(lock: &iwcq->lock, flags);
2012
2013	return 0;
2014	error:
2015	if (!udata) {
2016	dma_free_coherent(dev: dev->hw->device, size: kmem_buf.size, cpu_addr: kmem_buf.va,
2017	dma_handle: kmem_buf.pa);
2018	kmem_buf.va = NULL;
2019	}
2020	kfree(objp: cq_buf);
2021
2022	return ret;
2023	}
2024
2025	static inline int cq_validate_flags(u32 flags, u8 hw_rev)
2026	{
2027	/* GEN1 does not support CQ create flags */
2028	if (hw_rev == IRDMA_GEN_1)
2029	return flags ? -EOPNOTSUPP : 0;
2030
2031	return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0;
2032	}
2033
2034	/**
2035	* irdma_create_cq - create cq
2036	* @ibcq: CQ allocated
2037	* @attr: attributes for cq
2038	* @udata: user data
2039	*/
2040	static int irdma_create_cq(struct ib_cq *ibcq,
2041	const struct ib_cq_init_attr *attr,
2042	struct ib_udata *udata)
2043	{
2044	#define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf)
2045	#define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size)
2046	struct ib_device *ibdev = ibcq->device;
2047	struct irdma_device *iwdev = to_iwdev(ibdev);
2048	struct irdma_pci_f *rf = iwdev->rf;
2049	struct irdma_cq *iwcq = to_iwcq(ibcq);
2050	u32 cq_num = 0;
2051	struct irdma_sc_cq *cq;
2052	struct irdma_sc_dev *dev = &rf->sc_dev;
2053	struct irdma_cq_init_info info = {};
2054	struct irdma_cqp_request *cqp_request;
2055	struct cqp_cmds_info *cqp_info;
2056	struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
2057	unsigned long flags;
2058	int err_code;
2059	int entries = attr->cqe;
2060
2061	err_code = cq_validate_flags(flags: attr->flags, hw_rev: dev->hw_attrs.uk_attrs.hw_rev);
2062	if (err_code)
2063	return err_code;
2064
2065	if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN ||
2066	udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN))
2067	return -EINVAL;
2068
2069	err_code = irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_cqs, max_rsrc: rf->max_cq, req_rsrc_num: &cq_num,
2070	next: &rf->next_cq);
2071	if (err_code)
2072	return err_code;
2073
2074	cq = &iwcq->sc_cq;
2075	cq->back_cq = iwcq;
2076	refcount_set(r: &iwcq->refcnt, n: 1);
2077	spin_lock_init(&iwcq->lock);
2078	INIT_LIST_HEAD(list: &iwcq->resize_list);
2079	INIT_LIST_HEAD(list: &iwcq->cmpl_generated);
2080	info.dev = dev;
2081	ukinfo->cq_size = max(entries, 4);
2082	ukinfo->cq_id = cq_num;
2083	iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
2084	if (attr->comp_vector < rf->ceqs_count)
2085	info.ceq_id = attr->comp_vector;
2086	info.ceq_id_valid = true;
2087	info.ceqe_mask = 1;
2088	info.type = IRDMA_CQ_TYPE_IWARP;
2089	info.vsi = &iwdev->vsi;
2090
2091	if (udata) {
2092	struct irdma_ucontext *ucontext;
2093	struct irdma_create_cq_req req = {};
2094	struct irdma_cq_mr *cqmr;
2095	struct irdma_pbl *iwpbl;
2096	struct irdma_pbl *iwpbl_shadow;
2097	struct irdma_cq_mr *cqmr_shadow;
2098
2099	iwcq->user_mode = true;
2100	ucontext =
2101	rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2102	ibucontext);
2103	if (ib_copy_from_udata(dest: &req, udata,
2104	min(sizeof(req), udata->inlen))) {
2105	err_code = -EFAULT;
2106	goto cq_free_rsrc;
2107	}
2108
2109	spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
2110	iwpbl = irdma_get_pbl(va: (unsigned long)req.user_cq_buf,
2111	pbl_list: &ucontext->cq_reg_mem_list);
2112	spin_unlock_irqrestore(lock: &ucontext->cq_reg_mem_list_lock, flags);
2113	if (!iwpbl) {
2114	err_code = -EPROTO;
2115	goto cq_free_rsrc;
2116	}
2117
2118	iwcq->iwpbl = iwpbl;
2119	iwcq->cq_mem_size = 0;
2120	cqmr = &iwpbl->cq_mr;
2121
2122	if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
2123	IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) {
2124	spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
2125	iwpbl_shadow = irdma_get_pbl(
2126	va: (unsigned long)req.user_shadow_area,
2127	pbl_list: &ucontext->cq_reg_mem_list);
2128	spin_unlock_irqrestore(lock: &ucontext->cq_reg_mem_list_lock, flags);
2129
2130	if (!iwpbl_shadow) {
2131	err_code = -EPROTO;
2132	goto cq_free_rsrc;
2133	}
2134	iwcq->iwpbl_shadow = iwpbl_shadow;
2135	cqmr_shadow = &iwpbl_shadow->cq_mr;
2136	info.shadow_area_pa = cqmr_shadow->cq_pbl.addr;
2137	cqmr->split = true;
2138	} else {
2139	info.shadow_area_pa = cqmr->shadow;
2140	}
2141	if (iwpbl->pbl_allocated) {
2142	info.virtual_map = true;
2143	info.pbl_chunk_size = 1;
2144	info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
2145	} else {
2146	info.cq_base_pa = cqmr->cq_pbl.addr;
2147	}
2148	} else {
2149	/* Kmode allocations */
2150	int rsize;
2151
2152	if (entries < 1 || entries > rf->max_cqe) {
2153	err_code = -EINVAL;
2154	goto cq_free_rsrc;
2155	}
2156
2157	entries++;
2158	if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
2159	entries *= 2;
2160	ukinfo->cq_size = entries;
2161
2162	rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe);
2163	iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256);
2164	iwcq->kmem.va = dma_alloc_coherent(dev: dev->hw->device,
2165	size: iwcq->kmem.size,
2166	dma_handle: &iwcq->kmem.pa, GFP_KERNEL);
2167	if (!iwcq->kmem.va) {
2168	err_code = -ENOMEM;
2169	goto cq_free_rsrc;
2170	}
2171
2172	iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3,
2173	64);
2174	iwcq->kmem_shadow.va = dma_alloc_coherent(dev: dev->hw->device,
2175	size: iwcq->kmem_shadow.size,
2176	dma_handle: &iwcq->kmem_shadow.pa,
2177	GFP_KERNEL);
2178	if (!iwcq->kmem_shadow.va) {
2179	err_code = -ENOMEM;
2180	goto cq_free_rsrc;
2181	}
2182	info.shadow_area_pa = iwcq->kmem_shadow.pa;
2183	ukinfo->shadow_area = iwcq->kmem_shadow.va;
2184	ukinfo->cq_base = iwcq->kmem.va;
2185	info.cq_base_pa = iwcq->kmem.pa;
2186	}
2187
2188	info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2,
2189	(u32)IRDMA_MAX_CQ_READ_THRESH);
2190
2191	if (irdma_sc_cq_init(cq, info: &info)) {
2192	ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n");
2193	err_code = -EPROTO;
2194	goto cq_free_rsrc;
2195	}
2196
2197	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &rf->cqp, wait: true);
2198	if (!cqp_request) {
2199	err_code = -ENOMEM;
2200	goto cq_free_rsrc;
2201	}
2202
2203	cqp_info = &cqp_request->info;
2204	cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
2205	cqp_info->post_sq = 1;
2206	cqp_info->in.u.cq_create.cq = cq;
2207	cqp_info->in.u.cq_create.check_overflow = true;
2208	cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
2209	err_code = irdma_handle_cqp_op(rf, cqp_request);
2210	irdma_put_cqp_request(cqp: &rf->cqp, cqp_request);
2211	if (err_code)
2212	goto cq_free_rsrc;
2213
2214	if (udata) {
2215	struct irdma_create_cq_resp resp = {};
2216
2217	resp.cq_id = info.cq_uk_init_info.cq_id;
2218	resp.cq_size = info.cq_uk_init_info.cq_size;
2219	if (ib_copy_to_udata(udata, src: &resp,
2220	min(sizeof(resp), udata->outlen))) {
2221	ibdev_dbg(&iwdev->ibdev,
2222	"VERBS: copy to user data\n");
2223	err_code = -EPROTO;
2224	goto cq_destroy;
2225	}
2226	}
2227	rf->cq_table[cq_num] = iwcq;
2228	init_completion(x: &iwcq->free_cq);
2229
2230	return 0;
2231	cq_destroy:
2232	irdma_cq_wq_destroy(rf, cq);
2233	cq_free_rsrc:
2234	irdma_cq_free_rsrc(rf, iwcq);
2235
2236	return err_code;
2237	}
2238
2239	/**
2240	* irdma_get_mr_access - get hw MR access permissions from IB access flags
2241	* @access: IB access flags
2242	*/
2243	static inline u16 irdma_get_mr_access(int access)
2244	{
2245	u16 hw_access = 0;
2246
2247	hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ?
2248	IRDMA_ACCESS_FLAGS_LOCALWRITE : 0;
2249	hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ?
2250	IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0;
2251	hw_access |= (access & IB_ACCESS_REMOTE_READ) ?
2252	IRDMA_ACCESS_FLAGS_REMOTEREAD : 0;
2253	hw_access |= (access & IB_ACCESS_MW_BIND) ?
2254	IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0;
2255	hw_access |= (access & IB_ZERO_BASED) ?
2256	IRDMA_ACCESS_FLAGS_ZERO_BASED : 0;
2257	hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD;
2258
2259	return hw_access;
2260	}
2261
2262	/**
2263	* irdma_free_stag - free stag resource
2264	* @iwdev: irdma device
2265	* @stag: stag to free
2266	*/
2267	static void irdma_free_stag(struct irdma_device *iwdev, u32 stag)
2268	{
2269	u32 stag_idx;
2270
2271	stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S;
2272	irdma_free_rsrc(rf: iwdev->rf, rsrc_array: iwdev->rf->allocated_mrs, rsrc_num: stag_idx);
2273	}
2274
2275	/**
2276	* irdma_create_stag - create random stag
2277	* @iwdev: irdma device
2278	*/
2279	static u32 irdma_create_stag(struct irdma_device *iwdev)
2280	{
2281	u32 stag = 0;
2282	u32 stag_index = 0;
2283	u32 next_stag_index;
2284	u32 driver_key;
2285	u32 random;
2286	u8 consumer_key;
2287	int ret;
2288
2289	get_random_bytes(buf: &random, len: sizeof(random));
2290	consumer_key = (u8)random;
2291
2292	driver_key = random & ~iwdev->rf->mr_stagmask;
2293	next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8;
2294	next_stag_index %= iwdev->rf->max_mr;
2295
2296	ret = irdma_alloc_rsrc(rf: iwdev->rf, rsrc_array: iwdev->rf->allocated_mrs,
2297	max_rsrc: iwdev->rf->max_mr, req_rsrc_num: &stag_index,
2298	next: &next_stag_index);
2299	if (ret)
2300	return stag;
2301	stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S;
2302	stag |= driver_key;
2303	stag += (u32)consumer_key;
2304
2305	return stag;
2306	}
2307
2308	/**
2309	* irdma_next_pbl_addr - Get next pbl address
2310	* @pbl: pointer to a pble
2311	* @pinfo: info pointer
2312	* @idx: index
2313	*/
2314	static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo,
2315	u32 *idx)
2316	{
2317	*idx += 1;
2318	if (!(*pinfo) || *idx != (*pinfo)->cnt)
2319	return ++pbl;
2320	*idx = 0;
2321	(*pinfo)++;
2322
2323	return (*pinfo)->addr;
2324	}
2325
2326	/**
2327	* irdma_copy_user_pgaddrs - copy user page address to pble's os locally
2328	* @iwmr: iwmr for IB's user page addresses
2329	* @pbl: ple pointer to save 1 level or 0 level pble
2330	* @level: indicated level 0, 1 or 2
2331	*/
2332	static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl,
2333	enum irdma_pble_level level)
2334	{
2335	struct ib_umem *region = iwmr->region;
2336	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2337	struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
2338	struct irdma_pble_info *pinfo;
2339	struct ib_block_iter biter;
2340	u32 idx = 0;
2341	u32 pbl_cnt = 0;
2342
2343	pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf;
2344
2345	if (iwmr->type == IRDMA_MEMREG_TYPE_QP)
2346	iwpbl->qp_mr.sq_page = sg_page(sg: region->sgt_append.sgt.sgl);
2347
2348	rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) {
2349	*pbl = rdma_block_iter_dma_address(biter: &biter);
2350	if (++pbl_cnt == palloc->total_cnt)
2351	break;
2352	pbl = irdma_next_pbl_addr(pbl, pinfo: &pinfo, idx: &idx);
2353	}
2354	}
2355
2356	/**
2357	* irdma_check_mem_contiguous - check if pbls stored in arr are contiguous
2358	* @arr: lvl1 pbl array
2359	* @npages: page count
2360	* @pg_size: page size
2361	*
2362	*/
2363	static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)
2364	{
2365	u32 pg_idx;
2366
2367	for (pg_idx = 0; pg_idx < npages; pg_idx++) {
2368	if ((*arr + (pg_size * pg_idx)) != arr[pg_idx])
2369	return false;
2370	}
2371
2372	return true;
2373	}
2374
2375	/**
2376	* irdma_check_mr_contiguous - check if MR is physically contiguous
2377	* @palloc: pbl allocation struct
2378	* @pg_size: page size
2379	*/
2380	static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc,
2381	u32 pg_size)
2382	{
2383	struct irdma_pble_level2 *lvl2 = &palloc->level2;
2384	struct irdma_pble_info *leaf = lvl2->leaf;
2385	u64 *arr = NULL;
2386	u64 *start_addr = NULL;
2387	int i;
2388	bool ret;
2389
2390	if (palloc->level == PBLE_LEVEL_1) {
2391	arr = palloc->level1.addr;
2392	ret = irdma_check_mem_contiguous(arr, npages: palloc->total_cnt,
2393	pg_size);
2394	return ret;
2395	}
2396
2397	start_addr = leaf->addr;
2398
2399	for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
2400	arr = leaf->addr;
2401	if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr)
2402	return false;
2403	ret = irdma_check_mem_contiguous(arr, npages: leaf->cnt, pg_size);
2404	if (!ret)
2405	return false;
2406	}
2407
2408	return true;
2409	}
2410
2411	/**
2412	* irdma_setup_pbles - copy user pg address to pble's
2413	* @rf: RDMA PCI function
2414	* @iwmr: mr pointer for this memory registration
2415	* @lvl: requested pble levels
2416	*/
2417	static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
2418	u8 lvl)
2419	{
2420	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2421	struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
2422	struct irdma_pble_info *pinfo;
2423	u64 *pbl;
2424	int status;
2425	enum irdma_pble_level level = PBLE_LEVEL_1;
2426
2427	if (lvl) {
2428	status = irdma_get_pble(pble_rsrc: rf->pble_rsrc, palloc, pble_cnt: iwmr->page_cnt,
2429	lvl);
2430	if (status)
2431	return status;
2432
2433	iwpbl->pbl_allocated = true;
2434	level = palloc->level;
2435	pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 :
2436	palloc->level2.leaf;
2437	pbl = pinfo->addr;
2438	} else {
2439	pbl = iwmr->pgaddrmem;
2440	}
2441
2442	irdma_copy_user_pgaddrs(iwmr, pbl, level);
2443
2444	if (lvl)
2445	iwmr->pgaddrmem[0] = *pbl;
2446
2447	return 0;
2448	}
2449
2450	/**
2451	* irdma_handle_q_mem - handle memory for qp and cq
2452	* @iwdev: irdma device
2453	* @req: information for q memory management
2454	* @iwpbl: pble struct
2455	* @lvl: pble level mask
2456	*/
2457	static int irdma_handle_q_mem(struct irdma_device *iwdev,
2458	struct irdma_mem_reg_req *req,
2459	struct irdma_pbl *iwpbl, u8 lvl)
2460	{
2461	struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
2462	struct irdma_mr *iwmr = iwpbl->iwmr;
2463	struct irdma_qp_mr *qpmr = &iwpbl->qp_mr;
2464	struct irdma_cq_mr *cqmr = &iwpbl->cq_mr;
2465	struct irdma_hmc_pble *hmc_p;
2466	u64 *arr = iwmr->pgaddrmem;
2467	u32 pg_size, total;
2468	int err = 0;
2469	bool ret = true;
2470
2471	pg_size = iwmr->page_size;
2472	err = irdma_setup_pbles(rf: iwdev->rf, iwmr, lvl);
2473	if (err)
2474	return err;
2475
2476	if (lvl)
2477	arr = palloc->level1.addr;
2478
2479	switch (iwmr->type) {
2480	case IRDMA_MEMREG_TYPE_QP:
2481	total = req->sq_pages + req->rq_pages;
2482	hmc_p = &qpmr->sq_pbl;
2483	qpmr->shadow = (dma_addr_t)arr[total];
2484
2485	if (lvl) {
2486	ret = irdma_check_mem_contiguous(arr, npages: req->sq_pages,
2487	pg_size);
2488	if (ret)
2489	ret = irdma_check_mem_contiguous(arr: &arr[req->sq_pages],
2490	npages: req->rq_pages,
2491	pg_size);
2492	}
2493
2494	if (!ret) {
2495	hmc_p->idx = palloc->level1.idx;
2496	hmc_p = &qpmr->rq_pbl;
2497	hmc_p->idx = palloc->level1.idx + req->sq_pages;
2498	} else {
2499	hmc_p->addr = arr[0];
2500	hmc_p = &qpmr->rq_pbl;
2501	hmc_p->addr = arr[req->sq_pages];
2502	}
2503	break;
2504	case IRDMA_MEMREG_TYPE_CQ:
2505	hmc_p = &cqmr->cq_pbl;
2506
2507	if (!cqmr->split)
2508	cqmr->shadow = (dma_addr_t)arr[req->cq_pages];
2509
2510	if (lvl)
2511	ret = irdma_check_mem_contiguous(arr, npages: req->cq_pages,
2512	pg_size);
2513
2514	if (!ret)
2515	hmc_p->idx = palloc->level1.idx;
2516	else
2517	hmc_p->addr = arr[0];
2518	break;
2519	default:
2520	ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n");
2521	err = -EINVAL;
2522	}
2523
2524	if (lvl && ret) {
2525	irdma_free_pble(pble_rsrc: iwdev->rf->pble_rsrc, palloc);
2526	iwpbl->pbl_allocated = false;
2527	}
2528
2529	return err;
2530	}
2531
2532	/**
2533	* irdma_hw_alloc_mw - create the hw memory window
2534	* @iwdev: irdma device
2535	* @iwmr: pointer to memory window info
2536	*/
2537	static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr)
2538	{
2539	struct irdma_mw_alloc_info *info;
2540	struct irdma_pd *iwpd = to_iwpd(ibpd: iwmr->ibmr.pd);
2541	struct irdma_cqp_request *cqp_request;
2542	struct cqp_cmds_info *cqp_info;
2543	int status;
2544
2545	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &iwdev->rf->cqp, wait: true);
2546	if (!cqp_request)
2547	return -ENOMEM;
2548
2549	cqp_info = &cqp_request->info;
2550	info = &cqp_info->in.u.mw_alloc.info;
2551	memset(info, 0, sizeof(*info));
2552	if (iwmr->ibmw.type == IB_MW_TYPE_1)
2553	info->mw_wide = true;
2554
2555	info->page_size = PAGE_SIZE;
2556	info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
2557	info->pd_id = iwpd->sc_pd.pd_id;
2558	info->remote_access = true;
2559	cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC;
2560	cqp_info->post_sq = 1;
2561	cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev;
2562	cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request;
2563	status = irdma_handle_cqp_op(rf: iwdev->rf, cqp_request);
2564	irdma_put_cqp_request(cqp: &iwdev->rf->cqp, cqp_request);
2565
2566	return status;
2567	}
2568
2569	/**
2570	* irdma_alloc_mw - Allocate memory window
2571	* @ibmw: Memory Window
2572	* @udata: user data pointer
2573	*/
2574	static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
2575	{
2576	struct irdma_device *iwdev = to_iwdev(ibdev: ibmw->device);
2577	struct irdma_mr *iwmr = to_iwmw(ibmw);
2578	int err_code;
2579	u32 stag;
2580
2581	stag = irdma_create_stag(iwdev);
2582	if (!stag)
2583	return -ENOMEM;
2584
2585	iwmr->stag = stag;
2586	ibmw->rkey = stag;
2587
2588	err_code = irdma_hw_alloc_mw(iwdev, iwmr);
2589	if (err_code) {
2590	irdma_free_stag(iwdev, stag);
2591	return err_code;
2592	}
2593
2594	return 0;
2595	}
2596
2597	/**
2598	* irdma_dealloc_mw - Dealloc memory window
2599	* @ibmw: memory window structure.
2600	*/
2601	static int irdma_dealloc_mw(struct ib_mw *ibmw)
2602	{
2603	struct ib_pd *ibpd = ibmw->pd;
2604	struct irdma_pd *iwpd = to_iwpd(ibpd);
2605	struct irdma_mr *iwmr = to_iwmr(ibmr: (struct ib_mr *)ibmw);
2606	struct irdma_device *iwdev = to_iwdev(ibdev: ibmw->device);
2607	struct irdma_cqp_request *cqp_request;
2608	struct cqp_cmds_info *cqp_info;
2609	struct irdma_dealloc_stag_info *info;
2610
2611	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &iwdev->rf->cqp, wait: true);
2612	if (!cqp_request)
2613	return -ENOMEM;
2614
2615	cqp_info = &cqp_request->info;
2616	info = &cqp_info->in.u.dealloc_stag.info;
2617	memset(info, 0, sizeof(*info));
2618	info->pd_id = iwpd->sc_pd.pd_id;
2619	info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S;
2620	info->mr = false;
2621	cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
2622	cqp_info->post_sq = 1;
2623	cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;
2624	cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
2625	irdma_handle_cqp_op(rf: iwdev->rf, cqp_request);
2626	irdma_put_cqp_request(cqp: &iwdev->rf->cqp, cqp_request);
2627	irdma_free_stag(iwdev, stag: iwmr->stag);
2628
2629	return 0;
2630	}
2631
2632	/**
2633	* irdma_hw_alloc_stag - cqp command to allocate stag
2634	* @iwdev: irdma device
2635	* @iwmr: irdma mr pointer
2636	*/
2637	static int irdma_hw_alloc_stag(struct irdma_device *iwdev,
2638	struct irdma_mr *iwmr)
2639	{
2640	struct irdma_allocate_stag_info *info;
2641	struct ib_pd *pd = iwmr->ibmr.pd;
2642	struct irdma_pd *iwpd = to_iwpd(ibpd: pd);
2643	int status;
2644	struct irdma_cqp_request *cqp_request;
2645	struct cqp_cmds_info *cqp_info;
2646
2647	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &iwdev->rf->cqp, wait: true);
2648	if (!cqp_request)
2649	return -ENOMEM;
2650
2651	cqp_info = &cqp_request->info;
2652	info = &cqp_info->in.u.alloc_stag.info;
2653	memset(info, 0, sizeof(*info));
2654	info->page_size = PAGE_SIZE;
2655	info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
2656	info->pd_id = iwpd->sc_pd.pd_id;
2657	info->total_len = iwmr->len;
2658	info->all_memory = pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY;
2659	info->remote_access = true;
2660	cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG;
2661	cqp_info->post_sq = 1;
2662	cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev;
2663	cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request;
2664	status = irdma_handle_cqp_op(rf: iwdev->rf, cqp_request);
2665	irdma_put_cqp_request(cqp: &iwdev->rf->cqp, cqp_request);
2666	if (status)
2667	return status;
2668
2669	iwmr->is_hwreg = 1;
2670	return 0;
2671	}
2672
2673	/**
2674	* irdma_alloc_mr - register stag for fast memory registration
2675	* @pd: ibpd pointer
2676	* @mr_type: memory for stag registrion
2677	* @max_num_sg: man number of pages
2678	*/
2679	static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
2680	u32 max_num_sg)
2681	{
2682	struct irdma_device *iwdev = to_iwdev(ibdev: pd->device);
2683	struct irdma_pble_alloc *palloc;
2684	struct irdma_pbl *iwpbl;
2685	struct irdma_mr *iwmr;
2686	u32 stag;
2687	int err_code;
2688
2689	iwmr = kzalloc(size: sizeof(*iwmr), GFP_KERNEL);
2690	if (!iwmr)
2691	return ERR_PTR(error: -ENOMEM);
2692
2693	stag = irdma_create_stag(iwdev);
2694	if (!stag) {
2695	err_code = -ENOMEM;
2696	goto err;
2697	}
2698
2699	iwmr->stag = stag;
2700	iwmr->ibmr.rkey = stag;
2701	iwmr->ibmr.lkey = stag;
2702	iwmr->ibmr.pd = pd;
2703	iwmr->ibmr.device = pd->device;
2704	iwpbl = &iwmr->iwpbl;
2705	iwpbl->iwmr = iwmr;
2706	iwmr->type = IRDMA_MEMREG_TYPE_MEM;
2707	palloc = &iwpbl->pble_alloc;
2708	iwmr->page_cnt = max_num_sg;
2709	/* Use system PAGE_SIZE as the sg page sizes are unknown at this point */
2710	iwmr->len = max_num_sg * PAGE_SIZE;
2711	err_code = irdma_get_pble(pble_rsrc: iwdev->rf->pble_rsrc, palloc, pble_cnt: iwmr->page_cnt,
2712	lvl: false);
2713	if (err_code)
2714	goto err_get_pble;
2715
2716	err_code = irdma_hw_alloc_stag(iwdev, iwmr);
2717	if (err_code)
2718	goto err_alloc_stag;
2719
2720	iwpbl->pbl_allocated = true;
2721
2722	return &iwmr->ibmr;
2723	err_alloc_stag:
2724	irdma_free_pble(pble_rsrc: iwdev->rf->pble_rsrc, palloc);
2725	err_get_pble:
2726	irdma_free_stag(iwdev, stag);
2727	err:
2728	kfree(objp: iwmr);
2729
2730	return ERR_PTR(error: err_code);
2731	}
2732
2733	/**
2734	* irdma_set_page - populate pbl list for fmr
2735	* @ibmr: ib mem to access iwarp mr pointer
2736	* @addr: page dma address fro pbl list
2737	*/
2738	static int irdma_set_page(struct ib_mr *ibmr, u64 addr)
2739	{
2740	struct irdma_mr *iwmr = to_iwmr(ibmr);
2741	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2742	struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
2743	u64 *pbl;
2744
2745	if (unlikely(iwmr->npages == iwmr->page_cnt))
2746	return -ENOMEM;
2747
2748	if (palloc->level == PBLE_LEVEL_2) {
2749	struct irdma_pble_info *palloc_info =
2750	palloc->level2.leaf + (iwmr->npages >> PBLE_512_SHIFT);
2751
2752	palloc_info->addr[iwmr->npages & (PBLE_PER_PAGE - 1)] = addr;
2753	} else {
2754	pbl = palloc->level1.addr;
2755	pbl[iwmr->npages] = addr;
2756	}
2757	iwmr->npages++;
2758
2759	return 0;
2760	}
2761
2762	/**
2763	* irdma_map_mr_sg - map of sg list for fmr
2764	* @ibmr: ib mem to access iwarp mr pointer
2765	* @sg: scatter gather list
2766	* @sg_nents: number of sg pages
2767	* @sg_offset: scatter gather list for fmr
2768	*/
2769	static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2770	int sg_nents, unsigned int *sg_offset)
2771	{
2772	struct irdma_mr *iwmr = to_iwmr(ibmr);
2773
2774	iwmr->npages = 0;
2775
2776	return ib_sg_to_pages(mr: ibmr, sgl: sg, sg_nents, sg_offset, set_page: irdma_set_page);
2777	}
2778
2779	/**
2780	* irdma_hwreg_mr - send cqp command for memory registration
2781	* @iwdev: irdma device
2782	* @iwmr: irdma mr pointer
2783	* @access: access for MR
2784	*/
2785	static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,
2786	u16 access)
2787	{
2788	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2789	struct irdma_reg_ns_stag_info *stag_info;
2790	struct ib_pd *pd = iwmr->ibmr.pd;
2791	struct irdma_pd *iwpd = to_iwpd(ibpd: pd);
2792	struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
2793	struct irdma_cqp_request *cqp_request;
2794	struct cqp_cmds_info *cqp_info;
2795	int ret;
2796
2797	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &iwdev->rf->cqp, wait: true);
2798	if (!cqp_request)
2799	return -ENOMEM;
2800
2801	cqp_info = &cqp_request->info;
2802	stag_info = &cqp_info->in.u.mr_reg_non_shared.info;
2803	memset(stag_info, 0, sizeof(*stag_info));
2804	stag_info->va = iwpbl->user_base;
2805	stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
2806	stag_info->stag_key = (u8)iwmr->stag;
2807	stag_info->total_len = iwmr->len;
2808	stag_info->access_rights = irdma_get_mr_access(access);
2809	stag_info->pd_id = iwpd->sc_pd.pd_id;
2810	stag_info->all_memory = pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY;
2811	if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED)
2812	stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED;
2813	else
2814	stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED;
2815	stag_info->page_size = iwmr->page_size;
2816
2817	if (iwpbl->pbl_allocated) {
2818	if (palloc->level == PBLE_LEVEL_1) {
2819	stag_info->first_pm_pbl_index = palloc->level1.idx;
2820	stag_info->chunk_size = 1;
2821	} else {
2822	stag_info->first_pm_pbl_index = palloc->level2.root.idx;
2823	stag_info->chunk_size = 3;
2824	}
2825	} else {
2826	stag_info->reg_addr_pa = iwmr->pgaddrmem[0];
2827	}
2828
2829	cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED;
2830	cqp_info->post_sq = 1;
2831	cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev;
2832	cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request;
2833	ret = irdma_handle_cqp_op(rf: iwdev->rf, cqp_request);
2834	irdma_put_cqp_request(cqp: &iwdev->rf->cqp, cqp_request);
2835
2836	if (!ret)
2837	iwmr->is_hwreg = 1;
2838
2839	return ret;
2840	}
2841
2842	static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access,
2843	bool create_stag)
2844	{
2845	struct irdma_device *iwdev = to_iwdev(ibdev: iwmr->ibmr.device);
2846	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2847	u32 stag = 0;
2848	u8 lvl;
2849	int err;
2850
2851	lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;
2852
2853	err = irdma_setup_pbles(rf: iwdev->rf, iwmr, lvl);
2854	if (err)
2855	return err;
2856
2857	if (lvl) {
2858	err = irdma_check_mr_contiguous(palloc: &iwpbl->pble_alloc,
2859	pg_size: iwmr->page_size);
2860	if (err) {
2861	irdma_free_pble(pble_rsrc: iwdev->rf->pble_rsrc, palloc: &iwpbl->pble_alloc);
2862	iwpbl->pbl_allocated = false;
2863	}
2864	}
2865
2866	if (create_stag) {
2867	stag = irdma_create_stag(iwdev);
2868	if (!stag) {
2869	err = -ENOMEM;
2870	goto free_pble;
2871	}
2872
2873	iwmr->stag = stag;
2874	iwmr->ibmr.rkey = stag;
2875	iwmr->ibmr.lkey = stag;
2876	}
2877
2878	err = irdma_hwreg_mr(iwdev, iwmr, access);
2879	if (err)
2880	goto err_hwreg;
2881
2882	return 0;
2883
2884	err_hwreg:
2885	if (stag)
2886	irdma_free_stag(iwdev, stag);
2887
2888	free_pble:
2889	if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
2890	irdma_free_pble(pble_rsrc: iwdev->rf->pble_rsrc, palloc: &iwpbl->pble_alloc);
2891
2892	return err;
2893	}
2894
2895	static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region,
2896	struct ib_pd *pd, u64 virt,
2897	enum irdma_memreg_type reg_type)
2898	{
2899	struct irdma_device *iwdev = to_iwdev(ibdev: pd->device);
2900	struct irdma_pbl *iwpbl;
2901	struct irdma_mr *iwmr;
2902	unsigned long pgsz_bitmap;
2903
2904	iwmr = kzalloc(size: sizeof(*iwmr), GFP_KERNEL);
2905	if (!iwmr)
2906	return ERR_PTR(error: -ENOMEM);
2907
2908	iwpbl = &iwmr->iwpbl;
2909	iwpbl->iwmr = iwmr;
2910	iwmr->region = region;
2911	iwmr->ibmr.pd = pd;
2912	iwmr->ibmr.device = pd->device;
2913	iwmr->ibmr.iova = virt;
2914	iwmr->type = reg_type;
2915
2916	pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ?
2917	iwdev->rf->sc_dev.hw_attrs.page_size_cap : SZ_4K;
2918
2919	iwmr->page_size = ib_umem_find_best_pgsz(umem: region, pgsz_bitmap, virt);
2920	if (unlikely(!iwmr->page_size)) {
2921	kfree(objp: iwmr);
2922	return ERR_PTR(error: -EOPNOTSUPP);
2923	}
2924
2925	iwmr->len = region->length;
2926	iwpbl->user_base = virt;
2927	iwmr->page_cnt = ib_umem_num_dma_blocks(umem: region, pgsz: iwmr->page_size);
2928
2929	return iwmr;
2930	}
2931
2932	static void irdma_free_iwmr(struct irdma_mr *iwmr)
2933	{
2934	kfree(objp: iwmr);
2935	}
2936
2937	static int irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req,
2938	struct ib_udata *udata,
2939	struct irdma_mr *iwmr)
2940	{
2941	struct irdma_device *iwdev = to_iwdev(ibdev: iwmr->ibmr.device);
2942	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2943	struct irdma_ucontext *ucontext = NULL;
2944	unsigned long flags;
2945	u32 total;
2946	int err;
2947	u8 lvl;
2948
2949	/* iWarp: Catch page not starting on OS page boundary */
2950	if (!rdma_protocol_roce(device: &iwdev->ibdev, port_num: 1) &&
2951	ib_umem_offset(umem: iwmr->region))
2952	return -EINVAL;
2953
2954	total = req.sq_pages + req.rq_pages + 1;
2955	if (total > iwmr->page_cnt)
2956	return -EINVAL;
2957
2958	total = req.sq_pages + req.rq_pages;
2959	lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;
2960	err = irdma_handle_q_mem(iwdev, req: &req, iwpbl, lvl);
2961	if (err)
2962	return err;
2963
2964	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2965	ibucontext);
2966	spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
2967	list_add_tail(new: &iwpbl->list, head: &ucontext->qp_reg_mem_list);
2968	iwpbl->on_list = true;
2969	spin_unlock_irqrestore(lock: &ucontext->qp_reg_mem_list_lock, flags);
2970
2971	return 0;
2972	}
2973
2974	static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,
2975	struct ib_udata *udata,
2976	struct irdma_mr *iwmr)
2977	{
2978	struct irdma_device *iwdev = to_iwdev(ibdev: iwmr->ibmr.device);
2979	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
2980	struct irdma_ucontext *ucontext = NULL;
2981	u8 shadow_pgcnt = 1;
2982	unsigned long flags;
2983	u32 total;
2984	int err;
2985	u8 lvl;
2986
2987	if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)
2988	shadow_pgcnt = 0;
2989	total = req.cq_pages + shadow_pgcnt;
2990	if (total > iwmr->page_cnt)
2991	return -EINVAL;
2992
2993	lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;
2994	err = irdma_handle_q_mem(iwdev, req: &req, iwpbl, lvl);
2995	if (err)
2996	return err;
2997
2998	ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2999	ibucontext);
3000	spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
3001	list_add_tail(new: &iwpbl->list, head: &ucontext->cq_reg_mem_list);
3002	iwpbl->on_list = true;
3003	spin_unlock_irqrestore(lock: &ucontext->cq_reg_mem_list_lock, flags);
3004
3005	return 0;
3006	}
3007
3008	/**
3009	* irdma_reg_user_mr - Register a user memory region
3010	* @pd: ptr of pd
3011	* @start: virtual start address
3012	* @len: length of mr
3013	* @virt: virtual address
3014	* @access: access of mr
3015	* @udata: user data
3016	*/
3017	static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
3018	u64 virt, int access,
3019	struct ib_udata *udata)
3020	{
3021	#define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)
3022	struct irdma_device *iwdev = to_iwdev(ibdev: pd->device);
3023	struct irdma_mem_reg_req req = {};
3024	struct ib_umem *region = NULL;
3025	struct irdma_mr *iwmr = NULL;
3026	int err;
3027
3028	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
3029	return ERR_PTR(error: -EINVAL);
3030
3031	if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN)
3032	return ERR_PTR(error: -EINVAL);
3033
3034	region = ib_umem_get(device: pd->device, addr: start, size: len, access);
3035
3036	if (IS_ERR(ptr: region)) {
3037	ibdev_dbg(&iwdev->ibdev,
3038	"VERBS: Failed to create ib_umem region\n");
3039	return (struct ib_mr *)region;
3040	}
3041
3042	if (ib_copy_from_udata(dest: &req, udata, min(sizeof(req), udata->inlen))) {
3043	ib_umem_release(umem: region);
3044	return ERR_PTR(error: -EFAULT);
3045	}
3046
3047	iwmr = irdma_alloc_iwmr(region, pd, virt, reg_type: req.reg_type);
3048	if (IS_ERR(ptr: iwmr)) {
3049	ib_umem_release(umem: region);
3050	return (struct ib_mr *)iwmr;
3051	}
3052
3053	switch (req.reg_type) {
3054	case IRDMA_MEMREG_TYPE_QP:
3055	err = irdma_reg_user_mr_type_qp(req, udata, iwmr);
3056	if (err)
3057	goto error;
3058
3059	break;
3060	case IRDMA_MEMREG_TYPE_CQ:
3061	err = irdma_reg_user_mr_type_cq(req, udata, iwmr);
3062	if (err)
3063	goto error;
3064	break;
3065	case IRDMA_MEMREG_TYPE_MEM:
3066	err = irdma_reg_user_mr_type_mem(iwmr, access, create_stag: true);
3067	if (err)
3068	goto error;
3069
3070	break;
3071	default:
3072	err = -EINVAL;
3073	goto error;
3074	}
3075
3076	return &iwmr->ibmr;
3077	error:
3078	ib_umem_release(umem: region);
3079	irdma_free_iwmr(iwmr);
3080
3081	return ERR_PTR(error: err);
3082	}
3083
3084	static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
3085	u64 len, u64 virt,
3086	int fd, int access,
3087	struct ib_udata *udata)
3088	{
3089	struct irdma_device *iwdev = to_iwdev(ibdev: pd->device);
3090	struct ib_umem_dmabuf *umem_dmabuf;
3091	struct irdma_mr *iwmr;
3092	int err;
3093
3094	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
3095	return ERR_PTR(error: -EINVAL);
3096
3097	umem_dmabuf = ib_umem_dmabuf_get_pinned(device: pd->device, offset: start, size: len, fd, access);
3098	if (IS_ERR(ptr: umem_dmabuf)) {
3099	err = PTR_ERR(ptr: umem_dmabuf);
3100	ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n", err);
3101	return ERR_PTR(error: err);
3102	}
3103
3104	iwmr = irdma_alloc_iwmr(region: &umem_dmabuf->umem, pd, virt, reg_type: IRDMA_MEMREG_TYPE_MEM);
3105	if (IS_ERR(ptr: iwmr)) {
3106	err = PTR_ERR(ptr: iwmr);
3107	goto err_release;
3108	}
3109
3110	err = irdma_reg_user_mr_type_mem(iwmr, access, create_stag: true);
3111	if (err)
3112	goto err_iwmr;
3113
3114	return &iwmr->ibmr;
3115
3116	err_iwmr:
3117	irdma_free_iwmr(iwmr);
3118
3119	err_release:
3120	ib_umem_release(umem: &umem_dmabuf->umem);
3121
3122	return ERR_PTR(error: err);
3123	}
3124
3125	static int irdma_hwdereg_mr(struct ib_mr *ib_mr)
3126	{
3127	struct irdma_device *iwdev = to_iwdev(ibdev: ib_mr->device);
3128	struct irdma_mr *iwmr = to_iwmr(ibmr: ib_mr);
3129	struct irdma_pd *iwpd = to_iwpd(ibpd: ib_mr->pd);
3130	struct irdma_dealloc_stag_info *info;
3131	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
3132	struct irdma_cqp_request *cqp_request;
3133	struct cqp_cmds_info *cqp_info;
3134	int status;
3135
3136	/* Skip HW MR de-register when it is already de-registered
3137	* during an MR re-reregister and the re-registration fails
3138	*/
3139	if (!iwmr->is_hwreg)
3140	return 0;
3141
3142	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &iwdev->rf->cqp, wait: true);
3143	if (!cqp_request)
3144	return -ENOMEM;
3145
3146	cqp_info = &cqp_request->info;
3147	info = &cqp_info->in.u.dealloc_stag.info;
3148	memset(info, 0, sizeof(*info));
3149	info->pd_id = iwpd->sc_pd.pd_id;
3150	info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S;
3151	info->mr = true;
3152	if (iwpbl->pbl_allocated)
3153	info->dealloc_pbl = true;
3154
3155	cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
3156	cqp_info->post_sq = 1;
3157	cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;
3158	cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
3159	status = irdma_handle_cqp_op(rf: iwdev->rf, cqp_request);
3160	irdma_put_cqp_request(cqp: &iwdev->rf->cqp, cqp_request);
3161	if (status)
3162	return status;
3163
3164	iwmr->is_hwreg = 0;
3165	return 0;
3166	}
3167
3168	/*
3169	* irdma_rereg_mr_trans - Re-register a user MR for a change translation.
3170	* @iwmr: ptr of iwmr
3171	* @start: virtual start address
3172	* @len: length of mr
3173	* @virt: virtual address
3174	*
3175	* Re-register a user memory region when a change translation is requested.
3176	* Re-register a new region while reusing the stag from the original registration.
3177	*/
3178	static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
3179	u64 virt)
3180	{
3181	struct irdma_device *iwdev = to_iwdev(ibdev: iwmr->ibmr.device);
3182	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
3183	struct ib_pd *pd = iwmr->ibmr.pd;
3184	struct ib_umem *region;
3185	int err;
3186
3187	region = ib_umem_get(device: pd->device, addr: start, size: len, access: iwmr->access);
3188	if (IS_ERR(ptr: region))
3189	return PTR_ERR(ptr: region);
3190
3191	iwmr->region = region;
3192	iwmr->ibmr.iova = virt;
3193	iwmr->ibmr.pd = pd;
3194	iwmr->page_size = ib_umem_find_best_pgsz(umem: region,
3195	pgsz_bitmap: iwdev->rf->sc_dev.hw_attrs.page_size_cap,
3196	virt);
3197	if (unlikely(!iwmr->page_size)) {
3198	err = -EOPNOTSUPP;
3199	goto err;
3200	}
3201
3202	iwmr->len = region->length;
3203	iwpbl->user_base = virt;
3204	iwmr->page_cnt = ib_umem_num_dma_blocks(umem: region, pgsz: iwmr->page_size);
3205
3206	err = irdma_reg_user_mr_type_mem(iwmr, access: iwmr->access, create_stag: false);
3207	if (err)
3208	goto err;
3209
3210	return 0;
3211
3212	err:
3213	ib_umem_release(umem: region);
3214	return err;
3215	}
3216
3217	/*
3218	* irdma_rereg_user_mr - Re-Register a user memory region(MR)
3219	* @ibmr: ib mem to access iwarp mr pointer
3220	* @flags: bit mask to indicate which of the attr's of MR modified
3221	* @start: virtual start address
3222	* @len: length of mr
3223	* @virt: virtual address
3224	* @new_access: bit mask of access flags
3225	* @new_pd: ptr of pd
3226	* @udata: user data
3227	*
3228	* Return:
3229	* NULL - Success, existing MR updated
3230	* ERR_PTR - error occurred
3231	*/
3232	static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags,
3233	u64 start, u64 len, u64 virt,
3234	int new_access, struct ib_pd *new_pd,
3235	struct ib_udata *udata)
3236	{
3237	struct irdma_device *iwdev = to_iwdev(ibdev: ib_mr->device);
3238	struct irdma_mr *iwmr = to_iwmr(ibmr: ib_mr);
3239	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
3240	int ret;
3241
3242	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
3243	return ERR_PTR(error: -EINVAL);
3244
3245	if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))
3246	return ERR_PTR(error: -EOPNOTSUPP);
3247
3248	ret = irdma_hwdereg_mr(ib_mr);
3249	if (ret)
3250	return ERR_PTR(error: ret);
3251
3252	if (flags & IB_MR_REREG_ACCESS)
3253	iwmr->access = new_access;
3254
3255	if (flags & IB_MR_REREG_PD) {
3256	iwmr->ibmr.pd = new_pd;
3257	iwmr->ibmr.device = new_pd->device;
3258	}
3259
3260	if (flags & IB_MR_REREG_TRANS) {
3261	if (iwpbl->pbl_allocated) {
3262	irdma_free_pble(pble_rsrc: iwdev->rf->pble_rsrc,
3263	palloc: &iwpbl->pble_alloc);
3264	iwpbl->pbl_allocated = false;
3265	}
3266	if (iwmr->region) {
3267	ib_umem_release(umem: iwmr->region);
3268	iwmr->region = NULL;
3269	}
3270
3271	ret = irdma_rereg_mr_trans(iwmr, start, len, virt);
3272	} else
3273	ret = irdma_hwreg_mr(iwdev, iwmr, access: iwmr->access);
3274	if (ret)
3275	return ERR_PTR(error: ret);
3276
3277	return NULL;
3278	}
3279
3280	/**
3281	* irdma_reg_phys_mr - register kernel physical memory
3282	* @pd: ibpd pointer
3283	* @addr: physical address of memory to register
3284	* @size: size of memory to register
3285	* @access: Access rights
3286	* @iova_start: start of virtual address for physical buffers
3287	*/
3288	struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access,
3289	u64 *iova_start)
3290	{
3291	struct irdma_device *iwdev = to_iwdev(ibdev: pd->device);
3292	struct irdma_pbl *iwpbl;
3293	struct irdma_mr *iwmr;
3294	u32 stag;
3295	int ret;
3296
3297	iwmr = kzalloc(size: sizeof(*iwmr), GFP_KERNEL);
3298	if (!iwmr)
3299	return ERR_PTR(error: -ENOMEM);
3300
3301	iwmr->ibmr.pd = pd;
3302	iwmr->ibmr.device = pd->device;
3303	iwpbl = &iwmr->iwpbl;
3304	iwpbl->iwmr = iwmr;
3305	iwmr->type = IRDMA_MEMREG_TYPE_MEM;
3306	iwpbl->user_base = *iova_start;
3307	stag = irdma_create_stag(iwdev);
3308	if (!stag) {
3309	ret = -ENOMEM;
3310	goto err;
3311	}
3312
3313	iwmr->stag = stag;
3314	iwmr->ibmr.iova = *iova_start;
3315	iwmr->ibmr.rkey = stag;
3316	iwmr->ibmr.lkey = stag;
3317	iwmr->page_cnt = 1;
3318	iwmr->pgaddrmem[0] = addr;
3319	iwmr->len = size;
3320	iwmr->page_size = SZ_4K;
3321	ret = irdma_hwreg_mr(iwdev, iwmr, access);
3322	if (ret) {
3323	irdma_free_stag(iwdev, stag);
3324	goto err;
3325	}
3326
3327	return &iwmr->ibmr;
3328
3329	err:
3330	kfree(objp: iwmr);
3331
3332	return ERR_PTR(error: ret);
3333	}
3334
3335	/**
3336	* irdma_get_dma_mr - register physical mem
3337	* @pd: ptr of pd
3338	* @acc: access for memory
3339	*/
3340	static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc)
3341	{
3342	u64 kva = 0;
3343
3344	return irdma_reg_phys_mr(pd, addr: 0, size: 0, access: acc, iova_start: &kva);
3345	}
3346
3347	/**
3348	* irdma_del_memlist - Deleting pbl list entries for CQ/QP
3349	* @iwmr: iwmr for IB's user page addresses
3350	* @ucontext: ptr to user context
3351	*/
3352	static void irdma_del_memlist(struct irdma_mr *iwmr,
3353	struct irdma_ucontext *ucontext)
3354	{
3355	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
3356	unsigned long flags;
3357
3358	switch (iwmr->type) {
3359	case IRDMA_MEMREG_TYPE_CQ:
3360	spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
3361	if (iwpbl->on_list) {
3362	iwpbl->on_list = false;
3363	list_del(entry: &iwpbl->list);
3364	}
3365	spin_unlock_irqrestore(lock: &ucontext->cq_reg_mem_list_lock, flags);
3366	break;
3367	case IRDMA_MEMREG_TYPE_QP:
3368	spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
3369	if (iwpbl->on_list) {
3370	iwpbl->on_list = false;
3371	list_del(entry: &iwpbl->list);
3372	}
3373	spin_unlock_irqrestore(lock: &ucontext->qp_reg_mem_list_lock, flags);
3374	break;
3375	default:
3376	break;
3377	}
3378	}
3379
3380	/**
3381	* irdma_dereg_mr - deregister mr
3382	* @ib_mr: mr ptr for dereg
3383	* @udata: user data
3384	*/
3385	static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3386	{
3387	struct irdma_mr *iwmr = to_iwmr(ibmr: ib_mr);
3388	struct irdma_device *iwdev = to_iwdev(ibdev: ib_mr->device);
3389	struct irdma_pbl *iwpbl = &iwmr->iwpbl;
3390	int ret;
3391
3392	if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) {
3393	if (iwmr->region) {
3394	struct irdma_ucontext *ucontext;
3395
3396	ucontext = rdma_udata_to_drv_context(udata,
3397	struct irdma_ucontext,
3398	ibucontext);
3399	irdma_del_memlist(iwmr, ucontext);
3400	}
3401	goto done;
3402	}
3403
3404	ret = irdma_hwdereg_mr(ib_mr);
3405	if (ret)
3406	return ret;
3407
3408	irdma_free_stag(iwdev, stag: iwmr->stag);
3409	done:
3410	if (iwpbl->pbl_allocated)
3411	irdma_free_pble(pble_rsrc: iwdev->rf->pble_rsrc, palloc: &iwpbl->pble_alloc);
3412
3413	if (iwmr->region)
3414	ib_umem_release(umem: iwmr->region);
3415
3416	kfree(objp: iwmr);
3417
3418	return 0;
3419	}
3420
3421	/**
3422	* irdma_post_send - kernel application wr
3423	* @ibqp: qp ptr for wr
3424	* @ib_wr: work request ptr
3425	* @bad_wr: return of bad wr if err
3426	*/
3427	static int irdma_post_send(struct ib_qp *ibqp,
3428	const struct ib_send_wr *ib_wr,
3429	const struct ib_send_wr **bad_wr)
3430	{
3431	struct irdma_qp *iwqp;
3432	struct irdma_qp_uk *ukqp;
3433	struct irdma_sc_dev *dev;
3434	struct irdma_post_sq_info info;
3435	int err = 0;
3436	unsigned long flags;
3437	bool inv_stag;
3438	struct irdma_ah *ah;
3439
3440	iwqp = to_iwqp(ibqp);
3441	ukqp = &iwqp->sc_qp.qp_uk;
3442	dev = &iwqp->iwdev->rf->sc_dev;
3443
3444	spin_lock_irqsave(&iwqp->lock, flags);
3445	while (ib_wr) {
3446	memset(&info, 0, sizeof(info));
3447	inv_stag = false;
3448	info.wr_id = (ib_wr->wr_id);
3449	if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all)
3450	info.signaled = true;
3451	if (ib_wr->send_flags & IB_SEND_FENCE)
3452	info.read_fence = true;
3453	switch (ib_wr->opcode) {
3454	case IB_WR_SEND_WITH_IMM:
3455	if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) {
3456	info.imm_data_valid = true;
3457	info.imm_data = ntohl(ib_wr->ex.imm_data);
3458	} else {
3459	err = -EINVAL;
3460	break;
3461	}
3462	fallthrough;
3463	case IB_WR_SEND:
3464	case IB_WR_SEND_WITH_INV:
3465	if (ib_wr->opcode == IB_WR_SEND ||
3466	ib_wr->opcode == IB_WR_SEND_WITH_IMM) {
3467	if (ib_wr->send_flags & IB_SEND_SOLICITED)
3468	info.op_type = IRDMA_OP_TYPE_SEND_SOL;
3469	else
3470	info.op_type = IRDMA_OP_TYPE_SEND;
3471	} else {
3472	if (ib_wr->send_flags & IB_SEND_SOLICITED)
3473	info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV;
3474	else
3475	info.op_type = IRDMA_OP_TYPE_SEND_INV;
3476	info.stag_to_inv = ib_wr->ex.invalidate_rkey;
3477	}
3478
3479	info.op.send.num_sges = ib_wr->num_sge;
3480	info.op.send.sg_list = ib_wr->sg_list;
3481	if (iwqp->ibqp.qp_type == IB_QPT_UD ||
3482	iwqp->ibqp.qp_type == IB_QPT_GSI) {
3483	ah = to_iwah(ibah: ud_wr(wr: ib_wr)->ah);
3484	info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
3485	info.op.send.qkey = ud_wr(wr: ib_wr)->remote_qkey;
3486	info.op.send.dest_qp = ud_wr(wr: ib_wr)->remote_qpn;
3487	}
3488
3489	if (ib_wr->send_flags & IB_SEND_INLINE)
3490	err = irdma_uk_inline_send(qp: ukqp, info: &info, post_sq: false);
3491	else
3492	err = irdma_uk_send(qp: ukqp, info: &info, post_sq: false);
3493	break;
3494	case IB_WR_RDMA_WRITE_WITH_IMM:
3495	if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) {
3496	info.imm_data_valid = true;
3497	info.imm_data = ntohl(ib_wr->ex.imm_data);
3498	} else {
3499	err = -EINVAL;
3500	break;
3501	}
3502	fallthrough;
3503	case IB_WR_RDMA_WRITE:
3504	if (ib_wr->send_flags & IB_SEND_SOLICITED)
3505	info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL;
3506	else
3507	info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
3508
3509	info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
3510	info.op.rdma_write.lo_sg_list = ib_wr->sg_list;
3511	info.op.rdma_write.rem_addr.addr =
3512	rdma_wr(wr: ib_wr)->remote_addr;
3513	info.op.rdma_write.rem_addr.lkey = rdma_wr(wr: ib_wr)->rkey;
3514	if (ib_wr->send_flags & IB_SEND_INLINE)
3515	err = irdma_uk_inline_rdma_write(qp: ukqp, info: &info, post_sq: false);
3516	else
3517	err = irdma_uk_rdma_write(qp: ukqp, info: &info, post_sq: false);
3518	break;
3519	case IB_WR_RDMA_READ_WITH_INV:
3520	inv_stag = true;
3521	fallthrough;
3522	case IB_WR_RDMA_READ:
3523	if (ib_wr->num_sge >
3524	dev->hw_attrs.uk_attrs.max_hw_read_sges) {
3525	err = -EINVAL;
3526	break;
3527	}
3528	info.op_type = IRDMA_OP_TYPE_RDMA_READ;
3529	info.op.rdma_read.rem_addr.addr = rdma_wr(wr: ib_wr)->remote_addr;
3530	info.op.rdma_read.rem_addr.lkey = rdma_wr(wr: ib_wr)->rkey;
3531	info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
3532	info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
3533	err = irdma_uk_rdma_read(qp: ukqp, info: &info, inv_stag, post_sq: false);
3534	break;
3535	case IB_WR_LOCAL_INV:
3536	info.op_type = IRDMA_OP_TYPE_INV_STAG;
3537	info.local_fence = info.read_fence;
3538	info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey;
3539	err = irdma_uk_stag_local_invalidate(qp: ukqp, info: &info, post_sq: true);
3540	break;
3541	case IB_WR_REG_MR: {
3542	struct irdma_mr *iwmr = to_iwmr(ibmr: reg_wr(wr: ib_wr)->mr);
3543	struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc;
3544	struct irdma_fast_reg_stag_info stag_info = {};
3545
3546	stag_info.signaled = info.signaled;
3547	stag_info.read_fence = info.read_fence;
3548	stag_info.access_rights = irdma_get_mr_access(access: reg_wr(wr: ib_wr)->access);
3549	stag_info.stag_key = reg_wr(wr: ib_wr)->key & 0xff;
3550	stag_info.stag_idx = reg_wr(wr: ib_wr)->key >> 8;
3551	stag_info.page_size = reg_wr(wr: ib_wr)->mr->page_size;
3552	stag_info.wr_id = ib_wr->wr_id;
3553	stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED;
3554	stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova;
3555	stag_info.total_len = iwmr->ibmr.length;
3556	stag_info.reg_addr_pa = *palloc->level1.addr;
3557	stag_info.first_pm_pbl_index = palloc->level1.idx;
3558	stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
3559	if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR)
3560	stag_info.chunk_size = 1;
3561	err = irdma_sc_mr_fast_register(qp: &iwqp->sc_qp, info: &stag_info,
3562	post_sq: true);
3563	break;
3564	}
3565	default:
3566	err = -EINVAL;
3567	ibdev_dbg(&iwqp->iwdev->ibdev,
3568	"VERBS: upost_send bad opcode = 0x%x\n",
3569	ib_wr->opcode);
3570	break;
3571	}
3572
3573	if (err)
3574	break;
3575	ib_wr = ib_wr->next;
3576	}
3577
3578	if (!iwqp->flush_issued) {
3579	if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS)
3580	irdma_uk_qp_post_wr(qp: ukqp);
3581	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
3582	} else {
3583	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
3584	mod_delayed_work(wq: iwqp->iwdev->cleanup_wq, dwork: &iwqp->dwork_flush,
3585	delay: msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
3586	}
3587	if (err)
3588	*bad_wr = ib_wr;
3589
3590	return err;
3591	}
3592
3593	/**
3594	* irdma_post_recv - post receive wr for kernel application
3595	* @ibqp: ib qp pointer
3596	* @ib_wr: work request for receive
3597	* @bad_wr: bad wr caused an error
3598	*/
3599	static int irdma_post_recv(struct ib_qp *ibqp,
3600	const struct ib_recv_wr *ib_wr,
3601	const struct ib_recv_wr **bad_wr)
3602	{
3603	struct irdma_qp *iwqp;
3604	struct irdma_qp_uk *ukqp;
3605	struct irdma_post_rq_info post_recv = {};
3606	unsigned long flags;
3607	int err = 0;
3608
3609	iwqp = to_iwqp(ibqp);
3610	ukqp = &iwqp->sc_qp.qp_uk;
3611
3612	spin_lock_irqsave(&iwqp->lock, flags);
3613	while (ib_wr) {
3614	post_recv.num_sges = ib_wr->num_sge;
3615	post_recv.wr_id = ib_wr->wr_id;
3616	post_recv.sg_list = ib_wr->sg_list;
3617	err = irdma_uk_post_receive(qp: ukqp, info: &post_recv);
3618	if (err) {
3619	ibdev_dbg(&iwqp->iwdev->ibdev,
3620	"VERBS: post_recv err %d\n", err);
3621	goto out;
3622	}
3623
3624	ib_wr = ib_wr->next;
3625	}
3626
3627	out:
3628	spin_unlock_irqrestore(lock: &iwqp->lock, flags);
3629	if (iwqp->flush_issued)
3630	mod_delayed_work(wq: iwqp->iwdev->cleanup_wq, dwork: &iwqp->dwork_flush,
3631	delay: msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
3632
3633	if (err)
3634	*bad_wr = ib_wr;
3635
3636	return err;
3637	}
3638
3639	/**
3640	* irdma_flush_err_to_ib_wc_status - return change flush error code to IB status
3641	* @opcode: iwarp flush code
3642	*/
3643	static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode)
3644	{
3645	switch (opcode) {
3646	case FLUSH_PROT_ERR:
3647	return IB_WC_LOC_PROT_ERR;
3648	case FLUSH_REM_ACCESS_ERR:
3649	return IB_WC_REM_ACCESS_ERR;
3650	case FLUSH_LOC_QP_OP_ERR:
3651	return IB_WC_LOC_QP_OP_ERR;
3652	case FLUSH_REM_OP_ERR:
3653	return IB_WC_REM_OP_ERR;
3654	case FLUSH_LOC_LEN_ERR:
3655	return IB_WC_LOC_LEN_ERR;
3656	case FLUSH_GENERAL_ERR:
3657	return IB_WC_WR_FLUSH_ERR;
3658	case FLUSH_RETRY_EXC_ERR:
3659	return IB_WC_RETRY_EXC_ERR;
3660	case FLUSH_MW_BIND_ERR:
3661	return IB_WC_MW_BIND_ERR;
3662	case FLUSH_REM_INV_REQ_ERR:
3663	return IB_WC_REM_INV_REQ_ERR;
3664	case FLUSH_FATAL_ERR:
3665	default:
3666	return IB_WC_FATAL_ERR;
3667	}
3668	}
3669
3670	/**
3671	* irdma_process_cqe - process cqe info
3672	* @entry: processed cqe
3673	* @cq_poll_info: cqe info
3674	*/
3675	static void irdma_process_cqe(struct ib_wc *entry,
3676	struct irdma_cq_poll_info *cq_poll_info)
3677	{
3678	struct irdma_sc_qp *qp;
3679
3680	entry->wc_flags = 0;
3681	entry->pkey_index = 0;
3682	entry->wr_id = cq_poll_info->wr_id;
3683
3684	qp = cq_poll_info->qp_handle;
3685	entry->qp = qp->qp_uk.back_qp;
3686
3687	if (cq_poll_info->error) {
3688	entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ?
3689	irdma_flush_err_to_ib_wc_status(opcode: cq_poll_info->minor_err) : IB_WC_GENERAL_ERR;
3690
3691	entry->vendor_err = cq_poll_info->major_err << 16 |
3692	cq_poll_info->minor_err;
3693	} else {
3694	entry->status = IB_WC_SUCCESS;
3695	if (cq_poll_info->imm_valid) {
3696	entry->ex.imm_data = htonl(cq_poll_info->imm_data);
3697	entry->wc_flags |= IB_WC_WITH_IMM;
3698	}
3699	if (cq_poll_info->ud_smac_valid) {
3700	ether_addr_copy(dst: entry->smac, src: cq_poll_info->ud_smac);
3701	entry->wc_flags |= IB_WC_WITH_SMAC;
3702	}
3703
3704	if (cq_poll_info->ud_vlan_valid) {
3705	u16 vlan = cq_poll_info->ud_vlan & VLAN_VID_MASK;
3706
3707	entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT;
3708	if (vlan) {
3709	entry->vlan_id = vlan;
3710	entry->wc_flags |= IB_WC_WITH_VLAN;
3711	}
3712	} else {
3713	entry->sl = 0;
3714	}
3715	}
3716
3717	if (cq_poll_info->q_type == IRDMA_CQE_QTYPE_SQ) {
3718	set_ib_wc_op_sq(cq_poll_info, entry);
3719	} else {
3720	set_ib_wc_op_rq(cq_poll_info, entry,
3721	send_imm_support: qp->qp_uk.qp_caps & IRDMA_SEND_WITH_IMM);
3722	if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD &&
3723	cq_poll_info->stag_invalid_set) {
3724	entry->ex.invalidate_rkey = cq_poll_info->inv_stag;
3725	entry->wc_flags |= IB_WC_WITH_INVALIDATE;
3726	}
3727	}
3728
3729	if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) {
3730	entry->src_qp = cq_poll_info->ud_src_qpn;
3731	entry->slid = 0;
3732	entry->wc_flags |=
3733	(IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE);
3734	entry->network_hdr_type = cq_poll_info->ipv4 ?
3735	RDMA_NETWORK_IPV4 :
3736	RDMA_NETWORK_IPV6;
3737	} else {
3738	entry->src_qp = cq_poll_info->qp_id;
3739	}
3740
3741	entry->byte_len = cq_poll_info->bytes_xfered;
3742	}
3743
3744	/**
3745	* irdma_poll_one - poll one entry of the CQ
3746	* @ukcq: ukcq to poll
3747	* @cur_cqe: current CQE info to be filled in
3748	* @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ
3749	*
3750	* Returns the internal irdma device error code or 0 on success
3751	*/
3752	static inline int irdma_poll_one(struct irdma_cq_uk *ukcq,
3753	struct irdma_cq_poll_info *cur_cqe,
3754	struct ib_wc *entry)
3755	{
3756	int ret = irdma_uk_cq_poll_cmpl(cq: ukcq, info: cur_cqe);
3757
3758	if (ret)
3759	return ret;
3760
3761	irdma_process_cqe(entry, cq_poll_info: cur_cqe);
3762
3763	return 0;
3764	}
3765
3766	/**
3767	* __irdma_poll_cq - poll cq for completion (kernel apps)
3768	* @iwcq: cq to poll
3769	* @num_entries: number of entries to poll
3770	* @entry: wr of a completed entry
3771	*/
3772	static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry)
3773	{
3774	struct list_head *tmp_node, *list_node;
3775	struct irdma_cq_buf *last_buf = NULL;
3776	struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe;
3777	struct irdma_cq_buf *cq_buf;
3778	int ret;
3779	struct irdma_device *iwdev;
3780	struct irdma_cq_uk *ukcq;
3781	bool cq_new_cqe = false;
3782	int resized_bufs = 0;
3783	int npolled = 0;
3784
3785	iwdev = to_iwdev(ibdev: iwcq->ibcq.device);
3786	ukcq = &iwcq->sc_cq.cq_uk;
3787
3788	/* go through the list of previously resized CQ buffers */
3789	list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) {
3790	cq_buf = container_of(list_node, struct irdma_cq_buf, list);
3791	while (npolled < num_entries) {
3792	ret = irdma_poll_one(ukcq: &cq_buf->cq_uk, cur_cqe, entry: entry + npolled);
3793	if (!ret) {
3794	++npolled;
3795	cq_new_cqe = true;
3796	continue;
3797	}
3798	if (ret == -ENOENT)
3799	break;
3800	/* QP using the CQ is destroyed. Skip reporting this CQE */
3801	if (ret == -EFAULT) {
3802	cq_new_cqe = true;
3803	continue;
3804	}
3805	goto error;
3806	}
3807
3808	/* save the resized CQ buffer which received the last cqe */
3809	if (cq_new_cqe)
3810	last_buf = cq_buf;
3811	cq_new_cqe = false;
3812	}
3813
3814	/* check the current CQ for new cqes */
3815	while (npolled < num_entries) {
3816	ret = irdma_poll_one(ukcq, cur_cqe, entry: entry + npolled);
3817	if (ret == -ENOENT) {
3818	ret = irdma_generated_cmpls(iwcq, cq_poll_info: cur_cqe);
3819	if (!ret)
3820	irdma_process_cqe(entry: entry + npolled, cq_poll_info: cur_cqe);
3821	}
3822	if (!ret) {
3823	++npolled;
3824	cq_new_cqe = true;
3825	continue;
3826	}
3827
3828	if (ret == -ENOENT)
3829	break;
3830	/* QP using the CQ is destroyed. Skip reporting this CQE */
3831	if (ret == -EFAULT) {
3832	cq_new_cqe = true;
3833	continue;
3834	}
3835	goto error;
3836	}
3837
3838	if (cq_new_cqe)
3839	/* all previous CQ resizes are complete */
3840	resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL);
3841	else if (last_buf)
3842	/* only CQ resizes up to the last_buf are complete */
3843	resized_bufs = irdma_process_resize_list(iwcq, iwdev, lcqe_buf: last_buf);
3844	if (resized_bufs)
3845	/* report to the HW the number of complete CQ resizes */
3846	irdma_uk_cq_set_resized_cnt(qp: ukcq, cnt: resized_bufs);
3847
3848	return npolled;
3849	error:
3850	ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n",
3851	__func__, ret);
3852
3853	return ret;
3854	}
3855
3856	/**
3857	* irdma_poll_cq - poll cq for completion (kernel apps)
3858	* @ibcq: cq to poll
3859	* @num_entries: number of entries to poll
3860	* @entry: wr of a completed entry
3861	*/
3862	static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries,
3863	struct ib_wc *entry)
3864	{
3865	struct irdma_cq *iwcq;
3866	unsigned long flags;
3867	int ret;
3868
3869	iwcq = to_iwcq(ibcq);
3870
3871	spin_lock_irqsave(&iwcq->lock, flags);
3872	ret = __irdma_poll_cq(iwcq, num_entries, entry);
3873	spin_unlock_irqrestore(lock: &iwcq->lock, flags);
3874
3875	return ret;
3876	}
3877
3878	/**
3879	* irdma_req_notify_cq - arm cq kernel application
3880	* @ibcq: cq to arm
3881	* @notify_flags: notofication flags
3882	*/
3883	static int irdma_req_notify_cq(struct ib_cq *ibcq,
3884	enum ib_cq_notify_flags notify_flags)
3885	{
3886	struct irdma_cq *iwcq;
3887	struct irdma_cq_uk *ukcq;
3888	unsigned long flags;
3889	enum irdma_cmpl_notify cq_notify;
3890	bool promo_event = false;
3891	int ret = 0;
3892
3893	cq_notify = notify_flags == IB_CQ_SOLICITED ?
3894	IRDMA_CQ_COMPL_SOLICITED : IRDMA_CQ_COMPL_EVENT;
3895	iwcq = to_iwcq(ibcq);
3896	ukcq = &iwcq->sc_cq.cq_uk;
3897
3898	spin_lock_irqsave(&iwcq->lock, flags);
3899	/* Only promote to arm the CQ for any event if the last arm event was solicited. */
3900	if (iwcq->last_notify == IRDMA_CQ_COMPL_SOLICITED && notify_flags != IB_CQ_SOLICITED)
3901	promo_event = true;
3902
3903	if (!atomic_cmpxchg(v: &iwcq->armed, old: 0, new: 1) || promo_event) {
3904	iwcq->last_notify = cq_notify;
3905	irdma_uk_cq_request_notification(cq: ukcq, cq_notify);
3906	}
3907
3908	if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
3909	(!irdma_cq_empty(iwcq) || !list_empty(head: &iwcq->cmpl_generated)))
3910	ret = 1;
3911	spin_unlock_irqrestore(lock: &iwcq->lock, flags);
3912
3913	return ret;
3914	}
3915
3916	static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num,
3917	struct ib_port_immutable *immutable)
3918	{
3919	struct ib_port_attr attr;
3920	int err;
3921
3922	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3923	err = ib_query_port(device: ibdev, port_num, port_attr: &attr);
3924	if (err)
3925	return err;
3926
3927	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3928	immutable->pkey_tbl_len = attr.pkey_tbl_len;
3929	immutable->gid_tbl_len = attr.gid_tbl_len;
3930
3931	return 0;
3932	}
3933
3934	static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num,
3935	struct ib_port_immutable *immutable)
3936	{
3937	struct ib_port_attr attr;
3938	int err;
3939
3940	immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
3941	err = ib_query_port(device: ibdev, port_num, port_attr: &attr);
3942	if (err)
3943	return err;
3944	immutable->gid_tbl_len = attr.gid_tbl_len;
3945
3946	return 0;
3947	}
3948
3949	static const struct rdma_stat_desc irdma_hw_stat_names[] = {
3950	/* gen1 - 32-bit */
3951	[IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name = "ip4InDiscards",
3952	[IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name = "ip4InTruncatedPkts",
3953	[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name = "ip4OutNoRoutes",
3954	[IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name = "ip6InDiscards",
3955	[IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name = "ip6InTruncatedPkts",
3956	[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name = "ip6OutNoRoutes",
3957	[IRDMA_HW_STAT_INDEX_TCPRTXSEG].name = "tcpRetransSegs",
3958	[IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name = "tcpInOptErrors",
3959	[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name = "tcpInProtoErrors",
3960	[IRDMA_HW_STAT_INDEX_RXVLANERR].name = "rxVlanErrors",
3961	/* gen1 - 64-bit */
3962	[IRDMA_HW_STAT_INDEX_IP4RXOCTS].name = "ip4InOctets",
3963	[IRDMA_HW_STAT_INDEX_IP4RXPKTS].name = "ip4InPkts",
3964	[IRDMA_HW_STAT_INDEX_IP4RXFRAGS].name = "ip4InReasmRqd",
3965	[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS].name = "ip4InMcastPkts",
3966	[IRDMA_HW_STAT_INDEX_IP4TXOCTS].name = "ip4OutOctets",
3967	[IRDMA_HW_STAT_INDEX_IP4TXPKTS].name = "ip4OutPkts",
3968	[IRDMA_HW_STAT_INDEX_IP4TXFRAGS].name = "ip4OutSegRqd",
3969	[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS].name = "ip4OutMcastPkts",
3970	[IRDMA_HW_STAT_INDEX_IP6RXOCTS].name = "ip6InOctets",
3971	[IRDMA_HW_STAT_INDEX_IP6RXPKTS].name = "ip6InPkts",
3972	[IRDMA_HW_STAT_INDEX_IP6RXFRAGS].name = "ip6InReasmRqd",
3973	[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS].name = "ip6InMcastPkts",
3974	[IRDMA_HW_STAT_INDEX_IP6TXOCTS].name = "ip6OutOctets",
3975	[IRDMA_HW_STAT_INDEX_IP6TXPKTS].name = "ip6OutPkts",
3976	[IRDMA_HW_STAT_INDEX_IP6TXFRAGS].name = "ip6OutSegRqd",
3977	[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS].name = "ip6OutMcastPkts",
3978	[IRDMA_HW_STAT_INDEX_TCPRXSEGS].name = "tcpInSegs",
3979	[IRDMA_HW_STAT_INDEX_TCPTXSEG].name = "tcpOutSegs",
3980	[IRDMA_HW_STAT_INDEX_RDMARXRDS].name = "iwInRdmaReads",
3981	[IRDMA_HW_STAT_INDEX_RDMARXSNDS].name = "iwInRdmaSends",
3982	[IRDMA_HW_STAT_INDEX_RDMARXWRS].name = "iwInRdmaWrites",
3983	[IRDMA_HW_STAT_INDEX_RDMATXRDS].name = "iwOutRdmaReads",
3984	[IRDMA_HW_STAT_INDEX_RDMATXSNDS].name = "iwOutRdmaSends",
3985	[IRDMA_HW_STAT_INDEX_RDMATXWRS].name = "iwOutRdmaWrites",
3986	[IRDMA_HW_STAT_INDEX_RDMAVBND].name = "iwRdmaBnd",
3987	[IRDMA_HW_STAT_INDEX_RDMAVINV].name = "iwRdmaInv",
3988
3989	/* gen2 - 32-bit */
3990	[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name = "cnpHandled",
3991	[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name = "cnpIgnored",
3992	[IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name = "cnpSent",
3993	/* gen2 - 64-bit */
3994	[IRDMA_HW_STAT_INDEX_IP4RXMCOCTS].name = "ip4InMcastOctets",
3995	[IRDMA_HW_STAT_INDEX_IP4TXMCOCTS].name = "ip4OutMcastOctets",
3996	[IRDMA_HW_STAT_INDEX_IP6RXMCOCTS].name = "ip6InMcastOctets",
3997	[IRDMA_HW_STAT_INDEX_IP6TXMCOCTS].name = "ip6OutMcastOctets",
3998	[IRDMA_HW_STAT_INDEX_UDPRXPKTS].name = "RxUDP",
3999	[IRDMA_HW_STAT_INDEX_UDPTXPKTS].name = "TxUDP",
4000	[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS].name = "RxECNMrkd",
4001
4002	};
4003
4004	static void irdma_get_dev_fw_str(struct ib_device *dev, char *str)
4005	{
4006	struct irdma_device *iwdev = to_iwdev(ibdev: dev);
4007
4008	snprintf(buf: str, IB_FW_VERSION_NAME_MAX, fmt: "%u.%u",
4009	irdma_fw_major_ver(dev: &iwdev->rf->sc_dev),
4010	irdma_fw_minor_ver(dev: &iwdev->rf->sc_dev));
4011	}
4012
4013	/**
4014	* irdma_alloc_hw_port_stats - Allocate a hw stats structure
4015	* @ibdev: device pointer from stack
4016	* @port_num: port number
4017	*/
4018	static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev,
4019	u32 port_num)
4020	{
4021	struct irdma_device *iwdev = to_iwdev(ibdev);
4022	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
4023
4024	int num_counters = dev->hw_attrs.max_stat_idx;
4025	unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;
4026
4027	return rdma_alloc_hw_stats_struct(descs: irdma_hw_stat_names, num_counters,
4028	lifespan);
4029	}
4030
4031	/**
4032	* irdma_get_hw_stats - Populates the rdma_hw_stats structure
4033	* @ibdev: device pointer from stack
4034	* @stats: stats pointer from stack
4035	* @port_num: port number
4036	* @index: which hw counter the stack is requesting we update
4037	*/
4038	static int irdma_get_hw_stats(struct ib_device *ibdev,
4039	struct rdma_hw_stats *stats, u32 port_num,
4040	int index)
4041	{
4042	struct irdma_device *iwdev = to_iwdev(ibdev);
4043	struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats;
4044
4045	if (iwdev->rf->rdma_ver >= IRDMA_GEN_2)
4046	irdma_cqp_gather_stats_cmd(dev: &iwdev->rf->sc_dev, pestat: iwdev->vsi.pestat, wait: true);
4047	else
4048	irdma_cqp_gather_stats_gen1(dev: &iwdev->rf->sc_dev, pestat: iwdev->vsi.pestat);
4049
4050	memcpy(&stats->value[0], hw_stats, sizeof(u64) * stats->num_counters);
4051
4052	return stats->num_counters;
4053	}
4054
4055	/**
4056	* irdma_query_gid - Query port GID
4057	* @ibdev: device pointer from stack
4058	* @port: port number
4059	* @index: Entry index
4060	* @gid: Global ID
4061	*/
4062	static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index,
4063	union ib_gid *gid)
4064	{
4065	struct irdma_device *iwdev = to_iwdev(ibdev);
4066
4067	memset(gid->raw, 0, sizeof(gid->raw));
4068	ether_addr_copy(dst: gid->raw, src: iwdev->netdev->dev_addr);
4069
4070	return 0;
4071	}
4072
4073	/**
4074	* mcast_list_add - Add a new mcast item to list
4075	* @rf: RDMA PCI function
4076	* @new_elem: pointer to element to add
4077	*/
4078	static void mcast_list_add(struct irdma_pci_f *rf,
4079	struct mc_table_list *new_elem)
4080	{
4081	list_add(new: &new_elem->list, head: &rf->mc_qht_list.list);
4082	}
4083
4084	/**
4085	* mcast_list_del - Remove an mcast item from list
4086	* @mc_qht_elem: pointer to mcast table list element
4087	*/
4088	static void mcast_list_del(struct mc_table_list *mc_qht_elem)
4089	{
4090	if (mc_qht_elem)
4091	list_del(entry: &mc_qht_elem->list);
4092	}
4093
4094	/**
4095	* mcast_list_lookup_ip - Search mcast list for address
4096	* @rf: RDMA PCI function
4097	* @ip_mcast: pointer to mcast IP address
4098	*/
4099	static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf,
4100	u32 *ip_mcast)
4101	{
4102	struct mc_table_list *mc_qht_el;
4103	struct list_head *pos, *q;
4104
4105	list_for_each_safe (pos, q, &rf->mc_qht_list.list) {
4106	mc_qht_el = list_entry(pos, struct mc_table_list, list);
4107	if (!memcmp(p: mc_qht_el->mc_info.dest_ip, q: ip_mcast,
4108	size: sizeof(mc_qht_el->mc_info.dest_ip)))
4109	return mc_qht_el;
4110	}
4111
4112	return NULL;
4113	}
4114
4115	/**
4116	* irdma_mcast_cqp_op - perform a mcast cqp operation
4117	* @iwdev: irdma device
4118	* @mc_grp_ctx: mcast group info
4119	* @op: operation
4120	*
4121	* returns error status
4122	*/
4123	static int irdma_mcast_cqp_op(struct irdma_device *iwdev,
4124	struct irdma_mcast_grp_info *mc_grp_ctx, u8 op)
4125	{
4126	struct cqp_cmds_info *cqp_info;
4127	struct irdma_cqp_request *cqp_request;
4128	int status;
4129
4130	cqp_request = irdma_alloc_and_get_cqp_request(cqp: &iwdev->rf->cqp, wait: true);
4131	if (!cqp_request)
4132	return -ENOMEM;
4133
4134	cqp_request->info.in.u.mc_create.info = *mc_grp_ctx;
4135	cqp_info = &cqp_request->info;
4136	cqp_info->cqp_cmd = op;
4137	cqp_info->post_sq = 1;
4138	cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request;
4139	cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp;
4140	status = irdma_handle_cqp_op(rf: iwdev->rf, cqp_request);
4141	irdma_put_cqp_request(cqp: &iwdev->rf->cqp, cqp_request);
4142
4143	return status;
4144	}
4145
4146	/**
4147	* irdma_mcast_mac - Get the multicast MAC for an IP address
4148	* @ip_addr: IPv4 or IPv6 address
4149	* @mac: pointer to result MAC address
4150	* @ipv4: flag indicating IPv4 or IPv6
4151	*
4152	*/
4153	void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4)
4154	{
4155	u8 *ip = (u8 *)ip_addr;
4156
4157	if (ipv4) {
4158	unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00,
4159	0x00, 0x00};
4160
4161	mac4[3] = ip[2] & 0x7F;
4162	mac4[4] = ip[1];
4163	mac4[5] = ip[0];
4164	ether_addr_copy(dst: mac, src: mac4);
4165	} else {
4166	unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00,
4167	0x00, 0x00};
4168
4169	mac6[2] = ip[3];
4170	mac6[3] = ip[2];
4171	mac6[4] = ip[1];
4172	mac6[5] = ip[0];
4173	ether_addr_copy(dst: mac, src: mac6);
4174	}
4175	}
4176
4177	/**
4178	* irdma_attach_mcast - attach a qp to a multicast group
4179	* @ibqp: ptr to qp
4180	* @ibgid: pointer to global ID
4181	* @lid: local ID
4182	*
4183	* returns error status
4184	*/
4185	static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)
4186	{
4187	struct irdma_qp *iwqp = to_iwqp(ibqp);
4188	struct irdma_device *iwdev = iwqp->iwdev;
4189	struct irdma_pci_f *rf = iwdev->rf;
4190	struct mc_table_list *mc_qht_elem;
4191	struct irdma_mcast_grp_ctx_entry_info mcg_info = {};
4192	unsigned long flags;
4193	u32 ip_addr[4] = {};
4194	u32 mgn;
4195	u32 no_mgs;
4196	int ret = 0;
4197	bool ipv4;
4198	u16 vlan_id;
4199	union irdma_sockaddr sgid_addr;
4200	unsigned char dmac[ETH_ALEN];
4201
4202	rdma_gid2ip(out: (struct sockaddr *)&sgid_addr, gid: ibgid);
4203
4204	if (!ipv6_addr_v4mapped(a: (struct in6_addr *)ibgid)) {
4205	irdma_copy_ip_ntohl(dst: ip_addr,
4206	src: sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);
4207	irdma_get_vlan_mac_ipv6(addr: ip_addr, vlan_id: &vlan_id, NULL);
4208	ipv4 = false;
4209	ibdev_dbg(&iwdev->ibdev,
4210	"VERBS: qp_id=%d, IP6address=%pI6\n", ibqp->qp_num,
4211	ip_addr);
4212	irdma_mcast_mac(ip_addr, mac: dmac, ipv4: false);
4213	} else {
4214	ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr);
4215	ipv4 = true;
4216	vlan_id = irdma_get_vlan_ipv4(addr: ip_addr);
4217	irdma_mcast_mac(ip_addr, mac: dmac, ipv4: true);
4218	ibdev_dbg(&iwdev->ibdev,
4219	"VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n",
4220	ibqp->qp_num, ip_addr, dmac);
4221	}
4222
4223	spin_lock_irqsave(&rf->qh_list_lock, flags);
4224	mc_qht_elem = mcast_list_lookup_ip(rf, ip_mcast: ip_addr);
4225	if (!mc_qht_elem) {
4226	struct irdma_dma_mem *dma_mem_mc;
4227
4228	spin_unlock_irqrestore(lock: &rf->qh_list_lock, flags);
4229	mc_qht_elem = kzalloc(size: sizeof(*mc_qht_elem), GFP_KERNEL);
4230	if (!mc_qht_elem)
4231	return -ENOMEM;
4232
4233	mc_qht_elem->mc_info.ipv4_valid = ipv4;
4234	memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr,
4235	sizeof(mc_qht_elem->mc_info.dest_ip));
4236	ret = irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_mcgs, max_rsrc: rf->max_mcg,
4237	req_rsrc_num: &mgn, next: &rf->next_mcg);
4238	if (ret) {
4239	kfree(objp: mc_qht_elem);
4240	return -ENOMEM;
4241	}
4242
4243	mc_qht_elem->mc_info.mgn = mgn;
4244	dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc;
4245	dma_mem_mc->size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX,
4246	IRDMA_HW_PAGE_SIZE);
4247	dma_mem_mc->va = dma_alloc_coherent(dev: rf->hw.device,
4248	size: dma_mem_mc->size,
4249	dma_handle: &dma_mem_mc->pa,
4250	GFP_KERNEL);
4251	if (!dma_mem_mc->va) {
4252	irdma_free_rsrc(rf, rsrc_array: rf->allocated_mcgs, rsrc_num: mgn);
4253	kfree(objp: mc_qht_elem);
4254	return -ENOMEM;
4255	}
4256
4257	mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn;
4258	memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr,
4259	sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr));
4260	mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4;
4261	mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id;
4262	if (vlan_id < VLAN_N_VID)
4263	mc_qht_elem->mc_grp_ctx.vlan_valid = true;
4264	mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->rf->sc_dev.hmc_fn_id;
4265	mc_qht_elem->mc_grp_ctx.qs_handle =
4266	iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle;
4267	ether_addr_copy(dst: mc_qht_elem->mc_grp_ctx.dest_mac_addr, src: dmac);
4268
4269	spin_lock_irqsave(&rf->qh_list_lock, flags);
4270	mcast_list_add(rf, new_elem: mc_qht_elem);
4271	} else {
4272	if (mc_qht_elem->mc_grp_ctx.no_of_mgs ==
4273	IRDMA_MAX_MGS_PER_CTX) {
4274	spin_unlock_irqrestore(lock: &rf->qh_list_lock, flags);
4275	return -ENOMEM;
4276	}
4277	}
4278
4279	mcg_info.qp_id = iwqp->ibqp.qp_num;
4280	no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs;
4281	irdma_sc_add_mcast_grp(ctx: &mc_qht_elem->mc_grp_ctx, mg: &mcg_info);
4282	spin_unlock_irqrestore(lock: &rf->qh_list_lock, flags);
4283
4284	/* Only if there is a change do we need to modify or create */
4285	if (!no_mgs) {
4286	ret = irdma_mcast_cqp_op(iwdev, mc_grp_ctx: &mc_qht_elem->mc_grp_ctx,
4287	op: IRDMA_OP_MC_CREATE);
4288	} else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) {
4289	ret = irdma_mcast_cqp_op(iwdev, mc_grp_ctx: &mc_qht_elem->mc_grp_ctx,
4290	op: IRDMA_OP_MC_MODIFY);
4291	} else {
4292	return 0;
4293	}
4294
4295	if (ret)
4296	goto error;
4297
4298	return 0;
4299
4300	error:
4301	irdma_sc_del_mcast_grp(ctx: &mc_qht_elem->mc_grp_ctx, mg: &mcg_info);
4302	if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) {
4303	mcast_list_del(mc_qht_elem);
4304	dma_free_coherent(dev: rf->hw.device,
4305	size: mc_qht_elem->mc_grp_ctx.dma_mem_mc.size,
4306	cpu_addr: mc_qht_elem->mc_grp_ctx.dma_mem_mc.va,
4307	dma_handle: mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa);
4308	mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL;
4309	irdma_free_rsrc(rf, rsrc_array: rf->allocated_mcgs,
4310	rsrc_num: mc_qht_elem->mc_grp_ctx.mg_id);
4311	kfree(objp: mc_qht_elem);
4312	}
4313
4314	return ret;
4315	}
4316
4317	/**
4318	* irdma_detach_mcast - detach a qp from a multicast group
4319	* @ibqp: ptr to qp
4320	* @ibgid: pointer to global ID
4321	* @lid: local ID
4322	*
4323	* returns error status
4324	*/
4325	static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)
4326	{
4327	struct irdma_qp *iwqp = to_iwqp(ibqp);
4328	struct irdma_device *iwdev = iwqp->iwdev;
4329	struct irdma_pci_f *rf = iwdev->rf;
4330	u32 ip_addr[4] = {};
4331	struct mc_table_list *mc_qht_elem;
4332	struct irdma_mcast_grp_ctx_entry_info mcg_info = {};
4333	int ret;
4334	unsigned long flags;
4335	union irdma_sockaddr sgid_addr;
4336
4337	rdma_gid2ip(out: (struct sockaddr *)&sgid_addr, gid: ibgid);
4338	if (!ipv6_addr_v4mapped(a: (struct in6_addr *)ibgid))
4339	irdma_copy_ip_ntohl(dst: ip_addr,
4340	src: sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);
4341	else
4342	ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr);
4343
4344	spin_lock_irqsave(&rf->qh_list_lock, flags);
4345	mc_qht_elem = mcast_list_lookup_ip(rf, ip_mcast: ip_addr);
4346	if (!mc_qht_elem) {
4347	spin_unlock_irqrestore(lock: &rf->qh_list_lock, flags);
4348	ibdev_dbg(&iwdev->ibdev,
4349	"VERBS: address not found MCG\n");
4350	return 0;
4351	}
4352
4353	mcg_info.qp_id = iwqp->ibqp.qp_num;
4354	irdma_sc_del_mcast_grp(ctx: &mc_qht_elem->mc_grp_ctx, mg: &mcg_info);
4355	if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) {
4356	mcast_list_del(mc_qht_elem);
4357	spin_unlock_irqrestore(lock: &rf->qh_list_lock, flags);
4358	ret = irdma_mcast_cqp_op(iwdev, mc_grp_ctx: &mc_qht_elem->mc_grp_ctx,
4359	op: IRDMA_OP_MC_DESTROY);
4360	if (ret) {
4361	ibdev_dbg(&iwdev->ibdev,
4362	"VERBS: failed MC_DESTROY MCG\n");
4363	spin_lock_irqsave(&rf->qh_list_lock, flags);
4364	mcast_list_add(rf, new_elem: mc_qht_elem);
4365	spin_unlock_irqrestore(lock: &rf->qh_list_lock, flags);
4366	return -EAGAIN;
4367	}
4368
4369	dma_free_coherent(dev: rf->hw.device,
4370	size: mc_qht_elem->mc_grp_ctx.dma_mem_mc.size,
4371	cpu_addr: mc_qht_elem->mc_grp_ctx.dma_mem_mc.va,
4372	dma_handle: mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa);
4373	mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL;
4374	irdma_free_rsrc(rf, rsrc_array: rf->allocated_mcgs,
4375	rsrc_num: mc_qht_elem->mc_grp_ctx.mg_id);
4376	kfree(objp: mc_qht_elem);
4377	} else {
4378	spin_unlock_irqrestore(lock: &rf->qh_list_lock, flags);
4379	ret = irdma_mcast_cqp_op(iwdev, mc_grp_ctx: &mc_qht_elem->mc_grp_ctx,
4380	op: IRDMA_OP_MC_MODIFY);
4381	if (ret) {
4382	ibdev_dbg(&iwdev->ibdev,
4383	"VERBS: failed Modify MCG\n");
4384	return ret;
4385	}
4386	}
4387
4388	return 0;
4389	}
4390
4391	static int irdma_create_hw_ah(struct irdma_device *iwdev, struct irdma_ah *ah, bool sleep)
4392	{
4393	struct irdma_pci_f *rf = iwdev->rf;
4394	int err;
4395
4396	err = irdma_alloc_rsrc(rf, rsrc_array: rf->allocated_ahs, max_rsrc: rf->max_ah, req_rsrc_num: &ah->sc_ah.ah_info.ah_idx,
4397	next: &rf->next_ah);
4398	if (err)
4399	return err;
4400
4401	err = irdma_ah_cqp_op(rf, sc_ah: &ah->sc_ah, cmd: IRDMA_OP_AH_CREATE, wait: sleep,
4402	callback_fcn: irdma_gsi_ud_qp_ah_cb, cb_param: &ah->sc_ah);
4403
4404	if (err) {
4405	ibdev_dbg(&iwdev->ibdev, "VERBS: CQP-OP Create AH fail");
4406	goto err_ah_create;
4407	}
4408
4409	if (!sleep) {
4410	int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD;
4411
4412	do {
4413	irdma_cqp_ce_handler(rf, cq: &rf->ccq.sc_cq);
4414	mdelay(1);
4415	} while (!ah->sc_ah.ah_info.ah_valid && --cnt);
4416
4417	if (!cnt) {
4418	ibdev_dbg(&iwdev->ibdev, "VERBS: CQP create AH timed out");
4419	err = -ETIMEDOUT;
4420	goto err_ah_create;
4421	}
4422	}
4423	return 0;
4424
4425	err_ah_create:
4426	irdma_free_rsrc(rf: iwdev->rf, rsrc_array: iwdev->rf->allocated_ahs, rsrc_num: ah->sc_ah.ah_info.ah_idx);
4427
4428	return err;
4429	}
4430
4431	static int irdma_setup_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr)
4432	{
4433	struct irdma_pd *pd = to_iwpd(ibpd: ibah->pd);
4434	struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah);
4435	struct rdma_ah_attr *ah_attr = attr->ah_attr;
4436	const struct ib_gid_attr *sgid_attr;
4437	struct irdma_device *iwdev = to_iwdev(ibdev: ibah->pd->device);
4438	struct irdma_pci_f *rf = iwdev->rf;
4439	struct irdma_sc_ah *sc_ah;
4440	struct irdma_ah_info *ah_info;
4441	union irdma_sockaddr sgid_addr, dgid_addr;
4442	int err;
4443	u8 dmac[ETH_ALEN];
4444
4445	ah->pd = pd;
4446	sc_ah = &ah->sc_ah;
4447	sc_ah->ah_info.vsi = &iwdev->vsi;
4448	irdma_sc_init_ah(dev: &rf->sc_dev, ah: sc_ah);
4449	ah->sgid_index = ah_attr->grh.sgid_index;
4450	sgid_attr = ah_attr->grh.sgid_attr;
4451	memcpy(&ah->dgid, &ah_attr->grh.dgid, sizeof(ah->dgid));
4452	rdma_gid2ip(out: (struct sockaddr *)&sgid_addr, gid: &sgid_attr->gid);
4453	rdma_gid2ip(out: (struct sockaddr *)&dgid_addr, gid: &ah_attr->grh.dgid);
4454	ah->av.attrs = *ah_attr;
4455	ah->av.net_type = rdma_gid_attr_network_type(attr: sgid_attr);
4456	ah_info = &sc_ah->ah_info;
4457	ah_info->pd_idx = pd->sc_pd.pd_id;
4458	if (ah_attr->ah_flags & IB_AH_GRH) {
4459	ah_info->flow_label = ah_attr->grh.flow_label;
4460	ah_info->hop_ttl = ah_attr->grh.hop_limit;
4461	ah_info->tc_tos = ah_attr->grh.traffic_class;
4462	}
4463
4464	ether_addr_copy(dst: dmac, src: ah_attr->roce.dmac);
4465	if (ah->av.net_type == RDMA_NETWORK_IPV4) {
4466	ah_info->ipv4_valid = true;
4467	ah_info->dest_ip_addr[0] =
4468	ntohl(dgid_addr.saddr_in.sin_addr.s_addr);
4469	ah_info->src_ip_addr[0] =
4470	ntohl(sgid_addr.saddr_in.sin_addr.s_addr);
4471	ah_info->do_lpbk = irdma_ipv4_is_lpb(loc_addr: ah_info->src_ip_addr[0],
4472	rem_addr: ah_info->dest_ip_addr[0]);
4473	if (ipv4_is_multicast(addr: dgid_addr.saddr_in.sin_addr.s_addr)) {
4474	ah_info->do_lpbk = true;
4475	irdma_mcast_mac(ip_addr: ah_info->dest_ip_addr, mac: dmac, ipv4: true);
4476	}
4477	} else {
4478	irdma_copy_ip_ntohl(dst: ah_info->dest_ip_addr,
4479	src: dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);
4480	irdma_copy_ip_ntohl(dst: ah_info->src_ip_addr,
4481	src: sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32);
4482	ah_info->do_lpbk = irdma_ipv6_is_lpb(loc_addr: ah_info->src_ip_addr,
4483	rem_addr: ah_info->dest_ip_addr);
4484	if (rdma_is_multicast_addr(addr: &dgid_addr.saddr_in6.sin6_addr)) {
4485	ah_info->do_lpbk = true;
4486	irdma_mcast_mac(ip_addr: ah_info->dest_ip_addr, mac: dmac, ipv4: false);
4487	}
4488	}
4489
4490	err = rdma_read_gid_l2_fields(attr: sgid_attr, vlan_id: &ah_info->vlan_tag,
4491	smac: ah_info->mac_addr);
4492	if (err)
4493	return err;
4494
4495	ah_info->dst_arpindex = irdma_add_arp(rf: iwdev->rf, ip: ah_info->dest_ip_addr,
4496	ipv4: ah_info->ipv4_valid, mac: dmac);
4497
4498	if (ah_info->dst_arpindex == -1)
4499	return -EINVAL;
4500
4501	if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode)
4502	ah_info->vlan_tag = 0;
4503
4504	if (ah_info->vlan_tag < VLAN_N_VID) {
4505	u8 prio = rt_tos2priority(tos: ah_info->tc_tos);
4506
4507	prio = irdma_roce_get_vlan_prio(attr: sgid_attr, prio);
4508
4509	ah_info->vlan_tag |= (u16)prio << VLAN_PRIO_SHIFT;
4510	ah_info->insert_vlan_tag = true;
4511	}
4512
4513	return 0;
4514	}
4515
4516	/**
4517	* irdma_ah_exists - Check for existing identical AH
4518	* @iwdev: irdma device
4519	* @new_ah: AH to check for
4520	*
4521	* returns true if AH is found, false if not found.
4522	*/
4523	static bool irdma_ah_exists(struct irdma_device *iwdev,
4524	struct irdma_ah *new_ah)
4525	{
4526	struct irdma_ah *ah;
4527	u32 key = new_ah->sc_ah.ah_info.dest_ip_addr[0] ^
4528	new_ah->sc_ah.ah_info.dest_ip_addr[1] ^
4529	new_ah->sc_ah.ah_info.dest_ip_addr[2] ^
4530	new_ah->sc_ah.ah_info.dest_ip_addr[3];
4531
4532	hash_for_each_possible(iwdev->ah_hash_tbl, ah, list, key) {
4533	/* Set ah_valid and ah_id the same so memcmp can work */
4534	new_ah->sc_ah.ah_info.ah_idx = ah->sc_ah.ah_info.ah_idx;
4535	new_ah->sc_ah.ah_info.ah_valid = ah->sc_ah.ah_info.ah_valid;
4536	if (!memcmp(p: &ah->sc_ah.ah_info, q: &new_ah->sc_ah.ah_info,
4537	size: sizeof(ah->sc_ah.ah_info))) {
4538	refcount_inc(r: &ah->refcnt);
4539	new_ah->parent_ah = ah;
4540	return true;
4541	}
4542	}
4543
4544	return false;
4545	}
4546
4547	/**
4548	* irdma_destroy_ah - Destroy address handle
4549	* @ibah: pointer to address handle
4550	* @ah_flags: flags for sleepable
4551	*/
4552	static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags)
4553	{
4554	struct irdma_device *iwdev = to_iwdev(ibdev: ibah->device);
4555	struct irdma_ah *ah = to_iwah(ibah);
4556
4557	if ((ah_flags & RDMA_DESTROY_AH_SLEEPABLE) && ah->parent_ah) {
4558	mutex_lock(&iwdev->ah_tbl_lock);
4559	if (!refcount_dec_and_test(r: &ah->parent_ah->refcnt)) {
4560	mutex_unlock(lock: &iwdev->ah_tbl_lock);
4561	return 0;
4562	}
4563	hash_del(node: &ah->parent_ah->list);
4564	kfree(objp: ah->parent_ah);
4565	mutex_unlock(lock: &iwdev->ah_tbl_lock);
4566	}
4567
4568	irdma_ah_cqp_op(rf: iwdev->rf, sc_ah: &ah->sc_ah, cmd: IRDMA_OP_AH_DESTROY,
4569	wait: false, NULL, cb_param: ah);
4570
4571	irdma_free_rsrc(rf: iwdev->rf, rsrc_array: iwdev->rf->allocated_ahs,
4572	rsrc_num: ah->sc_ah.ah_info.ah_idx);
4573
4574	return 0;
4575	}
4576
4577	/**
4578	* irdma_create_user_ah - create user address handle
4579	* @ibah: address handle
4580	* @attr: address handle attributes
4581	* @udata: User data
4582	*
4583	* returns 0 on success, error otherwise
4584	*/
4585	static int irdma_create_user_ah(struct ib_ah *ibah,
4586	struct rdma_ah_init_attr *attr,
4587	struct ib_udata *udata)
4588	{
4589	#define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd)
4590	struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah);
4591	struct irdma_device *iwdev = to_iwdev(ibdev: ibah->pd->device);
4592	struct irdma_create_ah_resp uresp;
4593	struct irdma_ah *parent_ah;
4594	int err;
4595
4596	if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN)
4597	return -EINVAL;
4598
4599	err = irdma_setup_ah(ibah, attr);
4600	if (err)
4601	return err;
4602	mutex_lock(&iwdev->ah_tbl_lock);
4603	if (!irdma_ah_exists(iwdev, new_ah: ah)) {
4604	err = irdma_create_hw_ah(iwdev, ah, sleep: true);
4605	if (err) {
4606	mutex_unlock(lock: &iwdev->ah_tbl_lock);
4607	return err;
4608	}
4609	/* Add new AH to list */
4610	parent_ah = kmemdup(p: ah, size: sizeof(*ah), GFP_KERNEL);
4611	if (parent_ah) {
4612	u32 key = parent_ah->sc_ah.ah_info.dest_ip_addr[0] ^
4613	parent_ah->sc_ah.ah_info.dest_ip_addr[1] ^
4614	parent_ah->sc_ah.ah_info.dest_ip_addr[2] ^
4615	parent_ah->sc_ah.ah_info.dest_ip_addr[3];
4616
4617	ah->parent_ah = parent_ah;
4618	hash_add(iwdev->ah_hash_tbl, &parent_ah->list, key);
4619	refcount_set(r: &parent_ah->refcnt, n: 1);
4620	}
4621	}
4622	mutex_unlock(lock: &iwdev->ah_tbl_lock);
4623
4624	uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
4625	err = ib_copy_to_udata(udata, src: &uresp, min(sizeof(uresp), udata->outlen));
4626	if (err)
4627	irdma_destroy_ah(ibah, ah_flags: attr->flags);
4628
4629	return err;
4630	}
4631
4632	/**
4633	* irdma_create_ah - create address handle
4634	* @ibah: address handle
4635	* @attr: address handle attributes
4636	* @udata: NULL
4637	*
4638	* returns 0 on success, error otherwise
4639	*/
4640	static int irdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr,
4641	struct ib_udata *udata)
4642	{
4643	struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah);
4644	struct irdma_device *iwdev = to_iwdev(ibdev: ibah->pd->device);
4645	int err;
4646
4647	err = irdma_setup_ah(ibah, attr);
4648	if (err)
4649	return err;
4650	err = irdma_create_hw_ah(iwdev, ah, sleep: attr->flags & RDMA_CREATE_AH_SLEEPABLE);
4651
4652	return err;
4653	}
4654
4655	/**
4656	* irdma_query_ah - Query address handle
4657	* @ibah: pointer to address handle
4658	* @ah_attr: address handle attributes
4659	*/
4660	static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
4661	{
4662	struct irdma_ah *ah = to_iwah(ibah);
4663
4664	memset(ah_attr, 0, sizeof(*ah_attr));
4665	if (ah->av.attrs.ah_flags & IB_AH_GRH) {
4666	ah_attr->ah_flags = IB_AH_GRH;
4667	ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label;
4668	ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos;
4669	ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl;
4670	ah_attr->grh.sgid_index = ah->sgid_index;
4671	memcpy(&ah_attr->grh.dgid, &ah->dgid,
4672	sizeof(ah_attr->grh.dgid));
4673	}
4674
4675	return 0;
4676	}
4677
4678	static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev,
4679	u32 port_num)
4680	{
4681	return IB_LINK_LAYER_ETHERNET;
4682	}
4683
4684	static const struct ib_device_ops irdma_roce_dev_ops = {
4685	.attach_mcast = irdma_attach_mcast,
4686	.create_ah = irdma_create_ah,
4687	.create_user_ah = irdma_create_user_ah,
4688	.destroy_ah = irdma_destroy_ah,
4689	.detach_mcast = irdma_detach_mcast,
4690	.get_link_layer = irdma_get_link_layer,
4691	.get_port_immutable = irdma_roce_port_immutable,
4692	.modify_qp = irdma_modify_qp_roce,
4693	.query_ah = irdma_query_ah,
4694	.query_pkey = irdma_query_pkey,
4695	};
4696
4697	static const struct ib_device_ops irdma_iw_dev_ops = {
4698	.get_port_immutable = irdma_iw_port_immutable,
4699	.iw_accept = irdma_accept,
4700	.iw_add_ref = irdma_qp_add_ref,
4701	.iw_connect = irdma_connect,
4702	.iw_create_listen = irdma_create_listen,
4703	.iw_destroy_listen = irdma_destroy_listen,
4704	.iw_get_qp = irdma_get_qp,
4705	.iw_reject = irdma_reject,
4706	.iw_rem_ref = irdma_qp_rem_ref,
4707	.modify_qp = irdma_modify_qp,
4708	.query_gid = irdma_query_gid,
4709	};
4710
4711	static const struct ib_device_ops irdma_dev_ops = {
4712	.owner = THIS_MODULE,
4713	.driver_id = RDMA_DRIVER_IRDMA,
4714	.uverbs_abi_ver = IRDMA_ABI_VER,
4715
4716	.alloc_hw_port_stats = irdma_alloc_hw_port_stats,
4717	.alloc_mr = irdma_alloc_mr,
4718	.alloc_mw = irdma_alloc_mw,
4719	.alloc_pd = irdma_alloc_pd,
4720	.alloc_ucontext = irdma_alloc_ucontext,
4721	.create_cq = irdma_create_cq,
4722	.create_qp = irdma_create_qp,
4723	.dealloc_driver = irdma_ib_dealloc_device,
4724	.dealloc_mw = irdma_dealloc_mw,
4725	.dealloc_pd = irdma_dealloc_pd,
4726	.dealloc_ucontext = irdma_dealloc_ucontext,
4727	.dereg_mr = irdma_dereg_mr,
4728	.destroy_cq = irdma_destroy_cq,
4729	.destroy_qp = irdma_destroy_qp,
4730	.disassociate_ucontext = irdma_disassociate_ucontext,
4731	.get_dev_fw_str = irdma_get_dev_fw_str,
4732	.get_dma_mr = irdma_get_dma_mr,
4733	.get_hw_stats = irdma_get_hw_stats,
4734	.map_mr_sg = irdma_map_mr_sg,
4735	.mmap = irdma_mmap,
4736	.mmap_free = irdma_mmap_free,
4737	.poll_cq = irdma_poll_cq,
4738	.post_recv = irdma_post_recv,
4739	.post_send = irdma_post_send,
4740	.query_device = irdma_query_device,
4741	.query_port = irdma_query_port,
4742	.query_qp = irdma_query_qp,
4743	.reg_user_mr = irdma_reg_user_mr,
4744	.reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf,
4745	.rereg_user_mr = irdma_rereg_user_mr,
4746	.req_notify_cq = irdma_req_notify_cq,
4747	.resize_cq = irdma_resize_cq,
4748	INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),
4749	INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext),
4750	INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah),
4751	INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq),
4752	INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw),
4753	INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp),
4754	};
4755
4756	/**
4757	* irdma_init_roce_device - initialization of roce rdma device
4758	* @iwdev: irdma device
4759	*/
4760	static void irdma_init_roce_device(struct irdma_device *iwdev)
4761	{
4762	iwdev->ibdev.node_type = RDMA_NODE_IB_CA;
4763	addrconf_addr_eui48(eui: (u8 *)&iwdev->ibdev.node_guid,
4764	addr: iwdev->netdev->dev_addr);
4765	ib_set_device_ops(device: &iwdev->ibdev, ops: &irdma_roce_dev_ops);
4766	}
4767
4768	/**
4769	* irdma_init_iw_device - initialization of iwarp rdma device
4770	* @iwdev: irdma device
4771	*/
4772	static void irdma_init_iw_device(struct irdma_device *iwdev)
4773	{
4774	struct net_device *netdev = iwdev->netdev;
4775
4776	iwdev->ibdev.node_type = RDMA_NODE_RNIC;
4777	addrconf_addr_eui48(eui: (u8 *)&iwdev->ibdev.node_guid,
4778	addr: netdev->dev_addr);
4779	memcpy(iwdev->ibdev.iw_ifname, netdev->name,
4780	sizeof(iwdev->ibdev.iw_ifname));
4781	ib_set_device_ops(device: &iwdev->ibdev, ops: &irdma_iw_dev_ops);
4782	}
4783
4784	/**
4785	* irdma_init_rdma_device - initialization of rdma device
4786	* @iwdev: irdma device
4787	*/
4788	static void irdma_init_rdma_device(struct irdma_device *iwdev)
4789	{
4790	struct pci_dev *pcidev = iwdev->rf->pcidev;
4791
4792	if (iwdev->roce_mode)
4793	irdma_init_roce_device(iwdev);
4794	else
4795	irdma_init_iw_device(iwdev);
4796
4797	iwdev->ibdev.phys_port_cnt = 1;
4798	iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count;
4799	iwdev->ibdev.dev.parent = &pcidev->dev;
4800	ib_set_device_ops(device: &iwdev->ibdev, ops: &irdma_dev_ops);
4801	}
4802
4803	/**
4804	* irdma_port_ibevent - indicate port event
4805	* @iwdev: irdma device
4806	*/
4807	void irdma_port_ibevent(struct irdma_device *iwdev)
4808	{
4809	struct ib_event event;
4810
4811	event.device = &iwdev->ibdev;
4812	event.element.port_num = 1;
4813	event.event =
4814	iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
4815	ib_dispatch_event(event: &event);
4816	}
4817
4818	/**
4819	* irdma_ib_unregister_device - unregister rdma device from IB
4820	* core
4821	* @iwdev: irdma device
4822	*/
4823	void irdma_ib_unregister_device(struct irdma_device *iwdev)
4824	{
4825	iwdev->iw_status = 0;
4826	irdma_port_ibevent(iwdev);
4827	ib_unregister_device(device: &iwdev->ibdev);
4828	}
4829
4830	/**
4831	* irdma_ib_register_device - register irdma device to IB core
4832	* @iwdev: irdma device
4833	*/
4834	int irdma_ib_register_device(struct irdma_device *iwdev)
4835	{
4836	int ret;
4837
4838	irdma_init_rdma_device(iwdev);
4839
4840	ret = ib_device_set_netdev(ib_dev: &iwdev->ibdev, ndev: iwdev->netdev, port: 1);
4841	if (ret)
4842	goto error;
4843	dma_set_max_seg_size(dev: iwdev->rf->hw.device, UINT_MAX);
4844	ret = ib_register_device(device: &iwdev->ibdev, name: "irdma%d", dma_device: iwdev->rf->hw.device);
4845	if (ret)
4846	goto error;
4847
4848	iwdev->iw_status = 1;
4849	irdma_port_ibevent(iwdev);
4850
4851	return 0;
4852
4853	error:
4854	if (ret)
4855	ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n");
4856
4857	return ret;
4858	}
4859
4860	/**
4861	* irdma_ib_dealloc_device
4862	* @ibdev: ib device
4863	*
4864	* callback from ibdev dealloc_driver to deallocate resources
4865	* unber irdma device
4866	*/
4867	void irdma_ib_dealloc_device(struct ib_device *ibdev)
4868	{
4869	struct irdma_device *iwdev = to_iwdev(ibdev);
4870
4871	irdma_rt_deinit_hw(iwdev);
4872	irdma_ctrl_deinit_hw(rf: iwdev->rf);
4873	kfree(objp: iwdev->rf);
4874	}
4875