1	/* This file is part of the Emulex RoCE Device Driver for
2	* RoCE (RDMA over Converged Ethernet) adapters.
3	* Copyright (C) 2012-2015 Emulex. All rights reserved.
4	* EMULEX and SLI are trademarks of Emulex.
5	* www.emulex.com
6	*
7	* This software is available to you under a choice of one of two licenses.
8	* You may choose to be licensed under the terms of the GNU General Public
9	* License (GPL) Version 2, available from the file COPYING in the main
10	* directory of this source tree, or the BSD license below:
11	*
12	* Redistribution and use in source and binary forms, with or without
13	* modification, are permitted provided that the following conditions
14	* are met:
15	*
16	* - Redistributions of source code must retain the above copyright notice,
17	* this list of conditions and the following disclaimer.
18	*
19	* - Redistributions in binary form must reproduce the above copyright
20	* notice, this list of conditions and the following disclaimer in
21	* the documentation and/or other materials provided with the distribution.
22	*
23	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE
25	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
30	* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
31	* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
32	* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
33	* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34	*
35	* Contact Information:
36	* linux-drivers@emulex.com
37	*
38	* Emulex
39	* 3333 Susan Street
40	* Costa Mesa, CA 92626
41	*/
42
43	#include <linux/dma-mapping.h>
44	#include <net/addrconf.h>
45	#include <rdma/ib_verbs.h>
46	#include <rdma/ib_user_verbs.h>
47	#include <rdma/iw_cm.h>
48	#include <rdma/ib_umem.h>
49	#include <rdma/ib_addr.h>
50	#include <rdma/ib_cache.h>
51	#include <rdma/uverbs_ioctl.h>
52
53	#include "ocrdma.h"
54	#include "ocrdma_hw.h"
55	#include "ocrdma_verbs.h"
56	#include <rdma/ocrdma-abi.h>
57
58	int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
59	{
60	if (index > 0)
61	return -EINVAL;
62
63	*pkey = 0xffff;
64	return 0;
65	}
66
67	int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
68	struct ib_udata *uhw)
69	{
70	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
71
72	if (uhw->inlen || uhw->outlen)
73	return -EINVAL;
74
75	memset(attr, 0, sizeof *attr);
76	memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],
77	min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));
78	addrconf_addr_eui48(eui: (u8 *)&attr->sys_image_guid,
79	addr: dev->nic_info.mac_addr);
80	attr->max_mr_size = dev->attr.max_mr_size;
81	attr->page_size_cap = 0xffff000;
82	attr->vendor_id = dev->nic_info.pdev->vendor;
83	attr->vendor_part_id = dev->nic_info.pdev->device;
84	attr->hw_ver = dev->asic_id;
85	attr->max_qp = dev->attr.max_qp;
86	attr->max_ah = OCRDMA_MAX_AH;
87	attr->max_qp_wr = dev->attr.max_wqe;
88
89	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
90	IB_DEVICE_RC_RNR_NAK_GEN |
91	IB_DEVICE_SHUTDOWN_PORT |
92	IB_DEVICE_SYS_IMAGE_GUID |
93	IB_DEVICE_MEM_MGT_EXTENSIONS;
94	attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
95	attr->max_send_sge = dev->attr.max_send_sge;
96	attr->max_recv_sge = dev->attr.max_recv_sge;
97	attr->max_sge_rd = dev->attr.max_rdma_sge;
98	attr->max_cq = dev->attr.max_cq;
99	attr->max_cqe = dev->attr.max_cqe;
100	attr->max_mr = dev->attr.max_mr;
101	attr->max_mw = dev->attr.max_mw;
102	attr->max_pd = dev->attr.max_pd;
103	attr->atomic_cap = 0;
104	attr->max_qp_rd_atom =
105	min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);
106	attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;
107	attr->max_srq = dev->attr.max_srq;
108	attr->max_srq_sge = dev->attr.max_srq_sge;
109	attr->max_srq_wr = dev->attr.max_rqe;
110	attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay;
111	attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr;
112	attr->max_pkeys = 1;
113	return 0;
114	}
115
116	static inline void get_link_speed_and_width(struct ocrdma_dev *dev,
117	u16 *ib_speed, u8 *ib_width)
118	{
119	int status;
120	u8 speed;
121
122	status = ocrdma_mbx_get_link_speed(dev, lnk_speed: &speed, NULL);
123	if (status)
124	speed = OCRDMA_PHYS_LINK_SPEED_ZERO;
125
126	switch (speed) {
127	case OCRDMA_PHYS_LINK_SPEED_1GBPS:
128	*ib_speed = IB_SPEED_SDR;
129	*ib_width = IB_WIDTH_1X;
130	break;
131
132	case OCRDMA_PHYS_LINK_SPEED_10GBPS:
133	*ib_speed = IB_SPEED_QDR;
134	*ib_width = IB_WIDTH_1X;
135	break;
136
137	case OCRDMA_PHYS_LINK_SPEED_20GBPS:
138	*ib_speed = IB_SPEED_DDR;
139	*ib_width = IB_WIDTH_4X;
140	break;
141
142	case OCRDMA_PHYS_LINK_SPEED_40GBPS:
143	*ib_speed = IB_SPEED_QDR;
144	*ib_width = IB_WIDTH_4X;
145	break;
146
147	default:
148	/* Unsupported */
149	*ib_speed = IB_SPEED_SDR;
150	*ib_width = IB_WIDTH_1X;
151	}
152	}
153
154	int ocrdma_query_port(struct ib_device *ibdev,
155	u32 port, struct ib_port_attr *props)
156	{
157	enum ib_port_state port_state;
158	struct ocrdma_dev *dev;
159	struct net_device *netdev;
160
161	/* props being zeroed by the caller, avoid zeroing it here */
162	dev = get_ocrdma_dev(ibdev);
163	netdev = dev->nic_info.netdev;
164	if (netif_running(dev: netdev) && netif_oper_up(dev: netdev)) {
165	port_state = IB_PORT_ACTIVE;
166	props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
167	} else {
168	port_state = IB_PORT_DOWN;
169	props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
170	}
171	props->max_mtu = IB_MTU_4096;
172	props->active_mtu = iboe_get_mtu(mtu: netdev->mtu);
173	props->lid = 0;
174	props->lmc = 0;
175	props->sm_lid = 0;
176	props->sm_sl = 0;
177	props->state = port_state;
178	props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
179	IB_PORT_DEVICE_MGMT_SUP |
180	IB_PORT_VENDOR_CLASS_SUP;
181	props->ip_gids = true;
182	props->gid_tbl_len = OCRDMA_MAX_SGID;
183	props->pkey_tbl_len = 1;
184	props->bad_pkey_cntr = 0;
185	props->qkey_viol_cntr = 0;
186	get_link_speed_and_width(dev, ib_speed: &props->active_speed,
187	ib_width: &props->active_width);
188	props->max_msg_sz = 0x80000000;
189	props->max_vl_num = 4;
190	return 0;
191	}
192
193	static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
194	unsigned long len)
195	{
196	struct ocrdma_mm *mm;
197
198	mm = kzalloc(sizeof(*mm), GFP_KERNEL);
199	if (mm == NULL)
200	return -ENOMEM;
201	mm->key.phy_addr = phy_addr;
202	mm->key.len = len;
203	INIT_LIST_HEAD(list: &mm->entry);
204
205	mutex_lock(&uctx->mm_list_lock);
206	list_add_tail(new: &mm->entry, head: &uctx->mm_head);
207	mutex_unlock(lock: &uctx->mm_list_lock);
208	return 0;
209	}
210
211	static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
212	unsigned long len)
213	{
214	struct ocrdma_mm *mm, *tmp;
215
216	mutex_lock(&uctx->mm_list_lock);
217	list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
218	if (len != mm->key.len && phy_addr != mm->key.phy_addr)
219	continue;
220
221	list_del(entry: &mm->entry);
222	kfree(objp: mm);
223	break;
224	}
225	mutex_unlock(lock: &uctx->mm_list_lock);
226	}
227
228	static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
229	unsigned long len)
230	{
231	bool found = false;
232	struct ocrdma_mm *mm;
233
234	mutex_lock(&uctx->mm_list_lock);
235	list_for_each_entry(mm, &uctx->mm_head, entry) {
236	if (len != mm->key.len && phy_addr != mm->key.phy_addr)
237	continue;
238
239	found = true;
240	break;
241	}
242	mutex_unlock(lock: &uctx->mm_list_lock);
243	return found;
244	}
245
246
247	static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool)
248	{
249	u16 pd_bitmap_idx = 0;
250	unsigned long *pd_bitmap;
251
252	if (dpp_pool) {
253	pd_bitmap = dev->pd_mgr->pd_dpp_bitmap;
254	pd_bitmap_idx = find_first_zero_bit(addr: pd_bitmap,
255	size: dev->pd_mgr->max_dpp_pd);
256	__set_bit(pd_bitmap_idx, pd_bitmap);
257	dev->pd_mgr->pd_dpp_count++;
258	if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh)
259	dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count;
260	} else {
261	pd_bitmap = dev->pd_mgr->pd_norm_bitmap;
262	pd_bitmap_idx = find_first_zero_bit(addr: pd_bitmap,
263	size: dev->pd_mgr->max_normal_pd);
264	__set_bit(pd_bitmap_idx, pd_bitmap);
265	dev->pd_mgr->pd_norm_count++;
266	if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh)
267	dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count;
268	}
269	return pd_bitmap_idx;
270	}
271
272	static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id,
273	bool dpp_pool)
274	{
275	u16 pd_count;
276	u16 pd_bit_index;
277
278	pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count :
279	dev->pd_mgr->pd_norm_count;
280	if (pd_count == 0)
281	return -EINVAL;
282
283	if (dpp_pool) {
284	pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start;
285	if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) {
286	return -EINVAL;
287	} else {
288	__clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap);
289	dev->pd_mgr->pd_dpp_count--;
290	}
291	} else {
292	pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start;
293	if (pd_bit_index >= dev->pd_mgr->max_normal_pd) {
294	return -EINVAL;
295	} else {
296	__clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap);
297	dev->pd_mgr->pd_norm_count--;
298	}
299	}
300
301	return 0;
302	}
303
304	static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id,
305	bool dpp_pool)
306	{
307	int status;
308
309	mutex_lock(&dev->dev_lock);
310	status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool);
311	mutex_unlock(lock: &dev->dev_lock);
312	return status;
313	}
314
315	static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd)
316	{
317	u16 pd_idx = 0;
318	int status = 0;
319
320	mutex_lock(&dev->dev_lock);
321	if (pd->dpp_enabled) {
322	/* try allocating DPP PD, if not available then normal PD */
323	if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) {
324	pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, dpp_pool: true);
325	pd->id = dev->pd_mgr->pd_dpp_start + pd_idx;
326	pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx;
327	} else if (dev->pd_mgr->pd_norm_count <
328	dev->pd_mgr->max_normal_pd) {
329	pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, dpp_pool: false);
330	pd->id = dev->pd_mgr->pd_norm_start + pd_idx;
331	pd->dpp_enabled = false;
332	} else {
333	status = -EINVAL;
334	}
335	} else {
336	if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) {
337	pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, dpp_pool: false);
338	pd->id = dev->pd_mgr->pd_norm_start + pd_idx;
339	} else {
340	status = -EINVAL;
341	}
342	}
343	mutex_unlock(lock: &dev->dev_lock);
344	return status;
345	}
346
347	/*
348	* NOTE:
349	*
350	* ocrdma_ucontext must be used here because this function is also
351	* called from ocrdma_alloc_ucontext where ib_udata does not have
352	* valid ib_ucontext pointer. ib_uverbs_get_context does not call
353	* uobj_{alloc|get_xxx} helpers which are used to store the
354	* ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so
355	* ib_udata does NOT imply valid ib_ucontext here!
356	*/
357	static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
358	struct ocrdma_ucontext *uctx,
359	struct ib_udata *udata)
360	{
361	int status;
362
363	if (udata && uctx && dev->attr.max_dpp_pds) {
364	pd->dpp_enabled =
365	ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R;
366	pd->num_dpp_qp =
367	pd->dpp_enabled ? (dev->nic_info.db_page_size /
368	dev->attr.wqe_size) : 0;
369	}
370
371	if (dev->pd_mgr->pd_prealloc_valid)
372	return ocrdma_get_pd_num(dev, pd);
373
374	retry:
375	status = ocrdma_mbx_alloc_pd(dev, pd);
376	if (status) {
377	if (pd->dpp_enabled) {
378	pd->dpp_enabled = false;
379	pd->num_dpp_qp = 0;
380	goto retry;
381	}
382	return status;
383	}
384
385	return 0;
386	}
387
388	static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx,
389	struct ocrdma_pd *pd)
390	{
391	return (uctx->cntxt_pd == pd);
392	}
393
394	static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev,
395	struct ocrdma_pd *pd)
396	{
397	if (dev->pd_mgr->pd_prealloc_valid)
398	ocrdma_put_pd_num(dev, pd_id: pd->id, dpp_pool: pd->dpp_enabled);
399	else
400	ocrdma_mbx_dealloc_pd(dev, pd);
401	}
402
403	static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev,
404	struct ocrdma_ucontext *uctx,
405	struct ib_udata *udata)
406	{
407	struct ib_device *ibdev = &dev->ibdev;
408	struct ib_pd *pd;
409	int status;
410
411	pd = rdma_zalloc_drv_obj(ibdev, ib_pd);
412	if (!pd)
413	return -ENOMEM;
414
415	pd->device = ibdev;
416	uctx->cntxt_pd = get_ocrdma_pd(ibpd: pd);
417
418	status = _ocrdma_alloc_pd(dev, pd: uctx->cntxt_pd, uctx, udata);
419	if (status) {
420	kfree(objp: uctx->cntxt_pd);
421	goto err;
422	}
423
424	uctx->cntxt_pd->uctx = uctx;
425	uctx->cntxt_pd->ibpd.device = &dev->ibdev;
426	err:
427	return status;
428	}
429
430	static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx)
431	{
432	struct ocrdma_pd *pd = uctx->cntxt_pd;
433	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: pd->ibpd.device);
434
435	if (uctx->pd_in_use) {
436	pr_err("%s(%d) Freeing in use pdid=0x%x.\n",
437	__func__, dev->id, pd->id);
438	}
439	uctx->cntxt_pd = NULL;
440	_ocrdma_dealloc_pd(dev, pd);
441	kfree(objp: pd);
442	}
443
444	static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx)
445	{
446	struct ocrdma_pd *pd = NULL;
447
448	mutex_lock(&uctx->mm_list_lock);
449	if (!uctx->pd_in_use) {
450	uctx->pd_in_use = true;
451	pd = uctx->cntxt_pd;
452	}
453	mutex_unlock(lock: &uctx->mm_list_lock);
454
455	return pd;
456	}
457
458	static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx)
459	{
460	mutex_lock(&uctx->mm_list_lock);
461	uctx->pd_in_use = false;
462	mutex_unlock(lock: &uctx->mm_list_lock);
463	}
464
465	int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
466	{
467	struct ib_device *ibdev = uctx->device;
468	int status;
469	struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(ibucontext: uctx);
470	struct ocrdma_alloc_ucontext_resp resp = {};
471	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
472	struct pci_dev *pdev = dev->nic_info.pdev;
473	u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE);
474
475	if (!udata)
476	return -EFAULT;
477	INIT_LIST_HEAD(list: &ctx->mm_head);
478	mutex_init(&ctx->mm_list_lock);
479
480	ctx->ah_tbl.va = dma_alloc_coherent(dev: &pdev->dev, size: map_len,
481	dma_handle: &ctx->ah_tbl.pa, GFP_KERNEL);
482	if (!ctx->ah_tbl.va)
483	return -ENOMEM;
484
485	ctx->ah_tbl.len = map_len;
486
487	resp.ah_tbl_len = ctx->ah_tbl.len;
488	resp.ah_tbl_page = virt_to_phys(address: ctx->ah_tbl.va);
489
490	status = ocrdma_add_mmap(uctx: ctx, phy_addr: resp.ah_tbl_page, len: resp.ah_tbl_len);
491	if (status)
492	goto map_err;
493
494	status = ocrdma_alloc_ucontext_pd(dev, uctx: ctx, udata);
495	if (status)
496	goto pd_err;
497
498	resp.dev_id = dev->id;
499	resp.max_inline_data = dev->attr.max_inline_data;
500	resp.wqe_size = dev->attr.wqe_size;
501	resp.rqe_size = dev->attr.rqe_size;
502	resp.dpp_wqe_size = dev->attr.wqe_size;
503
504	memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver));
505	status = ib_copy_to_udata(udata, src: &resp, len: sizeof(resp));
506	if (status)
507	goto cpy_err;
508	return 0;
509
510	cpy_err:
511	ocrdma_dealloc_ucontext_pd(uctx: ctx);
512	pd_err:
513	ocrdma_del_mmap(uctx: ctx, phy_addr: ctx->ah_tbl.pa, len: ctx->ah_tbl.len);
514	map_err:
515	dma_free_coherent(dev: &pdev->dev, size: ctx->ah_tbl.len, cpu_addr: ctx->ah_tbl.va,
516	dma_handle: ctx->ah_tbl.pa);
517	return status;
518	}
519
520	void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)
521	{
522	struct ocrdma_mm *mm, *tmp;
523	struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibucontext: ibctx);
524	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibctx->device);
525	struct pci_dev *pdev = dev->nic_info.pdev;
526
527	ocrdma_dealloc_ucontext_pd(uctx);
528
529	ocrdma_del_mmap(uctx, phy_addr: uctx->ah_tbl.pa, len: uctx->ah_tbl.len);
530	dma_free_coherent(dev: &pdev->dev, size: uctx->ah_tbl.len, cpu_addr: uctx->ah_tbl.va,
531	dma_handle: uctx->ah_tbl.pa);
532
533	list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
534	list_del(entry: &mm->entry);
535	kfree(objp: mm);
536	}
537	}
538
539	int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
540	{
541	struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(ibucontext: context);
542	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: context->device);
543	unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
544	u64 unmapped_db = (u64) dev->nic_info.unmapped_db;
545	unsigned long len = (vma->vm_end - vma->vm_start);
546	int status;
547	bool found;
548
549	if (vma->vm_start & (PAGE_SIZE - 1))
550	return -EINVAL;
551	found = ocrdma_search_mmap(uctx: ucontext, phy_addr: vma->vm_pgoff << PAGE_SHIFT, len);
552	if (!found)
553	return -EINVAL;
554
555	if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
556	dev->nic_info.db_total_size)) &&
557	(len <= dev->nic_info.db_page_size)) {
558	if (vma->vm_flags & VM_READ)
559	return -EPERM;
560
561	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
562	status = io_remap_pfn_range(vma, addr: vma->vm_start, orig_pfn: vma->vm_pgoff,
563	size: len, orig_prot: vma->vm_page_prot);
564	} else if (dev->nic_info.dpp_unmapped_len &&
565	(vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) &&
566	(vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr +
567	dev->nic_info.dpp_unmapped_len)) &&
568	(len <= dev->nic_info.dpp_unmapped_len)) {
569	if (vma->vm_flags & VM_READ)
570	return -EPERM;
571
572	vma->vm_page_prot = pgprot_writecombine(prot: vma->vm_page_prot);
573	status = io_remap_pfn_range(vma, addr: vma->vm_start, orig_pfn: vma->vm_pgoff,
574	size: len, orig_prot: vma->vm_page_prot);
575	} else {
576	status = remap_pfn_range(vma, addr: vma->vm_start,
577	pfn: vma->vm_pgoff, size: len, pgprot: vma->vm_page_prot);
578	}
579	return status;
580	}
581
582	static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
583	struct ib_udata *udata)
584	{
585	int status;
586	u64 db_page_addr;
587	u64 dpp_page_addr = 0;
588	u32 db_page_size;
589	struct ocrdma_alloc_pd_uresp rsp;
590	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
591	udata, struct ocrdma_ucontext, ibucontext);
592
593	memset(&rsp, 0, sizeof(rsp));
594	rsp.id = pd->id;
595	rsp.dpp_enabled = pd->dpp_enabled;
596	db_page_addr = ocrdma_get_db_addr(dev, pdid: pd->id);
597	db_page_size = dev->nic_info.db_page_size;
598
599	status = ocrdma_add_mmap(uctx, phy_addr: db_page_addr, len: db_page_size);
600	if (status)
601	return status;
602
603	if (pd->dpp_enabled) {
604	dpp_page_addr = dev->nic_info.dpp_unmapped_addr +
605	(pd->id * PAGE_SIZE);
606	status = ocrdma_add_mmap(uctx, phy_addr: dpp_page_addr,
607	PAGE_SIZE);
608	if (status)
609	goto dpp_map_err;
610	rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr);
611	rsp.dpp_page_addr_lo = dpp_page_addr;
612	}
613
614	status = ib_copy_to_udata(udata, src: &rsp, len: sizeof(rsp));
615	if (status)
616	goto ucopy_err;
617
618	pd->uctx = uctx;
619	return 0;
620
621	ucopy_err:
622	if (pd->dpp_enabled)
623	ocrdma_del_mmap(uctx: pd->uctx, phy_addr: dpp_page_addr, PAGE_SIZE);
624	dpp_map_err:
625	ocrdma_del_mmap(uctx: pd->uctx, phy_addr: db_page_addr, len: db_page_size);
626	return status;
627	}
628
629	int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
630	{
631	struct ib_device *ibdev = ibpd->device;
632	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
633	struct ocrdma_pd *pd;
634	int status;
635	u8 is_uctx_pd = false;
636	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
637	udata, struct ocrdma_ucontext, ibucontext);
638
639	if (udata) {
640	pd = ocrdma_get_ucontext_pd(uctx);
641	if (pd) {
642	is_uctx_pd = true;
643	goto pd_mapping;
644	}
645	}
646
647	pd = get_ocrdma_pd(ibpd);
648	status = _ocrdma_alloc_pd(dev, pd, uctx, udata);
649	if (status)
650	goto exit;
651
652	pd_mapping:
653	if (udata) {
654	status = ocrdma_copy_pd_uresp(dev, pd, udata);
655	if (status)
656	goto err;
657	}
658	return 0;
659
660	err:
661	if (is_uctx_pd)
662	ocrdma_release_ucontext_pd(uctx);
663	else
664	_ocrdma_dealloc_pd(dev, pd);
665	exit:
666	return status;
667	}
668
669	int ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
670	{
671	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
672	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibpd->device);
673	struct ocrdma_ucontext *uctx = NULL;
674	u64 usr_db;
675
676	uctx = pd->uctx;
677	if (uctx) {
678	u64 dpp_db = dev->nic_info.dpp_unmapped_addr +
679	(pd->id * PAGE_SIZE);
680	if (pd->dpp_enabled)
681	ocrdma_del_mmap(uctx: pd->uctx, phy_addr: dpp_db, PAGE_SIZE);
682	usr_db = ocrdma_get_db_addr(dev, pdid: pd->id);
683	ocrdma_del_mmap(uctx: pd->uctx, phy_addr: usr_db, len: dev->nic_info.db_page_size);
684
685	if (is_ucontext_pd(uctx, pd)) {
686	ocrdma_release_ucontext_pd(uctx);
687	return 0;
688	}
689	}
690	_ocrdma_dealloc_pd(dev, pd);
691	return 0;
692	}
693
694	static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
695	u32 pdid, int acc, u32 num_pbls, u32 addr_check)
696	{
697	int status;
698
699	mr->hwmr.fr_mr = 0;
700	mr->hwmr.local_rd = 1;
701	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
702	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
703	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
704	mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
705	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
706	mr->hwmr.num_pbls = num_pbls;
707
708	status = ocrdma_mbx_alloc_lkey(dev, hwmr: &mr->hwmr, pd_id: pdid, addr_check);
709	if (status)
710	return status;
711
712	mr->ibmr.lkey = mr->hwmr.lkey;
713	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
714	mr->ibmr.rkey = mr->hwmr.lkey;
715	return 0;
716	}
717
718	struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc)
719	{
720	int status;
721	struct ocrdma_mr *mr;
722	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
723	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibpd->device);
724
725	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
726	pr_err("%s err, invalid access rights\n", __func__);
727	return ERR_PTR(error: -EINVAL);
728	}
729
730	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
731	if (!mr)
732	return ERR_PTR(error: -ENOMEM);
733
734	status = ocrdma_alloc_lkey(dev, mr, pdid: pd->id, acc, num_pbls: 0,
735	addr_check: OCRDMA_ADDR_CHECK_DISABLE);
736	if (status) {
737	kfree(objp: mr);
738	return ERR_PTR(error: status);
739	}
740
741	return &mr->ibmr;
742	}
743
744	static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev,
745	struct ocrdma_hw_mr *mr)
746	{
747	struct pci_dev *pdev = dev->nic_info.pdev;
748	int i = 0;
749
750	if (mr->pbl_table) {
751	for (i = 0; i < mr->num_pbls; i++) {
752	if (!mr->pbl_table[i].va)
753	continue;
754	dma_free_coherent(dev: &pdev->dev, size: mr->pbl_size,
755	cpu_addr: mr->pbl_table[i].va,
756	dma_handle: mr->pbl_table[i].pa);
757	}
758	kfree(objp: mr->pbl_table);
759	mr->pbl_table = NULL;
760	}
761	}
762
763	static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
764	u32 num_pbes)
765	{
766	u32 num_pbls = 0;
767	u32 idx = 0;
768	int status = 0;
769	u32 pbl_size;
770
771	do {
772	pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx);
773	if (pbl_size > MAX_OCRDMA_PBL_SIZE) {
774	status = -EFAULT;
775	break;
776	}
777	num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64)));
778	num_pbls = num_pbls / (pbl_size / sizeof(u64));
779	idx++;
780	} while (num_pbls >= dev->attr.max_num_mr_pbl);
781
782	mr->hwmr.num_pbes = num_pbes;
783	mr->hwmr.num_pbls = num_pbls;
784	mr->hwmr.pbl_size = pbl_size;
785	return status;
786	}
787
788	static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)
789	{
790	int status = 0;
791	int i;
792	u32 dma_len = mr->pbl_size;
793	struct pci_dev *pdev = dev->nic_info.pdev;
794	void *va;
795	dma_addr_t pa;
796
797	mr->pbl_table = kcalloc(mr->num_pbls, sizeof(struct ocrdma_pbl),
798	GFP_KERNEL);
799
800	if (!mr->pbl_table)
801	return -ENOMEM;
802
803	for (i = 0; i < mr->num_pbls; i++) {
804	va = dma_alloc_coherent(dev: &pdev->dev, size: dma_len, dma_handle: &pa, GFP_KERNEL);
805	if (!va) {
806	ocrdma_free_mr_pbl_tbl(dev, mr);
807	status = -ENOMEM;
808	break;
809	}
810	mr->pbl_table[i].va = va;
811	mr->pbl_table[i].pa = pa;
812	}
813	return status;
814	}
815
816	static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr)
817	{
818	struct ocrdma_pbe *pbe;
819	struct ib_block_iter biter;
820	struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
821	int pbe_cnt;
822	u64 pg_addr;
823
824	if (!mr->hwmr.num_pbes)
825	return;
826
827	pbe = (struct ocrdma_pbe *)pbl_tbl->va;
828	pbe_cnt = 0;
829
830	rdma_umem_for_each_dma_block (mr->umem, &biter, PAGE_SIZE) {
831	/* store the page address in pbe */
832	pg_addr = rdma_block_iter_dma_address(biter: &biter);
833	pbe->pa_lo = cpu_to_le32(pg_addr);
834	pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr));
835	pbe_cnt += 1;
836	pbe++;
837
838	/* if the given pbl is full storing the pbes,
839	* move to next pbl.
840	*/
841	if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) {
842	pbl_tbl++;
843	pbe = (struct ocrdma_pbe *)pbl_tbl->va;
844	pbe_cnt = 0;
845	}
846	}
847	}
848
849	struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
850	u64 usr_addr, int acc, struct ib_dmah *dmah,
851	struct ib_udata *udata)
852	{
853	int status = -ENOMEM;
854	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibpd->device);
855	struct ocrdma_mr *mr;
856	struct ocrdma_pd *pd;
857
858	if (dmah)
859	return ERR_PTR(error: -EOPNOTSUPP);
860
861	pd = get_ocrdma_pd(ibpd);
862
863	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
864	return ERR_PTR(error: -EINVAL);
865
866	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
867	if (!mr)
868	return ERR_PTR(error: status);
869	mr->umem = ib_umem_get(device: ibpd->device, addr: start, size: len, access: acc);
870	if (IS_ERR(ptr: mr->umem)) {
871	status = -EFAULT;
872	goto umem_err;
873	}
874	status = ocrdma_get_pbl_info(
875	dev, mr, num_pbes: ib_umem_num_dma_blocks(umem: mr->umem, PAGE_SIZE));
876	if (status)
877	goto umem_err;
878
879	mr->hwmr.pbe_size = PAGE_SIZE;
880	mr->hwmr.va = usr_addr;
881	mr->hwmr.len = len;
882	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
883	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
884	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
885	mr->hwmr.local_rd = 1;
886	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
887	status = ocrdma_build_pbl_tbl(dev, mr: &mr->hwmr);
888	if (status)
889	goto umem_err;
890	build_user_pbes(dev, mr);
891	status = ocrdma_reg_mr(dev, hwmr: &mr->hwmr, pd_id: pd->id, acc);
892	if (status)
893	goto mbx_err;
894	mr->ibmr.lkey = mr->hwmr.lkey;
895	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
896	mr->ibmr.rkey = mr->hwmr.lkey;
897
898	return &mr->ibmr;
899
900	mbx_err:
901	ocrdma_free_mr_pbl_tbl(dev, mr: &mr->hwmr);
902	umem_err:
903	kfree(objp: mr);
904	return ERR_PTR(error: status);
905	}
906
907	int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
908	{
909	struct ocrdma_mr *mr = get_ocrdma_mr(ibmr: ib_mr);
910	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ib_mr->device);
911
912	(void) ocrdma_mbx_dealloc_lkey(dev, fmr: mr->hwmr.fr_mr, lkey: mr->hwmr.lkey);
913
914	kfree(objp: mr->pages);
915	ocrdma_free_mr_pbl_tbl(dev, mr: &mr->hwmr);
916
917	/* it could be user registered memory. */
918	ib_umem_release(umem: mr->umem);
919	kfree(objp: mr);
920
921	/* Don't stop cleanup, in case FW is unresponsive */
922	if (dev->mqe_ctx.fw_error_state) {
923	pr_err("%s(%d) fw not responding.\n",
924	__func__, dev->id);
925	}
926	return 0;
927	}
928
929	static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
930	struct ib_udata *udata)
931	{
932	int status;
933	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
934	udata, struct ocrdma_ucontext, ibucontext);
935	struct ocrdma_create_cq_uresp uresp;
936
937	/* this must be user flow! */
938	if (!udata)
939	return -EINVAL;
940
941	memset(&uresp, 0, sizeof(uresp));
942	uresp.cq_id = cq->id;
943	uresp.page_size = PAGE_ALIGN(cq->len);
944	uresp.num_pages = 1;
945	uresp.max_hw_cqe = cq->max_hw_cqe;
946	uresp.page_addr[0] = virt_to_phys(address: cq->va);
947	uresp.db_page_addr = ocrdma_get_db_addr(dev, pdid: uctx->cntxt_pd->id);
948	uresp.db_page_size = dev->nic_info.db_page_size;
949	uresp.phase_change = cq->phase_change ? 1 : 0;
950	status = ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
951	if (status) {
952	pr_err("%s(%d) copy error cqid=0x%x.\n",
953	__func__, dev->id, cq->id);
954	goto err;
955	}
956	status = ocrdma_add_mmap(uctx, phy_addr: uresp.db_page_addr, len: uresp.db_page_size);
957	if (status)
958	goto err;
959	status = ocrdma_add_mmap(uctx, phy_addr: uresp.page_addr[0], len: uresp.page_size);
960	if (status) {
961	ocrdma_del_mmap(uctx, phy_addr: uresp.db_page_addr, len: uresp.db_page_size);
962	goto err;
963	}
964	cq->ucontext = uctx;
965	err:
966	return status;
967	}
968
969	int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
970	struct uverbs_attr_bundle *attrs)
971	{
972	struct ib_udata *udata = &attrs->driver_udata;
973	struct ib_device *ibdev = ibcq->device;
974	int entries = attr->cqe;
975	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
976	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
977	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
978	udata, struct ocrdma_ucontext, ibucontext);
979	u16 pd_id = 0;
980	int status;
981	struct ocrdma_create_cq_ureq ureq;
982
983	if (attr->flags)
984	return -EOPNOTSUPP;
985
986	if (udata) {
987	if (ib_copy_from_udata(dest: &ureq, udata, len: sizeof(ureq)))
988	return -EFAULT;
989	} else
990	ureq.dpp_cq = 0;
991
992	spin_lock_init(&cq->cq_lock);
993	spin_lock_init(&cq->comp_handler_lock);
994	INIT_LIST_HEAD(list: &cq->sq_head);
995	INIT_LIST_HEAD(list: &cq->rq_head);
996
997	if (udata)
998	pd_id = uctx->cntxt_pd->id;
999
1000	status = ocrdma_mbx_create_cq(dev, cq, entries, dpp_cq: ureq.dpp_cq, pd_id);
1001	if (status)
1002	return status;
1003
1004	if (udata) {
1005	status = ocrdma_copy_cq_uresp(dev, cq, udata);
1006	if (status)
1007	goto ctx_err;
1008	}
1009	cq->phase = OCRDMA_CQE_VALID;
1010	dev->cq_tbl[cq->id] = cq;
1011	return 0;
1012
1013	ctx_err:
1014	ocrdma_mbx_destroy_cq(dev, cq);
1015	return status;
1016	}
1017
1018	int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt,
1019	struct ib_udata *udata)
1020	{
1021	int status = 0;
1022	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
1023
1024	if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) {
1025	status = -EINVAL;
1026	return status;
1027	}
1028	ibcq->cqe = new_cnt;
1029	return status;
1030	}
1031
1032	static void ocrdma_flush_cq(struct ocrdma_cq *cq)
1033	{
1034	int cqe_cnt;
1035	int valid_count = 0;
1036	unsigned long flags;
1037
1038	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: cq->ibcq.device);
1039	struct ocrdma_cqe *cqe = NULL;
1040
1041	cqe = cq->va;
1042	cqe_cnt = cq->cqe_cnt;
1043
1044	/* Last irq might have scheduled a polling thread
1045	* sync-up with it before hard flushing.
1046	*/
1047	spin_lock_irqsave(&cq->cq_lock, flags);
1048	while (cqe_cnt) {
1049	if (is_cqe_valid(cq, cqe))
1050	valid_count++;
1051	cqe++;
1052	cqe_cnt--;
1053	}
1054	ocrdma_ring_cq_db(dev, cq_id: cq->id, armed: false, solicited: false, cqe_popped: valid_count);
1055	spin_unlock_irqrestore(lock: &cq->cq_lock, flags);
1056	}
1057
1058	int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1059	{
1060	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
1061	struct ocrdma_eq *eq = NULL;
1062	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibcq->device);
1063	int pdid = 0;
1064	u32 irq, indx;
1065
1066	dev->cq_tbl[cq->id] = NULL;
1067	indx = ocrdma_get_eq_table_index(dev, eqid: cq->eqn);
1068
1069	eq = &dev->eq_tbl[indx];
1070	irq = ocrdma_get_irq(dev, eq);
1071	synchronize_irq(irq);
1072	ocrdma_flush_cq(cq);
1073
1074	ocrdma_mbx_destroy_cq(dev, cq);
1075	if (cq->ucontext) {
1076	pdid = cq->ucontext->cntxt_pd->id;
1077	ocrdma_del_mmap(uctx: cq->ucontext, phy_addr: (u64) cq->pa,
1078	PAGE_ALIGN(cq->len));
1079	ocrdma_del_mmap(uctx: cq->ucontext,
1080	phy_addr: ocrdma_get_db_addr(dev, pdid),
1081	len: dev->nic_info.db_page_size);
1082	}
1083	return 0;
1084	}
1085
1086	static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1087	{
1088	int status = -EINVAL;
1089
1090	if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) {
1091	dev->qp_tbl[qp->id] = qp;
1092	status = 0;
1093	}
1094	return status;
1095	}
1096
1097	static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1098	{
1099	dev->qp_tbl[qp->id] = NULL;
1100	}
1101
1102	static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
1103	struct ib_qp_init_attr *attrs,
1104	struct ib_udata *udata)
1105	{
1106	if ((attrs->qp_type != IB_QPT_GSI) &&
1107	(attrs->qp_type != IB_QPT_RC) &&
1108	(attrs->qp_type != IB_QPT_UC) &&
1109	(attrs->qp_type != IB_QPT_UD)) {
1110	pr_err("%s(%d) unsupported qp type=0x%x requested\n",
1111	__func__, dev->id, attrs->qp_type);
1112	return -EOPNOTSUPP;
1113	}
1114	/* Skip the check for QP1 to support CM size of 128 */
1115	if ((attrs->qp_type != IB_QPT_GSI) &&
1116	(attrs->cap.max_send_wr > dev->attr.max_wqe)) {
1117	pr_err("%s(%d) unsupported send_wr=0x%x requested\n",
1118	__func__, dev->id, attrs->cap.max_send_wr);
1119	pr_err("%s(%d) supported send_wr=0x%x\n",
1120	__func__, dev->id, dev->attr.max_wqe);
1121	return -EINVAL;
1122	}
1123	if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) {
1124	pr_err("%s(%d) unsupported recv_wr=0x%x requested\n",
1125	__func__, dev->id, attrs->cap.max_recv_wr);
1126	pr_err("%s(%d) supported recv_wr=0x%x\n",
1127	__func__, dev->id, dev->attr.max_rqe);
1128	return -EINVAL;
1129	}
1130	if (attrs->cap.max_inline_data > dev->attr.max_inline_data) {
1131	pr_err("%s(%d) unsupported inline data size=0x%x requested\n",
1132	__func__, dev->id, attrs->cap.max_inline_data);
1133	pr_err("%s(%d) supported inline data size=0x%x\n",
1134	__func__, dev->id, dev->attr.max_inline_data);
1135	return -EINVAL;
1136	}
1137	if (attrs->cap.max_send_sge > dev->attr.max_send_sge) {
1138	pr_err("%s(%d) unsupported send_sge=0x%x requested\n",
1139	__func__, dev->id, attrs->cap.max_send_sge);
1140	pr_err("%s(%d) supported send_sge=0x%x\n",
1141	__func__, dev->id, dev->attr.max_send_sge);
1142	return -EINVAL;
1143	}
1144	if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) {
1145	pr_err("%s(%d) unsupported recv_sge=0x%x requested\n",
1146	__func__, dev->id, attrs->cap.max_recv_sge);
1147	pr_err("%s(%d) supported recv_sge=0x%x\n",
1148	__func__, dev->id, dev->attr.max_recv_sge);
1149	return -EINVAL;
1150	}
1151	/* unprivileged user space cannot create special QP */
1152	if (udata && attrs->qp_type == IB_QPT_GSI) {
1153	pr_err
1154	("%s(%d) Userspace can't create special QPs of type=0x%x\n",
1155	__func__, dev->id, attrs->qp_type);
1156	return -EINVAL;
1157	}
1158	/* allow creating only one GSI type of QP */
1159	if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
1160	pr_err("%s(%d) GSI special QPs already created.\n",
1161	__func__, dev->id);
1162	return -EINVAL;
1163	}
1164	/* verify consumer QPs are not trying to use GSI QP's CQ */
1165	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
1166	if ((dev->gsi_sqcq == get_ocrdma_cq(ibcq: attrs->send_cq)) ||
1167	(dev->gsi_rqcq == get_ocrdma_cq(ibcq: attrs->recv_cq))) {
1168	pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n",
1169	__func__, dev->id);
1170	return -EINVAL;
1171	}
1172	}
1173	return 0;
1174	}
1175
1176	static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
1177	struct ib_udata *udata, int dpp_offset,
1178	int dpp_credit_lmt, int srq)
1179	{
1180	int status;
1181	u64 usr_db;
1182	struct ocrdma_create_qp_uresp uresp;
1183	struct ocrdma_pd *pd = qp->pd;
1184	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: pd->ibpd.device);
1185
1186	memset(&uresp, 0, sizeof(uresp));
1187	usr_db = dev->nic_info.unmapped_db +
1188	(pd->id * dev->nic_info.db_page_size);
1189	uresp.qp_id = qp->id;
1190	uresp.sq_dbid = qp->sq.dbid;
1191	uresp.num_sq_pages = 1;
1192	uresp.sq_page_size = PAGE_ALIGN(qp->sq.len);
1193	uresp.sq_page_addr[0] = virt_to_phys(address: qp->sq.va);
1194	uresp.num_wqe_allocated = qp->sq.max_cnt;
1195	if (!srq) {
1196	uresp.rq_dbid = qp->rq.dbid;
1197	uresp.num_rq_pages = 1;
1198	uresp.rq_page_size = PAGE_ALIGN(qp->rq.len);
1199	uresp.rq_page_addr[0] = virt_to_phys(address: qp->rq.va);
1200	uresp.num_rqe_allocated = qp->rq.max_cnt;
1201	}
1202	uresp.db_page_addr = usr_db;
1203	uresp.db_page_size = dev->nic_info.db_page_size;
1204	uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET;
1205	uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1206	uresp.db_shift = OCRDMA_DB_RQ_SHIFT;
1207
1208	if (qp->dpp_enabled) {
1209	uresp.dpp_credit = dpp_credit_lmt;
1210	uresp.dpp_offset = dpp_offset;
1211	}
1212	status = ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
1213	if (status) {
1214	pr_err("%s(%d) user copy error.\n", __func__, dev->id);
1215	goto err;
1216	}
1217	status = ocrdma_add_mmap(uctx: pd->uctx, phy_addr: uresp.sq_page_addr[0],
1218	len: uresp.sq_page_size);
1219	if (status)
1220	goto err;
1221
1222	if (!srq) {
1223	status = ocrdma_add_mmap(uctx: pd->uctx, phy_addr: uresp.rq_page_addr[0],
1224	len: uresp.rq_page_size);
1225	if (status)
1226	goto rq_map_err;
1227	}
1228	return status;
1229	rq_map_err:
1230	ocrdma_del_mmap(uctx: pd->uctx, phy_addr: uresp.sq_page_addr[0], len: uresp.sq_page_size);
1231	err:
1232	return status;
1233	}
1234
1235	static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
1236	struct ocrdma_pd *pd)
1237	{
1238	if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1239	qp->sq_db = dev->nic_info.db +
1240	(pd->id * dev->nic_info.db_page_size) +
1241	OCRDMA_DB_GEN2_SQ_OFFSET;
1242	qp->rq_db = dev->nic_info.db +
1243	(pd->id * dev->nic_info.db_page_size) +
1244	OCRDMA_DB_GEN2_RQ_OFFSET;
1245	} else {
1246	qp->sq_db = dev->nic_info.db +
1247	(pd->id * dev->nic_info.db_page_size) +
1248	OCRDMA_DB_SQ_OFFSET;
1249	qp->rq_db = dev->nic_info.db +
1250	(pd->id * dev->nic_info.db_page_size) +
1251	OCRDMA_DB_RQ_OFFSET;
1252	}
1253	}
1254
1255	static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)
1256	{
1257	qp->wqe_wr_id_tbl =
1258	kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)),
1259	GFP_KERNEL);
1260	if (qp->wqe_wr_id_tbl == NULL)
1261	return -ENOMEM;
1262	qp->rqe_wr_id_tbl =
1263	kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL);
1264	if (qp->rqe_wr_id_tbl == NULL)
1265	return -ENOMEM;
1266
1267	return 0;
1268	}
1269
1270	static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,
1271	struct ocrdma_pd *pd,
1272	struct ib_qp_init_attr *attrs)
1273	{
1274	qp->pd = pd;
1275	spin_lock_init(&qp->q_lock);
1276	INIT_LIST_HEAD(list: &qp->sq_entry);
1277	INIT_LIST_HEAD(list: &qp->rq_entry);
1278
1279	qp->qp_type = attrs->qp_type;
1280	qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR;
1281	qp->max_inline_data = attrs->cap.max_inline_data;
1282	qp->sq.max_sges = attrs->cap.max_send_sge;
1283	qp->rq.max_sges = attrs->cap.max_recv_sge;
1284	qp->state = OCRDMA_QPS_RST;
1285	qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1286	}
1287
1288	static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev,
1289	struct ib_qp_init_attr *attrs)
1290	{
1291	if (attrs->qp_type == IB_QPT_GSI) {
1292	dev->gsi_qp_created = 1;
1293	dev->gsi_sqcq = get_ocrdma_cq(ibcq: attrs->send_cq);
1294	dev->gsi_rqcq = get_ocrdma_cq(ibcq: attrs->recv_cq);
1295	}
1296	}
1297
1298	int ocrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
1299	struct ib_udata *udata)
1300	{
1301	int status;
1302	struct ib_pd *ibpd = ibqp->pd;
1303	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1304	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1305	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibqp->device);
1306	struct ocrdma_create_qp_ureq ureq;
1307	u16 dpp_credit_lmt, dpp_offset;
1308
1309	if (attrs->create_flags)
1310	return -EOPNOTSUPP;
1311
1312	status = ocrdma_check_qp_params(ibpd, dev, attrs, udata);
1313	if (status)
1314	goto gen_err;
1315
1316	memset(&ureq, 0, sizeof(ureq));
1317	if (udata) {
1318	if (ib_copy_from_udata(dest: &ureq, udata, len: sizeof(ureq)))
1319	return -EFAULT;
1320	}
1321	ocrdma_set_qp_init_params(qp, pd, attrs);
1322	if (udata == NULL)
1323	qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1324	OCRDMA_QP_FAST_REG);
1325
1326	mutex_lock(&dev->dev_lock);
1327	status = ocrdma_mbx_create_qp(qp, attrs, enable_dpp_cq: ureq.enable_dpp_cq,
1328	dpp_cq_id: ureq.dpp_cq_id,
1329	dpp_offset: &dpp_offset, dpp_credit_lmt: &dpp_credit_lmt);
1330	if (status)
1331	goto mbx_err;
1332
1333	/* user space QP's wr_id table are managed in library */
1334	if (udata == NULL) {
1335	status = ocrdma_alloc_wr_id_tbl(qp);
1336	if (status)
1337	goto map_err;
1338	}
1339
1340	status = ocrdma_add_qpn_map(dev, qp);
1341	if (status)
1342	goto map_err;
1343	ocrdma_set_qp_db(dev, qp, pd);
1344	if (udata) {
1345	status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset,
1346	dpp_credit_lmt,
1347	srq: (attrs->srq != NULL));
1348	if (status)
1349	goto cpy_err;
1350	}
1351	ocrdma_store_gsi_qp_cq(dev, attrs);
1352	qp->ibqp.qp_num = qp->id;
1353	mutex_unlock(lock: &dev->dev_lock);
1354	return 0;
1355
1356	cpy_err:
1357	ocrdma_del_qpn_map(dev, qp);
1358	map_err:
1359	ocrdma_mbx_destroy_qp(dev, qp);
1360	mbx_err:
1361	mutex_unlock(lock: &dev->dev_lock);
1362	kfree(objp: qp->wqe_wr_id_tbl);
1363	kfree(objp: qp->rqe_wr_id_tbl);
1364	pr_err("%s(%d) error=%d\n", __func__, dev->id, status);
1365	gen_err:
1366	return status;
1367	}
1368
1369	int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1370	int attr_mask)
1371	{
1372	int status = 0;
1373	struct ocrdma_qp *qp;
1374	struct ocrdma_dev *dev;
1375	enum ib_qp_state old_qps;
1376
1377	qp = get_ocrdma_qp(ibqp);
1378	dev = get_ocrdma_dev(ibdev: ibqp->device);
1379	if (attr_mask & IB_QP_STATE)
1380	status = ocrdma_qp_state_change(qp, new_state: attr->qp_state, old_ib_state: &old_qps);
1381	/* if new and previous states are same hw doesn't need to
1382	* know about it.
1383	*/
1384	if (status < 0)
1385	return status;
1386	return ocrdma_mbx_modify_qp(dev, qp, attrs: attr, attr_mask);
1387	}
1388
1389	int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1390	int attr_mask, struct ib_udata *udata)
1391	{
1392	unsigned long flags;
1393	int status = -EINVAL;
1394	struct ocrdma_qp *qp;
1395	struct ocrdma_dev *dev;
1396	enum ib_qp_state old_qps, new_qps;
1397
1398	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1399	return -EOPNOTSUPP;
1400
1401	qp = get_ocrdma_qp(ibqp);
1402	dev = get_ocrdma_dev(ibdev: ibqp->device);
1403
1404	/* syncronize with multiple context trying to change, retrive qps */
1405	mutex_lock(&dev->dev_lock);
1406	/* syncronize with wqe, rqe posting and cqe processing contexts */
1407	spin_lock_irqsave(&qp->q_lock, flags);
1408	old_qps = get_ibqp_state(qps: qp->state);
1409	if (attr_mask & IB_QP_STATE)
1410	new_qps = attr->qp_state;
1411	else
1412	new_qps = old_qps;
1413	spin_unlock_irqrestore(lock: &qp->q_lock, flags);
1414
1415	if (!ib_modify_qp_is_ok(cur_state: old_qps, next_state: new_qps, type: ibqp->qp_type, mask: attr_mask)) {
1416	pr_err("%s(%d) invalid attribute mask=0x%x specified for\n"
1417	"qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n",
1418	__func__, dev->id, attr_mask, qp->id, ibqp->qp_type,
1419	old_qps, new_qps);
1420	goto param_err;
1421	}
1422
1423	status = _ocrdma_modify_qp(ibqp, attr, attr_mask);
1424	if (status > 0)
1425	status = 0;
1426	param_err:
1427	mutex_unlock(lock: &dev->dev_lock);
1428	return status;
1429	}
1430
1431	static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu)
1432	{
1433	switch (mtu) {
1434	case 256:
1435	return IB_MTU_256;
1436	case 512:
1437	return IB_MTU_512;
1438	case 1024:
1439	return IB_MTU_1024;
1440	case 2048:
1441	return IB_MTU_2048;
1442	case 4096:
1443	return IB_MTU_4096;
1444	default:
1445	return IB_MTU_1024;
1446	}
1447	}
1448
1449	static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags)
1450	{
1451	int ib_qp_acc_flags = 0;
1452
1453	if (qp_cap_flags & OCRDMA_QP_INB_WR)
1454	ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1455	if (qp_cap_flags & OCRDMA_QP_INB_RD)
1456	ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1457	return ib_qp_acc_flags;
1458	}
1459
1460	int ocrdma_query_qp(struct ib_qp *ibqp,
1461	struct ib_qp_attr *qp_attr,
1462	int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1463	{
1464	int status;
1465	u32 qp_state;
1466	struct ocrdma_qp_params params;
1467	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1468	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibqp->device);
1469
1470	memset(&params, 0, sizeof(params));
1471	mutex_lock(&dev->dev_lock);
1472	status = ocrdma_mbx_query_qp(dev, qp, param: &params);
1473	mutex_unlock(lock: &dev->dev_lock);
1474	if (status)
1475	goto mbx_err;
1476	if (qp->qp_type == IB_QPT_UD)
1477	qp_attr->qkey = params.qkey;
1478	qp_attr->path_mtu =
1479	ocrdma_mtu_int_to_enum(mtu: params.path_mtu_pkey_indx &
1480	OCRDMA_QP_PARAMS_PATH_MTU_MASK) >>
1481	OCRDMA_QP_PARAMS_PATH_MTU_SHIFT;
1482	qp_attr->path_mig_state = IB_MIG_MIGRATED;
1483	qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK;
1484	qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK;
1485	qp_attr->dest_qp_num =
1486	params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK;
1487
1488	qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp_cap_flags: qp->cap_flags);
1489	qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1;
1490	qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;
1491	qp_attr->cap.max_send_sge = qp->sq.max_sges;
1492	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1493	qp_attr->cap.max_inline_data = qp->max_inline_data;
1494	qp_init_attr->cap = qp_attr->cap;
1495	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
1496
1497	rdma_ah_set_grh(attr: &qp_attr->ah_attr, NULL,
1498	flow_label: params.rnt_rc_sl_fl &
1499	OCRDMA_QP_PARAMS_FLOW_LABEL_MASK,
1500	sgid_index: qp->sgid_idx,
1501	hop_limit: (params.hop_lmt_rq_psn &
1502	OCRDMA_QP_PARAMS_HOP_LMT_MASK) >>
1503	OCRDMA_QP_PARAMS_HOP_LMT_SHIFT,
1504	traffic_class: (params.tclass_sq_psn &
1505	OCRDMA_QP_PARAMS_TCLASS_MASK) >>
1506	OCRDMA_QP_PARAMS_TCLASS_SHIFT);
1507	rdma_ah_set_dgid_raw(attr: &qp_attr->ah_attr, dgid: &params.dgid[0]);
1508
1509	rdma_ah_set_port_num(attr: &qp_attr->ah_attr, port_num: 1);
1510	rdma_ah_set_sl(attr: &qp_attr->ah_attr, sl: (params.rnt_rc_sl_fl &
1511	OCRDMA_QP_PARAMS_SL_MASK) >>
1512	OCRDMA_QP_PARAMS_SL_SHIFT);
1513	qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn &
1514	OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >>
1515	OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT;
1516	qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn &
1517	OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >>
1518	OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT;
1519	qp_attr->retry_cnt =
1520	(params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >>
1521	OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT;
1522	qp_attr->min_rnr_timer = 0;
1523	qp_attr->pkey_index = 0;
1524	qp_attr->port_num = 1;
1525	rdma_ah_set_path_bits(attr: &qp_attr->ah_attr, src_path_bits: 0);
1526	rdma_ah_set_static_rate(attr: &qp_attr->ah_attr, static_rate: 0);
1527	qp_attr->alt_pkey_index = 0;
1528	qp_attr->alt_port_num = 0;
1529	qp_attr->alt_timeout = 0;
1530	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
1531	qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >>
1532	OCRDMA_QP_PARAMS_STATE_SHIFT;
1533	qp_attr->qp_state = get_ibqp_state(qps: qp_state);
1534	qp_attr->cur_qp_state = qp_attr->qp_state;
1535	qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0;
1536	qp_attr->max_dest_rd_atomic =
1537	params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT;
1538	qp_attr->max_rd_atomic =
1539	params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK;
1540	qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags &
1541	OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0;
1542	/* Sync driver QP state with FW */
1543	ocrdma_qp_state_change(qp, new_state: qp_attr->qp_state, NULL);
1544	mbx_err:
1545	return status;
1546	}
1547
1548	static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx)
1549	{
1550	unsigned int i = idx / 32;
1551	u32 mask = (1U << (idx % 32));
1552
1553	srq->idx_bit_fields[i] ^= mask;
1554	}
1555
1556	static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)
1557	{
1558	return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt;
1559	}
1560
1561	static int is_hw_sq_empty(struct ocrdma_qp *qp)
1562	{
1563	return (qp->sq.tail == qp->sq.head);
1564	}
1565
1566	static int is_hw_rq_empty(struct ocrdma_qp *qp)
1567	{
1568	return (qp->rq.tail == qp->rq.head);
1569	}
1570
1571	static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)
1572	{
1573	return q->va + (q->head * q->entry_size);
1574	}
1575
1576	static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q,
1577	u32 idx)
1578	{
1579	return q->va + (idx * q->entry_size);
1580	}
1581
1582	static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q)
1583	{
1584	q->head = (q->head + 1) & q->max_wqe_idx;
1585	}
1586
1587	static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q)
1588	{
1589	q->tail = (q->tail + 1) & q->max_wqe_idx;
1590	}
1591
1592	/* discard the cqe for a given QP */
1593	static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
1594	{
1595	unsigned long cq_flags;
1596	unsigned long flags;
1597	u32 cur_getp, stop_getp;
1598	struct ocrdma_cqe *cqe;
1599	u32 qpn = 0, wqe_idx = 0;
1600
1601	spin_lock_irqsave(&cq->cq_lock, cq_flags);
1602
1603	/* traverse through the CQEs in the hw CQ,
1604	* find the matching CQE for a given qp,
1605	* mark the matching one discarded by clearing qpn.
1606	* ring the doorbell in the poll_cq() as
1607	* we don't complete out of order cqe.
1608	*/
1609
1610	cur_getp = cq->getp;
1611	/* find upto when do we reap the cq. */
1612	stop_getp = cur_getp;
1613	do {
1614	if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
1615	break;
1616
1617	cqe = cq->va + cur_getp;
1618	/* if (a) done reaping whole hw cq, or
1619	* (b) qp_xq becomes empty.
1620	* then exit
1621	*/
1622	qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK;
1623	/* if previously discarded cqe found, skip that too. */
1624	/* check for matching qp */
1625	if (qpn == 0 || qpn != qp->id)
1626	goto skip_cqe;
1627
1628	if (is_cqe_for_sq(cqe)) {
1629	ocrdma_hwq_inc_tail(q: &qp->sq);
1630	} else {
1631	if (qp->srq) {
1632	wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
1633	OCRDMA_CQE_BUFTAG_SHIFT) &
1634	qp->srq->rq.max_wqe_idx;
1635	BUG_ON(wqe_idx < 1);
1636	spin_lock_irqsave(&qp->srq->q_lock, flags);
1637	ocrdma_hwq_inc_tail(q: &qp->srq->rq);
1638	ocrdma_srq_toggle_bit(srq: qp->srq, idx: wqe_idx - 1);
1639	spin_unlock_irqrestore(lock: &qp->srq->q_lock, flags);
1640
1641	} else {
1642	ocrdma_hwq_inc_tail(q: &qp->rq);
1643	}
1644	}
1645	/* mark cqe discarded so that it is not picked up later
1646	* in the poll_cq().
1647	*/
1648	cqe->cmn.qpn = 0;
1649	skip_cqe:
1650	cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
1651	} while (cur_getp != stop_getp);
1652	spin_unlock_irqrestore(lock: &cq->cq_lock, flags: cq_flags);
1653	}
1654
1655	void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
1656	{
1657	int found = false;
1658	unsigned long flags;
1659	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: qp->ibqp.device);
1660	/* sync with any active CQ poll */
1661
1662	spin_lock_irqsave(&dev->flush_q_lock, flags);
1663	found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
1664	if (found)
1665	list_del(entry: &qp->sq_entry);
1666	if (!qp->srq) {
1667	found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
1668	if (found)
1669	list_del(entry: &qp->rq_entry);
1670	}
1671	spin_unlock_irqrestore(lock: &dev->flush_q_lock, flags);
1672	}
1673
1674	int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1675	{
1676	struct ocrdma_pd *pd;
1677	struct ocrdma_qp *qp;
1678	struct ocrdma_dev *dev;
1679	struct ib_qp_attr attrs;
1680	int attr_mask;
1681	unsigned long flags;
1682
1683	qp = get_ocrdma_qp(ibqp);
1684	dev = get_ocrdma_dev(ibdev: ibqp->device);
1685
1686	pd = qp->pd;
1687
1688	/* change the QP state to ERROR */
1689	if (qp->state != OCRDMA_QPS_RST) {
1690	attrs.qp_state = IB_QPS_ERR;
1691	attr_mask = IB_QP_STATE;
1692	_ocrdma_modify_qp(ibqp, attr: &attrs, attr_mask);
1693	}
1694	/* ensure that CQEs for newly created QP (whose id may be same with
1695	* one which just getting destroyed are same), dont get
1696	* discarded until the old CQEs are discarded.
1697	*/
1698	mutex_lock(&dev->dev_lock);
1699	(void) ocrdma_mbx_destroy_qp(dev, qp);
1700
1701	/*
1702	* acquire CQ lock while destroy is in progress, in order to
1703	* protect against proessing in-flight CQEs for this QP.
1704	*/
1705	spin_lock_irqsave(&qp->sq_cq->cq_lock, flags);
1706	if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) {
1707	spin_lock(lock: &qp->rq_cq->cq_lock);
1708	ocrdma_del_qpn_map(dev, qp);
1709	spin_unlock(lock: &qp->rq_cq->cq_lock);
1710	} else {
1711	ocrdma_del_qpn_map(dev, qp);
1712	}
1713	spin_unlock_irqrestore(lock: &qp->sq_cq->cq_lock, flags);
1714
1715	if (!pd->uctx) {
1716	ocrdma_discard_cqes(qp, cq: qp->sq_cq);
1717	ocrdma_discard_cqes(qp, cq: qp->rq_cq);
1718	}
1719	mutex_unlock(lock: &dev->dev_lock);
1720
1721	if (pd->uctx) {
1722	ocrdma_del_mmap(uctx: pd->uctx, phy_addr: (u64) qp->sq.pa,
1723	PAGE_ALIGN(qp->sq.len));
1724	if (!qp->srq)
1725	ocrdma_del_mmap(uctx: pd->uctx, phy_addr: (u64) qp->rq.pa,
1726	PAGE_ALIGN(qp->rq.len));
1727	}
1728
1729	ocrdma_del_flush_qp(qp);
1730
1731	kfree(objp: qp->wqe_wr_id_tbl);
1732	kfree(objp: qp->rqe_wr_id_tbl);
1733	return 0;
1734	}
1735
1736	static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
1737	struct ib_udata *udata)
1738	{
1739	int status;
1740	struct ocrdma_create_srq_uresp uresp;
1741
1742	memset(&uresp, 0, sizeof(uresp));
1743	uresp.rq_dbid = srq->rq.dbid;
1744	uresp.num_rq_pages = 1;
1745	uresp.rq_page_addr[0] = virt_to_phys(address: srq->rq.va);
1746	uresp.rq_page_size = srq->rq.len;
1747	uresp.db_page_addr = dev->nic_info.unmapped_db +
1748	(srq->pd->id * dev->nic_info.db_page_size);
1749	uresp.db_page_size = dev->nic_info.db_page_size;
1750	uresp.num_rqe_allocated = srq->rq.max_cnt;
1751	if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1752	uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1753	uresp.db_shift = 24;
1754	} else {
1755	uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
1756	uresp.db_shift = 16;
1757	}
1758
1759	status = ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
1760	if (status)
1761	return status;
1762	status = ocrdma_add_mmap(uctx: srq->pd->uctx, phy_addr: uresp.rq_page_addr[0],
1763	len: uresp.rq_page_size);
1764	if (status)
1765	return status;
1766	return status;
1767	}
1768
1769	int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1770	struct ib_udata *udata)
1771	{
1772	int status;
1773	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd: ibsrq->pd);
1774	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibsrq->device);
1775	struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq);
1776
1777	if (init_attr->srq_type != IB_SRQT_BASIC)
1778	return -EOPNOTSUPP;
1779
1780	if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
1781	return -EINVAL;
1782	if (init_attr->attr.max_wr > dev->attr.max_rqe)
1783	return -EINVAL;
1784
1785	spin_lock_init(&srq->q_lock);
1786	srq->pd = pd;
1787	srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);
1788	status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd);
1789	if (status)
1790	return status;
1791
1792	if (!udata) {
1793	srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64),
1794	GFP_KERNEL);
1795	if (!srq->rqe_wr_id_tbl) {
1796	status = -ENOMEM;
1797	goto arm_err;
1798	}
1799
1800	srq->bit_fields_len = (srq->rq.max_cnt / 32) +
1801	(srq->rq.max_cnt % 32 ? 1 : 0);
1802	srq->idx_bit_fields =
1803	kmalloc_array(srq->bit_fields_len, sizeof(u32),
1804	GFP_KERNEL);
1805	if (!srq->idx_bit_fields) {
1806	status = -ENOMEM;
1807	goto arm_err;
1808	}
1809	memset(srq->idx_bit_fields, 0xff,
1810	srq->bit_fields_len * sizeof(u32));
1811	}
1812
1813	if (init_attr->attr.srq_limit) {
1814	status = ocrdma_mbx_modify_srq(srq, &init_attr->attr);
1815	if (status)
1816	goto arm_err;
1817	}
1818
1819	if (udata) {
1820	status = ocrdma_copy_srq_uresp(dev, srq, udata);
1821	if (status)
1822	goto arm_err;
1823	}
1824
1825	return 0;
1826
1827	arm_err:
1828	ocrdma_mbx_destroy_srq(dev, srq);
1829	kfree(objp: srq->rqe_wr_id_tbl);
1830	kfree(objp: srq->idx_bit_fields);
1831	return status;
1832	}
1833
1834	int ocrdma_modify_srq(struct ib_srq *ibsrq,
1835	struct ib_srq_attr *srq_attr,
1836	enum ib_srq_attr_mask srq_attr_mask,
1837	struct ib_udata *udata)
1838	{
1839	int status;
1840	struct ocrdma_srq *srq;
1841
1842	srq = get_ocrdma_srq(ibsrq);
1843	if (srq_attr_mask & IB_SRQ_MAX_WR)
1844	status = -EINVAL;
1845	else
1846	status = ocrdma_mbx_modify_srq(srq, srq_attr);
1847	return status;
1848	}
1849
1850	int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
1851	{
1852	struct ocrdma_srq *srq;
1853
1854	srq = get_ocrdma_srq(ibsrq);
1855	return ocrdma_mbx_query_srq(srq, srq_attr);
1856	}
1857
1858	int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1859	{
1860	struct ocrdma_srq *srq;
1861	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibsrq->device);
1862
1863	srq = get_ocrdma_srq(ibsrq);
1864
1865	ocrdma_mbx_destroy_srq(dev, srq);
1866
1867	if (srq->pd->uctx)
1868	ocrdma_del_mmap(uctx: srq->pd->uctx, phy_addr: (u64) srq->rq.pa,
1869	PAGE_ALIGN(srq->rq.len));
1870
1871	kfree(objp: srq->idx_bit_fields);
1872	kfree(objp: srq->rqe_wr_id_tbl);
1873	return 0;
1874	}
1875
1876	/* unprivileged verbs and their support functions. */
1877	static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,
1878	struct ocrdma_hdr_wqe *hdr,
1879	const struct ib_send_wr *wr)
1880	{
1881	struct ocrdma_ewqe_ud_hdr *ud_hdr =
1882	(struct ocrdma_ewqe_ud_hdr *)(hdr + 1);
1883	struct ocrdma_ah *ah = get_ocrdma_ah(ibah: ud_wr(wr)->ah);
1884
1885	ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn;
1886	if (qp->qp_type == IB_QPT_GSI)
1887	ud_hdr->qkey = qp->qkey;
1888	else
1889	ud_hdr->qkey = ud_wr(wr)->remote_qkey;
1890	ud_hdr->rsvd_ahid = ah->id;
1891	ud_hdr->hdr_type = ah->hdr_type;
1892	if (ah->av->valid & OCRDMA_AV_VLAN_VALID)
1893	hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT);
1894	}
1895
1896	static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,
1897	struct ocrdma_sge *sge, int num_sge,
1898	struct ib_sge *sg_list)
1899	{
1900	int i;
1901
1902	for (i = 0; i < num_sge; i++) {
1903	sge[i].lrkey = sg_list[i].lkey;
1904	sge[i].addr_lo = sg_list[i].addr;
1905	sge[i].addr_hi = upper_32_bits(sg_list[i].addr);
1906	sge[i].len = sg_list[i].length;
1907	hdr->total_len += sg_list[i].length;
1908	}
1909	if (num_sge == 0)
1910	memset(sge, 0, sizeof(*sge));
1911	}
1912
1913	static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge)
1914	{
1915	uint32_t total_len = 0, i;
1916
1917	for (i = 0; i < num_sge; i++)
1918	total_len += sg_list[i].length;
1919	return total_len;
1920	}
1921
1922
1923	static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
1924	struct ocrdma_hdr_wqe *hdr,
1925	struct ocrdma_sge *sge,
1926	const struct ib_send_wr *wr, u32 wqe_size)
1927	{
1928	int i;
1929	char *dpp_addr;
1930
1931	if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) {
1932	hdr->total_len = ocrdma_sglist_len(sg_list: wr->sg_list, num_sge: wr->num_sge);
1933	if (unlikely(hdr->total_len > qp->max_inline_data)) {
1934	pr_err("%s() supported_len=0x%x,\n"
1935	" unsupported len req=0x%x\n", __func__,
1936	qp->max_inline_data, hdr->total_len);
1937	return -EINVAL;
1938	}
1939	dpp_addr = (char *)sge;
1940	for (i = 0; i < wr->num_sge; i++) {
1941	memcpy(dpp_addr,
1942	(void *)(unsigned long)wr->sg_list[i].addr,
1943	wr->sg_list[i].length);
1944	dpp_addr += wr->sg_list[i].length;
1945	}
1946
1947	wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);
1948	if (0 == hdr->total_len)
1949	wqe_size += sizeof(struct ocrdma_sge);
1950	hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);
1951	} else {
1952	ocrdma_build_sges(hdr, sge, num_sge: wr->num_sge, sg_list: wr->sg_list);
1953	if (wr->num_sge)
1954	wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));
1955	else
1956	wqe_size += sizeof(struct ocrdma_sge);
1957	hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1958	}
1959	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1960	return 0;
1961	}
1962
1963	static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1964	const struct ib_send_wr *wr)
1965	{
1966	struct ocrdma_sge *sge;
1967	u32 wqe_size = sizeof(*hdr);
1968
1969	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
1970	ocrdma_build_ud_hdr(qp, hdr, wr);
1971	sge = (struct ocrdma_sge *)(hdr + 2);
1972	wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);
1973	} else {
1974	sge = (struct ocrdma_sge *)(hdr + 1);
1975	}
1976
1977	return ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1978	}
1979
1980	static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1981	const struct ib_send_wr *wr)
1982	{
1983	int status;
1984	struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1985	struct ocrdma_sge *sge = ext_rw + 1;
1986	u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw);
1987
1988	status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1989	if (status)
1990	return status;
1991	ext_rw->addr_lo = rdma_wr(wr)->remote_addr;
1992	ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);
1993	ext_rw->lrkey = rdma_wr(wr)->rkey;
1994	ext_rw->len = hdr->total_len;
1995	return 0;
1996	}
1997
1998	static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1999	const struct ib_send_wr *wr)
2000	{
2001	struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
2002	struct ocrdma_sge *sge = ext_rw + 1;
2003	u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) +
2004	sizeof(struct ocrdma_hdr_wqe);
2005
2006	ocrdma_build_sges(hdr, sge, num_sge: wr->num_sge, sg_list: wr->sg_list);
2007	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
2008	hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT);
2009	hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
2010
2011	ext_rw->addr_lo = rdma_wr(wr)->remote_addr;
2012	ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);
2013	ext_rw->lrkey = rdma_wr(wr)->rkey;
2014	ext_rw->len = hdr->total_len;
2015	}
2016
2017	static int get_encoded_page_size(int pg_sz)
2018	{
2019	/* Max size is 256M 4096 << 16 */
2020	int i = 0;
2021	for (; i < 17; i++)
2022	if (pg_sz == (4096 << i))
2023	break;
2024	return i;
2025	}
2026
2027	static int ocrdma_build_reg(struct ocrdma_qp *qp,
2028	struct ocrdma_hdr_wqe *hdr,
2029	const struct ib_reg_wr *wr)
2030	{
2031	u64 fbo;
2032	struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1);
2033	struct ocrdma_mr *mr = get_ocrdma_mr(ibmr: wr->mr);
2034	struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
2035	struct ocrdma_pbe *pbe;
2036	u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr);
2037	int num_pbes = 0, i;
2038
2039	wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES);
2040
2041	hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT);
2042	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
2043
2044	if (wr->access & IB_ACCESS_LOCAL_WRITE)
2045	hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR;
2046	if (wr->access & IB_ACCESS_REMOTE_WRITE)
2047	hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR;
2048	if (wr->access & IB_ACCESS_REMOTE_READ)
2049	hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD;
2050	hdr->lkey = wr->key;
2051	hdr->total_len = mr->ibmr.length;
2052
2053	fbo = mr->ibmr.iova - mr->pages[0];
2054
2055	fast_reg->va_hi = upper_32_bits(mr->ibmr.iova);
2056	fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff);
2057	fast_reg->fbo_hi = upper_32_bits(fbo);
2058	fast_reg->fbo_lo = (u32) fbo & 0xffffffff;
2059	fast_reg->num_sges = mr->npages;
2060	fast_reg->size_sge = get_encoded_page_size(pg_sz: mr->ibmr.page_size);
2061
2062	pbe = pbl_tbl->va;
2063	for (i = 0; i < mr->npages; i++) {
2064	u64 buf_addr = mr->pages[i];
2065
2066	pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK));
2067	pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr));
2068	num_pbes += 1;
2069	pbe++;
2070
2071	/* if the pbl is full storing the pbes,
2072	* move to next pbl.
2073	*/
2074	if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) {
2075	pbl_tbl++;
2076	pbe = (struct ocrdma_pbe *)pbl_tbl->va;
2077	}
2078	}
2079
2080	return 0;
2081	}
2082
2083	static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
2084	{
2085	u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT);
2086
2087	iowrite32(val, qp->sq_db);
2088	}
2089
2090	int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
2091	const struct ib_send_wr **bad_wr)
2092	{
2093	int status = 0;
2094	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2095	struct ocrdma_hdr_wqe *hdr;
2096	unsigned long flags;
2097
2098	spin_lock_irqsave(&qp->q_lock, flags);
2099	if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {
2100	spin_unlock_irqrestore(lock: &qp->q_lock, flags);
2101	*bad_wr = wr;
2102	return -EINVAL;
2103	}
2104
2105	while (wr) {
2106	if (qp->qp_type == IB_QPT_UD &&
2107	(wr->opcode != IB_WR_SEND &&
2108	wr->opcode != IB_WR_SEND_WITH_IMM)) {
2109	*bad_wr = wr;
2110	status = -EINVAL;
2111	break;
2112	}
2113	if (ocrdma_hwq_free_cnt(q: &qp->sq) == 0 ||
2114	wr->num_sge > qp->sq.max_sges) {
2115	*bad_wr = wr;
2116	status = -ENOMEM;
2117	break;
2118	}
2119	hdr = ocrdma_hwq_head(q: &qp->sq);
2120	hdr->cw = 0;
2121	if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2122	hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2123	if (wr->send_flags & IB_SEND_FENCE)
2124	hdr->cw |=
2125	(OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT);
2126	if (wr->send_flags & IB_SEND_SOLICITED)
2127	hdr->cw |=
2128	(OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT);
2129	hdr->total_len = 0;
2130	switch (wr->opcode) {
2131	case IB_WR_SEND_WITH_IMM:
2132	hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2133	hdr->immdt = ntohl(wr->ex.imm_data);
2134	fallthrough;
2135	case IB_WR_SEND:
2136	hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2137	ocrdma_build_send(qp, hdr, wr);
2138	break;
2139	case IB_WR_SEND_WITH_INV:
2140	hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
2141	hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2142	hdr->lkey = wr->ex.invalidate_rkey;
2143	status = ocrdma_build_send(qp, hdr, wr);
2144	break;
2145	case IB_WR_RDMA_WRITE_WITH_IMM:
2146	hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2147	hdr->immdt = ntohl(wr->ex.imm_data);
2148	fallthrough;
2149	case IB_WR_RDMA_WRITE:
2150	hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT);
2151	status = ocrdma_build_write(qp, hdr, wr);
2152	break;
2153	case IB_WR_RDMA_READ:
2154	ocrdma_build_read(qp, hdr, wr);
2155	break;
2156	case IB_WR_LOCAL_INV:
2157	hdr->cw |=
2158	(OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT);
2159	hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) +
2160	sizeof(struct ocrdma_sge)) /
2161	OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT;
2162	hdr->lkey = wr->ex.invalidate_rkey;
2163	break;
2164	case IB_WR_REG_MR:
2165	status = ocrdma_build_reg(qp, hdr, wr: reg_wr(wr));
2166	break;
2167	default:
2168	status = -EINVAL;
2169	break;
2170	}
2171	if (status) {
2172	*bad_wr = wr;
2173	break;
2174	}
2175	if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2176	qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
2177	else
2178	qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;
2179	qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;
2180	ocrdma_cpu_to_le32(dst: hdr, len: ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) &
2181	OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE);
2182	/* make sure wqe is written before adapter can access it */
2183	wmb();
2184	/* inform hw to start processing it */
2185	ocrdma_ring_sq_db(qp);
2186
2187	/* update pointer, counter for next wr */
2188	ocrdma_hwq_inc_head(q: &qp->sq);
2189	wr = wr->next;
2190	}
2191	spin_unlock_irqrestore(lock: &qp->q_lock, flags);
2192	return status;
2193	}
2194
2195	static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
2196	{
2197	u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT);
2198
2199	iowrite32(val, qp->rq_db);
2200	}
2201
2202	static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe,
2203	const struct ib_recv_wr *wr, u16 tag)
2204	{
2205	u32 wqe_size = 0;
2206	struct ocrdma_sge *sge;
2207	if (wr->num_sge)
2208	wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);
2209	else
2210	wqe_size = sizeof(*sge) + sizeof(*rqe);
2211
2212	rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) <<
2213	OCRDMA_WQE_SIZE_SHIFT);
2214	rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2215	rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
2216	rqe->total_len = 0;
2217	rqe->rsvd_tag = tag;
2218	sge = (struct ocrdma_sge *)(rqe + 1);
2219	ocrdma_build_sges(hdr: rqe, sge, num_sge: wr->num_sge, sg_list: wr->sg_list);
2220	ocrdma_cpu_to_le32(dst: rqe, len: wqe_size);
2221	}
2222
2223	int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
2224	const struct ib_recv_wr **bad_wr)
2225	{
2226	int status = 0;
2227	unsigned long flags;
2228	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2229	struct ocrdma_hdr_wqe *rqe;
2230
2231	spin_lock_irqsave(&qp->q_lock, flags);
2232	if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {
2233	spin_unlock_irqrestore(lock: &qp->q_lock, flags);
2234	*bad_wr = wr;
2235	return -EINVAL;
2236	}
2237	while (wr) {
2238	if (ocrdma_hwq_free_cnt(q: &qp->rq) == 0 ||
2239	wr->num_sge > qp->rq.max_sges) {
2240	*bad_wr = wr;
2241	status = -ENOMEM;
2242	break;
2243	}
2244	rqe = ocrdma_hwq_head(q: &qp->rq);
2245	ocrdma_build_rqe(rqe, wr, tag: 0);
2246
2247	qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;
2248	/* make sure rqe is written before adapter can access it */
2249	wmb();
2250
2251	/* inform hw to start processing it */
2252	ocrdma_ring_rq_db(qp);
2253
2254	/* update pointer, counter for next wr */
2255	ocrdma_hwq_inc_head(q: &qp->rq);
2256	wr = wr->next;
2257	}
2258	spin_unlock_irqrestore(lock: &qp->q_lock, flags);
2259	return status;
2260	}
2261
2262	/* cqe for srq's rqe can potentially arrive out of order.
2263	* index gives the entry in the shadow table where to store
2264	* the wr_id. tag/index is returned in cqe to reference back
2265	* for a given rqe.
2266	*/
2267	static int ocrdma_srq_get_idx(struct ocrdma_srq *srq)
2268	{
2269	int row = 0;
2270	int indx = 0;
2271
2272	for (row = 0; row < srq->bit_fields_len; row++) {
2273	if (srq->idx_bit_fields[row]) {
2274	indx = ffs(srq->idx_bit_fields[row]);
2275	indx = (row * 32) + (indx - 1);
2276	BUG_ON(indx >= srq->rq.max_cnt);
2277	ocrdma_srq_toggle_bit(srq, idx: indx);
2278	break;
2279	}
2280	}
2281
2282	BUG_ON(row == srq->bit_fields_len);
2283	return indx + 1; /* Use from index 1 */
2284	}
2285
2286	static void ocrdma_ring_srq_db(struct ocrdma_srq *srq)
2287	{
2288	u32 val = srq->rq.dbid | (1 << 16);
2289
2290	iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET);
2291	}
2292
2293	int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
2294	const struct ib_recv_wr **bad_wr)
2295	{
2296	int status = 0;
2297	unsigned long flags;
2298	struct ocrdma_srq *srq;
2299	struct ocrdma_hdr_wqe *rqe;
2300	u16 tag;
2301
2302	srq = get_ocrdma_srq(ibsrq);
2303
2304	spin_lock_irqsave(&srq->q_lock, flags);
2305	while (wr) {
2306	if (ocrdma_hwq_free_cnt(q: &srq->rq) == 0 ||
2307	wr->num_sge > srq->rq.max_sges) {
2308	status = -ENOMEM;
2309	*bad_wr = wr;
2310	break;
2311	}
2312	tag = ocrdma_srq_get_idx(srq);
2313	rqe = ocrdma_hwq_head(q: &srq->rq);
2314	ocrdma_build_rqe(rqe, wr, tag);
2315
2316	srq->rqe_wr_id_tbl[tag] = wr->wr_id;
2317	/* make sure rqe is written before adapter can perform DMA */
2318	wmb();
2319	/* inform hw to start processing it */
2320	ocrdma_ring_srq_db(srq);
2321	/* update pointer, counter for next wr */
2322	ocrdma_hwq_inc_head(q: &srq->rq);
2323	wr = wr->next;
2324	}
2325	spin_unlock_irqrestore(lock: &srq->q_lock, flags);
2326	return status;
2327	}
2328
2329	static enum ib_wc_status ocrdma_to_ibwc_err(u16 status)
2330	{
2331	enum ib_wc_status ibwc_status;
2332
2333	switch (status) {
2334	case OCRDMA_CQE_GENERAL_ERR:
2335	ibwc_status = IB_WC_GENERAL_ERR;
2336	break;
2337	case OCRDMA_CQE_LOC_LEN_ERR:
2338	ibwc_status = IB_WC_LOC_LEN_ERR;
2339	break;
2340	case OCRDMA_CQE_LOC_QP_OP_ERR:
2341	ibwc_status = IB_WC_LOC_QP_OP_ERR;
2342	break;
2343	case OCRDMA_CQE_LOC_EEC_OP_ERR:
2344	ibwc_status = IB_WC_LOC_EEC_OP_ERR;
2345	break;
2346	case OCRDMA_CQE_LOC_PROT_ERR:
2347	ibwc_status = IB_WC_LOC_PROT_ERR;
2348	break;
2349	case OCRDMA_CQE_WR_FLUSH_ERR:
2350	ibwc_status = IB_WC_WR_FLUSH_ERR;
2351	break;
2352	case OCRDMA_CQE_MW_BIND_ERR:
2353	ibwc_status = IB_WC_MW_BIND_ERR;
2354	break;
2355	case OCRDMA_CQE_BAD_RESP_ERR:
2356	ibwc_status = IB_WC_BAD_RESP_ERR;
2357	break;
2358	case OCRDMA_CQE_LOC_ACCESS_ERR:
2359	ibwc_status = IB_WC_LOC_ACCESS_ERR;
2360	break;
2361	case OCRDMA_CQE_REM_INV_REQ_ERR:
2362	ibwc_status = IB_WC_REM_INV_REQ_ERR;
2363	break;
2364	case OCRDMA_CQE_REM_ACCESS_ERR:
2365	ibwc_status = IB_WC_REM_ACCESS_ERR;
2366	break;
2367	case OCRDMA_CQE_REM_OP_ERR:
2368	ibwc_status = IB_WC_REM_OP_ERR;
2369	break;
2370	case OCRDMA_CQE_RETRY_EXC_ERR:
2371	ibwc_status = IB_WC_RETRY_EXC_ERR;
2372	break;
2373	case OCRDMA_CQE_RNR_RETRY_EXC_ERR:
2374	ibwc_status = IB_WC_RNR_RETRY_EXC_ERR;
2375	break;
2376	case OCRDMA_CQE_LOC_RDD_VIOL_ERR:
2377	ibwc_status = IB_WC_LOC_RDD_VIOL_ERR;
2378	break;
2379	case OCRDMA_CQE_REM_INV_RD_REQ_ERR:
2380	ibwc_status = IB_WC_REM_INV_RD_REQ_ERR;
2381	break;
2382	case OCRDMA_CQE_REM_ABORT_ERR:
2383	ibwc_status = IB_WC_REM_ABORT_ERR;
2384	break;
2385	case OCRDMA_CQE_INV_EECN_ERR:
2386	ibwc_status = IB_WC_INV_EECN_ERR;
2387	break;
2388	case OCRDMA_CQE_INV_EEC_STATE_ERR:
2389	ibwc_status = IB_WC_INV_EEC_STATE_ERR;
2390	break;
2391	case OCRDMA_CQE_FATAL_ERR:
2392	ibwc_status = IB_WC_FATAL_ERR;
2393	break;
2394	case OCRDMA_CQE_RESP_TIMEOUT_ERR:
2395	ibwc_status = IB_WC_RESP_TIMEOUT_ERR;
2396	break;
2397	default:
2398	ibwc_status = IB_WC_GENERAL_ERR;
2399	break;
2400	}
2401	return ibwc_status;
2402	}
2403
2404	static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,
2405	u32 wqe_idx)
2406	{
2407	struct ocrdma_hdr_wqe *hdr;
2408	struct ocrdma_sge *rw;
2409	int opcode;
2410
2411	hdr = ocrdma_hwq_head_from_idx(q: &qp->sq, idx: wqe_idx);
2412
2413	ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;
2414	/* Undo the hdr->cw swap */
2415	opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK;
2416	switch (opcode) {
2417	case OCRDMA_WRITE:
2418	ibwc->opcode = IB_WC_RDMA_WRITE;
2419	break;
2420	case OCRDMA_READ:
2421	rw = (struct ocrdma_sge *)(hdr + 1);
2422	ibwc->opcode = IB_WC_RDMA_READ;
2423	ibwc->byte_len = rw->len;
2424	break;
2425	case OCRDMA_SEND:
2426	ibwc->opcode = IB_WC_SEND;
2427	break;
2428	case OCRDMA_FR_MR:
2429	ibwc->opcode = IB_WC_REG_MR;
2430	break;
2431	case OCRDMA_LKEY_INV:
2432	ibwc->opcode = IB_WC_LOCAL_INV;
2433	break;
2434	default:
2435	ibwc->status = IB_WC_GENERAL_ERR;
2436	pr_err("%s() invalid opcode received = 0x%x\n",
2437	__func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK);
2438	break;
2439	}
2440	}
2441
2442	static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,
2443	struct ocrdma_cqe *cqe)
2444	{
2445	if (is_cqe_for_sq(cqe)) {
2446	cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2447	cqe->flags_status_srcqpn) &
2448	~OCRDMA_CQE_STATUS_MASK);
2449	cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2450	cqe->flags_status_srcqpn) |
2451	(OCRDMA_CQE_WR_FLUSH_ERR <<
2452	OCRDMA_CQE_STATUS_SHIFT));
2453	} else {
2454	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2455	cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2456	cqe->flags_status_srcqpn) &
2457	~OCRDMA_CQE_UD_STATUS_MASK);
2458	cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2459	cqe->flags_status_srcqpn) |
2460	(OCRDMA_CQE_WR_FLUSH_ERR <<
2461	OCRDMA_CQE_UD_STATUS_SHIFT));
2462	} else {
2463	cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2464	cqe->flags_status_srcqpn) &
2465	~OCRDMA_CQE_STATUS_MASK);
2466	cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2467	cqe->flags_status_srcqpn) |
2468	(OCRDMA_CQE_WR_FLUSH_ERR <<
2469	OCRDMA_CQE_STATUS_SHIFT));
2470	}
2471	}
2472	}
2473
2474	static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2475	struct ocrdma_qp *qp, int status)
2476	{
2477	bool expand = false;
2478
2479	ibwc->byte_len = 0;
2480	ibwc->qp = &qp->ibqp;
2481	ibwc->status = ocrdma_to_ibwc_err(status);
2482
2483	ocrdma_flush_qp(qp);
2484	ocrdma_qp_state_change(qp, new_state: IB_QPS_ERR, NULL);
2485
2486	/* if wqe/rqe pending for which cqe needs to be returned,
2487	* trigger inflating it.
2488	*/
2489	if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {
2490	expand = true;
2491	ocrdma_set_cqe_status_flushed(qp, cqe);
2492	}
2493	return expand;
2494	}
2495
2496	static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2497	struct ocrdma_qp *qp, int status)
2498	{
2499	ibwc->opcode = IB_WC_RECV;
2500	ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2501	ocrdma_hwq_inc_tail(q: &qp->rq);
2502
2503	return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2504	}
2505
2506	static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2507	struct ocrdma_qp *qp, int status)
2508	{
2509	ocrdma_update_wc(qp, ibwc, wqe_idx: qp->sq.tail);
2510	ocrdma_hwq_inc_tail(q: &qp->sq);
2511
2512	return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2513	}
2514
2515
2516	static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp,
2517	struct ocrdma_cqe *cqe, struct ib_wc *ibwc,
2518	bool *polled, bool *stop)
2519	{
2520	bool expand;
2521	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: qp->ibqp.device);
2522	int status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2523	OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2524	if (status < OCRDMA_MAX_CQE_ERR)
2525	atomic_inc(v: &dev->cqe_err_stats[status]);
2526
2527	/* when hw sq is empty, but rq is not empty, so we continue
2528	* to keep the cqe in order to get the cq event again.
2529	*/
2530	if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {
2531	/* when cq for rq and sq is same, it is safe to return
2532	* flush cqe for RQEs.
2533	*/
2534	if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2535	*polled = true;
2536	status = OCRDMA_CQE_WR_FLUSH_ERR;
2537	expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2538	} else {
2539	/* stop processing further cqe as this cqe is used for
2540	* triggering cq event on buddy cq of RQ.
2541	* When QP is destroyed, this cqe will be removed
2542	* from the cq's hardware q.
2543	*/
2544	*polled = false;
2545	*stop = true;
2546	expand = false;
2547	}
2548	} else if (is_hw_sq_empty(qp)) {
2549	/* Do nothing */
2550	expand = false;
2551	*polled = false;
2552	*stop = false;
2553	} else {
2554	*polled = true;
2555	expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2556	}
2557	return expand;
2558	}
2559
2560	static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
2561	struct ocrdma_cqe *cqe,
2562	struct ib_wc *ibwc, bool *polled)
2563	{
2564	bool expand = false;
2565	int tail = qp->sq.tail;
2566	u32 wqe_idx;
2567
2568	if (!qp->wqe_wr_id_tbl[tail].signaled) {
2569	*polled = false; /* WC cannot be consumed yet */
2570	} else {
2571	ibwc->status = IB_WC_SUCCESS;
2572	ibwc->wc_flags = 0;
2573	ibwc->qp = &qp->ibqp;
2574	ocrdma_update_wc(qp, ibwc, wqe_idx: tail);
2575	*polled = true;
2576	}
2577	wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) &
2578	OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;
2579	if (tail != wqe_idx)
2580	expand = true; /* Coalesced CQE can't be consumed yet */
2581
2582	ocrdma_hwq_inc_tail(q: &qp->sq);
2583	return expand;
2584	}
2585
2586	static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2587	struct ib_wc *ibwc, bool *polled, bool *stop)
2588	{
2589	int status;
2590	bool expand;
2591
2592	status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2593	OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2594
2595	if (status == OCRDMA_CQE_SUCCESS)
2596	expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);
2597	else
2598	expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);
2599	return expand;
2600	}
2601
2602	static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc,
2603	struct ocrdma_cqe *cqe)
2604	{
2605	int status;
2606	u16 hdr_type = 0;
2607
2608	status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2609	OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT;
2610	ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) &
2611	OCRDMA_CQE_SRCQP_MASK;
2612	ibwc->pkey_index = 0;
2613	ibwc->wc_flags = IB_WC_GRH;
2614	ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2615	OCRDMA_CQE_UD_XFER_LEN_SHIFT) &
2616	OCRDMA_CQE_UD_XFER_LEN_MASK;
2617
2618	if (ocrdma_is_udp_encap_supported(dev)) {
2619	hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2620	OCRDMA_CQE_UD_L3TYPE_SHIFT) &
2621	OCRDMA_CQE_UD_L3TYPE_MASK;
2622	ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
2623	ibwc->network_hdr_type = hdr_type;
2624	}
2625
2626	return status;
2627	}
2628
2629	static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc,
2630	struct ocrdma_cqe *cqe,
2631	struct ocrdma_qp *qp)
2632	{
2633	unsigned long flags;
2634	struct ocrdma_srq *srq;
2635	u32 wqe_idx;
2636
2637	srq = get_ocrdma_srq(ibsrq: qp->ibqp.srq);
2638	wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
2639	OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx;
2640	BUG_ON(wqe_idx < 1);
2641
2642	ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];
2643	spin_lock_irqsave(&srq->q_lock, flags);
2644	ocrdma_srq_toggle_bit(srq, idx: wqe_idx - 1);
2645	spin_unlock_irqrestore(lock: &srq->q_lock, flags);
2646	ocrdma_hwq_inc_tail(q: &srq->rq);
2647	}
2648
2649	static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2650	struct ib_wc *ibwc, bool *polled, bool *stop,
2651	int status)
2652	{
2653	bool expand;
2654	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: qp->ibqp.device);
2655
2656	if (status < OCRDMA_MAX_CQE_ERR)
2657	atomic_inc(v: &dev->cqe_err_stats[status]);
2658
2659	/* when hw_rq is empty, but wq is not empty, so continue
2660	* to keep the cqe to get the cq event again.
2661	*/
2662	if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {
2663	if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2664	*polled = true;
2665	status = OCRDMA_CQE_WR_FLUSH_ERR;
2666	expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2667	} else {
2668	*polled = false;
2669	*stop = true;
2670	expand = false;
2671	}
2672	} else if (is_hw_rq_empty(qp)) {
2673	/* Do nothing */
2674	expand = false;
2675	*polled = false;
2676	*stop = false;
2677	} else {
2678	*polled = true;
2679	expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2680	}
2681	return expand;
2682	}
2683
2684	static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
2685	struct ocrdma_cqe *cqe, struct ib_wc *ibwc)
2686	{
2687	struct ocrdma_dev *dev;
2688
2689	dev = get_ocrdma_dev(ibdev: qp->ibqp.device);
2690	ibwc->opcode = IB_WC_RECV;
2691	ibwc->qp = &qp->ibqp;
2692	ibwc->status = IB_WC_SUCCESS;
2693
2694	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
2695	ocrdma_update_ud_rcqe(dev, ibwc, cqe);
2696	else
2697	ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen);
2698
2699	if (is_cqe_imm(cqe)) {
2700	ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2701	ibwc->wc_flags |= IB_WC_WITH_IMM;
2702	} else if (is_cqe_wr_imm(cqe)) {
2703	ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2704	ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2705	ibwc->wc_flags |= IB_WC_WITH_IMM;
2706	} else if (is_cqe_invalidated(cqe)) {
2707	ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt);
2708	ibwc->wc_flags |= IB_WC_WITH_INVALIDATE;
2709	}
2710	if (qp->ibqp.srq) {
2711	ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
2712	} else {
2713	ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2714	ocrdma_hwq_inc_tail(q: &qp->rq);
2715	}
2716	}
2717
2718	static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2719	struct ib_wc *ibwc, bool *polled, bool *stop)
2720	{
2721	int status;
2722	bool expand = false;
2723
2724	ibwc->wc_flags = 0;
2725	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2726	status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2727	OCRDMA_CQE_UD_STATUS_MASK) >>
2728	OCRDMA_CQE_UD_STATUS_SHIFT;
2729	} else {
2730	status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2731	OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2732	}
2733
2734	if (status == OCRDMA_CQE_SUCCESS) {
2735	*polled = true;
2736	ocrdma_poll_success_rcqe(qp, cqe, ibwc);
2737	} else {
2738	expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,
2739	status);
2740	}
2741	return expand;
2742	}
2743
2744	static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,
2745	u16 cur_getp)
2746	{
2747	if (cq->phase_change) {
2748	if (cur_getp == 0)
2749	cq->phase = (~cq->phase & OCRDMA_CQE_VALID);
2750	} else {
2751	/* clear valid bit */
2752	cqe->flags_status_srcqpn = 0;
2753	}
2754	}
2755
2756	static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
2757	struct ib_wc *ibwc)
2758	{
2759	u16 qpn = 0;
2760	int i = 0;
2761	bool expand = false;
2762	int polled_hw_cqes = 0;
2763	struct ocrdma_qp *qp = NULL;
2764	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: cq->ibcq.device);
2765	struct ocrdma_cqe *cqe;
2766	u16 cur_getp; bool polled = false; bool stop = false;
2767
2768	cur_getp = cq->getp;
2769	while (num_entries) {
2770	cqe = cq->va + cur_getp;
2771	/* check whether valid cqe or not */
2772	if (!is_cqe_valid(cq, cqe))
2773	break;
2774	qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK);
2775	/* ignore discarded cqe */
2776	if (qpn == 0)
2777	goto skip_cqe;
2778	qp = dev->qp_tbl[qpn];
2779	BUG_ON(qp == NULL);
2780
2781	if (is_cqe_for_sq(cqe)) {
2782	expand = ocrdma_poll_scqe(qp, cqe, ibwc, polled: &polled,
2783	stop: &stop);
2784	} else {
2785	expand = ocrdma_poll_rcqe(qp, cqe, ibwc, polled: &polled,
2786	stop: &stop);
2787	}
2788	if (expand)
2789	goto expand_cqe;
2790	if (stop)
2791	goto stop_cqe;
2792	/* clear qpn to avoid duplicate processing by discard_cqe() */
2793	cqe->cmn.qpn = 0;
2794	skip_cqe:
2795	polled_hw_cqes += 1;
2796	cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
2797	ocrdma_change_cq_phase(cq, cqe, cur_getp);
2798	expand_cqe:
2799	if (polled) {
2800	num_entries -= 1;
2801	i += 1;
2802	ibwc = ibwc + 1;
2803	polled = false;
2804	}
2805	}
2806	stop_cqe:
2807	cq->getp = cur_getp;
2808
2809	if (polled_hw_cqes)
2810	ocrdma_ring_cq_db(dev, cq_id: cq->id, armed: false, solicited: false, cqe_popped: polled_hw_cqes);
2811
2812	return i;
2813	}
2814
2815	/* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */
2816	static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
2817	struct ocrdma_qp *qp, struct ib_wc *ibwc)
2818	{
2819	int err_cqes = 0;
2820
2821	while (num_entries) {
2822	if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))
2823	break;
2824	if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {
2825	ocrdma_update_wc(qp, ibwc, wqe_idx: qp->sq.tail);
2826	ocrdma_hwq_inc_tail(q: &qp->sq);
2827	} else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
2828	ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2829	ocrdma_hwq_inc_tail(q: &qp->rq);
2830	} else {
2831	return err_cqes;
2832	}
2833	ibwc->byte_len = 0;
2834	ibwc->status = IB_WC_WR_FLUSH_ERR;
2835	ibwc = ibwc + 1;
2836	err_cqes += 1;
2837	num_entries -= 1;
2838	}
2839	return err_cqes;
2840	}
2841
2842	int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2843	{
2844	int cqes_to_poll = num_entries;
2845	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2846	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibcq->device);
2847	int num_os_cqe = 0, err_cqes = 0;
2848	struct ocrdma_qp *qp;
2849	unsigned long flags;
2850
2851	/* poll cqes from adapter CQ */
2852	spin_lock_irqsave(&cq->cq_lock, flags);
2853	num_os_cqe = ocrdma_poll_hwcq(cq, num_entries: cqes_to_poll, ibwc: wc);
2854	spin_unlock_irqrestore(lock: &cq->cq_lock, flags);
2855	cqes_to_poll -= num_os_cqe;
2856
2857	if (cqes_to_poll) {
2858	wc = wc + num_os_cqe;
2859	/* adapter returns single error cqe when qp moves to
2860	* error state. So insert error cqes with wc_status as
2861	* FLUSHED for pending WQEs and RQEs of QP's SQ and RQ
2862	* respectively which uses this CQ.
2863	*/
2864	spin_lock_irqsave(&dev->flush_q_lock, flags);
2865	list_for_each_entry(qp, &cq->sq_head, sq_entry) {
2866	if (cqes_to_poll == 0)
2867	break;
2868	err_cqes = ocrdma_add_err_cqe(cq, num_entries: cqes_to_poll, qp, ibwc: wc);
2869	cqes_to_poll -= err_cqes;
2870	num_os_cqe += err_cqes;
2871	wc = wc + err_cqes;
2872	}
2873	spin_unlock_irqrestore(lock: &dev->flush_q_lock, flags);
2874	}
2875	return num_os_cqe;
2876	}
2877
2878	int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
2879	{
2880	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2881	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibcq->device);
2882	u16 cq_id;
2883	unsigned long flags;
2884	bool arm_needed = false, sol_needed = false;
2885
2886	cq_id = cq->id;
2887
2888	spin_lock_irqsave(&cq->cq_lock, flags);
2889	if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED)
2890	arm_needed = true;
2891	if (cq_flags & IB_CQ_SOLICITED)
2892	sol_needed = true;
2893
2894	ocrdma_ring_cq_db(dev, cq_id, armed: arm_needed, solicited: sol_needed, cqe_popped: 0);
2895	spin_unlock_irqrestore(lock: &cq->cq_lock, flags);
2896
2897	return 0;
2898	}
2899
2900	struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2901	u32 max_num_sg)
2902	{
2903	int status;
2904	struct ocrdma_mr *mr;
2905	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
2906	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev: ibpd->device);
2907
2908	if (mr_type != IB_MR_TYPE_MEM_REG)
2909	return ERR_PTR(error: -EINVAL);
2910
2911	if (max_num_sg > dev->attr.max_pages_per_frmr)
2912	return ERR_PTR(error: -EINVAL);
2913
2914	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2915	if (!mr)
2916	return ERR_PTR(error: -ENOMEM);
2917
2918	mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
2919	if (!mr->pages) {
2920	status = -ENOMEM;
2921	goto pl_err;
2922	}
2923
2924	status = ocrdma_get_pbl_info(dev, mr, num_pbes: max_num_sg);
2925	if (status)
2926	goto pbl_err;
2927	mr->hwmr.fr_mr = 1;
2928	mr->hwmr.remote_rd = 0;
2929	mr->hwmr.remote_wr = 0;
2930	mr->hwmr.local_rd = 0;
2931	mr->hwmr.local_wr = 0;
2932	mr->hwmr.mw_bind = 0;
2933	status = ocrdma_build_pbl_tbl(dev, mr: &mr->hwmr);
2934	if (status)
2935	goto pbl_err;
2936	status = ocrdma_reg_mr(dev, hwmr: &mr->hwmr, pd_id: pd->id, acc: 0);
2937	if (status)
2938	goto mbx_err;
2939	mr->ibmr.rkey = mr->hwmr.lkey;
2940	mr->ibmr.lkey = mr->hwmr.lkey;
2941	dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] =
2942	(unsigned long) mr;
2943	return &mr->ibmr;
2944	mbx_err:
2945	ocrdma_free_mr_pbl_tbl(dev, mr: &mr->hwmr);
2946	pbl_err:
2947	kfree(objp: mr->pages);
2948	pl_err:
2949	kfree(objp: mr);
2950	return ERR_PTR(error: -ENOMEM);
2951	}
2952
2953	static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr)
2954	{
2955	struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
2956
2957	if (unlikely(mr->npages == mr->hwmr.num_pbes))
2958	return -ENOMEM;
2959
2960	mr->pages[mr->npages++] = addr;
2961
2962	return 0;
2963	}
2964
2965	int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
2966	unsigned int *sg_offset)
2967	{
2968	struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
2969
2970	mr->npages = 0;
2971
2972	return ib_sg_to_pages(mr: ibmr, sgl: sg, sg_nents, sg_offset, set_page: ocrdma_set_page);
2973	}
2974