1	// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2
3	/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4	/* Copyright (c) 2008-2019, IBM Corporation */
5
6	#include <linux/errno.h>
7	#include <linux/types.h>
8	#include <linux/uaccess.h>
9	#include <linux/vmalloc.h>
10	#include <linux/xarray.h>
11	#include <net/addrconf.h>
12
13	#include <rdma/iw_cm.h>
14	#include <rdma/ib_verbs.h>
15	#include <rdma/ib_user_verbs.h>
16	#include <rdma/uverbs_ioctl.h>
17
18	#include "siw.h"
19	#include "siw_verbs.h"
20	#include "siw_mem.h"
21
22	static int siw_qp_state_to_ib_qp_state[SIW_QP_STATE_COUNT] = {
23	[SIW_QP_STATE_IDLE] = IB_QPS_INIT,
24	[SIW_QP_STATE_RTR] = IB_QPS_RTR,
25	[SIW_QP_STATE_RTS] = IB_QPS_RTS,
26	[SIW_QP_STATE_CLOSING] = IB_QPS_SQD,
27	[SIW_QP_STATE_TERMINATE] = IB_QPS_SQE,
28	[SIW_QP_STATE_ERROR] = IB_QPS_ERR
29	};
30
31	static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
32	[IB_QPS_RESET] = SIW_QP_STATE_IDLE,
33	[IB_QPS_INIT] = SIW_QP_STATE_IDLE,
34	[IB_QPS_RTR] = SIW_QP_STATE_RTR,
35	[IB_QPS_RTS] = SIW_QP_STATE_RTS,
36	[IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
37	[IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
38	[IB_QPS_ERR] = SIW_QP_STATE_ERROR
39	};
40
41	static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
42	[IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
43	[IB_QPS_RTS] = "RTS", [IB_QPS_SQD] = "SQD", [IB_QPS_SQE] = "SQE",
44	[IB_QPS_ERR] = "ERR"
45	};
46
47	void siw_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
48	{
49	struct siw_user_mmap_entry *entry = to_siw_mmap_entry(rdma_mmap: rdma_entry);
50
51	kfree(objp: entry);
52	}
53
54	int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
55	{
56	struct siw_ucontext *uctx = to_siw_ctx(base_ctx: ctx);
57	size_t size = vma->vm_end - vma->vm_start;
58	struct rdma_user_mmap_entry *rdma_entry;
59	struct siw_user_mmap_entry *entry;
60	int rv = -EINVAL;
61
62	/*
63	* Must be page aligned
64	*/
65	if (vma->vm_start & (PAGE_SIZE - 1)) {
66	pr_warn("siw: mmap not page aligned\n");
67	return -EINVAL;
68	}
69	rdma_entry = rdma_user_mmap_entry_get(ucontext: &uctx->base_ucontext, vma);
70	if (!rdma_entry) {
71	siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %#zx\n",
72	vma->vm_pgoff, size);
73	return -EINVAL;
74	}
75	entry = to_siw_mmap_entry(rdma_mmap: rdma_entry);
76
77	rv = remap_vmalloc_range(vma, addr: entry->address, pgoff: 0);
78	if (rv)
79	pr_warn("remap_vmalloc_range failed: %lu, %zu\n", vma->vm_pgoff,
80	size);
81	rdma_user_mmap_entry_put(entry: rdma_entry);
82
83	return rv;
84	}
85
86	int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
87	{
88	struct siw_device *sdev = to_siw_dev(base_dev: base_ctx->device);
89	struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
90	struct siw_uresp_alloc_ctx uresp = {};
91	int rv;
92
93	if (atomic_inc_return(v: &sdev->num_ctx) > SIW_MAX_CONTEXT) {
94	rv = -ENOMEM;
95	goto err_out;
96	}
97	ctx->sdev = sdev;
98
99	uresp.dev_id = sdev->vendor_part_id;
100
101	if (udata->outlen < sizeof(uresp)) {
102	rv = -EINVAL;
103	goto err_out;
104	}
105	rv = ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
106	if (rv)
107	goto err_out;
108
109	siw_dbg(base_ctx->device, "success. now %d context(s)\n",
110	atomic_read(&sdev->num_ctx));
111
112	return 0;
113
114	err_out:
115	atomic_dec(v: &sdev->num_ctx);
116	siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
117	atomic_read(&sdev->num_ctx));
118
119	return rv;
120	}
121
122	void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
123	{
124	struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
125
126	atomic_dec(v: &uctx->sdev->num_ctx);
127	}
128
129	int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
130	struct ib_udata *udata)
131	{
132	struct siw_device *sdev = to_siw_dev(base_dev);
133
134	if (udata->inlen || udata->outlen)
135	return -EINVAL;
136
137	memset(attr, 0, sizeof(*attr));
138
139	/* Revisit atomic caps if RFC 7306 gets supported */
140	attr->atomic_cap = 0;
141	attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
142	attr->kernel_cap_flags = IBK_ALLOW_USER_UNREG;
143	attr->max_cq = sdev->attrs.max_cq;
144	attr->max_cqe = sdev->attrs.max_cqe;
145	attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
146	attr->max_mr = sdev->attrs.max_mr;
147	attr->max_mw = sdev->attrs.max_mw;
148	attr->max_mr_size = ~0ull;
149	attr->max_pd = sdev->attrs.max_pd;
150	attr->max_qp = sdev->attrs.max_qp;
151	attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
152	attr->max_qp_rd_atom = sdev->attrs.max_ord;
153	attr->max_qp_wr = sdev->attrs.max_qp_wr;
154	attr->max_recv_sge = sdev->attrs.max_sge;
155	attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
156	attr->max_send_sge = sdev->attrs.max_sge;
157	attr->max_sge_rd = sdev->attrs.max_sge_rd;
158	attr->max_srq = sdev->attrs.max_srq;
159	attr->max_srq_sge = sdev->attrs.max_srq_sge;
160	attr->max_srq_wr = sdev->attrs.max_srq_wr;
161	attr->page_size_cap = PAGE_SIZE;
162	attr->vendor_id = SIW_VENDOR_ID;
163	attr->vendor_part_id = sdev->vendor_part_id;
164
165	addrconf_addr_eui48(eui: (u8 *)&attr->sys_image_guid,
166	addr: sdev->raw_gid);
167
168	return 0;
169	}
170
171	int siw_query_port(struct ib_device *base_dev, u32 port,
172	struct ib_port_attr *attr)
173	{
174	struct net_device *ndev;
175	int rv;
176
177	memset(attr, 0, sizeof(*attr));
178
179	rv = ib_get_eth_speed(dev: base_dev, port_num: port, speed: &attr->active_speed,
180	width: &attr->active_width);
181	if (rv)
182	return rv;
183
184	ndev = ib_device_get_netdev(ib_dev: base_dev, SIW_PORT);
185	if (!ndev)
186	return -ENODEV;
187
188	attr->gid_tbl_len = 1;
189	attr->max_msg_sz = -1;
190	attr->max_mtu = ib_mtu_int_to_enum(mtu: ndev->max_mtu);
191	attr->active_mtu = ib_mtu_int_to_enum(READ_ONCE(ndev->mtu));
192	attr->state = ib_get_curr_port_state(net_dev: ndev);
193	attr->phys_state = attr->state == IB_PORT_ACTIVE ?
194	IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
195	attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
196	/*
197	* All zero
198	*
199	* attr->lid = 0;
200	* attr->bad_pkey_cntr = 0;
201	* attr->qkey_viol_cntr = 0;
202	* attr->sm_lid = 0;
203	* attr->lmc = 0;
204	* attr->max_vl_num = 0;
205	* attr->sm_sl = 0;
206	* attr->subnet_timeout = 0;
207	* attr->init_type_repy = 0;
208	*/
209	dev_put(dev: ndev);
210	return rv;
211	}
212
213	int siw_get_port_immutable(struct ib_device *base_dev, u32 port,
214	struct ib_port_immutable *port_immutable)
215	{
216	struct ib_port_attr attr;
217	int rv = siw_query_port(base_dev, port, attr: &attr);
218
219	if (rv)
220	return rv;
221
222	port_immutable->gid_tbl_len = attr.gid_tbl_len;
223	port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
224
225	return 0;
226	}
227
228	int siw_query_gid(struct ib_device *base_dev, u32 port, int idx,
229	union ib_gid *gid)
230	{
231	struct siw_device *sdev = to_siw_dev(base_dev);
232
233	/* subnet_prefix == interface_id == 0; */
234	memset(gid, 0, sizeof(*gid));
235	memcpy(gid->raw, sdev->raw_gid, ETH_ALEN);
236
237	return 0;
238	}
239
240	int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
241	{
242	struct siw_device *sdev = to_siw_dev(base_dev: pd->device);
243
244	if (atomic_inc_return(v: &sdev->num_pd) > SIW_MAX_PD) {
245	atomic_dec(v: &sdev->num_pd);
246	return -ENOMEM;
247	}
248	siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
249
250	return 0;
251	}
252
253	int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
254	{
255	struct siw_device *sdev = to_siw_dev(base_dev: pd->device);
256
257	siw_dbg_pd(pd, "free PD\n");
258	atomic_dec(v: &sdev->num_pd);
259	return 0;
260	}
261
262	void siw_qp_get_ref(struct ib_qp *base_qp)
263	{
264	siw_qp_get(qp: to_siw_qp(base_qp));
265	}
266
267	void siw_qp_put_ref(struct ib_qp *base_qp)
268	{
269	siw_qp_put(qp: to_siw_qp(base_qp));
270	}
271
272	static struct rdma_user_mmap_entry *
273	siw_mmap_entry_insert(struct siw_ucontext *uctx,
274	void *address, size_t length,
275	u64 *offset)
276	{
277	struct siw_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
278	int rv;
279
280	*offset = SIW_INVAL_UOBJ_KEY;
281	if (!entry)
282	return NULL;
283
284	entry->address = address;
285
286	rv = rdma_user_mmap_entry_insert(ucontext: &uctx->base_ucontext,
287	entry: &entry->rdma_entry,
288	length);
289	if (rv) {
290	kfree(objp: entry);
291	return NULL;
292	}
293
294	*offset = rdma_user_mmap_get_offset(entry: &entry->rdma_entry);
295
296	return &entry->rdma_entry;
297	}
298
299	/*
300	* siw_create_qp()
301	*
302	* Create QP of requested size on given device.
303	*
304	* @qp: Queue pait
305	* @attrs: Initial QP attributes.
306	* @udata: used to provide QP ID, SQ and RQ size back to user.
307	*/
308
309	int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
310	struct ib_udata *udata)
311	{
312	struct ib_pd *pd = ibqp->pd;
313	struct siw_qp *qp = to_siw_qp(base_qp: ibqp);
314	struct ib_device *base_dev = pd->device;
315	struct siw_device *sdev = to_siw_dev(base_dev);
316	struct siw_ucontext *uctx =
317	rdma_udata_to_drv_context(udata, struct siw_ucontext,
318	base_ucontext);
319	unsigned long flags;
320	int num_sqe, num_rqe, rv = 0;
321	size_t length;
322
323	siw_dbg(base_dev, "create new QP\n");
324
325	if (attrs->create_flags)
326	return -EOPNOTSUPP;
327
328	if (atomic_inc_return(v: &sdev->num_qp) > SIW_MAX_QP) {
329	siw_dbg(base_dev, "too many QP's\n");
330	rv = -ENOMEM;
331	goto err_atomic;
332	}
333	if (attrs->qp_type != IB_QPT_RC) {
334	siw_dbg(base_dev, "only RC QP's supported\n");
335	rv = -EOPNOTSUPP;
336	goto err_atomic;
337	}
338	if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
339	(attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
340	(attrs->cap.max_send_sge > SIW_MAX_SGE) ||
341	(attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
342	siw_dbg(base_dev, "QP size error\n");
343	rv = -EINVAL;
344	goto err_atomic;
345	}
346	if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
347	siw_dbg(base_dev, "max inline send: %d > %d\n",
348	attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
349	rv = -EINVAL;
350	goto err_atomic;
351	}
352	/*
353	* NOTE: we don't allow for a QP unable to hold any SQ WQE
354	*/
355	if (attrs->cap.max_send_wr == 0) {
356	siw_dbg(base_dev, "QP must have send queue\n");
357	rv = -EINVAL;
358	goto err_atomic;
359	}
360
361	if (!attrs->send_cq || (!attrs->recv_cq && !attrs->srq)) {
362	siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
363	rv = -EINVAL;
364	goto err_atomic;
365	}
366
367	init_rwsem(&qp->state_lock);
368	spin_lock_init(&qp->sq_lock);
369	spin_lock_init(&qp->rq_lock);
370	spin_lock_init(&qp->orq_lock);
371
372	rv = siw_qp_add(sdev, qp);
373	if (rv)
374	goto err_atomic;
375
376
377	/* All queue indices are derived from modulo operations
378	* on a free running 'get' (consumer) and 'put' (producer)
379	* unsigned counter. Having queue sizes at power of two
380	* avoids handling counter wrap around.
381	*/
382	num_sqe = roundup_pow_of_two(attrs->cap.max_send_wr);
383	num_rqe = attrs->cap.max_recv_wr;
384	if (num_rqe)
385	num_rqe = roundup_pow_of_two(num_rqe);
386
387	if (udata)
388	qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
389	else
390	qp->sendq = vcalloc(num_sqe, sizeof(struct siw_sqe));
391
392	if (qp->sendq == NULL) {
393	rv = -ENOMEM;
394	goto err_out_xa;
395	}
396	if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
397	if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
398	qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
399	else {
400	rv = -EINVAL;
401	goto err_out_xa;
402	}
403	}
404	qp->pd = pd;
405	qp->scq = to_siw_cq(base_cq: attrs->send_cq);
406	qp->rcq = to_siw_cq(base_cq: attrs->recv_cq);
407
408	if (attrs->srq) {
409	/*
410	* SRQ support.
411	* Verbs 6.3.7: ignore RQ size, if SRQ present
412	* Verbs 6.3.5: do not check PD of SRQ against PD of QP
413	*/
414	qp->srq = to_siw_srq(base_srq: attrs->srq);
415	qp->attrs.rq_size = 0;
416	siw_dbg(base_dev, "QP [%u]: SRQ attached\n",
417	qp->base_qp.qp_num);
418	} else if (num_rqe) {
419	if (udata)
420	qp->recvq =
421	vmalloc_user(num_rqe * sizeof(struct siw_rqe));
422	else
423	qp->recvq = vcalloc(num_rqe, sizeof(struct siw_rqe));
424
425	if (qp->recvq == NULL) {
426	rv = -ENOMEM;
427	goto err_out_xa;
428	}
429	qp->attrs.rq_size = num_rqe;
430	}
431	qp->attrs.sq_size = num_sqe;
432	qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
433	qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
434
435	/* Make those two tunables fixed for now. */
436	qp->tx_ctx.gso_seg_limit = 1;
437	qp->tx_ctx.zcopy_tx = zcopy_tx;
438
439	qp->attrs.state = SIW_QP_STATE_IDLE;
440
441	if (udata) {
442	struct siw_uresp_create_qp uresp = {};
443
444	uresp.num_sqe = num_sqe;
445	uresp.num_rqe = num_rqe;
446	uresp.qp_id = qp_id(qp);
447
448	if (qp->sendq) {
449	length = num_sqe * sizeof(struct siw_sqe);
450	qp->sq_entry =
451	siw_mmap_entry_insert(uctx, address: qp->sendq,
452	length, offset: &uresp.sq_key);
453	if (!qp->sq_entry) {
454	rv = -ENOMEM;
455	goto err_out_xa;
456	}
457	}
458
459	if (qp->recvq) {
460	length = num_rqe * sizeof(struct siw_rqe);
461	qp->rq_entry =
462	siw_mmap_entry_insert(uctx, address: qp->recvq,
463	length, offset: &uresp.rq_key);
464	if (!qp->rq_entry) {
465	uresp.sq_key = SIW_INVAL_UOBJ_KEY;
466	rv = -ENOMEM;
467	goto err_out_xa;
468	}
469	}
470
471	if (udata->outlen < sizeof(uresp)) {
472	rv = -EINVAL;
473	goto err_out_xa;
474	}
475	rv = ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
476	if (rv)
477	goto err_out_xa;
478	}
479	qp->tx_cpu = siw_get_tx_cpu(sdev);
480	if (qp->tx_cpu < 0) {
481	rv = -EINVAL;
482	goto err_out_xa;
483	}
484	INIT_LIST_HEAD(list: &qp->devq);
485	spin_lock_irqsave(&sdev->lock, flags);
486	list_add_tail(new: &qp->devq, head: &sdev->qp_list);
487	spin_unlock_irqrestore(lock: &sdev->lock, flags);
488
489	init_completion(x: &qp->qp_free);
490
491	return 0;
492
493	err_out_xa:
494	xa_erase(&sdev->qp_xa, index: qp_id(qp));
495	if (uctx) {
496	rdma_user_mmap_entry_remove(entry: qp->sq_entry);
497	rdma_user_mmap_entry_remove(entry: qp->rq_entry);
498	}
499	vfree(addr: qp->sendq);
500	vfree(addr: qp->recvq);
501
502	err_atomic:
503	atomic_dec(v: &sdev->num_qp);
504	return rv;
505	}
506
507	/*
508	* Minimum siw_query_qp() verb interface.
509	*
510	* @qp_attr_mask is not used but all available information is provided
511	*/
512	int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
513	int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
514	{
515	struct siw_qp *qp;
516	struct net_device *ndev;
517
518	if (base_qp && qp_attr && qp_init_attr)
519	qp = to_siw_qp(base_qp);
520	else
521	return -EINVAL;
522
523	ndev = ib_device_get_netdev(ib_dev: base_qp->device, SIW_PORT);
524	if (!ndev)
525	return -ENODEV;
526
527	qp_attr->qp_state = siw_qp_state_to_ib_qp_state[qp->attrs.state];
528	qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
529	qp_attr->cap.max_send_wr = qp->attrs.sq_size;
530	qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
531	qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
532	qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
533	qp_attr->path_mtu = ib_mtu_int_to_enum(READ_ONCE(ndev->mtu));
534	qp_attr->max_rd_atomic = qp->attrs.irq_size;
535	qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
536
537	qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
538	IB_ACCESS_REMOTE_WRITE |
539	IB_ACCESS_REMOTE_READ;
540
541	qp_init_attr->qp_type = base_qp->qp_type;
542	qp_init_attr->send_cq = base_qp->send_cq;
543	qp_init_attr->recv_cq = base_qp->recv_cq;
544	qp_init_attr->srq = base_qp->srq;
545
546	qp_init_attr->cap = qp_attr->cap;
547
548	dev_put(dev: ndev);
549	return 0;
550	}
551
552	int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
553	int attr_mask, struct ib_udata *udata)
554	{
555	struct siw_qp_attrs new_attrs;
556	enum siw_qp_attr_mask siw_attr_mask = 0;
557	struct siw_qp *qp = to_siw_qp(base_qp);
558	int rv = 0;
559
560	if (!attr_mask)
561	return 0;
562
563	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
564	return -EOPNOTSUPP;
565
566	memset(&new_attrs, 0, sizeof(new_attrs));
567
568	if (attr_mask & IB_QP_ACCESS_FLAGS) {
569	siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
570
571	if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
572	new_attrs.flags |= SIW_RDMA_READ_ENABLED;
573	if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
574	new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
575	if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
576	new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
577	}
578	if (attr_mask & IB_QP_STATE) {
579	siw_dbg_qp(qp, "desired IB QP state: %s\n",
580	ib_qp_state_to_string[attr->qp_state]);
581
582	new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
583
584	if (new_attrs.state > SIW_QP_STATE_RTS)
585	qp->tx_ctx.tx_suspend = 1;
586
587	siw_attr_mask |= SIW_QP_ATTR_STATE;
588	}
589	if (!siw_attr_mask)
590	goto out;
591
592	down_write(sem: &qp->state_lock);
593
594	rv = siw_qp_modify(qp, attr: &new_attrs, mask: siw_attr_mask);
595
596	up_write(sem: &qp->state_lock);
597	out:
598	return rv;
599	}
600
601	int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
602	{
603	struct siw_qp *qp = to_siw_qp(base_qp);
604	struct siw_ucontext *uctx =
605	rdma_udata_to_drv_context(udata, struct siw_ucontext,
606	base_ucontext);
607	struct siw_qp_attrs qp_attrs;
608
609	siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
610
611	/*
612	* Mark QP as in process of destruction to prevent from
613	* any async callbacks to RDMA core
614	*/
615	qp->attrs.flags |= SIW_QP_IN_DESTROY;
616	qp->rx_stream.rx_suspend = 1;
617
618	if (uctx) {
619	rdma_user_mmap_entry_remove(entry: qp->sq_entry);
620	rdma_user_mmap_entry_remove(entry: qp->rq_entry);
621	}
622
623	down_write(sem: &qp->state_lock);
624
625	qp_attrs.state = SIW_QP_STATE_ERROR;
626	siw_qp_modify(qp, attr: &qp_attrs, mask: SIW_QP_ATTR_STATE);
627
628	if (qp->cep) {
629	siw_cep_put(cep: qp->cep);
630	qp->cep = NULL;
631	}
632	up_write(sem: &qp->state_lock);
633
634	qp->scq = qp->rcq = NULL;
635
636	siw_qp_put(qp);
637	wait_for_completion(&qp->qp_free);
638
639	return 0;
640	}
641
642	/*
643	* siw_copy_inline_sgl()
644	*
645	* Prepare sgl of inlined data for sending. For userland callers
646	* function checks if given buffer addresses and len's are within
647	* process context bounds.
648	* Data from all provided sge's are copied together into the wqe,
649	* referenced by a single sge.
650	*/
651	static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
652	struct siw_sqe *sqe)
653	{
654	struct ib_sge *core_sge = core_wr->sg_list;
655	void *kbuf = &sqe->sge[1];
656	int num_sge = core_wr->num_sge, bytes = 0;
657
658	sqe->sge[0].laddr = (uintptr_t)kbuf;
659	sqe->sge[0].lkey = 0;
660
661	while (num_sge--) {
662	if (!core_sge->length) {
663	core_sge++;
664	continue;
665	}
666	bytes += core_sge->length;
667	if (bytes > SIW_MAX_INLINE) {
668	bytes = -EINVAL;
669	break;
670	}
671	memcpy(kbuf, ib_virt_dma_to_ptr(core_sge->addr),
672	core_sge->length);
673
674	kbuf += core_sge->length;
675	core_sge++;
676	}
677	sqe->sge[0].length = max(bytes, 0);
678	sqe->num_sge = bytes > 0 ? 1 : 0;
679
680	return bytes;
681	}
682
683	/* Complete SQ WR's without processing */
684	static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
685	const struct ib_send_wr **bad_wr)
686	{
687	int rv = 0;
688
689	while (wr) {
690	struct siw_sqe sqe = {};
691
692	switch (wr->opcode) {
693	case IB_WR_RDMA_WRITE:
694	sqe.opcode = SIW_OP_WRITE;
695	break;
696	case IB_WR_RDMA_READ:
697	sqe.opcode = SIW_OP_READ;
698	break;
699	case IB_WR_RDMA_READ_WITH_INV:
700	sqe.opcode = SIW_OP_READ_LOCAL_INV;
701	break;
702	case IB_WR_SEND:
703	sqe.opcode = SIW_OP_SEND;
704	break;
705	case IB_WR_SEND_WITH_IMM:
706	sqe.opcode = SIW_OP_SEND_WITH_IMM;
707	break;
708	case IB_WR_SEND_WITH_INV:
709	sqe.opcode = SIW_OP_SEND_REMOTE_INV;
710	break;
711	case IB_WR_LOCAL_INV:
712	sqe.opcode = SIW_OP_INVAL_STAG;
713	break;
714	case IB_WR_REG_MR:
715	sqe.opcode = SIW_OP_REG_MR;
716	break;
717	default:
718	rv = -EINVAL;
719	break;
720	}
721	if (!rv) {
722	sqe.id = wr->wr_id;
723	rv = siw_sqe_complete(qp, sqe: &sqe, bytes: 0,
724	status: SIW_WC_WR_FLUSH_ERR);
725	}
726	if (rv) {
727	if (bad_wr)
728	*bad_wr = wr;
729	break;
730	}
731	wr = wr->next;
732	}
733	return rv;
734	}
735
736	/* Complete RQ WR's without processing */
737	static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
738	const struct ib_recv_wr **bad_wr)
739	{
740	struct siw_rqe rqe = {};
741	int rv = 0;
742
743	while (wr) {
744	rqe.id = wr->wr_id;
745	rv = siw_rqe_complete(qp, rqe: &rqe, bytes: 0, inval_stag: 0, status: SIW_WC_WR_FLUSH_ERR);
746	if (rv) {
747	if (bad_wr)
748	*bad_wr = wr;
749	break;
750	}
751	wr = wr->next;
752	}
753	return rv;
754	}
755
756	/*
757	* siw_post_send()
758	*
759	* Post a list of S-WR's to a SQ.
760	*
761	* @base_qp: Base QP contained in siw QP
762	* @wr: Null terminated list of user WR's
763	* @bad_wr: Points to failing WR in case of synchronous failure.
764	*/
765	int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
766	const struct ib_send_wr **bad_wr)
767	{
768	struct siw_qp *qp = to_siw_qp(base_qp);
769	struct siw_wqe *wqe = tx_wqe(qp);
770
771	unsigned long flags;
772	int rv = 0, imm_err = 0;
773
774	if (wr && !rdma_is_kernel_res(res: &qp->base_qp.res)) {
775	siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
776	*bad_wr = wr;
777	return -EINVAL;
778	}
779
780	/*
781	* Try to acquire QP state lock. Must be non-blocking
782	* to accommodate kernel clients needs.
783	*/
784	if (!down_read_trylock(sem: &qp->state_lock)) {
785	if (qp->attrs.state == SIW_QP_STATE_ERROR) {
786	/*
787	* ERROR state is final, so we can be sure
788	* this state will not change as long as the QP
789	* exists.
790	*
791	* This handles an ib_drain_sq() call with
792	* a concurrent request to set the QP state
793	* to ERROR.
794	*/
795	rv = siw_sq_flush_wr(qp, wr, bad_wr);
796	} else {
797	siw_dbg_qp(qp, "QP locked, state %d\n",
798	qp->attrs.state);
799	*bad_wr = wr;
800	rv = -ENOTCONN;
801	}
802	return rv;
803	}
804	if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
805	if (qp->attrs.state == SIW_QP_STATE_ERROR) {
806	/*
807	* Immediately flush this WR to CQ, if QP
808	* is in ERROR state. SQ is guaranteed to
809	* be empty, so WR complets in-order.
810	*
811	* Typically triggered by ib_drain_sq().
812	*/
813	rv = siw_sq_flush_wr(qp, wr, bad_wr);
814	} else {
815	siw_dbg_qp(qp, "QP out of state %d\n",
816	qp->attrs.state);
817	*bad_wr = wr;
818	rv = -ENOTCONN;
819	}
820	up_read(sem: &qp->state_lock);
821	return rv;
822	}
823	spin_lock_irqsave(&qp->sq_lock, flags);
824
825	while (wr) {
826	u32 idx = qp->sq_put % qp->attrs.sq_size;
827	struct siw_sqe *sqe = &qp->sendq[idx];
828
829	if (sqe->flags) {
830	siw_dbg_qp(qp, "sq full\n");
831	rv = -ENOMEM;
832	break;
833	}
834	if (wr->num_sge > qp->attrs.sq_max_sges) {
835	siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
836	rv = -EINVAL;
837	break;
838	}
839	sqe->id = wr->wr_id;
840
841	if ((wr->send_flags & IB_SEND_SIGNALED) ||
842	(qp->attrs.flags & SIW_SIGNAL_ALL_WR))
843	sqe->flags |= SIW_WQE_SIGNALLED;
844
845	if (wr->send_flags & IB_SEND_FENCE)
846	sqe->flags |= SIW_WQE_READ_FENCE;
847
848	switch (wr->opcode) {
849	case IB_WR_SEND:
850	case IB_WR_SEND_WITH_INV:
851	if (wr->send_flags & IB_SEND_SOLICITED)
852	sqe->flags |= SIW_WQE_SOLICITED;
853
854	if (!(wr->send_flags & IB_SEND_INLINE)) {
855	siw_copy_sgl(sge: wr->sg_list, siw_sge: sqe->sge,
856	num_sge: wr->num_sge);
857	sqe->num_sge = wr->num_sge;
858	} else {
859	rv = siw_copy_inline_sgl(core_wr: wr, sqe);
860	if (rv <= 0) {
861	rv = -EINVAL;
862	break;
863	}
864	sqe->flags |= SIW_WQE_INLINE;
865	sqe->num_sge = 1;
866	}
867	if (wr->opcode == IB_WR_SEND)
868	sqe->opcode = SIW_OP_SEND;
869	else {
870	sqe->opcode = SIW_OP_SEND_REMOTE_INV;
871	sqe->rkey = wr->ex.invalidate_rkey;
872	}
873	break;
874
875	case IB_WR_RDMA_READ_WITH_INV:
876	case IB_WR_RDMA_READ:
877	/*
878	* iWarp restricts RREAD sink to SGL containing
879	* 1 SGE only. we could relax to SGL with multiple
880	* elements referring the SAME ltag or even sending
881	* a private per-rreq tag referring to a checked
882	* local sgl with MULTIPLE ltag's.
883	*/
884	if (unlikely(wr->num_sge != 1)) {
885	rv = -EINVAL;
886	break;
887	}
888	siw_copy_sgl(sge: wr->sg_list, siw_sge: &sqe->sge[0], num_sge: 1);
889	/*
890	* NOTE: zero length RREAD is allowed!
891	*/
892	sqe->raddr = rdma_wr(wr)->remote_addr;
893	sqe->rkey = rdma_wr(wr)->rkey;
894	sqe->num_sge = 1;
895
896	if (wr->opcode == IB_WR_RDMA_READ)
897	sqe->opcode = SIW_OP_READ;
898	else
899	sqe->opcode = SIW_OP_READ_LOCAL_INV;
900	break;
901
902	case IB_WR_RDMA_WRITE:
903	if (!(wr->send_flags & IB_SEND_INLINE)) {
904	siw_copy_sgl(sge: wr->sg_list, siw_sge: &sqe->sge[0],
905	num_sge: wr->num_sge);
906	sqe->num_sge = wr->num_sge;
907	} else {
908	rv = siw_copy_inline_sgl(core_wr: wr, sqe);
909	if (unlikely(rv < 0)) {
910	rv = -EINVAL;
911	break;
912	}
913	sqe->flags |= SIW_WQE_INLINE;
914	sqe->num_sge = 1;
915	}
916	sqe->raddr = rdma_wr(wr)->remote_addr;
917	sqe->rkey = rdma_wr(wr)->rkey;
918	sqe->opcode = SIW_OP_WRITE;
919	break;
920
921	case IB_WR_REG_MR:
922	sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
923	sqe->rkey = reg_wr(wr)->key;
924	sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
925	sqe->opcode = SIW_OP_REG_MR;
926	break;
927
928	case IB_WR_LOCAL_INV:
929	sqe->rkey = wr->ex.invalidate_rkey;
930	sqe->opcode = SIW_OP_INVAL_STAG;
931	break;
932
933	default:
934	siw_dbg_qp(qp, "ib wr type %d unsupported\n",
935	wr->opcode);
936	rv = -EINVAL;
937	break;
938	}
939	siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%p\n",
940	sqe->opcode, sqe->flags,
941	(void *)(uintptr_t)sqe->id);
942
943	if (unlikely(rv < 0))
944	break;
945
946	/* make SQE only valid after completely written */
947	smp_wmb();
948	sqe->flags |= SIW_WQE_VALID;
949
950	qp->sq_put++;
951	wr = wr->next;
952	}
953
954	/*
955	* Send directly if SQ processing is not in progress.
956	* Eventual immediate errors (rv < 0) do not affect the involved
957	* RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
958	* processing, if new work is already pending. But rv and pointer
959	* to failed work request must be passed to caller.
960	*/
961	if (unlikely(rv < 0)) {
962	/*
963	* Immediate error
964	*/
965	siw_dbg_qp(qp, "Immediate error %d\n", rv);
966	imm_err = rv;
967	*bad_wr = wr;
968	}
969	if (wqe->wr_status != SIW_WR_IDLE) {
970	spin_unlock_irqrestore(lock: &qp->sq_lock, flags);
971	goto skip_direct_sending;
972	}
973	rv = siw_activate_tx(qp);
974	spin_unlock_irqrestore(lock: &qp->sq_lock, flags);
975
976	if (rv <= 0)
977	goto skip_direct_sending;
978
979	if (rdma_is_kernel_res(res: &qp->base_qp.res)) {
980	rv = siw_sq_start(qp);
981	} else {
982	qp->tx_ctx.in_syscall = 1;
983
984	if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
985	siw_qp_cm_drop(qp, schedule: 0);
986
987	qp->tx_ctx.in_syscall = 0;
988	}
989	skip_direct_sending:
990
991	up_read(sem: &qp->state_lock);
992
993	if (unlikely(imm_err))
994	return imm_err;
995
996	return (rv >= 0) ? 0 : rv;
997	}
998
999	/*
1000	* siw_post_receive()
1001	*
1002	* Post a list of R-WR's to a RQ.
1003	*
1004	* @base_qp: Base QP contained in siw QP
1005	* @wr: Null terminated list of user WR's
1006	* @bad_wr: Points to failing WR in case of synchronous failure.
1007	*/
1008	int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
1009	const struct ib_recv_wr **bad_wr)
1010	{
1011	struct siw_qp *qp = to_siw_qp(base_qp);
1012	unsigned long flags;
1013	int rv = 0;
1014
1015	if (qp->srq || qp->attrs.rq_size == 0) {
1016	*bad_wr = wr;
1017	return -EINVAL;
1018	}
1019	if (!rdma_is_kernel_res(res: &qp->base_qp.res)) {
1020	siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n");
1021	*bad_wr = wr;
1022	return -EINVAL;
1023	}
1024
1025	/*
1026	* Try to acquire QP state lock. Must be non-blocking
1027	* to accommodate kernel clients needs.
1028	*/
1029	if (!down_read_trylock(sem: &qp->state_lock)) {
1030	if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1031	/*
1032	* ERROR state is final, so we can be sure
1033	* this state will not change as long as the QP
1034	* exists.
1035	*
1036	* This handles an ib_drain_rq() call with
1037	* a concurrent request to set the QP state
1038	* to ERROR.
1039	*/
1040	rv = siw_rq_flush_wr(qp, wr, bad_wr);
1041	} else {
1042	siw_dbg_qp(qp, "QP locked, state %d\n",
1043	qp->attrs.state);
1044	*bad_wr = wr;
1045	rv = -ENOTCONN;
1046	}
1047	return rv;
1048	}
1049	if (qp->attrs.state > SIW_QP_STATE_RTS) {
1050	if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1051	/*
1052	* Immediately flush this WR to CQ, if QP
1053	* is in ERROR state. RQ is guaranteed to
1054	* be empty, so WR complets in-order.
1055	*
1056	* Typically triggered by ib_drain_rq().
1057	*/
1058	rv = siw_rq_flush_wr(qp, wr, bad_wr);
1059	} else {
1060	siw_dbg_qp(qp, "QP out of state %d\n",
1061	qp->attrs.state);
1062	*bad_wr = wr;
1063	rv = -ENOTCONN;
1064	}
1065	up_read(sem: &qp->state_lock);
1066	return rv;
1067	}
1068	/*
1069	* Serialize potentially multiple producers.
1070	* Not needed for single threaded consumer side.
1071	*/
1072	spin_lock_irqsave(&qp->rq_lock, flags);
1073
1074	while (wr) {
1075	u32 idx = qp->rq_put % qp->attrs.rq_size;
1076	struct siw_rqe *rqe = &qp->recvq[idx];
1077
1078	if (rqe->flags) {
1079	siw_dbg_qp(qp, "RQ full\n");
1080	rv = -ENOMEM;
1081	break;
1082	}
1083	if (wr->num_sge > qp->attrs.rq_max_sges) {
1084	siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1085	rv = -EINVAL;
1086	break;
1087	}
1088	rqe->id = wr->wr_id;
1089	rqe->num_sge = wr->num_sge;
1090	siw_copy_sgl(sge: wr->sg_list, siw_sge: rqe->sge, num_sge: wr->num_sge);
1091
1092	/* make sure RQE is completely written before valid */
1093	smp_wmb();
1094
1095	rqe->flags = SIW_WQE_VALID;
1096
1097	qp->rq_put++;
1098	wr = wr->next;
1099	}
1100	spin_unlock_irqrestore(lock: &qp->rq_lock, flags);
1101
1102	up_read(sem: &qp->state_lock);
1103
1104	if (rv < 0) {
1105	siw_dbg_qp(qp, "error %d\n", rv);
1106	*bad_wr = wr;
1107	}
1108	return rv;
1109	}
1110
1111	int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1112	{
1113	struct siw_cq *cq = to_siw_cq(base_cq);
1114	struct siw_device *sdev = to_siw_dev(base_dev: base_cq->device);
1115	struct siw_ucontext *ctx =
1116	rdma_udata_to_drv_context(udata, struct siw_ucontext,
1117	base_ucontext);
1118
1119	siw_dbg_cq(cq, "free CQ resources\n");
1120
1121	siw_cq_flush(cq);
1122
1123	if (ctx)
1124	rdma_user_mmap_entry_remove(entry: cq->cq_entry);
1125
1126	atomic_dec(v: &sdev->num_cq);
1127
1128	vfree(addr: cq->queue);
1129	return 0;
1130	}
1131
1132	/*
1133	* siw_create_cq()
1134	*
1135	* Populate CQ of requested size
1136	*
1137	* @base_cq: CQ as allocated by RDMA midlayer
1138	* @attr: Initial CQ attributes
1139	* @attrs: uverbs bundle
1140	*/
1141
1142	int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1143	struct uverbs_attr_bundle *attrs)
1144	{
1145	struct ib_udata *udata = &attrs->driver_udata;
1146	struct siw_device *sdev = to_siw_dev(base_dev: base_cq->device);
1147	struct siw_cq *cq = to_siw_cq(base_cq);
1148	int rv, size = attr->cqe;
1149
1150	if (attr->flags)
1151	return -EOPNOTSUPP;
1152
1153	if (atomic_inc_return(v: &sdev->num_cq) > SIW_MAX_CQ) {
1154	siw_dbg(base_cq->device, "too many CQ's\n");
1155	rv = -ENOMEM;
1156	goto err_out;
1157	}
1158	if (size < 1 || size > sdev->attrs.max_cqe) {
1159	siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1160	rv = -EINVAL;
1161	goto err_out;
1162	}
1163	size = roundup_pow_of_two(size);
1164	cq->base_cq.cqe = size;
1165	cq->num_cqe = size;
1166
1167	if (udata)
1168	cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1169	sizeof(struct siw_cq_ctrl));
1170	else
1171	cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1172	sizeof(struct siw_cq_ctrl));
1173
1174	if (cq->queue == NULL) {
1175	rv = -ENOMEM;
1176	goto err_out;
1177	}
1178	get_random_bytes(buf: &cq->id, len: 4);
1179	siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1180
1181	spin_lock_init(&cq->lock);
1182
1183	cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1184
1185	if (udata) {
1186	struct siw_uresp_create_cq uresp = {};
1187	struct siw_ucontext *ctx =
1188	rdma_udata_to_drv_context(udata, struct siw_ucontext,
1189	base_ucontext);
1190	size_t length = size * sizeof(struct siw_cqe) +
1191	sizeof(struct siw_cq_ctrl);
1192
1193	cq->cq_entry =
1194	siw_mmap_entry_insert(uctx: ctx, address: cq->queue,
1195	length, offset: &uresp.cq_key);
1196	if (!cq->cq_entry) {
1197	rv = -ENOMEM;
1198	goto err_out;
1199	}
1200
1201	uresp.cq_id = cq->id;
1202	uresp.num_cqe = size;
1203
1204	if (udata->outlen < sizeof(uresp)) {
1205	rv = -EINVAL;
1206	goto err_out;
1207	}
1208	rv = ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
1209	if (rv)
1210	goto err_out;
1211	}
1212	return 0;
1213
1214	err_out:
1215	siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1216
1217	if (cq->queue) {
1218	struct siw_ucontext *ctx =
1219	rdma_udata_to_drv_context(udata, struct siw_ucontext,
1220	base_ucontext);
1221	if (ctx)
1222	rdma_user_mmap_entry_remove(entry: cq->cq_entry);
1223	vfree(addr: cq->queue);
1224	}
1225	atomic_dec(v: &sdev->num_cq);
1226
1227	return rv;
1228	}
1229
1230	/*
1231	* siw_poll_cq()
1232	*
1233	* Reap CQ entries if available and copy work completion status into
1234	* array of WC's provided by caller. Returns number of reaped CQE's.
1235	*
1236	* @base_cq: Base CQ contained in siw CQ.
1237	* @num_cqe: Maximum number of CQE's to reap.
1238	* @wc: Array of work completions to be filled by siw.
1239	*/
1240	int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1241	{
1242	struct siw_cq *cq = to_siw_cq(base_cq);
1243	int i;
1244
1245	for (i = 0; i < num_cqe; i++) {
1246	if (!siw_reap_cqe(cq, wc))
1247	break;
1248	wc++;
1249	}
1250	return i;
1251	}
1252
1253	/*
1254	* siw_req_notify_cq()
1255	*
1256	* Request notification for new CQE's added to that CQ.
1257	* Defined flags:
1258	* o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1259	* event if a WQE with notification flag set enters the CQ
1260	* o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1261	* event if a WQE enters the CQ.
1262	* o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1263	* number of not reaped CQE's regardless of its notification
1264	* type and current or new CQ notification settings.
1265	*
1266	* @base_cq: Base CQ contained in siw CQ.
1267	* @flags: Requested notification flags.
1268	*/
1269	int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1270	{
1271	struct siw_cq *cq = to_siw_cq(base_cq);
1272
1273	siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1274
1275	if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1276	/*
1277	* Enable CQ event for next solicited completion.
1278	* and make it visible to all associated producers.
1279	*/
1280	smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1281	else
1282	/*
1283	* Enable CQ event for any signalled completion.
1284	* and make it visible to all associated producers.
1285	*/
1286	smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1287
1288	if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1289	return cq->cq_put - cq->cq_get;
1290
1291	return 0;
1292	}
1293
1294	/*
1295	* siw_dereg_mr()
1296	*
1297	* Release Memory Region.
1298	*
1299	* @base_mr: Base MR contained in siw MR.
1300	* @udata: points to user context, unused.
1301	*/
1302	int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1303	{
1304	struct siw_mr *mr = to_siw_mr(base_mr);
1305	struct siw_device *sdev = to_siw_dev(base_dev: base_mr->device);
1306
1307	siw_dbg_mem(mr->mem, "deregister MR\n");
1308
1309	atomic_dec(v: &sdev->num_mr);
1310
1311	siw_mr_drop_mem(mr);
1312	kfree_rcu(mr, rcu);
1313
1314	return 0;
1315	}
1316
1317	/*
1318	* siw_reg_user_mr()
1319	*
1320	* Register Memory Region.
1321	*
1322	* @pd: Protection Domain
1323	* @start: starting address of MR (virtual address)
1324	* @len: len of MR
1325	* @rnic_va: not used by siw
1326	* @rights: MR access rights
1327	* @dmah: dma handle
1328	* @udata: user buffer to communicate STag and Key.
1329	*/
1330	struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1331	u64 rnic_va, int rights, struct ib_dmah *dmah,
1332	struct ib_udata *udata)
1333	{
1334	struct siw_mr *mr = NULL;
1335	struct siw_umem *umem = NULL;
1336	struct siw_ureq_reg_mr ureq;
1337	struct siw_device *sdev = to_siw_dev(base_dev: pd->device);
1338	int rv;
1339
1340	siw_dbg_pd(pd, "start: 0x%p, va: 0x%p, len: %llu\n",
1341	(void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1342	(unsigned long long)len);
1343
1344	if (dmah)
1345	return ERR_PTR(error: -EOPNOTSUPP);
1346
1347	if (atomic_inc_return(v: &sdev->num_mr) > SIW_MAX_MR) {
1348	siw_dbg_pd(pd, "too many mr's\n");
1349	rv = -ENOMEM;
1350	goto err_out;
1351	}
1352	if (!len) {
1353	rv = -EINVAL;
1354	goto err_out;
1355	}
1356	umem = siw_umem_get(base_dave: pd->device, start, len, rights);
1357	if (IS_ERR(ptr: umem)) {
1358	rv = PTR_ERR(ptr: umem);
1359	siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1360	umem = NULL;
1361	goto err_out;
1362	}
1363	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1364	if (!mr) {
1365	rv = -ENOMEM;
1366	goto err_out;
1367	}
1368	rv = siw_mr_add_mem(mr, pd, mem_obj: umem, start, len, rights);
1369	if (rv)
1370	goto err_out;
1371
1372	if (udata) {
1373	struct siw_uresp_reg_mr uresp = {};
1374	struct siw_mem *mem = mr->mem;
1375
1376	if (udata->inlen < sizeof(ureq)) {
1377	rv = -EINVAL;
1378	goto err_out;
1379	}
1380	rv = ib_copy_from_udata(dest: &ureq, udata, len: sizeof(ureq));
1381	if (rv)
1382	goto err_out;
1383
1384	mr->base_mr.lkey |= ureq.stag_key;
1385	mr->base_mr.rkey |= ureq.stag_key;
1386	mem->stag |= ureq.stag_key;
1387	uresp.stag = mem->stag;
1388
1389	if (udata->outlen < sizeof(uresp)) {
1390	rv = -EINVAL;
1391	goto err_out;
1392	}
1393	rv = ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
1394	if (rv)
1395	goto err_out;
1396	}
1397	mr->mem->stag_valid = 1;
1398
1399	return &mr->base_mr;
1400
1401	err_out:
1402	atomic_dec(v: &sdev->num_mr);
1403	if (mr) {
1404	if (mr->mem)
1405	siw_mr_drop_mem(mr);
1406	kfree_rcu(mr, rcu);
1407	} else {
1408	if (umem)
1409	siw_umem_release(umem);
1410	}
1411	return ERR_PTR(error: rv);
1412	}
1413
1414	struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1415	u32 max_sge)
1416	{
1417	struct siw_device *sdev = to_siw_dev(base_dev: pd->device);
1418	struct siw_mr *mr = NULL;
1419	struct siw_pbl *pbl = NULL;
1420	int rv;
1421
1422	if (atomic_inc_return(v: &sdev->num_mr) > SIW_MAX_MR) {
1423	siw_dbg_pd(pd, "too many mr's\n");
1424	rv = -ENOMEM;
1425	goto err_out;
1426	}
1427	if (mr_type != IB_MR_TYPE_MEM_REG) {
1428	siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1429	rv = -EOPNOTSUPP;
1430	goto err_out;
1431	}
1432	if (max_sge > SIW_MAX_SGE_PBL) {
1433	siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1434	rv = -ENOMEM;
1435	goto err_out;
1436	}
1437	pbl = siw_pbl_alloc(num_buf: max_sge);
1438	if (IS_ERR(ptr: pbl)) {
1439	rv = PTR_ERR(ptr: pbl);
1440	siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1441	pbl = NULL;
1442	goto err_out;
1443	}
1444	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1445	if (!mr) {
1446	rv = -ENOMEM;
1447	goto err_out;
1448	}
1449	rv = siw_mr_add_mem(mr, pd, mem_obj: pbl, start: 0, len: max_sge * PAGE_SIZE, rights: 0);
1450	if (rv)
1451	goto err_out;
1452
1453	mr->mem->is_pbl = 1;
1454
1455	siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1456
1457	return &mr->base_mr;
1458
1459	err_out:
1460	atomic_dec(v: &sdev->num_mr);
1461
1462	if (!mr) {
1463	kfree(objp: pbl);
1464	} else {
1465	if (mr->mem)
1466	siw_mr_drop_mem(mr);
1467	kfree_rcu(mr, rcu);
1468	}
1469	siw_dbg_pd(pd, "failed: %d\n", rv);
1470
1471	return ERR_PTR(error: rv);
1472	}
1473
1474	/* Just used to count number of pages being mapped */
1475	static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1476	{
1477	return 0;
1478	}
1479
1480	int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1481	unsigned int *sg_off)
1482	{
1483	struct scatterlist *slp;
1484	struct siw_mr *mr = to_siw_mr(base_mr);
1485	struct siw_mem *mem = mr->mem;
1486	struct siw_pbl *pbl = mem->pbl;
1487	struct siw_pble *pble;
1488	unsigned long pbl_size;
1489	int i, rv;
1490
1491	if (!pbl) {
1492	siw_dbg_mem(mem, "no PBL allocated\n");
1493	return -EINVAL;
1494	}
1495	pble = pbl->pbe;
1496
1497	if (pbl->max_buf < num_sle) {
1498	siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1499	num_sle, pbl->max_buf);
1500	return -ENOMEM;
1501	}
1502	for_each_sg(sl, slp, num_sle, i) {
1503	if (sg_dma_len(slp) == 0) {
1504	siw_dbg_mem(mem, "empty SGE\n");
1505	return -EINVAL;
1506	}
1507	if (i == 0) {
1508	pble->addr = sg_dma_address(slp);
1509	pble->size = sg_dma_len(slp);
1510	pble->pbl_off = 0;
1511	pbl_size = pble->size;
1512	pbl->num_buf = 1;
1513	} else {
1514	/* Merge PBL entries if adjacent */
1515	if (pble->addr + pble->size == sg_dma_address(slp)) {
1516	pble->size += sg_dma_len(slp);
1517	} else {
1518	pble++;
1519	pbl->num_buf++;
1520	pble->addr = sg_dma_address(slp);
1521	pble->size = sg_dma_len(slp);
1522	pble->pbl_off = pbl_size;
1523	}
1524	pbl_size += sg_dma_len(slp);
1525	}
1526	siw_dbg_mem(mem,
1527	"sge[%d], size %u, addr 0x%p, total %lu\n",
1528	i, pble->size, ib_virt_dma_to_ptr(pble->addr),
1529	pbl_size);
1530	}
1531	rv = ib_sg_to_pages(mr: base_mr, sgl: sl, sg_nents: num_sle, sg_offset: sg_off, set_page: siw_set_pbl_page);
1532	if (rv > 0) {
1533	mem->len = base_mr->length;
1534	mem->va = base_mr->iova;
1535	siw_dbg_mem(mem,
1536	"%llu bytes, start 0x%p, %u SLE to %u entries\n",
1537	mem->len, (void *)(uintptr_t)mem->va, num_sle,
1538	pbl->num_buf);
1539	}
1540	return rv;
1541	}
1542
1543	/*
1544	* siw_get_dma_mr()
1545	*
1546	* Create a (empty) DMA memory region, where no umem is attached.
1547	*/
1548	struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1549	{
1550	struct siw_device *sdev = to_siw_dev(base_dev: pd->device);
1551	struct siw_mr *mr = NULL;
1552	int rv;
1553
1554	if (atomic_inc_return(v: &sdev->num_mr) > SIW_MAX_MR) {
1555	siw_dbg_pd(pd, "too many mr's\n");
1556	rv = -ENOMEM;
1557	goto err_out;
1558	}
1559	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1560	if (!mr) {
1561	rv = -ENOMEM;
1562	goto err_out;
1563	}
1564	rv = siw_mr_add_mem(mr, pd, NULL, start: 0, ULONG_MAX, rights);
1565	if (rv)
1566	goto err_out;
1567
1568	mr->mem->stag_valid = 1;
1569
1570	siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1571
1572	return &mr->base_mr;
1573
1574	err_out:
1575	if (rv)
1576	kfree(objp: mr);
1577
1578	atomic_dec(v: &sdev->num_mr);
1579
1580	return ERR_PTR(error: rv);
1581	}
1582
1583	/*
1584	* siw_create_srq()
1585	*
1586	* Create Shared Receive Queue of attributes @init_attrs
1587	* within protection domain given by @pd.
1588	*
1589	* @base_srq: Base SRQ contained in siw SRQ.
1590	* @init_attrs: SRQ init attributes.
1591	* @udata: points to user context
1592	*/
1593	int siw_create_srq(struct ib_srq *base_srq,
1594	struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1595	{
1596	struct siw_srq *srq = to_siw_srq(base_srq);
1597	struct ib_srq_attr *attrs = &init_attrs->attr;
1598	struct siw_device *sdev = to_siw_dev(base_dev: base_srq->device);
1599	struct siw_ucontext *ctx =
1600	rdma_udata_to_drv_context(udata, struct siw_ucontext,
1601	base_ucontext);
1602	int rv;
1603
1604	if (init_attrs->srq_type != IB_SRQT_BASIC)
1605	return -EOPNOTSUPP;
1606
1607	if (atomic_inc_return(v: &sdev->num_srq) > SIW_MAX_SRQ) {
1608	siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1609	rv = -ENOMEM;
1610	goto err_out;
1611	}
1612	if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1613	attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1614	rv = -EINVAL;
1615	goto err_out;
1616	}
1617	srq->max_sge = attrs->max_sge;
1618	srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1619	srq->limit = attrs->srq_limit;
1620	if (srq->limit)
1621	srq->armed = true;
1622
1623	srq->is_kernel_res = !udata;
1624
1625	if (udata)
1626	srq->recvq =
1627	vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1628	else
1629	srq->recvq = vcalloc(srq->num_rqe, sizeof(struct siw_rqe));
1630
1631	if (srq->recvq == NULL) {
1632	rv = -ENOMEM;
1633	goto err_out;
1634	}
1635	if (udata) {
1636	struct siw_uresp_create_srq uresp = {};
1637	size_t length = srq->num_rqe * sizeof(struct siw_rqe);
1638
1639	srq->srq_entry =
1640	siw_mmap_entry_insert(uctx: ctx, address: srq->recvq,
1641	length, offset: &uresp.srq_key);
1642	if (!srq->srq_entry) {
1643	rv = -ENOMEM;
1644	goto err_out;
1645	}
1646
1647	uresp.num_rqe = srq->num_rqe;
1648
1649	if (udata->outlen < sizeof(uresp)) {
1650	rv = -EINVAL;
1651	goto err_out;
1652	}
1653	rv = ib_copy_to_udata(udata, src: &uresp, len: sizeof(uresp));
1654	if (rv)
1655	goto err_out;
1656	}
1657	spin_lock_init(&srq->lock);
1658
1659	siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1660
1661	return 0;
1662
1663	err_out:
1664	if (srq->recvq) {
1665	if (ctx)
1666	rdma_user_mmap_entry_remove(entry: srq->srq_entry);
1667	vfree(addr: srq->recvq);
1668	}
1669	atomic_dec(v: &sdev->num_srq);
1670
1671	return rv;
1672	}
1673
1674	/*
1675	* siw_modify_srq()
1676	*
1677	* Modify SRQ. The caller may resize SRQ and/or set/reset notification
1678	* limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1679	*
1680	* NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1681	* parameter. siw_modify_srq() does not check the attrs->max_sge param.
1682	*/
1683	int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1684	enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1685	{
1686	struct siw_srq *srq = to_siw_srq(base_srq);
1687	unsigned long flags;
1688	int rv = 0;
1689
1690	spin_lock_irqsave(&srq->lock, flags);
1691
1692	if (attr_mask & IB_SRQ_MAX_WR) {
1693	/* resize request not yet supported */
1694	rv = -EOPNOTSUPP;
1695	goto out;
1696	}
1697	if (attr_mask & IB_SRQ_LIMIT) {
1698	if (attrs->srq_limit) {
1699	if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1700	rv = -EINVAL;
1701	goto out;
1702	}
1703	srq->armed = true;
1704	} else {
1705	srq->armed = false;
1706	}
1707	srq->limit = attrs->srq_limit;
1708	}
1709	out:
1710	spin_unlock_irqrestore(lock: &srq->lock, flags);
1711
1712	return rv;
1713	}
1714
1715	/*
1716	* siw_query_srq()
1717	*
1718	* Query SRQ attributes.
1719	*/
1720	int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1721	{
1722	struct siw_srq *srq = to_siw_srq(base_srq);
1723	unsigned long flags;
1724
1725	spin_lock_irqsave(&srq->lock, flags);
1726
1727	attrs->max_wr = srq->num_rqe;
1728	attrs->max_sge = srq->max_sge;
1729	attrs->srq_limit = srq->limit;
1730
1731	spin_unlock_irqrestore(lock: &srq->lock, flags);
1732
1733	return 0;
1734	}
1735
1736	/*
1737	* siw_destroy_srq()
1738	*
1739	* Destroy SRQ.
1740	* It is assumed that the SRQ is not referenced by any
1741	* QP anymore - the code trusts the RDMA core environment to keep track
1742	* of QP references.
1743	*/
1744	int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1745	{
1746	struct siw_srq *srq = to_siw_srq(base_srq);
1747	struct siw_device *sdev = to_siw_dev(base_dev: base_srq->device);
1748	struct siw_ucontext *ctx =
1749	rdma_udata_to_drv_context(udata, struct siw_ucontext,
1750	base_ucontext);
1751
1752	if (ctx)
1753	rdma_user_mmap_entry_remove(entry: srq->srq_entry);
1754	vfree(addr: srq->recvq);
1755	atomic_dec(v: &sdev->num_srq);
1756	return 0;
1757	}
1758
1759	/*
1760	* siw_post_srq_recv()
1761	*
1762	* Post a list of receive queue elements to SRQ.
1763	* NOTE: The function does not check or lock a certain SRQ state
1764	* during the post operation. The code simply trusts the
1765	* RDMA core environment.
1766	*
1767	* @base_srq: Base SRQ contained in siw SRQ
1768	* @wr: List of R-WR's
1769	* @bad_wr: Updated to failing WR if posting fails.
1770	*/
1771	int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1772	const struct ib_recv_wr **bad_wr)
1773	{
1774	struct siw_srq *srq = to_siw_srq(base_srq);
1775	unsigned long flags;
1776	int rv = 0;
1777
1778	if (unlikely(!srq->is_kernel_res)) {
1779	siw_dbg_pd(base_srq->pd,
1780	"[SRQ]: no kernel post_recv for mapped srq\n");
1781	rv = -EINVAL;
1782	goto out;
1783	}
1784	/*
1785	* Serialize potentially multiple producers.
1786	* Also needed to serialize potentially multiple
1787	* consumers.
1788	*/
1789	spin_lock_irqsave(&srq->lock, flags);
1790
1791	while (wr) {
1792	u32 idx = srq->rq_put % srq->num_rqe;
1793	struct siw_rqe *rqe = &srq->recvq[idx];
1794
1795	if (rqe->flags) {
1796	siw_dbg_pd(base_srq->pd, "SRQ full\n");
1797	rv = -ENOMEM;
1798	break;
1799	}
1800	if (unlikely(wr->num_sge > srq->max_sge)) {
1801	siw_dbg_pd(base_srq->pd,
1802	"[SRQ]: too many sge's: %d\n", wr->num_sge);
1803	rv = -EINVAL;
1804	break;
1805	}
1806	rqe->id = wr->wr_id;
1807	rqe->num_sge = wr->num_sge;
1808	siw_copy_sgl(sge: wr->sg_list, siw_sge: rqe->sge, num_sge: wr->num_sge);
1809
1810	/* Make sure S-RQE is completely written before valid */
1811	smp_wmb();
1812
1813	rqe->flags = SIW_WQE_VALID;
1814
1815	srq->rq_put++;
1816	wr = wr->next;
1817	}
1818	spin_unlock_irqrestore(lock: &srq->lock, flags);
1819	out:
1820	if (unlikely(rv < 0)) {
1821	siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1822	*bad_wr = wr;
1823	}
1824	return rv;
1825	}
1826
1827	void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1828	{
1829	struct ib_event event;
1830	struct ib_qp *base_qp = &qp->base_qp;
1831
1832	/*
1833	* Do not report asynchronous errors on QP which gets
1834	* destroyed via verbs interface (siw_destroy_qp())
1835	*/
1836	if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1837	return;
1838
1839	event.event = etype;
1840	event.device = base_qp->device;
1841	event.element.qp = base_qp;
1842
1843	if (base_qp->event_handler) {
1844	siw_dbg_qp(qp, "reporting event %d\n", etype);
1845	base_qp->event_handler(&event, base_qp->qp_context);
1846	}
1847	}
1848
1849	void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1850	{
1851	struct ib_event event;
1852	struct ib_cq *base_cq = &cq->base_cq;
1853
1854	event.event = etype;
1855	event.device = base_cq->device;
1856	event.element.cq = base_cq;
1857
1858	if (base_cq->event_handler) {
1859	siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1860	base_cq->event_handler(&event, base_cq->cq_context);
1861	}
1862	}
1863
1864	void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1865	{
1866	struct ib_event event;
1867	struct ib_srq *base_srq = &srq->base_srq;
1868
1869	event.event = etype;
1870	event.device = base_srq->device;
1871	event.element.srq = base_srq;
1872
1873	if (base_srq->event_handler) {
1874	siw_dbg_pd(srq->base_srq.pd,
1875	"reporting SRQ event %d\n", etype);
1876	base_srq->event_handler(&event, base_srq->srq_context);
1877	}
1878	}
1879
1880	void siw_port_event(struct siw_device *sdev, u32 port, enum ib_event_type etype)
1881	{
1882	struct ib_event event;
1883
1884	event.event = etype;
1885	event.device = &sdev->base_dev;
1886	event.element.port_num = port;
1887
1888	siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1889
1890	ib_dispatch_event(event: &event);
1891	}
1892