1	/*
2	* Copyright (c) 2005 Topspin Communications. All rights reserved.
3	* Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved.
4	* Copyright (c) 2005 PathScale, Inc. All rights reserved.
5	* Copyright (c) 2006 Mellanox Technologies. All rights reserved.
6	*
7	* This software is available to you under a choice of one of two
8	* licenses. You may choose to be licensed under the terms of the GNU
9	* General Public License (GPL) Version 2, available from the file
10	* COPYING in the main directory of this source tree, or the
11	* OpenIB.org BSD license below:
12	*
13	* Redistribution and use in source and binary forms, with or
14	* without modification, are permitted provided that the following
15	* conditions are met:
16	*
17	* - Redistributions of source code must retain the above
18	* copyright notice, this list of conditions and the following
19	* disclaimer.
20	*
21	* - Redistributions in binary form must reproduce the above
22	* copyright notice, this list of conditions and the following
23	* disclaimer in the documentation and/or other materials
24	* provided with the distribution.
25	*
26	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33	* SOFTWARE.
34	*/
35
36	#include <linux/file.h>
37	#include <linux/fs.h>
38	#include <linux/slab.h>
39	#include <linux/sched.h>
40
41	#include <linux/uaccess.h>
42
43	#include <rdma/uverbs_types.h>
44	#include <rdma/uverbs_std_types.h>
45	#include "rdma_core.h"
46
47	#include "uverbs.h"
48	#include "core_priv.h"
49
50	/*
51	* Copy a response to userspace. If the provided 'resp' is larger than the
52	* user buffer it is silently truncated. If the user provided a larger buffer
53	* then the trailing portion is zero filled.
54	*
55	* These semantics are intended to support future extension of the output
56	* structures.
57	*/
58	static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp,
59	size_t resp_len)
60	{
61	int ret;
62
63	if (uverbs_attr_is_valid(attrs_bundle: attrs, idx: UVERBS_ATTR_CORE_OUT))
64	return uverbs_copy_to_struct_or_zero(
65	bundle: attrs, idx: UVERBS_ATTR_CORE_OUT, from: resp, size: resp_len);
66
67	if (copy_to_user(to: attrs->ucore.outbuf, from: resp,
68	min(attrs->ucore.outlen, resp_len)))
69	return -EFAULT;
70
71	if (resp_len < attrs->ucore.outlen) {
72	/*
73	* Zero fill any extra memory that user
74	* space might have provided.
75	*/
76	ret = clear_user(to: attrs->ucore.outbuf + resp_len,
77	n: attrs->ucore.outlen - resp_len);
78	if (ret)
79	return -EFAULT;
80	}
81
82	return 0;
83	}
84
85	/*
86	* Copy a request from userspace. If the provided 'req' is larger than the
87	* user buffer then the user buffer is zero extended into the 'req'. If 'req'
88	* is smaller than the user buffer then the uncopied bytes in the user buffer
89	* must be zero.
90	*/
91	static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req,
92	size_t req_len)
93	{
94	if (copy_from_user(to: req, from: attrs->ucore.inbuf,
95	min(attrs->ucore.inlen, req_len)))
96	return -EFAULT;
97
98	if (attrs->ucore.inlen < req_len) {
99	memset(req + attrs->ucore.inlen, 0,
100	req_len - attrs->ucore.inlen);
101	} else if (attrs->ucore.inlen > req_len) {
102	if (!ib_is_buffer_cleared(p: attrs->ucore.inbuf + req_len,
103	len: attrs->ucore.inlen - req_len))
104	return -EOPNOTSUPP;
105	}
106	return 0;
107	}
108
109	/*
110	* Generate the value for the 'response_length' protocol used by write_ex.
111	* This is the number of bytes the kernel actually wrote. Userspace can use
112	* this to detect what structure members in the response the kernel
113	* understood.
114	*/
115	static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs,
116	size_t resp_len)
117	{
118	return min_t(size_t, attrs->ucore.outlen, resp_len);
119	}
120
121	/*
122	* The iterator version of the request interface is for handlers that need to
123	* step over a flex array at the end of a command header.
124	*/
125	struct uverbs_req_iter {
126	const void __user *cur;
127	const void __user *end;
128	};
129
130	static int uverbs_request_start(struct uverbs_attr_bundle *attrs,
131	struct uverbs_req_iter *iter,
132	void *req,
133	size_t req_len)
134	{
135	if (attrs->ucore.inlen < req_len)
136	return -ENOSPC;
137
138	if (copy_from_user(to: req, from: attrs->ucore.inbuf, n: req_len))
139	return -EFAULT;
140
141	iter->cur = attrs->ucore.inbuf + req_len;
142	iter->end = attrs->ucore.inbuf + attrs->ucore.inlen;
143	return 0;
144	}
145
146	static int uverbs_request_next(struct uverbs_req_iter *iter, void *val,
147	size_t len)
148	{
149	if (iter->cur + len > iter->end)
150	return -ENOSPC;
151
152	if (copy_from_user(to: val, from: iter->cur, n: len))
153	return -EFAULT;
154
155	iter->cur += len;
156	return 0;
157	}
158
159	static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter,
160	size_t len)
161	{
162	const void __user *res = iter->cur;
163
164	if (iter->cur + len > iter->end)
165	return (void __force __user *)ERR_PTR(error: -ENOSPC);
166	iter->cur += len;
167	return res;
168	}
169
170	static int uverbs_request_finish(struct uverbs_req_iter *iter)
171	{
172	if (!ib_is_buffer_cleared(p: iter->cur, len: iter->end - iter->cur))
173	return -EOPNOTSUPP;
174	return 0;
175	}
176
177	/*
178	* When calling a destroy function during an error unwind we need to pass in
179	* the udata that is sanitized of all user arguments. Ie from the driver
180	* perspective it looks like no udata was passed.
181	*/
182	struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)
183	{
184	attrs->driver_udata = (struct ib_udata){};
185	return &attrs->driver_udata;
186	}
187
188	static struct ib_uverbs_completion_event_file *
189	_ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)
190	{
191	struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,
192	fd, attrs);
193
194	if (IS_ERR(ptr: uobj))
195	return (void *)uobj;
196
197	uverbs_uobject_get(uobject: uobj);
198	uobj_put_read(uobj);
199
200	return container_of(uobj, struct ib_uverbs_completion_event_file,
201	uobj);
202	}
203	#define ib_uverbs_lookup_comp_file(_fd, _ufile) \
204	_ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile)
205
206	int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs)
207	{
208	struct ib_uverbs_file *ufile = attrs->ufile;
209	struct ib_ucontext *ucontext;
210	struct ib_device *ib_dev;
211
212	ib_dev = srcu_dereference(ufile->device->ib_dev,
213	&ufile->device->disassociate_srcu);
214	if (!ib_dev)
215	return -EIO;
216
217	ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext);
218	if (!ucontext)
219	return -ENOMEM;
220
221	ucontext->device = ib_dev;
222	ucontext->ufile = ufile;
223	xa_init_flags(xa: &ucontext->mmap_xa, XA_FLAGS_ALLOC);
224
225	rdma_restrack_new(res: &ucontext->res, type: RDMA_RESTRACK_CTX);
226	rdma_restrack_set_name(res: &ucontext->res, NULL);
227	attrs->context = ucontext;
228	return 0;
229	}
230
231	int ib_init_ucontext(struct uverbs_attr_bundle *attrs)
232	{
233	struct ib_ucontext *ucontext = attrs->context;
234	struct ib_uverbs_file *file = attrs->ufile;
235	int ret;
236
237	if (!down_read_trylock(sem: &file->hw_destroy_rwsem))
238	return -EIO;
239	mutex_lock(&file->ucontext_lock);
240	if (file->ucontext) {
241	ret = -EINVAL;
242	goto err;
243	}
244
245	ret = ib_rdmacg_try_charge(cg_obj: &ucontext->cg_obj, device: ucontext->device,
246	resource_index: RDMACG_RESOURCE_HCA_HANDLE);
247	if (ret)
248	goto err;
249
250	ret = ucontext->device->ops.alloc_ucontext(ucontext,
251	&attrs->driver_udata);
252	if (ret)
253	goto err_uncharge;
254
255	rdma_restrack_add(res: &ucontext->res);
256
257	/*
258	* Make sure that ib_uverbs_get_ucontext() sees the pointer update
259	* only after all writes to setup the ucontext have completed
260	*/
261	smp_store_release(&file->ucontext, ucontext);
262
263	mutex_unlock(lock: &file->ucontext_lock);
264	up_read(sem: &file->hw_destroy_rwsem);
265	return 0;
266
267	err_uncharge:
268	ib_rdmacg_uncharge(cg_obj: &ucontext->cg_obj, device: ucontext->device,
269	resource_index: RDMACG_RESOURCE_HCA_HANDLE);
270	err:
271	mutex_unlock(lock: &file->ucontext_lock);
272	up_read(sem: &file->hw_destroy_rwsem);
273	return ret;
274	}
275
276	static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
277	{
278	struct ib_uverbs_get_context_resp resp;
279	struct ib_uverbs_get_context cmd;
280	struct ib_device *ib_dev;
281	struct ib_uobject *uobj;
282	int ret;
283
284	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
285	if (ret)
286	return ret;
287
288	ret = ib_alloc_ucontext(attrs);
289	if (ret)
290	return ret;
291
292	uobj = uobj_alloc(UVERBS_OBJECT_ASYNC_EVENT, attrs, &ib_dev);
293	if (IS_ERR(ptr: uobj)) {
294	ret = PTR_ERR(ptr: uobj);
295	goto err_ucontext;
296	}
297
298	resp = (struct ib_uverbs_get_context_resp){
299	.num_comp_vectors = attrs->ufile->device->num_comp_vectors,
300	.async_fd = uobj->id,
301	};
302	ret = uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
303	if (ret)
304	goto err_uobj;
305
306	ret = ib_init_ucontext(attrs);
307	if (ret)
308	goto err_uobj;
309
310	ib_uverbs_init_async_event_file(
311	container_of(uobj, struct ib_uverbs_async_event_file, uobj));
312	rdma_alloc_commit_uobject(uobj, attrs);
313	return 0;
314
315	err_uobj:
316	rdma_alloc_abort_uobject(uobj, attrs, hw_obj_valid: false);
317	err_ucontext:
318	rdma_restrack_put(res: &attrs->context->res);
319	kfree(objp: attrs->context);
320	attrs->context = NULL;
321	return ret;
322	}
323
324	static void copy_query_dev_fields(struct ib_ucontext *ucontext,
325	struct ib_uverbs_query_device_resp *resp,
326	struct ib_device_attr *attr)
327	{
328	struct ib_device *ib_dev = ucontext->device;
329
330	resp->fw_ver = attr->fw_ver;
331	resp->node_guid = ib_dev->node_guid;
332	resp->sys_image_guid = attr->sys_image_guid;
333	resp->max_mr_size = attr->max_mr_size;
334	resp->page_size_cap = attr->page_size_cap;
335	resp->vendor_id = attr->vendor_id;
336	resp->vendor_part_id = attr->vendor_part_id;
337	resp->hw_ver = attr->hw_ver;
338	resp->max_qp = attr->max_qp;
339	resp->max_qp_wr = attr->max_qp_wr;
340	resp->device_cap_flags = lower_32_bits(attr->device_cap_flags);
341	resp->max_sge = min(attr->max_send_sge, attr->max_recv_sge);
342	resp->max_sge_rd = attr->max_sge_rd;
343	resp->max_cq = attr->max_cq;
344	resp->max_cqe = attr->max_cqe;
345	resp->max_mr = attr->max_mr;
346	resp->max_pd = attr->max_pd;
347	resp->max_qp_rd_atom = attr->max_qp_rd_atom;
348	resp->max_ee_rd_atom = attr->max_ee_rd_atom;
349	resp->max_res_rd_atom = attr->max_res_rd_atom;
350	resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom;
351	resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom;
352	resp->atomic_cap = attr->atomic_cap;
353	resp->max_ee = attr->max_ee;
354	resp->max_rdd = attr->max_rdd;
355	resp->max_mw = attr->max_mw;
356	resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp;
357	resp->max_raw_ethy_qp = attr->max_raw_ethy_qp;
358	resp->max_mcast_grp = attr->max_mcast_grp;
359	resp->max_mcast_qp_attach = attr->max_mcast_qp_attach;
360	resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
361	resp->max_ah = attr->max_ah;
362	resp->max_srq = attr->max_srq;
363	resp->max_srq_wr = attr->max_srq_wr;
364	resp->max_srq_sge = attr->max_srq_sge;
365	resp->max_pkeys = attr->max_pkeys;
366	resp->local_ca_ack_delay = attr->local_ca_ack_delay;
367	resp->phys_port_cnt = min_t(u32, ib_dev->phys_port_cnt, U8_MAX);
368	}
369
370	static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs)
371	{
372	struct ib_uverbs_query_device cmd;
373	struct ib_uverbs_query_device_resp resp;
374	struct ib_ucontext *ucontext;
375	int ret;
376
377	ucontext = ib_uverbs_get_ucontext(attrs);
378	if (IS_ERR(ptr: ucontext))
379	return PTR_ERR(ptr: ucontext);
380
381	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
382	if (ret)
383	return ret;
384
385	memset(&resp, 0, sizeof resp);
386	copy_query_dev_fields(ucontext, resp: &resp, attr: &ucontext->device->attrs);
387
388	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
389	}
390
391	static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs)
392	{
393	struct ib_uverbs_query_port cmd;
394	struct ib_uverbs_query_port_resp resp;
395	struct ib_port_attr attr;
396	int ret;
397	struct ib_ucontext *ucontext;
398	struct ib_device *ib_dev;
399
400	ucontext = ib_uverbs_get_ucontext(attrs);
401	if (IS_ERR(ptr: ucontext))
402	return PTR_ERR(ptr: ucontext);
403	ib_dev = ucontext->device;
404
405	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
406	if (ret)
407	return ret;
408
409	ret = ib_query_port(device: ib_dev, port_num: cmd.port_num, port_attr: &attr);
410	if (ret)
411	return ret;
412
413	memset(&resp, 0, sizeof resp);
414	copy_port_attr_to_resp(attr: &attr, resp: &resp, ib_dev, port_num: cmd.port_num);
415
416	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
417	}
418
419	static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
420	{
421	struct ib_uverbs_alloc_pd_resp resp = {};
422	struct ib_uverbs_alloc_pd cmd;
423	struct ib_uobject *uobj;
424	struct ib_pd *pd;
425	int ret;
426	struct ib_device *ib_dev;
427
428	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
429	if (ret)
430	return ret;
431
432	uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev);
433	if (IS_ERR(ptr: uobj))
434	return PTR_ERR(ptr: uobj);
435
436	pd = rdma_zalloc_drv_obj(ib_dev, ib_pd);
437	if (!pd) {
438	ret = -ENOMEM;
439	goto err;
440	}
441
442	pd->device = ib_dev;
443	pd->uobject = uobj;
444	atomic_set(v: &pd->usecnt, i: 0);
445
446	rdma_restrack_new(res: &pd->res, type: RDMA_RESTRACK_PD);
447	rdma_restrack_set_name(res: &pd->res, NULL);
448
449	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
450	if (ret)
451	goto err_alloc;
452	rdma_restrack_add(res: &pd->res);
453
454	uobj->object = pd;
455	uobj_finalize_uobj_create(uobj, attrs);
456
457	resp.pd_handle = uobj->id;
458	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
459
460	err_alloc:
461	rdma_restrack_put(res: &pd->res);
462	kfree(objp: pd);
463	err:
464	uobj_alloc_abort(uobj, attrs);
465	return ret;
466	}
467
468	static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs)
469	{
470	struct ib_uverbs_dealloc_pd cmd;
471	int ret;
472
473	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
474	if (ret)
475	return ret;
476
477	return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
478	}
479
480	struct xrcd_table_entry {
481	struct rb_node node;
482	struct ib_xrcd *xrcd;
483	struct inode *inode;
484	};
485
486	static int xrcd_table_insert(struct ib_uverbs_device *dev,
487	struct inode *inode,
488	struct ib_xrcd *xrcd)
489	{
490	struct xrcd_table_entry *entry, *scan;
491	struct rb_node **p = &dev->xrcd_tree.rb_node;
492	struct rb_node *parent = NULL;
493
494	entry = kmalloc(size: sizeof *entry, GFP_KERNEL);
495	if (!entry)
496	return -ENOMEM;
497
498	entry->xrcd = xrcd;
499	entry->inode = inode;
500
501	while (*p) {
502	parent = *p;
503	scan = rb_entry(parent, struct xrcd_table_entry, node);
504
505	if (inode < scan->inode) {
506	p = &(*p)->rb_left;
507	} else if (inode > scan->inode) {
508	p = &(*p)->rb_right;
509	} else {
510	kfree(objp: entry);
511	return -EEXIST;
512	}
513	}
514
515	rb_link_node(node: &entry->node, parent, rb_link: p);
516	rb_insert_color(&entry->node, &dev->xrcd_tree);
517	igrab(inode);
518	return 0;
519	}
520
521	static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
522	struct inode *inode)
523	{
524	struct xrcd_table_entry *entry;
525	struct rb_node *p = dev->xrcd_tree.rb_node;
526
527	while (p) {
528	entry = rb_entry(p, struct xrcd_table_entry, node);
529
530	if (inode < entry->inode)
531	p = p->rb_left;
532	else if (inode > entry->inode)
533	p = p->rb_right;
534	else
535	return entry;
536	}
537
538	return NULL;
539	}
540
541	static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
542	{
543	struct xrcd_table_entry *entry;
544
545	entry = xrcd_table_search(dev, inode);
546	if (!entry)
547	return NULL;
548
549	return entry->xrcd;
550	}
551
552	static void xrcd_table_delete(struct ib_uverbs_device *dev,
553	struct inode *inode)
554	{
555	struct xrcd_table_entry *entry;
556
557	entry = xrcd_table_search(dev, inode);
558	if (entry) {
559	iput(inode);
560	rb_erase(&entry->node, &dev->xrcd_tree);
561	kfree(objp: entry);
562	}
563	}
564
565	static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
566	{
567	struct ib_uverbs_device *ibudev = attrs->ufile->device;
568	struct ib_uverbs_open_xrcd_resp resp = {};
569	struct ib_uverbs_open_xrcd cmd;
570	struct ib_uxrcd_object *obj;
571	struct ib_xrcd *xrcd = NULL;
572	struct inode *inode = NULL;
573	int new_xrcd = 0;
574	struct ib_device *ib_dev;
575	struct fd f = {};
576	int ret;
577
578	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
579	if (ret)
580	return ret;
581
582	mutex_lock(&ibudev->xrcd_tree_mutex);
583
584	if (cmd.fd != -1) {
585	/* search for file descriptor */
586	f = fdget(fd: cmd.fd);
587	if (!f.file) {
588	ret = -EBADF;
589	goto err_tree_mutex_unlock;
590	}
591
592	inode = file_inode(f: f.file);
593	xrcd = find_xrcd(dev: ibudev, inode);
594	if (!xrcd && !(cmd.oflags & O_CREAT)) {
595	/* no file descriptor. Need CREATE flag */
596	ret = -EAGAIN;
597	goto err_tree_mutex_unlock;
598	}
599
600	if (xrcd && cmd.oflags & O_EXCL) {
601	ret = -EINVAL;
602	goto err_tree_mutex_unlock;
603	}
604	}
605
606	obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs,
607	&ib_dev);
608	if (IS_ERR(ptr: obj)) {
609	ret = PTR_ERR(ptr: obj);
610	goto err_tree_mutex_unlock;
611	}
612
613	if (!xrcd) {
614	xrcd = ib_alloc_xrcd_user(device: ib_dev, inode, udata: &attrs->driver_udata);
615	if (IS_ERR(ptr: xrcd)) {
616	ret = PTR_ERR(ptr: xrcd);
617	goto err;
618	}
619	new_xrcd = 1;
620	}
621
622	atomic_set(v: &obj->refcnt, i: 0);
623	obj->uobject.object = xrcd;
624
625	if (inode) {
626	if (new_xrcd) {
627	/* create new inode/xrcd table entry */
628	ret = xrcd_table_insert(dev: ibudev, inode, xrcd);
629	if (ret)
630	goto err_dealloc_xrcd;
631	}
632	atomic_inc(v: &xrcd->usecnt);
633	}
634
635	if (f.file)
636	fdput(fd: f);
637
638	mutex_unlock(lock: &ibudev->xrcd_tree_mutex);
639	uobj_finalize_uobj_create(uobj: &obj->uobject, attrs);
640
641	resp.xrcd_handle = obj->uobject.id;
642	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
643
644	err_dealloc_xrcd:
645	ib_dealloc_xrcd_user(xrcd, udata: uverbs_get_cleared_udata(attrs));
646
647	err:
648	uobj_alloc_abort(uobj: &obj->uobject, attrs);
649
650	err_tree_mutex_unlock:
651	if (f.file)
652	fdput(fd: f);
653
654	mutex_unlock(lock: &ibudev->xrcd_tree_mutex);
655
656	return ret;
657	}
658
659	static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)
660	{
661	struct ib_uverbs_close_xrcd cmd;
662	int ret;
663
664	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
665	if (ret)
666	return ret;
667
668	return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);
669	}
670
671	int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,
672	enum rdma_remove_reason why,
673	struct uverbs_attr_bundle *attrs)
674	{
675	struct inode *inode;
676	int ret;
677	struct ib_uverbs_device *dev = attrs->ufile->device;
678
679	inode = xrcd->inode;
680	if (inode && !atomic_dec_and_test(v: &xrcd->usecnt))
681	return 0;
682
683	ret = ib_dealloc_xrcd_user(xrcd, udata: &attrs->driver_udata);
684	if (ret) {
685	atomic_inc(v: &xrcd->usecnt);
686	return ret;
687	}
688
689	if (inode)
690	xrcd_table_delete(dev, inode);
691
692	return 0;
693	}
694
695	static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
696	{
697	struct ib_uverbs_reg_mr_resp resp = {};
698	struct ib_uverbs_reg_mr cmd;
699	struct ib_uobject *uobj;
700	struct ib_pd *pd;
701	struct ib_mr *mr;
702	int ret;
703	struct ib_device *ib_dev;
704
705	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
706	if (ret)
707	return ret;
708
709	if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
710	return -EINVAL;
711
712	uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
713	if (IS_ERR(ptr: uobj))
714	return PTR_ERR(ptr: uobj);
715
716	ret = ib_check_mr_access(ib_dev, flags: cmd.access_flags);
717	if (ret)
718	goto err_free;
719
720	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
721	if (!pd) {
722	ret = -EINVAL;
723	goto err_free;
724	}
725
726	mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
727	cmd.access_flags,
728	&attrs->driver_udata);
729	if (IS_ERR(ptr: mr)) {
730	ret = PTR_ERR(ptr: mr);
731	goto err_put;
732	}
733
734	mr->device = pd->device;
735	mr->pd = pd;
736	mr->type = IB_MR_TYPE_USER;
737	mr->dm = NULL;
738	mr->sig_attrs = NULL;
739	mr->uobject = uobj;
740	atomic_inc(v: &pd->usecnt);
741	mr->iova = cmd.hca_va;
742	mr->length = cmd.length;
743
744	rdma_restrack_new(res: &mr->res, type: RDMA_RESTRACK_MR);
745	rdma_restrack_set_name(res: &mr->res, NULL);
746	rdma_restrack_add(res: &mr->res);
747
748	uobj->object = mr;
749	uobj_put_obj_read(pd);
750	uobj_finalize_uobj_create(uobj, attrs);
751
752	resp.lkey = mr->lkey;
753	resp.rkey = mr->rkey;
754	resp.mr_handle = uobj->id;
755	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
756
757	err_put:
758	uobj_put_obj_read(pd);
759	err_free:
760	uobj_alloc_abort(uobj, attrs);
761	return ret;
762	}
763
764	static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs)
765	{
766	struct ib_uverbs_rereg_mr cmd;
767	struct ib_uverbs_rereg_mr_resp resp;
768	struct ib_mr *mr;
769	int ret;
770	struct ib_uobject *uobj;
771	struct ib_uobject *new_uobj;
772	struct ib_device *ib_dev;
773	struct ib_pd *orig_pd;
774	struct ib_pd *new_pd;
775	struct ib_mr *new_mr;
776
777	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
778	if (ret)
779	return ret;
780
781	if (!cmd.flags)
782	return -EINVAL;
783
784	if (cmd.flags & ~IB_MR_REREG_SUPPORTED)
785	return -EOPNOTSUPP;
786
787	if ((cmd.flags & IB_MR_REREG_TRANS) &&
788	(cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
789	return -EINVAL;
790
791	uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
792	if (IS_ERR(ptr: uobj))
793	return PTR_ERR(ptr: uobj);
794
795	mr = uobj->object;
796
797	if (mr->dm) {
798	ret = -EINVAL;
799	goto put_uobjs;
800	}
801
802	if (cmd.flags & IB_MR_REREG_ACCESS) {
803	ret = ib_check_mr_access(ib_dev: mr->device, flags: cmd.access_flags);
804	if (ret)
805	goto put_uobjs;
806	}
807
808	orig_pd = mr->pd;
809	if (cmd.flags & IB_MR_REREG_PD) {
810	new_pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle,
811	attrs);
812	if (!new_pd) {
813	ret = -EINVAL;
814	goto put_uobjs;
815	}
816	} else {
817	new_pd = mr->pd;
818	}
819
820	/*
821	* The driver might create a new HW object as part of the rereg, we need
822	* to have a uobject ready to hold it.
823	*/
824	new_uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev);
825	if (IS_ERR(ptr: new_uobj)) {
826	ret = PTR_ERR(ptr: new_uobj);
827	goto put_uobj_pd;
828	}
829
830	new_mr = ib_dev->ops.rereg_user_mr(mr, cmd.flags, cmd.start, cmd.length,
831	cmd.hca_va, cmd.access_flags, new_pd,
832	&attrs->driver_udata);
833	if (IS_ERR(ptr: new_mr)) {
834	ret = PTR_ERR(ptr: new_mr);
835	goto put_new_uobj;
836	}
837	if (new_mr) {
838	new_mr->device = new_pd->device;
839	new_mr->pd = new_pd;
840	new_mr->type = IB_MR_TYPE_USER;
841	new_mr->uobject = uobj;
842	atomic_inc(v: &new_pd->usecnt);
843	new_uobj->object = new_mr;
844
845	rdma_restrack_new(res: &new_mr->res, type: RDMA_RESTRACK_MR);
846	rdma_restrack_set_name(res: &new_mr->res, NULL);
847	rdma_restrack_add(res: &new_mr->res);
848
849	/*
850	* The new uobj for the new HW object is put into the same spot
851	* in the IDR and the old uobj & HW object is deleted.
852	*/
853	rdma_assign_uobject(to_uobj: uobj, new_uobj, attrs);
854	rdma_alloc_commit_uobject(uobj: new_uobj, attrs);
855	uobj_put_destroy(uobj);
856	new_uobj = NULL;
857	uobj = NULL;
858	mr = new_mr;
859	} else {
860	if (cmd.flags & IB_MR_REREG_PD) {
861	atomic_dec(v: &orig_pd->usecnt);
862	mr->pd = new_pd;
863	atomic_inc(v: &new_pd->usecnt);
864	}
865	if (cmd.flags & IB_MR_REREG_TRANS) {
866	mr->iova = cmd.hca_va;
867	mr->length = cmd.length;
868	}
869	}
870
871	memset(&resp, 0, sizeof(resp));
872	resp.lkey = mr->lkey;
873	resp.rkey = mr->rkey;
874
875	ret = uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
876
877	put_new_uobj:
878	if (new_uobj)
879	uobj_alloc_abort(uobj: new_uobj, attrs);
880	put_uobj_pd:
881	if (cmd.flags & IB_MR_REREG_PD)
882	uobj_put_obj_read(new_pd);
883
884	put_uobjs:
885	if (uobj)
886	uobj_put_write(uobj);
887
888	return ret;
889	}
890
891	static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs)
892	{
893	struct ib_uverbs_dereg_mr cmd;
894	int ret;
895
896	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
897	if (ret)
898	return ret;
899
900	return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs);
901	}
902
903	static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)
904	{
905	struct ib_uverbs_alloc_mw cmd;
906	struct ib_uverbs_alloc_mw_resp resp = {};
907	struct ib_uobject *uobj;
908	struct ib_pd *pd;
909	struct ib_mw *mw;
910	int ret;
911	struct ib_device *ib_dev;
912
913	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
914	if (ret)
915	return ret;
916
917	uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev);
918	if (IS_ERR(ptr: uobj))
919	return PTR_ERR(ptr: uobj);
920
921	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
922	if (!pd) {
923	ret = -EINVAL;
924	goto err_free;
925	}
926
927	if (cmd.mw_type != IB_MW_TYPE_1 && cmd.mw_type != IB_MW_TYPE_2) {
928	ret = -EINVAL;
929	goto err_put;
930	}
931
932	mw = rdma_zalloc_drv_obj(ib_dev, ib_mw);
933	if (!mw) {
934	ret = -ENOMEM;
935	goto err_put;
936	}
937
938	mw->device = ib_dev;
939	mw->pd = pd;
940	mw->uobject = uobj;
941	mw->type = cmd.mw_type;
942
943	ret = pd->device->ops.alloc_mw(mw, &attrs->driver_udata);
944	if (ret)
945	goto err_alloc;
946
947	atomic_inc(v: &pd->usecnt);
948
949	uobj->object = mw;
950	uobj_put_obj_read(pd);
951	uobj_finalize_uobj_create(uobj, attrs);
952
953	resp.rkey = mw->rkey;
954	resp.mw_handle = uobj->id;
955	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
956
957	err_alloc:
958	kfree(objp: mw);
959	err_put:
960	uobj_put_obj_read(pd);
961	err_free:
962	uobj_alloc_abort(uobj, attrs);
963	return ret;
964	}
965
966	static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs)
967	{
968	struct ib_uverbs_dealloc_mw cmd;
969	int ret;
970
971	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
972	if (ret)
973	return ret;
974
975	return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs);
976	}
977
978	static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)
979	{
980	struct ib_uverbs_create_comp_channel cmd;
981	struct ib_uverbs_create_comp_channel_resp resp;
982	struct ib_uobject *uobj;
983	struct ib_uverbs_completion_event_file *ev_file;
984	struct ib_device *ib_dev;
985	int ret;
986
987	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
988	if (ret)
989	return ret;
990
991	uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev);
992	if (IS_ERR(ptr: uobj))
993	return PTR_ERR(ptr: uobj);
994
995	ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
996	uobj);
997	ib_uverbs_init_event_queue(ev_queue: &ev_file->ev_queue);
998	uobj_finalize_uobj_create(uobj, attrs);
999
1000	resp.fd = uobj->id;
1001	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
1002	}
1003
1004	static int create_cq(struct uverbs_attr_bundle *attrs,
1005	struct ib_uverbs_ex_create_cq *cmd)
1006	{
1007	struct ib_ucq_object *obj;
1008	struct ib_uverbs_completion_event_file *ev_file = NULL;
1009	struct ib_cq *cq;
1010	int ret;
1011	struct ib_uverbs_ex_create_cq_resp resp = {};
1012	struct ib_cq_init_attr attr = {};
1013	struct ib_device *ib_dev;
1014
1015	if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors)
1016	return -EINVAL;
1017
1018	obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs,
1019	&ib_dev);
1020	if (IS_ERR(ptr: obj))
1021	return PTR_ERR(ptr: obj);
1022
1023	if (cmd->comp_channel >= 0) {
1024	ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs);
1025	if (IS_ERR(ptr: ev_file)) {
1026	ret = PTR_ERR(ptr: ev_file);
1027	goto err;
1028	}
1029	}
1030
1031	obj->uevent.uobject.user_handle = cmd->user_handle;
1032	INIT_LIST_HEAD(list: &obj->comp_list);
1033	INIT_LIST_HEAD(list: &obj->uevent.event_list);
1034
1035	attr.cqe = cmd->cqe;
1036	attr.comp_vector = cmd->comp_vector;
1037	attr.flags = cmd->flags;
1038
1039	cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);
1040	if (!cq) {
1041	ret = -ENOMEM;
1042	goto err_file;
1043	}
1044	cq->device = ib_dev;
1045	cq->uobject = obj;
1046	cq->comp_handler = ib_uverbs_comp_handler;
1047	cq->event_handler = ib_uverbs_cq_event_handler;
1048	cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
1049	atomic_set(v: &cq->usecnt, i: 0);
1050
1051	rdma_restrack_new(res: &cq->res, type: RDMA_RESTRACK_CQ);
1052	rdma_restrack_set_name(res: &cq->res, NULL);
1053
1054	ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata);
1055	if (ret)
1056	goto err_free;
1057	rdma_restrack_add(res: &cq->res);
1058
1059	obj->uevent.uobject.object = cq;
1060	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
1061	if (obj->uevent.event_file)
1062	uverbs_uobject_get(uobject: &obj->uevent.event_file->uobj);
1063	uobj_finalize_uobj_create(uobj: &obj->uevent.uobject, attrs);
1064
1065	resp.base.cq_handle = obj->uevent.uobject.id;
1066	resp.base.cqe = cq->cqe;
1067	resp.response_length = uverbs_response_length(attrs, resp_len: sizeof(resp));
1068	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
1069
1070	err_free:
1071	rdma_restrack_put(res: &cq->res);
1072	kfree(objp: cq);
1073	err_file:
1074	if (ev_file)
1075	ib_uverbs_release_ucq(ev_file, uobj: obj);
1076	err:
1077	uobj_alloc_abort(uobj: &obj->uevent.uobject, attrs);
1078	return ret;
1079	}
1080
1081	static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs)
1082	{
1083	struct ib_uverbs_create_cq cmd;
1084	struct ib_uverbs_ex_create_cq cmd_ex;
1085	int ret;
1086
1087	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1088	if (ret)
1089	return ret;
1090
1091	memset(&cmd_ex, 0, sizeof(cmd_ex));
1092	cmd_ex.user_handle = cmd.user_handle;
1093	cmd_ex.cqe = cmd.cqe;
1094	cmd_ex.comp_vector = cmd.comp_vector;
1095	cmd_ex.comp_channel = cmd.comp_channel;
1096
1097	return create_cq(attrs, cmd: &cmd_ex);
1098	}
1099
1100	static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs)
1101	{
1102	struct ib_uverbs_ex_create_cq cmd;
1103	int ret;
1104
1105	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1106	if (ret)
1107	return ret;
1108
1109	if (cmd.comp_mask)
1110	return -EINVAL;
1111
1112	if (cmd.reserved)
1113	return -EINVAL;
1114
1115	return create_cq(attrs, cmd: &cmd);
1116	}
1117
1118	static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs)
1119	{
1120	struct ib_uverbs_resize_cq cmd;
1121	struct ib_uverbs_resize_cq_resp resp = {};
1122	struct ib_cq *cq;
1123	int ret;
1124
1125	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1126	if (ret)
1127	return ret;
1128
1129	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1130	if (!cq)
1131	return -EINVAL;
1132
1133	ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata);
1134	if (ret)
1135	goto out;
1136
1137	resp.cqe = cq->cqe;
1138
1139	ret = uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
1140	out:
1141	rdma_lookup_put_uobject(uobj: &cq->uobject->uevent.uobject,
1142	mode: UVERBS_LOOKUP_READ);
1143
1144	return ret;
1145	}
1146
1147	static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
1148	struct ib_wc *wc)
1149	{
1150	struct ib_uverbs_wc tmp;
1151
1152	tmp.wr_id = wc->wr_id;
1153	tmp.status = wc->status;
1154	tmp.opcode = wc->opcode;
1155	tmp.vendor_err = wc->vendor_err;
1156	tmp.byte_len = wc->byte_len;
1157	tmp.ex.imm_data = wc->ex.imm_data;
1158	tmp.qp_num = wc->qp->qp_num;
1159	tmp.src_qp = wc->src_qp;
1160	tmp.wc_flags = wc->wc_flags;
1161	tmp.pkey_index = wc->pkey_index;
1162	if (rdma_cap_opa_ah(device: ib_dev, port_num: wc->port_num))
1163	tmp.slid = OPA_TO_IB_UCAST_LID(wc->slid);
1164	else
1165	tmp.slid = ib_lid_cpu16(lid: wc->slid);
1166	tmp.sl = wc->sl;
1167	tmp.dlid_path_bits = wc->dlid_path_bits;
1168	tmp.port_num = wc->port_num;
1169	tmp.reserved = 0;
1170
1171	if (copy_to_user(to: dest, from: &tmp, n: sizeof tmp))
1172	return -EFAULT;
1173
1174	return 0;
1175	}
1176
1177	static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs)
1178	{
1179	struct ib_uverbs_poll_cq cmd;
1180	struct ib_uverbs_poll_cq_resp resp;
1181	u8 __user *header_ptr;
1182	u8 __user *data_ptr;
1183	struct ib_cq *cq;
1184	struct ib_wc wc;
1185	int ret;
1186
1187	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1188	if (ret)
1189	return ret;
1190
1191	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1192	if (!cq)
1193	return -EINVAL;
1194
1195	/* we copy a struct ib_uverbs_poll_cq_resp to user space */
1196	header_ptr = attrs->ucore.outbuf;
1197	data_ptr = header_ptr + sizeof resp;
1198
1199	memset(&resp, 0, sizeof resp);
1200	while (resp.count < cmd.ne) {
1201	ret = ib_poll_cq(cq, num_entries: 1, wc: &wc);
1202	if (ret < 0)
1203	goto out_put;
1204	if (!ret)
1205	break;
1206
1207	ret = copy_wc_to_user(ib_dev: cq->device, dest: data_ptr, wc: &wc);
1208	if (ret)
1209	goto out_put;
1210
1211	data_ptr += sizeof(struct ib_uverbs_wc);
1212	++resp.count;
1213	}
1214
1215	if (copy_to_user(to: header_ptr, from: &resp, n: sizeof resp)) {
1216	ret = -EFAULT;
1217	goto out_put;
1218	}
1219	ret = 0;
1220
1221	if (uverbs_attr_is_valid(attrs_bundle: attrs, idx: UVERBS_ATTR_CORE_OUT))
1222	ret = uverbs_output_written(bundle: attrs, idx: UVERBS_ATTR_CORE_OUT);
1223
1224	out_put:
1225	rdma_lookup_put_uobject(uobj: &cq->uobject->uevent.uobject,
1226	mode: UVERBS_LOOKUP_READ);
1227	return ret;
1228	}
1229
1230	static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs)
1231	{
1232	struct ib_uverbs_req_notify_cq cmd;
1233	struct ib_cq *cq;
1234	int ret;
1235
1236	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1237	if (ret)
1238	return ret;
1239
1240	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1241	if (!cq)
1242	return -EINVAL;
1243
1244	ib_req_notify_cq(cq, flags: cmd.solicited_only ?
1245	IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1246
1247	rdma_lookup_put_uobject(uobj: &cq->uobject->uevent.uobject,
1248	mode: UVERBS_LOOKUP_READ);
1249	return 0;
1250	}
1251
1252	static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs)
1253	{
1254	struct ib_uverbs_destroy_cq cmd;
1255	struct ib_uverbs_destroy_cq_resp resp;
1256	struct ib_uobject *uobj;
1257	struct ib_ucq_object *obj;
1258	int ret;
1259
1260	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1261	if (ret)
1262	return ret;
1263
1264	uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
1265	if (IS_ERR(ptr: uobj))
1266	return PTR_ERR(ptr: uobj);
1267
1268	obj = container_of(uobj, struct ib_ucq_object, uevent.uobject);
1269	memset(&resp, 0, sizeof(resp));
1270	resp.comp_events_reported = obj->comp_events_reported;
1271	resp.async_events_reported = obj->uevent.events_reported;
1272
1273	uobj_put_destroy(uobj);
1274
1275	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
1276	}
1277
1278	static int create_qp(struct uverbs_attr_bundle *attrs,
1279	struct ib_uverbs_ex_create_qp *cmd)
1280	{
1281	struct ib_uqp_object *obj;
1282	struct ib_device *device;
1283	struct ib_pd *pd = NULL;
1284	struct ib_xrcd *xrcd = NULL;
1285	struct ib_uobject *xrcd_uobj = ERR_PTR(error: -ENOENT);
1286	struct ib_cq *scq = NULL, *rcq = NULL;
1287	struct ib_srq *srq = NULL;
1288	struct ib_qp *qp;
1289	struct ib_qp_init_attr attr = {};
1290	struct ib_uverbs_ex_create_qp_resp resp = {};
1291	int ret;
1292	struct ib_rwq_ind_table *ind_tbl = NULL;
1293	bool has_sq = true;
1294	struct ib_device *ib_dev;
1295
1296	switch (cmd->qp_type) {
1297	case IB_QPT_RAW_PACKET:
1298	if (!capable(CAP_NET_RAW))
1299	return -EPERM;
1300	break;
1301	case IB_QPT_RC:
1302	case IB_QPT_UC:
1303	case IB_QPT_UD:
1304	case IB_QPT_XRC_INI:
1305	case IB_QPT_XRC_TGT:
1306	case IB_QPT_DRIVER:
1307	break;
1308	default:
1309	return -EINVAL;
1310	}
1311
1312	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1313	&ib_dev);
1314	if (IS_ERR(ptr: obj))
1315	return PTR_ERR(ptr: obj);
1316	obj->uxrcd = NULL;
1317	obj->uevent.uobject.user_handle = cmd->user_handle;
1318	mutex_init(&obj->mcast_lock);
1319
1320	if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) {
1321	ind_tbl = uobj_get_obj_read(rwq_ind_table,
1322	UVERBS_OBJECT_RWQ_IND_TBL,
1323	cmd->rwq_ind_tbl_handle, attrs);
1324	if (!ind_tbl) {
1325	ret = -EINVAL;
1326	goto err_put;
1327	}
1328
1329	attr.rwq_ind_tbl = ind_tbl;
1330	}
1331
1332	if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) {
1333	ret = -EINVAL;
1334	goto err_put;
1335	}
1336
1337	if (ind_tbl && !cmd->max_send_wr)
1338	has_sq = false;
1339
1340	if (cmd->qp_type == IB_QPT_XRC_TGT) {
1341	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle,
1342	attrs);
1343
1344	if (IS_ERR(ptr: xrcd_uobj)) {
1345	ret = -EINVAL;
1346	goto err_put;
1347	}
1348
1349	xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1350	if (!xrcd) {
1351	ret = -EINVAL;
1352	goto err_put;
1353	}
1354	device = xrcd->device;
1355	} else {
1356	if (cmd->qp_type == IB_QPT_XRC_INI) {
1357	cmd->max_recv_wr = 0;
1358	cmd->max_recv_sge = 0;
1359	} else {
1360	if (cmd->is_srq) {
1361	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ,
1362	cmd->srq_handle, attrs);
1363	if (!srq || srq->srq_type == IB_SRQT_XRC) {
1364	ret = -EINVAL;
1365	goto err_put;
1366	}
1367	}
1368
1369	if (!ind_tbl) {
1370	if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1371	rcq = uobj_get_obj_read(
1372	cq, UVERBS_OBJECT_CQ,
1373	cmd->recv_cq_handle, attrs);
1374	if (!rcq) {
1375	ret = -EINVAL;
1376	goto err_put;
1377	}
1378	}
1379	}
1380	}
1381
1382	if (has_sq)
1383	scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
1384	cmd->send_cq_handle, attrs);
1385	if (!ind_tbl && cmd->qp_type != IB_QPT_XRC_INI)
1386	rcq = rcq ?: scq;
1387	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle,
1388	attrs);
1389	if (!pd || (!scq && has_sq)) {
1390	ret = -EINVAL;
1391	goto err_put;
1392	}
1393
1394	device = pd->device;
1395	}
1396
1397	attr.event_handler = ib_uverbs_qp_event_handler;
1398	attr.send_cq = scq;
1399	attr.recv_cq = rcq;
1400	attr.srq = srq;
1401	attr.xrcd = xrcd;
1402	attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1403	IB_SIGNAL_REQ_WR;
1404	attr.qp_type = cmd->qp_type;
1405
1406	attr.cap.max_send_wr = cmd->max_send_wr;
1407	attr.cap.max_recv_wr = cmd->max_recv_wr;
1408	attr.cap.max_send_sge = cmd->max_send_sge;
1409	attr.cap.max_recv_sge = cmd->max_recv_sge;
1410	attr.cap.max_inline_data = cmd->max_inline_data;
1411
1412	INIT_LIST_HEAD(list: &obj->uevent.event_list);
1413	INIT_LIST_HEAD(list: &obj->mcast_list);
1414
1415	attr.create_flags = cmd->create_flags;
1416	if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
1417	IB_QP_CREATE_CROSS_CHANNEL |
1418	IB_QP_CREATE_MANAGED_SEND |
1419	IB_QP_CREATE_MANAGED_RECV |
1420	IB_QP_CREATE_SCATTER_FCS |
1421	IB_QP_CREATE_CVLAN_STRIPPING |
1422	IB_QP_CREATE_SOURCE_QPN |
1423	IB_QP_CREATE_PCI_WRITE_END_PADDING)) {
1424	ret = -EINVAL;
1425	goto err_put;
1426	}
1427
1428	if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
1429	if (!capable(CAP_NET_RAW)) {
1430	ret = -EPERM;
1431	goto err_put;
1432	}
1433
1434	attr.source_qpn = cmd->source_qpn;
1435	}
1436
1437	qp = ib_create_qp_user(dev: device, pd, attr: &attr, udata: &attrs->driver_udata, uobj: obj,
1438	KBUILD_MODNAME);
1439	if (IS_ERR(ptr: qp)) {
1440	ret = PTR_ERR(ptr: qp);
1441	goto err_put;
1442	}
1443	ib_qp_usecnt_inc(qp);
1444
1445	obj->uevent.uobject.object = qp;
1446	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
1447	if (obj->uevent.event_file)
1448	uverbs_uobject_get(uobject: &obj->uevent.event_file->uobj);
1449
1450	if (xrcd) {
1451	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1452	uobject);
1453	atomic_inc(v: &obj->uxrcd->refcnt);
1454	uobj_put_read(uobj: xrcd_uobj);
1455	}
1456
1457	if (pd)
1458	uobj_put_obj_read(pd);
1459	if (scq)
1460	rdma_lookup_put_uobject(uobj: &scq->uobject->uevent.uobject,
1461	mode: UVERBS_LOOKUP_READ);
1462	if (rcq && rcq != scq)
1463	rdma_lookup_put_uobject(uobj: &rcq->uobject->uevent.uobject,
1464	mode: UVERBS_LOOKUP_READ);
1465	if (srq)
1466	rdma_lookup_put_uobject(uobj: &srq->uobject->uevent.uobject,
1467	mode: UVERBS_LOOKUP_READ);
1468	if (ind_tbl)
1469	uobj_put_obj_read(ind_tbl);
1470	uobj_finalize_uobj_create(uobj: &obj->uevent.uobject, attrs);
1471
1472	resp.base.qpn = qp->qp_num;
1473	resp.base.qp_handle = obj->uevent.uobject.id;
1474	resp.base.max_recv_sge = attr.cap.max_recv_sge;
1475	resp.base.max_send_sge = attr.cap.max_send_sge;
1476	resp.base.max_recv_wr = attr.cap.max_recv_wr;
1477	resp.base.max_send_wr = attr.cap.max_send_wr;
1478	resp.base.max_inline_data = attr.cap.max_inline_data;
1479	resp.response_length = uverbs_response_length(attrs, resp_len: sizeof(resp));
1480	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
1481
1482	err_put:
1483	if (!IS_ERR(ptr: xrcd_uobj))
1484	uobj_put_read(uobj: xrcd_uobj);
1485	if (pd)
1486	uobj_put_obj_read(pd);
1487	if (scq)
1488	rdma_lookup_put_uobject(uobj: &scq->uobject->uevent.uobject,
1489	mode: UVERBS_LOOKUP_READ);
1490	if (rcq && rcq != scq)
1491	rdma_lookup_put_uobject(uobj: &rcq->uobject->uevent.uobject,
1492	mode: UVERBS_LOOKUP_READ);
1493	if (srq)
1494	rdma_lookup_put_uobject(uobj: &srq->uobject->uevent.uobject,
1495	mode: UVERBS_LOOKUP_READ);
1496	if (ind_tbl)
1497	uobj_put_obj_read(ind_tbl);
1498
1499	uobj_alloc_abort(uobj: &obj->uevent.uobject, attrs);
1500	return ret;
1501	}
1502
1503	static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs)
1504	{
1505	struct ib_uverbs_create_qp cmd;
1506	struct ib_uverbs_ex_create_qp cmd_ex;
1507	int ret;
1508
1509	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1510	if (ret)
1511	return ret;
1512
1513	memset(&cmd_ex, 0, sizeof(cmd_ex));
1514	cmd_ex.user_handle = cmd.user_handle;
1515	cmd_ex.pd_handle = cmd.pd_handle;
1516	cmd_ex.send_cq_handle = cmd.send_cq_handle;
1517	cmd_ex.recv_cq_handle = cmd.recv_cq_handle;
1518	cmd_ex.srq_handle = cmd.srq_handle;
1519	cmd_ex.max_send_wr = cmd.max_send_wr;
1520	cmd_ex.max_recv_wr = cmd.max_recv_wr;
1521	cmd_ex.max_send_sge = cmd.max_send_sge;
1522	cmd_ex.max_recv_sge = cmd.max_recv_sge;
1523	cmd_ex.max_inline_data = cmd.max_inline_data;
1524	cmd_ex.sq_sig_all = cmd.sq_sig_all;
1525	cmd_ex.qp_type = cmd.qp_type;
1526	cmd_ex.is_srq = cmd.is_srq;
1527
1528	return create_qp(attrs, cmd: &cmd_ex);
1529	}
1530
1531	static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs)
1532	{
1533	struct ib_uverbs_ex_create_qp cmd;
1534	int ret;
1535
1536	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1537	if (ret)
1538	return ret;
1539
1540	if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
1541	return -EINVAL;
1542
1543	if (cmd.reserved)
1544	return -EINVAL;
1545
1546	return create_qp(attrs, cmd: &cmd);
1547	}
1548
1549	static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
1550	{
1551	struct ib_uverbs_create_qp_resp resp = {};
1552	struct ib_uverbs_open_qp cmd;
1553	struct ib_uqp_object *obj;
1554	struct ib_xrcd *xrcd;
1555	struct ib_qp *qp;
1556	struct ib_qp_open_attr attr = {};
1557	int ret;
1558	struct ib_uobject *xrcd_uobj;
1559	struct ib_device *ib_dev;
1560
1561	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1562	if (ret)
1563	return ret;
1564
1565	obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs,
1566	&ib_dev);
1567	if (IS_ERR(ptr: obj))
1568	return PTR_ERR(ptr: obj);
1569
1570	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs);
1571	if (IS_ERR(ptr: xrcd_uobj)) {
1572	ret = -EINVAL;
1573	goto err_put;
1574	}
1575
1576	xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1577	if (!xrcd) {
1578	ret = -EINVAL;
1579	goto err_xrcd;
1580	}
1581
1582	attr.event_handler = ib_uverbs_qp_event_handler;
1583	attr.qp_num = cmd.qpn;
1584	attr.qp_type = cmd.qp_type;
1585
1586	INIT_LIST_HEAD(list: &obj->uevent.event_list);
1587	INIT_LIST_HEAD(list: &obj->mcast_list);
1588
1589	qp = ib_open_qp(xrcd, qp_open_attr: &attr);
1590	if (IS_ERR(ptr: qp)) {
1591	ret = PTR_ERR(ptr: qp);
1592	goto err_xrcd;
1593	}
1594
1595	obj->uevent.uobject.object = qp;
1596	obj->uevent.uobject.user_handle = cmd.user_handle;
1597
1598	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1599	atomic_inc(v: &obj->uxrcd->refcnt);
1600	qp->uobject = obj;
1601	uobj_put_read(uobj: xrcd_uobj);
1602	uobj_finalize_uobj_create(uobj: &obj->uevent.uobject, attrs);
1603
1604	resp.qpn = qp->qp_num;
1605	resp.qp_handle = obj->uevent.uobject.id;
1606	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
1607
1608	err_xrcd:
1609	uobj_put_read(uobj: xrcd_uobj);
1610	err_put:
1611	uobj_alloc_abort(uobj: &obj->uevent.uobject, attrs);
1612	return ret;
1613	}
1614
1615	static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,
1616	struct rdma_ah_attr *rdma_attr)
1617	{
1618	const struct ib_global_route *grh;
1619
1620	uverb_attr->dlid = rdma_ah_get_dlid(attr: rdma_attr);
1621	uverb_attr->sl = rdma_ah_get_sl(attr: rdma_attr);
1622	uverb_attr->src_path_bits = rdma_ah_get_path_bits(attr: rdma_attr);
1623	uverb_attr->static_rate = rdma_ah_get_static_rate(attr: rdma_attr);
1624	uverb_attr->is_global = !!(rdma_ah_get_ah_flags(attr: rdma_attr) &
1625	IB_AH_GRH);
1626	if (uverb_attr->is_global) {
1627	grh = rdma_ah_read_grh(attr: rdma_attr);
1628	memcpy(uverb_attr->dgid, grh->dgid.raw, 16);
1629	uverb_attr->flow_label = grh->flow_label;
1630	uverb_attr->sgid_index = grh->sgid_index;
1631	uverb_attr->hop_limit = grh->hop_limit;
1632	uverb_attr->traffic_class = grh->traffic_class;
1633	}
1634	uverb_attr->port_num = rdma_ah_get_port_num(attr: rdma_attr);
1635	}
1636
1637	static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs)
1638	{
1639	struct ib_uverbs_query_qp cmd;
1640	struct ib_uverbs_query_qp_resp resp;
1641	struct ib_qp *qp;
1642	struct ib_qp_attr *attr;
1643	struct ib_qp_init_attr *init_attr;
1644	int ret;
1645
1646	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1647	if (ret)
1648	return ret;
1649
1650	attr = kmalloc(size: sizeof *attr, GFP_KERNEL);
1651	init_attr = kmalloc(size: sizeof *init_attr, GFP_KERNEL);
1652	if (!attr || !init_attr) {
1653	ret = -ENOMEM;
1654	goto out;
1655	}
1656
1657	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1658	if (!qp) {
1659	ret = -EINVAL;
1660	goto out;
1661	}
1662
1663	ret = ib_query_qp(qp, qp_attr: attr, qp_attr_mask: cmd.attr_mask, qp_init_attr: init_attr);
1664
1665	rdma_lookup_put_uobject(uobj: &qp->uobject->uevent.uobject,
1666	mode: UVERBS_LOOKUP_READ);
1667
1668	if (ret)
1669	goto out;
1670
1671	memset(&resp, 0, sizeof resp);
1672
1673	resp.qp_state = attr->qp_state;
1674	resp.cur_qp_state = attr->cur_qp_state;
1675	resp.path_mtu = attr->path_mtu;
1676	resp.path_mig_state = attr->path_mig_state;
1677	resp.qkey = attr->qkey;
1678	resp.rq_psn = attr->rq_psn;
1679	resp.sq_psn = attr->sq_psn;
1680	resp.dest_qp_num = attr->dest_qp_num;
1681	resp.qp_access_flags = attr->qp_access_flags;
1682	resp.pkey_index = attr->pkey_index;
1683	resp.alt_pkey_index = attr->alt_pkey_index;
1684	resp.sq_draining = attr->sq_draining;
1685	resp.max_rd_atomic = attr->max_rd_atomic;
1686	resp.max_dest_rd_atomic = attr->max_dest_rd_atomic;
1687	resp.min_rnr_timer = attr->min_rnr_timer;
1688	resp.port_num = attr->port_num;
1689	resp.timeout = attr->timeout;
1690	resp.retry_cnt = attr->retry_cnt;
1691	resp.rnr_retry = attr->rnr_retry;
1692	resp.alt_port_num = attr->alt_port_num;
1693	resp.alt_timeout = attr->alt_timeout;
1694
1695	copy_ah_attr_to_uverbs(uverb_attr: &resp.dest, rdma_attr: &attr->ah_attr);
1696	copy_ah_attr_to_uverbs(uverb_attr: &resp.alt_dest, rdma_attr: &attr->alt_ah_attr);
1697
1698	resp.max_send_wr = init_attr->cap.max_send_wr;
1699	resp.max_recv_wr = init_attr->cap.max_recv_wr;
1700	resp.max_send_sge = init_attr->cap.max_send_sge;
1701	resp.max_recv_sge = init_attr->cap.max_recv_sge;
1702	resp.max_inline_data = init_attr->cap.max_inline_data;
1703	resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1704
1705	ret = uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
1706
1707	out:
1708	kfree(objp: attr);
1709	kfree(objp: init_attr);
1710
1711	return ret;
1712	}
1713
1714	/* Remove ignored fields set in the attribute mask */
1715	static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
1716	{
1717	switch (qp_type) {
1718	case IB_QPT_XRC_INI:
1719	return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
1720	case IB_QPT_XRC_TGT:
1721	return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
1722	IB_QP_RNR_RETRY);
1723	default:
1724	return mask;
1725	}
1726	}
1727
1728	static void copy_ah_attr_from_uverbs(struct ib_device *dev,
1729	struct rdma_ah_attr *rdma_attr,
1730	struct ib_uverbs_qp_dest *uverb_attr)
1731	{
1732	rdma_attr->type = rdma_ah_find_type(dev, port_num: uverb_attr->port_num);
1733	if (uverb_attr->is_global) {
1734	rdma_ah_set_grh(attr: rdma_attr, NULL,
1735	flow_label: uverb_attr->flow_label,
1736	sgid_index: uverb_attr->sgid_index,
1737	hop_limit: uverb_attr->hop_limit,
1738	traffic_class: uverb_attr->traffic_class);
1739	rdma_ah_set_dgid_raw(attr: rdma_attr, dgid: uverb_attr->dgid);
1740	} else {
1741	rdma_ah_set_ah_flags(attr: rdma_attr, flag: 0);
1742	}
1743	rdma_ah_set_dlid(attr: rdma_attr, dlid: uverb_attr->dlid);
1744	rdma_ah_set_sl(attr: rdma_attr, sl: uverb_attr->sl);
1745	rdma_ah_set_path_bits(attr: rdma_attr, src_path_bits: uverb_attr->src_path_bits);
1746	rdma_ah_set_static_rate(attr: rdma_attr, static_rate: uverb_attr->static_rate);
1747	rdma_ah_set_port_num(attr: rdma_attr, port_num: uverb_attr->port_num);
1748	rdma_ah_set_make_grd(attr: rdma_attr, make_grd: false);
1749	}
1750
1751	static int modify_qp(struct uverbs_attr_bundle *attrs,
1752	struct ib_uverbs_ex_modify_qp *cmd)
1753	{
1754	struct ib_qp_attr *attr;
1755	struct ib_qp *qp;
1756	int ret;
1757
1758	attr = kzalloc(size: sizeof(*attr), GFP_KERNEL);
1759	if (!attr)
1760	return -ENOMEM;
1761
1762	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle,
1763	attrs);
1764	if (!qp) {
1765	ret = -EINVAL;
1766	goto out;
1767	}
1768
1769	if ((cmd->base.attr_mask & IB_QP_PORT) &&
1770	!rdma_is_port_valid(device: qp->device, port: cmd->base.port_num)) {
1771	ret = -EINVAL;
1772	goto release_qp;
1773	}
1774
1775	if ((cmd->base.attr_mask & IB_QP_AV)) {
1776	if (!rdma_is_port_valid(device: qp->device, port: cmd->base.dest.port_num)) {
1777	ret = -EINVAL;
1778	goto release_qp;
1779	}
1780
1781	if (cmd->base.attr_mask & IB_QP_STATE &&
1782	cmd->base.qp_state == IB_QPS_RTR) {
1783	/* We are in INIT->RTR TRANSITION (if we are not,
1784	* this transition will be rejected in subsequent checks).
1785	* In the INIT->RTR transition, we cannot have IB_QP_PORT set,
1786	* but the IB_QP_STATE flag is required.
1787	*
1788	* Since kernel 3.14 (commit dbf727de7440), the uverbs driver,
1789	* when IB_QP_AV is set, has required inclusion of a valid
1790	* port number in the primary AV. (AVs are created and handled
1791	* differently for infiniband and ethernet (RoCE) ports).
1792	*
1793	* Check the port number included in the primary AV against
1794	* the port number in the qp struct, which was set (and saved)
1795	* in the RST->INIT transition.
1796	*/
1797	if (cmd->base.dest.port_num != qp->real_qp->port) {
1798	ret = -EINVAL;
1799	goto release_qp;
1800	}
1801	} else {
1802	/* We are in SQD->SQD. (If we are not, this transition will
1803	* be rejected later in the verbs layer checks).
1804	* Check for both IB_QP_PORT and IB_QP_AV, these can be set
1805	* together in the SQD->SQD transition.
1806	*
1807	* If only IP_QP_AV was set, add in IB_QP_PORT as well (the
1808	* verbs layer driver does not track primary port changes
1809	* resulting from path migration. Thus, in SQD, if the primary
1810	* AV is modified, the primary port should also be modified).
1811	*
1812	* Note that in this transition, the IB_QP_STATE flag
1813	* is not allowed.
1814	*/
1815	if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1816	== (IB_QP_AV | IB_QP_PORT)) &&
1817	cmd->base.port_num != cmd->base.dest.port_num) {
1818	ret = -EINVAL;
1819	goto release_qp;
1820	}
1821	if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
1822	== IB_QP_AV) {
1823	cmd->base.attr_mask |= IB_QP_PORT;
1824	cmd->base.port_num = cmd->base.dest.port_num;
1825	}
1826	}
1827	}
1828
1829	if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&
1830	(!rdma_is_port_valid(device: qp->device, port: cmd->base.alt_port_num) ||
1831	!rdma_is_port_valid(device: qp->device, port: cmd->base.alt_dest.port_num) ||
1832	cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) {
1833	ret = -EINVAL;
1834	goto release_qp;
1835	}
1836
1837	if ((cmd->base.attr_mask & IB_QP_CUR_STATE &&
1838	cmd->base.cur_qp_state > IB_QPS_ERR) ||
1839	(cmd->base.attr_mask & IB_QP_STATE &&
1840	cmd->base.qp_state > IB_QPS_ERR)) {
1841	ret = -EINVAL;
1842	goto release_qp;
1843	}
1844
1845	if (cmd->base.attr_mask & IB_QP_STATE)
1846	attr->qp_state = cmd->base.qp_state;
1847	if (cmd->base.attr_mask & IB_QP_CUR_STATE)
1848	attr->cur_qp_state = cmd->base.cur_qp_state;
1849	if (cmd->base.attr_mask & IB_QP_PATH_MTU)
1850	attr->path_mtu = cmd->base.path_mtu;
1851	if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE)
1852	attr->path_mig_state = cmd->base.path_mig_state;
1853	if (cmd->base.attr_mask & IB_QP_QKEY) {
1854	if (cmd->base.qkey & IB_QP_SET_QKEY &&
1855	!rdma_nl_get_privileged_qkey()) {
1856	ret = -EPERM;
1857	goto release_qp;
1858	}
1859	attr->qkey = cmd->base.qkey;
1860	}
1861	if (cmd->base.attr_mask & IB_QP_RQ_PSN)
1862	attr->rq_psn = cmd->base.rq_psn;
1863	if (cmd->base.attr_mask & IB_QP_SQ_PSN)
1864	attr->sq_psn = cmd->base.sq_psn;
1865	if (cmd->base.attr_mask & IB_QP_DEST_QPN)
1866	attr->dest_qp_num = cmd->base.dest_qp_num;
1867	if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS)
1868	attr->qp_access_flags = cmd->base.qp_access_flags;
1869	if (cmd->base.attr_mask & IB_QP_PKEY_INDEX)
1870	attr->pkey_index = cmd->base.pkey_index;
1871	if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1872	attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;
1873	if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
1874	attr->max_rd_atomic = cmd->base.max_rd_atomic;
1875	if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1876	attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic;
1877	if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER)
1878	attr->min_rnr_timer = cmd->base.min_rnr_timer;
1879	if (cmd->base.attr_mask & IB_QP_PORT)
1880	attr->port_num = cmd->base.port_num;
1881	if (cmd->base.attr_mask & IB_QP_TIMEOUT)
1882	attr->timeout = cmd->base.timeout;
1883	if (cmd->base.attr_mask & IB_QP_RETRY_CNT)
1884	attr->retry_cnt = cmd->base.retry_cnt;
1885	if (cmd->base.attr_mask & IB_QP_RNR_RETRY)
1886	attr->rnr_retry = cmd->base.rnr_retry;
1887	if (cmd->base.attr_mask & IB_QP_ALT_PATH) {
1888	attr->alt_port_num = cmd->base.alt_port_num;
1889	attr->alt_timeout = cmd->base.alt_timeout;
1890	attr->alt_pkey_index = cmd->base.alt_pkey_index;
1891	}
1892	if (cmd->base.attr_mask & IB_QP_RATE_LIMIT)
1893	attr->rate_limit = cmd->rate_limit;
1894
1895	if (cmd->base.attr_mask & IB_QP_AV)
1896	copy_ah_attr_from_uverbs(dev: qp->device, rdma_attr: &attr->ah_attr,
1897	uverb_attr: &cmd->base.dest);
1898
1899	if (cmd->base.attr_mask & IB_QP_ALT_PATH)
1900	copy_ah_attr_from_uverbs(dev: qp->device, rdma_attr: &attr->alt_ah_attr,
1901	uverb_attr: &cmd->base.alt_dest);
1902
1903	ret = ib_modify_qp_with_udata(qp, attr,
1904	attr_mask: modify_qp_mask(qp_type: qp->qp_type,
1905	mask: cmd->base.attr_mask),
1906	udata: &attrs->driver_udata);
1907
1908	release_qp:
1909	rdma_lookup_put_uobject(uobj: &qp->uobject->uevent.uobject,
1910	mode: UVERBS_LOOKUP_READ);
1911	out:
1912	kfree(objp: attr);
1913
1914	return ret;
1915	}
1916
1917	static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs)
1918	{
1919	struct ib_uverbs_ex_modify_qp cmd;
1920	int ret;
1921
1922	ret = uverbs_request(attrs, req: &cmd.base, req_len: sizeof(cmd.base));
1923	if (ret)
1924	return ret;
1925
1926	if (cmd.base.attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1927	return -EOPNOTSUPP;
1928
1929	return modify_qp(attrs, cmd: &cmd);
1930	}
1931
1932	static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs)
1933	{
1934	struct ib_uverbs_ex_modify_qp cmd;
1935	struct ib_uverbs_ex_modify_qp_resp resp = {
1936	.response_length = uverbs_response_length(attrs, resp_len: sizeof(resp))
1937	};
1938	int ret;
1939
1940	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1941	if (ret)
1942	return ret;
1943
1944	/*
1945	* Last bit is reserved for extending the attr_mask by
1946	* using another field.
1947	*/
1948	if (cmd.base.attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
1949	return -EOPNOTSUPP;
1950
1951	ret = modify_qp(attrs, cmd: &cmd);
1952	if (ret)
1953	return ret;
1954
1955	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
1956	}
1957
1958	static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs)
1959	{
1960	struct ib_uverbs_destroy_qp cmd;
1961	struct ib_uverbs_destroy_qp_resp resp;
1962	struct ib_uobject *uobj;
1963	struct ib_uqp_object *obj;
1964	int ret;
1965
1966	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
1967	if (ret)
1968	return ret;
1969
1970	uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
1971	if (IS_ERR(ptr: uobj))
1972	return PTR_ERR(ptr: uobj);
1973
1974	obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
1975	memset(&resp, 0, sizeof(resp));
1976	resp.events_reported = obj->uevent.events_reported;
1977
1978	uobj_put_destroy(uobj);
1979
1980	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
1981	}
1982
1983	static void *alloc_wr(size_t wr_size, __u32 num_sge)
1984	{
1985	if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof(struct ib_sge))) /
1986	sizeof(struct ib_sge))
1987	return NULL;
1988
1989	return kmalloc(ALIGN(wr_size, sizeof(struct ib_sge)) +
1990	num_sge * sizeof(struct ib_sge),
1991	GFP_KERNEL);
1992	}
1993
1994	static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs)
1995	{
1996	struct ib_uverbs_post_send cmd;
1997	struct ib_uverbs_post_send_resp resp;
1998	struct ib_uverbs_send_wr *user_wr;
1999	struct ib_send_wr *wr = NULL, *last, *next;
2000	const struct ib_send_wr *bad_wr;
2001	struct ib_qp *qp;
2002	int i, sg_ind;
2003	int is_ud;
2004	int ret, ret2;
2005	size_t next_size;
2006	const struct ib_sge __user *sgls;
2007	const void __user *wqes;
2008	struct uverbs_req_iter iter;
2009
2010	ret = uverbs_request_start(attrs, iter: &iter, req: &cmd, req_len: sizeof(cmd));
2011	if (ret)
2012	return ret;
2013	wqes = uverbs_request_next_ptr(iter: &iter, len: cmd.wqe_size * cmd.wr_count);
2014	if (IS_ERR(ptr: wqes))
2015	return PTR_ERR(ptr: wqes);
2016	sgls = uverbs_request_next_ptr(
2017	iter: &iter, len: cmd.sge_count * sizeof(struct ib_uverbs_sge));
2018	if (IS_ERR(ptr: sgls))
2019	return PTR_ERR(ptr: sgls);
2020	ret = uverbs_request_finish(iter: &iter);
2021	if (ret)
2022	return ret;
2023
2024	user_wr = kmalloc(size: cmd.wqe_size, GFP_KERNEL);
2025	if (!user_wr)
2026	return -ENOMEM;
2027
2028	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2029	if (!qp) {
2030	ret = -EINVAL;
2031	goto out;
2032	}
2033
2034	is_ud = qp->qp_type == IB_QPT_UD;
2035	sg_ind = 0;
2036	last = NULL;
2037	for (i = 0; i < cmd.wr_count; ++i) {
2038	if (copy_from_user(to: user_wr, from: wqes + i * cmd.wqe_size,
2039	n: cmd.wqe_size)) {
2040	ret = -EFAULT;
2041	goto out_put;
2042	}
2043
2044	if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2045	ret = -EINVAL;
2046	goto out_put;
2047	}
2048
2049	if (is_ud) {
2050	struct ib_ud_wr *ud;
2051
2052	if (user_wr->opcode != IB_WR_SEND &&
2053	user_wr->opcode != IB_WR_SEND_WITH_IMM) {
2054	ret = -EINVAL;
2055	goto out_put;
2056	}
2057
2058	next_size = sizeof(*ud);
2059	ud = alloc_wr(wr_size: next_size, num_sge: user_wr->num_sge);
2060	if (!ud) {
2061	ret = -ENOMEM;
2062	goto out_put;
2063	}
2064
2065	ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH,
2066	user_wr->wr.ud.ah, attrs);
2067	if (!ud->ah) {
2068	kfree(objp: ud);
2069	ret = -EINVAL;
2070	goto out_put;
2071	}
2072	ud->remote_qpn = user_wr->wr.ud.remote_qpn;
2073	ud->remote_qkey = user_wr->wr.ud.remote_qkey;
2074
2075	next = &ud->wr;
2076	} else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2077	user_wr->opcode == IB_WR_RDMA_WRITE ||
2078	user_wr->opcode == IB_WR_RDMA_READ) {
2079	struct ib_rdma_wr *rdma;
2080
2081	next_size = sizeof(*rdma);
2082	rdma = alloc_wr(wr_size: next_size, num_sge: user_wr->num_sge);
2083	if (!rdma) {
2084	ret = -ENOMEM;
2085	goto out_put;
2086	}
2087
2088	rdma->remote_addr = user_wr->wr.rdma.remote_addr;
2089	rdma->rkey = user_wr->wr.rdma.rkey;
2090
2091	next = &rdma->wr;
2092	} else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2093	user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2094	struct ib_atomic_wr *atomic;
2095
2096	next_size = sizeof(*atomic);
2097	atomic = alloc_wr(wr_size: next_size, num_sge: user_wr->num_sge);
2098	if (!atomic) {
2099	ret = -ENOMEM;
2100	goto out_put;
2101	}
2102
2103	atomic->remote_addr = user_wr->wr.atomic.remote_addr;
2104	atomic->compare_add = user_wr->wr.atomic.compare_add;
2105	atomic->swap = user_wr->wr.atomic.swap;
2106	atomic->rkey = user_wr->wr.atomic.rkey;
2107
2108	next = &atomic->wr;
2109	} else if (user_wr->opcode == IB_WR_SEND ||
2110	user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2111	user_wr->opcode == IB_WR_SEND_WITH_INV) {
2112	next_size = sizeof(*next);
2113	next = alloc_wr(wr_size: next_size, num_sge: user_wr->num_sge);
2114	if (!next) {
2115	ret = -ENOMEM;
2116	goto out_put;
2117	}
2118	} else {
2119	ret = -EINVAL;
2120	goto out_put;
2121	}
2122
2123	if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2124	user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
2125	next->ex.imm_data =
2126	(__be32 __force) user_wr->ex.imm_data;
2127	} else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {
2128	next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;
2129	}
2130
2131	if (!last)
2132	wr = next;
2133	else
2134	last->next = next;
2135	last = next;
2136
2137	next->next = NULL;
2138	next->wr_id = user_wr->wr_id;
2139	next->num_sge = user_wr->num_sge;
2140	next->opcode = user_wr->opcode;
2141	next->send_flags = user_wr->send_flags;
2142
2143	if (next->num_sge) {
2144	next->sg_list = (void *) next +
2145	ALIGN(next_size, sizeof(struct ib_sge));
2146	if (copy_from_user(to: next->sg_list, from: sgls + sg_ind,
2147	n: next->num_sge *
2148	sizeof(struct ib_sge))) {
2149	ret = -EFAULT;
2150	goto out_put;
2151	}
2152	sg_ind += next->num_sge;
2153	} else
2154	next->sg_list = NULL;
2155	}
2156
2157	resp.bad_wr = 0;
2158	ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr);
2159	if (ret)
2160	for (next = wr; next; next = next->next) {
2161	++resp.bad_wr;
2162	if (next == bad_wr)
2163	break;
2164	}
2165
2166	ret2 = uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
2167	if (ret2)
2168	ret = ret2;
2169
2170	out_put:
2171	rdma_lookup_put_uobject(uobj: &qp->uobject->uevent.uobject,
2172	mode: UVERBS_LOOKUP_READ);
2173
2174	while (wr) {
2175	if (is_ud && ud_wr(wr)->ah)
2176	uobj_put_obj_read(ud_wr(wr)->ah);
2177	next = wr->next;
2178	kfree(objp: wr);
2179	wr = next;
2180	}
2181
2182	out:
2183	kfree(objp: user_wr);
2184
2185	return ret;
2186	}
2187
2188	static struct ib_recv_wr *
2189	ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count,
2190	u32 wqe_size, u32 sge_count)
2191	{
2192	struct ib_uverbs_recv_wr *user_wr;
2193	struct ib_recv_wr *wr = NULL, *last, *next;
2194	int sg_ind;
2195	int i;
2196	int ret;
2197	const struct ib_sge __user *sgls;
2198	const void __user *wqes;
2199
2200	if (wqe_size < sizeof(struct ib_uverbs_recv_wr))
2201	return ERR_PTR(error: -EINVAL);
2202
2203	wqes = uverbs_request_next_ptr(iter, len: wqe_size * wr_count);
2204	if (IS_ERR(ptr: wqes))
2205	return ERR_CAST(ptr: wqes);
2206	sgls = uverbs_request_next_ptr(
2207	iter, len: sge_count * sizeof(struct ib_uverbs_sge));
2208	if (IS_ERR(ptr: sgls))
2209	return ERR_CAST(ptr: sgls);
2210	ret = uverbs_request_finish(iter);
2211	if (ret)
2212	return ERR_PTR(error: ret);
2213
2214	user_wr = kmalloc(size: wqe_size, GFP_KERNEL);
2215	if (!user_wr)
2216	return ERR_PTR(error: -ENOMEM);
2217
2218	sg_ind = 0;
2219	last = NULL;
2220	for (i = 0; i < wr_count; ++i) {
2221	if (copy_from_user(to: user_wr, from: wqes + i * wqe_size,
2222	n: wqe_size)) {
2223	ret = -EFAULT;
2224	goto err;
2225	}
2226
2227	if (user_wr->num_sge + sg_ind > sge_count) {
2228	ret = -EINVAL;
2229	goto err;
2230	}
2231
2232	if (user_wr->num_sge >=
2233	(U32_MAX - ALIGN(sizeof(*next), sizeof(struct ib_sge))) /
2234	sizeof(struct ib_sge)) {
2235	ret = -EINVAL;
2236	goto err;
2237	}
2238
2239	next = kmalloc(ALIGN(sizeof(*next), sizeof(struct ib_sge)) +
2240	user_wr->num_sge * sizeof(struct ib_sge),
2241	GFP_KERNEL);
2242	if (!next) {
2243	ret = -ENOMEM;
2244	goto err;
2245	}
2246
2247	if (!last)
2248	wr = next;
2249	else
2250	last->next = next;
2251	last = next;
2252
2253	next->next = NULL;
2254	next->wr_id = user_wr->wr_id;
2255	next->num_sge = user_wr->num_sge;
2256
2257	if (next->num_sge) {
2258	next->sg_list = (void *)next +
2259	ALIGN(sizeof(*next), sizeof(struct ib_sge));
2260	if (copy_from_user(to: next->sg_list, from: sgls + sg_ind,
2261	n: next->num_sge *
2262	sizeof(struct ib_sge))) {
2263	ret = -EFAULT;
2264	goto err;
2265	}
2266	sg_ind += next->num_sge;
2267	} else
2268	next->sg_list = NULL;
2269	}
2270
2271	kfree(objp: user_wr);
2272	return wr;
2273
2274	err:
2275	kfree(objp: user_wr);
2276
2277	while (wr) {
2278	next = wr->next;
2279	kfree(objp: wr);
2280	wr = next;
2281	}
2282
2283	return ERR_PTR(error: ret);
2284	}
2285
2286	static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs)
2287	{
2288	struct ib_uverbs_post_recv cmd;
2289	struct ib_uverbs_post_recv_resp resp;
2290	struct ib_recv_wr *wr, *next;
2291	const struct ib_recv_wr *bad_wr;
2292	struct ib_qp *qp;
2293	int ret, ret2;
2294	struct uverbs_req_iter iter;
2295
2296	ret = uverbs_request_start(attrs, iter: &iter, req: &cmd, req_len: sizeof(cmd));
2297	if (ret)
2298	return ret;
2299
2300	wr = ib_uverbs_unmarshall_recv(iter: &iter, wr_count: cmd.wr_count, wqe_size: cmd.wqe_size,
2301	sge_count: cmd.sge_count);
2302	if (IS_ERR(ptr: wr))
2303	return PTR_ERR(ptr: wr);
2304
2305	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2306	if (!qp) {
2307	ret = -EINVAL;
2308	goto out;
2309	}
2310
2311	resp.bad_wr = 0;
2312	ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr);
2313
2314	rdma_lookup_put_uobject(uobj: &qp->uobject->uevent.uobject,
2315	mode: UVERBS_LOOKUP_READ);
2316	if (ret) {
2317	for (next = wr; next; next = next->next) {
2318	++resp.bad_wr;
2319	if (next == bad_wr)
2320	break;
2321	}
2322	}
2323
2324	ret2 = uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
2325	if (ret2)
2326	ret = ret2;
2327	out:
2328	while (wr) {
2329	next = wr->next;
2330	kfree(objp: wr);
2331	wr = next;
2332	}
2333
2334	return ret;
2335	}
2336
2337	static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs)
2338	{
2339	struct ib_uverbs_post_srq_recv cmd;
2340	struct ib_uverbs_post_srq_recv_resp resp;
2341	struct ib_recv_wr *wr, *next;
2342	const struct ib_recv_wr *bad_wr;
2343	struct ib_srq *srq;
2344	int ret, ret2;
2345	struct uverbs_req_iter iter;
2346
2347	ret = uverbs_request_start(attrs, iter: &iter, req: &cmd, req_len: sizeof(cmd));
2348	if (ret)
2349	return ret;
2350
2351	wr = ib_uverbs_unmarshall_recv(iter: &iter, wr_count: cmd.wr_count, wqe_size: cmd.wqe_size,
2352	sge_count: cmd.sge_count);
2353	if (IS_ERR(ptr: wr))
2354	return PTR_ERR(ptr: wr);
2355
2356	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
2357	if (!srq) {
2358	ret = -EINVAL;
2359	goto out;
2360	}
2361
2362	resp.bad_wr = 0;
2363	ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr);
2364
2365	rdma_lookup_put_uobject(uobj: &srq->uobject->uevent.uobject,
2366	mode: UVERBS_LOOKUP_READ);
2367
2368	if (ret)
2369	for (next = wr; next; next = next->next) {
2370	++resp.bad_wr;
2371	if (next == bad_wr)
2372	break;
2373	}
2374
2375	ret2 = uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
2376	if (ret2)
2377	ret = ret2;
2378
2379	out:
2380	while (wr) {
2381	next = wr->next;
2382	kfree(objp: wr);
2383	wr = next;
2384	}
2385
2386	return ret;
2387	}
2388
2389	static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)
2390	{
2391	struct ib_uverbs_create_ah cmd;
2392	struct ib_uverbs_create_ah_resp resp;
2393	struct ib_uobject *uobj;
2394	struct ib_pd *pd;
2395	struct ib_ah *ah;
2396	struct rdma_ah_attr attr = {};
2397	int ret;
2398	struct ib_device *ib_dev;
2399
2400	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
2401	if (ret)
2402	return ret;
2403
2404	uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev);
2405	if (IS_ERR(ptr: uobj))
2406	return PTR_ERR(ptr: uobj);
2407
2408	if (!rdma_is_port_valid(device: ib_dev, port: cmd.attr.port_num)) {
2409	ret = -EINVAL;
2410	goto err;
2411	}
2412
2413	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2414	if (!pd) {
2415	ret = -EINVAL;
2416	goto err;
2417	}
2418
2419	attr.type = rdma_ah_find_type(dev: ib_dev, port_num: cmd.attr.port_num);
2420	rdma_ah_set_make_grd(attr: &attr, make_grd: false);
2421	rdma_ah_set_dlid(attr: &attr, dlid: cmd.attr.dlid);
2422	rdma_ah_set_sl(attr: &attr, sl: cmd.attr.sl);
2423	rdma_ah_set_path_bits(attr: &attr, src_path_bits: cmd.attr.src_path_bits);
2424	rdma_ah_set_static_rate(attr: &attr, static_rate: cmd.attr.static_rate);
2425	rdma_ah_set_port_num(attr: &attr, port_num: cmd.attr.port_num);
2426
2427	if (cmd.attr.is_global) {
2428	rdma_ah_set_grh(attr: &attr, NULL, flow_label: cmd.attr.grh.flow_label,
2429	sgid_index: cmd.attr.grh.sgid_index,
2430	hop_limit: cmd.attr.grh.hop_limit,
2431	traffic_class: cmd.attr.grh.traffic_class);
2432	rdma_ah_set_dgid_raw(attr: &attr, dgid: cmd.attr.grh.dgid);
2433	} else {
2434	rdma_ah_set_ah_flags(attr: &attr, flag: 0);
2435	}
2436
2437	ah = rdma_create_user_ah(pd, ah_attr: &attr, udata: &attrs->driver_udata);
2438	if (IS_ERR(ptr: ah)) {
2439	ret = PTR_ERR(ptr: ah);
2440	goto err_put;
2441	}
2442
2443	ah->uobject = uobj;
2444	uobj->user_handle = cmd.user_handle;
2445	uobj->object = ah;
2446	uobj_put_obj_read(pd);
2447	uobj_finalize_uobj_create(uobj, attrs);
2448
2449	resp.ah_handle = uobj->id;
2450	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
2451
2452	err_put:
2453	uobj_put_obj_read(pd);
2454	err:
2455	uobj_alloc_abort(uobj, attrs);
2456	return ret;
2457	}
2458
2459	static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs)
2460	{
2461	struct ib_uverbs_destroy_ah cmd;
2462	int ret;
2463
2464	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
2465	if (ret)
2466	return ret;
2467
2468	return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs);
2469	}
2470
2471	static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs)
2472	{
2473	struct ib_uverbs_attach_mcast cmd;
2474	struct ib_qp *qp;
2475	struct ib_uqp_object *obj;
2476	struct ib_uverbs_mcast_entry *mcast;
2477	int ret;
2478
2479	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
2480	if (ret)
2481	return ret;
2482
2483	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2484	if (!qp)
2485	return -EINVAL;
2486
2487	obj = qp->uobject;
2488
2489	mutex_lock(&obj->mcast_lock);
2490	list_for_each_entry(mcast, &obj->mcast_list, list)
2491	if (cmd.mlid == mcast->lid &&
2492	!memcmp(p: cmd.gid, q: mcast->gid.raw, size: sizeof mcast->gid.raw)) {
2493	ret = 0;
2494	goto out_put;
2495	}
2496
2497	mcast = kmalloc(size: sizeof *mcast, GFP_KERNEL);
2498	if (!mcast) {
2499	ret = -ENOMEM;
2500	goto out_put;
2501	}
2502
2503	mcast->lid = cmd.mlid;
2504	memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
2505
2506	ret = ib_attach_mcast(qp, gid: &mcast->gid, lid: cmd.mlid);
2507	if (!ret)
2508	list_add_tail(new: &mcast->list, head: &obj->mcast_list);
2509	else
2510	kfree(objp: mcast);
2511
2512	out_put:
2513	mutex_unlock(lock: &obj->mcast_lock);
2514	rdma_lookup_put_uobject(uobj: &qp->uobject->uevent.uobject,
2515	mode: UVERBS_LOOKUP_READ);
2516
2517	return ret;
2518	}
2519
2520	static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs)
2521	{
2522	struct ib_uverbs_detach_mcast cmd;
2523	struct ib_uqp_object *obj;
2524	struct ib_qp *qp;
2525	struct ib_uverbs_mcast_entry *mcast;
2526	int ret;
2527	bool found = false;
2528
2529	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
2530	if (ret)
2531	return ret;
2532
2533	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
2534	if (!qp)
2535	return -EINVAL;
2536
2537	obj = qp->uobject;
2538	mutex_lock(&obj->mcast_lock);
2539
2540	list_for_each_entry(mcast, &obj->mcast_list, list)
2541	if (cmd.mlid == mcast->lid &&
2542	!memcmp(p: cmd.gid, q: mcast->gid.raw, size: sizeof mcast->gid.raw)) {
2543	list_del(entry: &mcast->list);
2544	kfree(objp: mcast);
2545	found = true;
2546	break;
2547	}
2548
2549	if (!found) {
2550	ret = -EINVAL;
2551	goto out_put;
2552	}
2553
2554	ret = ib_detach_mcast(qp, gid: (union ib_gid *)cmd.gid, lid: cmd.mlid);
2555
2556	out_put:
2557	mutex_unlock(lock: &obj->mcast_lock);
2558	rdma_lookup_put_uobject(uobj: &qp->uobject->uevent.uobject,
2559	mode: UVERBS_LOOKUP_READ);
2560	return ret;
2561	}
2562
2563	struct ib_uflow_resources *flow_resources_alloc(size_t num_specs)
2564	{
2565	struct ib_uflow_resources *resources;
2566
2567	resources = kzalloc(size: sizeof(*resources), GFP_KERNEL);
2568
2569	if (!resources)
2570	return NULL;
2571
2572	if (!num_specs)
2573	goto out;
2574
2575	resources->counters =
2576	kcalloc(n: num_specs, size: sizeof(*resources->counters), GFP_KERNEL);
2577	resources->collection =
2578	kcalloc(n: num_specs, size: sizeof(*resources->collection), GFP_KERNEL);
2579
2580	if (!resources->counters || !resources->collection)
2581	goto err;
2582
2583	out:
2584	resources->max = num_specs;
2585	return resources;
2586
2587	err:
2588	kfree(objp: resources->counters);
2589	kfree(objp: resources);
2590
2591	return NULL;
2592	}
2593	EXPORT_SYMBOL(flow_resources_alloc);
2594
2595	void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res)
2596	{
2597	unsigned int i;
2598
2599	if (!uflow_res)
2600	return;
2601
2602	for (i = 0; i < uflow_res->collection_num; i++)
2603	atomic_dec(v: &uflow_res->collection[i]->usecnt);
2604
2605	for (i = 0; i < uflow_res->counters_num; i++)
2606	atomic_dec(v: &uflow_res->counters[i]->usecnt);
2607
2608	kfree(objp: uflow_res->collection);
2609	kfree(objp: uflow_res->counters);
2610	kfree(objp: uflow_res);
2611	}
2612	EXPORT_SYMBOL(ib_uverbs_flow_resources_free);
2613
2614	void flow_resources_add(struct ib_uflow_resources *uflow_res,
2615	enum ib_flow_spec_type type,
2616	void *ibobj)
2617	{
2618	WARN_ON(uflow_res->num >= uflow_res->max);
2619
2620	switch (type) {
2621	case IB_FLOW_SPEC_ACTION_HANDLE:
2622	atomic_inc(v: &((struct ib_flow_action *)ibobj)->usecnt);
2623	uflow_res->collection[uflow_res->collection_num++] =
2624	(struct ib_flow_action *)ibobj;
2625	break;
2626	case IB_FLOW_SPEC_ACTION_COUNT:
2627	atomic_inc(v: &((struct ib_counters *)ibobj)->usecnt);
2628	uflow_res->counters[uflow_res->counters_num++] =
2629	(struct ib_counters *)ibobj;
2630	break;
2631	default:
2632	WARN_ON(1);
2633	}
2634
2635	uflow_res->num++;
2636	}
2637	EXPORT_SYMBOL(flow_resources_add);
2638
2639	static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs,
2640	struct ib_uverbs_flow_spec *kern_spec,
2641	union ib_flow_spec *ib_spec,
2642	struct ib_uflow_resources *uflow_res)
2643	{
2644	ib_spec->type = kern_spec->type;
2645	switch (ib_spec->type) {
2646	case IB_FLOW_SPEC_ACTION_TAG:
2647	if (kern_spec->flow_tag.size !=
2648	sizeof(struct ib_uverbs_flow_spec_action_tag))
2649	return -EINVAL;
2650
2651	ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag);
2652	ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id;
2653	break;
2654	case IB_FLOW_SPEC_ACTION_DROP:
2655	if (kern_spec->drop.size !=
2656	sizeof(struct ib_uverbs_flow_spec_action_drop))
2657	return -EINVAL;
2658
2659	ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop);
2660	break;
2661	case IB_FLOW_SPEC_ACTION_HANDLE:
2662	if (kern_spec->action.size !=
2663	sizeof(struct ib_uverbs_flow_spec_action_handle))
2664	return -EOPNOTSUPP;
2665	ib_spec->action.act = uobj_get_obj_read(flow_action,
2666	UVERBS_OBJECT_FLOW_ACTION,
2667	kern_spec->action.handle,
2668	attrs);
2669	if (!ib_spec->action.act)
2670	return -EINVAL;
2671	ib_spec->action.size =
2672	sizeof(struct ib_flow_spec_action_handle);
2673	flow_resources_add(uflow_res,
2674	IB_FLOW_SPEC_ACTION_HANDLE,
2675	ib_spec->action.act);
2676	uobj_put_obj_read(ib_spec->action.act);
2677	break;
2678	case IB_FLOW_SPEC_ACTION_COUNT:
2679	if (kern_spec->flow_count.size !=
2680	sizeof(struct ib_uverbs_flow_spec_action_count))
2681	return -EINVAL;
2682	ib_spec->flow_count.counters =
2683	uobj_get_obj_read(counters,
2684	UVERBS_OBJECT_COUNTERS,
2685	kern_spec->flow_count.handle,
2686	attrs);
2687	if (!ib_spec->flow_count.counters)
2688	return -EINVAL;
2689	ib_spec->flow_count.size =
2690	sizeof(struct ib_flow_spec_action_count);
2691	flow_resources_add(uflow_res,
2692	IB_FLOW_SPEC_ACTION_COUNT,
2693	ib_spec->flow_count.counters);
2694	uobj_put_obj_read(ib_spec->flow_count.counters);
2695	break;
2696	default:
2697	return -EINVAL;
2698	}
2699	return 0;
2700	}
2701
2702	static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size,
2703	u16 ib_real_filter_sz)
2704	{
2705	/*
2706	* User space filter structures must be 64 bit aligned, otherwise this
2707	* may pass, but we won't handle additional new attributes.
2708	*/
2709
2710	if (kern_filter_size > ib_real_filter_sz) {
2711	if (memchr_inv(p: kern_spec_filter +
2712	ib_real_filter_sz, c: 0,
2713	size: kern_filter_size - ib_real_filter_sz))
2714	return -EINVAL;
2715	return ib_real_filter_sz;
2716	}
2717	return kern_filter_size;
2718	}
2719
2720	int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,
2721	const void *kern_spec_mask,
2722	const void *kern_spec_val,
2723	size_t kern_filter_sz,
2724	union ib_flow_spec *ib_spec)
2725	{
2726	ssize_t actual_filter_sz;
2727	ssize_t ib_filter_sz;
2728
2729	/* User flow spec size must be aligned to 4 bytes */
2730	if (kern_filter_sz != ALIGN(kern_filter_sz, 4))
2731	return -EINVAL;
2732
2733	ib_spec->type = type;
2734
2735	if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))
2736	return -EINVAL;
2737
2738	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
2739	case IB_FLOW_SPEC_ETH:
2740	ib_filter_sz = sizeof(struct ib_flow_eth_filter);
2741	actual_filter_sz = spec_filter_size(kern_spec_filter: kern_spec_mask,
2742	kern_filter_size: kern_filter_sz,
2743	ib_real_filter_sz: ib_filter_sz);
2744	if (actual_filter_sz <= 0)
2745	return -EINVAL;
2746	ib_spec->size = sizeof(struct ib_flow_spec_eth);
2747	memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);
2748	memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);
2749	break;
2750	case IB_FLOW_SPEC_IPV4:
2751	ib_filter_sz = sizeof(struct ib_flow_ipv4_filter);
2752	actual_filter_sz = spec_filter_size(kern_spec_filter: kern_spec_mask,
2753	kern_filter_size: kern_filter_sz,
2754	ib_real_filter_sz: ib_filter_sz);
2755	if (actual_filter_sz <= 0)
2756	return -EINVAL;
2757	ib_spec->size = sizeof(struct ib_flow_spec_ipv4);
2758	memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);
2759	memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);
2760	break;
2761	case IB_FLOW_SPEC_IPV6:
2762	ib_filter_sz = sizeof(struct ib_flow_ipv6_filter);
2763	actual_filter_sz = spec_filter_size(kern_spec_filter: kern_spec_mask,
2764	kern_filter_size: kern_filter_sz,
2765	ib_real_filter_sz: ib_filter_sz);
2766	if (actual_filter_sz <= 0)
2767	return -EINVAL;
2768	ib_spec->size = sizeof(struct ib_flow_spec_ipv6);
2769	memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);
2770	memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);
2771
2772	if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||
2773	(ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))
2774	return -EINVAL;
2775	break;
2776	case IB_FLOW_SPEC_TCP:
2777	case IB_FLOW_SPEC_UDP:
2778	ib_filter_sz = sizeof(struct ib_flow_tcp_udp_filter);
2779	actual_filter_sz = spec_filter_size(kern_spec_filter: kern_spec_mask,
2780	kern_filter_size: kern_filter_sz,
2781	ib_real_filter_sz: ib_filter_sz);
2782	if (actual_filter_sz <= 0)
2783	return -EINVAL;
2784	ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);
2785	memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
2786	memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
2787	break;
2788	case IB_FLOW_SPEC_VXLAN_TUNNEL:
2789	ib_filter_sz = sizeof(struct ib_flow_tunnel_filter);
2790	actual_filter_sz = spec_filter_size(kern_spec_filter: kern_spec_mask,
2791	kern_filter_size: kern_filter_sz,
2792	ib_real_filter_sz: ib_filter_sz);
2793	if (actual_filter_sz <= 0)
2794	return -EINVAL;
2795	ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);
2796	memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);
2797	memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);
2798
2799	if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||
2800	(ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))
2801	return -EINVAL;
2802	break;
2803	case IB_FLOW_SPEC_ESP:
2804	ib_filter_sz = sizeof(struct ib_flow_esp_filter);
2805	actual_filter_sz = spec_filter_size(kern_spec_filter: kern_spec_mask,
2806	kern_filter_size: kern_filter_sz,
2807	ib_real_filter_sz: ib_filter_sz);
2808	if (actual_filter_sz <= 0)
2809	return -EINVAL;
2810	ib_spec->esp.size = sizeof(struct ib_flow_spec_esp);
2811	memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz);
2812	memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz);
2813	break;
2814	case IB_FLOW_SPEC_GRE:
2815	ib_filter_sz = sizeof(struct ib_flow_gre_filter);
2816	actual_filter_sz = spec_filter_size(kern_spec_filter: kern_spec_mask,
2817	kern_filter_size: kern_filter_sz,
2818	ib_real_filter_sz: ib_filter_sz);
2819	if (actual_filter_sz <= 0)
2820	return -EINVAL;
2821	ib_spec->gre.size = sizeof(struct ib_flow_spec_gre);
2822	memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz);
2823	memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz);
2824	break;
2825	case IB_FLOW_SPEC_MPLS:
2826	ib_filter_sz = sizeof(struct ib_flow_mpls_filter);
2827	actual_filter_sz = spec_filter_size(kern_spec_filter: kern_spec_mask,
2828	kern_filter_size: kern_filter_sz,
2829	ib_real_filter_sz: ib_filter_sz);
2830	if (actual_filter_sz <= 0)
2831	return -EINVAL;
2832	ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls);
2833	memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz);
2834	memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz);
2835	break;
2836	default:
2837	return -EINVAL;
2838	}
2839	return 0;
2840	}
2841
2842	static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,
2843	union ib_flow_spec *ib_spec)
2844	{
2845	size_t kern_filter_sz;
2846	void *kern_spec_mask;
2847	void *kern_spec_val;
2848
2849	if (check_sub_overflow((size_t)kern_spec->hdr.size,
2850	sizeof(struct ib_uverbs_flow_spec_hdr),
2851	&kern_filter_sz))
2852	return -EINVAL;
2853
2854	kern_filter_sz /= 2;
2855
2856	kern_spec_val = (void *)kern_spec +
2857	sizeof(struct ib_uverbs_flow_spec_hdr);
2858	kern_spec_mask = kern_spec_val + kern_filter_sz;
2859
2860	return ib_uverbs_kern_spec_to_ib_spec_filter(type: kern_spec->type,
2861	kern_spec_mask,
2862	kern_spec_val,
2863	kern_filter_sz, ib_spec);
2864	}
2865
2866	static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs,
2867	struct ib_uverbs_flow_spec *kern_spec,
2868	union ib_flow_spec *ib_spec,
2869	struct ib_uflow_resources *uflow_res)
2870	{
2871	if (kern_spec->reserved)
2872	return -EINVAL;
2873
2874	if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
2875	return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec,
2876	uflow_res);
2877	else
2878	return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
2879	}
2880
2881	static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
2882	{
2883	struct ib_uverbs_ex_create_wq cmd;
2884	struct ib_uverbs_ex_create_wq_resp resp = {};
2885	struct ib_uwq_object *obj;
2886	int err = 0;
2887	struct ib_cq *cq;
2888	struct ib_pd *pd;
2889	struct ib_wq *wq;
2890	struct ib_wq_init_attr wq_init_attr = {};
2891	struct ib_device *ib_dev;
2892
2893	err = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
2894	if (err)
2895	return err;
2896
2897	if (cmd.comp_mask)
2898	return -EOPNOTSUPP;
2899
2900	obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs,
2901	&ib_dev);
2902	if (IS_ERR(ptr: obj))
2903	return PTR_ERR(ptr: obj);
2904
2905	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs);
2906	if (!pd) {
2907	err = -EINVAL;
2908	goto err_uobj;
2909	}
2910
2911	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
2912	if (!cq) {
2913	err = -EINVAL;
2914	goto err_put_pd;
2915	}
2916
2917	wq_init_attr.cq = cq;
2918	wq_init_attr.max_sge = cmd.max_sge;
2919	wq_init_attr.max_wr = cmd.max_wr;
2920	wq_init_attr.wq_type = cmd.wq_type;
2921	wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
2922	wq_init_attr.create_flags = cmd.create_flags;
2923	INIT_LIST_HEAD(list: &obj->uevent.event_list);
2924	obj->uevent.uobject.user_handle = cmd.user_handle;
2925
2926	wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata);
2927	if (IS_ERR(ptr: wq)) {
2928	err = PTR_ERR(ptr: wq);
2929	goto err_put_cq;
2930	}
2931
2932	wq->uobject = obj;
2933	obj->uevent.uobject.object = wq;
2934	wq->wq_type = wq_init_attr.wq_type;
2935	wq->cq = cq;
2936	wq->pd = pd;
2937	wq->device = pd->device;
2938	atomic_set(v: &wq->usecnt, i: 0);
2939	atomic_inc(v: &pd->usecnt);
2940	atomic_inc(v: &cq->usecnt);
2941	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
2942	if (obj->uevent.event_file)
2943	uverbs_uobject_get(uobject: &obj->uevent.event_file->uobj);
2944
2945	uobj_put_obj_read(pd);
2946	rdma_lookup_put_uobject(uobj: &cq->uobject->uevent.uobject,
2947	mode: UVERBS_LOOKUP_READ);
2948	uobj_finalize_uobj_create(uobj: &obj->uevent.uobject, attrs);
2949
2950	resp.wq_handle = obj->uevent.uobject.id;
2951	resp.max_sge = wq_init_attr.max_sge;
2952	resp.max_wr = wq_init_attr.max_wr;
2953	resp.wqn = wq->wq_num;
2954	resp.response_length = uverbs_response_length(attrs, resp_len: sizeof(resp));
2955	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
2956
2957	err_put_cq:
2958	rdma_lookup_put_uobject(uobj: &cq->uobject->uevent.uobject,
2959	mode: UVERBS_LOOKUP_READ);
2960	err_put_pd:
2961	uobj_put_obj_read(pd);
2962	err_uobj:
2963	uobj_alloc_abort(uobj: &obj->uevent.uobject, attrs);
2964
2965	return err;
2966	}
2967
2968	static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs)
2969	{
2970	struct ib_uverbs_ex_destroy_wq cmd;
2971	struct ib_uverbs_ex_destroy_wq_resp resp = {};
2972	struct ib_uobject *uobj;
2973	struct ib_uwq_object *obj;
2974	int ret;
2975
2976	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
2977	if (ret)
2978	return ret;
2979
2980	if (cmd.comp_mask)
2981	return -EOPNOTSUPP;
2982
2983	resp.response_length = uverbs_response_length(attrs, resp_len: sizeof(resp));
2984	uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
2985	if (IS_ERR(ptr: uobj))
2986	return PTR_ERR(ptr: uobj);
2987
2988	obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);
2989	resp.events_reported = obj->uevent.events_reported;
2990
2991	uobj_put_destroy(uobj);
2992
2993	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
2994	}
2995
2996	static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs)
2997	{
2998	struct ib_uverbs_ex_modify_wq cmd;
2999	struct ib_wq *wq;
3000	struct ib_wq_attr wq_attr = {};
3001	int ret;
3002
3003	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3004	if (ret)
3005	return ret;
3006
3007	if (!cmd.attr_mask)
3008	return -EINVAL;
3009
3010	if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS))
3011	return -EINVAL;
3012
3013	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs);
3014	if (!wq)
3015	return -EINVAL;
3016
3017	if (cmd.attr_mask & IB_WQ_FLAGS) {
3018	wq_attr.flags = cmd.flags;
3019	wq_attr.flags_mask = cmd.flags_mask;
3020	}
3021
3022	if (cmd.attr_mask & IB_WQ_CUR_STATE) {
3023	if (cmd.curr_wq_state > IB_WQS_ERR)
3024	return -EINVAL;
3025
3026	wq_attr.curr_wq_state = cmd.curr_wq_state;
3027	} else {
3028	wq_attr.curr_wq_state = wq->state;
3029	}
3030
3031	if (cmd.attr_mask & IB_WQ_STATE) {
3032	if (cmd.wq_state > IB_WQS_ERR)
3033	return -EINVAL;
3034
3035	wq_attr.wq_state = cmd.wq_state;
3036	} else {
3037	wq_attr.wq_state = wq_attr.curr_wq_state;
3038	}
3039
3040	ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask,
3041	&attrs->driver_udata);
3042	rdma_lookup_put_uobject(uobj: &wq->uobject->uevent.uobject,
3043	mode: UVERBS_LOOKUP_READ);
3044	return ret;
3045	}
3046
3047	static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3048	{
3049	struct ib_uverbs_ex_create_rwq_ind_table cmd;
3050	struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {};
3051	struct ib_uobject *uobj;
3052	int err;
3053	struct ib_rwq_ind_table_init_attr init_attr = {};
3054	struct ib_rwq_ind_table *rwq_ind_tbl;
3055	struct ib_wq **wqs = NULL;
3056	u32 *wqs_handles = NULL;
3057	struct ib_wq *wq = NULL;
3058	int i, num_read_wqs;
3059	u32 num_wq_handles;
3060	struct uverbs_req_iter iter;
3061	struct ib_device *ib_dev;
3062
3063	err = uverbs_request_start(attrs, iter: &iter, req: &cmd, req_len: sizeof(cmd));
3064	if (err)
3065	return err;
3066
3067	if (cmd.comp_mask)
3068	return -EOPNOTSUPP;
3069
3070	if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE)
3071	return -EINVAL;
3072
3073	num_wq_handles = 1 << cmd.log_ind_tbl_size;
3074	wqs_handles = kcalloc(n: num_wq_handles, size: sizeof(*wqs_handles),
3075	GFP_KERNEL);
3076	if (!wqs_handles)
3077	return -ENOMEM;
3078
3079	err = uverbs_request_next(iter: &iter, val: wqs_handles,
3080	len: num_wq_handles * sizeof(__u32));
3081	if (err)
3082	goto err_free;
3083
3084	err = uverbs_request_finish(iter: &iter);
3085	if (err)
3086	goto err_free;
3087
3088	wqs = kcalloc(n: num_wq_handles, size: sizeof(*wqs), GFP_KERNEL);
3089	if (!wqs) {
3090	err = -ENOMEM;
3091	goto err_free;
3092	}
3093
3094	for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3095	num_read_wqs++) {
3096	wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ,
3097	wqs_handles[num_read_wqs], attrs);
3098	if (!wq) {
3099	err = -EINVAL;
3100	goto put_wqs;
3101	}
3102
3103	wqs[num_read_wqs] = wq;
3104	atomic_inc(v: &wqs[num_read_wqs]->usecnt);
3105	}
3106
3107	uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev);
3108	if (IS_ERR(ptr: uobj)) {
3109	err = PTR_ERR(ptr: uobj);
3110	goto put_wqs;
3111	}
3112
3113	rwq_ind_tbl = rdma_zalloc_drv_obj(ib_dev, ib_rwq_ind_table);
3114	if (!rwq_ind_tbl) {
3115	err = -ENOMEM;
3116	goto err_uobj;
3117	}
3118
3119	init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3120	init_attr.ind_tbl = wqs;
3121
3122	rwq_ind_tbl->ind_tbl = wqs;
3123	rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
3124	rwq_ind_tbl->uobject = uobj;
3125	uobj->object = rwq_ind_tbl;
3126	rwq_ind_tbl->device = ib_dev;
3127	atomic_set(v: &rwq_ind_tbl->usecnt, i: 0);
3128
3129	err = ib_dev->ops.create_rwq_ind_table(rwq_ind_tbl, &init_attr,
3130	&attrs->driver_udata);
3131	if (err)
3132	goto err_create;
3133
3134	for (i = 0; i < num_wq_handles; i++)
3135	rdma_lookup_put_uobject(uobj: &wqs[i]->uobject->uevent.uobject,
3136	mode: UVERBS_LOOKUP_READ);
3137	kfree(objp: wqs_handles);
3138	uobj_finalize_uobj_create(uobj, attrs);
3139
3140	resp.ind_tbl_handle = uobj->id;
3141	resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3142	resp.response_length = uverbs_response_length(attrs, resp_len: sizeof(resp));
3143	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
3144
3145	err_create:
3146	kfree(objp: rwq_ind_tbl);
3147	err_uobj:
3148	uobj_alloc_abort(uobj, attrs);
3149	put_wqs:
3150	for (i = 0; i < num_read_wqs; i++) {
3151	rdma_lookup_put_uobject(uobj: &wqs[i]->uobject->uevent.uobject,
3152	mode: UVERBS_LOOKUP_READ);
3153	atomic_dec(v: &wqs[i]->usecnt);
3154	}
3155	err_free:
3156	kfree(objp: wqs_handles);
3157	kfree(objp: wqs);
3158	return err;
3159	}
3160
3161	static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs)
3162	{
3163	struct ib_uverbs_ex_destroy_rwq_ind_table cmd;
3164	int ret;
3165
3166	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3167	if (ret)
3168	return ret;
3169
3170	if (cmd.comp_mask)
3171	return -EOPNOTSUPP;
3172
3173	return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL,
3174	cmd.ind_tbl_handle, attrs);
3175	}
3176
3177	static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
3178	{
3179	struct ib_uverbs_create_flow cmd;
3180	struct ib_uverbs_create_flow_resp resp = {};
3181	struct ib_uobject *uobj;
3182	struct ib_flow *flow_id;
3183	struct ib_uverbs_flow_attr *kern_flow_attr;
3184	struct ib_flow_attr *flow_attr;
3185	struct ib_qp *qp;
3186	struct ib_uflow_resources *uflow_res;
3187	struct ib_uverbs_flow_spec_hdr *kern_spec;
3188	struct uverbs_req_iter iter;
3189	int err;
3190	void *ib_spec;
3191	int i;
3192	struct ib_device *ib_dev;
3193
3194	err = uverbs_request_start(attrs, iter: &iter, req: &cmd, req_len: sizeof(cmd));
3195	if (err)
3196	return err;
3197
3198	if (cmd.comp_mask)
3199	return -EINVAL;
3200
3201	if (!capable(CAP_NET_RAW))
3202	return -EPERM;
3203
3204	if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)
3205	return -EINVAL;
3206
3207	if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
3208	((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||
3209	(cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))
3210	return -EINVAL;
3211
3212	if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3213	return -EINVAL;
3214
3215	if (cmd.flow_attr.size >
3216	(cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3217	return -EINVAL;
3218
3219	if (cmd.flow_attr.reserved[0] ||
3220	cmd.flow_attr.reserved[1])
3221	return -EINVAL;
3222
3223	if (cmd.flow_attr.num_of_specs) {
3224	kern_flow_attr = kmalloc(size: sizeof(*kern_flow_attr) + cmd.flow_attr.size,
3225	GFP_KERNEL);
3226	if (!kern_flow_attr)
3227	return -ENOMEM;
3228
3229	*kern_flow_attr = cmd.flow_attr;
3230	err = uverbs_request_next(iter: &iter, val: &kern_flow_attr->flow_specs,
3231	len: cmd.flow_attr.size);
3232	if (err)
3233	goto err_free_attr;
3234	} else {
3235	kern_flow_attr = &cmd.flow_attr;
3236	}
3237
3238	err = uverbs_request_finish(iter: &iter);
3239	if (err)
3240	goto err_free_attr;
3241
3242	uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev);
3243	if (IS_ERR(ptr: uobj)) {
3244	err = PTR_ERR(ptr: uobj);
3245	goto err_free_attr;
3246	}
3247
3248	if (!rdma_is_port_valid(device: uobj->context->device, port: cmd.flow_attr.port)) {
3249	err = -EINVAL;
3250	goto err_uobj;
3251	}
3252
3253	qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs);
3254	if (!qp) {
3255	err = -EINVAL;
3256	goto err_uobj;
3257	}
3258
3259	if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {
3260	err = -EINVAL;
3261	goto err_put;
3262	}
3263
3264	flow_attr = kzalloc(struct_size(flow_attr, flows,
3265	cmd.flow_attr.num_of_specs), GFP_KERNEL);
3266	if (!flow_attr) {
3267	err = -ENOMEM;
3268	goto err_put;
3269	}
3270	uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs);
3271	if (!uflow_res) {
3272	err = -ENOMEM;
3273	goto err_free_flow_attr;
3274	}
3275
3276	flow_attr->type = kern_flow_attr->type;
3277	flow_attr->priority = kern_flow_attr->priority;
3278	flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3279	flow_attr->port = kern_flow_attr->port;
3280	flow_attr->flags = kern_flow_attr->flags;
3281	flow_attr->size = sizeof(*flow_attr);
3282
3283	kern_spec = kern_flow_attr->flow_specs;
3284	ib_spec = flow_attr + 1;
3285	for (i = 0; i < flow_attr->num_of_specs &&
3286	cmd.flow_attr.size >= sizeof(*kern_spec) &&
3287	cmd.flow_attr.size >= kern_spec->size;
3288	i++) {
3289	err = kern_spec_to_ib_spec(
3290	attrs, kern_spec: (struct ib_uverbs_flow_spec *)kern_spec,
3291	ib_spec, uflow_res);
3292	if (err)
3293	goto err_free;
3294
3295	flow_attr->size +=
3296	((union ib_flow_spec *) ib_spec)->size;
3297	cmd.flow_attr.size -= kern_spec->size;
3298	kern_spec = ((void *)kern_spec) + kern_spec->size;
3299	ib_spec += ((union ib_flow_spec *) ib_spec)->size;
3300	}
3301	if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3302	pr_warn("create flow failed, flow %d: %u bytes left from uverb cmd\n",
3303	i, cmd.flow_attr.size);
3304	err = -EINVAL;
3305	goto err_free;
3306	}
3307
3308	flow_id = qp->device->ops.create_flow(qp, flow_attr,
3309	&attrs->driver_udata);
3310
3311	if (IS_ERR(ptr: flow_id)) {
3312	err = PTR_ERR(ptr: flow_id);
3313	goto err_free;
3314	}
3315
3316	ib_set_flow(uobj, ibflow: flow_id, qp, device: qp->device, uflow_res);
3317
3318	rdma_lookup_put_uobject(uobj: &qp->uobject->uevent.uobject,
3319	mode: UVERBS_LOOKUP_READ);
3320	kfree(objp: flow_attr);
3321
3322	if (cmd.flow_attr.num_of_specs)
3323	kfree(objp: kern_flow_attr);
3324	uobj_finalize_uobj_create(uobj, attrs);
3325
3326	resp.flow_handle = uobj->id;
3327	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
3328
3329	err_free:
3330	ib_uverbs_flow_resources_free(uflow_res);
3331	err_free_flow_attr:
3332	kfree(objp: flow_attr);
3333	err_put:
3334	rdma_lookup_put_uobject(uobj: &qp->uobject->uevent.uobject,
3335	mode: UVERBS_LOOKUP_READ);
3336	err_uobj:
3337	uobj_alloc_abort(uobj, attrs);
3338	err_free_attr:
3339	if (cmd.flow_attr.num_of_specs)
3340	kfree(objp: kern_flow_attr);
3341	return err;
3342	}
3343
3344	static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs)
3345	{
3346	struct ib_uverbs_destroy_flow cmd;
3347	int ret;
3348
3349	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3350	if (ret)
3351	return ret;
3352
3353	if (cmd.comp_mask)
3354	return -EINVAL;
3355
3356	return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs);
3357	}
3358
3359	static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
3360	struct ib_uverbs_create_xsrq *cmd,
3361	struct ib_udata *udata)
3362	{
3363	struct ib_uverbs_create_srq_resp resp = {};
3364	struct ib_usrq_object *obj;
3365	struct ib_pd *pd;
3366	struct ib_srq *srq;
3367	struct ib_srq_init_attr attr;
3368	int ret;
3369	struct ib_uobject *xrcd_uobj;
3370	struct ib_device *ib_dev;
3371
3372	obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs,
3373	&ib_dev);
3374	if (IS_ERR(ptr: obj))
3375	return PTR_ERR(ptr: obj);
3376
3377	if (cmd->srq_type == IB_SRQT_TM)
3378	attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;
3379
3380	if (cmd->srq_type == IB_SRQT_XRC) {
3381	xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle,
3382	attrs);
3383	if (IS_ERR(ptr: xrcd_uobj)) {
3384	ret = -EINVAL;
3385	goto err;
3386	}
3387
3388	attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;
3389	if (!attr.ext.xrc.xrcd) {
3390	ret = -EINVAL;
3391	goto err_put_xrcd;
3392	}
3393
3394	obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3395	atomic_inc(v: &obj->uxrcd->refcnt);
3396	}
3397
3398	if (ib_srq_has_cq(srq_type: cmd->srq_type)) {
3399	attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ,
3400	cmd->cq_handle, attrs);
3401	if (!attr.ext.cq) {
3402	ret = -EINVAL;
3403	goto err_put_xrcd;
3404	}
3405	}
3406
3407	pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs);
3408	if (!pd) {
3409	ret = -EINVAL;
3410	goto err_put_cq;
3411	}
3412
3413	attr.event_handler = ib_uverbs_srq_event_handler;
3414	attr.srq_type = cmd->srq_type;
3415	attr.attr.max_wr = cmd->max_wr;
3416	attr.attr.max_sge = cmd->max_sge;
3417	attr.attr.srq_limit = cmd->srq_limit;
3418
3419	INIT_LIST_HEAD(list: &obj->uevent.event_list);
3420	obj->uevent.uobject.user_handle = cmd->user_handle;
3421
3422	srq = ib_create_srq_user(pd, srq_init_attr: &attr, uobject: obj, udata);
3423	if (IS_ERR(ptr: srq)) {
3424	ret = PTR_ERR(ptr: srq);
3425	goto err_put_pd;
3426	}
3427
3428	obj->uevent.uobject.object = srq;
3429	obj->uevent.uobject.user_handle = cmd->user_handle;
3430	obj->uevent.event_file = READ_ONCE(attrs->ufile->default_async_file);
3431	if (obj->uevent.event_file)
3432	uverbs_uobject_get(uobject: &obj->uevent.event_file->uobj);
3433
3434	if (cmd->srq_type == IB_SRQT_XRC)
3435	resp.srqn = srq->ext.xrc.srq_num;
3436
3437	if (cmd->srq_type == IB_SRQT_XRC)
3438	uobj_put_read(uobj: xrcd_uobj);
3439
3440	if (ib_srq_has_cq(srq_type: cmd->srq_type))
3441	rdma_lookup_put_uobject(uobj: &attr.ext.cq->uobject->uevent.uobject,
3442	mode: UVERBS_LOOKUP_READ);
3443
3444	uobj_put_obj_read(pd);
3445	uobj_finalize_uobj_create(uobj: &obj->uevent.uobject, attrs);
3446
3447	resp.srq_handle = obj->uevent.uobject.id;
3448	resp.max_wr = attr.attr.max_wr;
3449	resp.max_sge = attr.attr.max_sge;
3450	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
3451
3452	err_put_pd:
3453	uobj_put_obj_read(pd);
3454	err_put_cq:
3455	if (ib_srq_has_cq(srq_type: cmd->srq_type))
3456	rdma_lookup_put_uobject(uobj: &attr.ext.cq->uobject->uevent.uobject,
3457	mode: UVERBS_LOOKUP_READ);
3458
3459	err_put_xrcd:
3460	if (cmd->srq_type == IB_SRQT_XRC) {
3461	atomic_dec(v: &obj->uxrcd->refcnt);
3462	uobj_put_read(uobj: xrcd_uobj);
3463	}
3464
3465	err:
3466	uobj_alloc_abort(uobj: &obj->uevent.uobject, attrs);
3467	return ret;
3468	}
3469
3470	static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs)
3471	{
3472	struct ib_uverbs_create_srq cmd;
3473	struct ib_uverbs_create_xsrq xcmd;
3474	int ret;
3475
3476	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3477	if (ret)
3478	return ret;
3479
3480	memset(&xcmd, 0, sizeof(xcmd));
3481	xcmd.response = cmd.response;
3482	xcmd.user_handle = cmd.user_handle;
3483	xcmd.srq_type = IB_SRQT_BASIC;
3484	xcmd.pd_handle = cmd.pd_handle;
3485	xcmd.max_wr = cmd.max_wr;
3486	xcmd.max_sge = cmd.max_sge;
3487	xcmd.srq_limit = cmd.srq_limit;
3488
3489	return __uverbs_create_xsrq(attrs, cmd: &xcmd, udata: &attrs->driver_udata);
3490	}
3491
3492	static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs)
3493	{
3494	struct ib_uverbs_create_xsrq cmd;
3495	int ret;
3496
3497	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3498	if (ret)
3499	return ret;
3500
3501	return __uverbs_create_xsrq(attrs, cmd: &cmd, udata: &attrs->driver_udata);
3502	}
3503
3504	static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs)
3505	{
3506	struct ib_uverbs_modify_srq cmd;
3507	struct ib_srq *srq;
3508	struct ib_srq_attr attr;
3509	int ret;
3510
3511	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3512	if (ret)
3513	return ret;
3514
3515	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3516	if (!srq)
3517	return -EINVAL;
3518
3519	attr.max_wr = cmd.max_wr;
3520	attr.srq_limit = cmd.srq_limit;
3521
3522	ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask,
3523	&attrs->driver_udata);
3524
3525	rdma_lookup_put_uobject(uobj: &srq->uobject->uevent.uobject,
3526	mode: UVERBS_LOOKUP_READ);
3527
3528	return ret;
3529	}
3530
3531	static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs)
3532	{
3533	struct ib_uverbs_query_srq cmd;
3534	struct ib_uverbs_query_srq_resp resp;
3535	struct ib_srq_attr attr;
3536	struct ib_srq *srq;
3537	int ret;
3538
3539	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3540	if (ret)
3541	return ret;
3542
3543	srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3544	if (!srq)
3545	return -EINVAL;
3546
3547	ret = ib_query_srq(srq, srq_attr: &attr);
3548
3549	rdma_lookup_put_uobject(uobj: &srq->uobject->uevent.uobject,
3550	mode: UVERBS_LOOKUP_READ);
3551
3552	if (ret)
3553	return ret;
3554
3555	memset(&resp, 0, sizeof resp);
3556
3557	resp.max_wr = attr.max_wr;
3558	resp.max_sge = attr.max_sge;
3559	resp.srq_limit = attr.srq_limit;
3560
3561	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
3562	}
3563
3564	static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs)
3565	{
3566	struct ib_uverbs_destroy_srq cmd;
3567	struct ib_uverbs_destroy_srq_resp resp;
3568	struct ib_uobject *uobj;
3569	struct ib_uevent_object *obj;
3570	int ret;
3571
3572	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3573	if (ret)
3574	return ret;
3575
3576	uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs);
3577	if (IS_ERR(ptr: uobj))
3578	return PTR_ERR(ptr: uobj);
3579
3580	obj = container_of(uobj, struct ib_uevent_object, uobject);
3581	memset(&resp, 0, sizeof(resp));
3582	resp.events_reported = obj->events_reported;
3583
3584	uobj_put_destroy(uobj);
3585
3586	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
3587	}
3588
3589	static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs)
3590	{
3591	struct ib_uverbs_ex_query_device_resp resp = {};
3592	struct ib_uverbs_ex_query_device cmd;
3593	struct ib_device_attr attr = {0};
3594	struct ib_ucontext *ucontext;
3595	struct ib_device *ib_dev;
3596	int err;
3597
3598	ucontext = ib_uverbs_get_ucontext(attrs);
3599	if (IS_ERR(ptr: ucontext))
3600	return PTR_ERR(ptr: ucontext);
3601	ib_dev = ucontext->device;
3602
3603	err = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3604	if (err)
3605	return err;
3606
3607	if (cmd.comp_mask)
3608	return -EINVAL;
3609
3610	if (cmd.reserved)
3611	return -EINVAL;
3612
3613	err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata);
3614	if (err)
3615	return err;
3616
3617	copy_query_dev_fields(ucontext, resp: &resp.base, attr: &attr);
3618
3619	resp.odp_caps.general_caps = attr.odp_caps.general_caps;
3620	resp.odp_caps.per_transport_caps.rc_odp_caps =
3621	attr.odp_caps.per_transport_caps.rc_odp_caps;
3622	resp.odp_caps.per_transport_caps.uc_odp_caps =
3623	attr.odp_caps.per_transport_caps.uc_odp_caps;
3624	resp.odp_caps.per_transport_caps.ud_odp_caps =
3625	attr.odp_caps.per_transport_caps.ud_odp_caps;
3626	resp.xrc_odp_caps = attr.odp_caps.per_transport_caps.xrc_odp_caps;
3627
3628	resp.timestamp_mask = attr.timestamp_mask;
3629	resp.hca_core_clock = attr.hca_core_clock;
3630	resp.device_cap_flags_ex = attr.device_cap_flags;
3631	resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
3632	resp.rss_caps.max_rwq_indirection_tables =
3633	attr.rss_caps.max_rwq_indirection_tables;
3634	resp.rss_caps.max_rwq_indirection_table_size =
3635	attr.rss_caps.max_rwq_indirection_table_size;
3636	resp.max_wq_type_rq = attr.max_wq_type_rq;
3637	resp.raw_packet_caps = attr.raw_packet_caps;
3638	resp.tm_caps.max_rndv_hdr_size = attr.tm_caps.max_rndv_hdr_size;
3639	resp.tm_caps.max_num_tags = attr.tm_caps.max_num_tags;
3640	resp.tm_caps.max_ops = attr.tm_caps.max_ops;
3641	resp.tm_caps.max_sge = attr.tm_caps.max_sge;
3642	resp.tm_caps.flags = attr.tm_caps.flags;
3643	resp.cq_moderation_caps.max_cq_moderation_count =
3644	attr.cq_caps.max_cq_moderation_count;
3645	resp.cq_moderation_caps.max_cq_moderation_period =
3646	attr.cq_caps.max_cq_moderation_period;
3647	resp.max_dm_size = attr.max_dm_size;
3648	resp.response_length = uverbs_response_length(attrs, resp_len: sizeof(resp));
3649
3650	return uverbs_response(attrs, resp: &resp, resp_len: sizeof(resp));
3651	}
3652
3653	static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs)
3654	{
3655	struct ib_uverbs_ex_modify_cq cmd;
3656	struct ib_cq *cq;
3657	int ret;
3658
3659	ret = uverbs_request(attrs, req: &cmd, req_len: sizeof(cmd));
3660	if (ret)
3661	return ret;
3662
3663	if (!cmd.attr_mask || cmd.reserved)
3664	return -EINVAL;
3665
3666	if (cmd.attr_mask > IB_CQ_MODERATE)
3667	return -EOPNOTSUPP;
3668
3669	cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs);
3670	if (!cq)
3671	return -EINVAL;
3672
3673	ret = rdma_set_cq_moderation(cq, cq_count: cmd.attr.cq_count, cq_period: cmd.attr.cq_period);
3674
3675	rdma_lookup_put_uobject(uobj: &cq->uobject->uevent.uobject,
3676	mode: UVERBS_LOOKUP_READ);
3677	return ret;
3678	}
3679
3680	/*
3681	* Describe the input structs for write(). Some write methods have an input
3682	* only struct, most have an input and output. If the struct has an output then
3683	* the 'response' u64 must be the first field in the request structure.
3684	*
3685	* If udata is present then both the request and response structs have a
3686	* trailing driver_data flex array. In this case the size of the base struct
3687	* cannot be changed.
3688	*/
3689	#define UAPI_DEF_WRITE_IO(req, resp) \
3690	.write.has_resp = 1 + \
3691	BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) + \
3692	BUILD_BUG_ON_ZERO(sizeof_field(req, response) != \
3693	sizeof(u64)), \
3694	.write.req_size = sizeof(req), .write.resp_size = sizeof(resp)
3695
3696	#define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req)
3697
3698	#define UAPI_DEF_WRITE_UDATA_IO(req, resp) \
3699	UAPI_DEF_WRITE_IO(req, resp), \
3700	.write.has_udata = \
3701	1 + \
3702	BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
3703	sizeof(req)) + \
3704	BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) != \
3705	sizeof(resp))
3706
3707	#define UAPI_DEF_WRITE_UDATA_I(req) \
3708	UAPI_DEF_WRITE_I(req), \
3709	.write.has_udata = \
3710	1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \
3711	sizeof(req))
3712
3713	/*
3714	* The _EX versions are for use with WRITE_EX and allow the last struct member
3715	* to be specified. Buffers that do not include that member will be rejected.
3716	*/
3717	#define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member) \
3718	.write.has_resp = 1, \
3719	.write.req_size = offsetofend(req, req_last_member), \
3720	.write.resp_size = offsetofend(resp, resp_last_member)
3721
3722	#define UAPI_DEF_WRITE_I_EX(req, req_last_member) \
3723	.write.req_size = offsetofend(req, req_last_member)
3724
3725	const struct uapi_definition uverbs_def_write_intf[] = {
3726	DECLARE_UVERBS_OBJECT(
3727	UVERBS_OBJECT_AH,
3728	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH,
3729	ib_uverbs_create_ah,
3730	UAPI_DEF_WRITE_UDATA_IO(
3731	struct ib_uverbs_create_ah,
3732	struct ib_uverbs_create_ah_resp)),
3733	DECLARE_UVERBS_WRITE(
3734	IB_USER_VERBS_CMD_DESTROY_AH,
3735	ib_uverbs_destroy_ah,
3736	UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah)),
3737	UAPI_DEF_OBJ_NEEDS_FN(create_user_ah),
3738	UAPI_DEF_OBJ_NEEDS_FN(destroy_ah)),
3739
3740	DECLARE_UVERBS_OBJECT(
3741	UVERBS_OBJECT_COMP_CHANNEL,
3742	DECLARE_UVERBS_WRITE(
3743	IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
3744	ib_uverbs_create_comp_channel,
3745	UAPI_DEF_WRITE_IO(
3746	struct ib_uverbs_create_comp_channel,
3747	struct ib_uverbs_create_comp_channel_resp))),
3748
3749	DECLARE_UVERBS_OBJECT(
3750	UVERBS_OBJECT_CQ,
3751	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ,
3752	ib_uverbs_create_cq,
3753	UAPI_DEF_WRITE_UDATA_IO(
3754	struct ib_uverbs_create_cq,
3755	struct ib_uverbs_create_cq_resp),
3756	UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3757	DECLARE_UVERBS_WRITE(
3758	IB_USER_VERBS_CMD_DESTROY_CQ,
3759	ib_uverbs_destroy_cq,
3760	UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq,
3761	struct ib_uverbs_destroy_cq_resp),
3762	UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)),
3763	DECLARE_UVERBS_WRITE(
3764	IB_USER_VERBS_CMD_POLL_CQ,
3765	ib_uverbs_poll_cq,
3766	UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq,
3767	struct ib_uverbs_poll_cq_resp),
3768	UAPI_DEF_METHOD_NEEDS_FN(poll_cq)),
3769	DECLARE_UVERBS_WRITE(
3770	IB_USER_VERBS_CMD_REQ_NOTIFY_CQ,
3771	ib_uverbs_req_notify_cq,
3772	UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq),
3773	UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)),
3774	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ,
3775	ib_uverbs_resize_cq,
3776	UAPI_DEF_WRITE_UDATA_IO(
3777	struct ib_uverbs_resize_cq,
3778	struct ib_uverbs_resize_cq_resp),
3779	UAPI_DEF_METHOD_NEEDS_FN(resize_cq)),
3780	DECLARE_UVERBS_WRITE_EX(
3781	IB_USER_VERBS_EX_CMD_CREATE_CQ,
3782	ib_uverbs_ex_create_cq,
3783	UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq,
3784	reserved,
3785	struct ib_uverbs_ex_create_cq_resp,
3786	response_length),
3787	UAPI_DEF_METHOD_NEEDS_FN(create_cq)),
3788	DECLARE_UVERBS_WRITE_EX(
3789	IB_USER_VERBS_EX_CMD_MODIFY_CQ,
3790	ib_uverbs_ex_modify_cq,
3791	UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq),
3792	UAPI_DEF_METHOD_NEEDS_FN(modify_cq))),
3793
3794	DECLARE_UVERBS_OBJECT(
3795	UVERBS_OBJECT_DEVICE,
3796	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT,
3797	ib_uverbs_get_context,
3798	UAPI_DEF_WRITE_UDATA_IO(
3799	struct ib_uverbs_get_context,
3800	struct ib_uverbs_get_context_resp)),
3801	DECLARE_UVERBS_WRITE(
3802	IB_USER_VERBS_CMD_QUERY_DEVICE,
3803	ib_uverbs_query_device,
3804	UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device,
3805	struct ib_uverbs_query_device_resp)),
3806	DECLARE_UVERBS_WRITE(
3807	IB_USER_VERBS_CMD_QUERY_PORT,
3808	ib_uverbs_query_port,
3809	UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port,
3810	struct ib_uverbs_query_port_resp),
3811	UAPI_DEF_METHOD_NEEDS_FN(query_port)),
3812	DECLARE_UVERBS_WRITE_EX(
3813	IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
3814	ib_uverbs_ex_query_device,
3815	UAPI_DEF_WRITE_IO_EX(
3816	struct ib_uverbs_ex_query_device,
3817	reserved,
3818	struct ib_uverbs_ex_query_device_resp,
3819	response_length),
3820	UAPI_DEF_METHOD_NEEDS_FN(query_device)),
3821	UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext),
3822	UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)),
3823
3824	DECLARE_UVERBS_OBJECT(
3825	UVERBS_OBJECT_FLOW,
3826	DECLARE_UVERBS_WRITE_EX(
3827	IB_USER_VERBS_EX_CMD_CREATE_FLOW,
3828	ib_uverbs_ex_create_flow,
3829	UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow,
3830	flow_attr,
3831	struct ib_uverbs_create_flow_resp,
3832	flow_handle),
3833	UAPI_DEF_METHOD_NEEDS_FN(create_flow)),
3834	DECLARE_UVERBS_WRITE_EX(
3835	IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
3836	ib_uverbs_ex_destroy_flow,
3837	UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow),
3838	UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))),
3839
3840	DECLARE_UVERBS_OBJECT(
3841	UVERBS_OBJECT_MR,
3842	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR,
3843	ib_uverbs_dereg_mr,
3844	UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr),
3845	UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)),
3846	DECLARE_UVERBS_WRITE(
3847	IB_USER_VERBS_CMD_REG_MR,
3848	ib_uverbs_reg_mr,
3849	UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr,
3850	struct ib_uverbs_reg_mr_resp),
3851	UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)),
3852	DECLARE_UVERBS_WRITE(
3853	IB_USER_VERBS_CMD_REREG_MR,
3854	ib_uverbs_rereg_mr,
3855	UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr,
3856	struct ib_uverbs_rereg_mr_resp),
3857	UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))),
3858
3859	DECLARE_UVERBS_OBJECT(
3860	UVERBS_OBJECT_MW,
3861	DECLARE_UVERBS_WRITE(
3862	IB_USER_VERBS_CMD_ALLOC_MW,
3863	ib_uverbs_alloc_mw,
3864	UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw,
3865	struct ib_uverbs_alloc_mw_resp),
3866	UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)),
3867	DECLARE_UVERBS_WRITE(
3868	IB_USER_VERBS_CMD_DEALLOC_MW,
3869	ib_uverbs_dealloc_mw,
3870	UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw),
3871	UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))),
3872
3873	DECLARE_UVERBS_OBJECT(
3874	UVERBS_OBJECT_PD,
3875	DECLARE_UVERBS_WRITE(
3876	IB_USER_VERBS_CMD_ALLOC_PD,
3877	ib_uverbs_alloc_pd,
3878	UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd,
3879	struct ib_uverbs_alloc_pd_resp),
3880	UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)),
3881	DECLARE_UVERBS_WRITE(
3882	IB_USER_VERBS_CMD_DEALLOC_PD,
3883	ib_uverbs_dealloc_pd,
3884	UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd),
3885	UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))),
3886
3887	DECLARE_UVERBS_OBJECT(
3888	UVERBS_OBJECT_QP,
3889	DECLARE_UVERBS_WRITE(
3890	IB_USER_VERBS_CMD_ATTACH_MCAST,
3891	ib_uverbs_attach_mcast,
3892	UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast),
3893	UAPI_DEF_METHOD_NEEDS_FN(attach_mcast),
3894	UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3895	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP,
3896	ib_uverbs_create_qp,
3897	UAPI_DEF_WRITE_UDATA_IO(
3898	struct ib_uverbs_create_qp,
3899	struct ib_uverbs_create_qp_resp),
3900	UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3901	DECLARE_UVERBS_WRITE(
3902	IB_USER_VERBS_CMD_DESTROY_QP,
3903	ib_uverbs_destroy_qp,
3904	UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp,
3905	struct ib_uverbs_destroy_qp_resp),
3906	UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)),
3907	DECLARE_UVERBS_WRITE(
3908	IB_USER_VERBS_CMD_DETACH_MCAST,
3909	ib_uverbs_detach_mcast,
3910	UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast),
3911	UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)),
3912	DECLARE_UVERBS_WRITE(
3913	IB_USER_VERBS_CMD_MODIFY_QP,
3914	ib_uverbs_modify_qp,
3915	UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp),
3916	UAPI_DEF_METHOD_NEEDS_FN(modify_qp)),
3917	DECLARE_UVERBS_WRITE(
3918	IB_USER_VERBS_CMD_POST_RECV,
3919	ib_uverbs_post_recv,
3920	UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv,
3921	struct ib_uverbs_post_recv_resp),
3922	UAPI_DEF_METHOD_NEEDS_FN(post_recv)),
3923	DECLARE_UVERBS_WRITE(
3924	IB_USER_VERBS_CMD_POST_SEND,
3925	ib_uverbs_post_send,
3926	UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send,
3927	struct ib_uverbs_post_send_resp),
3928	UAPI_DEF_METHOD_NEEDS_FN(post_send)),
3929	DECLARE_UVERBS_WRITE(
3930	IB_USER_VERBS_CMD_QUERY_QP,
3931	ib_uverbs_query_qp,
3932	UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp,
3933	struct ib_uverbs_query_qp_resp),
3934	UAPI_DEF_METHOD_NEEDS_FN(query_qp)),
3935	DECLARE_UVERBS_WRITE_EX(
3936	IB_USER_VERBS_EX_CMD_CREATE_QP,
3937	ib_uverbs_ex_create_qp,
3938	UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp,
3939	comp_mask,
3940	struct ib_uverbs_ex_create_qp_resp,
3941	response_length),
3942	UAPI_DEF_METHOD_NEEDS_FN(create_qp)),
3943	DECLARE_UVERBS_WRITE_EX(
3944	IB_USER_VERBS_EX_CMD_MODIFY_QP,
3945	ib_uverbs_ex_modify_qp,
3946	UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp,
3947	base,
3948	struct ib_uverbs_ex_modify_qp_resp,
3949	response_length),
3950	UAPI_DEF_METHOD_NEEDS_FN(modify_qp))),
3951
3952	DECLARE_UVERBS_OBJECT(
3953	UVERBS_OBJECT_RWQ_IND_TBL,
3954	DECLARE_UVERBS_WRITE_EX(
3955	IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
3956	ib_uverbs_ex_create_rwq_ind_table,
3957	UAPI_DEF_WRITE_IO_EX(
3958	struct ib_uverbs_ex_create_rwq_ind_table,
3959	log_ind_tbl_size,
3960	struct ib_uverbs_ex_create_rwq_ind_table_resp,
3961	ind_tbl_num),
3962	UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)),
3963	DECLARE_UVERBS_WRITE_EX(
3964	IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,
3965	ib_uverbs_ex_destroy_rwq_ind_table,
3966	UAPI_DEF_WRITE_I(
3967	struct ib_uverbs_ex_destroy_rwq_ind_table),
3968	UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))),
3969
3970	DECLARE_UVERBS_OBJECT(
3971	UVERBS_OBJECT_WQ,
3972	DECLARE_UVERBS_WRITE_EX(
3973	IB_USER_VERBS_EX_CMD_CREATE_WQ,
3974	ib_uverbs_ex_create_wq,
3975	UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq,
3976	max_sge,
3977	struct ib_uverbs_ex_create_wq_resp,
3978	wqn),
3979	UAPI_DEF_METHOD_NEEDS_FN(create_wq)),
3980	DECLARE_UVERBS_WRITE_EX(
3981	IB_USER_VERBS_EX_CMD_DESTROY_WQ,
3982	ib_uverbs_ex_destroy_wq,
3983	UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq,
3984	wq_handle,
3985	struct ib_uverbs_ex_destroy_wq_resp,
3986	reserved),
3987	UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)),
3988	DECLARE_UVERBS_WRITE_EX(
3989	IB_USER_VERBS_EX_CMD_MODIFY_WQ,
3990	ib_uverbs_ex_modify_wq,
3991	UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq,
3992	curr_wq_state),
3993	UAPI_DEF_METHOD_NEEDS_FN(modify_wq))),
3994
3995	DECLARE_UVERBS_OBJECT(
3996	UVERBS_OBJECT_SRQ,
3997	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ,
3998	ib_uverbs_create_srq,
3999	UAPI_DEF_WRITE_UDATA_IO(
4000	struct ib_uverbs_create_srq,
4001	struct ib_uverbs_create_srq_resp),
4002	UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4003	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ,
4004	ib_uverbs_create_xsrq,
4005	UAPI_DEF_WRITE_UDATA_IO(
4006	struct ib_uverbs_create_xsrq,
4007	struct ib_uverbs_create_srq_resp),
4008	UAPI_DEF_METHOD_NEEDS_FN(create_srq)),
4009	DECLARE_UVERBS_WRITE(
4010	IB_USER_VERBS_CMD_DESTROY_SRQ,
4011	ib_uverbs_destroy_srq,
4012	UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq,
4013	struct ib_uverbs_destroy_srq_resp),
4014	UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)),
4015	DECLARE_UVERBS_WRITE(
4016	IB_USER_VERBS_CMD_MODIFY_SRQ,
4017	ib_uverbs_modify_srq,
4018	UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq),
4019	UAPI_DEF_METHOD_NEEDS_FN(modify_srq)),
4020	DECLARE_UVERBS_WRITE(
4021	IB_USER_VERBS_CMD_POST_SRQ_RECV,
4022	ib_uverbs_post_srq_recv,
4023	UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv,
4024	struct ib_uverbs_post_srq_recv_resp),
4025	UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)),
4026	DECLARE_UVERBS_WRITE(
4027	IB_USER_VERBS_CMD_QUERY_SRQ,
4028	ib_uverbs_query_srq,
4029	UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq,
4030	struct ib_uverbs_query_srq_resp),
4031	UAPI_DEF_METHOD_NEEDS_FN(query_srq))),
4032
4033	DECLARE_UVERBS_OBJECT(
4034	UVERBS_OBJECT_XRCD,
4035	DECLARE_UVERBS_WRITE(
4036	IB_USER_VERBS_CMD_CLOSE_XRCD,
4037	ib_uverbs_close_xrcd,
4038	UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd)),
4039	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP,
4040	ib_uverbs_open_qp,
4041	UAPI_DEF_WRITE_UDATA_IO(
4042	struct ib_uverbs_open_qp,
4043	struct ib_uverbs_create_qp_resp)),
4044	DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD,
4045	ib_uverbs_open_xrcd,
4046	UAPI_DEF_WRITE_UDATA_IO(
4047	struct ib_uverbs_open_xrcd,
4048	struct ib_uverbs_open_xrcd_resp)),
4049	UAPI_DEF_OBJ_NEEDS_FN(alloc_xrcd),
4050	UAPI_DEF_OBJ_NEEDS_FN(dealloc_xrcd)),
4051
4052	{},
4053	};
4054