1	/*
2	* Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/
32
33	#include <linux/completion.h>
34	#include <linux/file.h>
35	#include <linux/mutex.h>
36	#include <linux/poll.h>
37	#include <linux/sched.h>
38	#include <linux/idr.h>
39	#include <linux/in.h>
40	#include <linux/in6.h>
41	#include <linux/miscdevice.h>
42	#include <linux/slab.h>
43	#include <linux/sysctl.h>
44	#include <linux/module.h>
45	#include <linux/nsproxy.h>
46
47	#include <linux/nospec.h>
48
49	#include <rdma/rdma_user_cm.h>
50	#include <rdma/ib_marshall.h>
51	#include <rdma/rdma_cm.h>
52	#include <rdma/rdma_cm_ib.h>
53	#include <rdma/ib_addr.h>
54	#include <rdma/ib.h>
55	#include <rdma/ib_cm.h>
56	#include <rdma/rdma_netlink.h>
57	#include "core_priv.h"
58
59	MODULE_AUTHOR("Sean Hefty");
60	MODULE_DESCRIPTION("RDMA Userspace Connection Manager Access");
61	MODULE_LICENSE("Dual BSD/GPL");
62
63	static unsigned int max_backlog = 1024;
64
65	static struct ctl_table_header *ucma_ctl_table_hdr;
66	static struct ctl_table ucma_ctl_table[] = {
67	{
68	.procname = "max_backlog",
69	.data = &max_backlog,
70	.maxlen = sizeof max_backlog,
71	.mode = 0644,
72	.proc_handler = proc_dointvec_minmax,
73	.extra1 = SYSCTL_ZERO,
74	.extra2 = SYSCTL_INT_MAX,
75	},
76	};
77
78	struct ucma_file {
79	struct mutex mut;
80	struct file *filp;
81	struct list_head ctx_list;
82	struct list_head event_list;
83	wait_queue_head_t poll_wait;
84	};
85
86	struct ucma_context {
87	u32 id;
88	struct completion comp;
89	refcount_t ref;
90	int events_reported;
91	atomic_t backlog;
92
93	struct ucma_file *file;
94	struct rdma_cm_id *cm_id;
95	struct mutex mutex;
96	u64 uid;
97
98	struct list_head list;
99	struct list_head mc_list;
100	struct work_struct close_work;
101	};
102
103	struct ucma_multicast {
104	struct ucma_context *ctx;
105	u32 id;
106	int events_reported;
107
108	u64 uid;
109	u8 join_state;
110	struct list_head list;
111	struct sockaddr_storage addr;
112	};
113
114	struct ucma_event {
115	struct ucma_context *ctx;
116	struct ucma_context *conn_req_ctx;
117	struct ucma_multicast *mc;
118	struct list_head list;
119	struct rdma_ucm_event_resp resp;
120	};
121
122	static DEFINE_XARRAY_ALLOC(ctx_table);
123	static DEFINE_XARRAY_ALLOC(multicast_table);
124
125	static const struct file_operations ucma_fops;
126	static int ucma_destroy_private_ctx(struct ucma_context *ctx);
127
128	static inline struct ucma_context *_ucma_find_context(int id,
129	struct ucma_file *file)
130	{
131	struct ucma_context *ctx;
132
133	ctx = xa_load(&ctx_table, index: id);
134	if (!ctx)
135	ctx = ERR_PTR(error: -ENOENT);
136	else if (ctx->file != file)
137	ctx = ERR_PTR(error: -EINVAL);
138	return ctx;
139	}
140
141	static struct ucma_context *ucma_get_ctx(struct ucma_file *file, int id)
142	{
143	struct ucma_context *ctx;
144
145	xa_lock(&ctx_table);
146	ctx = _ucma_find_context(id, file);
147	if (!IS_ERR(ptr: ctx))
148	if (!refcount_inc_not_zero(r: &ctx->ref))
149	ctx = ERR_PTR(error: -ENXIO);
150	xa_unlock(&ctx_table);
151	return ctx;
152	}
153
154	static void ucma_put_ctx(struct ucma_context *ctx)
155	{
156	if (refcount_dec_and_test(r: &ctx->ref))
157	complete(&ctx->comp);
158	}
159
160	/*
161	* Same as ucm_get_ctx but requires that ->cm_id->device is valid, eg that the
162	* CM_ID is bound.
163	*/
164	static struct ucma_context *ucma_get_ctx_dev(struct ucma_file *file, int id)
165	{
166	struct ucma_context *ctx = ucma_get_ctx(file, id);
167
168	if (IS_ERR(ptr: ctx))
169	return ctx;
170	if (!ctx->cm_id->device) {
171	ucma_put_ctx(ctx);
172	return ERR_PTR(error: -EINVAL);
173	}
174	return ctx;
175	}
176
177	static void ucma_close_id(struct work_struct *work)
178	{
179	struct ucma_context *ctx = container_of(work, struct ucma_context, close_work);
180
181	/* once all inflight tasks are finished, we close all underlying
182	* resources. The context is still alive till its explicit destryoing
183	* by its creator. This puts back the xarray's reference.
184	*/
185	ucma_put_ctx(ctx);
186	wait_for_completion(&ctx->comp);
187	/* No new events will be generated after destroying the id. */
188	rdma_destroy_id(id: ctx->cm_id);
189
190	/* Reading the cm_id without holding a positive ref is not allowed */
191	ctx->cm_id = NULL;
192	}
193
194	static struct ucma_context *ucma_alloc_ctx(struct ucma_file *file)
195	{
196	struct ucma_context *ctx;
197
198	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
199	if (!ctx)
200	return NULL;
201
202	INIT_WORK(&ctx->close_work, ucma_close_id);
203	init_completion(x: &ctx->comp);
204	INIT_LIST_HEAD(list: &ctx->mc_list);
205	/* So list_del() will work if we don't do ucma_finish_ctx() */
206	INIT_LIST_HEAD(list: &ctx->list);
207	ctx->file = file;
208	mutex_init(&ctx->mutex);
209
210	if (xa_alloc(xa: &ctx_table, id: &ctx->id, NULL, xa_limit_32b, GFP_KERNEL)) {
211	kfree(objp: ctx);
212	return NULL;
213	}
214	return ctx;
215	}
216
217	static void ucma_set_ctx_cm_id(struct ucma_context *ctx,
218	struct rdma_cm_id *cm_id)
219	{
220	refcount_set(r: &ctx->ref, n: 1);
221	ctx->cm_id = cm_id;
222	}
223
224	static void ucma_finish_ctx(struct ucma_context *ctx)
225	{
226	lockdep_assert_held(&ctx->file->mut);
227	list_add_tail(new: &ctx->list, head: &ctx->file->ctx_list);
228	xa_store(&ctx_table, index: ctx->id, entry: ctx, GFP_KERNEL);
229	}
230
231	static void ucma_copy_conn_event(struct rdma_ucm_conn_param *dst,
232	struct rdma_conn_param *src)
233	{
234	if (src->private_data_len)
235	memcpy(dst->private_data, src->private_data,
236	src->private_data_len);
237	dst->private_data_len = src->private_data_len;
238	dst->responder_resources = src->responder_resources;
239	dst->initiator_depth = src->initiator_depth;
240	dst->flow_control = src->flow_control;
241	dst->retry_count = src->retry_count;
242	dst->rnr_retry_count = src->rnr_retry_count;
243	dst->srq = src->srq;
244	dst->qp_num = src->qp_num;
245	}
246
247	static void ucma_copy_ud_event(struct ib_device *device,
248	struct rdma_ucm_ud_param *dst,
249	struct rdma_ud_param *src)
250	{
251	if (src->private_data_len)
252	memcpy(dst->private_data, src->private_data,
253	src->private_data_len);
254	dst->private_data_len = src->private_data_len;
255	ib_copy_ah_attr_to_user(device, dst: &dst->ah_attr, src: &src->ah_attr);
256	dst->qp_num = src->qp_num;
257	dst->qkey = src->qkey;
258	}
259
260	static struct ucma_event *ucma_create_uevent(struct ucma_context *ctx,
261	struct rdma_cm_event *event)
262	{
263	struct ucma_event *uevent;
264
265	uevent = kzalloc(sizeof(*uevent), GFP_KERNEL);
266	if (!uevent)
267	return NULL;
268
269	uevent->ctx = ctx;
270	switch (event->event) {
271	case RDMA_CM_EVENT_MULTICAST_JOIN:
272	case RDMA_CM_EVENT_MULTICAST_ERROR:
273	uevent->mc = (struct ucma_multicast *)
274	event->param.ud.private_data;
275	uevent->resp.uid = uevent->mc->uid;
276	uevent->resp.id = uevent->mc->id;
277	break;
278	default:
279	uevent->resp.uid = ctx->uid;
280	uevent->resp.id = ctx->id;
281	break;
282	}
283	uevent->resp.event = event->event;
284	uevent->resp.status = event->status;
285
286	if (event->event == RDMA_CM_EVENT_ADDRINFO_RESOLVED)
287	goto out;
288
289	if (ctx->cm_id->qp_type == IB_QPT_UD)
290	ucma_copy_ud_event(device: ctx->cm_id->device, dst: &uevent->resp.param.ud,
291	src: &event->param.ud);
292	else
293	ucma_copy_conn_event(dst: &uevent->resp.param.conn,
294	src: &event->param.conn);
295
296	out:
297	uevent->resp.ece.vendor_id = event->ece.vendor_id;
298	uevent->resp.ece.attr_mod = event->ece.attr_mod;
299	return uevent;
300	}
301
302	static int ucma_connect_event_handler(struct rdma_cm_id *cm_id,
303	struct rdma_cm_event *event)
304	{
305	struct ucma_context *listen_ctx = cm_id->context;
306	struct ucma_context *ctx;
307	struct ucma_event *uevent;
308
309	if (!atomic_add_unless(v: &listen_ctx->backlog, a: -1, u: 0))
310	return -ENOMEM;
311	ctx = ucma_alloc_ctx(file: listen_ctx->file);
312	if (!ctx)
313	goto err_backlog;
314	ucma_set_ctx_cm_id(ctx, cm_id);
315
316	uevent = ucma_create_uevent(ctx: listen_ctx, event);
317	if (!uevent)
318	goto err_alloc;
319	uevent->conn_req_ctx = ctx;
320	uevent->resp.id = ctx->id;
321
322	ctx->cm_id->context = ctx;
323
324	mutex_lock(&ctx->file->mut);
325	ucma_finish_ctx(ctx);
326	list_add_tail(new: &uevent->list, head: &ctx->file->event_list);
327	mutex_unlock(lock: &ctx->file->mut);
328	wake_up_interruptible(&ctx->file->poll_wait);
329	return 0;
330
331	err_alloc:
332	ucma_destroy_private_ctx(ctx);
333	err_backlog:
334	atomic_inc(v: &listen_ctx->backlog);
335	/* Returning error causes the new ID to be destroyed */
336	return -ENOMEM;
337	}
338
339	static int ucma_event_handler(struct rdma_cm_id *cm_id,
340	struct rdma_cm_event *event)
341	{
342	struct ucma_event *uevent;
343	struct ucma_context *ctx = cm_id->context;
344
345	if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST)
346	return ucma_connect_event_handler(cm_id, event);
347
348	/*
349	* We ignore events for new connections until userspace has set their
350	* context. This can only happen if an error occurs on a new connection
351	* before the user accepts it. This is okay, since the accept will just
352	* fail later. However, we do need to release the underlying HW
353	* resources in case of a device removal event.
354	*/
355	if (ctx->uid) {
356	uevent = ucma_create_uevent(ctx, event);
357	if (!uevent)
358	return 0;
359
360	mutex_lock(&ctx->file->mut);
361	list_add_tail(new: &uevent->list, head: &ctx->file->event_list);
362	mutex_unlock(lock: &ctx->file->mut);
363	wake_up_interruptible(&ctx->file->poll_wait);
364	}
365
366	if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) {
367	xa_lock(&ctx_table);
368	if (xa_load(&ctx_table, index: ctx->id) == ctx)
369	queue_work(wq: system_dfl_wq, work: &ctx->close_work);
370	xa_unlock(&ctx_table);
371	}
372	return 0;
373	}
374
375	static ssize_t ucma_get_event(struct ucma_file *file, const char __user *inbuf,
376	int in_len, int out_len)
377	{
378	struct rdma_ucm_get_event cmd;
379	struct ucma_event *uevent;
380
381	/*
382	* Old 32 bit user space does not send the 4 byte padding in the
383	* reserved field. We don't care, allow it to keep working.
384	*/
385	if (out_len < sizeof(uevent->resp) - sizeof(uevent->resp.reserved) -
386	sizeof(uevent->resp.ece))
387	return -ENOSPC;
388
389	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
390	return -EFAULT;
391
392	mutex_lock(&file->mut);
393	while (list_empty(head: &file->event_list)) {
394	mutex_unlock(lock: &file->mut);
395
396	if (file->filp->f_flags & O_NONBLOCK)
397	return -EAGAIN;
398
399	if (wait_event_interruptible(file->poll_wait,
400	!list_empty(&file->event_list)))
401	return -ERESTARTSYS;
402
403	mutex_lock(&file->mut);
404	}
405
406	uevent = list_first_entry(&file->event_list, struct ucma_event, list);
407
408	if (copy_to_user(u64_to_user_ptr(cmd.response),
409	from: &uevent->resp,
410	min_t(size_t, out_len, sizeof(uevent->resp)))) {
411	mutex_unlock(lock: &file->mut);
412	return -EFAULT;
413	}
414
415	list_del(entry: &uevent->list);
416	uevent->ctx->events_reported++;
417	if (uevent->mc)
418	uevent->mc->events_reported++;
419	if (uevent->resp.event == RDMA_CM_EVENT_CONNECT_REQUEST)
420	atomic_inc(v: &uevent->ctx->backlog);
421	mutex_unlock(lock: &file->mut);
422
423	kfree(objp: uevent);
424	return 0;
425	}
426
427	static int ucma_get_qp_type(struct rdma_ucm_create_id *cmd, enum ib_qp_type *qp_type)
428	{
429	switch (cmd->ps) {
430	case RDMA_PS_TCP:
431	*qp_type = IB_QPT_RC;
432	return 0;
433	case RDMA_PS_UDP:
434	case RDMA_PS_IPOIB:
435	*qp_type = IB_QPT_UD;
436	return 0;
437	case RDMA_PS_IB:
438	*qp_type = cmd->qp_type;
439	return 0;
440	default:
441	return -EINVAL;
442	}
443	}
444
445	static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf,
446	int in_len, int out_len)
447	{
448	struct rdma_ucm_create_id cmd;
449	struct rdma_ucm_create_id_resp resp;
450	struct ucma_context *ctx;
451	struct rdma_cm_id *cm_id;
452	enum ib_qp_type qp_type;
453	int ret;
454
455	if (out_len < sizeof(resp))
456	return -ENOSPC;
457
458	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
459	return -EFAULT;
460
461	ret = ucma_get_qp_type(cmd: &cmd, qp_type: &qp_type);
462	if (ret)
463	return ret;
464
465	ctx = ucma_alloc_ctx(file);
466	if (!ctx)
467	return -ENOMEM;
468
469	ctx->uid = cmd.uid;
470	cm_id = rdma_create_user_id(event_handler: ucma_event_handler, context: ctx, ps: cmd.ps, qp_type);
471	if (IS_ERR(ptr: cm_id)) {
472	ret = PTR_ERR(ptr: cm_id);
473	goto err1;
474	}
475	ucma_set_ctx_cm_id(ctx, cm_id);
476
477	resp.id = ctx->id;
478	if (copy_to_user(u64_to_user_ptr(cmd.response),
479	from: &resp, n: sizeof(resp))) {
480	ret = -EFAULT;
481	goto err1;
482	}
483
484	mutex_lock(&file->mut);
485	ucma_finish_ctx(ctx);
486	mutex_unlock(lock: &file->mut);
487	return 0;
488
489	err1:
490	ucma_destroy_private_ctx(ctx);
491	return ret;
492	}
493
494	static void ucma_cleanup_multicast(struct ucma_context *ctx)
495	{
496	struct ucma_multicast *mc, *tmp;
497
498	xa_lock(&multicast_table);
499	list_for_each_entry_safe(mc, tmp, &ctx->mc_list, list) {
500	list_del(entry: &mc->list);
501	/*
502	* At this point mc->ctx->ref is 0 so the mc cannot leave the
503	* lock on the reader and this is enough serialization
504	*/
505	__xa_erase(&multicast_table, index: mc->id);
506	kfree(objp: mc);
507	}
508	xa_unlock(&multicast_table);
509	}
510
511	static void ucma_cleanup_mc_events(struct ucma_multicast *mc)
512	{
513	struct ucma_event *uevent, *tmp;
514
515	rdma_lock_handler(id: mc->ctx->cm_id);
516	mutex_lock(&mc->ctx->file->mut);
517	list_for_each_entry_safe(uevent, tmp, &mc->ctx->file->event_list, list) {
518	if (uevent->mc != mc)
519	continue;
520
521	list_del(entry: &uevent->list);
522	kfree(objp: uevent);
523	}
524	mutex_unlock(lock: &mc->ctx->file->mut);
525	rdma_unlock_handler(id: mc->ctx->cm_id);
526	}
527
528	static int ucma_cleanup_ctx_events(struct ucma_context *ctx)
529	{
530	int events_reported;
531	struct ucma_event *uevent, *tmp;
532	LIST_HEAD(list);
533
534	/* Cleanup events not yet reported to the user.*/
535	mutex_lock(&ctx->file->mut);
536	list_for_each_entry_safe(uevent, tmp, &ctx->file->event_list, list) {
537	if (uevent->ctx != ctx)
538	continue;
539
540	if (uevent->resp.event == RDMA_CM_EVENT_CONNECT_REQUEST &&
541	xa_cmpxchg(xa: &ctx_table, index: uevent->conn_req_ctx->id,
542	old: uevent->conn_req_ctx, XA_ZERO_ENTRY,
543	GFP_KERNEL) == uevent->conn_req_ctx) {
544	list_move_tail(list: &uevent->list, head: &list);
545	continue;
546	}
547	list_del(entry: &uevent->list);
548	kfree(objp: uevent);
549	}
550	list_del(entry: &ctx->list);
551	events_reported = ctx->events_reported;
552	mutex_unlock(lock: &ctx->file->mut);
553
554	/*
555	* If this was a listening ID then any connections spawned from it that
556	* have not been delivered to userspace are cleaned up too. Must be done
557	* outside any locks.
558	*/
559	list_for_each_entry_safe(uevent, tmp, &list, list) {
560	ucma_destroy_private_ctx(ctx: uevent->conn_req_ctx);
561	kfree(objp: uevent);
562	}
563	return events_reported;
564	}
565
566	/*
567	* When this is called the xarray must have a XA_ZERO_ENTRY in the ctx->id (ie
568	* the ctx is not public to the user). This either because:
569	* - ucma_finish_ctx() hasn't been called
570	* - xa_cmpxchg() succeed to remove the entry (only one thread can succeed)
571	*/
572	static int ucma_destroy_private_ctx(struct ucma_context *ctx)
573	{
574	int events_reported;
575
576	/*
577	* Destroy the underlying cm_id. New work queuing is prevented now by
578	* the removal from the xarray. Once the work is cancled ref will either
579	* be 0 because the work ran to completion and consumed the ref from the
580	* xarray, or it will be positive because we still have the ref from the
581	* xarray. This can also be 0 in cases where cm_id was never set
582	*/
583	cancel_work_sync(work: &ctx->close_work);
584	if (refcount_read(r: &ctx->ref))
585	ucma_close_id(work: &ctx->close_work);
586
587	events_reported = ucma_cleanup_ctx_events(ctx);
588	ucma_cleanup_multicast(ctx);
589
590	WARN_ON(xa_cmpxchg(&ctx_table, ctx->id, XA_ZERO_ENTRY, NULL,
591	GFP_KERNEL) != NULL);
592	mutex_destroy(lock: &ctx->mutex);
593	kfree(objp: ctx);
594	return events_reported;
595	}
596
597	static ssize_t ucma_destroy_id(struct ucma_file *file, const char __user *inbuf,
598	int in_len, int out_len)
599	{
600	struct rdma_ucm_destroy_id cmd;
601	struct rdma_ucm_destroy_id_resp resp;
602	struct ucma_context *ctx;
603	int ret = 0;
604
605	if (out_len < sizeof(resp))
606	return -ENOSPC;
607
608	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
609	return -EFAULT;
610
611	xa_lock(&ctx_table);
612	ctx = _ucma_find_context(id: cmd.id, file);
613	if (!IS_ERR(ptr: ctx)) {
614	if (__xa_cmpxchg(&ctx_table, index: ctx->id, old: ctx, XA_ZERO_ENTRY,
615	GFP_KERNEL) != ctx)
616	ctx = ERR_PTR(error: -ENOENT);
617	}
618	xa_unlock(&ctx_table);
619
620	if (IS_ERR(ptr: ctx))
621	return PTR_ERR(ptr: ctx);
622
623	resp.events_reported = ucma_destroy_private_ctx(ctx);
624	if (copy_to_user(u64_to_user_ptr(cmd.response),
625	from: &resp, n: sizeof(resp)))
626	ret = -EFAULT;
627
628	return ret;
629	}
630
631	static ssize_t ucma_bind_ip(struct ucma_file *file, const char __user *inbuf,
632	int in_len, int out_len)
633	{
634	struct rdma_ucm_bind_ip cmd;
635	struct ucma_context *ctx;
636	int ret;
637
638	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
639	return -EFAULT;
640
641	if (!rdma_addr_size_in6(addr: &cmd.addr))
642	return -EINVAL;
643
644	ctx = ucma_get_ctx(file, id: cmd.id);
645	if (IS_ERR(ptr: ctx))
646	return PTR_ERR(ptr: ctx);
647
648	mutex_lock(&ctx->mutex);
649	ret = rdma_bind_addr(id: ctx->cm_id, addr: (struct sockaddr *) &cmd.addr);
650	mutex_unlock(lock: &ctx->mutex);
651
652	ucma_put_ctx(ctx);
653	return ret;
654	}
655
656	static ssize_t ucma_bind(struct ucma_file *file, const char __user *inbuf,
657	int in_len, int out_len)
658	{
659	struct rdma_ucm_bind cmd;
660	struct ucma_context *ctx;
661	int ret;
662
663	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
664	return -EFAULT;
665
666	if (cmd.reserved || !cmd.addr_size ||
667	cmd.addr_size != rdma_addr_size_kss(addr: &cmd.addr))
668	return -EINVAL;
669
670	ctx = ucma_get_ctx(file, id: cmd.id);
671	if (IS_ERR(ptr: ctx))
672	return PTR_ERR(ptr: ctx);
673
674	mutex_lock(&ctx->mutex);
675	ret = rdma_bind_addr(id: ctx->cm_id, addr: (struct sockaddr *) &cmd.addr);
676	mutex_unlock(lock: &ctx->mutex);
677	ucma_put_ctx(ctx);
678	return ret;
679	}
680
681	static ssize_t ucma_resolve_ip(struct ucma_file *file,
682	const char __user *inbuf,
683	int in_len, int out_len)
684	{
685	struct rdma_ucm_resolve_ip cmd;
686	struct ucma_context *ctx;
687	int ret;
688
689	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
690	return -EFAULT;
691
692	if ((cmd.src_addr.sin6_family && !rdma_addr_size_in6(addr: &cmd.src_addr)) ||
693	!rdma_addr_size_in6(addr: &cmd.dst_addr))
694	return -EINVAL;
695
696	ctx = ucma_get_ctx(file, id: cmd.id);
697	if (IS_ERR(ptr: ctx))
698	return PTR_ERR(ptr: ctx);
699
700	mutex_lock(&ctx->mutex);
701	ret = rdma_resolve_addr(id: ctx->cm_id, src_addr: (struct sockaddr *) &cmd.src_addr,
702	dst_addr: (struct sockaddr *) &cmd.dst_addr, timeout_ms: cmd.timeout_ms);
703	mutex_unlock(lock: &ctx->mutex);
704	ucma_put_ctx(ctx);
705	return ret;
706	}
707
708	static ssize_t ucma_resolve_addr(struct ucma_file *file,
709	const char __user *inbuf,
710	int in_len, int out_len)
711	{
712	struct rdma_ucm_resolve_addr cmd;
713	struct ucma_context *ctx;
714	int ret;
715
716	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
717	return -EFAULT;
718
719	if (cmd.reserved ||
720	(cmd.src_size && (cmd.src_size != rdma_addr_size_kss(addr: &cmd.src_addr))) ||
721	!cmd.dst_size || (cmd.dst_size != rdma_addr_size_kss(addr: &cmd.dst_addr)))
722	return -EINVAL;
723
724	ctx = ucma_get_ctx(file, id: cmd.id);
725	if (IS_ERR(ptr: ctx))
726	return PTR_ERR(ptr: ctx);
727
728	mutex_lock(&ctx->mutex);
729	ret = rdma_resolve_addr(id: ctx->cm_id, src_addr: (struct sockaddr *) &cmd.src_addr,
730	dst_addr: (struct sockaddr *) &cmd.dst_addr, timeout_ms: cmd.timeout_ms);
731	mutex_unlock(lock: &ctx->mutex);
732	ucma_put_ctx(ctx);
733	return ret;
734	}
735
736	static ssize_t ucma_resolve_ib_service(struct ucma_file *file,
737	const char __user *inbuf, int in_len,
738	int out_len)
739	{
740	struct rdma_ucm_resolve_ib_service cmd;
741	struct ucma_context *ctx;
742	int ret;
743
744	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
745	return -EFAULT;
746
747	ctx = ucma_get_ctx(file, id: cmd.id);
748	if (IS_ERR(ptr: ctx))
749	return PTR_ERR(ptr: ctx);
750
751	mutex_lock(&ctx->mutex);
752	ret = rdma_resolve_ib_service(id: ctx->cm_id, ibs: &cmd.ibs);
753	mutex_unlock(lock: &ctx->mutex);
754	ucma_put_ctx(ctx);
755	return ret;
756	}
757
758	static ssize_t ucma_resolve_route(struct ucma_file *file,
759	const char __user *inbuf,
760	int in_len, int out_len)
761	{
762	struct rdma_ucm_resolve_route cmd;
763	struct ucma_context *ctx;
764	int ret;
765
766	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
767	return -EFAULT;
768
769	ctx = ucma_get_ctx_dev(file, id: cmd.id);
770	if (IS_ERR(ptr: ctx))
771	return PTR_ERR(ptr: ctx);
772
773	mutex_lock(&ctx->mutex);
774	ret = rdma_resolve_route(id: ctx->cm_id, timeout_ms: cmd.timeout_ms);
775	mutex_unlock(lock: &ctx->mutex);
776	ucma_put_ctx(ctx);
777	return ret;
778	}
779
780	static void ucma_copy_ib_route(struct rdma_ucm_query_route_resp *resp,
781	struct rdma_route *route)
782	{
783	struct rdma_dev_addr *dev_addr;
784
785	resp->num_paths = route->num_pri_alt_paths;
786	switch (route->num_pri_alt_paths) {
787	case 0:
788	dev_addr = &route->addr.dev_addr;
789	rdma_addr_get_dgid(dev_addr,
790	gid: (union ib_gid *) &resp->ib_route[0].dgid);
791	rdma_addr_get_sgid(dev_addr,
792	gid: (union ib_gid *) &resp->ib_route[0].sgid);
793	resp->ib_route[0].pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
794	break;
795	case 2:
796	ib_copy_path_rec_to_user(dst: &resp->ib_route[1],
797	src: &route->path_rec[1]);
798	fallthrough;
799	case 1:
800	ib_copy_path_rec_to_user(dst: &resp->ib_route[0],
801	src: &route->path_rec[0]);
802	break;
803	default:
804	break;
805	}
806	}
807
808	static void ucma_copy_iboe_route(struct rdma_ucm_query_route_resp *resp,
809	struct rdma_route *route)
810	{
811
812	resp->num_paths = route->num_pri_alt_paths;
813	switch (route->num_pri_alt_paths) {
814	case 0:
815	rdma_ip2gid(addr: (struct sockaddr *)&route->addr.dst_addr,
816	gid: (union ib_gid *)&resp->ib_route[0].dgid);
817	rdma_ip2gid(addr: (struct sockaddr *)&route->addr.src_addr,
818	gid: (union ib_gid *)&resp->ib_route[0].sgid);
819	resp->ib_route[0].pkey = cpu_to_be16(0xffff);
820	break;
821	case 2:
822	ib_copy_path_rec_to_user(dst: &resp->ib_route[1],
823	src: &route->path_rec[1]);
824	fallthrough;
825	case 1:
826	ib_copy_path_rec_to_user(dst: &resp->ib_route[0],
827	src: &route->path_rec[0]);
828	break;
829	default:
830	break;
831	}
832	}
833
834	static void ucma_copy_iw_route(struct rdma_ucm_query_route_resp *resp,
835	struct rdma_route *route)
836	{
837	struct rdma_dev_addr *dev_addr;
838
839	dev_addr = &route->addr.dev_addr;
840	rdma_addr_get_dgid(dev_addr, gid: (union ib_gid *) &resp->ib_route[0].dgid);
841	rdma_addr_get_sgid(dev_addr, gid: (union ib_gid *) &resp->ib_route[0].sgid);
842	}
843
844	static ssize_t ucma_query_route(struct ucma_file *file,
845	const char __user *inbuf,
846	int in_len, int out_len)
847	{
848	struct rdma_ucm_query cmd;
849	struct rdma_ucm_query_route_resp resp;
850	struct ucma_context *ctx;
851	struct sockaddr *addr;
852	int ret = 0;
853
854	if (out_len < offsetof(struct rdma_ucm_query_route_resp, ibdev_index))
855	return -ENOSPC;
856
857	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
858	return -EFAULT;
859
860	ctx = ucma_get_ctx(file, id: cmd.id);
861	if (IS_ERR(ptr: ctx))
862	return PTR_ERR(ptr: ctx);
863
864	mutex_lock(&ctx->mutex);
865	memset(&resp, 0, sizeof resp);
866	addr = (struct sockaddr *) &ctx->cm_id->route.addr.src_addr;
867	memcpy(&resp.src_addr, addr, addr->sa_family == AF_INET ?
868	sizeof(struct sockaddr_in) :
869	sizeof(struct sockaddr_in6));
870	addr = (struct sockaddr *) &ctx->cm_id->route.addr.dst_addr;
871	memcpy(&resp.dst_addr, addr, addr->sa_family == AF_INET ?
872	sizeof(struct sockaddr_in) :
873	sizeof(struct sockaddr_in6));
874	if (!ctx->cm_id->device)
875	goto out;
876
877	resp.node_guid = (__force __u64) ctx->cm_id->device->node_guid;
878	resp.ibdev_index = ctx->cm_id->device->index;
879	resp.port_num = ctx->cm_id->port_num;
880
881	if (rdma_cap_ib_sa(device: ctx->cm_id->device, port_num: ctx->cm_id->port_num))
882	ucma_copy_ib_route(resp: &resp, route: &ctx->cm_id->route);
883	else if (rdma_protocol_roce(device: ctx->cm_id->device, port_num: ctx->cm_id->port_num))
884	ucma_copy_iboe_route(resp: &resp, route: &ctx->cm_id->route);
885	else if (rdma_protocol_iwarp(device: ctx->cm_id->device, port_num: ctx->cm_id->port_num))
886	ucma_copy_iw_route(resp: &resp, route: &ctx->cm_id->route);
887
888	out:
889	mutex_unlock(lock: &ctx->mutex);
890	if (copy_to_user(u64_to_user_ptr(cmd.response), from: &resp,
891	min_t(size_t, out_len, sizeof(resp))))
892	ret = -EFAULT;
893
894	ucma_put_ctx(ctx);
895	return ret;
896	}
897
898	static void ucma_query_device_addr(struct rdma_cm_id *cm_id,
899	struct rdma_ucm_query_addr_resp *resp)
900	{
901	if (!cm_id->device)
902	return;
903
904	resp->node_guid = (__force __u64) cm_id->device->node_guid;
905	resp->ibdev_index = cm_id->device->index;
906	resp->port_num = cm_id->port_num;
907	resp->pkey = (__force __u16) cpu_to_be16(
908	ib_addr_get_pkey(&cm_id->route.addr.dev_addr));
909	}
910
911	static ssize_t ucma_query_addr(struct ucma_context *ctx,
912	void __user *response, int out_len)
913	{
914	struct rdma_ucm_query_addr_resp resp;
915	struct sockaddr *addr;
916	int ret = 0;
917
918	if (out_len < offsetof(struct rdma_ucm_query_addr_resp, ibdev_index))
919	return -ENOSPC;
920
921	memset(&resp, 0, sizeof resp);
922
923	addr = (struct sockaddr *) &ctx->cm_id->route.addr.src_addr;
924	resp.src_size = rdma_addr_size(addr);
925	memcpy(&resp.src_addr, addr, resp.src_size);
926
927	addr = (struct sockaddr *) &ctx->cm_id->route.addr.dst_addr;
928	resp.dst_size = rdma_addr_size(addr);
929	memcpy(&resp.dst_addr, addr, resp.dst_size);
930
931	ucma_query_device_addr(cm_id: ctx->cm_id, resp: &resp);
932
933	if (copy_to_user(to: response, from: &resp, min_t(size_t, out_len, sizeof(resp))))
934	ret = -EFAULT;
935
936	return ret;
937	}
938
939	static ssize_t ucma_query_path(struct ucma_context *ctx,
940	void __user *response, int out_len)
941	{
942	struct rdma_ucm_query_path_resp *resp;
943	int i, ret = 0;
944
945	if (out_len < sizeof(*resp))
946	return -ENOSPC;
947
948	resp = kzalloc(out_len, GFP_KERNEL);
949	if (!resp)
950	return -ENOMEM;
951
952	resp->num_paths = ctx->cm_id->route.num_pri_alt_paths;
953	for (i = 0, out_len -= sizeof(*resp);
954	i < resp->num_paths && out_len > sizeof(struct ib_path_rec_data);
955	i++, out_len -= sizeof(struct ib_path_rec_data)) {
956	struct sa_path_rec *rec = &ctx->cm_id->route.path_rec[i];
957
958	resp->path_data[i].flags = IB_PATH_GMP | IB_PATH_PRIMARY |
959	IB_PATH_BIDIRECTIONAL;
960	if (rec->rec_type == SA_PATH_REC_TYPE_OPA) {
961	struct sa_path_rec ib;
962
963	sa_convert_path_opa_to_ib(dest: &ib, src: rec);
964	ib_sa_pack_path(rec: &ib, attribute: &resp->path_data[i].path_rec);
965
966	} else {
967	ib_sa_pack_path(rec, attribute: &resp->path_data[i].path_rec);
968	}
969	}
970
971	if (copy_to_user(to: response, from: resp, struct_size(resp, path_data, i)))
972	ret = -EFAULT;
973
974	kfree(objp: resp);
975	return ret;
976	}
977
978	static ssize_t ucma_query_gid(struct ucma_context *ctx,
979	void __user *response, int out_len)
980	{
981	struct rdma_ucm_query_addr_resp resp;
982	struct sockaddr_ib *addr;
983	int ret = 0;
984
985	if (out_len < offsetof(struct rdma_ucm_query_addr_resp, ibdev_index))
986	return -ENOSPC;
987
988	memset(&resp, 0, sizeof resp);
989
990	ucma_query_device_addr(cm_id: ctx->cm_id, resp: &resp);
991
992	addr = (struct sockaddr_ib *) &resp.src_addr;
993	resp.src_size = sizeof(*addr);
994	if (ctx->cm_id->route.addr.src_addr.ss_family == AF_IB) {
995	memcpy(addr, &ctx->cm_id->route.addr.src_addr, resp.src_size);
996	} else {
997	addr->sib_family = AF_IB;
998	addr->sib_pkey = (__force __be16) resp.pkey;
999	rdma_read_gids(cm_id: ctx->cm_id, sgid: (union ib_gid *)&addr->sib_addr,
1000	NULL);
1001	addr->sib_sid = rdma_get_service_id(id: ctx->cm_id, addr: (struct sockaddr *)
1002	&ctx->cm_id->route.addr.src_addr);
1003	}
1004
1005	addr = (struct sockaddr_ib *) &resp.dst_addr;
1006	resp.dst_size = sizeof(*addr);
1007	if (ctx->cm_id->route.addr.dst_addr.ss_family == AF_IB) {
1008	memcpy(addr, &ctx->cm_id->route.addr.dst_addr, resp.dst_size);
1009	} else {
1010	addr->sib_family = AF_IB;
1011	addr->sib_pkey = (__force __be16) resp.pkey;
1012	rdma_read_gids(cm_id: ctx->cm_id, NULL,
1013	dgid: (union ib_gid *)&addr->sib_addr);
1014	addr->sib_sid = rdma_get_service_id(id: ctx->cm_id, addr: (struct sockaddr *)
1015	&ctx->cm_id->route.addr.dst_addr);
1016	}
1017
1018	if (copy_to_user(to: response, from: &resp, min_t(size_t, out_len, sizeof(resp))))
1019	ret = -EFAULT;
1020
1021	return ret;
1022	}
1023
1024	static ssize_t ucma_query_ib_service(struct ucma_context *ctx,
1025	void __user *response, int out_len)
1026	{
1027	struct rdma_ucm_query_ib_service_resp *resp;
1028	int n, ret = 0;
1029
1030	if (out_len < sizeof(struct rdma_ucm_query_ib_service_resp))
1031	return -ENOSPC;
1032
1033	if (!ctx->cm_id->route.service_recs)
1034	return -ENODATA;
1035
1036	resp = kzalloc(out_len, GFP_KERNEL);
1037	if (!resp)
1038	return -ENOMEM;
1039
1040	resp->num_service_recs = ctx->cm_id->route.num_service_recs;
1041
1042	n = (out_len - sizeof(struct rdma_ucm_query_ib_service_resp)) /
1043	sizeof(struct ib_user_service_rec);
1044
1045	if (!n)
1046	goto out;
1047
1048	if (n > ctx->cm_id->route.num_service_recs)
1049	n = ctx->cm_id->route.num_service_recs;
1050
1051	memcpy(resp->recs, ctx->cm_id->route.service_recs,
1052	sizeof(*resp->recs) * n);
1053	if (copy_to_user(to: response, from: resp, struct_size(resp, recs, n)))
1054	ret = -EFAULT;
1055
1056	out:
1057	kfree(objp: resp);
1058	return ret;
1059	}
1060
1061	static ssize_t ucma_query(struct ucma_file *file,
1062	const char __user *inbuf,
1063	int in_len, int out_len)
1064	{
1065	struct rdma_ucm_query cmd;
1066	struct ucma_context *ctx;
1067	void __user *response;
1068	int ret;
1069
1070	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1071	return -EFAULT;
1072
1073	response = u64_to_user_ptr(cmd.response);
1074	ctx = ucma_get_ctx(file, id: cmd.id);
1075	if (IS_ERR(ptr: ctx))
1076	return PTR_ERR(ptr: ctx);
1077
1078	mutex_lock(&ctx->mutex);
1079	switch (cmd.option) {
1080	case RDMA_USER_CM_QUERY_ADDR:
1081	ret = ucma_query_addr(ctx, response, out_len);
1082	break;
1083	case RDMA_USER_CM_QUERY_PATH:
1084	ret = ucma_query_path(ctx, response, out_len);
1085	break;
1086	case RDMA_USER_CM_QUERY_GID:
1087	ret = ucma_query_gid(ctx, response, out_len);
1088	break;
1089	case RDMA_USER_CM_QUERY_IB_SERVICE:
1090	ret = ucma_query_ib_service(ctx, response, out_len);
1091	break;
1092	default:
1093	ret = -ENOSYS;
1094	break;
1095	}
1096	mutex_unlock(lock: &ctx->mutex);
1097
1098	ucma_put_ctx(ctx);
1099	return ret;
1100	}
1101
1102	static void ucma_copy_conn_param(struct rdma_cm_id *id,
1103	struct rdma_conn_param *dst,
1104	struct rdma_ucm_conn_param *src)
1105	{
1106	dst->private_data = src->private_data;
1107	dst->private_data_len = src->private_data_len;
1108	dst->responder_resources = src->responder_resources;
1109	dst->initiator_depth = src->initiator_depth;
1110	dst->flow_control = src->flow_control;
1111	dst->retry_count = src->retry_count;
1112	dst->rnr_retry_count = src->rnr_retry_count;
1113	dst->srq = src->srq;
1114	dst->qp_num = src->qp_num & 0xFFFFFF;
1115	dst->qkey = (id->route.addr.src_addr.ss_family == AF_IB) ? src->qkey : 0;
1116	}
1117
1118	static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf,
1119	int in_len, int out_len)
1120	{
1121	struct rdma_conn_param conn_param;
1122	struct rdma_ucm_ece ece = {};
1123	struct rdma_ucm_connect cmd;
1124	struct ucma_context *ctx;
1125	size_t in_size;
1126	int ret;
1127
1128	if (in_len < offsetofend(typeof(cmd), reserved))
1129	return -EINVAL;
1130	in_size = min_t(size_t, in_len, sizeof(cmd));
1131	if (copy_from_user(to: &cmd, from: inbuf, n: in_size))
1132	return -EFAULT;
1133
1134	if (!cmd.conn_param.valid)
1135	return -EINVAL;
1136
1137	ctx = ucma_get_ctx_dev(file, id: cmd.id);
1138	if (IS_ERR(ptr: ctx))
1139	return PTR_ERR(ptr: ctx);
1140
1141	ucma_copy_conn_param(id: ctx->cm_id, dst: &conn_param, src: &cmd.conn_param);
1142	if (offsetofend(typeof(cmd), ece) <= in_size) {
1143	ece.vendor_id = cmd.ece.vendor_id;
1144	ece.attr_mod = cmd.ece.attr_mod;
1145	}
1146
1147	mutex_lock(&ctx->mutex);
1148	ret = rdma_connect_ece(id: ctx->cm_id, conn_param: &conn_param, ece: &ece);
1149	mutex_unlock(lock: &ctx->mutex);
1150	ucma_put_ctx(ctx);
1151	return ret;
1152	}
1153
1154	static ssize_t ucma_listen(struct ucma_file *file, const char __user *inbuf,
1155	int in_len, int out_len)
1156	{
1157	struct rdma_ucm_listen cmd;
1158	struct ucma_context *ctx;
1159	int ret;
1160
1161	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1162	return -EFAULT;
1163
1164	ctx = ucma_get_ctx(file, id: cmd.id);
1165	if (IS_ERR(ptr: ctx))
1166	return PTR_ERR(ptr: ctx);
1167
1168	if (cmd.backlog <= 0 || cmd.backlog > max_backlog)
1169	cmd.backlog = max_backlog;
1170	atomic_set(v: &ctx->backlog, i: cmd.backlog);
1171
1172	mutex_lock(&ctx->mutex);
1173	ret = rdma_listen(id: ctx->cm_id, backlog: cmd.backlog);
1174	mutex_unlock(lock: &ctx->mutex);
1175	ucma_put_ctx(ctx);
1176	return ret;
1177	}
1178
1179	static ssize_t ucma_accept(struct ucma_file *file, const char __user *inbuf,
1180	int in_len, int out_len)
1181	{
1182	struct rdma_ucm_accept cmd;
1183	struct rdma_conn_param conn_param;
1184	struct rdma_ucm_ece ece = {};
1185	struct ucma_context *ctx;
1186	size_t in_size;
1187	int ret;
1188
1189	if (in_len < offsetofend(typeof(cmd), reserved))
1190	return -EINVAL;
1191	in_size = min_t(size_t, in_len, sizeof(cmd));
1192	if (copy_from_user(to: &cmd, from: inbuf, n: in_size))
1193	return -EFAULT;
1194
1195	ctx = ucma_get_ctx_dev(file, id: cmd.id);
1196	if (IS_ERR(ptr: ctx))
1197	return PTR_ERR(ptr: ctx);
1198
1199	if (offsetofend(typeof(cmd), ece) <= in_size) {
1200	ece.vendor_id = cmd.ece.vendor_id;
1201	ece.attr_mod = cmd.ece.attr_mod;
1202	}
1203
1204	if (cmd.conn_param.valid) {
1205	ucma_copy_conn_param(id: ctx->cm_id, dst: &conn_param, src: &cmd.conn_param);
1206	mutex_lock(&ctx->mutex);
1207	rdma_lock_handler(id: ctx->cm_id);
1208	ret = rdma_accept_ece(id: ctx->cm_id, conn_param: &conn_param, ece: &ece);
1209	if (!ret) {
1210	/* The uid must be set atomically with the handler */
1211	ctx->uid = cmd.uid;
1212	}
1213	rdma_unlock_handler(id: ctx->cm_id);
1214	mutex_unlock(lock: &ctx->mutex);
1215	} else {
1216	mutex_lock(&ctx->mutex);
1217	rdma_lock_handler(id: ctx->cm_id);
1218	ret = rdma_accept_ece(id: ctx->cm_id, NULL, ece: &ece);
1219	rdma_unlock_handler(id: ctx->cm_id);
1220	mutex_unlock(lock: &ctx->mutex);
1221	}
1222	ucma_put_ctx(ctx);
1223	return ret;
1224	}
1225
1226	static ssize_t ucma_reject(struct ucma_file *file, const char __user *inbuf,
1227	int in_len, int out_len)
1228	{
1229	struct rdma_ucm_reject cmd;
1230	struct ucma_context *ctx;
1231	int ret;
1232
1233	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1234	return -EFAULT;
1235
1236	if (!cmd.reason)
1237	cmd.reason = IB_CM_REJ_CONSUMER_DEFINED;
1238
1239	switch (cmd.reason) {
1240	case IB_CM_REJ_CONSUMER_DEFINED:
1241	case IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED:
1242	break;
1243	default:
1244	return -EINVAL;
1245	}
1246
1247	ctx = ucma_get_ctx_dev(file, id: cmd.id);
1248	if (IS_ERR(ptr: ctx))
1249	return PTR_ERR(ptr: ctx);
1250
1251	mutex_lock(&ctx->mutex);
1252	ret = rdma_reject(id: ctx->cm_id, private_data: cmd.private_data, private_data_len: cmd.private_data_len,
1253	reason: cmd.reason);
1254	mutex_unlock(lock: &ctx->mutex);
1255	ucma_put_ctx(ctx);
1256	return ret;
1257	}
1258
1259	static ssize_t ucma_disconnect(struct ucma_file *file, const char __user *inbuf,
1260	int in_len, int out_len)
1261	{
1262	struct rdma_ucm_disconnect cmd;
1263	struct ucma_context *ctx;
1264	int ret;
1265
1266	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1267	return -EFAULT;
1268
1269	ctx = ucma_get_ctx_dev(file, id: cmd.id);
1270	if (IS_ERR(ptr: ctx))
1271	return PTR_ERR(ptr: ctx);
1272
1273	mutex_lock(&ctx->mutex);
1274	ret = rdma_disconnect(id: ctx->cm_id);
1275	mutex_unlock(lock: &ctx->mutex);
1276	ucma_put_ctx(ctx);
1277	return ret;
1278	}
1279
1280	static ssize_t ucma_init_qp_attr(struct ucma_file *file,
1281	const char __user *inbuf,
1282	int in_len, int out_len)
1283	{
1284	struct rdma_ucm_init_qp_attr cmd;
1285	struct ib_uverbs_qp_attr resp;
1286	struct ucma_context *ctx;
1287	struct ib_qp_attr qp_attr;
1288	int ret;
1289
1290	if (out_len < sizeof(resp))
1291	return -ENOSPC;
1292
1293	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1294	return -EFAULT;
1295
1296	if (cmd.qp_state > IB_QPS_ERR)
1297	return -EINVAL;
1298
1299	ctx = ucma_get_ctx_dev(file, id: cmd.id);
1300	if (IS_ERR(ptr: ctx))
1301	return PTR_ERR(ptr: ctx);
1302
1303	resp.qp_attr_mask = 0;
1304	memset(&qp_attr, 0, sizeof qp_attr);
1305	qp_attr.qp_state = cmd.qp_state;
1306	mutex_lock(&ctx->mutex);
1307	ret = rdma_init_qp_attr(id: ctx->cm_id, qp_attr: &qp_attr, qp_attr_mask: &resp.qp_attr_mask);
1308	mutex_unlock(lock: &ctx->mutex);
1309	if (ret)
1310	goto out;
1311
1312	ib_copy_qp_attr_to_user(device: ctx->cm_id->device, dst: &resp, src: &qp_attr);
1313	if (copy_to_user(u64_to_user_ptr(cmd.response),
1314	from: &resp, n: sizeof(resp)))
1315	ret = -EFAULT;
1316
1317	out:
1318	ucma_put_ctx(ctx);
1319	return ret;
1320	}
1321
1322	static int ucma_set_option_id(struct ucma_context *ctx, int optname,
1323	void *optval, size_t optlen)
1324	{
1325	int ret = 0;
1326
1327	switch (optname) {
1328	case RDMA_OPTION_ID_TOS:
1329	if (optlen != sizeof(u8)) {
1330	ret = -EINVAL;
1331	break;
1332	}
1333	rdma_set_service_type(id: ctx->cm_id, tos: *((u8 *) optval));
1334	break;
1335	case RDMA_OPTION_ID_REUSEADDR:
1336	if (optlen != sizeof(int)) {
1337	ret = -EINVAL;
1338	break;
1339	}
1340	ret = rdma_set_reuseaddr(id: ctx->cm_id, reuse: *((int *) optval) ? 1 : 0);
1341	break;
1342	case RDMA_OPTION_ID_AFONLY:
1343	if (optlen != sizeof(int)) {
1344	ret = -EINVAL;
1345	break;
1346	}
1347	ret = rdma_set_afonly(id: ctx->cm_id, afonly: *((int *) optval) ? 1 : 0);
1348	break;
1349	case RDMA_OPTION_ID_ACK_TIMEOUT:
1350	if (optlen != sizeof(u8)) {
1351	ret = -EINVAL;
1352	break;
1353	}
1354	ret = rdma_set_ack_timeout(id: ctx->cm_id, timeout: *((u8 *)optval));
1355	break;
1356	default:
1357	ret = -ENOSYS;
1358	}
1359
1360	return ret;
1361	}
1362
1363	static int ucma_set_ib_path(struct ucma_context *ctx,
1364	struct ib_path_rec_data *path_data, size_t optlen)
1365	{
1366	struct sa_path_rec sa_path;
1367	struct rdma_cm_event event;
1368	int ret;
1369
1370	if (optlen % sizeof(*path_data))
1371	return -EINVAL;
1372
1373	for (; optlen; optlen -= sizeof(*path_data), path_data++) {
1374	if (path_data->flags == (IB_PATH_GMP | IB_PATH_PRIMARY |
1375	IB_PATH_BIDIRECTIONAL))
1376	break;
1377	}
1378
1379	if (!optlen)
1380	return -EINVAL;
1381
1382	if (!ctx->cm_id->device)
1383	return -EINVAL;
1384
1385	memset(&sa_path, 0, sizeof(sa_path));
1386
1387	sa_path.rec_type = SA_PATH_REC_TYPE_IB;
1388	ib_sa_unpack_path(attribute: path_data->path_rec, rec: &sa_path);
1389
1390	if (rdma_cap_opa_ah(device: ctx->cm_id->device, port_num: ctx->cm_id->port_num)) {
1391	struct sa_path_rec opa;
1392
1393	sa_convert_path_ib_to_opa(dest: &opa, src: &sa_path);
1394	mutex_lock(&ctx->mutex);
1395	ret = rdma_set_ib_path(id: ctx->cm_id, path_rec: &opa);
1396	mutex_unlock(lock: &ctx->mutex);
1397	} else {
1398	mutex_lock(&ctx->mutex);
1399	ret = rdma_set_ib_path(id: ctx->cm_id, path_rec: &sa_path);
1400	mutex_unlock(lock: &ctx->mutex);
1401	}
1402	if (ret)
1403	return ret;
1404
1405	memset(&event, 0, sizeof event);
1406	event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1407	return ucma_event_handler(cm_id: ctx->cm_id, event: &event);
1408	}
1409
1410	static int ucma_set_option_ib(struct ucma_context *ctx, int optname,
1411	void *optval, size_t optlen)
1412	{
1413	int ret;
1414
1415	switch (optname) {
1416	case RDMA_OPTION_IB_PATH:
1417	ret = ucma_set_ib_path(ctx, path_data: optval, optlen);
1418	break;
1419	default:
1420	ret = -ENOSYS;
1421	}
1422
1423	return ret;
1424	}
1425
1426	static int ucma_set_option_level(struct ucma_context *ctx, int level,
1427	int optname, void *optval, size_t optlen)
1428	{
1429	int ret;
1430
1431	switch (level) {
1432	case RDMA_OPTION_ID:
1433	mutex_lock(&ctx->mutex);
1434	ret = ucma_set_option_id(ctx, optname, optval, optlen);
1435	mutex_unlock(lock: &ctx->mutex);
1436	break;
1437	case RDMA_OPTION_IB:
1438	ret = ucma_set_option_ib(ctx, optname, optval, optlen);
1439	break;
1440	default:
1441	ret = -ENOSYS;
1442	}
1443
1444	return ret;
1445	}
1446
1447	static ssize_t ucma_set_option(struct ucma_file *file, const char __user *inbuf,
1448	int in_len, int out_len)
1449	{
1450	struct rdma_ucm_set_option cmd;
1451	struct ucma_context *ctx;
1452	void *optval;
1453	int ret;
1454
1455	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1456	return -EFAULT;
1457
1458	if (unlikely(cmd.optlen > KMALLOC_MAX_SIZE))
1459	return -EINVAL;
1460
1461	ctx = ucma_get_ctx(file, id: cmd.id);
1462	if (IS_ERR(ptr: ctx))
1463	return PTR_ERR(ptr: ctx);
1464
1465	optval = memdup_user(u64_to_user_ptr(cmd.optval),
1466	cmd.optlen);
1467	if (IS_ERR(ptr: optval)) {
1468	ret = PTR_ERR(ptr: optval);
1469	goto out;
1470	}
1471
1472	ret = ucma_set_option_level(ctx, level: cmd.level, optname: cmd.optname, optval,
1473	optlen: cmd.optlen);
1474	kfree(objp: optval);
1475
1476	out:
1477	ucma_put_ctx(ctx);
1478	return ret;
1479	}
1480
1481	static ssize_t ucma_notify(struct ucma_file *file, const char __user *inbuf,
1482	int in_len, int out_len)
1483	{
1484	struct rdma_ucm_notify cmd;
1485	struct ucma_context *ctx;
1486	int ret = -EINVAL;
1487
1488	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1489	return -EFAULT;
1490
1491	ctx = ucma_get_ctx(file, id: cmd.id);
1492	if (IS_ERR(ptr: ctx))
1493	return PTR_ERR(ptr: ctx);
1494
1495	mutex_lock(&ctx->mutex);
1496	if (ctx->cm_id->device)
1497	ret = rdma_notify(id: ctx->cm_id, event: (enum ib_event_type)cmd.event);
1498	mutex_unlock(lock: &ctx->mutex);
1499
1500	ucma_put_ctx(ctx);
1501	return ret;
1502	}
1503
1504	static ssize_t ucma_process_join(struct ucma_file *file,
1505	struct rdma_ucm_join_mcast *cmd, int out_len)
1506	{
1507	struct rdma_ucm_create_id_resp resp;
1508	struct ucma_context *ctx;
1509	struct ucma_multicast *mc;
1510	struct sockaddr *addr;
1511	int ret;
1512	u8 join_state;
1513
1514	if (out_len < sizeof(resp))
1515	return -ENOSPC;
1516
1517	addr = (struct sockaddr *) &cmd->addr;
1518	if (cmd->addr_size != rdma_addr_size(addr))
1519	return -EINVAL;
1520
1521	if (cmd->join_flags == RDMA_MC_JOIN_FLAG_FULLMEMBER)
1522	join_state = BIT(FULLMEMBER_JOIN);
1523	else if (cmd->join_flags == RDMA_MC_JOIN_FLAG_SENDONLY_FULLMEMBER)
1524	join_state = BIT(SENDONLY_FULLMEMBER_JOIN);
1525	else
1526	return -EINVAL;
1527
1528	ctx = ucma_get_ctx_dev(file, id: cmd->id);
1529	if (IS_ERR(ptr: ctx))
1530	return PTR_ERR(ptr: ctx);
1531
1532	mc = kzalloc(sizeof(*mc), GFP_KERNEL);
1533	if (!mc) {
1534	ret = -ENOMEM;
1535	goto err_put_ctx;
1536	}
1537
1538	mc->ctx = ctx;
1539	mc->join_state = join_state;
1540	mc->uid = cmd->uid;
1541	memcpy(&mc->addr, addr, cmd->addr_size);
1542
1543	xa_lock(&multicast_table);
1544	if (__xa_alloc(&multicast_table, id: &mc->id, NULL, xa_limit_32b,
1545	GFP_KERNEL)) {
1546	ret = -ENOMEM;
1547	goto err_free_mc;
1548	}
1549
1550	list_add_tail(new: &mc->list, head: &ctx->mc_list);
1551	xa_unlock(&multicast_table);
1552
1553	mutex_lock(&ctx->mutex);
1554	ret = rdma_join_multicast(id: ctx->cm_id, addr: (struct sockaddr *)&mc->addr,
1555	join_state, context: mc);
1556	mutex_unlock(lock: &ctx->mutex);
1557	if (ret)
1558	goto err_xa_erase;
1559
1560	resp.id = mc->id;
1561	if (copy_to_user(u64_to_user_ptr(cmd->response),
1562	from: &resp, n: sizeof(resp))) {
1563	ret = -EFAULT;
1564	goto err_leave_multicast;
1565	}
1566
1567	xa_store(&multicast_table, index: mc->id, entry: mc, 0);
1568
1569	ucma_put_ctx(ctx);
1570	return 0;
1571
1572	err_leave_multicast:
1573	mutex_lock(&ctx->mutex);
1574	rdma_leave_multicast(id: ctx->cm_id, addr: (struct sockaddr *) &mc->addr);
1575	mutex_unlock(lock: &ctx->mutex);
1576	ucma_cleanup_mc_events(mc);
1577	err_xa_erase:
1578	xa_lock(&multicast_table);
1579	list_del(entry: &mc->list);
1580	__xa_erase(&multicast_table, index: mc->id);
1581	err_free_mc:
1582	xa_unlock(&multicast_table);
1583	kfree(objp: mc);
1584	err_put_ctx:
1585	ucma_put_ctx(ctx);
1586	return ret;
1587	}
1588
1589	static ssize_t ucma_join_ip_multicast(struct ucma_file *file,
1590	const char __user *inbuf,
1591	int in_len, int out_len)
1592	{
1593	struct rdma_ucm_join_ip_mcast cmd;
1594	struct rdma_ucm_join_mcast join_cmd;
1595
1596	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1597	return -EFAULT;
1598
1599	join_cmd.response = cmd.response;
1600	join_cmd.uid = cmd.uid;
1601	join_cmd.id = cmd.id;
1602	join_cmd.addr_size = rdma_addr_size_in6(addr: &cmd.addr);
1603	if (!join_cmd.addr_size)
1604	return -EINVAL;
1605
1606	join_cmd.join_flags = RDMA_MC_JOIN_FLAG_FULLMEMBER;
1607	memcpy(&join_cmd.addr, &cmd.addr, join_cmd.addr_size);
1608
1609	return ucma_process_join(file, cmd: &join_cmd, out_len);
1610	}
1611
1612	static ssize_t ucma_join_multicast(struct ucma_file *file,
1613	const char __user *inbuf,
1614	int in_len, int out_len)
1615	{
1616	struct rdma_ucm_join_mcast cmd;
1617
1618	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1619	return -EFAULT;
1620
1621	if (!rdma_addr_size_kss(addr: &cmd.addr))
1622	return -EINVAL;
1623
1624	return ucma_process_join(file, cmd: &cmd, out_len);
1625	}
1626
1627	static ssize_t ucma_leave_multicast(struct ucma_file *file,
1628	const char __user *inbuf,
1629	int in_len, int out_len)
1630	{
1631	struct rdma_ucm_destroy_id cmd;
1632	struct rdma_ucm_destroy_id_resp resp;
1633	struct ucma_multicast *mc;
1634	int ret = 0;
1635
1636	if (out_len < sizeof(resp))
1637	return -ENOSPC;
1638
1639	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1640	return -EFAULT;
1641
1642	xa_lock(&multicast_table);
1643	mc = xa_load(&multicast_table, index: cmd.id);
1644	if (!mc)
1645	mc = ERR_PTR(error: -ENOENT);
1646	else if (READ_ONCE(mc->ctx->file) != file)
1647	mc = ERR_PTR(error: -EINVAL);
1648	else if (!refcount_inc_not_zero(r: &mc->ctx->ref))
1649	mc = ERR_PTR(error: -ENXIO);
1650
1651	if (IS_ERR(ptr: mc)) {
1652	xa_unlock(&multicast_table);
1653	ret = PTR_ERR(ptr: mc);
1654	goto out;
1655	}
1656
1657	list_del(entry: &mc->list);
1658	__xa_erase(&multicast_table, index: mc->id);
1659	xa_unlock(&multicast_table);
1660
1661	mutex_lock(&mc->ctx->mutex);
1662	rdma_leave_multicast(id: mc->ctx->cm_id, addr: (struct sockaddr *) &mc->addr);
1663	mutex_unlock(lock: &mc->ctx->mutex);
1664
1665	ucma_cleanup_mc_events(mc);
1666
1667	ucma_put_ctx(ctx: mc->ctx);
1668	resp.events_reported = mc->events_reported;
1669	kfree(objp: mc);
1670
1671	if (copy_to_user(u64_to_user_ptr(cmd.response),
1672	from: &resp, n: sizeof(resp)))
1673	ret = -EFAULT;
1674	out:
1675	return ret;
1676	}
1677
1678	static ssize_t ucma_migrate_id(struct ucma_file *new_file,
1679	const char __user *inbuf,
1680	int in_len, int out_len)
1681	{
1682	struct rdma_ucm_migrate_id cmd;
1683	struct rdma_ucm_migrate_resp resp;
1684	struct ucma_event *uevent, *tmp;
1685	struct ucma_context *ctx;
1686	LIST_HEAD(event_list);
1687	struct ucma_file *cur_file;
1688	int ret = 0;
1689
1690	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1691	return -EFAULT;
1692
1693	/* Get current fd to protect against it being closed */
1694	CLASS(fd, f)(fd: cmd.fd);
1695	if (fd_empty(f))
1696	return -ENOENT;
1697	if (fd_file(f)->f_op != &ucma_fops)
1698	return -EINVAL;
1699	cur_file = fd_file(f)->private_data;
1700
1701	/* Validate current fd and prevent destruction of id. */
1702	ctx = ucma_get_ctx(file: cur_file, id: cmd.id);
1703	if (IS_ERR(ptr: ctx))
1704	return PTR_ERR(ptr: ctx);
1705
1706	rdma_lock_handler(id: ctx->cm_id);
1707	/*
1708	* ctx->file can only be changed under the handler & xa_lock. xa_load()
1709	* must be checked again to ensure the ctx hasn't begun destruction
1710	* since the ucma_get_ctx().
1711	*/
1712	xa_lock(&ctx_table);
1713	if (_ucma_find_context(id: cmd.id, file: cur_file) != ctx) {
1714	xa_unlock(&ctx_table);
1715	ret = -ENOENT;
1716	goto err_unlock;
1717	}
1718	ctx->file = new_file;
1719	xa_unlock(&ctx_table);
1720
1721	mutex_lock(&cur_file->mut);
1722	list_del(entry: &ctx->list);
1723	/*
1724	* At this point lock_handler() prevents addition of new uevents for
1725	* this ctx.
1726	*/
1727	list_for_each_entry_safe(uevent, tmp, &cur_file->event_list, list)
1728	if (uevent->ctx == ctx)
1729	list_move_tail(list: &uevent->list, head: &event_list);
1730	resp.events_reported = ctx->events_reported;
1731	mutex_unlock(lock: &cur_file->mut);
1732
1733	mutex_lock(&new_file->mut);
1734	list_add_tail(new: &ctx->list, head: &new_file->ctx_list);
1735	list_splice_tail(list: &event_list, head: &new_file->event_list);
1736	mutex_unlock(lock: &new_file->mut);
1737
1738	if (copy_to_user(u64_to_user_ptr(cmd.response),
1739	from: &resp, n: sizeof(resp)))
1740	ret = -EFAULT;
1741
1742	err_unlock:
1743	rdma_unlock_handler(id: ctx->cm_id);
1744	ucma_put_ctx(ctx);
1745	return ret;
1746	}
1747
1748	static ssize_t ucma_write_cm_event(struct ucma_file *file,
1749	const char __user *inbuf, int in_len,
1750	int out_len)
1751	{
1752	struct rdma_ucm_write_cm_event cmd;
1753	struct rdma_cm_event event = {};
1754	struct ucma_event *uevent;
1755	struct ucma_context *ctx;
1756	int ret = 0;
1757
1758	if (copy_from_user(to: &cmd, from: inbuf, n: sizeof(cmd)))
1759	return -EFAULT;
1760
1761	if ((cmd.event != RDMA_CM_EVENT_USER) &&
1762	(cmd.event != RDMA_CM_EVENT_INTERNAL))
1763	return -EINVAL;
1764
1765	ctx = ucma_get_ctx(file, id: cmd.id);
1766	if (IS_ERR(ptr: ctx))
1767	return PTR_ERR(ptr: ctx);
1768
1769	event.event = cmd.event;
1770	event.status = cmd.status;
1771	event.param.arg = cmd.param.arg;
1772
1773	uevent = kzalloc(sizeof(*uevent), GFP_KERNEL);
1774	if (!uevent) {
1775	ret = -ENOMEM;
1776	goto out;
1777	}
1778
1779	uevent->ctx = ctx;
1780	uevent->resp.uid = ctx->uid;
1781	uevent->resp.id = ctx->id;
1782	uevent->resp.event = event.event;
1783	uevent->resp.status = event.status;
1784	memcpy(uevent->resp.param.arg32, &event.param.arg,
1785	sizeof(event.param.arg));
1786
1787	mutex_lock(&ctx->file->mut);
1788	list_add_tail(new: &uevent->list, head: &ctx->file->event_list);
1789	mutex_unlock(lock: &ctx->file->mut);
1790	wake_up_interruptible(&ctx->file->poll_wait);
1791
1792	out:
1793	ucma_put_ctx(ctx);
1794	return ret;
1795	}
1796
1797	static ssize_t (*ucma_cmd_table[])(struct ucma_file *file,
1798	const char __user *inbuf,
1799	int in_len, int out_len) = {
1800	[RDMA_USER_CM_CMD_CREATE_ID] = ucma_create_id,
1801	[RDMA_USER_CM_CMD_DESTROY_ID] = ucma_destroy_id,
1802	[RDMA_USER_CM_CMD_BIND_IP] = ucma_bind_ip,
1803	[RDMA_USER_CM_CMD_RESOLVE_IP] = ucma_resolve_ip,
1804	[RDMA_USER_CM_CMD_RESOLVE_ROUTE] = ucma_resolve_route,
1805	[RDMA_USER_CM_CMD_QUERY_ROUTE] = ucma_query_route,
1806	[RDMA_USER_CM_CMD_CONNECT] = ucma_connect,
1807	[RDMA_USER_CM_CMD_LISTEN] = ucma_listen,
1808	[RDMA_USER_CM_CMD_ACCEPT] = ucma_accept,
1809	[RDMA_USER_CM_CMD_REJECT] = ucma_reject,
1810	[RDMA_USER_CM_CMD_DISCONNECT] = ucma_disconnect,
1811	[RDMA_USER_CM_CMD_INIT_QP_ATTR] = ucma_init_qp_attr,
1812	[RDMA_USER_CM_CMD_GET_EVENT] = ucma_get_event,
1813	[RDMA_USER_CM_CMD_GET_OPTION] = NULL,
1814	[RDMA_USER_CM_CMD_SET_OPTION] = ucma_set_option,
1815	[RDMA_USER_CM_CMD_NOTIFY] = ucma_notify,
1816	[RDMA_USER_CM_CMD_JOIN_IP_MCAST] = ucma_join_ip_multicast,
1817	[RDMA_USER_CM_CMD_LEAVE_MCAST] = ucma_leave_multicast,
1818	[RDMA_USER_CM_CMD_MIGRATE_ID] = ucma_migrate_id,
1819	[RDMA_USER_CM_CMD_QUERY] = ucma_query,
1820	[RDMA_USER_CM_CMD_BIND] = ucma_bind,
1821	[RDMA_USER_CM_CMD_RESOLVE_ADDR] = ucma_resolve_addr,
1822	[RDMA_USER_CM_CMD_JOIN_MCAST] = ucma_join_multicast,
1823	[RDMA_USER_CM_CMD_RESOLVE_IB_SERVICE] = ucma_resolve_ib_service,
1824	[RDMA_USER_CM_CMD_WRITE_CM_EVENT] = ucma_write_cm_event,
1825	};
1826
1827	static ssize_t ucma_write(struct file *filp, const char __user *buf,
1828	size_t len, loff_t *pos)
1829	{
1830	struct ucma_file *file = filp->private_data;
1831	struct rdma_ucm_cmd_hdr hdr;
1832	ssize_t ret;
1833
1834	if (!ib_safe_file_access(filp)) {
1835	pr_err_once("%s: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
1836	__func__, task_tgid_vnr(current), current->comm);
1837	return -EACCES;
1838	}
1839
1840	if (len < sizeof(hdr))
1841	return -EINVAL;
1842
1843	if (copy_from_user(to: &hdr, from: buf, n: sizeof(hdr)))
1844	return -EFAULT;
1845
1846	if (hdr.cmd >= ARRAY_SIZE(ucma_cmd_table))
1847	return -EINVAL;
1848	hdr.cmd = array_index_nospec(hdr.cmd, ARRAY_SIZE(ucma_cmd_table));
1849
1850	if (hdr.in + sizeof(hdr) > len)
1851	return -EINVAL;
1852
1853	if (!ucma_cmd_table[hdr.cmd])
1854	return -ENOSYS;
1855
1856	ret = ucma_cmd_table[hdr.cmd](file, buf + sizeof(hdr), hdr.in, hdr.out);
1857	if (!ret)
1858	ret = len;
1859
1860	return ret;
1861	}
1862
1863	static __poll_t ucma_poll(struct file *filp, struct poll_table_struct *wait)
1864	{
1865	struct ucma_file *file = filp->private_data;
1866	__poll_t mask = 0;
1867
1868	poll_wait(filp, wait_address: &file->poll_wait, p: wait);
1869
1870	if (!list_empty(head: &file->event_list))
1871	mask = EPOLLIN | EPOLLRDNORM;
1872
1873	return mask;
1874	}
1875
1876	/*
1877	* ucma_open() does not need the BKL:
1878	*
1879	* - no global state is referred to;
1880	* - there is no ioctl method to race against;
1881	* - no further module initialization is required for open to work
1882	* after the device is registered.
1883	*/
1884	static int ucma_open(struct inode *inode, struct file *filp)
1885	{
1886	struct ucma_file *file;
1887
1888	file = kmalloc(sizeof *file, GFP_KERNEL);
1889	if (!file)
1890	return -ENOMEM;
1891
1892	INIT_LIST_HEAD(list: &file->event_list);
1893	INIT_LIST_HEAD(list: &file->ctx_list);
1894	init_waitqueue_head(&file->poll_wait);
1895	mutex_init(&file->mut);
1896
1897	filp->private_data = file;
1898	file->filp = filp;
1899
1900	return stream_open(inode, filp);
1901	}
1902
1903	static int ucma_close(struct inode *inode, struct file *filp)
1904	{
1905	struct ucma_file *file = filp->private_data;
1906
1907	/*
1908	* All paths that touch ctx_list or ctx_list starting from write() are
1909	* prevented by this being a FD release function. The list_add_tail() in
1910	* ucma_connect_event_handler() can run concurrently, however it only
1911	* adds to the list *after* a listening ID. By only reading the first of
1912	* the list, and relying on ucma_destroy_private_ctx() to block
1913	* ucma_connect_event_handler(), no additional locking is needed.
1914	*/
1915	while (!list_empty(head: &file->ctx_list)) {
1916	struct ucma_context *ctx = list_first_entry(
1917	&file->ctx_list, struct ucma_context, list);
1918
1919	WARN_ON(xa_cmpxchg(&ctx_table, ctx->id, ctx, XA_ZERO_ENTRY,
1920	GFP_KERNEL) != ctx);
1921	ucma_destroy_private_ctx(ctx);
1922	}
1923	kfree(objp: file);
1924	return 0;
1925	}
1926
1927	static const struct file_operations ucma_fops = {
1928	.owner = THIS_MODULE,
1929	.open = ucma_open,
1930	.release = ucma_close,
1931	.write = ucma_write,
1932	.poll = ucma_poll,
1933	};
1934
1935	static struct miscdevice ucma_misc = {
1936	.minor = MISC_DYNAMIC_MINOR,
1937	.name = "rdma_cm",
1938	.nodename = "infiniband/rdma_cm",
1939	.mode = 0666,
1940	.fops = &ucma_fops,
1941	};
1942
1943	static int ucma_get_global_nl_info(struct ib_client_nl_info *res)
1944	{
1945	res->abi = RDMA_USER_CM_ABI_VERSION;
1946	res->cdev = ucma_misc.this_device;
1947	return 0;
1948	}
1949
1950	static struct ib_client rdma_cma_client = {
1951	.name = "rdma_cm",
1952	.get_global_nl_info = ucma_get_global_nl_info,
1953	};
1954	MODULE_ALIAS_RDMA_CLIENT("rdma_cm");
1955
1956	static ssize_t abi_version_show(struct device *dev,
1957	struct device_attribute *attr, char *buf)
1958	{
1959	return sysfs_emit(buf, fmt: "%d\n", RDMA_USER_CM_ABI_VERSION);
1960	}
1961	static DEVICE_ATTR_RO(abi_version);
1962
1963	static int __init ucma_init(void)
1964	{
1965	int ret;
1966
1967	ret = misc_register(misc: &ucma_misc);
1968	if (ret)
1969	return ret;
1970
1971	ret = device_create_file(device: ucma_misc.this_device, entry: &dev_attr_abi_version);
1972	if (ret) {
1973	pr_err("rdma_ucm: couldn't create abi_version attr\n");
1974	goto err1;
1975	}
1976
1977	ucma_ctl_table_hdr = register_net_sysctl(&init_net, "net/rdma_ucm", ucma_ctl_table);
1978	if (!ucma_ctl_table_hdr) {
1979	pr_err("rdma_ucm: couldn't register sysctl paths\n");
1980	ret = -ENOMEM;
1981	goto err2;
1982	}
1983
1984	ret = ib_register_client(client: &rdma_cma_client);
1985	if (ret)
1986	goto err3;
1987
1988	return 0;
1989	err3:
1990	unregister_net_sysctl_table(header: ucma_ctl_table_hdr);
1991	err2:
1992	device_remove_file(dev: ucma_misc.this_device, attr: &dev_attr_abi_version);
1993	err1:
1994	misc_deregister(misc: &ucma_misc);
1995	return ret;
1996	}
1997
1998	static void __exit ucma_cleanup(void)
1999	{
2000	ib_unregister_client(client: &rdma_cma_client);
2001	unregister_net_sysctl_table(header: ucma_ctl_table_hdr);
2002	device_remove_file(dev: ucma_misc.this_device, attr: &dev_attr_abi_version);
2003	misc_deregister(misc: &ucma_misc);
2004	}
2005
2006	module_init(ucma_init);
2007	module_exit(ucma_cleanup);
2008