1/*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
6 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
7 *
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
13 *
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
16 * conditions are met:
17 *
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer.
21 *
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * SOFTWARE.
35 */
36
37#include <linux/module.h>
38#include <linux/init.h>
39#include <linux/device.h>
40#include <linux/err.h>
41#include <linux/fs.h>
42#include <linux/poll.h>
43#include <linux/sched.h>
44#include <linux/file.h>
45#include <linux/cdev.h>
46#include <linux/anon_inodes.h>
47#include <linux/slab.h>
48#include <linux/sched/mm.h>
49
50#include <linux/uaccess.h>
51
52#include <rdma/ib.h>
53#include <rdma/uverbs_std_types.h>
54#include <rdma/rdma_netlink.h>
55#include <rdma/ib_ucaps.h>
56
57#include "uverbs.h"
58#include "core_priv.h"
59#include "rdma_core.h"
60
61MODULE_AUTHOR("Roland Dreier");
62MODULE_DESCRIPTION("InfiniBand userspace verbs access");
63MODULE_LICENSE("Dual BSD/GPL");
64
65enum {
66 IB_UVERBS_MAJOR = 231,
67 IB_UVERBS_BASE_MINOR = 192,
68 IB_UVERBS_MAX_DEVICES = RDMA_MAX_PORTS,
69 IB_UVERBS_NUM_FIXED_MINOR = 32,
70 IB_UVERBS_NUM_DYNAMIC_MINOR = IB_UVERBS_MAX_DEVICES - IB_UVERBS_NUM_FIXED_MINOR,
71};
72
73#define IB_UVERBS_BASE_DEV MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR)
74
75static dev_t dynamic_uverbs_dev;
76
77static DEFINE_IDA(uverbs_ida);
78static int ib_uverbs_add_one(struct ib_device *device);
79static void ib_uverbs_remove_one(struct ib_device *device, void *client_data);
80static struct ib_client uverbs_client;
81
82static char *uverbs_devnode(const struct device *dev, umode_t *mode)
83{
84 if (mode)
85 *mode = 0666;
86 return kasprintf(GFP_KERNEL, fmt: "infiniband/%s", dev_name(dev));
87}
88
89static const struct class uverbs_class = {
90 .name = "infiniband_verbs",
91 .devnode = uverbs_devnode,
92};
93
94/*
95 * Must be called with the ufile->device->disassociate_srcu held, and the lock
96 * must be held until use of the ucontext is finished.
97 */
98struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile)
99{
100 /*
101 * We do not hold the hw_destroy_rwsem lock for this flow, instead
102 * srcu is used. It does not matter if someone races this with
103 * get_context, we get NULL or valid ucontext.
104 */
105 struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext);
106
107 if (!srcu_dereference(ufile->device->ib_dev,
108 &ufile->device->disassociate_srcu))
109 return ERR_PTR(error: -EIO);
110
111 if (!ucontext)
112 return ERR_PTR(error: -EINVAL);
113
114 return ucontext;
115}
116EXPORT_SYMBOL(ib_uverbs_get_ucontext_file);
117
118int uverbs_dealloc_mw(struct ib_mw *mw)
119{
120 struct ib_pd *pd = mw->pd;
121 int ret;
122
123 ret = mw->device->ops.dealloc_mw(mw);
124 if (ret)
125 return ret;
126
127 atomic_dec(v: &pd->usecnt);
128 kfree(objp: mw);
129 return ret;
130}
131
132static void ib_uverbs_release_dev(struct device *device)
133{
134 struct ib_uverbs_device *dev =
135 container_of(device, struct ib_uverbs_device, dev);
136
137 uverbs_destroy_api(uapi: dev->uapi);
138 cleanup_srcu_struct(ssp: &dev->disassociate_srcu);
139 mutex_destroy(lock: &dev->lists_mutex);
140 mutex_destroy(lock: &dev->xrcd_tree_mutex);
141 kfree(objp: dev);
142}
143
144void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file,
145 struct ib_ucq_object *uobj)
146{
147 struct ib_uverbs_event *evt, *tmp;
148
149 if (ev_file) {
150 spin_lock_irq(lock: &ev_file->ev_queue.lock);
151 list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) {
152 list_del(entry: &evt->list);
153 kfree(objp: evt);
154 }
155 spin_unlock_irq(lock: &ev_file->ev_queue.lock);
156
157 uverbs_uobject_put(uobject: &ev_file->uobj);
158 }
159
160 ib_uverbs_release_uevent(uobj: &uobj->uevent);
161}
162
163void ib_uverbs_release_uevent(struct ib_uevent_object *uobj)
164{
165 struct ib_uverbs_async_event_file *async_file = uobj->event_file;
166 struct ib_uverbs_event *evt, *tmp;
167
168 if (!async_file)
169 return;
170
171 spin_lock_irq(lock: &async_file->ev_queue.lock);
172 list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) {
173 list_del(entry: &evt->list);
174 kfree(objp: evt);
175 }
176 spin_unlock_irq(lock: &async_file->ev_queue.lock);
177 uverbs_uobject_put(uobject: &async_file->uobj);
178}
179
180void ib_uverbs_detach_umcast(struct ib_qp *qp,
181 struct ib_uqp_object *uobj)
182{
183 struct ib_uverbs_mcast_entry *mcast, *tmp;
184
185 list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) {
186 ib_detach_mcast(qp, gid: &mcast->gid, lid: mcast->lid);
187 list_del(entry: &mcast->list);
188 kfree(objp: mcast);
189 }
190}
191
192static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev)
193{
194 complete(&dev->comp);
195}
196
197void ib_uverbs_release_file(struct kref *ref)
198{
199 struct ib_uverbs_file *file =
200 container_of(ref, struct ib_uverbs_file, ref);
201 struct ib_device *ib_dev;
202 int srcu_key;
203
204 release_ufile_idr_uobject(ufile: file);
205
206 srcu_key = srcu_read_lock(ssp: &file->device->disassociate_srcu);
207 ib_dev = srcu_dereference(file->device->ib_dev,
208 &file->device->disassociate_srcu);
209 if (ib_dev && !ib_dev->ops.disassociate_ucontext)
210 module_put(module: ib_dev->ops.owner);
211 srcu_read_unlock(ssp: &file->device->disassociate_srcu, idx: srcu_key);
212
213 if (refcount_dec_and_test(r: &file->device->refcount))
214 ib_uverbs_comp_dev(dev: file->device);
215
216 if (file->default_async_file)
217 uverbs_uobject_put(uobject: &file->default_async_file->uobj);
218 put_device(dev: &file->device->dev);
219
220 if (file->disassociate_page)
221 __free_pages(page: file->disassociate_page, order: 0);
222 mutex_destroy(lock: &file->disassociation_lock);
223 mutex_destroy(lock: &file->umap_lock);
224 mutex_destroy(lock: &file->ucontext_lock);
225 kfree(objp: file);
226}
227
228static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue,
229 struct file *filp, char __user *buf,
230 size_t count, loff_t *pos,
231 size_t eventsz)
232{
233 struct ib_uverbs_event *event;
234 int ret = 0;
235
236 spin_lock_irq(lock: &ev_queue->lock);
237
238 while (list_empty(head: &ev_queue->event_list)) {
239 if (ev_queue->is_closed) {
240 spin_unlock_irq(lock: &ev_queue->lock);
241 return -EIO;
242 }
243
244 spin_unlock_irq(lock: &ev_queue->lock);
245 if (filp->f_flags & O_NONBLOCK)
246 return -EAGAIN;
247
248 if (wait_event_interruptible(ev_queue->poll_wait,
249 (!list_empty(&ev_queue->event_list) ||
250 ev_queue->is_closed)))
251 return -ERESTARTSYS;
252
253 spin_lock_irq(lock: &ev_queue->lock);
254 }
255
256 event = list_entry(ev_queue->event_list.next, struct ib_uverbs_event, list);
257
258 if (eventsz > count) {
259 ret = -EINVAL;
260 event = NULL;
261 } else {
262 list_del(entry: ev_queue->event_list.next);
263 if (event->counter) {
264 ++(*event->counter);
265 list_del(entry: &event->obj_list);
266 }
267 }
268
269 spin_unlock_irq(lock: &ev_queue->lock);
270
271 if (event) {
272 if (copy_to_user(to: buf, from: event, n: eventsz))
273 ret = -EFAULT;
274 else
275 ret = eventsz;
276 }
277
278 kfree(objp: event);
279
280 return ret;
281}
282
283static ssize_t ib_uverbs_async_event_read(struct file *filp, char __user *buf,
284 size_t count, loff_t *pos)
285{
286 struct ib_uverbs_async_event_file *file = filp->private_data;
287
288 return ib_uverbs_event_read(ev_queue: &file->ev_queue, filp, buf, count, pos,
289 eventsz: sizeof(struct ib_uverbs_async_event_desc));
290}
291
292static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf,
293 size_t count, loff_t *pos)
294{
295 struct ib_uverbs_completion_event_file *comp_ev_file =
296 filp->private_data;
297
298 return ib_uverbs_event_read(ev_queue: &comp_ev_file->ev_queue, filp, buf, count,
299 pos,
300 eventsz: sizeof(struct ib_uverbs_comp_event_desc));
301}
302
303static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue,
304 struct file *filp,
305 struct poll_table_struct *wait)
306{
307 __poll_t pollflags = 0;
308
309 poll_wait(filp, wait_address: &ev_queue->poll_wait, p: wait);
310
311 spin_lock_irq(lock: &ev_queue->lock);
312 if (!list_empty(head: &ev_queue->event_list))
313 pollflags = EPOLLIN | EPOLLRDNORM;
314 else if (ev_queue->is_closed)
315 pollflags = EPOLLERR;
316 spin_unlock_irq(lock: &ev_queue->lock);
317
318 return pollflags;
319}
320
321static __poll_t ib_uverbs_async_event_poll(struct file *filp,
322 struct poll_table_struct *wait)
323{
324 struct ib_uverbs_async_event_file *file = filp->private_data;
325
326 return ib_uverbs_event_poll(ev_queue: &file->ev_queue, filp, wait);
327}
328
329static __poll_t ib_uverbs_comp_event_poll(struct file *filp,
330 struct poll_table_struct *wait)
331{
332 struct ib_uverbs_completion_event_file *comp_ev_file =
333 filp->private_data;
334
335 return ib_uverbs_event_poll(ev_queue: &comp_ev_file->ev_queue, filp, wait);
336}
337
338static int ib_uverbs_async_event_fasync(int fd, struct file *filp, int on)
339{
340 struct ib_uverbs_async_event_file *file = filp->private_data;
341
342 return fasync_helper(fd, filp, on, &file->ev_queue.async_queue);
343}
344
345static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on)
346{
347 struct ib_uverbs_completion_event_file *comp_ev_file =
348 filp->private_data;
349
350 return fasync_helper(fd, filp, on, &comp_ev_file->ev_queue.async_queue);
351}
352
353const struct file_operations uverbs_event_fops = {
354 .owner = THIS_MODULE,
355 .read = ib_uverbs_comp_event_read,
356 .poll = ib_uverbs_comp_event_poll,
357 .release = uverbs_uobject_fd_release,
358 .fasync = ib_uverbs_comp_event_fasync,
359};
360
361const struct file_operations uverbs_async_event_fops = {
362 .owner = THIS_MODULE,
363 .read = ib_uverbs_async_event_read,
364 .poll = ib_uverbs_async_event_poll,
365 .release = uverbs_async_event_release,
366 .fasync = ib_uverbs_async_event_fasync,
367};
368
369void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context)
370{
371 struct ib_uverbs_event_queue *ev_queue = cq_context;
372 struct ib_ucq_object *uobj;
373 struct ib_uverbs_event *entry;
374 unsigned long flags;
375
376 if (!ev_queue)
377 return;
378
379 spin_lock_irqsave(&ev_queue->lock, flags);
380 if (ev_queue->is_closed) {
381 spin_unlock_irqrestore(lock: &ev_queue->lock, flags);
382 return;
383 }
384
385 entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
386 if (!entry) {
387 spin_unlock_irqrestore(lock: &ev_queue->lock, flags);
388 return;
389 }
390
391 uobj = cq->uobject;
392
393 entry->desc.comp.cq_handle = cq->uobject->uevent.uobject.user_handle;
394 entry->counter = &uobj->comp_events_reported;
395
396 list_add_tail(new: &entry->list, head: &ev_queue->event_list);
397 list_add_tail(new: &entry->obj_list, head: &uobj->comp_list);
398 spin_unlock_irqrestore(lock: &ev_queue->lock, flags);
399
400 wake_up_interruptible(&ev_queue->poll_wait);
401 kill_fasync(&ev_queue->async_queue, SIGIO, POLL_IN);
402}
403
404void ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file,
405 __u64 element, __u64 event,
406 struct list_head *obj_list, u32 *counter)
407{
408 struct ib_uverbs_event *entry;
409 unsigned long flags;
410
411 if (!async_file)
412 return;
413
414 spin_lock_irqsave(&async_file->ev_queue.lock, flags);
415 if (async_file->ev_queue.is_closed) {
416 spin_unlock_irqrestore(lock: &async_file->ev_queue.lock, flags);
417 return;
418 }
419
420 entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
421 if (!entry) {
422 spin_unlock_irqrestore(lock: &async_file->ev_queue.lock, flags);
423 return;
424 }
425
426 entry->desc.async.element = element;
427 entry->desc.async.event_type = event;
428 entry->desc.async.reserved = 0;
429 entry->counter = counter;
430
431 list_add_tail(new: &entry->list, head: &async_file->ev_queue.event_list);
432 if (obj_list)
433 list_add_tail(new: &entry->obj_list, head: obj_list);
434 spin_unlock_irqrestore(lock: &async_file->ev_queue.lock, flags);
435
436 wake_up_interruptible(&async_file->ev_queue.poll_wait);
437 kill_fasync(&async_file->ev_queue.async_queue, SIGIO, POLL_IN);
438}
439
440static void uverbs_uobj_event(struct ib_uevent_object *eobj,
441 struct ib_event *event)
442{
443 ib_uverbs_async_handler(async_file: eobj->event_file,
444 element: eobj->uobject.user_handle, event: event->event,
445 obj_list: &eobj->event_list, counter: &eobj->events_reported);
446}
447
448void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr)
449{
450 uverbs_uobj_event(eobj: &event->element.cq->uobject->uevent, event);
451}
452
453void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr)
454{
455 /* for XRC target qp's, check that qp is live */
456 if (!event->element.qp->uobject)
457 return;
458
459 uverbs_uobj_event(eobj: &event->element.qp->uobject->uevent, event);
460}
461
462void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr)
463{
464 uverbs_uobj_event(eobj: &event->element.wq->uobject->uevent, event);
465}
466
467void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr)
468{
469 uverbs_uobj_event(eobj: &event->element.srq->uobject->uevent, event);
470}
471
472static void ib_uverbs_event_handler(struct ib_event_handler *handler,
473 struct ib_event *event)
474{
475 ib_uverbs_async_handler(
476 container_of(handler, struct ib_uverbs_async_event_file,
477 event_handler),
478 element: event->element.port_num, event: event->event, NULL, NULL);
479}
480
481void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue)
482{
483 spin_lock_init(&ev_queue->lock);
484 INIT_LIST_HEAD(list: &ev_queue->event_list);
485 init_waitqueue_head(&ev_queue->poll_wait);
486 ev_queue->is_closed = 0;
487 ev_queue->async_queue = NULL;
488}
489
490void ib_uverbs_init_async_event_file(
491 struct ib_uverbs_async_event_file *async_file)
492{
493 struct ib_uverbs_file *uverbs_file = async_file->uobj.ufile;
494 struct ib_device *ib_dev = async_file->uobj.context->device;
495
496 ib_uverbs_init_event_queue(ev_queue: &async_file->ev_queue);
497
498 /* The first async_event_file becomes the default one for the file. */
499 mutex_lock(&uverbs_file->ucontext_lock);
500 if (!uverbs_file->default_async_file) {
501 /* Pairs with the put in ib_uverbs_release_file */
502 uverbs_uobject_get(uobject: &async_file->uobj);
503 smp_store_release(&uverbs_file->default_async_file, async_file);
504 }
505 mutex_unlock(lock: &uverbs_file->ucontext_lock);
506
507 INIT_IB_EVENT_HANDLER(&async_file->event_handler, ib_dev,
508 ib_uverbs_event_handler);
509 ib_register_event_handler(event_handler: &async_file->event_handler);
510}
511
512static ssize_t verify_hdr(struct ib_uverbs_cmd_hdr *hdr,
513 struct ib_uverbs_ex_cmd_hdr *ex_hdr, size_t count,
514 const struct uverbs_api_write_method *method_elm)
515{
516 if (method_elm->is_ex) {
517 count -= sizeof(*hdr) + sizeof(*ex_hdr);
518
519 if ((hdr->in_words + ex_hdr->provider_in_words) * 8 != count)
520 return -EINVAL;
521
522 if (hdr->in_words * 8 < method_elm->req_size)
523 return -ENOSPC;
524
525 if (ex_hdr->cmd_hdr_reserved)
526 return -EINVAL;
527
528 if (ex_hdr->response) {
529 if (!hdr->out_words && !ex_hdr->provider_out_words)
530 return -EINVAL;
531
532 if (hdr->out_words * 8 < method_elm->resp_size)
533 return -ENOSPC;
534
535 if (!access_ok(u64_to_user_ptr(ex_hdr->response),
536 (hdr->out_words + ex_hdr->provider_out_words) * 8))
537 return -EFAULT;
538 } else {
539 if (hdr->out_words || ex_hdr->provider_out_words)
540 return -EINVAL;
541 }
542
543 return 0;
544 }
545
546 /* not extended command */
547 if (hdr->in_words * 4 != count)
548 return -EINVAL;
549
550 if (count < method_elm->req_size + sizeof(*hdr)) {
551 /*
552 * rdma-core v18 and v19 have a bug where they send DESTROY_CQ
553 * with a 16 byte write instead of 24. Old kernels didn't
554 * check the size so they allowed this. Now that the size is
555 * checked provide a compatibility work around to not break
556 * those userspaces.
557 */
558 if (hdr->command == IB_USER_VERBS_CMD_DESTROY_CQ &&
559 count == 16) {
560 hdr->in_words = 6;
561 return 0;
562 }
563 return -ENOSPC;
564 }
565 if (hdr->out_words * 4 < method_elm->resp_size)
566 return -ENOSPC;
567
568 return 0;
569}
570
571static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
572 size_t count, loff_t *pos)
573{
574 struct ib_uverbs_file *file = filp->private_data;
575 const struct uverbs_api_write_method *method_elm;
576 struct uverbs_api *uapi = file->device->uapi;
577 struct ib_uverbs_ex_cmd_hdr ex_hdr;
578 struct ib_uverbs_cmd_hdr hdr;
579 struct uverbs_attr_bundle bundle;
580 int srcu_key;
581 ssize_t ret;
582
583 if (!ib_safe_file_access(filp)) {
584 pr_err_once("uverbs_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
585 task_tgid_vnr(current), current->comm);
586 return -EACCES;
587 }
588
589 if (count < sizeof(hdr))
590 return -EINVAL;
591
592 if (copy_from_user(to: &hdr, from: buf, n: sizeof(hdr)))
593 return -EFAULT;
594
595 method_elm = uapi_get_method(uapi, command: hdr.command);
596 if (IS_ERR(ptr: method_elm))
597 return PTR_ERR(ptr: method_elm);
598
599 if (method_elm->is_ex) {
600 if (count < (sizeof(hdr) + sizeof(ex_hdr)))
601 return -EINVAL;
602 if (copy_from_user(to: &ex_hdr, from: buf + sizeof(hdr), n: sizeof(ex_hdr)))
603 return -EFAULT;
604 }
605
606 ret = verify_hdr(hdr: &hdr, ex_hdr: &ex_hdr, count, method_elm);
607 if (ret)
608 return ret;
609
610 srcu_key = srcu_read_lock(ssp: &file->device->disassociate_srcu);
611
612 buf += sizeof(hdr);
613
614 memset(bundle.attr_present, 0, sizeof(bundle.attr_present));
615 bundle.ufile = file;
616 bundle.context = NULL; /* only valid if bundle has uobject */
617 bundle.uobject = NULL;
618 if (!method_elm->is_ex) {
619 size_t in_len = hdr.in_words * 4 - sizeof(hdr);
620 size_t out_len = hdr.out_words * 4;
621 u64 response = 0;
622
623 if (method_elm->has_udata) {
624 bundle.driver_udata.inlen =
625 in_len - method_elm->req_size;
626 in_len = method_elm->req_size;
627 if (bundle.driver_udata.inlen)
628 bundle.driver_udata.inbuf = buf + in_len;
629 else
630 bundle.driver_udata.inbuf = NULL;
631 } else {
632 memset(&bundle.driver_udata, 0,
633 sizeof(bundle.driver_udata));
634 }
635
636 if (method_elm->has_resp) {
637 /*
638 * The macros check that if has_resp is set
639 * then the command request structure starts
640 * with a '__aligned u64 response' member.
641 */
642 ret = get_user(response, (const u64 __user *)buf);
643 if (ret)
644 goto out_unlock;
645
646 if (method_elm->has_udata) {
647 bundle.driver_udata.outlen =
648 out_len - method_elm->resp_size;
649 out_len = method_elm->resp_size;
650 if (bundle.driver_udata.outlen)
651 bundle.driver_udata.outbuf =
652 u64_to_user_ptr(response +
653 out_len);
654 else
655 bundle.driver_udata.outbuf = NULL;
656 }
657 } else {
658 bundle.driver_udata.outlen = 0;
659 bundle.driver_udata.outbuf = NULL;
660 }
661
662 ib_uverbs_init_udata_buf_or_null(
663 udata: &bundle.ucore, ibuf: buf, u64_to_user_ptr(response),
664 ilen: in_len, olen: out_len);
665 } else {
666 buf += sizeof(ex_hdr);
667
668 ib_uverbs_init_udata_buf_or_null(udata: &bundle.ucore, ibuf: buf,
669 u64_to_user_ptr(ex_hdr.response),
670 ilen: hdr.in_words * 8, olen: hdr.out_words * 8);
671
672 ib_uverbs_init_udata_buf_or_null(
673 udata: &bundle.driver_udata, ibuf: buf + bundle.ucore.inlen,
674 u64_to_user_ptr(ex_hdr.response) + bundle.ucore.outlen,
675 ilen: ex_hdr.provider_in_words * 8,
676 olen: ex_hdr.provider_out_words * 8);
677
678 }
679
680 ret = method_elm->handler(&bundle);
681 if (bundle.uobject)
682 uverbs_finalize_object(uobj: bundle.uobject, access: UVERBS_ACCESS_NEW, hw_obj_valid: true,
683 commit: !ret, attrs: &bundle);
684out_unlock:
685 srcu_read_unlock(ssp: &file->device->disassociate_srcu, idx: srcu_key);
686 return (ret) ? : count;
687}
688
689static const struct vm_operations_struct rdma_umap_ops;
690
691static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)
692{
693 struct ib_uverbs_file *file = filp->private_data;
694 struct ib_ucontext *ucontext;
695 int ret = 0;
696 int srcu_key;
697
698 srcu_key = srcu_read_lock(ssp: &file->device->disassociate_srcu);
699 ucontext = ib_uverbs_get_ucontext_file(file);
700 if (IS_ERR(ptr: ucontext)) {
701 ret = PTR_ERR(ptr: ucontext);
702 goto out;
703 }
704
705 mutex_lock(&file->disassociation_lock);
706
707 vma->vm_ops = &rdma_umap_ops;
708 ret = ucontext->device->ops.mmap(ucontext, vma);
709
710 mutex_unlock(lock: &file->disassociation_lock);
711out:
712 srcu_read_unlock(ssp: &file->device->disassociate_srcu, idx: srcu_key);
713 return ret;
714}
715
716/*
717 * The VMA has been dup'd, initialize the vm_private_data with a new tracking
718 * struct
719 */
720static void rdma_umap_open(struct vm_area_struct *vma)
721{
722 struct ib_uverbs_file *ufile = vma->vm_file->private_data;
723 struct rdma_umap_priv *opriv = vma->vm_private_data;
724 struct rdma_umap_priv *priv;
725
726 if (!opriv)
727 return;
728
729 /* We are racing with disassociation */
730 if (!down_read_trylock(sem: &ufile->hw_destroy_rwsem))
731 goto out_zap;
732 mutex_lock(&ufile->disassociation_lock);
733
734 /*
735 * Disassociation already completed, the VMA should already be zapped.
736 */
737 if (!ufile->ucontext)
738 goto out_unlock;
739
740 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
741 if (!priv)
742 goto out_unlock;
743 rdma_umap_priv_init(priv, vma, entry: opriv->entry);
744
745 mutex_unlock(lock: &ufile->disassociation_lock);
746 up_read(sem: &ufile->hw_destroy_rwsem);
747 return;
748
749out_unlock:
750 mutex_unlock(lock: &ufile->disassociation_lock);
751 up_read(sem: &ufile->hw_destroy_rwsem);
752out_zap:
753 /*
754 * We can't allow the VMA to be created with the actual IO pages, that
755 * would break our API contract, and it can't be stopped at this
756 * point, so zap it.
757 */
758 vma->vm_private_data = NULL;
759 zap_vma_ptes(vma, address: vma->vm_start, size: vma->vm_end - vma->vm_start);
760}
761
762static void rdma_umap_close(struct vm_area_struct *vma)
763{
764 struct ib_uverbs_file *ufile = vma->vm_file->private_data;
765 struct rdma_umap_priv *priv = vma->vm_private_data;
766
767 if (!priv)
768 return;
769
770 /*
771 * The vma holds a reference on the struct file that created it, which
772 * in turn means that the ib_uverbs_file is guaranteed to exist at
773 * this point.
774 */
775 mutex_lock(&ufile->umap_lock);
776 if (priv->entry)
777 rdma_user_mmap_entry_put(entry: priv->entry);
778
779 list_del(entry: &priv->list);
780 mutex_unlock(lock: &ufile->umap_lock);
781 kfree(objp: priv);
782}
783
784/*
785 * Once the zap_vma_ptes has been called touches to the VMA will come here and
786 * we return a dummy writable zero page for all the pfns.
787 */
788static vm_fault_t rdma_umap_fault(struct vm_fault *vmf)
789{
790 struct ib_uverbs_file *ufile = vmf->vma->vm_file->private_data;
791 struct rdma_umap_priv *priv = vmf->vma->vm_private_data;
792 vm_fault_t ret = 0;
793
794 if (!priv)
795 return VM_FAULT_SIGBUS;
796
797 /* Read only pages can just use the system zero page. */
798 if (!(vmf->vma->vm_flags & (VM_WRITE | VM_MAYWRITE))) {
799 vmf->page = ZERO_PAGE(vmf->address);
800 get_page(page: vmf->page);
801 return 0;
802 }
803
804 mutex_lock(&ufile->umap_lock);
805 if (!ufile->disassociate_page)
806 ufile->disassociate_page =
807 alloc_pages(vmf->gfp_mask | __GFP_ZERO, 0);
808
809 if (ufile->disassociate_page) {
810 /*
811 * This VMA is forced to always be shared so this doesn't have
812 * to worry about COW.
813 */
814 vmf->page = ufile->disassociate_page;
815 get_page(page: vmf->page);
816 } else {
817 ret = VM_FAULT_SIGBUS;
818 }
819 mutex_unlock(lock: &ufile->umap_lock);
820
821 return ret;
822}
823
824static const struct vm_operations_struct rdma_umap_ops = {
825 .open = rdma_umap_open,
826 .close = rdma_umap_close,
827 .fault = rdma_umap_fault,
828};
829
830void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
831{
832 struct rdma_umap_priv *priv, *next_priv;
833
834 mutex_lock(&ufile->disassociation_lock);
835
836 while (1) {
837 struct mm_struct *mm = NULL;
838
839 /* Get an arbitrary mm pointer that hasn't been cleaned yet */
840 mutex_lock(&ufile->umap_lock);
841 while (!list_empty(head: &ufile->umaps)) {
842 int ret;
843
844 priv = list_first_entry(&ufile->umaps,
845 struct rdma_umap_priv, list);
846 mm = priv->vma->vm_mm;
847 ret = mmget_not_zero(mm);
848 if (!ret) {
849 list_del_init(entry: &priv->list);
850 if (priv->entry) {
851 rdma_user_mmap_entry_put(entry: priv->entry);
852 priv->entry = NULL;
853 }
854 mm = NULL;
855 continue;
856 }
857 break;
858 }
859 mutex_unlock(lock: &ufile->umap_lock);
860 if (!mm) {
861 mutex_unlock(lock: &ufile->disassociation_lock);
862 return;
863 }
864
865 /*
866 * The umap_lock is nested under mmap_lock since it used within
867 * the vma_ops callbacks, so we have to clean the list one mm
868 * at a time to get the lock ordering right. Typically there
869 * will only be one mm, so no big deal.
870 */
871 mmap_read_lock(mm);
872 mutex_lock(&ufile->umap_lock);
873 list_for_each_entry_safe (priv, next_priv, &ufile->umaps,
874 list) {
875 struct vm_area_struct *vma = priv->vma;
876
877 if (vma->vm_mm != mm)
878 continue;
879 list_del_init(entry: &priv->list);
880
881 zap_vma_ptes(vma, address: vma->vm_start,
882 size: vma->vm_end - vma->vm_start);
883
884 if (priv->entry) {
885 rdma_user_mmap_entry_put(entry: priv->entry);
886 priv->entry = NULL;
887 }
888 }
889 mutex_unlock(lock: &ufile->umap_lock);
890 mmap_read_unlock(mm);
891 mmput(mm);
892 }
893
894 mutex_unlock(lock: &ufile->disassociation_lock);
895}
896
897/**
898 * rdma_user_mmap_disassociate() - Revoke mmaps for a device
899 * @device: device to revoke
900 *
901 * This function should be called by drivers that need to disable mmaps for the
902 * device, for instance because it is going to be reset.
903 */
904void rdma_user_mmap_disassociate(struct ib_device *device)
905{
906 struct ib_uverbs_device *uverbs_dev =
907 ib_get_client_data(device, client: &uverbs_client);
908 struct ib_uverbs_file *ufile;
909
910 mutex_lock(&uverbs_dev->lists_mutex);
911 list_for_each_entry(ufile, &uverbs_dev->uverbs_file_list, list) {
912 if (ufile->ucontext)
913 uverbs_user_mmap_disassociate(ufile);
914 }
915 mutex_unlock(lock: &uverbs_dev->lists_mutex);
916}
917EXPORT_SYMBOL(rdma_user_mmap_disassociate);
918
919/*
920 * ib_uverbs_open() does not need the BKL:
921 *
922 * - the ib_uverbs_device structures are properly reference counted and
923 * everything else is purely local to the file being created, so
924 * races against other open calls are not a problem;
925 * - there is no ioctl method to race against;
926 * - the open method will either immediately run -ENXIO, or all
927 * required initialization will be done.
928 */
929static int ib_uverbs_open(struct inode *inode, struct file *filp)
930{
931 struct ib_uverbs_device *dev;
932 struct ib_uverbs_file *file;
933 struct ib_device *ib_dev;
934 int ret;
935 int module_dependent;
936 int srcu_key;
937
938 dev = container_of(inode->i_cdev, struct ib_uverbs_device, cdev);
939 if (!refcount_inc_not_zero(r: &dev->refcount))
940 return -ENXIO;
941
942 get_device(dev: &dev->dev);
943 srcu_key = srcu_read_lock(ssp: &dev->disassociate_srcu);
944 mutex_lock(&dev->lists_mutex);
945 ib_dev = srcu_dereference(dev->ib_dev,
946 &dev->disassociate_srcu);
947 if (!ib_dev) {
948 ret = -EIO;
949 goto err;
950 }
951
952 if (!rdma_dev_access_netns(device: ib_dev, current->nsproxy->net_ns)) {
953 ret = -EPERM;
954 goto err;
955 }
956
957 /* In case IB device supports disassociate ucontext, there is no hard
958 * dependency between uverbs device and its low level device.
959 */
960 module_dependent = !(ib_dev->ops.disassociate_ucontext);
961
962 if (module_dependent) {
963 if (!try_module_get(module: ib_dev->ops.owner)) {
964 ret = -ENODEV;
965 goto err;
966 }
967 }
968
969 file = kzalloc(sizeof(*file), GFP_KERNEL);
970 if (!file) {
971 ret = -ENOMEM;
972 if (module_dependent)
973 goto err_module;
974
975 goto err;
976 }
977
978 file->device = dev;
979 kref_init(kref: &file->ref);
980 mutex_init(&file->ucontext_lock);
981
982 spin_lock_init(&file->uobjects_lock);
983 INIT_LIST_HEAD(list: &file->uobjects);
984 init_rwsem(&file->hw_destroy_rwsem);
985 mutex_init(&file->umap_lock);
986 INIT_LIST_HEAD(list: &file->umaps);
987
988 mutex_init(&file->disassociation_lock);
989
990 filp->private_data = file;
991 list_add_tail(new: &file->list, head: &dev->uverbs_file_list);
992 mutex_unlock(lock: &dev->lists_mutex);
993 srcu_read_unlock(ssp: &dev->disassociate_srcu, idx: srcu_key);
994
995 setup_ufile_idr_uobject(file);
996
997 return stream_open(inode, filp);
998
999err_module:
1000 module_put(module: ib_dev->ops.owner);
1001
1002err:
1003 mutex_unlock(lock: &dev->lists_mutex);
1004 srcu_read_unlock(ssp: &dev->disassociate_srcu, idx: srcu_key);
1005 if (refcount_dec_and_test(r: &dev->refcount))
1006 ib_uverbs_comp_dev(dev);
1007
1008 put_device(dev: &dev->dev);
1009 return ret;
1010}
1011
1012static int ib_uverbs_close(struct inode *inode, struct file *filp)
1013{
1014 struct ib_uverbs_file *file = filp->private_data;
1015
1016 uverbs_destroy_ufile_hw(ufile: file, reason: RDMA_REMOVE_CLOSE);
1017
1018 mutex_lock(&file->device->lists_mutex);
1019 list_del_init(entry: &file->list);
1020 mutex_unlock(lock: &file->device->lists_mutex);
1021
1022 kref_put(kref: &file->ref, release: ib_uverbs_release_file);
1023
1024 return 0;
1025}
1026
1027static const struct file_operations uverbs_fops = {
1028 .owner = THIS_MODULE,
1029 .write = ib_uverbs_write,
1030 .open = ib_uverbs_open,
1031 .release = ib_uverbs_close,
1032 .unlocked_ioctl = ib_uverbs_ioctl,
1033 .compat_ioctl = compat_ptr_ioctl,
1034};
1035
1036static const struct file_operations uverbs_mmap_fops = {
1037 .owner = THIS_MODULE,
1038 .write = ib_uverbs_write,
1039 .mmap = ib_uverbs_mmap,
1040 .open = ib_uverbs_open,
1041 .release = ib_uverbs_close,
1042 .unlocked_ioctl = ib_uverbs_ioctl,
1043 .compat_ioctl = compat_ptr_ioctl,
1044};
1045
1046static int ib_uverbs_get_nl_info(struct ib_device *ibdev, void *client_data,
1047 struct ib_client_nl_info *res)
1048{
1049 struct ib_uverbs_device *uverbs_dev = client_data;
1050 int ret;
1051
1052 if (res->port != -1)
1053 return -EINVAL;
1054
1055 res->abi = ibdev->ops.uverbs_abi_ver;
1056 res->cdev = &uverbs_dev->dev;
1057
1058 /*
1059 * To support DRIVER_ID binding in userspace some of the driver need
1060 * upgrading to expose their PCI dependent revision information
1061 * through get_context instead of relying on modalias matching. When
1062 * the drivers are fixed they can drop this flag.
1063 */
1064 if (!ibdev->ops.uverbs_no_driver_id_binding) {
1065 ret = nla_put_u32(skb: res->nl_msg, attrtype: RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID,
1066 value: ibdev->ops.driver_id);
1067 if (ret)
1068 return ret;
1069 }
1070 return 0;
1071}
1072
1073static struct ib_client uverbs_client = {
1074 .name = "uverbs",
1075 .no_kverbs_req = true,
1076 .add = ib_uverbs_add_one,
1077 .remove = ib_uverbs_remove_one,
1078 .get_nl_info = ib_uverbs_get_nl_info,
1079};
1080MODULE_ALIAS_RDMA_CLIENT("uverbs");
1081
1082static ssize_t ibdev_show(struct device *device, struct device_attribute *attr,
1083 char *buf)
1084{
1085 struct ib_uverbs_device *dev =
1086 container_of(device, struct ib_uverbs_device, dev);
1087 int ret = -ENODEV;
1088 int srcu_key;
1089 struct ib_device *ib_dev;
1090
1091 srcu_key = srcu_read_lock(ssp: &dev->disassociate_srcu);
1092 ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1093 if (ib_dev)
1094 ret = sysfs_emit(buf, fmt: "%s\n", dev_name(dev: &ib_dev->dev));
1095 srcu_read_unlock(ssp: &dev->disassociate_srcu, idx: srcu_key);
1096
1097 return ret;
1098}
1099static DEVICE_ATTR_RO(ibdev);
1100
1101static ssize_t abi_version_show(struct device *device,
1102 struct device_attribute *attr, char *buf)
1103{
1104 struct ib_uverbs_device *dev =
1105 container_of(device, struct ib_uverbs_device, dev);
1106 int ret = -ENODEV;
1107 int srcu_key;
1108 struct ib_device *ib_dev;
1109
1110 srcu_key = srcu_read_lock(ssp: &dev->disassociate_srcu);
1111 ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1112 if (ib_dev)
1113 ret = sysfs_emit(buf, fmt: "%u\n", ib_dev->ops.uverbs_abi_ver);
1114 srcu_read_unlock(ssp: &dev->disassociate_srcu, idx: srcu_key);
1115
1116 return ret;
1117}
1118static DEVICE_ATTR_RO(abi_version);
1119
1120static struct attribute *ib_dev_attrs[] = {
1121 &dev_attr_abi_version.attr,
1122 &dev_attr_ibdev.attr,
1123 NULL,
1124};
1125
1126static const struct attribute_group dev_attr_group = {
1127 .attrs = ib_dev_attrs,
1128};
1129
1130static CLASS_ATTR_STRING(abi_version, S_IRUGO,
1131 __stringify(IB_USER_VERBS_ABI_VERSION));
1132
1133static int ib_uverbs_create_uapi(struct ib_device *device,
1134 struct ib_uverbs_device *uverbs_dev)
1135{
1136 struct uverbs_api *uapi;
1137
1138 uapi = uverbs_alloc_api(ibdev: device);
1139 if (IS_ERR(ptr: uapi))
1140 return PTR_ERR(ptr: uapi);
1141
1142 uverbs_dev->uapi = uapi;
1143 return 0;
1144}
1145
1146static int ib_uverbs_add_one(struct ib_device *device)
1147{
1148 int devnum;
1149 dev_t base;
1150 struct ib_uverbs_device *uverbs_dev;
1151 int ret;
1152
1153 if (!device->ops.alloc_ucontext ||
1154 device->type == RDMA_DEVICE_TYPE_SMI)
1155 return -EOPNOTSUPP;
1156
1157 uverbs_dev = kzalloc(sizeof(*uverbs_dev), GFP_KERNEL);
1158 if (!uverbs_dev)
1159 return -ENOMEM;
1160
1161 ret = init_srcu_struct(&uverbs_dev->disassociate_srcu);
1162 if (ret) {
1163 kfree(objp: uverbs_dev);
1164 return -ENOMEM;
1165 }
1166
1167 device_initialize(dev: &uverbs_dev->dev);
1168 uverbs_dev->dev.class = &uverbs_class;
1169 uverbs_dev->dev.parent = device->dev.parent;
1170 uverbs_dev->dev.release = ib_uverbs_release_dev;
1171 uverbs_dev->groups[0] = &dev_attr_group;
1172 uverbs_dev->dev.groups = uverbs_dev->groups;
1173 refcount_set(r: &uverbs_dev->refcount, n: 1);
1174 init_completion(x: &uverbs_dev->comp);
1175 uverbs_dev->xrcd_tree = RB_ROOT;
1176 mutex_init(&uverbs_dev->xrcd_tree_mutex);
1177 mutex_init(&uverbs_dev->lists_mutex);
1178 INIT_LIST_HEAD(list: &uverbs_dev->uverbs_file_list);
1179 rcu_assign_pointer(uverbs_dev->ib_dev, device);
1180 uverbs_dev->num_comp_vectors = device->num_comp_vectors;
1181
1182 devnum = ida_alloc_max(ida: &uverbs_ida, max: IB_UVERBS_MAX_DEVICES - 1,
1183 GFP_KERNEL);
1184 if (devnum < 0) {
1185 ret = -ENOMEM;
1186 goto err;
1187 }
1188 uverbs_dev->devnum = devnum;
1189 if (devnum >= IB_UVERBS_NUM_FIXED_MINOR)
1190 base = dynamic_uverbs_dev + devnum - IB_UVERBS_NUM_FIXED_MINOR;
1191 else
1192 base = IB_UVERBS_BASE_DEV + devnum;
1193
1194 ret = ib_uverbs_create_uapi(device, uverbs_dev);
1195 if (ret)
1196 goto err_uapi;
1197
1198 uverbs_dev->dev.devt = base;
1199 dev_set_name(dev: &uverbs_dev->dev, name: "uverbs%d", uverbs_dev->devnum);
1200
1201 cdev_init(&uverbs_dev->cdev,
1202 device->ops.mmap ? &uverbs_mmap_fops : &uverbs_fops);
1203 uverbs_dev->cdev.owner = THIS_MODULE;
1204
1205 ret = cdev_device_add(cdev: &uverbs_dev->cdev, dev: &uverbs_dev->dev);
1206 if (ret)
1207 goto err_uapi;
1208
1209 ib_set_client_data(device, client: &uverbs_client, data: uverbs_dev);
1210 return 0;
1211
1212err_uapi:
1213 ida_free(&uverbs_ida, id: devnum);
1214err:
1215 if (refcount_dec_and_test(r: &uverbs_dev->refcount))
1216 ib_uverbs_comp_dev(dev: uverbs_dev);
1217 wait_for_completion(&uverbs_dev->comp);
1218 put_device(dev: &uverbs_dev->dev);
1219 return ret;
1220}
1221
1222static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,
1223 struct ib_device *ib_dev)
1224{
1225 struct ib_uverbs_file *file;
1226
1227 /* Pending running commands to terminate */
1228 uverbs_disassociate_api_pre(uverbs_dev);
1229
1230 mutex_lock(&uverbs_dev->lists_mutex);
1231 while (!list_empty(head: &uverbs_dev->uverbs_file_list)) {
1232 file = list_first_entry(&uverbs_dev->uverbs_file_list,
1233 struct ib_uverbs_file, list);
1234 list_del_init(entry: &file->list);
1235 kref_get(kref: &file->ref);
1236
1237 /* We must release the mutex before going ahead and calling
1238 * uverbs_cleanup_ufile, as it might end up indirectly calling
1239 * uverbs_close, for example due to freeing the resources (e.g
1240 * mmput).
1241 */
1242 mutex_unlock(lock: &uverbs_dev->lists_mutex);
1243
1244 uverbs_destroy_ufile_hw(ufile: file, reason: RDMA_REMOVE_DRIVER_REMOVE);
1245 kref_put(kref: &file->ref, release: ib_uverbs_release_file);
1246
1247 mutex_lock(&uverbs_dev->lists_mutex);
1248 }
1249 mutex_unlock(lock: &uverbs_dev->lists_mutex);
1250
1251 uverbs_disassociate_api(uapi: uverbs_dev->uapi);
1252}
1253
1254static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)
1255{
1256 struct ib_uverbs_device *uverbs_dev = client_data;
1257 int wait_clients = 1;
1258
1259 cdev_device_del(cdev: &uverbs_dev->cdev, dev: &uverbs_dev->dev);
1260 ida_free(&uverbs_ida, id: uverbs_dev->devnum);
1261
1262 if (device->ops.disassociate_ucontext) {
1263 /* We disassociate HW resources and immediately return.
1264 * Userspace will see a EIO errno for all future access.
1265 * Upon returning, ib_device may be freed internally and is not
1266 * valid any more.
1267 * uverbs_device is still available until all clients close
1268 * their files, then the uverbs device ref count will be zero
1269 * and its resources will be freed.
1270 * Note: At this point no more files can be opened since the
1271 * cdev was deleted, however active clients can still issue
1272 * commands and close their open files.
1273 */
1274 ib_uverbs_free_hw_resources(uverbs_dev, ib_dev: device);
1275 wait_clients = 0;
1276 }
1277
1278 if (refcount_dec_and_test(r: &uverbs_dev->refcount))
1279 ib_uverbs_comp_dev(dev: uverbs_dev);
1280 if (wait_clients)
1281 wait_for_completion(&uverbs_dev->comp);
1282
1283 put_device(dev: &uverbs_dev->dev);
1284}
1285
1286static int __init ib_uverbs_init(void)
1287{
1288 int ret;
1289
1290 ret = register_chrdev_region(IB_UVERBS_BASE_DEV,
1291 IB_UVERBS_NUM_FIXED_MINOR,
1292 "infiniband_verbs");
1293 if (ret) {
1294 pr_err("user_verbs: couldn't register device number\n");
1295 goto out;
1296 }
1297
1298 ret = alloc_chrdev_region(&dynamic_uverbs_dev, 0,
1299 IB_UVERBS_NUM_DYNAMIC_MINOR,
1300 "infiniband_verbs");
1301 if (ret) {
1302 pr_err("couldn't register dynamic device number\n");
1303 goto out_alloc;
1304 }
1305
1306 ret = class_register(class: &uverbs_class);
1307 if (ret) {
1308 pr_err("user_verbs: couldn't create class infiniband_verbs\n");
1309 goto out_chrdev;
1310 }
1311
1312 ret = class_create_file(class: &uverbs_class, attr: &class_attr_abi_version.attr);
1313 if (ret) {
1314 pr_err("user_verbs: couldn't create abi_version attribute\n");
1315 goto out_class;
1316 }
1317
1318 ret = ib_register_client(client: &uverbs_client);
1319 if (ret) {
1320 pr_err("user_verbs: couldn't register client\n");
1321 goto out_class;
1322 }
1323
1324 return 0;
1325
1326out_class:
1327 class_unregister(class: &uverbs_class);
1328
1329out_chrdev:
1330 unregister_chrdev_region(dynamic_uverbs_dev,
1331 IB_UVERBS_NUM_DYNAMIC_MINOR);
1332
1333out_alloc:
1334 unregister_chrdev_region(IB_UVERBS_BASE_DEV,
1335 IB_UVERBS_NUM_FIXED_MINOR);
1336
1337out:
1338 return ret;
1339}
1340
1341static void __exit ib_uverbs_cleanup(void)
1342{
1343 ib_unregister_client(client: &uverbs_client);
1344 class_unregister(class: &uverbs_class);
1345 unregister_chrdev_region(IB_UVERBS_BASE_DEV,
1346 IB_UVERBS_NUM_FIXED_MINOR);
1347 unregister_chrdev_region(dynamic_uverbs_dev,
1348 IB_UVERBS_NUM_DYNAMIC_MINOR);
1349 ib_cleanup_ucaps();
1350 mmu_notifier_synchronize();
1351}
1352
1353module_init(ib_uverbs_init);
1354module_exit(ib_uverbs_cleanup);
1355

source code of linux/drivers/infiniband/core/uverbs_main.c