linux/drivers/infiniband/core/uverbs_main.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2005 Topspin Communications.  All rights reserved.
   3 * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
   4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5 * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
   6 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
   7 *
   8 * This software is available to you under a choice of one of two
   9 * licenses.  You may choose to be licensed under the terms of the GNU
  10 * General Public License (GPL) Version 2, available from the file
  11 * COPYING in the main directory of this source tree, or the
  12 * OpenIB.org BSD license below:
  13 *
  14 *     Redistribution and use in source and binary forms, with or
  15 *     without modification, are permitted provided that the following
  16 *     conditions are met:
  17 *
  18 *      - Redistributions of source code must retain the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer.
  21 *
  22 *      - Redistributions in binary form must reproduce the above
  23 *        copyright notice, this list of conditions and the following
  24 *        disclaimer in the documentation and/or other materials
  25 *        provided with the distribution.
  26 *
  27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34 * SOFTWARE.
  35 */
  36
  37#include <linux/module.h>
  38#include <linux/init.h>
  39#include <linux/device.h>
  40#include <linux/err.h>
  41#include <linux/fs.h>
  42#include <linux/poll.h>
  43#include <linux/sched.h>
  44#include <linux/file.h>
  45#include <linux/cdev.h>
  46#include <linux/anon_inodes.h>
  47#include <linux/slab.h>
  48#include <linux/sched/mm.h>
  49
  50#include <linux/uaccess.h>
  51
  52#include <rdma/ib.h>
  53#include <rdma/uverbs_std_types.h>
  54#include <rdma/rdma_netlink.h>
  55
  56#include "uverbs.h"
  57#include "core_priv.h"
  58#include "rdma_core.h"
  59
  60MODULE_AUTHOR("Roland Dreier");
  61MODULE_DESCRIPTION("InfiniBand userspace verbs access");
  62MODULE_LICENSE("Dual BSD/GPL");
  63
  64enum {
  65        IB_UVERBS_MAJOR       = 231,
  66        IB_UVERBS_BASE_MINOR  = 192,
  67        IB_UVERBS_MAX_DEVICES = RDMA_MAX_PORTS,
  68        IB_UVERBS_NUM_FIXED_MINOR = 32,
  69        IB_UVERBS_NUM_DYNAMIC_MINOR = IB_UVERBS_MAX_DEVICES - IB_UVERBS_NUM_FIXED_MINOR,
  70};
  71
  72#define IB_UVERBS_BASE_DEV      MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR)
  73
  74static dev_t dynamic_uverbs_dev;
  75static struct class *uverbs_class;
  76
  77static DEFINE_IDA(uverbs_ida);
  78static int ib_uverbs_add_one(struct ib_device *device);
  79static void ib_uverbs_remove_one(struct ib_device *device, void *client_data);
  80
  81/*
  82 * Must be called with the ufile->device->disassociate_srcu held, and the lock
  83 * must be held until use of the ucontext is finished.
  84 */
  85struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile)
  86{
  87        /*
  88         * We do not hold the hw_destroy_rwsem lock for this flow, instead
  89         * srcu is used. It does not matter if someone races this with
  90         * get_context, we get NULL or valid ucontext.
  91         */
  92        struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext);
  93
  94        if (!srcu_dereference(ufile->device->ib_dev,
  95                              &ufile->device->disassociate_srcu))
  96                return ERR_PTR(-EIO);
  97
  98        if (!ucontext)
  99                return ERR_PTR(-EINVAL);
 100
 101        return ucontext;
 102}
 103EXPORT_SYMBOL(ib_uverbs_get_ucontext_file);
 104
 105int uverbs_dealloc_mw(struct ib_mw *mw)
 106{
 107        struct ib_pd *pd = mw->pd;
 108        int ret;
 109
 110        ret = mw->device->ops.dealloc_mw(mw);
 111        if (ret)
 112                return ret;
 113
 114        atomic_dec(&pd->usecnt);
 115        kfree(mw);
 116        return ret;
 117}
 118
 119static void ib_uverbs_release_dev(struct device *device)
 120{
 121        struct ib_uverbs_device *dev =
 122                        container_of(device, struct ib_uverbs_device, dev);
 123
 124        uverbs_destroy_api(dev->uapi);
 125        cleanup_srcu_struct(&dev->disassociate_srcu);
 126        mutex_destroy(&dev->lists_mutex);
 127        mutex_destroy(&dev->xrcd_tree_mutex);
 128        kfree(dev);
 129}
 130
 131void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file,
 132                           struct ib_ucq_object *uobj)
 133{
 134        struct ib_uverbs_event *evt, *tmp;
 135
 136        if (ev_file) {
 137                spin_lock_irq(&ev_file->ev_queue.lock);
 138                list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) {
 139                        list_del(&evt->list);
 140                        kfree(evt);
 141                }
 142                spin_unlock_irq(&ev_file->ev_queue.lock);
 143
 144                uverbs_uobject_put(&ev_file->uobj);
 145        }
 146
 147        ib_uverbs_release_uevent(&uobj->uevent);
 148}
 149
 150void ib_uverbs_release_uevent(struct ib_uevent_object *uobj)
 151{
 152        struct ib_uverbs_async_event_file *async_file = uobj->event_file;
 153        struct ib_uverbs_event *evt, *tmp;
 154
 155        if (!async_file)
 156                return;
 157
 158        spin_lock_irq(&async_file->ev_queue.lock);
 159        list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) {
 160                list_del(&evt->list);
 161                kfree(evt);
 162        }
 163        spin_unlock_irq(&async_file->ev_queue.lock);
 164        uverbs_uobject_put(&async_file->uobj);
 165}
 166
 167void ib_uverbs_detach_umcast(struct ib_qp *qp,
 168                             struct ib_uqp_object *uobj)
 169{
 170        struct ib_uverbs_mcast_entry *mcast, *tmp;
 171
 172        list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) {
 173                ib_detach_mcast(qp, &mcast->gid, mcast->lid);
 174                list_del(&mcast->list);
 175                kfree(mcast);
 176        }
 177}
 178
 179static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev)
 180{
 181        complete(&dev->comp);
 182}
 183
 184void ib_uverbs_release_file(struct kref *ref)
 185{
 186        struct ib_uverbs_file *file =
 187                container_of(ref, struct ib_uverbs_file, ref);
 188        struct ib_device *ib_dev;
 189        int srcu_key;
 190
 191        release_ufile_idr_uobject(file);
 192
 193        srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
 194        ib_dev = srcu_dereference(file->device->ib_dev,
 195                                  &file->device->disassociate_srcu);
 196        if (ib_dev && !ib_dev->ops.disassociate_ucontext)
 197                module_put(ib_dev->ops.owner);
 198        srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 199
 200        if (refcount_dec_and_test(&file->device->refcount))
 201                ib_uverbs_comp_dev(file->device);
 202
 203        if (file->default_async_file)
 204                uverbs_uobject_put(&file->default_async_file->uobj);
 205        put_device(&file->device->dev);
 206
 207        if (file->disassociate_page)
 208                __free_pages(file->disassociate_page, 0);
 209        mutex_destroy(&file->umap_lock);
 210        mutex_destroy(&file->ucontext_lock);
 211        kfree(file);
 212}
 213
 214static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue,
 215                                    struct file *filp, char __user *buf,
 216                                    size_t count, loff_t *pos,
 217                                    size_t eventsz)
 218{
 219        struct ib_uverbs_event *event;
 220        int ret = 0;
 221
 222        spin_lock_irq(&ev_queue->lock);
 223
 224        while (list_empty(&ev_queue->event_list)) {
 225                spin_unlock_irq(&ev_queue->lock);
 226
 227                if (filp->f_flags & O_NONBLOCK)
 228                        return -EAGAIN;
 229
 230                if (wait_event_interruptible(ev_queue->poll_wait,
 231                                             (!list_empty(&ev_queue->event_list) ||
 232                                              ev_queue->is_closed)))
 233                        return -ERESTARTSYS;
 234
 235                spin_lock_irq(&ev_queue->lock);
 236
 237                /* If device was disassociated and no event exists set an error */
 238                if (list_empty(&ev_queue->event_list) && ev_queue->is_closed) {
 239                        spin_unlock_irq(&ev_queue->lock);
 240                        return -EIO;
 241                }
 242        }
 243
 244        event = list_entry(ev_queue->event_list.next, struct ib_uverbs_event, list);
 245
 246        if (eventsz > count) {
 247                ret   = -EINVAL;
 248                event = NULL;
 249        } else {
 250                list_del(ev_queue->event_list.next);
 251                if (event->counter) {
 252                        ++(*event->counter);
 253                        list_del(&event->obj_list);
 254                }
 255        }
 256
 257        spin_unlock_irq(&ev_queue->lock);
 258
 259        if (event) {
 260                if (copy_to_user(buf, event, eventsz))
 261                        ret = -EFAULT;
 262                else
 263                        ret = eventsz;
 264        }
 265
 266        kfree(event);
 267
 268        return ret;
 269}
 270
 271static ssize_t ib_uverbs_async_event_read(struct file *filp, char __user *buf,
 272                                          size_t count, loff_t *pos)
 273{
 274        struct ib_uverbs_async_event_file *file = filp->private_data;
 275
 276        return ib_uverbs_event_read(&file->ev_queue, filp, buf, count, pos,
 277                                    sizeof(struct ib_uverbs_async_event_desc));
 278}
 279
 280static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf,
 281                                         size_t count, loff_t *pos)
 282{
 283        struct ib_uverbs_completion_event_file *comp_ev_file =
 284                filp->private_data;
 285
 286        return ib_uverbs_event_read(&comp_ev_file->ev_queue, filp, buf, count,
 287                                    pos,
 288                                    sizeof(struct ib_uverbs_comp_event_desc));
 289}
 290
 291static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue,
 292                                         struct file *filp,
 293                                         struct poll_table_struct *wait)
 294{
 295        __poll_t pollflags = 0;
 296
 297        poll_wait(filp, &ev_queue->poll_wait, wait);
 298
 299        spin_lock_irq(&ev_queue->lock);
 300        if (!list_empty(&ev_queue->event_list))
 301                pollflags = EPOLLIN | EPOLLRDNORM;
 302        else if (ev_queue->is_closed)
 303                pollflags = EPOLLERR;
 304        spin_unlock_irq(&ev_queue->lock);
 305
 306        return pollflags;
 307}
 308
 309static __poll_t ib_uverbs_async_event_poll(struct file *filp,
 310                                               struct poll_table_struct *wait)
 311{
 312        struct ib_uverbs_async_event_file *file = filp->private_data;
 313
 314        return ib_uverbs_event_poll(&file->ev_queue, filp, wait);
 315}
 316
 317static __poll_t ib_uverbs_comp_event_poll(struct file *filp,
 318                                              struct poll_table_struct *wait)
 319{
 320        struct ib_uverbs_completion_event_file *comp_ev_file =
 321                filp->private_data;
 322
 323        return ib_uverbs_event_poll(&comp_ev_file->ev_queue, filp, wait);
 324}
 325
 326static int ib_uverbs_async_event_fasync(int fd, struct file *filp, int on)
 327{
 328        struct ib_uverbs_async_event_file *file = filp->private_data;
 329
 330        return fasync_helper(fd, filp, on, &file->ev_queue.async_queue);
 331}
 332
 333static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on)
 334{
 335        struct ib_uverbs_completion_event_file *comp_ev_file =
 336                filp->private_data;
 337
 338        return fasync_helper(fd, filp, on, &comp_ev_file->ev_queue.async_queue);
 339}
 340
 341const struct file_operations uverbs_event_fops = {
 342        .owner   = THIS_MODULE,
 343        .read    = ib_uverbs_comp_event_read,
 344        .poll    = ib_uverbs_comp_event_poll,
 345        .release = uverbs_uobject_fd_release,
 346        .fasync  = ib_uverbs_comp_event_fasync,
 347        .llseek  = no_llseek,
 348};
 349
 350const struct file_operations uverbs_async_event_fops = {
 351        .owner   = THIS_MODULE,
 352        .read    = ib_uverbs_async_event_read,
 353        .poll    = ib_uverbs_async_event_poll,
 354        .release = uverbs_async_event_release,
 355        .fasync  = ib_uverbs_async_event_fasync,
 356        .llseek  = no_llseek,
 357};
 358
 359void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context)
 360{
 361        struct ib_uverbs_event_queue   *ev_queue = cq_context;
 362        struct ib_ucq_object           *uobj;
 363        struct ib_uverbs_event         *entry;
 364        unsigned long                   flags;
 365
 366        if (!ev_queue)
 367                return;
 368
 369        spin_lock_irqsave(&ev_queue->lock, flags);
 370        if (ev_queue->is_closed) {
 371                spin_unlock_irqrestore(&ev_queue->lock, flags);
 372                return;
 373        }
 374
 375        entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
 376        if (!entry) {
 377                spin_unlock_irqrestore(&ev_queue->lock, flags);
 378                return;
 379        }
 380
 381        uobj = cq->uobject;
 382
 383        entry->desc.comp.cq_handle = cq->uobject->uevent.uobject.user_handle;
 384        entry->counter             = &uobj->comp_events_reported;
 385
 386        list_add_tail(&entry->list, &ev_queue->event_list);
 387        list_add_tail(&entry->obj_list, &uobj->comp_list);
 388        spin_unlock_irqrestore(&ev_queue->lock, flags);
 389
 390        wake_up_interruptible(&ev_queue->poll_wait);
 391        kill_fasync(&ev_queue->async_queue, SIGIO, POLL_IN);
 392}
 393
 394void ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file,
 395                             __u64 element, __u64 event,
 396                             struct list_head *obj_list, u32 *counter)
 397{
 398        struct ib_uverbs_event *entry;
 399        unsigned long flags;
 400
 401        if (!async_file)
 402                return;
 403
 404        spin_lock_irqsave(&async_file->ev_queue.lock, flags);
 405        if (async_file->ev_queue.is_closed) {
 406                spin_unlock_irqrestore(&async_file->ev_queue.lock, flags);
 407                return;
 408        }
 409
 410        entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
 411        if (!entry) {
 412                spin_unlock_irqrestore(&async_file->ev_queue.lock, flags);
 413                return;
 414        }
 415
 416        entry->desc.async.element = element;
 417        entry->desc.async.event_type = event;
 418        entry->desc.async.reserved = 0;
 419        entry->counter = counter;
 420
 421        list_add_tail(&entry->list, &async_file->ev_queue.event_list);
 422        if (obj_list)
 423                list_add_tail(&entry->obj_list, obj_list);
 424        spin_unlock_irqrestore(&async_file->ev_queue.lock, flags);
 425
 426        wake_up_interruptible(&async_file->ev_queue.poll_wait);
 427        kill_fasync(&async_file->ev_queue.async_queue, SIGIO, POLL_IN);
 428}
 429
 430static void uverbs_uobj_event(struct ib_uevent_object *eobj,
 431                              struct ib_event *event)
 432{
 433        ib_uverbs_async_handler(eobj->event_file,
 434                                eobj->uobject.user_handle, event->event,
 435                                &eobj->event_list, &eobj->events_reported);
 436}
 437
 438void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr)
 439{
 440        uverbs_uobj_event(&event->element.cq->uobject->uevent, event);
 441}
 442
 443void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr)
 444{
 445        /* for XRC target qp's, check that qp is live */
 446        if (!event->element.qp->uobject)
 447                return;
 448
 449        uverbs_uobj_event(&event->element.qp->uobject->uevent, event);
 450}
 451
 452void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr)
 453{
 454        uverbs_uobj_event(&event->element.wq->uobject->uevent, event);
 455}
 456
 457void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr)
 458{
 459        uverbs_uobj_event(&event->element.srq->uobject->uevent, event);
 460}
 461
 462static void ib_uverbs_event_handler(struct ib_event_handler *handler,
 463                                    struct ib_event *event)
 464{
 465        ib_uverbs_async_handler(
 466                container_of(handler, struct ib_uverbs_async_event_file,
 467                             event_handler),
 468                event->element.port_num, event->event, NULL, NULL);
 469}
 470
 471void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue)
 472{
 473        spin_lock_init(&ev_queue->lock);
 474        INIT_LIST_HEAD(&ev_queue->event_list);
 475        init_waitqueue_head(&ev_queue->poll_wait);
 476        ev_queue->is_closed   = 0;
 477        ev_queue->async_queue = NULL;
 478}
 479
 480void ib_uverbs_init_async_event_file(
 481        struct ib_uverbs_async_event_file *async_file)
 482{
 483        struct ib_uverbs_file *uverbs_file = async_file->uobj.ufile;
 484        struct ib_device *ib_dev = async_file->uobj.context->device;
 485
 486        ib_uverbs_init_event_queue(&async_file->ev_queue);
 487
 488        /* The first async_event_file becomes the default one for the file. */
 489        mutex_lock(&uverbs_file->ucontext_lock);
 490        if (!uverbs_file->default_async_file) {
 491                /* Pairs with the put in ib_uverbs_release_file */
 492                uverbs_uobject_get(&async_file->uobj);
 493                smp_store_release(&uverbs_file->default_async_file, async_file);
 494        }
 495        mutex_unlock(&uverbs_file->ucontext_lock);
 496
 497        INIT_IB_EVENT_HANDLER(&async_file->event_handler, ib_dev,
 498                              ib_uverbs_event_handler);
 499        ib_register_event_handler(&async_file->event_handler);
 500}
 501
 502static ssize_t verify_hdr(struct ib_uverbs_cmd_hdr *hdr,
 503                          struct ib_uverbs_ex_cmd_hdr *ex_hdr, size_t count,
 504                          const struct uverbs_api_write_method *method_elm)
 505{
 506        if (method_elm->is_ex) {
 507                count -= sizeof(*hdr) + sizeof(*ex_hdr);
 508
 509                if ((hdr->in_words + ex_hdr->provider_in_words) * 8 != count)
 510                        return -EINVAL;
 511
 512                if (hdr->in_words * 8 < method_elm->req_size)
 513                        return -ENOSPC;
 514
 515                if (ex_hdr->cmd_hdr_reserved)
 516                        return -EINVAL;
 517
 518                if (ex_hdr->response) {
 519                        if (!hdr->out_words && !ex_hdr->provider_out_words)
 520                                return -EINVAL;
 521
 522                        if (hdr->out_words * 8 < method_elm->resp_size)
 523                                return -ENOSPC;
 524
 525                        if (!access_ok(u64_to_user_ptr(ex_hdr->response),
 526                                       (hdr->out_words + ex_hdr->provider_out_words) * 8))
 527                                return -EFAULT;
 528                } else {
 529                        if (hdr->out_words || ex_hdr->provider_out_words)
 530                                return -EINVAL;
 531                }
 532
 533                return 0;
 534        }
 535
 536        /* not extended command */
 537        if (hdr->in_words * 4 != count)
 538                return -EINVAL;
 539
 540        if (count < method_elm->req_size + sizeof(hdr)) {
 541                /*
 542                 * rdma-core v18 and v19 have a bug where they send DESTROY_CQ
 543                 * with a 16 byte write instead of 24. Old kernels didn't
 544                 * check the size so they allowed this. Now that the size is
 545                 * checked provide a compatibility work around to not break
 546                 * those userspaces.
 547                 */
 548                if (hdr->command == IB_USER_VERBS_CMD_DESTROY_CQ &&
 549                    count == 16) {
 550                        hdr->in_words = 6;
 551                        return 0;
 552                }
 553                return -ENOSPC;
 554        }
 555        if (hdr->out_words * 4 < method_elm->resp_size)
 556                return -ENOSPC;
 557
 558        return 0;
 559}
 560
 561static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
 562                             size_t count, loff_t *pos)
 563{
 564        struct ib_uverbs_file *file = filp->private_data;
 565        const struct uverbs_api_write_method *method_elm;
 566        struct uverbs_api *uapi = file->device->uapi;
 567        struct ib_uverbs_ex_cmd_hdr ex_hdr;
 568        struct ib_uverbs_cmd_hdr hdr;
 569        struct uverbs_attr_bundle bundle;
 570        int srcu_key;
 571        ssize_t ret;
 572
 573        if (!ib_safe_file_access(filp)) {
 574                pr_err_once("uverbs_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
 575                            task_tgid_vnr(current), current->comm);
 576                return -EACCES;
 577        }
 578
 579        if (count < sizeof(hdr))
 580                return -EINVAL;
 581
 582        if (copy_from_user(&hdr, buf, sizeof(hdr)))
 583                return -EFAULT;
 584
 585        method_elm = uapi_get_method(uapi, hdr.command);
 586        if (IS_ERR(method_elm))
 587                return PTR_ERR(method_elm);
 588
 589        if (method_elm->is_ex) {
 590                if (count < (sizeof(hdr) + sizeof(ex_hdr)))
 591                        return -EINVAL;
 592                if (copy_from_user(&ex_hdr, buf + sizeof(hdr), sizeof(ex_hdr)))
 593                        return -EFAULT;
 594        }
 595
 596        ret = verify_hdr(&hdr, &ex_hdr, count, method_elm);
 597        if (ret)
 598                return ret;
 599
 600        srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
 601
 602        buf += sizeof(hdr);
 603
 604        memset(bundle.attr_present, 0, sizeof(bundle.attr_present));
 605        bundle.ufile = file;
 606        bundle.context = NULL; /* only valid if bundle has uobject */
 607        bundle.uobject = NULL;
 608        if (!method_elm->is_ex) {
 609                size_t in_len = hdr.in_words * 4 - sizeof(hdr);
 610                size_t out_len = hdr.out_words * 4;
 611                u64 response = 0;
 612
 613                if (method_elm->has_udata) {
 614                        bundle.driver_udata.inlen =
 615                                in_len - method_elm->req_size;
 616                        in_len = method_elm->req_size;
 617                        if (bundle.driver_udata.inlen)
 618                                bundle.driver_udata.inbuf = buf + in_len;
 619                        else
 620                                bundle.driver_udata.inbuf = NULL;
 621                } else {
 622                        memset(&bundle.driver_udata, 0,
 623                               sizeof(bundle.driver_udata));
 624                }
 625
 626                if (method_elm->has_resp) {
 627                        /*
 628                         * The macros check that if has_resp is set
 629                         * then the command request structure starts
 630                         * with a '__aligned u64 response' member.
 631                         */
 632                        ret = get_user(response, (const u64 __user *)buf);
 633                        if (ret)
 634                                goto out_unlock;
 635
 636                        if (method_elm->has_udata) {
 637                                bundle.driver_udata.outlen =
 638                                        out_len - method_elm->resp_size;
 639                                out_len = method_elm->resp_size;
 640                                if (bundle.driver_udata.outlen)
 641                                        bundle.driver_udata.outbuf =
 642                                                u64_to_user_ptr(response +
 643                                                                out_len);
 644                                else
 645                                        bundle.driver_udata.outbuf = NULL;
 646                        }
 647                } else {
 648                        bundle.driver_udata.outlen = 0;
 649                        bundle.driver_udata.outbuf = NULL;
 650                }
 651
 652                ib_uverbs_init_udata_buf_or_null(
 653                        &bundle.ucore, buf, u64_to_user_ptr(response),
 654                        in_len, out_len);
 655        } else {
 656                buf += sizeof(ex_hdr);
 657
 658                ib_uverbs_init_udata_buf_or_null(&bundle.ucore, buf,
 659                                        u64_to_user_ptr(ex_hdr.response),
 660                                        hdr.in_words * 8, hdr.out_words * 8);
 661
 662                ib_uverbs_init_udata_buf_or_null(
 663                        &bundle.driver_udata, buf + bundle.ucore.inlen,
 664                        u64_to_user_ptr(ex_hdr.response) + bundle.ucore.outlen,
 665                        ex_hdr.provider_in_words * 8,
 666                        ex_hdr.provider_out_words * 8);
 667
 668        }
 669
 670        ret = method_elm->handler(&bundle);
 671        if (bundle.uobject)
 672                uverbs_finalize_object(bundle.uobject, UVERBS_ACCESS_NEW, true,
 673                                       !ret, &bundle);
 674out_unlock:
 675        srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 676        return (ret) ? : count;
 677}
 678
 679static const struct vm_operations_struct rdma_umap_ops;
 680
 681static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)
 682{
 683        struct ib_uverbs_file *file = filp->private_data;
 684        struct ib_ucontext *ucontext;
 685        int ret = 0;
 686        int srcu_key;
 687
 688        srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
 689        ucontext = ib_uverbs_get_ucontext_file(file);
 690        if (IS_ERR(ucontext)) {
 691                ret = PTR_ERR(ucontext);
 692                goto out;
 693        }
 694        vma->vm_ops = &rdma_umap_ops;
 695        ret = ucontext->device->ops.mmap(ucontext, vma);
 696out:
 697        srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 698        return ret;
 699}
 700
 701/*
 702 * The VMA has been dup'd, initialize the vm_private_data with a new tracking
 703 * struct
 704 */
 705static void rdma_umap_open(struct vm_area_struct *vma)
 706{
 707        struct ib_uverbs_file *ufile = vma->vm_file->private_data;
 708        struct rdma_umap_priv *opriv = vma->vm_private_data;
 709        struct rdma_umap_priv *priv;
 710
 711        if (!opriv)
 712                return;
 713
 714        /* We are racing with disassociation */
 715        if (!down_read_trylock(&ufile->hw_destroy_rwsem))
 716                goto out_zap;
 717        /*
 718         * Disassociation already completed, the VMA should already be zapped.
 719         */
 720        if (!ufile->ucontext)
 721                goto out_unlock;
 722
 723        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 724        if (!priv)
 725                goto out_unlock;
 726        rdma_umap_priv_init(priv, vma, opriv->entry);
 727
 728        up_read(&ufile->hw_destroy_rwsem);
 729        return;
 730
 731out_unlock:
 732        up_read(&ufile->hw_destroy_rwsem);
 733out_zap:
 734        /*
 735         * We can't allow the VMA to be created with the actual IO pages, that
 736         * would break our API contract, and it can't be stopped at this
 737         * point, so zap it.
 738         */
 739        vma->vm_private_data = NULL;
 740        zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
 741}
 742
 743static void rdma_umap_close(struct vm_area_struct *vma)
 744{
 745        struct ib_uverbs_file *ufile = vma->vm_file->private_data;
 746        struct rdma_umap_priv *priv = vma->vm_private_data;
 747
 748        if (!priv)
 749                return;
 750
 751        /*
 752         * The vma holds a reference on the struct file that created it, which
 753         * in turn means that the ib_uverbs_file is guaranteed to exist at
 754         * this point.
 755         */
 756        mutex_lock(&ufile->umap_lock);
 757        if (priv->entry)
 758                rdma_user_mmap_entry_put(priv->entry);
 759
 760        list_del(&priv->list);
 761        mutex_unlock(&ufile->umap_lock);
 762        kfree(priv);
 763}
 764
 765/*
 766 * Once the zap_vma_ptes has been called touches to the VMA will come here and
 767 * we return a dummy writable zero page for all the pfns.
 768 */
 769static vm_fault_t rdma_umap_fault(struct vm_fault *vmf)
 770{
 771        struct ib_uverbs_file *ufile = vmf->vma->vm_file->private_data;
 772        struct rdma_umap_priv *priv = vmf->vma->vm_private_data;
 773        vm_fault_t ret = 0;
 774
 775        if (!priv)
 776                return VM_FAULT_SIGBUS;
 777
 778        /* Read only pages can just use the system zero page. */
 779        if (!(vmf->vma->vm_flags & (VM_WRITE | VM_MAYWRITE))) {
 780                vmf->page = ZERO_PAGE(vmf->address);
 781                get_page(vmf->page);
 782                return 0;
 783        }
 784
 785        mutex_lock(&ufile->umap_lock);
 786        if (!ufile->disassociate_page)
 787                ufile->disassociate_page =
 788                        alloc_pages(vmf->gfp_mask | __GFP_ZERO, 0);
 789
 790        if (ufile->disassociate_page) {
 791                /*
 792                 * This VMA is forced to always be shared so this doesn't have
 793                 * to worry about COW.
 794                 */
 795                vmf->page = ufile->disassociate_page;
 796                get_page(vmf->page);
 797        } else {
 798                ret = VM_FAULT_SIGBUS;
 799        }
 800        mutex_unlock(&ufile->umap_lock);
 801
 802        return ret;
 803}
 804
 805static const struct vm_operations_struct rdma_umap_ops = {
 806        .open = rdma_umap_open,
 807        .close = rdma_umap_close,
 808        .fault = rdma_umap_fault,
 809};
 810
 811void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
 812{
 813        struct rdma_umap_priv *priv, *next_priv;
 814
 815        lockdep_assert_held(&ufile->hw_destroy_rwsem);
 816
 817        while (1) {
 818                struct mm_struct *mm = NULL;
 819
 820                /* Get an arbitrary mm pointer that hasn't been cleaned yet */
 821                mutex_lock(&ufile->umap_lock);
 822                while (!list_empty(&ufile->umaps)) {
 823                        int ret;
 824
 825                        priv = list_first_entry(&ufile->umaps,
 826                                                struct rdma_umap_priv, list);
 827                        mm = priv->vma->vm_mm;
 828                        ret = mmget_not_zero(mm);
 829                        if (!ret) {
 830                                list_del_init(&priv->list);
 831                                if (priv->entry) {
 832                                        rdma_user_mmap_entry_put(priv->entry);
 833                                        priv->entry = NULL;
 834                                }
 835                                mm = NULL;
 836                                continue;
 837                        }
 838                        break;
 839                }
 840                mutex_unlock(&ufile->umap_lock);
 841                if (!mm)
 842                        return;
 843
 844                /*
 845                 * The umap_lock is nested under mmap_lock since it used within
 846                 * the vma_ops callbacks, so we have to clean the list one mm
 847                 * at a time to get the lock ordering right. Typically there
 848                 * will only be one mm, so no big deal.
 849                 */
 850                mmap_read_lock(mm);
 851                mutex_lock(&ufile->umap_lock);
 852                list_for_each_entry_safe (priv, next_priv, &ufile->umaps,
 853                                          list) {
 854                        struct vm_area_struct *vma = priv->vma;
 855
 856                        if (vma->vm_mm != mm)
 857                                continue;
 858                        list_del_init(&priv->list);
 859
 860                        zap_vma_ptes(vma, vma->vm_start,
 861                                     vma->vm_end - vma->vm_start);
 862
 863                        if (priv->entry) {
 864                                rdma_user_mmap_entry_put(priv->entry);
 865                                priv->entry = NULL;
 866                        }
 867                }
 868                mutex_unlock(&ufile->umap_lock);
 869                mmap_read_unlock(mm);
 870                mmput(mm);
 871        }
 872}
 873
 874/*
 875 * ib_uverbs_open() does not need the BKL:
 876 *
 877 *  - the ib_uverbs_device structures are properly reference counted and
 878 *    everything else is purely local to the file being created, so
 879 *    races against other open calls are not a problem;
 880 *  - there is no ioctl method to race against;
 881 *  - the open method will either immediately run -ENXIO, or all
 882 *    required initialization will be done.
 883 */
 884static int ib_uverbs_open(struct inode *inode, struct file *filp)
 885{
 886        struct ib_uverbs_device *dev;
 887        struct ib_uverbs_file *file;
 888        struct ib_device *ib_dev;
 889        int ret;
 890        int module_dependent;
 891        int srcu_key;
 892
 893        dev = container_of(inode->i_cdev, struct ib_uverbs_device, cdev);
 894        if (!refcount_inc_not_zero(&dev->refcount))
 895                return -ENXIO;
 896
 897        get_device(&dev->dev);
 898        srcu_key = srcu_read_lock(&dev->disassociate_srcu);
 899        mutex_lock(&dev->lists_mutex);
 900        ib_dev = srcu_dereference(dev->ib_dev,
 901                                  &dev->disassociate_srcu);
 902        if (!ib_dev) {
 903                ret = -EIO;
 904                goto err;
 905        }
 906
 907        if (!rdma_dev_access_netns(ib_dev, current->nsproxy->net_ns)) {
 908                ret = -EPERM;
 909                goto err;
 910        }
 911
 912        /* In case IB device supports disassociate ucontext, there is no hard
 913         * dependency between uverbs device and its low level device.
 914         */
 915        module_dependent = !(ib_dev->ops.disassociate_ucontext);
 916
 917        if (module_dependent) {
 918                if (!try_module_get(ib_dev->ops.owner)) {
 919                        ret = -ENODEV;
 920                        goto err;
 921                }
 922        }
 923
 924        file = kzalloc(sizeof(*file), GFP_KERNEL);
 925        if (!file) {
 926                ret = -ENOMEM;
 927                if (module_dependent)
 928                        goto err_module;
 929
 930                goto err;
 931        }
 932
 933        file->device     = dev;
 934        kref_init(&file->ref);
 935        mutex_init(&file->ucontext_lock);
 936
 937        spin_lock_init(&file->uobjects_lock);
 938        INIT_LIST_HEAD(&file->uobjects);
 939        init_rwsem(&file->hw_destroy_rwsem);
 940        mutex_init(&file->umap_lock);
 941        INIT_LIST_HEAD(&file->umaps);
 942
 943        filp->private_data = file;
 944        list_add_tail(&file->list, &dev->uverbs_file_list);
 945        mutex_unlock(&dev->lists_mutex);
 946        srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
 947
 948        setup_ufile_idr_uobject(file);
 949
 950        return stream_open(inode, filp);
 951
 952err_module:
 953        module_put(ib_dev->ops.owner);
 954
 955err:
 956        mutex_unlock(&dev->lists_mutex);
 957        srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
 958        if (refcount_dec_and_test(&dev->refcount))
 959                ib_uverbs_comp_dev(dev);
 960
 961        put_device(&dev->dev);
 962        return ret;
 963}
 964
 965static int ib_uverbs_close(struct inode *inode, struct file *filp)
 966{
 967        struct ib_uverbs_file *file = filp->private_data;
 968
 969        uverbs_destroy_ufile_hw(file, RDMA_REMOVE_CLOSE);
 970
 971        mutex_lock(&file->device->lists_mutex);
 972        list_del_init(&file->list);
 973        mutex_unlock(&file->device->lists_mutex);
 974
 975        kref_put(&file->ref, ib_uverbs_release_file);
 976
 977        return 0;
 978}
 979
 980static const struct file_operations uverbs_fops = {
 981        .owner   = THIS_MODULE,
 982        .write   = ib_uverbs_write,
 983        .open    = ib_uverbs_open,
 984        .release = ib_uverbs_close,
 985        .llseek  = no_llseek,
 986        .unlocked_ioctl = ib_uverbs_ioctl,
 987        .compat_ioctl = compat_ptr_ioctl,
 988};
 989
 990static const struct file_operations uverbs_mmap_fops = {
 991        .owner   = THIS_MODULE,
 992        .write   = ib_uverbs_write,
 993        .mmap    = ib_uverbs_mmap,
 994        .open    = ib_uverbs_open,
 995        .release = ib_uverbs_close,
 996        .llseek  = no_llseek,
 997        .unlocked_ioctl = ib_uverbs_ioctl,
 998        .compat_ioctl = compat_ptr_ioctl,
 999};
1000
1001static int ib_uverbs_get_nl_info(struct ib_device *ibdev, void *client_data,
1002                                 struct ib_client_nl_info *res)
1003{
1004        struct ib_uverbs_device *uverbs_dev = client_data;
1005        int ret;
1006
1007        if (res->port != -1)
1008                return -EINVAL;
1009
1010        res->abi = ibdev->ops.uverbs_abi_ver;
1011        res->cdev = &uverbs_dev->dev;
1012
1013        /*
1014         * To support DRIVER_ID binding in userspace some of the driver need
1015         * upgrading to expose their PCI dependent revision information
1016         * through get_context instead of relying on modalias matching. When
1017         * the drivers are fixed they can drop this flag.
1018         */
1019        if (!ibdev->ops.uverbs_no_driver_id_binding) {
1020                ret = nla_put_u32(res->nl_msg, RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID,
1021                                  ibdev->ops.driver_id);
1022                if (ret)
1023                        return ret;
1024        }
1025        return 0;
1026}
1027
1028static struct ib_client uverbs_client = {
1029        .name   = "uverbs",
1030        .no_kverbs_req = true,
1031        .add    = ib_uverbs_add_one,
1032        .remove = ib_uverbs_remove_one,
1033        .get_nl_info = ib_uverbs_get_nl_info,
1034};
1035MODULE_ALIAS_RDMA_CLIENT("uverbs");
1036
1037static ssize_t ibdev_show(struct device *device, struct device_attribute *attr,
1038                          char *buf)
1039{
1040        struct ib_uverbs_device *dev =
1041                        container_of(device, struct ib_uverbs_device, dev);
1042        int ret = -ENODEV;
1043        int srcu_key;
1044        struct ib_device *ib_dev;
1045
1046        srcu_key = srcu_read_lock(&dev->disassociate_srcu);
1047        ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1048        if (ib_dev)
1049                ret = sysfs_emit(buf, "%s\n", dev_name(&ib_dev->dev));
1050        srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
1051
1052        return ret;
1053}
1054static DEVICE_ATTR_RO(ibdev);
1055
1056static ssize_t abi_version_show(struct device *device,
1057                                struct device_attribute *attr, char *buf)
1058{
1059        struct ib_uverbs_device *dev =
1060                        container_of(device, struct ib_uverbs_device, dev);
1061        int ret = -ENODEV;
1062        int srcu_key;
1063        struct ib_device *ib_dev;
1064
1065        srcu_key = srcu_read_lock(&dev->disassociate_srcu);
1066        ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1067        if (ib_dev)
1068                ret = sysfs_emit(buf, "%u\n", ib_dev->ops.uverbs_abi_ver);
1069        srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
1070
1071        return ret;
1072}
1073static DEVICE_ATTR_RO(abi_version);
1074
1075static struct attribute *ib_dev_attrs[] = {
1076        &dev_attr_abi_version.attr,
1077        &dev_attr_ibdev.attr,
1078        NULL,
1079};
1080
1081static const struct attribute_group dev_attr_group = {
1082        .attrs = ib_dev_attrs,
1083};
1084
1085static CLASS_ATTR_STRING(abi_version, S_IRUGO,
1086                         __stringify(IB_USER_VERBS_ABI_VERSION));
1087
1088static int ib_uverbs_create_uapi(struct ib_device *device,
1089                                 struct ib_uverbs_device *uverbs_dev)
1090{
1091        struct uverbs_api *uapi;
1092
1093        uapi = uverbs_alloc_api(device);
1094        if (IS_ERR(uapi))
1095                return PTR_ERR(uapi);
1096
1097        uverbs_dev->uapi = uapi;
1098        return 0;
1099}
1100
1101static int ib_uverbs_add_one(struct ib_device *device)
1102{
1103        int devnum;
1104        dev_t base;
1105        struct ib_uverbs_device *uverbs_dev;
1106        int ret;
1107
1108        if (!device->ops.alloc_ucontext)
1109                return -EOPNOTSUPP;
1110
1111        uverbs_dev = kzalloc(sizeof(*uverbs_dev), GFP_KERNEL);
1112        if (!uverbs_dev)
1113                return -ENOMEM;
1114
1115        ret = init_srcu_struct(&uverbs_dev->disassociate_srcu);
1116        if (ret) {
1117                kfree(uverbs_dev);
1118                return -ENOMEM;
1119        }
1120
1121        device_initialize(&uverbs_dev->dev);
1122        uverbs_dev->dev.class = uverbs_class;
1123        uverbs_dev->dev.parent = device->dev.parent;
1124        uverbs_dev->dev.release = ib_uverbs_release_dev;
1125        uverbs_dev->groups[0] = &dev_attr_group;
1126        uverbs_dev->dev.groups = uverbs_dev->groups;
1127        refcount_set(&uverbs_dev->refcount, 1);
1128        init_completion(&uverbs_dev->comp);
1129        uverbs_dev->xrcd_tree = RB_ROOT;
1130        mutex_init(&uverbs_dev->xrcd_tree_mutex);
1131        mutex_init(&uverbs_dev->lists_mutex);
1132        INIT_LIST_HEAD(&uverbs_dev->uverbs_file_list);
1133        rcu_assign_pointer(uverbs_dev->ib_dev, device);
1134        uverbs_dev->num_comp_vectors = device->num_comp_vectors;
1135
1136        devnum = ida_alloc_max(&uverbs_ida, IB_UVERBS_MAX_DEVICES - 1,
1137                               GFP_KERNEL);
1138        if (devnum < 0) {
1139                ret = -ENOMEM;
1140                goto err;
1141        }
1142        uverbs_dev->devnum = devnum;
1143        if (devnum >= IB_UVERBS_NUM_FIXED_MINOR)
1144                base = dynamic_uverbs_dev + devnum - IB_UVERBS_NUM_FIXED_MINOR;
1145        else
1146                base = IB_UVERBS_BASE_DEV + devnum;
1147
1148        ret = ib_uverbs_create_uapi(device, uverbs_dev);
1149        if (ret)
1150                goto err_uapi;
1151
1152        uverbs_dev->dev.devt = base;
1153        dev_set_name(&uverbs_dev->dev, "uverbs%d", uverbs_dev->devnum);
1154
1155        cdev_init(&uverbs_dev->cdev,
1156                  device->ops.mmap ? &uverbs_mmap_fops : &uverbs_fops);
1157        uverbs_dev->cdev.owner = THIS_MODULE;
1158
1159        ret = cdev_device_add(&uverbs_dev->cdev, &uverbs_dev->dev);
1160        if (ret)
1161                goto err_uapi;
1162
1163        ib_set_client_data(device, &uverbs_client, uverbs_dev);
1164        return 0;
1165
1166err_uapi:
1167        ida_free(&uverbs_ida, devnum);
1168err:
1169        if (refcount_dec_and_test(&uverbs_dev->refcount))
1170                ib_uverbs_comp_dev(uverbs_dev);
1171        wait_for_completion(&uverbs_dev->comp);
1172        put_device(&uverbs_dev->dev);
1173        return ret;
1174}
1175
1176static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,
1177                                        struct ib_device *ib_dev)
1178{
1179        struct ib_uverbs_file *file;
1180
1181        /* Pending running commands to terminate */
1182        uverbs_disassociate_api_pre(uverbs_dev);
1183
1184        mutex_lock(&uverbs_dev->lists_mutex);
1185        while (!list_empty(&uverbs_dev->uverbs_file_list)) {
1186                file = list_first_entry(&uverbs_dev->uverbs_file_list,
1187                                        struct ib_uverbs_file, list);
1188                list_del_init(&file->list);
1189                kref_get(&file->ref);
1190
1191                /* We must release the mutex before going ahead and calling
1192                 * uverbs_cleanup_ufile, as it might end up indirectly calling
1193                 * uverbs_close, for example due to freeing the resources (e.g
1194                 * mmput).
1195                 */
1196                mutex_unlock(&uverbs_dev->lists_mutex);
1197
1198                uverbs_destroy_ufile_hw(file, RDMA_REMOVE_DRIVER_REMOVE);
1199                kref_put(&file->ref, ib_uverbs_release_file);
1200
1201                mutex_lock(&uverbs_dev->lists_mutex);
1202        }
1203        mutex_unlock(&uverbs_dev->lists_mutex);
1204
1205        uverbs_disassociate_api(uverbs_dev->uapi);
1206}
1207
1208static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)
1209{
1210        struct ib_uverbs_device *uverbs_dev = client_data;
1211        int wait_clients = 1;
1212
1213        cdev_device_del(&uverbs_dev->cdev, &uverbs_dev->dev);
1214        ida_free(&uverbs_ida, uverbs_dev->devnum);
1215
1216        if (device->ops.disassociate_ucontext) {
1217                /* We disassociate HW resources and immediately return.
1218                 * Userspace will see a EIO errno for all future access.
1219                 * Upon returning, ib_device may be freed internally and is not
1220                 * valid any more.
1221                 * uverbs_device is still available until all clients close
1222                 * their files, then the uverbs device ref count will be zero
1223                 * and its resources will be freed.
1224                 * Note: At this point no more files can be opened since the
1225                 * cdev was deleted, however active clients can still issue
1226                 * commands and close their open files.
1227                 */
1228                ib_uverbs_free_hw_resources(uverbs_dev, device);
1229                wait_clients = 0;
1230        }
1231
1232        if (refcount_dec_and_test(&uverbs_dev->refcount))
1233                ib_uverbs_comp_dev(uverbs_dev);
1234        if (wait_clients)
1235                wait_for_completion(&uverbs_dev->comp);
1236
1237        put_device(&uverbs_dev->dev);
1238}
1239
1240static char *uverbs_devnode(struct device *dev, umode_t *mode)
1241{
1242        if (mode)
1243                *mode = 0666;
1244        return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
1245}
1246
1247static int __init ib_uverbs_init(void)
1248{
1249        int ret;
1250
1251        ret = register_chrdev_region(IB_UVERBS_BASE_DEV,
1252                                     IB_UVERBS_NUM_FIXED_MINOR,
1253                                     "infiniband_verbs");
1254        if (ret) {
1255                pr_err("user_verbs: couldn't register device number\n");
1256                goto out;
1257        }
1258
1259        ret = alloc_chrdev_region(&dynamic_uverbs_dev, 0,
1260                                  IB_UVERBS_NUM_DYNAMIC_MINOR,
1261                                  "infiniband_verbs");
1262        if (ret) {
1263                pr_err("couldn't register dynamic device number\n");
1264                goto out_alloc;
1265        }
1266
1267        uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");
1268        if (IS_ERR(uverbs_class)) {
1269                ret = PTR_ERR(uverbs_class);
1270                pr_err("user_verbs: couldn't create class infiniband_verbs\n");
1271                goto out_chrdev;
1272        }
1273
1274        uverbs_class->devnode = uverbs_devnode;
1275
1276        ret = class_create_file(uverbs_class, &class_attr_abi_version.attr);
1277        if (ret) {
1278                pr_err("user_verbs: couldn't create abi_version attribute\n");
1279                goto out_class;
1280        }
1281
1282        ret = ib_register_client(&uverbs_client);
1283        if (ret) {
1284                pr_err("user_verbs: couldn't register client\n");
1285                goto out_class;
1286        }
1287
1288        return 0;
1289
1290out_class:
1291        class_destroy(uverbs_class);
1292
1293out_chrdev:
1294        unregister_chrdev_region(dynamic_uverbs_dev,
1295                                 IB_UVERBS_NUM_DYNAMIC_MINOR);
1296
1297out_alloc:
1298        unregister_chrdev_region(IB_UVERBS_BASE_DEV,
1299                                 IB_UVERBS_NUM_FIXED_MINOR);
1300
1301out:
1302        return ret;
1303}
1304
1305static void __exit ib_uverbs_cleanup(void)
1306{
1307        ib_unregister_client(&uverbs_client);
1308        class_destroy(uverbs_class);
1309        unregister_chrdev_region(IB_UVERBS_BASE_DEV,
1310                                 IB_UVERBS_NUM_FIXED_MINOR);
1311        unregister_chrdev_region(dynamic_uverbs_dev,
1312                                 IB_UVERBS_NUM_DYNAMIC_MINOR);
1313        mmu_notifier_synchronize();
1314}
1315
1316module_init(ib_uverbs_init);
1317module_exit(ib_uverbs_cleanup);
1318