linux/drivers/misc/vmw_vmci/vmci_doorbell.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * VMware VMCI Driver
   4 *
   5 * Copyright (C) 2012 VMware, Inc. All rights reserved.
   6 */
   7
   8#include <linux/vmw_vmci_defs.h>
   9#include <linux/vmw_vmci_api.h>
  10#include <linux/completion.h>
  11#include <linux/hash.h>
  12#include <linux/kernel.h>
  13#include <linux/list.h>
  14#include <linux/module.h>
  15#include <linux/sched.h>
  16#include <linux/slab.h>
  17
  18#include "vmci_datagram.h"
  19#include "vmci_doorbell.h"
  20#include "vmci_resource.h"
  21#include "vmci_driver.h"
  22#include "vmci_route.h"
  23
  24
  25#define VMCI_DOORBELL_INDEX_BITS        6
  26#define VMCI_DOORBELL_INDEX_TABLE_SIZE  (1 << VMCI_DOORBELL_INDEX_BITS)
  27#define VMCI_DOORBELL_HASH(_idx)        hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)
  28
  29/*
  30 * DoorbellEntry describes the a doorbell notification handle allocated by the
  31 * host.
  32 */
  33struct dbell_entry {
  34        struct vmci_resource resource;
  35        struct hlist_node node;
  36        struct work_struct work;
  37        vmci_callback notify_cb;
  38        void *client_data;
  39        u32 idx;
  40        u32 priv_flags;
  41        bool run_delayed;
  42        atomic_t active;        /* Only used by guest personality */
  43};
  44
  45/* The VMCI index table keeps track of currently registered doorbells. */
  46struct dbell_index_table {
  47        spinlock_t lock;        /* Index table lock */
  48        struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
  49};
  50
  51static struct dbell_index_table vmci_doorbell_it = {
  52        .lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),
  53};
  54
  55/*
  56 * The max_notify_idx is one larger than the currently known bitmap index in
  57 * use, and is used to determine how much of the bitmap needs to be scanned.
  58 */
  59static u32 max_notify_idx;
  60
  61/*
  62 * The notify_idx_count is used for determining whether there are free entries
  63 * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
  64 */
  65static u32 notify_idx_count;
  66
  67/*
  68 * The last_notify_idx_reserved is used to track the last index handed out - in
  69 * the case where multiple handles share a notification index, we hand out
  70 * indexes round robin based on last_notify_idx_reserved.
  71 */
  72static u32 last_notify_idx_reserved;
  73
  74/* This is a one entry cache used to by the index allocation. */
  75static u32 last_notify_idx_released = PAGE_SIZE;
  76
  77
  78/*
  79 * Utility function that retrieves the privilege flags associated
  80 * with a given doorbell handle. For guest endpoints, the
  81 * privileges are determined by the context ID, but for host
  82 * endpoints privileges are associated with the complete
  83 * handle. Hypervisor endpoints are not yet supported.
  84 */
  85int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)
  86{
  87        if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
  88                return VMCI_ERROR_INVALID_ARGS;
  89
  90        if (handle.context == VMCI_HOST_CONTEXT_ID) {
  91                struct dbell_entry *entry;
  92                struct vmci_resource *resource;
  93
  94                resource = vmci_resource_by_handle(handle,
  95                                                   VMCI_RESOURCE_TYPE_DOORBELL);
  96                if (!resource)
  97                        return VMCI_ERROR_NOT_FOUND;
  98
  99                entry = container_of(resource, struct dbell_entry, resource);
 100                *priv_flags = entry->priv_flags;
 101                vmci_resource_put(resource);
 102        } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
 103                /*
 104                 * Hypervisor endpoints for notifications are not
 105                 * supported (yet).
 106                 */
 107                return VMCI_ERROR_INVALID_ARGS;
 108        } else {
 109                *priv_flags = vmci_context_get_priv_flags(handle.context);
 110        }
 111
 112        return VMCI_SUCCESS;
 113}
 114
 115/*
 116 * Find doorbell entry by bitmap index.
 117 */
 118static struct dbell_entry *dbell_index_table_find(u32 idx)
 119{
 120        u32 bucket = VMCI_DOORBELL_HASH(idx);
 121        struct dbell_entry *dbell;
 122
 123        hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],
 124                             node) {
 125                if (idx == dbell->idx)
 126                        return dbell;
 127        }
 128
 129        return NULL;
 130}
 131
 132/*
 133 * Add the given entry to the index table.  This willi take a reference to the
 134 * entry's resource so that the entry is not deleted before it is removed from
 135 * the * table.
 136 */
 137static void dbell_index_table_add(struct dbell_entry *entry)
 138{
 139        u32 bucket;
 140        u32 new_notify_idx;
 141
 142        vmci_resource_get(&entry->resource);
 143
 144        spin_lock_bh(&vmci_doorbell_it.lock);
 145
 146        /*
 147         * Below we try to allocate an index in the notification
 148         * bitmap with "not too much" sharing between resources. If we
 149         * use less that the full bitmap, we either add to the end if
 150         * there are no unused flags within the currently used area,
 151         * or we search for unused ones. If we use the full bitmap, we
 152         * allocate the index round robin.
 153         */
 154        if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
 155                if (last_notify_idx_released < max_notify_idx &&
 156                    !dbell_index_table_find(last_notify_idx_released)) {
 157                        new_notify_idx = last_notify_idx_released;
 158                        last_notify_idx_released = PAGE_SIZE;
 159                } else {
 160                        bool reused = false;
 161                        new_notify_idx = last_notify_idx_reserved;
 162                        if (notify_idx_count + 1 < max_notify_idx) {
 163                                do {
 164                                        if (!dbell_index_table_find
 165                                            (new_notify_idx)) {
 166                                                reused = true;
 167                                                break;
 168                                        }
 169                                        new_notify_idx = (new_notify_idx + 1) %
 170                                            max_notify_idx;
 171                                } while (new_notify_idx !=
 172                                         last_notify_idx_released);
 173                        }
 174                        if (!reused) {
 175                                new_notify_idx = max_notify_idx;
 176                                max_notify_idx++;
 177                        }
 178                }
 179        } else {
 180                new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
 181        }
 182
 183        last_notify_idx_reserved = new_notify_idx;
 184        notify_idx_count++;
 185
 186        entry->idx = new_notify_idx;
 187        bucket = VMCI_DOORBELL_HASH(entry->idx);
 188        hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);
 189
 190        spin_unlock_bh(&vmci_doorbell_it.lock);
 191}
 192
 193/*
 194 * Remove the given entry from the index table.  This will release() the
 195 * entry's resource.
 196 */
 197static void dbell_index_table_remove(struct dbell_entry *entry)
 198{
 199        spin_lock_bh(&vmci_doorbell_it.lock);
 200
 201        hlist_del_init(&entry->node);
 202
 203        notify_idx_count--;
 204        if (entry->idx == max_notify_idx - 1) {
 205                /*
 206                 * If we delete an entry with the maximum known
 207                 * notification index, we take the opportunity to
 208                 * prune the current max. As there might be other
 209                 * unused indices immediately below, we lower the
 210                 * maximum until we hit an index in use.
 211                 */
 212                while (max_notify_idx > 0 &&
 213                       !dbell_index_table_find(max_notify_idx - 1))
 214                        max_notify_idx--;
 215        }
 216
 217        last_notify_idx_released = entry->idx;
 218
 219        spin_unlock_bh(&vmci_doorbell_it.lock);
 220
 221        vmci_resource_put(&entry->resource);
 222}
 223
 224/*
 225 * Creates a link between the given doorbell handle and the given
 226 * index in the bitmap in the device backend. A notification state
 227 * is created in hypervisor.
 228 */
 229static int dbell_link(struct vmci_handle handle, u32 notify_idx)
 230{
 231        struct vmci_doorbell_link_msg link_msg;
 232
 233        link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
 234                                            VMCI_DOORBELL_LINK);
 235        link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 236        link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
 237        link_msg.handle = handle;
 238        link_msg.notify_idx = notify_idx;
 239
 240        return vmci_send_datagram(&link_msg.hdr);
 241}
 242
 243/*
 244 * Unlinks the given doorbell handle from an index in the bitmap in
 245 * the device backend. The notification state is destroyed in hypervisor.
 246 */
 247static int dbell_unlink(struct vmci_handle handle)
 248{
 249        struct vmci_doorbell_unlink_msg unlink_msg;
 250
 251        unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
 252                                              VMCI_DOORBELL_UNLINK);
 253        unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 254        unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
 255        unlink_msg.handle = handle;
 256
 257        return vmci_send_datagram(&unlink_msg.hdr);
 258}
 259
 260/*
 261 * Notify another guest or the host.  We send a datagram down to the
 262 * host via the hypervisor with the notification info.
 263 */
 264static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags)
 265{
 266        struct vmci_doorbell_notify_msg notify_msg;
 267
 268        notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
 269                                              VMCI_DOORBELL_NOTIFY);
 270        notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 271        notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
 272        notify_msg.handle = handle;
 273
 274        return vmci_send_datagram(&notify_msg.hdr);
 275}
 276
 277/*
 278 * Calls the specified callback in a delayed context.
 279 */
 280static void dbell_delayed_dispatch(struct work_struct *work)
 281{
 282        struct dbell_entry *entry = container_of(work,
 283                                                 struct dbell_entry, work);
 284
 285        entry->notify_cb(entry->client_data);
 286        vmci_resource_put(&entry->resource);
 287}
 288
 289/*
 290 * Dispatches a doorbell notification to the host context.
 291 */
 292int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)
 293{
 294        struct dbell_entry *entry;
 295        struct vmci_resource *resource;
 296
 297        if (vmci_handle_is_invalid(handle)) {
 298                pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",
 299                         handle.context, handle.resource);
 300                return VMCI_ERROR_INVALID_ARGS;
 301        }
 302
 303        resource = vmci_resource_by_handle(handle,
 304                                           VMCI_RESOURCE_TYPE_DOORBELL);
 305        if (!resource) {
 306                pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",
 307                         handle.context, handle.resource);
 308                return VMCI_ERROR_NOT_FOUND;
 309        }
 310
 311        entry = container_of(resource, struct dbell_entry, resource);
 312        if (entry->run_delayed) {
 313                if (!schedule_work(&entry->work))
 314                        vmci_resource_put(resource);
 315        } else {
 316                entry->notify_cb(entry->client_data);
 317                vmci_resource_put(resource);
 318        }
 319
 320        return VMCI_SUCCESS;
 321}
 322
 323/*
 324 * Register the notification bitmap with the host.
 325 */
 326bool vmci_dbell_register_notification_bitmap(u64 bitmap_ppn)
 327{
 328        int result;
 329        struct vmci_notify_bm_set_msg bitmap_set_msg = { };
 330
 331        bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
 332                                                  VMCI_SET_NOTIFY_BITMAP);
 333        bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
 334        bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -
 335            VMCI_DG_HEADERSIZE;
 336        if (vmci_use_ppn64())
 337                bitmap_set_msg.bitmap_ppn64 = bitmap_ppn;
 338        else
 339                bitmap_set_msg.bitmap_ppn32 = (u32) bitmap_ppn;
 340
 341        result = vmci_send_datagram(&bitmap_set_msg.hdr);
 342        if (result != VMCI_SUCCESS) {
 343                pr_devel("Failed to register (PPN=%llu) as notification bitmap (error=%d)\n",
 344                         bitmap_ppn, result);
 345                return false;
 346        }
 347        return true;
 348}
 349
 350/*
 351 * Executes or schedules the handlers for a given notify index.
 352 */
 353static void dbell_fire_entries(u32 notify_idx)
 354{
 355        u32 bucket = VMCI_DOORBELL_HASH(notify_idx);
 356        struct dbell_entry *dbell;
 357
 358        spin_lock_bh(&vmci_doorbell_it.lock);
 359
 360        hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {
 361                if (dbell->idx == notify_idx &&
 362                    atomic_read(&dbell->active) == 1) {
 363                        if (dbell->run_delayed) {
 364                                vmci_resource_get(&dbell->resource);
 365                                if (!schedule_work(&dbell->work))
 366                                        vmci_resource_put(&dbell->resource);
 367                        } else {
 368                                dbell->notify_cb(dbell->client_data);
 369                        }
 370                }
 371        }
 372
 373        spin_unlock_bh(&vmci_doorbell_it.lock);
 374}
 375
 376/*
 377 * Scans the notification bitmap, collects pending notifications,
 378 * resets the bitmap and invokes appropriate callbacks.
 379 */
 380void vmci_dbell_scan_notification_entries(u8 *bitmap)
 381{
 382        u32 idx;
 383
 384        for (idx = 0; idx < max_notify_idx; idx++) {
 385                if (bitmap[idx] & 0x1) {
 386                        bitmap[idx] &= ~1;
 387                        dbell_fire_entries(idx);
 388                }
 389        }
 390}
 391
 392/*
 393 * vmci_doorbell_create() - Creates a doorbell
 394 * @handle:     A handle used to track the resource.  Can be invalid.
 395 * @flags:      Flag that determines context of callback.
 396 * @priv_flags: Privileges flags.
 397 * @notify_cb:  The callback to be ivoked when the doorbell fires.
 398 * @client_data:        A parameter to be passed to the callback.
 399 *
 400 * Creates a doorbell with the given callback. If the handle is
 401 * VMCI_INVALID_HANDLE, a free handle will be assigned, if
 402 * possible. The callback can be run immediately (potentially with
 403 * locks held - the default) or delayed (in a kernel thread) by
 404 * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution
 405 * is selected, a given callback may not be run if the kernel is
 406 * unable to allocate memory for the delayed execution (highly
 407 * unlikely).
 408 */
 409int vmci_doorbell_create(struct vmci_handle *handle,
 410                         u32 flags,
 411                         u32 priv_flags,
 412                         vmci_callback notify_cb, void *client_data)
 413{
 414        struct dbell_entry *entry;
 415        struct vmci_handle new_handle;
 416        int result;
 417
 418        if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
 419            priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
 420                return VMCI_ERROR_INVALID_ARGS;
 421
 422        entry = kmalloc(sizeof(*entry), GFP_KERNEL);
 423        if (entry == NULL) {
 424                pr_warn("Failed allocating memory for datagram entry\n");
 425                return VMCI_ERROR_NO_MEM;
 426        }
 427
 428        if (vmci_handle_is_invalid(*handle)) {
 429                u32 context_id = vmci_get_context_id();
 430
 431                if (context_id == VMCI_INVALID_ID) {
 432                        pr_warn("Failed to get context ID\n");
 433                        result = VMCI_ERROR_NO_RESOURCES;
 434                        goto free_mem;
 435                }
 436
 437                /* Let resource code allocate a free ID for us */
 438                new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
 439        } else {
 440                bool valid_context = false;
 441
 442                /*
 443                 * Validate the handle.  We must do both of the checks below
 444                 * because we can be acting as both a host and a guest at the
 445                 * same time. We always allow the host context ID, since the
 446                 * host functionality is in practice always there with the
 447                 * unified driver.
 448                 */
 449                if (handle->context == VMCI_HOST_CONTEXT_ID ||
 450                    (vmci_guest_code_active() &&
 451                     vmci_get_context_id() == handle->context)) {
 452                        valid_context = true;
 453                }
 454
 455                if (!valid_context || handle->resource == VMCI_INVALID_ID) {
 456                        pr_devel("Invalid argument (handle=0x%x:0x%x)\n",
 457                                 handle->context, handle->resource);
 458                        result = VMCI_ERROR_INVALID_ARGS;
 459                        goto free_mem;
 460                }
 461
 462                new_handle = *handle;
 463        }
 464
 465        entry->idx = 0;
 466        INIT_HLIST_NODE(&entry->node);
 467        entry->priv_flags = priv_flags;
 468        INIT_WORK(&entry->work, dbell_delayed_dispatch);
 469        entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;
 470        entry->notify_cb = notify_cb;
 471        entry->client_data = client_data;
 472        atomic_set(&entry->active, 0);
 473
 474        result = vmci_resource_add(&entry->resource,
 475                                   VMCI_RESOURCE_TYPE_DOORBELL,
 476                                   new_handle);
 477        if (result != VMCI_SUCCESS) {
 478                pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",
 479                        new_handle.context, new_handle.resource, result);
 480                goto free_mem;
 481        }
 482
 483        new_handle = vmci_resource_handle(&entry->resource);
 484        if (vmci_guest_code_active()) {
 485                dbell_index_table_add(entry);
 486                result = dbell_link(new_handle, entry->idx);
 487                if (VMCI_SUCCESS != result)
 488                        goto destroy_resource;
 489
 490                atomic_set(&entry->active, 1);
 491        }
 492
 493        *handle = new_handle;
 494
 495        return result;
 496
 497 destroy_resource:
 498        dbell_index_table_remove(entry);
 499        vmci_resource_remove(&entry->resource);
 500 free_mem:
 501        kfree(entry);
 502        return result;
 503}
 504EXPORT_SYMBOL_GPL(vmci_doorbell_create);
 505
 506/*
 507 * vmci_doorbell_destroy() - Destroy a doorbell.
 508 * @handle:     The handle tracking the resource.
 509 *
 510 * Destroys a doorbell previously created with vmcii_doorbell_create. This
 511 * operation may block waiting for a callback to finish.
 512 */
 513int vmci_doorbell_destroy(struct vmci_handle handle)
 514{
 515        struct dbell_entry *entry;
 516        struct vmci_resource *resource;
 517
 518        if (vmci_handle_is_invalid(handle))
 519                return VMCI_ERROR_INVALID_ARGS;
 520
 521        resource = vmci_resource_by_handle(handle,
 522                                           VMCI_RESOURCE_TYPE_DOORBELL);
 523        if (!resource) {
 524                pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",
 525                         handle.context, handle.resource);
 526                return VMCI_ERROR_NOT_FOUND;
 527        }
 528
 529        entry = container_of(resource, struct dbell_entry, resource);
 530
 531        if (!hlist_unhashed(&entry->node)) {
 532                int result;
 533
 534                dbell_index_table_remove(entry);
 535
 536                result = dbell_unlink(handle);
 537                if (VMCI_SUCCESS != result) {
 538
 539                        /*
 540                         * The only reason this should fail would be
 541                         * an inconsistency between guest and
 542                         * hypervisor state, where the guest believes
 543                         * it has an active registration whereas the
 544                         * hypervisor doesn't. One case where this may
 545                         * happen is if a doorbell is unregistered
 546                         * following a hibernation at a time where the
 547                         * doorbell state hasn't been restored on the
 548                         * hypervisor side yet. Since the handle has
 549                         * now been removed in the guest, we just
 550                         * print a warning and return success.
 551                         */
 552                        pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",
 553                                 handle.context, handle.resource, result);
 554                }
 555        }
 556
 557        /*
 558         * Now remove the resource from the table.  It might still be in use
 559         * after this, in a callback or still on the delayed work queue.
 560         */
 561        vmci_resource_put(&entry->resource);
 562        vmci_resource_remove(&entry->resource);
 563
 564        kfree(entry);
 565
 566        return VMCI_SUCCESS;
 567}
 568EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);
 569
 570/*
 571 * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes).
 572 * @dst:        The handlle identifying the doorbell resource
 573 * @priv_flags: Priviledge flags.
 574 *
 575 * Generates a notification on the doorbell identified by the
 576 * handle. For host side generation of notifications, the caller
 577 * can specify what the privilege of the calling side is.
 578 */
 579int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags)
 580{
 581        int retval;
 582        enum vmci_route route;
 583        struct vmci_handle src;
 584
 585        if (vmci_handle_is_invalid(dst) ||
 586            (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
 587                return VMCI_ERROR_INVALID_ARGS;
 588
 589        src = VMCI_INVALID_HANDLE;
 590        retval = vmci_route(&src, &dst, false, &route);
 591        if (retval < VMCI_SUCCESS)
 592                return retval;
 593
 594        if (VMCI_ROUTE_AS_HOST == route)
 595                return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID,
 596                                             dst, priv_flags);
 597
 598        if (VMCI_ROUTE_AS_GUEST == route)
 599                return dbell_notify_as_guest(dst, priv_flags);
 600
 601        pr_warn("Unknown route (%d) for doorbell\n", route);
 602        return VMCI_ERROR_DST_UNREACHABLE;
 603}
 604EXPORT_SYMBOL_GPL(vmci_doorbell_notify);
 605