linux/drivers/virtio/virtio_pci.c
<<
>>
Prefs
   1/*
   2 * Virtio PCI driver
   3 *
   4 * This module allows virtio devices to be used over a virtual PCI device.
   5 * This can be used with QEMU based VMMs like KVM or Xen.
   6 *
   7 * Copyright IBM Corp. 2007
   8 *
   9 * Authors:
  10 *  Anthony Liguori  <aliguori@us.ibm.com>
  11 *
  12 * This work is licensed under the terms of the GNU GPL, version 2 or later.
  13 * See the COPYING file in the top-level directory.
  14 *
  15 */
  16
  17#include <linux/module.h>
  18#include <linux/list.h>
  19#include <linux/pci.h>
  20#include <linux/slab.h>
  21#include <linux/interrupt.h>
  22#include <linux/virtio.h>
  23#include <linux/virtio_config.h>
  24#include <linux/virtio_ring.h>
  25#include <linux/virtio_pci.h>
  26#include <linux/highmem.h>
  27#include <linux/spinlock.h>
  28
  29MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
  30MODULE_DESCRIPTION("virtio-pci");
  31MODULE_LICENSE("GPL");
  32MODULE_VERSION("1");
  33
  34/* Our device structure */
  35struct virtio_pci_device
  36{
  37        struct virtio_device vdev;
  38        struct pci_dev *pci_dev;
  39
  40        /* the IO mapping for the PCI config space */
  41        void __iomem *ioaddr;
  42
  43        /* a list of queues so we can dispatch IRQs */
  44        spinlock_t lock;
  45        struct list_head virtqueues;
  46
  47        /* MSI-X support */
  48        int msix_enabled;
  49        int intx_enabled;
  50        struct msix_entry *msix_entries;
  51        cpumask_var_t *msix_affinity_masks;
  52        /* Name strings for interrupts. This size should be enough,
  53         * and I'm too lazy to allocate each name separately. */
  54        char (*msix_names)[256];
  55        /* Number of available vectors */
  56        unsigned msix_vectors;
  57        /* Vectors allocated, excluding per-vq vectors if any */
  58        unsigned msix_used_vectors;
  59
  60        /* Status saved during hibernate/restore */
  61        u8 saved_status;
  62
  63        /* Whether we have vector per vq */
  64        bool per_vq_vectors;
  65};
  66
  67/* Constants for MSI-X */
  68/* Use first vector for configuration changes, second and the rest for
  69 * virtqueues Thus, we need at least 2 vectors for MSI. */
  70enum {
  71        VP_MSIX_CONFIG_VECTOR = 0,
  72        VP_MSIX_VQ_VECTOR = 1,
  73};
  74
  75struct virtio_pci_vq_info
  76{
  77        /* the actual virtqueue */
  78        struct virtqueue *vq;
  79
  80        /* the number of entries in the queue */
  81        int num;
  82
  83        /* the virtual address of the ring queue */
  84        void *queue;
  85
  86        /* the list node for the virtqueues list */
  87        struct list_head node;
  88
  89        /* MSI-X vector (or none) */
  90        unsigned msix_vector;
  91};
  92
  93/* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
  94static DEFINE_PCI_DEVICE_TABLE(virtio_pci_id_table) = {
  95        { PCI_DEVICE(0x1af4, PCI_ANY_ID) },
  96        { 0 }
  97};
  98
  99MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
 100
 101/* Convert a generic virtio device to our structure */
 102static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
 103{
 104        return container_of(vdev, struct virtio_pci_device, vdev);
 105}
 106
 107/* virtio config->get_features() implementation */
 108static u32 vp_get_features(struct virtio_device *vdev)
 109{
 110        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 111
 112        /* When someone needs more than 32 feature bits, we'll need to
 113         * steal a bit to indicate that the rest are somewhere else. */
 114        return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
 115}
 116
 117/* virtio config->finalize_features() implementation */
 118static void vp_finalize_features(struct virtio_device *vdev)
 119{
 120        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 121
 122        /* Give virtio_ring a chance to accept features. */
 123        vring_transport_features(vdev);
 124
 125        /* We only support 32 feature bits. */
 126        BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1);
 127        iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES);
 128}
 129
 130/* virtio config->get() implementation */
 131static void vp_get(struct virtio_device *vdev, unsigned offset,
 132                   void *buf, unsigned len)
 133{
 134        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 135        void __iomem *ioaddr = vp_dev->ioaddr +
 136                                VIRTIO_PCI_CONFIG(vp_dev) + offset;
 137        u8 *ptr = buf;
 138        int i;
 139
 140        for (i = 0; i < len; i++)
 141                ptr[i] = ioread8(ioaddr + i);
 142}
 143
 144/* the config->set() implementation.  it's symmetric to the config->get()
 145 * implementation */
 146static void vp_set(struct virtio_device *vdev, unsigned offset,
 147                   const void *buf, unsigned len)
 148{
 149        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 150        void __iomem *ioaddr = vp_dev->ioaddr +
 151                                VIRTIO_PCI_CONFIG(vp_dev) + offset;
 152        const u8 *ptr = buf;
 153        int i;
 154
 155        for (i = 0; i < len; i++)
 156                iowrite8(ptr[i], ioaddr + i);
 157}
 158
 159/* config->{get,set}_status() implementations */
 160static u8 vp_get_status(struct virtio_device *vdev)
 161{
 162        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 163        return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
 164}
 165
 166static void vp_set_status(struct virtio_device *vdev, u8 status)
 167{
 168        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 169        /* We should never be setting status to 0. */
 170        BUG_ON(status == 0);
 171        iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
 172}
 173
 174/* wait for pending irq handlers */
 175static void vp_synchronize_vectors(struct virtio_device *vdev)
 176{
 177        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 178        int i;
 179
 180        if (vp_dev->intx_enabled)
 181                synchronize_irq(vp_dev->pci_dev->irq);
 182
 183        for (i = 0; i < vp_dev->msix_vectors; ++i)
 184                synchronize_irq(vp_dev->msix_entries[i].vector);
 185}
 186
 187static void vp_reset(struct virtio_device *vdev)
 188{
 189        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 190        /* 0 status means a reset. */
 191        iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
 192        /* Flush out the status write, and flush in device writes,
 193         * including MSi-X interrupts, if any. */
 194        ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
 195        /* Flush pending VQ/configuration callbacks. */
 196        vp_synchronize_vectors(vdev);
 197}
 198
 199/* the notify function used when creating a virt queue */
 200static void vp_notify(struct virtqueue *vq)
 201{
 202        struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
 203
 204        /* we write the queue's selector into the notification register to
 205         * signal the other end */
 206        iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
 207}
 208
 209/* Handle a configuration change: Tell driver if it wants to know. */
 210static irqreturn_t vp_config_changed(int irq, void *opaque)
 211{
 212        struct virtio_pci_device *vp_dev = opaque;
 213        struct virtio_driver *drv;
 214        drv = container_of(vp_dev->vdev.dev.driver,
 215                           struct virtio_driver, driver);
 216
 217        if (drv && drv->config_changed)
 218                drv->config_changed(&vp_dev->vdev);
 219        return IRQ_HANDLED;
 220}
 221
 222/* Notify all virtqueues on an interrupt. */
 223static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
 224{
 225        struct virtio_pci_device *vp_dev = opaque;
 226        struct virtio_pci_vq_info *info;
 227        irqreturn_t ret = IRQ_NONE;
 228        unsigned long flags;
 229
 230        spin_lock_irqsave(&vp_dev->lock, flags);
 231        list_for_each_entry(info, &vp_dev->virtqueues, node) {
 232                if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
 233                        ret = IRQ_HANDLED;
 234        }
 235        spin_unlock_irqrestore(&vp_dev->lock, flags);
 236
 237        return ret;
 238}
 239
 240/* A small wrapper to also acknowledge the interrupt when it's handled.
 241 * I really need an EIO hook for the vring so I can ack the interrupt once we
 242 * know that we'll be handling the IRQ but before we invoke the callback since
 243 * the callback may notify the host which results in the host attempting to
 244 * raise an interrupt that we would then mask once we acknowledged the
 245 * interrupt. */
 246static irqreturn_t vp_interrupt(int irq, void *opaque)
 247{
 248        struct virtio_pci_device *vp_dev = opaque;
 249        u8 isr;
 250
 251        /* reading the ISR has the effect of also clearing it so it's very
 252         * important to save off the value. */
 253        isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
 254
 255        /* It's definitely not us if the ISR was not high */
 256        if (!isr)
 257                return IRQ_NONE;
 258
 259        /* Configuration change?  Tell driver if it wants to know. */
 260        if (isr & VIRTIO_PCI_ISR_CONFIG)
 261                vp_config_changed(irq, opaque);
 262
 263        return vp_vring_interrupt(irq, opaque);
 264}
 265
 266static void vp_free_vectors(struct virtio_device *vdev)
 267{
 268        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 269        int i;
 270
 271        if (vp_dev->intx_enabled) {
 272                free_irq(vp_dev->pci_dev->irq, vp_dev);
 273                vp_dev->intx_enabled = 0;
 274        }
 275
 276        for (i = 0; i < vp_dev->msix_used_vectors; ++i)
 277                free_irq(vp_dev->msix_entries[i].vector, vp_dev);
 278
 279        for (i = 0; i < vp_dev->msix_vectors; i++)
 280                if (vp_dev->msix_affinity_masks[i])
 281                        free_cpumask_var(vp_dev->msix_affinity_masks[i]);
 282
 283        if (vp_dev->msix_enabled) {
 284                /* Disable the vector used for configuration */
 285                iowrite16(VIRTIO_MSI_NO_VECTOR,
 286                          vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
 287                /* Flush the write out to device */
 288                ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
 289
 290                pci_disable_msix(vp_dev->pci_dev);
 291                vp_dev->msix_enabled = 0;
 292        }
 293
 294        vp_dev->msix_vectors = 0;
 295        vp_dev->msix_used_vectors = 0;
 296        kfree(vp_dev->msix_names);
 297        vp_dev->msix_names = NULL;
 298        kfree(vp_dev->msix_entries);
 299        vp_dev->msix_entries = NULL;
 300        kfree(vp_dev->msix_affinity_masks);
 301        vp_dev->msix_affinity_masks = NULL;
 302}
 303
 304static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 305                                   bool per_vq_vectors)
 306{
 307        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 308        const char *name = dev_name(&vp_dev->vdev.dev);
 309        unsigned i, v;
 310        int err = -ENOMEM;
 311
 312        vp_dev->msix_vectors = nvectors;
 313
 314        vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
 315                                       GFP_KERNEL);
 316        if (!vp_dev->msix_entries)
 317                goto error;
 318        vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
 319                                     GFP_KERNEL);
 320        if (!vp_dev->msix_names)
 321                goto error;
 322        vp_dev->msix_affinity_masks
 323                = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
 324                          GFP_KERNEL);
 325        if (!vp_dev->msix_affinity_masks)
 326                goto error;
 327        for (i = 0; i < nvectors; ++i)
 328                if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
 329                                        GFP_KERNEL))
 330                        goto error;
 331
 332        for (i = 0; i < nvectors; ++i)
 333                vp_dev->msix_entries[i].entry = i;
 334
 335        /* pci_enable_msix returns positive if we can't get this many. */
 336        err = pci_enable_msix(vp_dev->pci_dev, vp_dev->msix_entries, nvectors);
 337        if (err > 0)
 338                err = -ENOSPC;
 339        if (err)
 340                goto error;
 341        vp_dev->msix_enabled = 1;
 342
 343        /* Set the vector used for configuration */
 344        v = vp_dev->msix_used_vectors;
 345        snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 346                 "%s-config", name);
 347        err = request_irq(vp_dev->msix_entries[v].vector,
 348                          vp_config_changed, 0, vp_dev->msix_names[v],
 349                          vp_dev);
 350        if (err)
 351                goto error;
 352        ++vp_dev->msix_used_vectors;
 353
 354        iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
 355        /* Verify we had enough resources to assign the vector */
 356        v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
 357        if (v == VIRTIO_MSI_NO_VECTOR) {
 358                err = -EBUSY;
 359                goto error;
 360        }
 361
 362        if (!per_vq_vectors) {
 363                /* Shared vector for all VQs */
 364                v = vp_dev->msix_used_vectors;
 365                snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 366                         "%s-virtqueues", name);
 367                err = request_irq(vp_dev->msix_entries[v].vector,
 368                                  vp_vring_interrupt, 0, vp_dev->msix_names[v],
 369                                  vp_dev);
 370                if (err)
 371                        goto error;
 372                ++vp_dev->msix_used_vectors;
 373        }
 374        return 0;
 375error:
 376        vp_free_vectors(vdev);
 377        return err;
 378}
 379
 380static int vp_request_intx(struct virtio_device *vdev)
 381{
 382        int err;
 383        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 384
 385        err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
 386                          IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
 387        if (!err)
 388                vp_dev->intx_enabled = 1;
 389        return err;
 390}
 391
 392static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
 393                                  void (*callback)(struct virtqueue *vq),
 394                                  const char *name,
 395                                  u16 msix_vec)
 396{
 397        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 398        struct virtio_pci_vq_info *info;
 399        struct virtqueue *vq;
 400        unsigned long flags, size;
 401        u16 num;
 402        int err;
 403
 404        /* Select the queue we're interested in */
 405        iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
 406
 407        /* Check if queue is either not available or already active. */
 408        num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
 409        if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
 410                return ERR_PTR(-ENOENT);
 411
 412        /* allocate and fill out our structure the represents an active
 413         * queue */
 414        info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL);
 415        if (!info)
 416                return ERR_PTR(-ENOMEM);
 417
 418        info->num = num;
 419        info->msix_vector = msix_vec;
 420
 421        size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
 422        info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
 423        if (info->queue == NULL) {
 424                err = -ENOMEM;
 425                goto out_info;
 426        }
 427
 428        /* activate the queue */
 429        iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT,
 430                  vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 431
 432        /* create the vring */
 433        vq = vring_new_virtqueue(index, info->num, VIRTIO_PCI_VRING_ALIGN, vdev,
 434                                 true, info->queue, vp_notify, callback, name);
 435        if (!vq) {
 436                err = -ENOMEM;
 437                goto out_activate_queue;
 438        }
 439
 440        vq->priv = info;
 441        info->vq = vq;
 442
 443        if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
 444                iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
 445                msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
 446                if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
 447                        err = -EBUSY;
 448                        goto out_assign;
 449                }
 450        }
 451
 452        if (callback) {
 453                spin_lock_irqsave(&vp_dev->lock, flags);
 454                list_add(&info->node, &vp_dev->virtqueues);
 455                spin_unlock_irqrestore(&vp_dev->lock, flags);
 456        } else {
 457                INIT_LIST_HEAD(&info->node);
 458        }
 459
 460        return vq;
 461
 462out_assign:
 463        vring_del_virtqueue(vq);
 464out_activate_queue:
 465        iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 466        free_pages_exact(info->queue, size);
 467out_info:
 468        kfree(info);
 469        return ERR_PTR(err);
 470}
 471
 472static void vp_del_vq(struct virtqueue *vq)
 473{
 474        struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
 475        struct virtio_pci_vq_info *info = vq->priv;
 476        unsigned long flags, size;
 477
 478        spin_lock_irqsave(&vp_dev->lock, flags);
 479        list_del(&info->node);
 480        spin_unlock_irqrestore(&vp_dev->lock, flags);
 481
 482        iowrite16(vq->index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
 483
 484        if (vp_dev->msix_enabled) {
 485                iowrite16(VIRTIO_MSI_NO_VECTOR,
 486                          vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
 487                /* Flush the write out to device */
 488                ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
 489        }
 490
 491        vring_del_virtqueue(vq);
 492
 493        /* Select and deactivate the queue */
 494        iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
 495
 496        size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN));
 497        free_pages_exact(info->queue, size);
 498        kfree(info);
 499}
 500
 501/* the config->del_vqs() implementation */
 502static void vp_del_vqs(struct virtio_device *vdev)
 503{
 504        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 505        struct virtqueue *vq, *n;
 506        struct virtio_pci_vq_info *info;
 507
 508        list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
 509                info = vq->priv;
 510                if (vp_dev->per_vq_vectors &&
 511                        info->msix_vector != VIRTIO_MSI_NO_VECTOR)
 512                        free_irq(vp_dev->msix_entries[info->msix_vector].vector,
 513                                 vq);
 514                vp_del_vq(vq);
 515        }
 516        vp_dev->per_vq_vectors = false;
 517
 518        vp_free_vectors(vdev);
 519}
 520
 521static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 522                              struct virtqueue *vqs[],
 523                              vq_callback_t *callbacks[],
 524                              const char *names[],
 525                              bool use_msix,
 526                              bool per_vq_vectors)
 527{
 528        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 529        u16 msix_vec;
 530        int i, err, nvectors, allocated_vectors;
 531
 532        if (!use_msix) {
 533                /* Old style: one normal interrupt for change and all vqs. */
 534                err = vp_request_intx(vdev);
 535                if (err)
 536                        goto error_request;
 537        } else {
 538                if (per_vq_vectors) {
 539                        /* Best option: one for change interrupt, one per vq. */
 540                        nvectors = 1;
 541                        for (i = 0; i < nvqs; ++i)
 542                                if (callbacks[i])
 543                                        ++nvectors;
 544                } else {
 545                        /* Second best: one for change, shared for all vqs. */
 546                        nvectors = 2;
 547                }
 548
 549                err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
 550                if (err)
 551                        goto error_request;
 552        }
 553
 554        vp_dev->per_vq_vectors = per_vq_vectors;
 555        allocated_vectors = vp_dev->msix_used_vectors;
 556        for (i = 0; i < nvqs; ++i) {
 557                if (!names[i]) {
 558                        vqs[i] = NULL;
 559                        continue;
 560                } else if (!callbacks[i] || !vp_dev->msix_enabled)
 561                        msix_vec = VIRTIO_MSI_NO_VECTOR;
 562                else if (vp_dev->per_vq_vectors)
 563                        msix_vec = allocated_vectors++;
 564                else
 565                        msix_vec = VP_MSIX_VQ_VECTOR;
 566                vqs[i] = setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
 567                if (IS_ERR(vqs[i])) {
 568                        err = PTR_ERR(vqs[i]);
 569                        goto error_find;
 570                }
 571
 572                if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
 573                        continue;
 574
 575                /* allocate per-vq irq if available and necessary */
 576                snprintf(vp_dev->msix_names[msix_vec],
 577                         sizeof *vp_dev->msix_names,
 578                         "%s-%s",
 579                         dev_name(&vp_dev->vdev.dev), names[i]);
 580                err = request_irq(vp_dev->msix_entries[msix_vec].vector,
 581                                  vring_interrupt, 0,
 582                                  vp_dev->msix_names[msix_vec],
 583                                  vqs[i]);
 584                if (err) {
 585                        vp_del_vq(vqs[i]);
 586                        goto error_find;
 587                }
 588        }
 589        return 0;
 590
 591error_find:
 592        vp_del_vqs(vdev);
 593
 594error_request:
 595        return err;
 596}
 597
 598/* the config->find_vqs() implementation */
 599static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 600                       struct virtqueue *vqs[],
 601                       vq_callback_t *callbacks[],
 602                       const char *names[])
 603{
 604        int err;
 605
 606        /* Try MSI-X with one vector per queue. */
 607        err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
 608        if (!err)
 609                return 0;
 610        /* Fallback: MSI-X with one vector for config, one shared for queues. */
 611        err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
 612                                 true, false);
 613        if (!err)
 614                return 0;
 615        /* Finally fall back to regular interrupts. */
 616        return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
 617                                  false, false);
 618}
 619
 620static const char *vp_bus_name(struct virtio_device *vdev)
 621{
 622        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 623
 624        return pci_name(vp_dev->pci_dev);
 625}
 626
 627/* Setup the affinity for a virtqueue:
 628 * - force the affinity for per vq vector
 629 * - OR over all affinities for shared MSI
 630 * - ignore the affinity request if we're using INTX
 631 */
 632static int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
 633{
 634        struct virtio_device *vdev = vq->vdev;
 635        struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 636        struct virtio_pci_vq_info *info = vq->priv;
 637        struct cpumask *mask;
 638        unsigned int irq;
 639
 640        if (!vq->callback)
 641                return -EINVAL;
 642
 643        if (vp_dev->msix_enabled) {
 644                mask = vp_dev->msix_affinity_masks[info->msix_vector];
 645                irq = vp_dev->msix_entries[info->msix_vector].vector;
 646                if (cpu == -1)
 647                        irq_set_affinity_hint(irq, NULL);
 648                else {
 649                        cpumask_set_cpu(cpu, mask);
 650                        irq_set_affinity_hint(irq, mask);
 651                }
 652        }
 653        return 0;
 654}
 655
 656static const struct virtio_config_ops virtio_pci_config_ops = {
 657        .get            = vp_get,
 658        .set            = vp_set,
 659        .get_status     = vp_get_status,
 660        .set_status     = vp_set_status,
 661        .reset          = vp_reset,
 662        .find_vqs       = vp_find_vqs,
 663        .del_vqs        = vp_del_vqs,
 664        .get_features   = vp_get_features,
 665        .finalize_features = vp_finalize_features,
 666        .bus_name       = vp_bus_name,
 667        .set_vq_affinity = vp_set_vq_affinity,
 668};
 669
 670static void virtio_pci_release_dev(struct device *_d)
 671{
 672        /*
 673         * No need for a release method as we allocate/free
 674         * all devices together with the pci devices.
 675         * Provide an empty one to avoid getting a warning from core.
 676         */
 677}
 678
 679/* the PCI probing function */
 680static int virtio_pci_probe(struct pci_dev *pci_dev,
 681                            const struct pci_device_id *id)
 682{
 683        struct virtio_pci_device *vp_dev;
 684        int err;
 685
 686        /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
 687        if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f)
 688                return -ENODEV;
 689
 690        if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) {
 691                printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n",
 692                       VIRTIO_PCI_ABI_VERSION, pci_dev->revision);
 693                return -ENODEV;
 694        }
 695
 696        /* allocate our structure and fill it out */
 697        vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
 698        if (vp_dev == NULL)
 699                return -ENOMEM;
 700
 701        vp_dev->vdev.dev.parent = &pci_dev->dev;
 702        vp_dev->vdev.dev.release = virtio_pci_release_dev;
 703        vp_dev->vdev.config = &virtio_pci_config_ops;
 704        vp_dev->pci_dev = pci_dev;
 705        INIT_LIST_HEAD(&vp_dev->virtqueues);
 706        spin_lock_init(&vp_dev->lock);
 707
 708        /* Disable MSI/MSIX to bring device to a known good state. */
 709        pci_msi_off(pci_dev);
 710
 711        /* enable the device */
 712        err = pci_enable_device(pci_dev);
 713        if (err)
 714                goto out;
 715
 716        err = pci_request_regions(pci_dev, "virtio-pci");
 717        if (err)
 718                goto out_enable_device;
 719
 720        vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
 721        if (vp_dev->ioaddr == NULL) {
 722                err = -ENOMEM;
 723                goto out_req_regions;
 724        }
 725
 726        pci_set_drvdata(pci_dev, vp_dev);
 727        pci_set_master(pci_dev);
 728
 729        /* we use the subsystem vendor/device id as the virtio vendor/device
 730         * id.  this allows us to use the same PCI vendor/device id for all
 731         * virtio devices and to identify the particular virtio driver by
 732         * the subsystem ids */
 733        vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
 734        vp_dev->vdev.id.device = pci_dev->subsystem_device;
 735
 736        /* finally register the virtio device */
 737        err = register_virtio_device(&vp_dev->vdev);
 738        if (err)
 739                goto out_set_drvdata;
 740
 741        return 0;
 742
 743out_set_drvdata:
 744        pci_set_drvdata(pci_dev, NULL);
 745        pci_iounmap(pci_dev, vp_dev->ioaddr);
 746out_req_regions:
 747        pci_release_regions(pci_dev);
 748out_enable_device:
 749        pci_disable_device(pci_dev);
 750out:
 751        kfree(vp_dev);
 752        return err;
 753}
 754
 755static void virtio_pci_remove(struct pci_dev *pci_dev)
 756{
 757        struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 758
 759        unregister_virtio_device(&vp_dev->vdev);
 760
 761        vp_del_vqs(&vp_dev->vdev);
 762        pci_set_drvdata(pci_dev, NULL);
 763        pci_iounmap(pci_dev, vp_dev->ioaddr);
 764        pci_release_regions(pci_dev);
 765        pci_disable_device(pci_dev);
 766        kfree(vp_dev);
 767}
 768
 769#ifdef CONFIG_PM
 770static int virtio_pci_freeze(struct device *dev)
 771{
 772        struct pci_dev *pci_dev = to_pci_dev(dev);
 773        struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 774        struct virtio_driver *drv;
 775        int ret;
 776
 777        drv = container_of(vp_dev->vdev.dev.driver,
 778                           struct virtio_driver, driver);
 779
 780        ret = 0;
 781        vp_dev->saved_status = vp_get_status(&vp_dev->vdev);
 782        if (drv && drv->freeze)
 783                ret = drv->freeze(&vp_dev->vdev);
 784
 785        if (!ret)
 786                pci_disable_device(pci_dev);
 787        return ret;
 788}
 789
 790static int virtio_pci_restore(struct device *dev)
 791{
 792        struct pci_dev *pci_dev = to_pci_dev(dev);
 793        struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 794        struct virtio_driver *drv;
 795        int ret;
 796
 797        drv = container_of(vp_dev->vdev.dev.driver,
 798                           struct virtio_driver, driver);
 799
 800        ret = pci_enable_device(pci_dev);
 801        if (ret)
 802                return ret;
 803
 804        pci_set_master(pci_dev);
 805        vp_finalize_features(&vp_dev->vdev);
 806
 807        if (drv && drv->restore)
 808                ret = drv->restore(&vp_dev->vdev);
 809
 810        /* Finally, tell the device we're all set */
 811        if (!ret)
 812                vp_set_status(&vp_dev->vdev, vp_dev->saved_status);
 813
 814        return ret;
 815}
 816
 817static const struct dev_pm_ops virtio_pci_pm_ops = {
 818        SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
 819};
 820#endif
 821
 822static struct pci_driver virtio_pci_driver = {
 823        .name           = "virtio-pci",
 824        .id_table       = virtio_pci_id_table,
 825        .probe          = virtio_pci_probe,
 826        .remove         = virtio_pci_remove,
 827#ifdef CONFIG_PM
 828        .driver.pm      = &virtio_pci_pm_ops,
 829#endif
 830};
 831
 832module_pci_driver(virtio_pci_driver);
 833
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.