linux/drivers/xen/gntalloc.c
<<
>>
Prefs
   1/******************************************************************************
   2 * gntalloc.c
   3 *
   4 * Device for creating grant references (in user-space) that may be shared
   5 * with other domains.
   6 *
   7 * This program is distributed in the hope that it will be useful,
   8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10 * GNU General Public License for more details.
  11 *
  12 * You should have received a copy of the GNU General Public License
  13 * along with this program; if not, write to the Free Software
  14 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  15 */
  16
  17/*
  18 * This driver exists to allow userspace programs in Linux to allocate kernel
  19 * memory that will later be shared with another domain.  Without this device,
  20 * Linux userspace programs cannot create grant references.
  21 *
  22 * How this stuff works:
  23 *   X -> granting a page to Y
  24 *   Y -> mapping the grant from X
  25 *
  26 *   1. X uses the gntalloc device to allocate a page of kernel memory, P.
  27 *   2. X creates an entry in the grant table that says domid(Y) can access P.
  28 *      This is done without a hypercall unless the grant table needs expansion.
  29 *   3. X gives the grant reference identifier, GREF, to Y.
  30 *   4. Y maps the page, either directly into kernel memory for use in a backend
  31 *      driver, or via a the gntdev device to map into the address space of an
  32 *      application running in Y. This is the first point at which Xen does any
  33 *      tracking of the page.
  34 *   5. A program in X mmap()s a segment of the gntalloc device that corresponds
  35 *      to the shared page, and can now communicate with Y over the shared page.
  36 *
  37 *
  38 * NOTE TO USERSPACE LIBRARIES:
  39 *   The grant allocation and mmap()ing are, naturally, two separate operations.
  40 *   You set up the sharing by calling the create ioctl() and then the mmap().
  41 *   Teardown requires munmap() and either close() or ioctl().
  42 *
  43 * WARNING: Since Xen does not allow a guest to forcibly end the use of a grant
  44 * reference, this device can be used to consume kernel memory by leaving grant
  45 * references mapped by another domain when an application exits. Therefore,
  46 * there is a global limit on the number of pages that can be allocated. When
  47 * all references to the page are unmapped, it will be freed during the next
  48 * grant operation.
  49 */
  50
  51#include <linux/atomic.h>
  52#include <linux/module.h>
  53#include <linux/miscdevice.h>
  54#include <linux/kernel.h>
  55#include <linux/init.h>
  56#include <linux/slab.h>
  57#include <linux/fs.h>
  58#include <linux/device.h>
  59#include <linux/mm.h>
  60#include <linux/uaccess.h>
  61#include <linux/types.h>
  62#include <linux/list.h>
  63#include <linux/highmem.h>
  64
  65#include <xen/xen.h>
  66#include <xen/page.h>
  67#include <xen/grant_table.h>
  68#include <xen/gntalloc.h>
  69#include <xen/events.h>
  70
  71static int limit = 1024;
  72module_param(limit, int, 0644);
  73MODULE_PARM_DESC(limit, "Maximum number of grants that may be allocated by "
  74                "the gntalloc device");
  75
  76static LIST_HEAD(gref_list);
  77static DEFINE_MUTEX(gref_mutex);
  78static int gref_size;
  79
  80struct notify_info {
  81        uint16_t pgoff:12;    /* Bits 0-11: Offset of the byte to clear */
  82        uint16_t flags:2;     /* Bits 12-13: Unmap notification flags */
  83        int event;            /* Port (event channel) to notify */
  84};
  85
  86/* Metadata on a grant reference. */
  87struct gntalloc_gref {
  88        struct list_head next_gref;  /* list entry gref_list */
  89        struct list_head next_file;  /* list entry file->list, if open */
  90        struct page *page;           /* The shared page */
  91        uint64_t file_index;         /* File offset for mmap() */
  92        unsigned int users;          /* Use count - when zero, waiting on Xen */
  93        grant_ref_t gref_id;         /* The grant reference number */
  94        struct notify_info notify;   /* Unmap notification */
  95};
  96
  97struct gntalloc_file_private_data {
  98        struct list_head list;
  99        uint64_t index;
 100};
 101
 102struct gntalloc_vma_private_data {
 103        struct gntalloc_gref *gref;
 104        int users;
 105        int count;
 106};
 107
 108static void __del_gref(struct gntalloc_gref *gref);
 109
 110static void do_cleanup(void)
 111{
 112        struct gntalloc_gref *gref, *n;
 113        list_for_each_entry_safe(gref, n, &gref_list, next_gref) {
 114                if (!gref->users)
 115                        __del_gref(gref);
 116        }
 117}
 118
 119static int add_grefs(struct ioctl_gntalloc_alloc_gref *op,
 120        uint32_t *gref_ids, struct gntalloc_file_private_data *priv)
 121{
 122        int i, rc, readonly;
 123        LIST_HEAD(queue_gref);
 124        LIST_HEAD(queue_file);
 125        struct gntalloc_gref *gref;
 126
 127        readonly = !(op->flags & GNTALLOC_FLAG_WRITABLE);
 128        rc = -ENOMEM;
 129        for (i = 0; i < op->count; i++) {
 130                gref = kzalloc(sizeof(*gref), GFP_KERNEL);
 131                if (!gref)
 132                        goto undo;
 133                list_add_tail(&gref->next_gref, &queue_gref);
 134                list_add_tail(&gref->next_file, &queue_file);
 135                gref->users = 1;
 136                gref->file_index = op->index + i * PAGE_SIZE;
 137                gref->page = alloc_page(GFP_KERNEL|__GFP_ZERO);
 138                if (!gref->page)
 139                        goto undo;
 140
 141                /* Grant foreign access to the page. */
 142                gref->gref_id = gnttab_grant_foreign_access(op->domid,
 143                        pfn_to_mfn(page_to_pfn(gref->page)), readonly);
 144                if ((int)gref->gref_id < 0) {
 145                        rc = gref->gref_id;
 146                        goto undo;
 147                }
 148                gref_ids[i] = gref->gref_id;
 149        }
 150
 151        /* Add to gref lists. */
 152        mutex_lock(&gref_mutex);
 153        list_splice_tail(&queue_gref, &gref_list);
 154        list_splice_tail(&queue_file, &priv->list);
 155        mutex_unlock(&gref_mutex);
 156
 157        return 0;
 158
 159undo:
 160        mutex_lock(&gref_mutex);
 161        gref_size -= (op->count - i);
 162
 163        list_for_each_entry(gref, &queue_file, next_file) {
 164                /* __del_gref does not remove from queue_file */
 165                __del_gref(gref);
 166        }
 167
 168        /* It's possible for the target domain to map the just-allocated grant
 169         * references by blindly guessing their IDs; if this is done, then
 170         * __del_gref will leave them in the queue_gref list. They need to be
 171         * added to the global list so that we can free them when they are no
 172         * longer referenced.
 173         */
 174        if (unlikely(!list_empty(&queue_gref)))
 175                list_splice_tail(&queue_gref, &gref_list);
 176        mutex_unlock(&gref_mutex);
 177        return rc;
 178}
 179
 180static void __del_gref(struct gntalloc_gref *gref)
 181{
 182        if (gref->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {
 183                uint8_t *tmp = kmap(gref->page);
 184                tmp[gref->notify.pgoff] = 0;
 185                kunmap(gref->page);
 186        }
 187        if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {
 188                notify_remote_via_evtchn(gref->notify.event);
 189                evtchn_put(gref->notify.event);
 190        }
 191
 192        gref->notify.flags = 0;
 193
 194        if (gref->gref_id > 0) {
 195                if (gnttab_query_foreign_access(gref->gref_id))
 196                        return;
 197
 198                if (!gnttab_end_foreign_access_ref(gref->gref_id, 0))
 199                        return;
 200
 201                gnttab_free_grant_reference(gref->gref_id);
 202        }
 203
 204        gref_size--;
 205        list_del(&gref->next_gref);
 206
 207        if (gref->page)
 208                __free_page(gref->page);
 209
 210        kfree(gref);
 211}
 212
 213/* finds contiguous grant references in a file, returns the first */
 214static struct gntalloc_gref *find_grefs(struct gntalloc_file_private_data *priv,
 215                uint64_t index, uint32_t count)
 216{
 217        struct gntalloc_gref *rv = NULL, *gref;
 218        list_for_each_entry(gref, &priv->list, next_file) {
 219                if (gref->file_index == index && !rv)
 220                        rv = gref;
 221                if (rv) {
 222                        if (gref->file_index != index)
 223                                return NULL;
 224                        index += PAGE_SIZE;
 225                        count--;
 226                        if (count == 0)
 227                                return rv;
 228                }
 229        }
 230        return NULL;
 231}
 232
 233/*
 234 * -------------------------------------
 235 *  File operations.
 236 * -------------------------------------
 237 */
 238static int gntalloc_open(struct inode *inode, struct file *filp)
 239{
 240        struct gntalloc_file_private_data *priv;
 241
 242        priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 243        if (!priv)
 244                goto out_nomem;
 245        INIT_LIST_HEAD(&priv->list);
 246
 247        filp->private_data = priv;
 248
 249        pr_debug("%s: priv %p\n", __func__, priv);
 250
 251        return 0;
 252
 253out_nomem:
 254        return -ENOMEM;
 255}
 256
 257static int gntalloc_release(struct inode *inode, struct file *filp)
 258{
 259        struct gntalloc_file_private_data *priv = filp->private_data;
 260        struct gntalloc_gref *gref;
 261
 262        pr_debug("%s: priv %p\n", __func__, priv);
 263
 264        mutex_lock(&gref_mutex);
 265        while (!list_empty(&priv->list)) {
 266                gref = list_entry(priv->list.next,
 267                        struct gntalloc_gref, next_file);
 268                list_del(&gref->next_file);
 269                gref->users--;
 270                if (gref->users == 0)
 271                        __del_gref(gref);
 272        }
 273        kfree(priv);
 274        mutex_unlock(&gref_mutex);
 275
 276        return 0;
 277}
 278
 279static long gntalloc_ioctl_alloc(struct gntalloc_file_private_data *priv,
 280                struct ioctl_gntalloc_alloc_gref __user *arg)
 281{
 282        int rc = 0;
 283        struct ioctl_gntalloc_alloc_gref op;
 284        uint32_t *gref_ids;
 285
 286        pr_debug("%s: priv %p\n", __func__, priv);
 287
 288        if (copy_from_user(&op, arg, sizeof(op))) {
 289                rc = -EFAULT;
 290                goto out;
 291        }
 292
 293        gref_ids = kcalloc(op.count, sizeof(gref_ids[0]), GFP_TEMPORARY);
 294        if (!gref_ids) {
 295                rc = -ENOMEM;
 296                goto out;
 297        }
 298
 299        mutex_lock(&gref_mutex);
 300        /* Clean up pages that were at zero (local) users but were still mapped
 301         * by remote domains. Since those pages count towards the limit that we
 302         * are about to enforce, removing them here is a good idea.
 303         */
 304        do_cleanup();
 305        if (gref_size + op.count > limit) {
 306                mutex_unlock(&gref_mutex);
 307                rc = -ENOSPC;
 308                goto out_free;
 309        }
 310        gref_size += op.count;
 311        op.index = priv->index;
 312        priv->index += op.count * PAGE_SIZE;
 313        mutex_unlock(&gref_mutex);
 314
 315        rc = add_grefs(&op, gref_ids, priv);
 316        if (rc < 0)
 317                goto out_free;
 318
 319        /* Once we finish add_grefs, it is unsafe to touch the new reference,
 320         * since it is possible for a concurrent ioctl to remove it (by guessing
 321         * its index). If the userspace application doesn't provide valid memory
 322         * to write the IDs to, then it will need to close the file in order to
 323         * release - which it will do by segfaulting when it tries to access the
 324         * IDs to close them.
 325         */
 326        if (copy_to_user(arg, &op, sizeof(op))) {
 327                rc = -EFAULT;
 328                goto out_free;
 329        }
 330        if (copy_to_user(arg->gref_ids, gref_ids,
 331                        sizeof(gref_ids[0]) * op.count)) {
 332                rc = -EFAULT;
 333                goto out_free;
 334        }
 335
 336out_free:
 337        kfree(gref_ids);
 338out:
 339        return rc;
 340}
 341
 342static long gntalloc_ioctl_dealloc(struct gntalloc_file_private_data *priv,
 343                void __user *arg)
 344{
 345        int i, rc = 0;
 346        struct ioctl_gntalloc_dealloc_gref op;
 347        struct gntalloc_gref *gref, *n;
 348
 349        pr_debug("%s: priv %p\n", __func__, priv);
 350
 351        if (copy_from_user(&op, arg, sizeof(op))) {
 352                rc = -EFAULT;
 353                goto dealloc_grant_out;
 354        }
 355
 356        mutex_lock(&gref_mutex);
 357        gref = find_grefs(priv, op.index, op.count);
 358        if (gref) {
 359                /* Remove from the file list only, and decrease reference count.
 360                 * The later call to do_cleanup() will remove from gref_list and
 361                 * free the memory if the pages aren't mapped anywhere.
 362                 */
 363                for (i = 0; i < op.count; i++) {
 364                        n = list_entry(gref->next_file.next,
 365                                struct gntalloc_gref, next_file);
 366                        list_del(&gref->next_file);
 367                        gref->users--;
 368                        gref = n;
 369                }
 370        } else {
 371                rc = -EINVAL;
 372        }
 373
 374        do_cleanup();
 375
 376        mutex_unlock(&gref_mutex);
 377dealloc_grant_out:
 378        return rc;
 379}
 380
 381static long gntalloc_ioctl_unmap_notify(struct gntalloc_file_private_data *priv,
 382                void __user *arg)
 383{
 384        struct ioctl_gntalloc_unmap_notify op;
 385        struct gntalloc_gref *gref;
 386        uint64_t index;
 387        int pgoff;
 388        int rc;
 389
 390        if (copy_from_user(&op, arg, sizeof(op)))
 391                return -EFAULT;
 392
 393        index = op.index & ~(PAGE_SIZE - 1);
 394        pgoff = op.index & (PAGE_SIZE - 1);
 395
 396        mutex_lock(&gref_mutex);
 397
 398        gref = find_grefs(priv, index, 1);
 399        if (!gref) {
 400                rc = -ENOENT;
 401                goto unlock_out;
 402        }
 403
 404        if (op.action & ~(UNMAP_NOTIFY_CLEAR_BYTE|UNMAP_NOTIFY_SEND_EVENT)) {
 405                rc = -EINVAL;
 406                goto unlock_out;
 407        }
 408
 409        /* We need to grab a reference to the event channel we are going to use
 410         * to send the notify before releasing the reference we may already have
 411         * (if someone has called this ioctl twice). This is required so that
 412         * it is possible to change the clear_byte part of the notification
 413         * without disturbing the event channel part, which may now be the last
 414         * reference to that event channel.
 415         */
 416        if (op.action & UNMAP_NOTIFY_SEND_EVENT) {
 417                if (evtchn_get(op.event_channel_port)) {
 418                        rc = -EINVAL;
 419                        goto unlock_out;
 420                }
 421        }
 422
 423        if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT)
 424                evtchn_put(gref->notify.event);
 425
 426        gref->notify.flags = op.action;
 427        gref->notify.pgoff = pgoff;
 428        gref->notify.event = op.event_channel_port;
 429        rc = 0;
 430
 431 unlock_out:
 432        mutex_unlock(&gref_mutex);
 433        return rc;
 434}
 435
 436static long gntalloc_ioctl(struct file *filp, unsigned int cmd,
 437                unsigned long arg)
 438{
 439        struct gntalloc_file_private_data *priv = filp->private_data;
 440
 441        switch (cmd) {
 442        case IOCTL_GNTALLOC_ALLOC_GREF:
 443                return gntalloc_ioctl_alloc(priv, (void __user *)arg);
 444
 445        case IOCTL_GNTALLOC_DEALLOC_GREF:
 446                return gntalloc_ioctl_dealloc(priv, (void __user *)arg);
 447
 448        case IOCTL_GNTALLOC_SET_UNMAP_NOTIFY:
 449                return gntalloc_ioctl_unmap_notify(priv, (void __user *)arg);
 450
 451        default:
 452                return -ENOIOCTLCMD;
 453        }
 454
 455        return 0;
 456}
 457
 458static void gntalloc_vma_open(struct vm_area_struct *vma)
 459{
 460        struct gntalloc_vma_private_data *priv = vma->vm_private_data;
 461
 462        if (!priv)
 463                return;
 464
 465        mutex_lock(&gref_mutex);
 466        priv->users++;
 467        mutex_unlock(&gref_mutex);
 468}
 469
 470static void gntalloc_vma_close(struct vm_area_struct *vma)
 471{
 472        struct gntalloc_vma_private_data *priv = vma->vm_private_data;
 473        struct gntalloc_gref *gref, *next;
 474        int i;
 475
 476        if (!priv)
 477                return;
 478
 479        mutex_lock(&gref_mutex);
 480        priv->users--;
 481        if (priv->users == 0) {
 482                gref = priv->gref;
 483                for (i = 0; i < priv->count; i++) {
 484                        gref->users--;
 485                        next = list_entry(gref->next_gref.next,
 486                                          struct gntalloc_gref, next_gref);
 487                        if (gref->users == 0)
 488                                __del_gref(gref);
 489                        gref = next;
 490                }
 491                kfree(priv);
 492        }
 493        mutex_unlock(&gref_mutex);
 494}
 495
 496static struct vm_operations_struct gntalloc_vmops = {
 497        .open = gntalloc_vma_open,
 498        .close = gntalloc_vma_close,
 499};
 500
 501static int gntalloc_mmap(struct file *filp, struct vm_area_struct *vma)
 502{
 503        struct gntalloc_file_private_data *priv = filp->private_data;
 504        struct gntalloc_vma_private_data *vm_priv;
 505        struct gntalloc_gref *gref;
 506        int count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
 507        int rv, i;
 508
 509        if (!(vma->vm_flags & VM_SHARED)) {
 510                printk(KERN_ERR "%s: Mapping must be shared.\n", __func__);
 511                return -EINVAL;
 512        }
 513
 514        vm_priv = kmalloc(sizeof(*vm_priv), GFP_KERNEL);
 515        if (!vm_priv)
 516                return -ENOMEM;
 517
 518        mutex_lock(&gref_mutex);
 519
 520        pr_debug("%s: priv %p,%p, page %lu+%d\n", __func__,
 521                       priv, vm_priv, vma->vm_pgoff, count);
 522
 523        gref = find_grefs(priv, vma->vm_pgoff << PAGE_SHIFT, count);
 524        if (gref == NULL) {
 525                rv = -ENOENT;
 526                pr_debug("%s: Could not find grant reference",
 527                                __func__);
 528                kfree(vm_priv);
 529                goto out_unlock;
 530        }
 531
 532        vm_priv->gref = gref;
 533        vm_priv->users = 1;
 534        vm_priv->count = count;
 535
 536        vma->vm_private_data = vm_priv;
 537
 538        vma->vm_flags |= VM_RESERVED | VM_DONTEXPAND;
 539
 540        vma->vm_ops = &gntalloc_vmops;
 541
 542        for (i = 0; i < count; i++) {
 543                gref->users++;
 544                rv = vm_insert_page(vma, vma->vm_start + i * PAGE_SIZE,
 545                                gref->page);
 546                if (rv)
 547                        goto out_unlock;
 548
 549                gref = list_entry(gref->next_file.next,
 550                                struct gntalloc_gref, next_file);
 551        }
 552        rv = 0;
 553
 554out_unlock:
 555        mutex_unlock(&gref_mutex);
 556        return rv;
 557}
 558
 559static const struct file_operations gntalloc_fops = {
 560        .owner = THIS_MODULE,
 561        .open = gntalloc_open,
 562        .release = gntalloc_release,
 563        .unlocked_ioctl = gntalloc_ioctl,
 564        .mmap = gntalloc_mmap
 565};
 566
 567/*
 568 * -------------------------------------
 569 * Module creation/destruction.
 570 * -------------------------------------
 571 */
 572static struct miscdevice gntalloc_miscdev = {
 573        .minor  = MISC_DYNAMIC_MINOR,
 574        .name   = "xen/gntalloc",
 575        .fops   = &gntalloc_fops,
 576};
 577
 578static int __init gntalloc_init(void)
 579{
 580        int err;
 581
 582        if (!xen_domain())
 583                return -ENODEV;
 584
 585        err = misc_register(&gntalloc_miscdev);
 586        if (err != 0) {
 587                printk(KERN_ERR "Could not register misc gntalloc device\n");
 588                return err;
 589        }
 590
 591        pr_debug("Created grant allocation device at %d,%d\n",
 592                        MISC_MAJOR, gntalloc_miscdev.minor);
 593
 594        return 0;
 595}
 596
 597static void __exit gntalloc_exit(void)
 598{
 599        misc_deregister(&gntalloc_miscdev);
 600}
 601
 602module_init(gntalloc_init);
 603module_exit(gntalloc_exit);
 604
 605MODULE_LICENSE("GPL");
 606MODULE_AUTHOR("Carter Weatherly <carter.weatherly@jhuapl.edu>, "
 607                "Daniel De Graaf <dgdegra@tycho.nsa.gov>");
 608MODULE_DESCRIPTION("User-space grant reference allocator driver");
 609
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.