linux/tools/virtio/vringh_test.c
<<
>>
Prefs
   1/* Simple test of virtio code, entirely in userpsace. */
   2#define _GNU_SOURCE
   3#include <sched.h>
   4#include <err.h>
   5#include <linux/kernel.h>
   6#include <linux/err.h>
   7#include <linux/virtio.h>
   8#include <linux/vringh.h>
   9#include <linux/virtio_ring.h>
  10#include <linux/uaccess.h>
  11#include <sys/types.h>
  12#include <sys/stat.h>
  13#include <sys/mman.h>
  14#include <sys/wait.h>
  15#include <fcntl.h>
  16
  17#define USER_MEM (1024*1024)
  18void *__user_addr_min, *__user_addr_max;
  19void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end;
  20static u64 user_addr_offset;
  21
  22#define RINGSIZE 256
  23#define ALIGN 4096
  24
  25static void never_notify_host(struct virtqueue *vq)
  26{
  27        abort();
  28}
  29
  30static void never_callback_guest(struct virtqueue *vq)
  31{
  32        abort();
  33}
  34
  35static bool getrange_iov(struct vringh *vrh, u64 addr, struct vringh_range *r)
  36{
  37        if (addr < (u64)(unsigned long)__user_addr_min - user_addr_offset)
  38                return false;
  39        if (addr >= (u64)(unsigned long)__user_addr_max - user_addr_offset)
  40                return false;
  41
  42        r->start = (u64)(unsigned long)__user_addr_min - user_addr_offset;
  43        r->end_incl = (u64)(unsigned long)__user_addr_max - 1 - user_addr_offset;
  44        r->offset = user_addr_offset;
  45        return true;
  46}
  47
  48/* We return single byte ranges. */
  49static bool getrange_slow(struct vringh *vrh, u64 addr, struct vringh_range *r)
  50{
  51        if (addr < (u64)(unsigned long)__user_addr_min - user_addr_offset)
  52                return false;
  53        if (addr >= (u64)(unsigned long)__user_addr_max - user_addr_offset)
  54                return false;
  55
  56        r->start = addr;
  57        r->end_incl = r->start;
  58        r->offset = user_addr_offset;
  59        return true;
  60}
  61
  62struct guest_virtio_device {
  63        struct virtio_device vdev;
  64        int to_host_fd;
  65        unsigned long notifies;
  66};
  67
  68static void parallel_notify_host(struct virtqueue *vq)
  69{
  70        struct guest_virtio_device *gvdev;
  71
  72        gvdev = container_of(vq->vdev, struct guest_virtio_device, vdev);
  73        write(gvdev->to_host_fd, "", 1);
  74        gvdev->notifies++;
  75}
  76
  77static void no_notify_host(struct virtqueue *vq)
  78{
  79}
  80
  81#define NUM_XFERS (10000000)
  82
  83/* We aim for two "distant" cpus. */
  84static void find_cpus(unsigned int *first, unsigned int *last)
  85{
  86        unsigned int i;
  87
  88        *first = -1U;
  89        *last = 0;
  90        for (i = 0; i < 4096; i++) {
  91                cpu_set_t set;
  92                CPU_ZERO(&set);
  93                CPU_SET(i, &set);
  94                if (sched_setaffinity(getpid(), sizeof(set), &set) == 0) {
  95                        if (i < *first)
  96                                *first = i;
  97                        if (i > *last)
  98                                *last = i;
  99                }
 100        }
 101}
 102
 103/* Opencoded version for fast mode */
 104static inline int vringh_get_head(struct vringh *vrh, u16 *head)
 105{
 106        u16 avail_idx, i;
 107        int err;
 108
 109        err = get_user(avail_idx, &vrh->vring.avail->idx);
 110        if (err)
 111                return err;
 112
 113        if (vrh->last_avail_idx == avail_idx)
 114                return 0;
 115
 116        /* Only get avail ring entries after they have been exposed by guest. */
 117        virtio_rmb(vrh->weak_barriers);
 118
 119        i = vrh->last_avail_idx & (vrh->vring.num - 1);
 120
 121        err = get_user(*head, &vrh->vring.avail->ring[i]);
 122        if (err)
 123                return err;
 124
 125        vrh->last_avail_idx++;
 126        return 1;
 127}
 128
 129static int parallel_test(unsigned long features,
 130                         bool (*getrange)(struct vringh *vrh,
 131                                          u64 addr, struct vringh_range *r),
 132                         bool fast_vringh)
 133{
 134        void *host_map, *guest_map;
 135        int fd, mapsize, to_guest[2], to_host[2];
 136        unsigned long xfers = 0, notifies = 0, receives = 0;
 137        unsigned int first_cpu, last_cpu;
 138        cpu_set_t cpu_set;
 139        char buf[128];
 140
 141        /* Create real file to mmap. */
 142        fd = open("/tmp/vringh_test-file", O_RDWR|O_CREAT|O_TRUNC, 0600);
 143        if (fd < 0)
 144                err(1, "Opening /tmp/vringh_test-file");
 145
 146        /* Extra room at the end for some data, and indirects */
 147        mapsize = vring_size(RINGSIZE, ALIGN)
 148                + RINGSIZE * 2 * sizeof(int)
 149                + RINGSIZE * 6 * sizeof(struct vring_desc);
 150        mapsize = (mapsize + getpagesize() - 1) & ~(getpagesize() - 1);
 151        ftruncate(fd, mapsize);
 152
 153        /* Parent and child use separate addresses, to check our mapping logic! */
 154        host_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
 155        guest_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
 156
 157        pipe(to_guest);
 158        pipe(to_host);
 159
 160        CPU_ZERO(&cpu_set);
 161        find_cpus(&first_cpu, &last_cpu);
 162        printf("Using CPUS %u and %u\n", first_cpu, last_cpu);
 163        fflush(stdout);
 164
 165        if (fork() != 0) {
 166                struct vringh vrh;
 167                int status, err, rlen = 0;
 168                char rbuf[5];
 169
 170                /* We are the host: never access guest addresses! */
 171                munmap(guest_map, mapsize);
 172
 173                __user_addr_min = host_map;
 174                __user_addr_max = __user_addr_min + mapsize;
 175                user_addr_offset = host_map - guest_map;
 176                assert(user_addr_offset);
 177
 178                close(to_guest[0]);
 179                close(to_host[1]);
 180
 181                vring_init(&vrh.vring, RINGSIZE, host_map, ALIGN);
 182                vringh_init_user(&vrh, features, RINGSIZE, true,
 183                                 vrh.vring.desc, vrh.vring.avail, vrh.vring.used);
 184                CPU_SET(first_cpu, &cpu_set);
 185                if (sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set))
 186                        errx(1, "Could not set affinity to cpu %u", first_cpu);
 187
 188                while (xfers < NUM_XFERS) {
 189                        struct iovec host_riov[2], host_wiov[2];
 190                        struct vringh_iov riov, wiov;
 191                        u16 head, written;
 192
 193                        if (fast_vringh) {
 194                                for (;;) {
 195                                        err = vringh_get_head(&vrh, &head);
 196                                        if (err != 0)
 197                                                break;
 198                                        err = vringh_need_notify_user(&vrh);
 199                                        if (err < 0)
 200                                                errx(1, "vringh_need_notify_user: %i",
 201                                                     err);
 202                                        if (err) {
 203                                                write(to_guest[1], "", 1);
 204                                                notifies++;
 205                                        }
 206                                }
 207                                if (err != 1)
 208                                        errx(1, "vringh_get_head");
 209                                written = 0;
 210                                goto complete;
 211                        } else {
 212                                vringh_iov_init(&riov,
 213                                                host_riov,
 214                                                ARRAY_SIZE(host_riov));
 215                                vringh_iov_init(&wiov,
 216                                                host_wiov,
 217                                                ARRAY_SIZE(host_wiov));
 218
 219                                err = vringh_getdesc_user(&vrh, &riov, &wiov,
 220                                                          getrange, &head);
 221                        }
 222                        if (err == 0) {
 223                                err = vringh_need_notify_user(&vrh);
 224                                if (err < 0)
 225                                        errx(1, "vringh_need_notify_user: %i",
 226                                             err);
 227                                if (err) {
 228                                        write(to_guest[1], "", 1);
 229                                        notifies++;
 230                                }
 231
 232                                if (!vringh_notify_enable_user(&vrh))
 233                                        continue;
 234
 235                                /* Swallow all notifies at once. */
 236                                if (read(to_host[0], buf, sizeof(buf)) < 1)
 237                                        break;
 238
 239                                vringh_notify_disable_user(&vrh);
 240                                receives++;
 241                                continue;
 242                        }
 243                        if (err != 1)
 244                                errx(1, "vringh_getdesc_user: %i", err);
 245
 246                        /* We simply copy bytes. */
 247                        if (riov.used) {
 248                                rlen = vringh_iov_pull_user(&riov, rbuf,
 249                                                            sizeof(rbuf));
 250                                if (rlen != 4)
 251                                        errx(1, "vringh_iov_pull_user: %i",
 252                                             rlen);
 253                                assert(riov.i == riov.used);
 254                                written = 0;
 255                        } else {
 256                                err = vringh_iov_push_user(&wiov, rbuf, rlen);
 257                                if (err != rlen)
 258                                        errx(1, "vringh_iov_push_user: %i",
 259                                             err);
 260                                assert(wiov.i == wiov.used);
 261                                written = err;
 262                        }
 263                complete:
 264                        xfers++;
 265
 266                        err = vringh_complete_user(&vrh, head, written);
 267                        if (err != 0)
 268                                errx(1, "vringh_complete_user: %i", err);
 269                }
 270
 271                err = vringh_need_notify_user(&vrh);
 272                if (err < 0)
 273                        errx(1, "vringh_need_notify_user: %i", err);
 274                if (err) {
 275                        write(to_guest[1], "", 1);
 276                        notifies++;
 277                }
 278                wait(&status);
 279                if (!WIFEXITED(status))
 280                        errx(1, "Child died with signal %i?", WTERMSIG(status));
 281                if (WEXITSTATUS(status) != 0)
 282                        errx(1, "Child exited %i?", WEXITSTATUS(status));
 283                printf("Host: notified %lu, pinged %lu\n", notifies, receives);
 284                return 0;
 285        } else {
 286                struct guest_virtio_device gvdev;
 287                struct virtqueue *vq;
 288                unsigned int *data;
 289                struct vring_desc *indirects;
 290                unsigned int finished = 0;
 291
 292                /* We pass sg[]s pointing into here, but we need RINGSIZE+1 */
 293                data = guest_map + vring_size(RINGSIZE, ALIGN);
 294                indirects = (void *)data + (RINGSIZE + 1) * 2 * sizeof(int);
 295
 296                /* We are the guest. */
 297                munmap(host_map, mapsize);
 298
 299                close(to_guest[1]);
 300                close(to_host[0]);
 301
 302                gvdev.vdev.features[0] = features;
 303                gvdev.to_host_fd = to_host[1];
 304                gvdev.notifies = 0;
 305
 306                CPU_SET(first_cpu, &cpu_set);
 307                if (sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set))
 308                        err(1, "Could not set affinity to cpu %u", first_cpu);
 309
 310                vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &gvdev.vdev, true,
 311                                         guest_map, fast_vringh ? no_notify_host
 312                                         : parallel_notify_host,
 313                                         never_callback_guest, "guest vq");
 314
 315                /* Don't kfree indirects. */
 316                __kfree_ignore_start = indirects;
 317                __kfree_ignore_end = indirects + RINGSIZE * 6;
 318
 319                while (xfers < NUM_XFERS) {
 320                        struct scatterlist sg[4];
 321                        unsigned int num_sg, len;
 322                        int *dbuf, err;
 323                        bool output = !(xfers % 2);
 324
 325                        /* Consume bufs. */
 326                        while ((dbuf = virtqueue_get_buf(vq, &len)) != NULL) {
 327                                if (len == 4)
 328                                        assert(*dbuf == finished - 1);
 329                                else if (!fast_vringh)
 330                                        assert(*dbuf == finished);
 331                                finished++;
 332                        }
 333
 334                        /* Produce a buffer. */
 335                        dbuf = data + (xfers % (RINGSIZE + 1));
 336
 337                        if (output)
 338                                *dbuf = xfers;
 339                        else
 340                                *dbuf = -1;
 341
 342                        switch ((xfers / sizeof(*dbuf)) % 4) {
 343                        case 0:
 344                                /* Nasty three-element sg list. */
 345                                sg_init_table(sg, num_sg = 3);
 346                                sg_set_buf(&sg[0], (void *)dbuf, 1);
 347                                sg_set_buf(&sg[1], (void *)dbuf + 1, 2);
 348                                sg_set_buf(&sg[2], (void *)dbuf + 3, 1);
 349                                break;
 350                        case 1:
 351                                sg_init_table(sg, num_sg = 2);
 352                                sg_set_buf(&sg[0], (void *)dbuf, 1);
 353                                sg_set_buf(&sg[1], (void *)dbuf + 1, 3);
 354                                break;
 355                        case 2:
 356                                sg_init_table(sg, num_sg = 1);
 357                                sg_set_buf(&sg[0], (void *)dbuf, 4);
 358                                break;
 359                        case 3:
 360                                sg_init_table(sg, num_sg = 4);
 361                                sg_set_buf(&sg[0], (void *)dbuf, 1);
 362                                sg_set_buf(&sg[1], (void *)dbuf + 1, 1);
 363                                sg_set_buf(&sg[2], (void *)dbuf + 2, 1);
 364                                sg_set_buf(&sg[3], (void *)dbuf + 3, 1);
 365                                break;
 366                        }
 367
 368                        /* May allocate an indirect, so force it to allocate
 369                         * user addr */
 370                        __kmalloc_fake = indirects + (xfers % RINGSIZE) * 4;
 371                        if (output)
 372                                err = virtqueue_add_outbuf(vq, sg, num_sg, dbuf,
 373                                                           GFP_KERNEL);
 374                        else
 375                                err = virtqueue_add_inbuf(vq, sg, num_sg,
 376                                                          dbuf, GFP_KERNEL);
 377
 378                        if (err == -ENOSPC) {
 379                                if (!virtqueue_enable_cb_delayed(vq))
 380                                        continue;
 381                                /* Swallow all notifies at once. */
 382                                if (read(to_guest[0], buf, sizeof(buf)) < 1)
 383                                        break;
 384                                
 385                                receives++;
 386                                virtqueue_disable_cb(vq);
 387                                continue;
 388                        }
 389
 390                        if (err)
 391                                errx(1, "virtqueue_add_in/outbuf: %i", err);
 392
 393                        xfers++;
 394                        virtqueue_kick(vq);
 395                }
 396
 397                /* Any extra? */
 398                while (finished != xfers) {
 399                        int *dbuf;
 400                        unsigned int len;
 401
 402                        /* Consume bufs. */
 403                        dbuf = virtqueue_get_buf(vq, &len);
 404                        if (dbuf) {
 405                                if (len == 4)
 406                                        assert(*dbuf == finished - 1);
 407                                else
 408                                        assert(len == 0);
 409                                finished++;
 410                                continue;
 411                        }
 412
 413                        if (!virtqueue_enable_cb_delayed(vq))
 414                                continue;
 415                        if (read(to_guest[0], buf, sizeof(buf)) < 1)
 416                                break;
 417                                
 418                        receives++;
 419                        virtqueue_disable_cb(vq);
 420                }
 421
 422                printf("Guest: notified %lu, pinged %lu\n",
 423                       gvdev.notifies, receives);
 424                vring_del_virtqueue(vq);
 425                return 0;
 426        }
 427}
 428
 429int main(int argc, char *argv[])
 430{
 431        struct virtio_device vdev;
 432        struct virtqueue *vq;
 433        struct vringh vrh;
 434        struct scatterlist guest_sg[RINGSIZE], *sgs[2];
 435        struct iovec host_riov[2], host_wiov[2];
 436        struct vringh_iov riov, wiov;
 437        struct vring_used_elem used[RINGSIZE];
 438        char buf[28];
 439        u16 head;
 440        int err;
 441        unsigned i;
 442        void *ret;
 443        bool (*getrange)(struct vringh *vrh, u64 addr, struct vringh_range *r);
 444        bool fast_vringh = false, parallel = false;
 445
 446        getrange = getrange_iov;
 447        vdev.features[0] = 0;
 448
 449        while (argv[1]) {
 450                if (strcmp(argv[1], "--indirect") == 0)
 451                        vdev.features[0] |= (1 << VIRTIO_RING_F_INDIRECT_DESC);
 452                else if (strcmp(argv[1], "--eventidx") == 0)
 453                        vdev.features[0] |= (1 << VIRTIO_RING_F_EVENT_IDX);
 454                else if (strcmp(argv[1], "--slow-range") == 0)
 455                        getrange = getrange_slow;
 456                else if (strcmp(argv[1], "--fast-vringh") == 0)
 457                        fast_vringh = true;
 458                else if (strcmp(argv[1], "--parallel") == 0)
 459                        parallel = true;
 460                else
 461                        errx(1, "Unknown arg %s", argv[1]);
 462                argv++;
 463        }
 464
 465        if (parallel)
 466                return parallel_test(vdev.features[0], getrange, fast_vringh);
 467
 468        if (posix_memalign(&__user_addr_min, PAGE_SIZE, USER_MEM) != 0)
 469                abort();
 470        __user_addr_max = __user_addr_min + USER_MEM;
 471        memset(__user_addr_min, 0, vring_size(RINGSIZE, ALIGN));
 472
 473        /* Set up guest side. */
 474        vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &vdev, true,
 475                                 __user_addr_min,
 476                                 never_notify_host, never_callback_guest,
 477                                 "guest vq");
 478
 479        /* Set up host side. */
 480        vring_init(&vrh.vring, RINGSIZE, __user_addr_min, ALIGN);
 481        vringh_init_user(&vrh, vdev.features[0], RINGSIZE, true,
 482                         vrh.vring.desc, vrh.vring.avail, vrh.vring.used);
 483
 484        /* No descriptor to get yet... */
 485        err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
 486        if (err != 0)
 487                errx(1, "vringh_getdesc_user: %i", err);
 488
 489        /* Guest puts in a descriptor. */
 490        memcpy(__user_addr_max - 1, "a", 1);
 491        sg_init_table(guest_sg, 1);
 492        sg_set_buf(&guest_sg[0], __user_addr_max - 1, 1);
 493        sg_init_table(guest_sg+1, 1);
 494        sg_set_buf(&guest_sg[1], __user_addr_max - 3, 2);
 495        sgs[0] = &guest_sg[0];
 496        sgs[1] = &guest_sg[1];
 497
 498        /* May allocate an indirect, so force it to allocate user addr */
 499        __kmalloc_fake = __user_addr_min + vring_size(RINGSIZE, ALIGN);
 500        err = virtqueue_add_sgs(vq, sgs, 1, 1, &err, GFP_KERNEL);
 501        if (err)
 502                errx(1, "virtqueue_add_sgs: %i", err);
 503        __kmalloc_fake = NULL;
 504
 505        /* Host retreives it. */
 506        vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov));
 507        vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov));
 508
 509        err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
 510        if (err != 1)
 511                errx(1, "vringh_getdesc_user: %i", err);
 512
 513        assert(riov.used == 1);
 514        assert(riov.iov[0].iov_base == __user_addr_max - 1);
 515        assert(riov.iov[0].iov_len == 1);
 516        if (getrange != getrange_slow) {
 517                assert(wiov.used == 1);
 518                assert(wiov.iov[0].iov_base == __user_addr_max - 3);
 519                assert(wiov.iov[0].iov_len == 2);
 520        } else {
 521                assert(wiov.used == 2);
 522                assert(wiov.iov[0].iov_base == __user_addr_max - 3);
 523                assert(wiov.iov[0].iov_len == 1);
 524                assert(wiov.iov[1].iov_base == __user_addr_max - 2);
 525                assert(wiov.iov[1].iov_len == 1);
 526        }
 527
 528        err = vringh_iov_pull_user(&riov, buf, 5);
 529        if (err != 1)
 530                errx(1, "vringh_iov_pull_user: %i", err);
 531        assert(buf[0] == 'a');
 532        assert(riov.i == 1);
 533        assert(vringh_iov_pull_user(&riov, buf, 5) == 0);
 534
 535        memcpy(buf, "bcdef", 5);
 536        err = vringh_iov_push_user(&wiov, buf, 5);
 537        if (err != 2)
 538                errx(1, "vringh_iov_push_user: %i", err);
 539        assert(memcmp(__user_addr_max - 3, "bc", 2) == 0);
 540        assert(wiov.i == wiov.used);
 541        assert(vringh_iov_push_user(&wiov, buf, 5) == 0);
 542
 543        /* Host is done. */
 544        err = vringh_complete_user(&vrh, head, err);
 545        if (err != 0)
 546                errx(1, "vringh_complete_user: %i", err);
 547
 548        /* Guest should see used token now. */
 549        __kfree_ignore_start = __user_addr_min + vring_size(RINGSIZE, ALIGN);
 550        __kfree_ignore_end = __kfree_ignore_start + 1;
 551        ret = virtqueue_get_buf(vq, &i);
 552        if (ret != &err)
 553                errx(1, "virtqueue_get_buf: %p", ret);
 554        assert(i == 2);
 555
 556        /* Guest puts in a huge descriptor. */
 557        sg_init_table(guest_sg, RINGSIZE);
 558        for (i = 0; i < RINGSIZE; i++) {
 559                sg_set_buf(&guest_sg[i],
 560                           __user_addr_max - USER_MEM/4, USER_MEM/4);
 561        }
 562
 563        /* Fill contents with recognisable garbage. */
 564        for (i = 0; i < USER_MEM/4; i++)
 565                ((char *)__user_addr_max - USER_MEM/4)[i] = i;
 566
 567        /* This will allocate an indirect, so force it to allocate user addr */
 568        __kmalloc_fake = __user_addr_min + vring_size(RINGSIZE, ALIGN);
 569        err = virtqueue_add_outbuf(vq, guest_sg, RINGSIZE, &err, GFP_KERNEL);
 570        if (err)
 571                errx(1, "virtqueue_add_outbuf (large): %i", err);
 572        __kmalloc_fake = NULL;
 573
 574        /* Host picks it up (allocates new iov). */
 575        vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov));
 576        vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov));
 577
 578        err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
 579        if (err != 1)
 580                errx(1, "vringh_getdesc_user: %i", err);
 581
 582        assert(riov.max_num & VRINGH_IOV_ALLOCATED);
 583        assert(riov.iov != host_riov);
 584        if (getrange != getrange_slow)
 585                assert(riov.used == RINGSIZE);
 586        else
 587                assert(riov.used == RINGSIZE * USER_MEM/4);
 588
 589        assert(!(wiov.max_num & VRINGH_IOV_ALLOCATED));
 590        assert(wiov.used == 0);
 591
 592        /* Pull data back out (in odd chunks), should be as expected. */
 593        for (i = 0; i < RINGSIZE * USER_MEM/4; i += 3) {
 594                err = vringh_iov_pull_user(&riov, buf, 3);
 595                if (err != 3 && i + err != RINGSIZE * USER_MEM/4)
 596                        errx(1, "vringh_iov_pull_user large: %i", err);
 597                assert(buf[0] == (char)i);
 598                assert(err < 2 || buf[1] == (char)(i + 1));
 599                assert(err < 3 || buf[2] == (char)(i + 2));
 600        }
 601        assert(riov.i == riov.used);
 602        vringh_iov_cleanup(&riov);
 603        vringh_iov_cleanup(&wiov);
 604
 605        /* Complete using multi interface, just because we can. */
 606        used[0].id = head;
 607        used[0].len = 0;
 608        err = vringh_complete_multi_user(&vrh, used, 1);
 609        if (err)
 610                errx(1, "vringh_complete_multi_user(1): %i", err);
 611
 612        /* Free up those descriptors. */
 613        ret = virtqueue_get_buf(vq, &i);
 614        if (ret != &err)
 615                errx(1, "virtqueue_get_buf: %p", ret);
 616
 617        /* Add lots of descriptors. */
 618        sg_init_table(guest_sg, 1);
 619        sg_set_buf(&guest_sg[0], __user_addr_max - 1, 1);
 620        for (i = 0; i < RINGSIZE; i++) {
 621                err = virtqueue_add_outbuf(vq, guest_sg, 1, &err, GFP_KERNEL);
 622                if (err)
 623                        errx(1, "virtqueue_add_outbuf (multiple): %i", err);
 624        }
 625
 626        /* Now get many, and consume them all at once. */
 627        vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov));
 628        vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov));
 629
 630        for (i = 0; i < RINGSIZE; i++) {
 631                err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
 632                if (err != 1)
 633                        errx(1, "vringh_getdesc_user: %i", err);
 634                used[i].id = head;
 635                used[i].len = 0;
 636        }
 637        /* Make sure it wraps around ring, to test! */
 638        assert(vrh.vring.used->idx % RINGSIZE != 0);
 639        err = vringh_complete_multi_user(&vrh, used, RINGSIZE);
 640        if (err)
 641                errx(1, "vringh_complete_multi_user: %i", err);
 642
 643        /* Free those buffers. */
 644        for (i = 0; i < RINGSIZE; i++) {
 645                unsigned len;
 646                assert(virtqueue_get_buf(vq, &len) != NULL);
 647        }
 648
 649        /* Test weird (but legal!) indirect. */
 650        if (vdev.features[0] & (1 << VIRTIO_RING_F_INDIRECT_DESC)) {
 651                char *data = __user_addr_max - USER_MEM/4;
 652                struct vring_desc *d = __user_addr_max - USER_MEM/2;
 653                struct vring vring;
 654
 655                /* Force creation of direct, which we modify. */
 656                vdev.features[0] &= ~(1 << VIRTIO_RING_F_INDIRECT_DESC);
 657                vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &vdev, true,
 658                                         __user_addr_min,
 659                                         never_notify_host,
 660                                         never_callback_guest,
 661                                         "guest vq");
 662
 663                sg_init_table(guest_sg, 4);
 664                sg_set_buf(&guest_sg[0], d, sizeof(*d)*2);
 665                sg_set_buf(&guest_sg[1], d + 2, sizeof(*d)*1);
 666                sg_set_buf(&guest_sg[2], data + 6, 4);
 667                sg_set_buf(&guest_sg[3], d + 3, sizeof(*d)*3);
 668
 669                err = virtqueue_add_outbuf(vq, guest_sg, 4, &err, GFP_KERNEL);
 670                if (err)
 671                        errx(1, "virtqueue_add_outbuf (indirect): %i", err);
 672
 673                vring_init(&vring, RINGSIZE, __user_addr_min, ALIGN);
 674
 675                /* They're used in order, but double-check... */
 676                assert(vring.desc[0].addr == (unsigned long)d);
 677                assert(vring.desc[1].addr == (unsigned long)(d+2));
 678                assert(vring.desc[2].addr == (unsigned long)data + 6);
 679                assert(vring.desc[3].addr == (unsigned long)(d+3));
 680                vring.desc[0].flags |= VRING_DESC_F_INDIRECT;
 681                vring.desc[1].flags |= VRING_DESC_F_INDIRECT;
 682                vring.desc[3].flags |= VRING_DESC_F_INDIRECT;
 683
 684                /* First indirect */
 685                d[0].addr = (unsigned long)data;
 686                d[0].len = 1;
 687                d[0].flags = VRING_DESC_F_NEXT;
 688                d[0].next = 1;
 689                d[1].addr = (unsigned long)data + 1;
 690                d[1].len = 2;
 691                d[1].flags = 0;
 692
 693                /* Second indirect */
 694                d[2].addr = (unsigned long)data + 3;
 695                d[2].len = 3;
 696                d[2].flags = 0;
 697
 698                /* Third indirect */
 699                d[3].addr = (unsigned long)data + 10;
 700                d[3].len = 5;
 701                d[3].flags = VRING_DESC_F_NEXT;
 702                d[3].next = 1;
 703                d[4].addr = (unsigned long)data + 15;
 704                d[4].len = 6;
 705                d[4].flags = VRING_DESC_F_NEXT;
 706                d[4].next = 2;
 707                d[5].addr = (unsigned long)data + 21;
 708                d[5].len = 7;
 709                d[5].flags = 0;
 710
 711                /* Host picks it up (allocates new iov). */
 712                vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov));
 713                vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov));
 714
 715                err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
 716                if (err != 1)
 717                        errx(1, "vringh_getdesc_user: %i", err);
 718
 719                if (head != 0)
 720                        errx(1, "vringh_getdesc_user: head %i not 0", head);
 721
 722                assert(riov.max_num & VRINGH_IOV_ALLOCATED);
 723                if (getrange != getrange_slow)
 724                        assert(riov.used == 7);
 725                else
 726                        assert(riov.used == 28);
 727                err = vringh_iov_pull_user(&riov, buf, 29);
 728                assert(err == 28);
 729
 730                /* Data should be linear. */
 731                for (i = 0; i < err; i++)
 732                        assert(buf[i] == i);
 733                vringh_iov_cleanup(&riov);
 734        }
 735
 736        /* Don't leak memory... */
 737        vring_del_virtqueue(vq);
 738        free(__user_addr_min);
 739
 740        return 0;
 741}
 742
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.