linux/drivers/xen/pvcalls-front.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * (c) 2017 Stefano Stabellini <stefano@aporeto.com>
   4 */
   5
   6#include <linux/module.h>
   7#include <linux/net.h>
   8#include <linux/socket.h>
   9
  10#include <net/sock.h>
  11
  12#include <xen/events.h>
  13#include <xen/grant_table.h>
  14#include <xen/xen.h>
  15#include <xen/xenbus.h>
  16#include <xen/interface/io/pvcalls.h>
  17
  18#include "pvcalls-front.h"
  19
  20#define PVCALLS_INVALID_ID UINT_MAX
  21#define PVCALLS_RING_ORDER XENBUS_MAX_RING_GRANT_ORDER
  22#define PVCALLS_NR_RSP_PER_RING __CONST_RING_SIZE(xen_pvcalls, XEN_PAGE_SIZE)
  23#define PVCALLS_FRONT_MAX_SPIN 5000
  24
  25static struct proto pvcalls_proto = {
  26        .name   = "PVCalls",
  27        .owner  = THIS_MODULE,
  28        .obj_size = sizeof(struct sock),
  29};
  30
  31struct pvcalls_bedata {
  32        struct xen_pvcalls_front_ring ring;
  33        grant_ref_t ref;
  34        int irq;
  35
  36        struct list_head socket_mappings;
  37        spinlock_t socket_lock;
  38
  39        wait_queue_head_t inflight_req;
  40        struct xen_pvcalls_response rsp[PVCALLS_NR_RSP_PER_RING];
  41};
  42/* Only one front/back connection supported. */
  43static struct xenbus_device *pvcalls_front_dev;
  44static atomic_t pvcalls_refcount;
  45
  46/* first increment refcount, then proceed */
  47#define pvcalls_enter() {               \
  48        atomic_inc(&pvcalls_refcount);      \
  49}
  50
  51/* first complete other operations, then decrement refcount */
  52#define pvcalls_exit() {                \
  53        atomic_dec(&pvcalls_refcount);      \
  54}
  55
  56struct sock_mapping {
  57        bool active_socket;
  58        struct list_head list;
  59        struct socket *sock;
  60        atomic_t refcount;
  61        union {
  62                struct {
  63                        int irq;
  64                        grant_ref_t ref;
  65                        struct pvcalls_data_intf *ring;
  66                        struct pvcalls_data data;
  67                        struct mutex in_mutex;
  68                        struct mutex out_mutex;
  69
  70                        wait_queue_head_t inflight_conn_req;
  71                } active;
  72                struct {
  73                /*
  74                 * Socket status, needs to be 64-bit aligned due to the
  75                 * test_and_* functions which have this requirement on arm64.
  76                 */
  77#define PVCALLS_STATUS_UNINITALIZED  0
  78#define PVCALLS_STATUS_BIND          1
  79#define PVCALLS_STATUS_LISTEN        2
  80                        uint8_t status __attribute__((aligned(8)));
  81                /*
  82                 * Internal state-machine flags.
  83                 * Only one accept operation can be inflight for a socket.
  84                 * Only one poll operation can be inflight for a given socket.
  85                 * flags needs to be 64-bit aligned due to the test_and_*
  86                 * functions which have this requirement on arm64.
  87                 */
  88#define PVCALLS_FLAG_ACCEPT_INFLIGHT 0
  89#define PVCALLS_FLAG_POLL_INFLIGHT   1
  90#define PVCALLS_FLAG_POLL_RET        2
  91                        uint8_t flags __attribute__((aligned(8)));
  92                        uint32_t inflight_req_id;
  93                        struct sock_mapping *accept_map;
  94                        wait_queue_head_t inflight_accept_req;
  95                } passive;
  96        };
  97};
  98
  99static inline struct sock_mapping *pvcalls_enter_sock(struct socket *sock)
 100{
 101        struct sock_mapping *map;
 102
 103        if (!pvcalls_front_dev ||
 104                dev_get_drvdata(&pvcalls_front_dev->dev) == NULL)
 105                return ERR_PTR(-ENOTCONN);
 106
 107        map = (struct sock_mapping *)sock->sk->sk_send_head;
 108        if (map == NULL)
 109                return ERR_PTR(-ENOTSOCK);
 110
 111        pvcalls_enter();
 112        atomic_inc(&map->refcount);
 113        return map;
 114}
 115
 116static inline void pvcalls_exit_sock(struct socket *sock)
 117{
 118        struct sock_mapping *map;
 119
 120        map = (struct sock_mapping *)sock->sk->sk_send_head;
 121        atomic_dec(&map->refcount);
 122        pvcalls_exit();
 123}
 124
 125static inline int get_request(struct pvcalls_bedata *bedata, int *req_id)
 126{
 127        *req_id = bedata->ring.req_prod_pvt & (RING_SIZE(&bedata->ring) - 1);
 128        if (RING_FULL(&bedata->ring) ||
 129            bedata->rsp[*req_id].req_id != PVCALLS_INVALID_ID)
 130                return -EAGAIN;
 131        return 0;
 132}
 133
 134static bool pvcalls_front_write_todo(struct sock_mapping *map)
 135{
 136        struct pvcalls_data_intf *intf = map->active.ring;
 137        RING_IDX cons, prod, size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 138        int32_t error;
 139
 140        error = intf->out_error;
 141        if (error == -ENOTCONN)
 142                return false;
 143        if (error != 0)
 144                return true;
 145
 146        cons = intf->out_cons;
 147        prod = intf->out_prod;
 148        return !!(size - pvcalls_queued(prod, cons, size));
 149}
 150
 151static bool pvcalls_front_read_todo(struct sock_mapping *map)
 152{
 153        struct pvcalls_data_intf *intf = map->active.ring;
 154        RING_IDX cons, prod;
 155        int32_t error;
 156
 157        cons = intf->in_cons;
 158        prod = intf->in_prod;
 159        error = intf->in_error;
 160        return (error != 0 ||
 161                pvcalls_queued(prod, cons,
 162                               XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER)) != 0);
 163}
 164
 165static irqreturn_t pvcalls_front_event_handler(int irq, void *dev_id)
 166{
 167        struct xenbus_device *dev = dev_id;
 168        struct pvcalls_bedata *bedata;
 169        struct xen_pvcalls_response *rsp;
 170        uint8_t *src, *dst;
 171        int req_id = 0, more = 0, done = 0;
 172
 173        if (dev == NULL)
 174                return IRQ_HANDLED;
 175
 176        pvcalls_enter();
 177        bedata = dev_get_drvdata(&dev->dev);
 178        if (bedata == NULL) {
 179                pvcalls_exit();
 180                return IRQ_HANDLED;
 181        }
 182
 183again:
 184        while (RING_HAS_UNCONSUMED_RESPONSES(&bedata->ring)) {
 185                rsp = RING_GET_RESPONSE(&bedata->ring, bedata->ring.rsp_cons);
 186
 187                req_id = rsp->req_id;
 188                if (rsp->cmd == PVCALLS_POLL) {
 189                        struct sock_mapping *map = (struct sock_mapping *)(uintptr_t)
 190                                                   rsp->u.poll.id;
 191
 192                        clear_bit(PVCALLS_FLAG_POLL_INFLIGHT,
 193                                  (void *)&map->passive.flags);
 194                        /*
 195                         * clear INFLIGHT, then set RET. It pairs with
 196                         * the checks at the beginning of
 197                         * pvcalls_front_poll_passive.
 198                         */
 199                        smp_wmb();
 200                        set_bit(PVCALLS_FLAG_POLL_RET,
 201                                (void *)&map->passive.flags);
 202                } else {
 203                        dst = (uint8_t *)&bedata->rsp[req_id] +
 204                              sizeof(rsp->req_id);
 205                        src = (uint8_t *)rsp + sizeof(rsp->req_id);
 206                        memcpy(dst, src, sizeof(*rsp) - sizeof(rsp->req_id));
 207                        /*
 208                         * First copy the rest of the data, then req_id. It is
 209                         * paired with the barrier when accessing bedata->rsp.
 210                         */
 211                        smp_wmb();
 212                        bedata->rsp[req_id].req_id = req_id;
 213                }
 214
 215                done = 1;
 216                bedata->ring.rsp_cons++;
 217        }
 218
 219        RING_FINAL_CHECK_FOR_RESPONSES(&bedata->ring, more);
 220        if (more)
 221                goto again;
 222        if (done)
 223                wake_up(&bedata->inflight_req);
 224        pvcalls_exit();
 225        return IRQ_HANDLED;
 226}
 227
 228static void pvcalls_front_free_map(struct pvcalls_bedata *bedata,
 229                                   struct sock_mapping *map)
 230{
 231        int i;
 232
 233        unbind_from_irqhandler(map->active.irq, map);
 234
 235        spin_lock(&bedata->socket_lock);
 236        if (!list_empty(&map->list))
 237                list_del_init(&map->list);
 238        spin_unlock(&bedata->socket_lock);
 239
 240        for (i = 0; i < (1 << PVCALLS_RING_ORDER); i++)
 241                gnttab_end_foreign_access(map->active.ring->ref[i], 0, 0);
 242        gnttab_end_foreign_access(map->active.ref, 0, 0);
 243        free_page((unsigned long)map->active.ring);
 244
 245        kfree(map);
 246}
 247
 248static irqreturn_t pvcalls_front_conn_handler(int irq, void *sock_map)
 249{
 250        struct sock_mapping *map = sock_map;
 251
 252        if (map == NULL)
 253                return IRQ_HANDLED;
 254
 255        wake_up_interruptible(&map->active.inflight_conn_req);
 256
 257        return IRQ_HANDLED;
 258}
 259
 260int pvcalls_front_socket(struct socket *sock)
 261{
 262        struct pvcalls_bedata *bedata;
 263        struct sock_mapping *map = NULL;
 264        struct xen_pvcalls_request *req;
 265        int notify, req_id, ret;
 266
 267        /*
 268         * PVCalls only supports domain AF_INET,
 269         * type SOCK_STREAM and protocol 0 sockets for now.
 270         *
 271         * Check socket type here, AF_INET and protocol checks are done
 272         * by the caller.
 273         */
 274        if (sock->type != SOCK_STREAM)
 275                return -EOPNOTSUPP;
 276
 277        pvcalls_enter();
 278        if (!pvcalls_front_dev) {
 279                pvcalls_exit();
 280                return -EACCES;
 281        }
 282        bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 283
 284        map = kzalloc(sizeof(*map), GFP_KERNEL);
 285        if (map == NULL) {
 286                pvcalls_exit();
 287                return -ENOMEM;
 288        }
 289
 290        spin_lock(&bedata->socket_lock);
 291
 292        ret = get_request(bedata, &req_id);
 293        if (ret < 0) {
 294                kfree(map);
 295                spin_unlock(&bedata->socket_lock);
 296                pvcalls_exit();
 297                return ret;
 298        }
 299
 300        /*
 301         * sock->sk->sk_send_head is not used for ip sockets: reuse the
 302         * field to store a pointer to the struct sock_mapping
 303         * corresponding to the socket. This way, we can easily get the
 304         * struct sock_mapping from the struct socket.
 305         */
 306        sock->sk->sk_send_head = (void *)map;
 307        list_add_tail(&map->list, &bedata->socket_mappings);
 308
 309        req = RING_GET_REQUEST(&bedata->ring, req_id);
 310        req->req_id = req_id;
 311        req->cmd = PVCALLS_SOCKET;
 312        req->u.socket.id = (uintptr_t) map;
 313        req->u.socket.domain = AF_INET;
 314        req->u.socket.type = SOCK_STREAM;
 315        req->u.socket.protocol = IPPROTO_IP;
 316
 317        bedata->ring.req_prod_pvt++;
 318        RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 319        spin_unlock(&bedata->socket_lock);
 320        if (notify)
 321                notify_remote_via_irq(bedata->irq);
 322
 323        wait_event(bedata->inflight_req,
 324                   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
 325
 326        /* read req_id, then the content */
 327        smp_rmb();
 328        ret = bedata->rsp[req_id].ret;
 329        bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 330
 331        pvcalls_exit();
 332        return ret;
 333}
 334
 335static void free_active_ring(struct sock_mapping *map)
 336{
 337        if (!map->active.ring)
 338                return;
 339
 340        free_pages((unsigned long)map->active.data.in,
 341                        map->active.ring->ring_order);
 342        free_page((unsigned long)map->active.ring);
 343}
 344
 345static int alloc_active_ring(struct sock_mapping *map)
 346{
 347        void *bytes;
 348
 349        map->active.ring = (struct pvcalls_data_intf *)
 350                get_zeroed_page(GFP_KERNEL);
 351        if (!map->active.ring)
 352                goto out;
 353
 354        map->active.ring->ring_order = PVCALLS_RING_ORDER;
 355        bytes = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
 356                                        PVCALLS_RING_ORDER);
 357        if (!bytes)
 358                goto out;
 359
 360        map->active.data.in = bytes;
 361        map->active.data.out = bytes +
 362                XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 363
 364        return 0;
 365
 366out:
 367        free_active_ring(map);
 368        return -ENOMEM;
 369}
 370
 371static int create_active(struct sock_mapping *map, evtchn_port_t *evtchn)
 372{
 373        void *bytes;
 374        int ret, irq = -1, i;
 375
 376        *evtchn = 0;
 377        init_waitqueue_head(&map->active.inflight_conn_req);
 378
 379        bytes = map->active.data.in;
 380        for (i = 0; i < (1 << PVCALLS_RING_ORDER); i++)
 381                map->active.ring->ref[i] = gnttab_grant_foreign_access(
 382                        pvcalls_front_dev->otherend_id,
 383                        pfn_to_gfn(virt_to_pfn(bytes) + i), 0);
 384
 385        map->active.ref = gnttab_grant_foreign_access(
 386                pvcalls_front_dev->otherend_id,
 387                pfn_to_gfn(virt_to_pfn((void *)map->active.ring)), 0);
 388
 389        ret = xenbus_alloc_evtchn(pvcalls_front_dev, evtchn);
 390        if (ret)
 391                goto out_error;
 392        irq = bind_evtchn_to_irqhandler(*evtchn, pvcalls_front_conn_handler,
 393                                        0, "pvcalls-frontend", map);
 394        if (irq < 0) {
 395                ret = irq;
 396                goto out_error;
 397        }
 398
 399        map->active.irq = irq;
 400        map->active_socket = true;
 401        mutex_init(&map->active.in_mutex);
 402        mutex_init(&map->active.out_mutex);
 403
 404        return 0;
 405
 406out_error:
 407        if (*evtchn > 0)
 408                xenbus_free_evtchn(pvcalls_front_dev, *evtchn);
 409        return ret;
 410}
 411
 412int pvcalls_front_connect(struct socket *sock, struct sockaddr *addr,
 413                                int addr_len, int flags)
 414{
 415        struct pvcalls_bedata *bedata;
 416        struct sock_mapping *map = NULL;
 417        struct xen_pvcalls_request *req;
 418        int notify, req_id, ret;
 419        evtchn_port_t evtchn;
 420
 421        if (addr->sa_family != AF_INET || sock->type != SOCK_STREAM)
 422                return -EOPNOTSUPP;
 423
 424        map = pvcalls_enter_sock(sock);
 425        if (IS_ERR(map))
 426                return PTR_ERR(map);
 427
 428        bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 429        ret = alloc_active_ring(map);
 430        if (ret < 0) {
 431                pvcalls_exit_sock(sock);
 432                return ret;
 433        }
 434
 435        spin_lock(&bedata->socket_lock);
 436        ret = get_request(bedata, &req_id);
 437        if (ret < 0) {
 438                spin_unlock(&bedata->socket_lock);
 439                free_active_ring(map);
 440                pvcalls_exit_sock(sock);
 441                return ret;
 442        }
 443        ret = create_active(map, &evtchn);
 444        if (ret < 0) {
 445                spin_unlock(&bedata->socket_lock);
 446                free_active_ring(map);
 447                pvcalls_exit_sock(sock);
 448                return ret;
 449        }
 450
 451        req = RING_GET_REQUEST(&bedata->ring, req_id);
 452        req->req_id = req_id;
 453        req->cmd = PVCALLS_CONNECT;
 454        req->u.connect.id = (uintptr_t)map;
 455        req->u.connect.len = addr_len;
 456        req->u.connect.flags = flags;
 457        req->u.connect.ref = map->active.ref;
 458        req->u.connect.evtchn = evtchn;
 459        memcpy(req->u.connect.addr, addr, sizeof(*addr));
 460
 461        map->sock = sock;
 462
 463        bedata->ring.req_prod_pvt++;
 464        RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 465        spin_unlock(&bedata->socket_lock);
 466
 467        if (notify)
 468                notify_remote_via_irq(bedata->irq);
 469
 470        wait_event(bedata->inflight_req,
 471                   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
 472
 473        /* read req_id, then the content */
 474        smp_rmb();
 475        ret = bedata->rsp[req_id].ret;
 476        bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 477        pvcalls_exit_sock(sock);
 478        return ret;
 479}
 480
 481static int __write_ring(struct pvcalls_data_intf *intf,
 482                        struct pvcalls_data *data,
 483                        struct iov_iter *msg_iter,
 484                        int len)
 485{
 486        RING_IDX cons, prod, size, masked_prod, masked_cons;
 487        RING_IDX array_size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 488        int32_t error;
 489
 490        error = intf->out_error;
 491        if (error < 0)
 492                return error;
 493        cons = intf->out_cons;
 494        prod = intf->out_prod;
 495        /* read indexes before continuing */
 496        virt_mb();
 497
 498        size = pvcalls_queued(prod, cons, array_size);
 499        if (size > array_size)
 500                return -EINVAL;
 501        if (size == array_size)
 502                return 0;
 503        if (len > array_size - size)
 504                len = array_size - size;
 505
 506        masked_prod = pvcalls_mask(prod, array_size);
 507        masked_cons = pvcalls_mask(cons, array_size);
 508
 509        if (masked_prod < masked_cons) {
 510                len = copy_from_iter(data->out + masked_prod, len, msg_iter);
 511        } else {
 512                if (len > array_size - masked_prod) {
 513                        int ret = copy_from_iter(data->out + masked_prod,
 514                                       array_size - masked_prod, msg_iter);
 515                        if (ret != array_size - masked_prod) {
 516                                len = ret;
 517                                goto out;
 518                        }
 519                        len = ret + copy_from_iter(data->out, len - ret, msg_iter);
 520                } else {
 521                        len = copy_from_iter(data->out + masked_prod, len, msg_iter);
 522                }
 523        }
 524out:
 525        /* write to ring before updating pointer */
 526        virt_wmb();
 527        intf->out_prod += len;
 528
 529        return len;
 530}
 531
 532int pvcalls_front_sendmsg(struct socket *sock, struct msghdr *msg,
 533                          size_t len)
 534{
 535        struct sock_mapping *map;
 536        int sent, tot_sent = 0;
 537        int count = 0, flags;
 538
 539        flags = msg->msg_flags;
 540        if (flags & (MSG_CONFIRM|MSG_DONTROUTE|MSG_EOR|MSG_OOB))
 541                return -EOPNOTSUPP;
 542
 543        map = pvcalls_enter_sock(sock);
 544        if (IS_ERR(map))
 545                return PTR_ERR(map);
 546
 547        mutex_lock(&map->active.out_mutex);
 548        if ((flags & MSG_DONTWAIT) && !pvcalls_front_write_todo(map)) {
 549                mutex_unlock(&map->active.out_mutex);
 550                pvcalls_exit_sock(sock);
 551                return -EAGAIN;
 552        }
 553        if (len > INT_MAX)
 554                len = INT_MAX;
 555
 556again:
 557        count++;
 558        sent = __write_ring(map->active.ring,
 559                            &map->active.data, &msg->msg_iter,
 560                            len);
 561        if (sent > 0) {
 562                len -= sent;
 563                tot_sent += sent;
 564                notify_remote_via_irq(map->active.irq);
 565        }
 566        if (sent >= 0 && len > 0 && count < PVCALLS_FRONT_MAX_SPIN)
 567                goto again;
 568        if (sent < 0)
 569                tot_sent = sent;
 570
 571        mutex_unlock(&map->active.out_mutex);
 572        pvcalls_exit_sock(sock);
 573        return tot_sent;
 574}
 575
 576static int __read_ring(struct pvcalls_data_intf *intf,
 577                       struct pvcalls_data *data,
 578                       struct iov_iter *msg_iter,
 579                       size_t len, int flags)
 580{
 581        RING_IDX cons, prod, size, masked_prod, masked_cons;
 582        RING_IDX array_size = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 583        int32_t error;
 584
 585        cons = intf->in_cons;
 586        prod = intf->in_prod;
 587        error = intf->in_error;
 588        /* get pointers before reading from the ring */
 589        virt_rmb();
 590
 591        size = pvcalls_queued(prod, cons, array_size);
 592        masked_prod = pvcalls_mask(prod, array_size);
 593        masked_cons = pvcalls_mask(cons, array_size);
 594
 595        if (size == 0)
 596                return error ?: size;
 597
 598        if (len > size)
 599                len = size;
 600
 601        if (masked_prod > masked_cons) {
 602                len = copy_to_iter(data->in + masked_cons, len, msg_iter);
 603        } else {
 604                if (len > (array_size - masked_cons)) {
 605                        int ret = copy_to_iter(data->in + masked_cons,
 606                                     array_size - masked_cons, msg_iter);
 607                        if (ret != array_size - masked_cons) {
 608                                len = ret;
 609                                goto out;
 610                        }
 611                        len = ret + copy_to_iter(data->in, len - ret, msg_iter);
 612                } else {
 613                        len = copy_to_iter(data->in + masked_cons, len, msg_iter);
 614                }
 615        }
 616out:
 617        /* read data from the ring before increasing the index */
 618        virt_mb();
 619        if (!(flags & MSG_PEEK))
 620                intf->in_cons += len;
 621
 622        return len;
 623}
 624
 625int pvcalls_front_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
 626                     int flags)
 627{
 628        int ret;
 629        struct sock_mapping *map;
 630
 631        if (flags & (MSG_CMSG_CLOEXEC|MSG_ERRQUEUE|MSG_OOB|MSG_TRUNC))
 632                return -EOPNOTSUPP;
 633
 634        map = pvcalls_enter_sock(sock);
 635        if (IS_ERR(map))
 636                return PTR_ERR(map);
 637
 638        mutex_lock(&map->active.in_mutex);
 639        if (len > XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER))
 640                len = XEN_FLEX_RING_SIZE(PVCALLS_RING_ORDER);
 641
 642        while (!(flags & MSG_DONTWAIT) && !pvcalls_front_read_todo(map)) {
 643                wait_event_interruptible(map->active.inflight_conn_req,
 644                                         pvcalls_front_read_todo(map));
 645        }
 646        ret = __read_ring(map->active.ring, &map->active.data,
 647                          &msg->msg_iter, len, flags);
 648
 649        if (ret > 0)
 650                notify_remote_via_irq(map->active.irq);
 651        if (ret == 0)
 652                ret = (flags & MSG_DONTWAIT) ? -EAGAIN : 0;
 653        if (ret == -ENOTCONN)
 654                ret = 0;
 655
 656        mutex_unlock(&map->active.in_mutex);
 657        pvcalls_exit_sock(sock);
 658        return ret;
 659}
 660
 661int pvcalls_front_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
 662{
 663        struct pvcalls_bedata *bedata;
 664        struct sock_mapping *map = NULL;
 665        struct xen_pvcalls_request *req;
 666        int notify, req_id, ret;
 667
 668        if (addr->sa_family != AF_INET || sock->type != SOCK_STREAM)
 669                return -EOPNOTSUPP;
 670
 671        map = pvcalls_enter_sock(sock);
 672        if (IS_ERR(map))
 673                return PTR_ERR(map);
 674        bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 675
 676        spin_lock(&bedata->socket_lock);
 677        ret = get_request(bedata, &req_id);
 678        if (ret < 0) {
 679                spin_unlock(&bedata->socket_lock);
 680                pvcalls_exit_sock(sock);
 681                return ret;
 682        }
 683        req = RING_GET_REQUEST(&bedata->ring, req_id);
 684        req->req_id = req_id;
 685        map->sock = sock;
 686        req->cmd = PVCALLS_BIND;
 687        req->u.bind.id = (uintptr_t)map;
 688        memcpy(req->u.bind.addr, addr, sizeof(*addr));
 689        req->u.bind.len = addr_len;
 690
 691        init_waitqueue_head(&map->passive.inflight_accept_req);
 692
 693        map->active_socket = false;
 694
 695        bedata->ring.req_prod_pvt++;
 696        RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 697        spin_unlock(&bedata->socket_lock);
 698        if (notify)
 699                notify_remote_via_irq(bedata->irq);
 700
 701        wait_event(bedata->inflight_req,
 702                   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
 703
 704        /* read req_id, then the content */
 705        smp_rmb();
 706        ret = bedata->rsp[req_id].ret;
 707        bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 708
 709        map->passive.status = PVCALLS_STATUS_BIND;
 710        pvcalls_exit_sock(sock);
 711        return 0;
 712}
 713
 714int pvcalls_front_listen(struct socket *sock, int backlog)
 715{
 716        struct pvcalls_bedata *bedata;
 717        struct sock_mapping *map;
 718        struct xen_pvcalls_request *req;
 719        int notify, req_id, ret;
 720
 721        map = pvcalls_enter_sock(sock);
 722        if (IS_ERR(map))
 723                return PTR_ERR(map);
 724        bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 725
 726        if (map->passive.status != PVCALLS_STATUS_BIND) {
 727                pvcalls_exit_sock(sock);
 728                return -EOPNOTSUPP;
 729        }
 730
 731        spin_lock(&bedata->socket_lock);
 732        ret = get_request(bedata, &req_id);
 733        if (ret < 0) {
 734                spin_unlock(&bedata->socket_lock);
 735                pvcalls_exit_sock(sock);
 736                return ret;
 737        }
 738        req = RING_GET_REQUEST(&bedata->ring, req_id);
 739        req->req_id = req_id;
 740        req->cmd = PVCALLS_LISTEN;
 741        req->u.listen.id = (uintptr_t) map;
 742        req->u.listen.backlog = backlog;
 743
 744        bedata->ring.req_prod_pvt++;
 745        RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 746        spin_unlock(&bedata->socket_lock);
 747        if (notify)
 748                notify_remote_via_irq(bedata->irq);
 749
 750        wait_event(bedata->inflight_req,
 751                   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
 752
 753        /* read req_id, then the content */
 754        smp_rmb();
 755        ret = bedata->rsp[req_id].ret;
 756        bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 757
 758        map->passive.status = PVCALLS_STATUS_LISTEN;
 759        pvcalls_exit_sock(sock);
 760        return ret;
 761}
 762
 763int pvcalls_front_accept(struct socket *sock, struct socket *newsock, int flags)
 764{
 765        struct pvcalls_bedata *bedata;
 766        struct sock_mapping *map;
 767        struct sock_mapping *map2 = NULL;
 768        struct xen_pvcalls_request *req;
 769        int notify, req_id, ret, nonblock;
 770        evtchn_port_t evtchn;
 771
 772        map = pvcalls_enter_sock(sock);
 773        if (IS_ERR(map))
 774                return PTR_ERR(map);
 775        bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 776
 777        if (map->passive.status != PVCALLS_STATUS_LISTEN) {
 778                pvcalls_exit_sock(sock);
 779                return -EINVAL;
 780        }
 781
 782        nonblock = flags & SOCK_NONBLOCK;
 783        /*
 784         * Backend only supports 1 inflight accept request, will return
 785         * errors for the others
 786         */
 787        if (test_and_set_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 788                             (void *)&map->passive.flags)) {
 789                req_id = READ_ONCE(map->passive.inflight_req_id);
 790                if (req_id != PVCALLS_INVALID_ID &&
 791                    READ_ONCE(bedata->rsp[req_id].req_id) == req_id) {
 792                        map2 = map->passive.accept_map;
 793                        goto received;
 794                }
 795                if (nonblock) {
 796                        pvcalls_exit_sock(sock);
 797                        return -EAGAIN;
 798                }
 799                if (wait_event_interruptible(map->passive.inflight_accept_req,
 800                        !test_and_set_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 801                                          (void *)&map->passive.flags))) {
 802                        pvcalls_exit_sock(sock);
 803                        return -EINTR;
 804                }
 805        }
 806
 807        map2 = kzalloc(sizeof(*map2), GFP_KERNEL);
 808        if (map2 == NULL) {
 809                clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 810                          (void *)&map->passive.flags);
 811                pvcalls_exit_sock(sock);
 812                return -ENOMEM;
 813        }
 814        ret = alloc_active_ring(map2);
 815        if (ret < 0) {
 816                clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 817                                (void *)&map->passive.flags);
 818                kfree(map2);
 819                pvcalls_exit_sock(sock);
 820                return ret;
 821        }
 822        spin_lock(&bedata->socket_lock);
 823        ret = get_request(bedata, &req_id);
 824        if (ret < 0) {
 825                clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 826                          (void *)&map->passive.flags);
 827                spin_unlock(&bedata->socket_lock);
 828                free_active_ring(map2);
 829                kfree(map2);
 830                pvcalls_exit_sock(sock);
 831                return ret;
 832        }
 833
 834        ret = create_active(map2, &evtchn);
 835        if (ret < 0) {
 836                free_active_ring(map2);
 837                kfree(map2);
 838                clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 839                          (void *)&map->passive.flags);
 840                spin_unlock(&bedata->socket_lock);
 841                pvcalls_exit_sock(sock);
 842                return ret;
 843        }
 844        list_add_tail(&map2->list, &bedata->socket_mappings);
 845
 846        req = RING_GET_REQUEST(&bedata->ring, req_id);
 847        req->req_id = req_id;
 848        req->cmd = PVCALLS_ACCEPT;
 849        req->u.accept.id = (uintptr_t) map;
 850        req->u.accept.ref = map2->active.ref;
 851        req->u.accept.id_new = (uintptr_t) map2;
 852        req->u.accept.evtchn = evtchn;
 853        map->passive.accept_map = map2;
 854
 855        bedata->ring.req_prod_pvt++;
 856        RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 857        spin_unlock(&bedata->socket_lock);
 858        if (notify)
 859                notify_remote_via_irq(bedata->irq);
 860        /* We could check if we have received a response before returning. */
 861        if (nonblock) {
 862                WRITE_ONCE(map->passive.inflight_req_id, req_id);
 863                pvcalls_exit_sock(sock);
 864                return -EAGAIN;
 865        }
 866
 867        if (wait_event_interruptible(bedata->inflight_req,
 868                READ_ONCE(bedata->rsp[req_id].req_id) == req_id)) {
 869                pvcalls_exit_sock(sock);
 870                return -EINTR;
 871        }
 872        /* read req_id, then the content */
 873        smp_rmb();
 874
 875received:
 876        map2->sock = newsock;
 877        newsock->sk = sk_alloc(sock_net(sock->sk), PF_INET, GFP_KERNEL, &pvcalls_proto, false);
 878        if (!newsock->sk) {
 879                bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 880                map->passive.inflight_req_id = PVCALLS_INVALID_ID;
 881                clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 882                          (void *)&map->passive.flags);
 883                pvcalls_front_free_map(bedata, map2);
 884                pvcalls_exit_sock(sock);
 885                return -ENOMEM;
 886        }
 887        newsock->sk->sk_send_head = (void *)map2;
 888
 889        ret = bedata->rsp[req_id].ret;
 890        bedata->rsp[req_id].req_id = PVCALLS_INVALID_ID;
 891        map->passive.inflight_req_id = PVCALLS_INVALID_ID;
 892
 893        clear_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT, (void *)&map->passive.flags);
 894        wake_up(&map->passive.inflight_accept_req);
 895
 896        pvcalls_exit_sock(sock);
 897        return ret;
 898}
 899
 900static __poll_t pvcalls_front_poll_passive(struct file *file,
 901                                               struct pvcalls_bedata *bedata,
 902                                               struct sock_mapping *map,
 903                                               poll_table *wait)
 904{
 905        int notify, req_id, ret;
 906        struct xen_pvcalls_request *req;
 907
 908        if (test_bit(PVCALLS_FLAG_ACCEPT_INFLIGHT,
 909                     (void *)&map->passive.flags)) {
 910                uint32_t req_id = READ_ONCE(map->passive.inflight_req_id);
 911
 912                if (req_id != PVCALLS_INVALID_ID &&
 913                    READ_ONCE(bedata->rsp[req_id].req_id) == req_id)
 914                        return EPOLLIN | EPOLLRDNORM;
 915
 916                poll_wait(file, &map->passive.inflight_accept_req, wait);
 917                return 0;
 918        }
 919
 920        if (test_and_clear_bit(PVCALLS_FLAG_POLL_RET,
 921                               (void *)&map->passive.flags))
 922                return EPOLLIN | EPOLLRDNORM;
 923
 924        /*
 925         * First check RET, then INFLIGHT. No barriers necessary to
 926         * ensure execution ordering because of the conditional
 927         * instructions creating control dependencies.
 928         */
 929
 930        if (test_and_set_bit(PVCALLS_FLAG_POLL_INFLIGHT,
 931                             (void *)&map->passive.flags)) {
 932                poll_wait(file, &bedata->inflight_req, wait);
 933                return 0;
 934        }
 935
 936        spin_lock(&bedata->socket_lock);
 937        ret = get_request(bedata, &req_id);
 938        if (ret < 0) {
 939                spin_unlock(&bedata->socket_lock);
 940                return ret;
 941        }
 942        req = RING_GET_REQUEST(&bedata->ring, req_id);
 943        req->req_id = req_id;
 944        req->cmd = PVCALLS_POLL;
 945        req->u.poll.id = (uintptr_t) map;
 946
 947        bedata->ring.req_prod_pvt++;
 948        RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
 949        spin_unlock(&bedata->socket_lock);
 950        if (notify)
 951                notify_remote_via_irq(bedata->irq);
 952
 953        poll_wait(file, &bedata->inflight_req, wait);
 954        return 0;
 955}
 956
 957static __poll_t pvcalls_front_poll_active(struct file *file,
 958                                              struct pvcalls_bedata *bedata,
 959                                              struct sock_mapping *map,
 960                                              poll_table *wait)
 961{
 962        __poll_t mask = 0;
 963        int32_t in_error, out_error;
 964        struct pvcalls_data_intf *intf = map->active.ring;
 965
 966        out_error = intf->out_error;
 967        in_error = intf->in_error;
 968
 969        poll_wait(file, &map->active.inflight_conn_req, wait);
 970        if (pvcalls_front_write_todo(map))
 971                mask |= EPOLLOUT | EPOLLWRNORM;
 972        if (pvcalls_front_read_todo(map))
 973                mask |= EPOLLIN | EPOLLRDNORM;
 974        if (in_error != 0 || out_error != 0)
 975                mask |= EPOLLERR;
 976
 977        return mask;
 978}
 979
 980__poll_t pvcalls_front_poll(struct file *file, struct socket *sock,
 981                               poll_table *wait)
 982{
 983        struct pvcalls_bedata *bedata;
 984        struct sock_mapping *map;
 985        __poll_t ret;
 986
 987        map = pvcalls_enter_sock(sock);
 988        if (IS_ERR(map))
 989                return EPOLLNVAL;
 990        bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
 991
 992        if (map->active_socket)
 993                ret = pvcalls_front_poll_active(file, bedata, map, wait);
 994        else
 995                ret = pvcalls_front_poll_passive(file, bedata, map, wait);
 996        pvcalls_exit_sock(sock);
 997        return ret;
 998}
 999
1000int pvcalls_front_release(struct socket *sock)
1001{
1002        struct pvcalls_bedata *bedata;
1003        struct sock_mapping *map;
1004        int req_id, notify, ret;
1005        struct xen_pvcalls_request *req;
1006
1007        if (sock->sk == NULL)
1008                return 0;
1009
1010        map = pvcalls_enter_sock(sock);
1011        if (IS_ERR(map)) {
1012                if (PTR_ERR(map) == -ENOTCONN)
1013                        return -EIO;
1014                else
1015                        return 0;
1016        }
1017        bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
1018
1019        spin_lock(&bedata->socket_lock);
1020        ret = get_request(bedata, &req_id);
1021        if (ret < 0) {
1022                spin_unlock(&bedata->socket_lock);
1023                pvcalls_exit_sock(sock);
1024                return ret;
1025        }
1026        sock->sk->sk_send_head = NULL;
1027
1028        req = RING_GET_REQUEST(&bedata->ring, req_id);
1029        req->req_id = req_id;
1030        req->cmd = PVCALLS_RELEASE;
1031        req->u.release.id = (uintptr_t)map;
1032
1033        bedata->ring.req_prod_pvt++;
1034        RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&bedata->ring, notify);
1035        spin_unlock(&bedata->socket_lock);
1036        if (notify)
1037                notify_remote_via_irq(bedata->irq);
1038
1039        wait_event(bedata->inflight_req,
1040                   READ_ONCE(bedata->rsp[req_id].req_id) == req_id);
1041
1042        if (map->active_socket) {
1043                /*
1044                 * Set in_error and wake up inflight_conn_req to force
1045                 * recvmsg waiters to exit.
1046                 */
1047                map->active.ring->in_error = -EBADF;
1048                wake_up_interruptible(&map->active.inflight_conn_req);
1049
1050                /*
1051                 * We need to make sure that sendmsg/recvmsg on this socket have
1052                 * not started before we've cleared sk_send_head here. The
1053                 * easiest way to guarantee this is to see that no pvcalls
1054                 * (other than us) is in progress on this socket.
1055                 */
1056                while (atomic_read(&map->refcount) > 1)
1057                        cpu_relax();
1058
1059                pvcalls_front_free_map(bedata, map);
1060        } else {
1061                wake_up(&bedata->inflight_req);
1062                wake_up(&map->passive.inflight_accept_req);
1063
1064                while (atomic_read(&map->refcount) > 1)
1065                        cpu_relax();
1066
1067                spin_lock(&bedata->socket_lock);
1068                list_del(&map->list);
1069                spin_unlock(&bedata->socket_lock);
1070                if (READ_ONCE(map->passive.inflight_req_id) != PVCALLS_INVALID_ID &&
1071                        READ_ONCE(map->passive.inflight_req_id) != 0) {
1072                        pvcalls_front_free_map(bedata,
1073                                               map->passive.accept_map);
1074                }
1075                kfree(map);
1076        }
1077        WRITE_ONCE(bedata->rsp[req_id].req_id, PVCALLS_INVALID_ID);
1078
1079        pvcalls_exit();
1080        return 0;
1081}
1082
1083static const struct xenbus_device_id pvcalls_front_ids[] = {
1084        { "pvcalls" },
1085        { "" }
1086};
1087
1088static int pvcalls_front_remove(struct xenbus_device *dev)
1089{
1090        struct pvcalls_bedata *bedata;
1091        struct sock_mapping *map = NULL, *n;
1092
1093        bedata = dev_get_drvdata(&pvcalls_front_dev->dev);
1094        dev_set_drvdata(&dev->dev, NULL);
1095        pvcalls_front_dev = NULL;
1096        if (bedata->irq >= 0)
1097                unbind_from_irqhandler(bedata->irq, dev);
1098
1099        list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {
1100                map->sock->sk->sk_send_head = NULL;
1101                if (map->active_socket) {
1102                        map->active.ring->in_error = -EBADF;
1103                        wake_up_interruptible(&map->active.inflight_conn_req);
1104                }
1105        }
1106
1107        smp_mb();
1108        while (atomic_read(&pvcalls_refcount) > 0)
1109                cpu_relax();
1110        list_for_each_entry_safe(map, n, &bedata->socket_mappings, list) {
1111                if (map->active_socket) {
1112                        /* No need to lock, refcount is 0 */
1113                        pvcalls_front_free_map(bedata, map);
1114                } else {
1115                        list_del(&map->list);
1116                        kfree(map);
1117                }
1118        }
1119        if (bedata->ref != -1)
1120                gnttab_end_foreign_access(bedata->ref, 0, 0);
1121        kfree(bedata->ring.sring);
1122        kfree(bedata);
1123        xenbus_switch_state(dev, XenbusStateClosed);
1124        return 0;
1125}
1126
1127static int pvcalls_front_probe(struct xenbus_device *dev,
1128                          const struct xenbus_device_id *id)
1129{
1130        int ret = -ENOMEM, i;
1131        evtchn_port_t evtchn;
1132        unsigned int max_page_order, function_calls, len;
1133        char *versions;
1134        grant_ref_t gref_head = 0;
1135        struct xenbus_transaction xbt;
1136        struct pvcalls_bedata *bedata = NULL;
1137        struct xen_pvcalls_sring *sring;
1138
1139        if (pvcalls_front_dev != NULL) {
1140                dev_err(&dev->dev, "only one PV Calls connection supported\n");
1141                return -EINVAL;
1142        }
1143
1144        versions = xenbus_read(XBT_NIL, dev->otherend, "versions", &len);
1145        if (IS_ERR(versions))
1146                return PTR_ERR(versions);
1147        if (!len)
1148                return -EINVAL;
1149        if (strcmp(versions, "1")) {
1150                kfree(versions);
1151                return -EINVAL;
1152        }
1153        kfree(versions);
1154        max_page_order = xenbus_read_unsigned(dev->otherend,
1155                                              "max-page-order", 0);
1156        if (max_page_order < PVCALLS_RING_ORDER)
1157                return -ENODEV;
1158        function_calls = xenbus_read_unsigned(dev->otherend,
1159                                              "function-calls", 0);
1160        /* See XENBUS_FUNCTIONS_CALLS in pvcalls.h */
1161        if (function_calls != 1)
1162                return -ENODEV;
1163        pr_info("%s max-page-order is %u\n", __func__, max_page_order);
1164
1165        bedata = kzalloc(sizeof(struct pvcalls_bedata), GFP_KERNEL);
1166        if (!bedata)
1167                return -ENOMEM;
1168
1169        dev_set_drvdata(&dev->dev, bedata);
1170        pvcalls_front_dev = dev;
1171        init_waitqueue_head(&bedata->inflight_req);
1172        INIT_LIST_HEAD(&bedata->socket_mappings);
1173        spin_lock_init(&bedata->socket_lock);
1174        bedata->irq = -1;
1175        bedata->ref = -1;
1176
1177        for (i = 0; i < PVCALLS_NR_RSP_PER_RING; i++)
1178                bedata->rsp[i].req_id = PVCALLS_INVALID_ID;
1179
1180        sring = (struct xen_pvcalls_sring *) __get_free_page(GFP_KERNEL |
1181                                                             __GFP_ZERO);
1182        if (!sring)
1183                goto error;
1184        SHARED_RING_INIT(sring);
1185        FRONT_RING_INIT(&bedata->ring, sring, XEN_PAGE_SIZE);
1186
1187        ret = xenbus_alloc_evtchn(dev, &evtchn);
1188        if (ret)
1189                goto error;
1190
1191        bedata->irq = bind_evtchn_to_irqhandler(evtchn,
1192                                                pvcalls_front_event_handler,
1193                                                0, "pvcalls-frontend", dev);
1194        if (bedata->irq < 0) {
1195                ret = bedata->irq;
1196                goto error;
1197        }
1198
1199        ret = gnttab_alloc_grant_references(1, &gref_head);
1200        if (ret < 0)
1201                goto error;
1202        ret = gnttab_claim_grant_reference(&gref_head);
1203        if (ret < 0)
1204                goto error;
1205        bedata->ref = ret;
1206        gnttab_grant_foreign_access_ref(bedata->ref, dev->otherend_id,
1207                                        virt_to_gfn((void *)sring), 0);
1208
1209 again:
1210        ret = xenbus_transaction_start(&xbt);
1211        if (ret) {
1212                xenbus_dev_fatal(dev, ret, "starting transaction");
1213                goto error;
1214        }
1215        ret = xenbus_printf(xbt, dev->nodename, "version", "%u", 1);
1216        if (ret)
1217                goto error_xenbus;
1218        ret = xenbus_printf(xbt, dev->nodename, "ring-ref", "%d", bedata->ref);
1219        if (ret)
1220                goto error_xenbus;
1221        ret = xenbus_printf(xbt, dev->nodename, "port", "%u",
1222                            evtchn);
1223        if (ret)
1224                goto error_xenbus;
1225        ret = xenbus_transaction_end(xbt, 0);
1226        if (ret) {
1227                if (ret == -EAGAIN)
1228                        goto again;
1229                xenbus_dev_fatal(dev, ret, "completing transaction");
1230                goto error;
1231        }
1232        xenbus_switch_state(dev, XenbusStateInitialised);
1233
1234        return 0;
1235
1236 error_xenbus:
1237        xenbus_transaction_end(xbt, 1);
1238        xenbus_dev_fatal(dev, ret, "writing xenstore");
1239 error:
1240        pvcalls_front_remove(dev);
1241        return ret;
1242}
1243
1244static void pvcalls_front_changed(struct xenbus_device *dev,
1245                            enum xenbus_state backend_state)
1246{
1247        switch (backend_state) {
1248        case XenbusStateReconfiguring:
1249        case XenbusStateReconfigured:
1250        case XenbusStateInitialising:
1251        case XenbusStateInitialised:
1252        case XenbusStateUnknown:
1253                break;
1254
1255        case XenbusStateInitWait:
1256                break;
1257
1258        case XenbusStateConnected:
1259                xenbus_switch_state(dev, XenbusStateConnected);
1260                break;
1261
1262        case XenbusStateClosed:
1263                if (dev->state == XenbusStateClosed)
1264                        break;
1265                /* Missed the backend's CLOSING state */
1266                fallthrough;
1267        case XenbusStateClosing:
1268                xenbus_frontend_closed(dev);
1269                break;
1270        }
1271}
1272
1273static struct xenbus_driver pvcalls_front_driver = {
1274        .ids = pvcalls_front_ids,
1275        .probe = pvcalls_front_probe,
1276        .remove = pvcalls_front_remove,
1277        .otherend_changed = pvcalls_front_changed,
1278};
1279
1280static int __init pvcalls_frontend_init(void)
1281{
1282        if (!xen_domain())
1283                return -ENODEV;
1284
1285        pr_info("Initialising Xen pvcalls frontend driver\n");
1286
1287        return xenbus_register_frontend(&pvcalls_front_driver);
1288}
1289
1290module_init(pvcalls_frontend_init);
1291
1292MODULE_DESCRIPTION("Xen PV Calls frontend driver");
1293MODULE_AUTHOR("Stefano Stabellini <sstabellini@kernel.org>");
1294MODULE_LICENSE("GPL");
1295
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.