linux/drivers/block/aoe/aoecmd.c
<<
>>
Prefs
   1/* Copyright (c) 2013 Coraid, Inc.  See COPYING for GPL terms. */
   2/*
   3 * aoecmd.c
   4 * Filesystem request handling methods
   5 */
   6
   7#include <linux/ata.h>
   8#include <linux/slab.h>
   9#include <linux/hdreg.h>
  10#include <linux/blk-mq.h>
  11#include <linux/skbuff.h>
  12#include <linux/netdevice.h>
  13#include <linux/genhd.h>
  14#include <linux/moduleparam.h>
  15#include <linux/workqueue.h>
  16#include <linux/kthread.h>
  17#include <net/net_namespace.h>
  18#include <asm/unaligned.h>
  19#include <linux/uio.h>
  20#include "aoe.h"
  21
  22#define MAXIOC (8192)   /* default meant to avoid most soft lockups */
  23
  24static void ktcomplete(struct frame *, struct sk_buff *);
  25static int count_targets(struct aoedev *d, int *untainted);
  26
  27static struct buf *nextbuf(struct aoedev *);
  28
  29static int aoe_deadsecs = 60 * 3;
  30module_param(aoe_deadsecs, int, 0644);
  31MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev.");
  32
  33static int aoe_maxout = 64;
  34module_param(aoe_maxout, int, 0644);
  35MODULE_PARM_DESC(aoe_maxout,
  36        "Only aoe_maxout outstanding packets for every MAC on eX.Y.");
  37
  38/* The number of online cpus during module initialization gives us a
  39 * convenient heuristic cap on the parallelism used for ktio threads
  40 * doing I/O completion.  It is not important that the cap equal the
  41 * actual number of running CPUs at any given time, but because of CPU
  42 * hotplug, we take care to use ncpus instead of using
  43 * num_online_cpus() after module initialization.
  44 */
  45static int ncpus;
  46
  47/* mutex lock used for synchronization while thread spawning */
  48static DEFINE_MUTEX(ktio_spawn_lock);
  49
  50static wait_queue_head_t *ktiowq;
  51static struct ktstate *kts;
  52
  53/* io completion queue */
  54struct iocq_ktio {
  55        struct list_head head;
  56        spinlock_t lock;
  57};
  58static struct iocq_ktio *iocq;
  59
  60static struct page *empty_page;
  61
  62static struct sk_buff *
  63new_skb(ulong len)
  64{
  65        struct sk_buff *skb;
  66
  67        skb = alloc_skb(len + MAX_HEADER, GFP_ATOMIC);
  68        if (skb) {
  69                skb_reserve(skb, MAX_HEADER);
  70                skb_reset_mac_header(skb);
  71                skb_reset_network_header(skb);
  72                skb->protocol = __constant_htons(ETH_P_AOE);
  73                skb_checksum_none_assert(skb);
  74        }
  75        return skb;
  76}
  77
  78static struct frame *
  79getframe_deferred(struct aoedev *d, u32 tag)
  80{
  81        struct list_head *head, *pos, *nx;
  82        struct frame *f;
  83
  84        head = &d->rexmitq;
  85        list_for_each_safe(pos, nx, head) {
  86                f = list_entry(pos, struct frame, head);
  87                if (f->tag == tag) {
  88                        list_del(pos);
  89                        return f;
  90                }
  91        }
  92        return NULL;
  93}
  94
  95static struct frame *
  96getframe(struct aoedev *d, u32 tag)
  97{
  98        struct frame *f;
  99        struct list_head *head, *pos, *nx;
 100        u32 n;
 101
 102        n = tag % NFACTIVE;
 103        head = &d->factive[n];
 104        list_for_each_safe(pos, nx, head) {
 105                f = list_entry(pos, struct frame, head);
 106                if (f->tag == tag) {
 107                        list_del(pos);
 108                        return f;
 109                }
 110        }
 111        return NULL;
 112}
 113
 114/*
 115 * Leave the top bit clear so we have tagspace for userland.
 116 * The bottom 16 bits are the xmit tick for rexmit/rttavg processing.
 117 * This driver reserves tag -1 to mean "unused frame."
 118 */
 119static int
 120newtag(struct aoedev *d)
 121{
 122        register ulong n;
 123
 124        n = jiffies & 0xffff;
 125        return n |= (++d->lasttag & 0x7fff) << 16;
 126}
 127
 128static u32
 129aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h)
 130{
 131        u32 host_tag = newtag(d);
 132
 133        memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
 134        memcpy(h->dst, t->addr, sizeof h->dst);
 135        h->type = __constant_cpu_to_be16(ETH_P_AOE);
 136        h->verfl = AOE_HVER;
 137        h->major = cpu_to_be16(d->aoemajor);
 138        h->minor = d->aoeminor;
 139        h->cmd = AOECMD_ATA;
 140        h->tag = cpu_to_be32(host_tag);
 141
 142        return host_tag;
 143}
 144
 145static inline void
 146put_lba(struct aoe_atahdr *ah, sector_t lba)
 147{
 148        ah->lba0 = lba;
 149        ah->lba1 = lba >>= 8;
 150        ah->lba2 = lba >>= 8;
 151        ah->lba3 = lba >>= 8;
 152        ah->lba4 = lba >>= 8;
 153        ah->lba5 = lba >>= 8;
 154}
 155
 156static struct aoeif *
 157ifrotate(struct aoetgt *t)
 158{
 159        struct aoeif *ifp;
 160
 161        ifp = t->ifp;
 162        ifp++;
 163        if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL)
 164                ifp = t->ifs;
 165        if (ifp->nd == NULL)
 166                return NULL;
 167        return t->ifp = ifp;
 168}
 169
 170static void
 171skb_pool_put(struct aoedev *d, struct sk_buff *skb)
 172{
 173        __skb_queue_tail(&d->skbpool, skb);
 174}
 175
 176static struct sk_buff *
 177skb_pool_get(struct aoedev *d)
 178{
 179        struct sk_buff *skb = skb_peek(&d->skbpool);
 180
 181        if (skb && atomic_read(&skb_shinfo(skb)->dataref) == 1) {
 182                __skb_unlink(skb, &d->skbpool);
 183                return skb;
 184        }
 185        if (skb_queue_len(&d->skbpool) < NSKBPOOLMAX &&
 186            (skb = new_skb(ETH_ZLEN)))
 187                return skb;
 188
 189        return NULL;
 190}
 191
 192void
 193aoe_freetframe(struct frame *f)
 194{
 195        struct aoetgt *t;
 196
 197        t = f->t;
 198        f->buf = NULL;
 199        memset(&f->iter, 0, sizeof(f->iter));
 200        f->r_skb = NULL;
 201        f->flags = 0;
 202        list_add(&f->head, &t->ffree);
 203}
 204
 205static struct frame *
 206newtframe(struct aoedev *d, struct aoetgt *t)
 207{
 208        struct frame *f;
 209        struct sk_buff *skb;
 210        struct list_head *pos;
 211
 212        if (list_empty(&t->ffree)) {
 213                if (t->falloc >= NSKBPOOLMAX*2)
 214                        return NULL;
 215                f = kcalloc(1, sizeof(*f), GFP_ATOMIC);
 216                if (f == NULL)
 217                        return NULL;
 218                t->falloc++;
 219                f->t = t;
 220        } else {
 221                pos = t->ffree.next;
 222                list_del(pos);
 223                f = list_entry(pos, struct frame, head);
 224        }
 225
 226        skb = f->skb;
 227        if (skb == NULL) {
 228                f->skb = skb = new_skb(ETH_ZLEN);
 229                if (!skb) {
 230bail:                   aoe_freetframe(f);
 231                        return NULL;
 232                }
 233        }
 234
 235        if (atomic_read(&skb_shinfo(skb)->dataref) != 1) {
 236                skb = skb_pool_get(d);
 237                if (skb == NULL)
 238                        goto bail;
 239                skb_pool_put(d, f->skb);
 240                f->skb = skb;
 241        }
 242
 243        skb->truesize -= skb->data_len;
 244        skb_shinfo(skb)->nr_frags = skb->data_len = 0;
 245        skb_trim(skb, 0);
 246        return f;
 247}
 248
 249static struct frame *
 250newframe(struct aoedev *d)
 251{
 252        struct frame *f;
 253        struct aoetgt *t, **tt;
 254        int totout = 0;
 255        int use_tainted;
 256        int has_untainted;
 257
 258        if (!d->targets || !d->targets[0]) {
 259                printk(KERN_ERR "aoe: NULL TARGETS!\n");
 260                return NULL;
 261        }
 262        tt = d->tgt;    /* last used target */
 263        for (use_tainted = 0, has_untainted = 0;;) {
 264                tt++;
 265                if (tt >= &d->targets[d->ntargets] || !*tt)
 266                        tt = d->targets;
 267                t = *tt;
 268                if (!t->taint) {
 269                        has_untainted = 1;
 270                        totout += t->nout;
 271                }
 272                if (t->nout < t->maxout
 273                && (use_tainted || !t->taint)
 274                && t->ifp->nd) {
 275                        f = newtframe(d, t);
 276                        if (f) {
 277                                ifrotate(t);
 278                                d->tgt = tt;
 279                                return f;
 280                        }
 281                }
 282                if (tt == d->tgt) {     /* we've looped and found nada */
 283                        if (!use_tainted && !has_untainted)
 284                                use_tainted = 1;
 285                        else
 286                                break;
 287                }
 288        }
 289        if (totout == 0) {
 290                d->kicked++;
 291                d->flags |= DEVFL_KICKME;
 292        }
 293        return NULL;
 294}
 295
 296static void
 297skb_fillup(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter)
 298{
 299        int frag = 0;
 300        struct bio_vec bv;
 301
 302        __bio_for_each_segment(bv, bio, iter, iter)
 303                skb_fill_page_desc(skb, frag++, bv.bv_page,
 304                                   bv.bv_offset, bv.bv_len);
 305}
 306
 307static void
 308fhash(struct frame *f)
 309{
 310        struct aoedev *d = f->t->d;
 311        u32 n;
 312
 313        n = f->tag % NFACTIVE;
 314        list_add_tail(&f->head, &d->factive[n]);
 315}
 316
 317static void
 318ata_rw_frameinit(struct frame *f)
 319{
 320        struct aoetgt *t;
 321        struct aoe_hdr *h;
 322        struct aoe_atahdr *ah;
 323        struct sk_buff *skb;
 324        char writebit, extbit;
 325
 326        skb = f->skb;
 327        h = (struct aoe_hdr *) skb_mac_header(skb);
 328        ah = (struct aoe_atahdr *) (h + 1);
 329        skb_put(skb, sizeof(*h) + sizeof(*ah));
 330        memset(h, 0, skb->len);
 331
 332        writebit = 0x10;
 333        extbit = 0x4;
 334
 335        t = f->t;
 336        f->tag = aoehdr_atainit(t->d, t, h);
 337        fhash(f);
 338        t->nout++;
 339        f->waited = 0;
 340        f->waited_total = 0;
 341
 342        /* set up ata header */
 343        ah->scnt = f->iter.bi_size >> 9;
 344        put_lba(ah, f->iter.bi_sector);
 345        if (t->d->flags & DEVFL_EXT) {
 346                ah->aflags |= AOEAFL_EXT;
 347        } else {
 348                extbit = 0;
 349                ah->lba3 &= 0x0f;
 350                ah->lba3 |= 0xe0;       /* LBA bit + obsolete 0xa0 */
 351        }
 352        if (f->buf && bio_data_dir(f->buf->bio) == WRITE) {
 353                skb_fillup(skb, f->buf->bio, f->iter);
 354                ah->aflags |= AOEAFL_WRITE;
 355                skb->len += f->iter.bi_size;
 356                skb->data_len = f->iter.bi_size;
 357                skb->truesize += f->iter.bi_size;
 358                t->wpkts++;
 359        } else {
 360                t->rpkts++;
 361                writebit = 0;
 362        }
 363
 364        ah->cmdstat = ATA_CMD_PIO_READ | writebit | extbit;
 365        skb->dev = t->ifp->nd;
 366}
 367
 368static int
 369aoecmd_ata_rw(struct aoedev *d)
 370{
 371        struct frame *f;
 372        struct buf *buf;
 373        struct sk_buff *skb;
 374        struct sk_buff_head queue;
 375
 376        buf = nextbuf(d);
 377        if (buf == NULL)
 378                return 0;
 379        f = newframe(d);
 380        if (f == NULL)
 381                return 0;
 382
 383        /* initialize the headers & frame */
 384        f->buf = buf;
 385        f->iter = buf->iter;
 386        f->iter.bi_size = min_t(unsigned long,
 387                                d->maxbcnt ?: DEFAULTBCNT,
 388                                f->iter.bi_size);
 389        bio_advance_iter(buf->bio, &buf->iter, f->iter.bi_size);
 390
 391        if (!buf->iter.bi_size)
 392                d->ip.buf = NULL;
 393
 394        /* mark all tracking fields and load out */
 395        buf->nframesout += 1;
 396
 397        ata_rw_frameinit(f);
 398
 399        skb = skb_clone(f->skb, GFP_ATOMIC);
 400        if (skb) {
 401                f->sent = ktime_get();
 402                __skb_queue_head_init(&queue);
 403                __skb_queue_tail(&queue, skb);
 404                aoenet_xmit(&queue);
 405        }
 406        return 1;
 407}
 408
 409/* some callers cannot sleep, and they can call this function,
 410 * transmitting the packets later, when interrupts are on
 411 */
 412static void
 413aoecmd_cfg_pkts(ushort aoemajor, unsigned char aoeminor, struct sk_buff_head *queue)
 414{
 415        struct aoe_hdr *h;
 416        struct aoe_cfghdr *ch;
 417        struct sk_buff *skb;
 418        struct net_device *ifp;
 419
 420        rcu_read_lock();
 421        for_each_netdev_rcu(&init_net, ifp) {
 422                dev_hold(ifp);
 423                if (!is_aoe_netif(ifp))
 424                        goto cont;
 425
 426                skb = new_skb(sizeof *h + sizeof *ch);
 427                if (skb == NULL) {
 428                        printk(KERN_INFO "aoe: skb alloc failure\n");
 429                        goto cont;
 430                }
 431                skb_put(skb, sizeof *h + sizeof *ch);
 432                skb->dev = ifp;
 433                __skb_queue_tail(queue, skb);
 434                h = (struct aoe_hdr *) skb_mac_header(skb);
 435                memset(h, 0, sizeof *h + sizeof *ch);
 436
 437                memset(h->dst, 0xff, sizeof h->dst);
 438                memcpy(h->src, ifp->dev_addr, sizeof h->src);
 439                h->type = __constant_cpu_to_be16(ETH_P_AOE);
 440                h->verfl = AOE_HVER;
 441                h->major = cpu_to_be16(aoemajor);
 442                h->minor = aoeminor;
 443                h->cmd = AOECMD_CFG;
 444
 445cont:
 446                dev_put(ifp);
 447        }
 448        rcu_read_unlock();
 449}
 450
 451static void
 452resend(struct aoedev *d, struct frame *f)
 453{
 454        struct sk_buff *skb;
 455        struct sk_buff_head queue;
 456        struct aoe_hdr *h;
 457        struct aoetgt *t;
 458        char buf[128];
 459        u32 n;
 460
 461        t = f->t;
 462        n = newtag(d);
 463        skb = f->skb;
 464        if (ifrotate(t) == NULL) {
 465                /* probably can't happen, but set it up to fail anyway */
 466                pr_info("aoe: resend: no interfaces to rotate to.\n");
 467                ktcomplete(f, NULL);
 468                return;
 469        }
 470        h = (struct aoe_hdr *) skb_mac_header(skb);
 471
 472        if (!(f->flags & FFL_PROBE)) {
 473                snprintf(buf, sizeof(buf),
 474                        "%15s e%ld.%d oldtag=%08x@%08lx newtag=%08x s=%pm d=%pm nout=%d\n",
 475                        "retransmit", d->aoemajor, d->aoeminor,
 476                        f->tag, jiffies, n,
 477                        h->src, h->dst, t->nout);
 478                aoechr_error(buf);
 479        }
 480
 481        f->tag = n;
 482        fhash(f);
 483        h->tag = cpu_to_be32(n);
 484        memcpy(h->dst, t->addr, sizeof h->dst);
 485        memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
 486
 487        skb->dev = t->ifp->nd;
 488        skb = skb_clone(skb, GFP_ATOMIC);
 489        if (skb == NULL)
 490                return;
 491        f->sent = ktime_get();
 492        __skb_queue_head_init(&queue);
 493        __skb_queue_tail(&queue, skb);
 494        aoenet_xmit(&queue);
 495}
 496
 497static int
 498tsince_hr(struct frame *f)
 499{
 500        u64 delta = ktime_to_ns(ktime_sub(ktime_get(), f->sent));
 501
 502        /* delta is normally under 4.2 seconds, avoid 64-bit division */
 503        if (likely(delta <= UINT_MAX))
 504                return (u32)delta / NSEC_PER_USEC;
 505
 506        /* avoid overflow after 71 minutes */
 507        if (delta > ((u64)INT_MAX * NSEC_PER_USEC))
 508                return INT_MAX;
 509
 510        return div_u64(delta, NSEC_PER_USEC);
 511}
 512
 513static int
 514tsince(u32 tag)
 515{
 516        int n;
 517
 518        n = jiffies & 0xffff;
 519        n -= tag & 0xffff;
 520        if (n < 0)
 521                n += 1<<16;
 522        return jiffies_to_usecs(n + 1);
 523}
 524
 525static struct aoeif *
 526getif(struct aoetgt *t, struct net_device *nd)
 527{
 528        struct aoeif *p, *e;
 529
 530        p = t->ifs;
 531        e = p + NAOEIFS;
 532        for (; p < e; p++)
 533                if (p->nd == nd)
 534                        return p;
 535        return NULL;
 536}
 537
 538static void
 539ejectif(struct aoetgt *t, struct aoeif *ifp)
 540{
 541        struct aoeif *e;
 542        struct net_device *nd;
 543        ulong n;
 544
 545        nd = ifp->nd;
 546        e = t->ifs + NAOEIFS - 1;
 547        n = (e - ifp) * sizeof *ifp;
 548        memmove(ifp, ifp+1, n);
 549        e->nd = NULL;
 550        dev_put(nd);
 551}
 552
 553static struct frame *
 554reassign_frame(struct frame *f)
 555{
 556        struct frame *nf;
 557        struct sk_buff *skb;
 558
 559        nf = newframe(f->t->d);
 560        if (!nf)
 561                return NULL;
 562        if (nf->t == f->t) {
 563                aoe_freetframe(nf);
 564                return NULL;
 565        }
 566
 567        skb = nf->skb;
 568        nf->skb = f->skb;
 569        nf->buf = f->buf;
 570        nf->iter = f->iter;
 571        nf->waited = 0;
 572        nf->waited_total = f->waited_total;
 573        nf->sent = f->sent;
 574        f->skb = skb;
 575
 576        return nf;
 577}
 578
 579static void
 580probe(struct aoetgt *t)
 581{
 582        struct aoedev *d;
 583        struct frame *f;
 584        struct sk_buff *skb;
 585        struct sk_buff_head queue;
 586        size_t n, m;
 587        int frag;
 588
 589        d = t->d;
 590        f = newtframe(d, t);
 591        if (!f) {
 592                pr_err("%s %pm for e%ld.%d: %s\n",
 593                        "aoe: cannot probe remote address",
 594                        t->addr,
 595                        (long) d->aoemajor, d->aoeminor,
 596                        "no frame available");
 597                return;
 598        }
 599        f->flags |= FFL_PROBE;
 600        ifrotate(t);
 601        f->iter.bi_size = t->d->maxbcnt ? t->d->maxbcnt : DEFAULTBCNT;
 602        ata_rw_frameinit(f);
 603        skb = f->skb;
 604        for (frag = 0, n = f->iter.bi_size; n > 0; ++frag, n -= m) {
 605                if (n < PAGE_SIZE)
 606                        m = n;
 607                else
 608                        m = PAGE_SIZE;
 609                skb_fill_page_desc(skb, frag, empty_page, 0, m);
 610        }
 611        skb->len += f->iter.bi_size;
 612        skb->data_len = f->iter.bi_size;
 613        skb->truesize += f->iter.bi_size;
 614
 615        skb = skb_clone(f->skb, GFP_ATOMIC);
 616        if (skb) {
 617                f->sent = ktime_get();
 618                __skb_queue_head_init(&queue);
 619                __skb_queue_tail(&queue, skb);
 620                aoenet_xmit(&queue);
 621        }
 622}
 623
 624static long
 625rto(struct aoedev *d)
 626{
 627        long t;
 628
 629        t = 2 * d->rttavg >> RTTSCALE;
 630        t += 8 * d->rttdev >> RTTDSCALE;
 631        if (t == 0)
 632                t = 1;
 633
 634        return t;
 635}
 636
 637static void
 638rexmit_deferred(struct aoedev *d)
 639{
 640        struct aoetgt *t;
 641        struct frame *f;
 642        struct frame *nf;
 643        struct list_head *pos, *nx, *head;
 644        int since;
 645        int untainted;
 646
 647        count_targets(d, &untainted);
 648
 649        head = &d->rexmitq;
 650        list_for_each_safe(pos, nx, head) {
 651                f = list_entry(pos, struct frame, head);
 652                t = f->t;
 653                if (t->taint) {
 654                        if (!(f->flags & FFL_PROBE)) {
 655                                nf = reassign_frame(f);
 656                                if (nf) {
 657                                        if (t->nout_probes == 0
 658                                        && untainted > 0) {
 659                                                probe(t);
 660                                                t->nout_probes++;
 661                                        }
 662                                        list_replace(&f->head, &nf->head);
 663                                        pos = &nf->head;
 664                                        aoe_freetframe(f);
 665                                        f = nf;
 666                                        t = f->t;
 667                                }
 668                        } else if (untainted < 1) {
 669                                /* don't probe w/o other untainted aoetgts */
 670                                goto stop_probe;
 671                        } else if (tsince_hr(f) < t->taint * rto(d)) {
 672                                /* reprobe slowly when taint is high */
 673                                continue;
 674                        }
 675                } else if (f->flags & FFL_PROBE) {
 676stop_probe:             /* don't probe untainted aoetgts */
 677                        list_del(pos);
 678                        aoe_freetframe(f);
 679                        /* leaving d->kicked, because this is routine */
 680                        f->t->d->flags |= DEVFL_KICKME;
 681                        continue;
 682                }
 683                if (t->nout >= t->maxout)
 684                        continue;
 685                list_del(pos);
 686                t->nout++;
 687                if (f->flags & FFL_PROBE)
 688                        t->nout_probes++;
 689                since = tsince_hr(f);
 690                f->waited += since;
 691                f->waited_total += since;
 692                resend(d, f);
 693        }
 694}
 695
 696/* An aoetgt accumulates demerits quickly, and successful
 697 * probing redeems the aoetgt slowly.
 698 */
 699static void
 700scorn(struct aoetgt *t)
 701{
 702        int n;
 703
 704        n = t->taint++;
 705        t->taint += t->taint * 2;
 706        if (n > t->taint)
 707                t->taint = n;
 708        if (t->taint > MAX_TAINT)
 709                t->taint = MAX_TAINT;
 710}
 711
 712static int
 713count_targets(struct aoedev *d, int *untainted)
 714{
 715        int i, good;
 716
 717        for (i = good = 0; i < d->ntargets && d->targets[i]; ++i)
 718                if (d->targets[i]->taint == 0)
 719                        good++;
 720
 721        if (untainted)
 722                *untainted = good;
 723        return i;
 724}
 725
 726static void
 727rexmit_timer(struct timer_list *timer)
 728{
 729        struct aoedev *d;
 730        struct aoetgt *t;
 731        struct aoeif *ifp;
 732        struct frame *f;
 733        struct list_head *head, *pos, *nx;
 734        LIST_HEAD(flist);
 735        register long timeout;
 736        ulong flags, n;
 737        int i;
 738        int utgts;      /* number of aoetgt descriptors (not slots) */
 739        int since;
 740
 741        d = from_timer(d, timer, timer);
 742
 743        spin_lock_irqsave(&d->lock, flags);
 744
 745        /* timeout based on observed timings and variations */
 746        timeout = rto(d);
 747
 748        utgts = count_targets(d, NULL);
 749
 750        if (d->flags & DEVFL_TKILL) {
 751                spin_unlock_irqrestore(&d->lock, flags);
 752                return;
 753        }
 754
 755        /* collect all frames to rexmit into flist */
 756        for (i = 0; i < NFACTIVE; i++) {
 757                head = &d->factive[i];
 758                list_for_each_safe(pos, nx, head) {
 759                        f = list_entry(pos, struct frame, head);
 760                        if (tsince_hr(f) < timeout)
 761                                break;  /* end of expired frames */
 762                        /* move to flist for later processing */
 763                        list_move_tail(pos, &flist);
 764                }
 765        }
 766
 767        /* process expired frames */
 768        while (!list_empty(&flist)) {
 769                pos = flist.next;
 770                f = list_entry(pos, struct frame, head);
 771                since = tsince_hr(f);
 772                n = f->waited_total + since;
 773                n /= USEC_PER_SEC;
 774                if (aoe_deadsecs
 775                && n > aoe_deadsecs
 776                && !(f->flags & FFL_PROBE)) {
 777                        /* Waited too long.  Device failure.
 778                         * Hang all frames on first hash bucket for downdev
 779                         * to clean up.
 780                         */
 781                        list_splice(&flist, &d->factive[0]);
 782                        aoedev_downdev(d);
 783                        goto out;
 784                }
 785
 786                t = f->t;
 787                n = f->waited + since;
 788                n /= USEC_PER_SEC;
 789                if (aoe_deadsecs && utgts > 0
 790                && (n > aoe_deadsecs / utgts || n > HARD_SCORN_SECS))
 791                        scorn(t); /* avoid this target */
 792
 793                if (t->maxout != 1) {
 794                        t->ssthresh = t->maxout / 2;
 795                        t->maxout = 1;
 796                }
 797
 798                if (f->flags & FFL_PROBE) {
 799                        t->nout_probes--;
 800                } else {
 801                        ifp = getif(t, f->skb->dev);
 802                        if (ifp && ++ifp->lost > (t->nframes << 1)
 803                        && (ifp != t->ifs || t->ifs[1].nd)) {
 804                                ejectif(t, ifp);
 805                                ifp = NULL;
 806                        }
 807                }
 808                list_move_tail(pos, &d->rexmitq);
 809                t->nout--;
 810        }
 811        rexmit_deferred(d);
 812
 813out:
 814        if ((d->flags & DEVFL_KICKME) && d->blkq) {
 815                d->flags &= ~DEVFL_KICKME;
 816                blk_mq_run_hw_queues(d->blkq, true);
 817        }
 818
 819        d->timer.expires = jiffies + TIMERTICK;
 820        add_timer(&d->timer);
 821
 822        spin_unlock_irqrestore(&d->lock, flags);
 823}
 824
 825static void
 826bufinit(struct buf *buf, struct request *rq, struct bio *bio)
 827{
 828        memset(buf, 0, sizeof(*buf));
 829        buf->rq = rq;
 830        buf->bio = bio;
 831        buf->iter = bio->bi_iter;
 832}
 833
 834static struct buf *
 835nextbuf(struct aoedev *d)
 836{
 837        struct request *rq;
 838        struct request_queue *q;
 839        struct aoe_req *req;
 840        struct buf *buf;
 841        struct bio *bio;
 842
 843        q = d->blkq;
 844        if (q == NULL)
 845                return NULL;    /* initializing */
 846        if (d->ip.buf)
 847                return d->ip.buf;
 848        rq = d->ip.rq;
 849        if (rq == NULL) {
 850                rq = list_first_entry_or_null(&d->rq_list, struct request,
 851                                                queuelist);
 852                if (rq == NULL)
 853                        return NULL;
 854                list_del_init(&rq->queuelist);
 855                blk_mq_start_request(rq);
 856                d->ip.rq = rq;
 857                d->ip.nxbio = rq->bio;
 858
 859                req = blk_mq_rq_to_pdu(rq);
 860                req->nr_bios = 0;
 861                __rq_for_each_bio(bio, rq)
 862                        req->nr_bios++;
 863        }
 864        buf = mempool_alloc(d->bufpool, GFP_ATOMIC);
 865        if (buf == NULL) {
 866                pr_err("aoe: nextbuf: unable to mempool_alloc!\n");
 867                return NULL;
 868        }
 869        bio = d->ip.nxbio;
 870        bufinit(buf, rq, bio);
 871        bio = bio->bi_next;
 872        d->ip.nxbio = bio;
 873        if (bio == NULL)
 874                d->ip.rq = NULL;
 875        return d->ip.buf = buf;
 876}
 877
 878/* enters with d->lock held */
 879void
 880aoecmd_work(struct aoedev *d)
 881{
 882        rexmit_deferred(d);
 883        while (aoecmd_ata_rw(d))
 884                ;
 885}
 886
 887/* this function performs work that has been deferred until sleeping is OK
 888 */
 889void
 890aoecmd_sleepwork(struct work_struct *work)
 891{
 892        struct aoedev *d = container_of(work, struct aoedev, work);
 893
 894        if (d->flags & DEVFL_GDALLOC)
 895                aoeblk_gdalloc(d);
 896
 897        if (d->flags & DEVFL_NEWSIZE) {
 898                set_capacity_and_notify(d->gd, d->ssize);
 899
 900                spin_lock_irq(&d->lock);
 901                d->flags |= DEVFL_UP;
 902                d->flags &= ~DEVFL_NEWSIZE;
 903                spin_unlock_irq(&d->lock);
 904        }
 905}
 906
 907static void
 908ata_ident_fixstring(u16 *id, int ns)
 909{
 910        u16 s;
 911
 912        while (ns-- > 0) {
 913                s = *id;
 914                *id++ = s >> 8 | s << 8;
 915        }
 916}
 917
 918static void
 919ataid_complete(struct aoedev *d, struct aoetgt *t, unsigned char *id)
 920{
 921        u64 ssize;
 922        u16 n;
 923
 924        /* word 83: command set supported */
 925        n = get_unaligned_le16(&id[83 << 1]);
 926
 927        /* word 86: command set/feature enabled */
 928        n |= get_unaligned_le16(&id[86 << 1]);
 929
 930        if (n & (1<<10)) {      /* bit 10: LBA 48 */
 931                d->flags |= DEVFL_EXT;
 932
 933                /* word 100: number lba48 sectors */
 934                ssize = get_unaligned_le64(&id[100 << 1]);
 935
 936                /* set as in ide-disk.c:init_idedisk_capacity */
 937                d->geo.cylinders = ssize;
 938                d->geo.cylinders /= (255 * 63);
 939                d->geo.heads = 255;
 940                d->geo.sectors = 63;
 941        } else {
 942                d->flags &= ~DEVFL_EXT;
 943
 944                /* number lba28 sectors */
 945                ssize = get_unaligned_le32(&id[60 << 1]);
 946
 947                /* NOTE: obsolete in ATA 6 */
 948                d->geo.cylinders = get_unaligned_le16(&id[54 << 1]);
 949                d->geo.heads = get_unaligned_le16(&id[55 << 1]);
 950                d->geo.sectors = get_unaligned_le16(&id[56 << 1]);
 951        }
 952
 953        ata_ident_fixstring((u16 *) &id[10<<1], 10);    /* serial */
 954        ata_ident_fixstring((u16 *) &id[23<<1], 4);     /* firmware */
 955        ata_ident_fixstring((u16 *) &id[27<<1], 20);    /* model */
 956        memcpy(d->ident, id, sizeof(d->ident));
 957
 958        if (d->ssize != ssize)
 959                printk(KERN_INFO
 960                        "aoe: %pm e%ld.%d v%04x has %llu sectors\n",
 961                        t->addr,
 962                        d->aoemajor, d->aoeminor,
 963                        d->fw_ver, (long long)ssize);
 964        d->ssize = ssize;
 965        d->geo.start = 0;
 966        if (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE))
 967                return;
 968        if (d->gd != NULL)
 969                d->flags |= DEVFL_NEWSIZE;
 970        else
 971                d->flags |= DEVFL_GDALLOC;
 972        schedule_work(&d->work);
 973}
 974
 975static void
 976calc_rttavg(struct aoedev *d, struct aoetgt *t, int rtt)
 977{
 978        register long n;
 979
 980        n = rtt;
 981
 982        /* cf. Congestion Avoidance and Control, Jacobson & Karels, 1988 */
 983        n -= d->rttavg >> RTTSCALE;
 984        d->rttavg += n;
 985        if (n < 0)
 986                n = -n;
 987        n -= d->rttdev >> RTTDSCALE;
 988        d->rttdev += n;
 989
 990        if (!t || t->maxout >= t->nframes)
 991                return;
 992        if (t->maxout < t->ssthresh)
 993                t->maxout += 1;
 994        else if (t->nout == t->maxout && t->next_cwnd-- == 0) {
 995                t->maxout += 1;
 996                t->next_cwnd = t->maxout;
 997        }
 998}
 999
1000static struct aoetgt *
1001gettgt(struct aoedev *d, char *addr)
1002{
1003        struct aoetgt **t, **e;
1004
1005        t = d->targets;
1006        e = t + d->ntargets;
1007        for (; t < e && *t; t++)
1008                if (memcmp((*t)->addr, addr, sizeof((*t)->addr)) == 0)
1009                        return *t;
1010        return NULL;
1011}
1012
1013static void
1014bvcpy(struct sk_buff *skb, struct bio *bio, struct bvec_iter iter, long cnt)
1015{
1016        int soff = 0;
1017        struct bio_vec bv;
1018
1019        iter.bi_size = cnt;
1020
1021        __bio_for_each_segment(bv, bio, iter, iter) {
1022                char *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
1023                skb_copy_bits(skb, soff, p, bv.bv_len);
1024                kunmap_atomic(p);
1025                soff += bv.bv_len;
1026        }
1027}
1028
1029void
1030aoe_end_request(struct aoedev *d, struct request *rq, int fastfail)
1031{
1032        struct bio *bio;
1033        int bok;
1034        struct request_queue *q;
1035        blk_status_t err = BLK_STS_OK;
1036
1037        q = d->blkq;
1038        if (rq == d->ip.rq)
1039                d->ip.rq = NULL;
1040        do {
1041                bio = rq->bio;
1042                bok = !fastfail && !bio->bi_status;
1043                if (!bok)
1044                        err = BLK_STS_IOERR;
1045        } while (blk_update_request(rq, bok ? BLK_STS_OK : BLK_STS_IOERR, bio->bi_iter.bi_size));
1046
1047        __blk_mq_end_request(rq, err);
1048
1049        /* cf. https://lore.kernel.org/lkml/20061031071040.GS14055@kernel.dk/ */
1050        if (!fastfail)
1051                blk_mq_run_hw_queues(q, true);
1052}
1053
1054static void
1055aoe_end_buf(struct aoedev *d, struct buf *buf)
1056{
1057        struct request *rq = buf->rq;
1058        struct aoe_req *req = blk_mq_rq_to_pdu(rq);
1059
1060        if (buf == d->ip.buf)
1061                d->ip.buf = NULL;
1062        mempool_free(buf, d->bufpool);
1063        if (--req->nr_bios == 0)
1064                aoe_end_request(d, rq, 0);
1065}
1066
1067static void
1068ktiocomplete(struct frame *f)
1069{
1070        struct aoe_hdr *hin, *hout;
1071        struct aoe_atahdr *ahin, *ahout;
1072        struct buf *buf;
1073        struct sk_buff *skb;
1074        struct aoetgt *t;
1075        struct aoeif *ifp;
1076        struct aoedev *d;
1077        long n;
1078        int untainted;
1079
1080        if (f == NULL)
1081                return;
1082
1083        t = f->t;
1084        d = t->d;
1085        skb = f->r_skb;
1086        buf = f->buf;
1087        if (f->flags & FFL_PROBE)
1088                goto out;
1089        if (!skb)               /* just fail the buf. */
1090                goto noskb;
1091
1092        hout = (struct aoe_hdr *) skb_mac_header(f->skb);
1093        ahout = (struct aoe_atahdr *) (hout+1);
1094
1095        hin = (struct aoe_hdr *) skb->data;
1096        skb_pull(skb, sizeof(*hin));
1097        ahin = (struct aoe_atahdr *) skb->data;
1098        skb_pull(skb, sizeof(*ahin));
1099        if (ahin->cmdstat & 0xa9) {     /* these bits cleared on success */
1100                pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
1101                        ahout->cmdstat, ahin->cmdstat,
1102                        d->aoemajor, d->aoeminor);
1103noskb:          if (buf)
1104                        buf->bio->bi_status = BLK_STS_IOERR;
1105                goto out;
1106        }
1107
1108        n = ahout->scnt << 9;
1109        switch (ahout->cmdstat) {
1110        case ATA_CMD_PIO_READ:
1111        case ATA_CMD_PIO_READ_EXT:
1112                if (skb->len < n) {
1113                        pr_err("%s e%ld.%d.  skb->len=%d need=%ld\n",
1114                                "aoe: runt data size in read from",
1115                                (long) d->aoemajor, d->aoeminor,
1116                               skb->len, n);
1117                        buf->bio->bi_status = BLK_STS_IOERR;
1118                        break;
1119                }
1120                if (n > f->iter.bi_size) {
1121                        pr_err_ratelimited("%s e%ld.%d.  bytes=%ld need=%u\n",
1122                                "aoe: too-large data size in read from",
1123                                (long) d->aoemajor, d->aoeminor,
1124                                n, f->iter.bi_size);
1125                        buf->bio->bi_status = BLK_STS_IOERR;
1126                        break;
1127                }
1128                bvcpy(skb, f->buf->bio, f->iter, n);
1129                fallthrough;
1130        case ATA_CMD_PIO_WRITE:
1131        case ATA_CMD_PIO_WRITE_EXT:
1132                spin_lock_irq(&d->lock);
1133                ifp = getif(t, skb->dev);
1134                if (ifp)
1135                        ifp->lost = 0;
1136                spin_unlock_irq(&d->lock);
1137                break;
1138        case ATA_CMD_ID_ATA:
1139                if (skb->len < 512) {
1140                        pr_info("%s e%ld.%d.  skb->len=%d need=512\n",
1141                                "aoe: runt data size in ataid from",
1142                                (long) d->aoemajor, d->aoeminor,
1143                                skb->len);
1144                        break;
1145                }
1146                if (skb_linearize(skb))
1147                        break;
1148                spin_lock_irq(&d->lock);
1149                ataid_complete(d, t, skb->data);
1150                spin_unlock_irq(&d->lock);
1151                break;
1152        default:
1153                pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n",
1154                        ahout->cmdstat,
1155                        be16_to_cpu(get_unaligned(&hin->major)),
1156                        hin->minor);
1157        }
1158out:
1159        spin_lock_irq(&d->lock);
1160        if (t->taint > 0
1161        && --t->taint > 0
1162        && t->nout_probes == 0) {
1163                count_targets(d, &untainted);
1164                if (untainted > 0) {
1165                        probe(t);
1166                        t->nout_probes++;
1167                }
1168        }
1169
1170        aoe_freetframe(f);
1171
1172        if (buf && --buf->nframesout == 0 && buf->iter.bi_size == 0)
1173                aoe_end_buf(d, buf);
1174
1175        spin_unlock_irq(&d->lock);
1176        aoedev_put(d);
1177        dev_kfree_skb(skb);
1178}
1179
1180/* Enters with iocq.lock held.
1181 * Returns true iff responses needing processing remain.
1182 */
1183static int
1184ktio(int id)
1185{
1186        struct frame *f;
1187        struct list_head *pos;
1188        int i;
1189        int actual_id;
1190
1191        for (i = 0; ; ++i) {
1192                if (i == MAXIOC)
1193                        return 1;
1194                if (list_empty(&iocq[id].head))
1195                        return 0;
1196                pos = iocq[id].head.next;
1197                list_del(pos);
1198                f = list_entry(pos, struct frame, head);
1199                spin_unlock_irq(&iocq[id].lock);
1200                ktiocomplete(f);
1201
1202                /* Figure out if extra threads are required. */
1203                actual_id = f->t->d->aoeminor % ncpus;
1204
1205                if (!kts[actual_id].active) {
1206                        BUG_ON(id != 0);
1207                        mutex_lock(&ktio_spawn_lock);
1208                        if (!kts[actual_id].active
1209                                && aoe_ktstart(&kts[actual_id]) == 0)
1210                                kts[actual_id].active = 1;
1211                        mutex_unlock(&ktio_spawn_lock);
1212                }
1213                spin_lock_irq(&iocq[id].lock);
1214        }
1215}
1216
1217static int
1218kthread(void *vp)
1219{
1220        struct ktstate *k;
1221        DECLARE_WAITQUEUE(wait, current);
1222        int more;
1223
1224        k = vp;
1225        current->flags |= PF_NOFREEZE;
1226        set_user_nice(current, -10);
1227        complete(&k->rendez);   /* tell spawner we're running */
1228        do {
1229                spin_lock_irq(k->lock);
1230                more = k->fn(k->id);
1231                if (!more) {
1232                        add_wait_queue(k->waitq, &wait);
1233                        __set_current_state(TASK_INTERRUPTIBLE);
1234                }
1235                spin_unlock_irq(k->lock);
1236                if (!more) {
1237                        schedule();
1238                        remove_wait_queue(k->waitq, &wait);
1239                } else
1240                        cond_resched();
1241        } while (!kthread_should_stop());
1242        complete(&k->rendez);   /* tell spawner we're stopping */
1243        return 0;
1244}
1245
1246void
1247aoe_ktstop(struct ktstate *k)
1248{
1249        kthread_stop(k->task);
1250        wait_for_completion(&k->rendez);
1251}
1252
1253int
1254aoe_ktstart(struct ktstate *k)
1255{
1256        struct task_struct *task;
1257
1258        init_completion(&k->rendez);
1259        task = kthread_run(kthread, k, "%s", k->name);
1260        if (task == NULL || IS_ERR(task))
1261                return -ENOMEM;
1262        k->task = task;
1263        wait_for_completion(&k->rendez); /* allow kthread to start */
1264        init_completion(&k->rendez);    /* for waiting for exit later */
1265        return 0;
1266}
1267
1268/* pass it off to kthreads for processing */
1269static void
1270ktcomplete(struct frame *f, struct sk_buff *skb)
1271{
1272        int id;
1273        ulong flags;
1274
1275        f->r_skb = skb;
1276        id = f->t->d->aoeminor % ncpus;
1277        spin_lock_irqsave(&iocq[id].lock, flags);
1278        if (!kts[id].active) {
1279                spin_unlock_irqrestore(&iocq[id].lock, flags);
1280                /* The thread with id has not been spawned yet,
1281                 * so delegate the work to the main thread and
1282                 * try spawning a new thread.
1283                 */
1284                id = 0;
1285                spin_lock_irqsave(&iocq[id].lock, flags);
1286        }
1287        list_add_tail(&f->head, &iocq[id].head);
1288        spin_unlock_irqrestore(&iocq[id].lock, flags);
1289        wake_up(&ktiowq[id]);
1290}
1291
1292struct sk_buff *
1293aoecmd_ata_rsp(struct sk_buff *skb)
1294{
1295        struct aoedev *d;
1296        struct aoe_hdr *h;
1297        struct frame *f;
1298        u32 n;
1299        ulong flags;
1300        char ebuf[128];
1301        u16 aoemajor;
1302
1303        h = (struct aoe_hdr *) skb->data;
1304        aoemajor = be16_to_cpu(get_unaligned(&h->major));
1305        d = aoedev_by_aoeaddr(aoemajor, h->minor, 0);
1306        if (d == NULL) {
1307                snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response "
1308                        "for unknown device %d.%d\n",
1309                        aoemajor, h->minor);
1310                aoechr_error(ebuf);
1311                return skb;
1312        }
1313
1314        spin_lock_irqsave(&d->lock, flags);
1315
1316        n = be32_to_cpu(get_unaligned(&h->tag));
1317        f = getframe(d, n);
1318        if (f) {
1319                calc_rttavg(d, f->t, tsince_hr(f));
1320                f->t->nout--;
1321                if (f->flags & FFL_PROBE)
1322                        f->t->nout_probes--;
1323        } else {
1324                f = getframe_deferred(d, n);
1325                if (f) {
1326                        calc_rttavg(d, NULL, tsince_hr(f));
1327                } else {
1328                        calc_rttavg(d, NULL, tsince(n));
1329                        spin_unlock_irqrestore(&d->lock, flags);
1330                        aoedev_put(d);
1331                        snprintf(ebuf, sizeof(ebuf),
1332                                 "%15s e%d.%d    tag=%08x@%08lx s=%pm d=%pm\n",
1333                                 "unexpected rsp",
1334                                 get_unaligned_be16(&h->major),
1335                                 h->minor,
1336                                 get_unaligned_be32(&h->tag),
1337                                 jiffies,
1338                                 h->src,
1339                                 h->dst);
1340                        aoechr_error(ebuf);
1341                        return skb;
1342                }
1343        }
1344        aoecmd_work(d);
1345
1346        spin_unlock_irqrestore(&d->lock, flags);
1347
1348        ktcomplete(f, skb);
1349
1350        /*
1351         * Note here that we do not perform an aoedev_put, as we are
1352         * leaving this reference for the ktio to release.
1353         */
1354        return NULL;
1355}
1356
1357void
1358aoecmd_cfg(ushort aoemajor, unsigned char aoeminor)
1359{
1360        struct sk_buff_head queue;
1361
1362        __skb_queue_head_init(&queue);
1363        aoecmd_cfg_pkts(aoemajor, aoeminor, &queue);
1364        aoenet_xmit(&queue);
1365}
1366
1367struct sk_buff *
1368aoecmd_ata_id(struct aoedev *d)
1369{
1370        struct aoe_hdr *h;
1371        struct aoe_atahdr *ah;
1372        struct frame *f;
1373        struct sk_buff *skb;
1374        struct aoetgt *t;
1375
1376        f = newframe(d);
1377        if (f == NULL)
1378                return NULL;
1379
1380        t = *d->tgt;
1381
1382        /* initialize the headers & frame */
1383        skb = f->skb;
1384        h = (struct aoe_hdr *) skb_mac_header(skb);
1385        ah = (struct aoe_atahdr *) (h+1);
1386        skb_put(skb, sizeof *h + sizeof *ah);
1387        memset(h, 0, skb->len);
1388        f->tag = aoehdr_atainit(d, t, h);
1389        fhash(f);
1390        t->nout++;
1391        f->waited = 0;
1392        f->waited_total = 0;
1393
1394        /* set up ata header */
1395        ah->scnt = 1;
1396        ah->cmdstat = ATA_CMD_ID_ATA;
1397        ah->lba3 = 0xa0;
1398
1399        skb->dev = t->ifp->nd;
1400
1401        d->rttavg = RTTAVG_INIT;
1402        d->rttdev = RTTDEV_INIT;
1403        d->timer.function = rexmit_timer;
1404
1405        skb = skb_clone(skb, GFP_ATOMIC);
1406        if (skb)
1407                f->sent = ktime_get();
1408
1409        return skb;
1410}
1411
1412static struct aoetgt **
1413grow_targets(struct aoedev *d)
1414{
1415        ulong oldn, newn;
1416        struct aoetgt **tt;
1417
1418        oldn = d->ntargets;
1419        newn = oldn * 2;
1420        tt = kcalloc(newn, sizeof(*d->targets), GFP_ATOMIC);
1421        if (!tt)
1422                return NULL;
1423        memmove(tt, d->targets, sizeof(*d->targets) * oldn);
1424        d->tgt = tt + (d->tgt - d->targets);
1425        kfree(d->targets);
1426        d->targets = tt;
1427        d->ntargets = newn;
1428
1429        return &d->targets[oldn];
1430}
1431
1432static struct aoetgt *
1433addtgt(struct aoedev *d, char *addr, ulong nframes)
1434{
1435        struct aoetgt *t, **tt, **te;
1436
1437        tt = d->targets;
1438        te = tt + d->ntargets;
1439        for (; tt < te && *tt; tt++)
1440                ;
1441
1442        if (tt == te) {
1443                tt = grow_targets(d);
1444                if (!tt)
1445                        goto nomem;
1446        }
1447        t = kzalloc(sizeof(*t), GFP_ATOMIC);
1448        if (!t)
1449                goto nomem;
1450        t->nframes = nframes;
1451        t->d = d;
1452        memcpy(t->addr, addr, sizeof t->addr);
1453        t->ifp = t->ifs;
1454        aoecmd_wreset(t);
1455        t->maxout = t->nframes / 2;
1456        INIT_LIST_HEAD(&t->ffree);
1457        return *tt = t;
1458
1459 nomem:
1460        pr_info("aoe: cannot allocate memory to add target\n");
1461        return NULL;
1462}
1463
1464static void
1465setdbcnt(struct aoedev *d)
1466{
1467        struct aoetgt **t, **e;
1468        int bcnt = 0;
1469
1470        t = d->targets;
1471        e = t + d->ntargets;
1472        for (; t < e && *t; t++)
1473                if (bcnt == 0 || bcnt > (*t)->minbcnt)
1474                        bcnt = (*t)->minbcnt;
1475        if (bcnt != d->maxbcnt) {
1476                d->maxbcnt = bcnt;
1477                pr_info("aoe: e%ld.%d: setting %d byte data frames\n",
1478                        d->aoemajor, d->aoeminor, bcnt);
1479        }
1480}
1481
1482static void
1483setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt)
1484{
1485        struct aoedev *d;
1486        struct aoeif *p, *e;
1487        int minbcnt;
1488
1489        d = t->d;
1490        minbcnt = bcnt;
1491        p = t->ifs;
1492        e = p + NAOEIFS;
1493        for (; p < e; p++) {
1494                if (p->nd == NULL)
1495                        break;          /* end of the valid interfaces */
1496                if (p->nd == nd) {
1497                        p->bcnt = bcnt; /* we're updating */
1498                        nd = NULL;
1499                } else if (minbcnt > p->bcnt)
1500                        minbcnt = p->bcnt; /* find the min interface */
1501        }
1502        if (nd) {
1503                if (p == e) {
1504                        pr_err("aoe: device setifbcnt failure; too many interfaces.\n");
1505                        return;
1506                }
1507                dev_hold(nd);
1508                p->nd = nd;
1509                p->bcnt = bcnt;
1510        }
1511        t->minbcnt = minbcnt;
1512        setdbcnt(d);
1513}
1514
1515void
1516aoecmd_cfg_rsp(struct sk_buff *skb)
1517{
1518        struct aoedev *d;
1519        struct aoe_hdr *h;
1520        struct aoe_cfghdr *ch;
1521        struct aoetgt *t;
1522        ulong flags, aoemajor;
1523        struct sk_buff *sl;
1524        struct sk_buff_head queue;
1525        u16 n;
1526
1527        sl = NULL;
1528        h = (struct aoe_hdr *) skb_mac_header(skb);
1529        ch = (struct aoe_cfghdr *) (h+1);
1530
1531        /*
1532         * Enough people have their dip switches set backwards to
1533         * warrant a loud message for this special case.
1534         */
1535        aoemajor = get_unaligned_be16(&h->major);
1536        if (aoemajor == 0xfff) {
1537                printk(KERN_ERR "aoe: Warning: shelf address is all ones.  "
1538                        "Check shelf dip switches.\n");
1539                return;
1540        }
1541        if (aoemajor == 0xffff) {
1542                pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n",
1543                        aoemajor, (int) h->minor);
1544                return;
1545        }
1546        if (h->minor == 0xff) {
1547                pr_info("aoe: e%ld.%d: broadcast slot number invalid\n",
1548                        aoemajor, (int) h->minor);
1549                return;
1550        }
1551
1552        n = be16_to_cpu(ch->bufcnt);
1553        if (n > aoe_maxout)     /* keep it reasonable */
1554                n = aoe_maxout;
1555
1556        d = aoedev_by_aoeaddr(aoemajor, h->minor, 1);
1557        if (d == NULL) {
1558                pr_info("aoe: device allocation failure\n");
1559                return;
1560        }
1561
1562        spin_lock_irqsave(&d->lock, flags);
1563
1564        t = gettgt(d, h->src);
1565        if (t) {
1566                t->nframes = n;
1567                if (n < t->maxout)
1568                        aoecmd_wreset(t);
1569        } else {
1570                t = addtgt(d, h->src, n);
1571                if (!t)
1572                        goto bail;
1573        }
1574        n = skb->dev->mtu;
1575        n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr);
1576        n /= 512;
1577        if (n > ch->scnt)
1578                n = ch->scnt;
1579        n = n ? n * 512 : DEFAULTBCNT;
1580        setifbcnt(t, skb->dev, n);
1581
1582        /* don't change users' perspective */
1583        if (d->nopen == 0) {
1584                d->fw_ver = be16_to_cpu(ch->fwver);
1585                sl = aoecmd_ata_id(d);
1586        }
1587bail:
1588        spin_unlock_irqrestore(&d->lock, flags);
1589        aoedev_put(d);
1590        if (sl) {
1591                __skb_queue_head_init(&queue);
1592                __skb_queue_tail(&queue, sl);
1593                aoenet_xmit(&queue);
1594        }
1595}
1596
1597void
1598aoecmd_wreset(struct aoetgt *t)
1599{
1600        t->maxout = 1;
1601        t->ssthresh = t->nframes / 2;
1602        t->next_cwnd = t->nframes;
1603}
1604
1605void
1606aoecmd_cleanslate(struct aoedev *d)
1607{
1608        struct aoetgt **t, **te;
1609
1610        d->rttavg = RTTAVG_INIT;
1611        d->rttdev = RTTDEV_INIT;
1612        d->maxbcnt = 0;
1613
1614        t = d->targets;
1615        te = t + d->ntargets;
1616        for (; t < te && *t; t++)
1617                aoecmd_wreset(*t);
1618}
1619
1620void
1621aoe_failbuf(struct aoedev *d, struct buf *buf)
1622{
1623        if (buf == NULL)
1624                return;
1625        buf->iter.bi_size = 0;
1626        buf->bio->bi_status = BLK_STS_IOERR;
1627        if (buf->nframesout == 0)
1628                aoe_end_buf(d, buf);
1629}
1630
1631void
1632aoe_flush_iocq(void)
1633{
1634        int i;
1635
1636        for (i = 0; i < ncpus; i++) {
1637                if (kts[i].active)
1638                        aoe_flush_iocq_by_index(i);
1639        }
1640}
1641
1642void
1643aoe_flush_iocq_by_index(int id)
1644{
1645        struct frame *f;
1646        struct aoedev *d;
1647        LIST_HEAD(flist);
1648        struct list_head *pos;
1649        struct sk_buff *skb;
1650        ulong flags;
1651
1652        spin_lock_irqsave(&iocq[id].lock, flags);
1653        list_splice_init(&iocq[id].head, &flist);
1654        spin_unlock_irqrestore(&iocq[id].lock, flags);
1655        while (!list_empty(&flist)) {
1656                pos = flist.next;
1657                list_del(pos);
1658                f = list_entry(pos, struct frame, head);
1659                d = f->t->d;
1660                skb = f->r_skb;
1661                spin_lock_irqsave(&d->lock, flags);
1662                if (f->buf) {
1663                        f->buf->nframesout--;
1664                        aoe_failbuf(d, f->buf);
1665                }
1666                aoe_freetframe(f);
1667                spin_unlock_irqrestore(&d->lock, flags);
1668                dev_kfree_skb(skb);
1669                aoedev_put(d);
1670        }
1671}
1672
1673int __init
1674aoecmd_init(void)
1675{
1676        void *p;
1677        int i;
1678        int ret;
1679
1680        /* get_zeroed_page returns page with ref count 1 */
1681        p = (void *) get_zeroed_page(GFP_KERNEL);
1682        if (!p)
1683                return -ENOMEM;
1684        empty_page = virt_to_page(p);
1685
1686        ncpus = num_online_cpus();
1687
1688        iocq = kcalloc(ncpus, sizeof(struct iocq_ktio), GFP_KERNEL);
1689        if (!iocq)
1690                return -ENOMEM;
1691
1692        kts = kcalloc(ncpus, sizeof(struct ktstate), GFP_KERNEL);
1693        if (!kts) {
1694                ret = -ENOMEM;
1695                goto kts_fail;
1696        }
1697
1698        ktiowq = kcalloc(ncpus, sizeof(wait_queue_head_t), GFP_KERNEL);
1699        if (!ktiowq) {
1700                ret = -ENOMEM;
1701                goto ktiowq_fail;
1702        }
1703
1704        mutex_init(&ktio_spawn_lock);
1705
1706        for (i = 0; i < ncpus; i++) {
1707                INIT_LIST_HEAD(&iocq[i].head);
1708                spin_lock_init(&iocq[i].lock);
1709                init_waitqueue_head(&ktiowq[i]);
1710                snprintf(kts[i].name, sizeof(kts[i].name), "aoe_ktio%d", i);
1711                kts[i].fn = ktio;
1712                kts[i].waitq = &ktiowq[i];
1713                kts[i].lock = &iocq[i].lock;
1714                kts[i].id = i;
1715                kts[i].active = 0;
1716        }
1717        kts[0].active = 1;
1718        if (aoe_ktstart(&kts[0])) {
1719                ret = -ENOMEM;
1720                goto ktstart_fail;
1721        }
1722        return 0;
1723
1724ktstart_fail:
1725        kfree(ktiowq);
1726ktiowq_fail:
1727        kfree(kts);
1728kts_fail:
1729        kfree(iocq);
1730
1731        return ret;
1732}
1733
1734void
1735aoecmd_exit(void)
1736{
1737        int i;
1738
1739        for (i = 0; i < ncpus; i++)
1740                if (kts[i].active)
1741                        aoe_ktstop(&kts[i]);
1742
1743        aoe_flush_iocq();
1744
1745        /* Free up the iocq and thread speicific configuration
1746        * allocated during startup.
1747        */
1748        kfree(iocq);
1749        kfree(kts);
1750        kfree(ktiowq);
1751
1752        free_page((unsigned long) page_address(empty_page));
1753        empty_page = NULL;
1754}
1755