linux/drivers/block/sx8.c
<<
>>
Prefs
   1/*
   2 *  sx8.c: Driver for Promise SATA SX8 looks-like-I2O hardware
   3 *
   4 *  Copyright 2004-2005 Red Hat, Inc.
   5 *
   6 *  Author/maintainer:  Jeff Garzik <jgarzik@pobox.com>
   7 *
   8 *  This file is subject to the terms and conditions of the GNU General Public
   9 *  License.  See the file "COPYING" in the main directory of this archive
  10 *  for more details.
  11 */
  12
  13#include <linux/kernel.h>
  14#include <linux/module.h>
  15#include <linux/init.h>
  16#include <linux/pci.h>
  17#include <linux/slab.h>
  18#include <linux/spinlock.h>
  19#include <linux/blk-mq.h>
  20#include <linux/sched.h>
  21#include <linux/interrupt.h>
  22#include <linux/compiler.h>
  23#include <linux/workqueue.h>
  24#include <linux/bitops.h>
  25#include <linux/delay.h>
  26#include <linux/ktime.h>
  27#include <linux/hdreg.h>
  28#include <linux/dma-mapping.h>
  29#include <linux/completion.h>
  30#include <linux/scatterlist.h>
  31#include <asm/io.h>
  32#include <linux/uaccess.h>
  33
  34#if 0
  35#define CARM_DEBUG
  36#define CARM_VERBOSE_DEBUG
  37#else
  38#undef CARM_DEBUG
  39#undef CARM_VERBOSE_DEBUG
  40#endif
  41#undef CARM_NDEBUG
  42
  43#define DRV_NAME "sx8"
  44#define DRV_VERSION "1.0"
  45#define PFX DRV_NAME ": "
  46
  47MODULE_AUTHOR("Jeff Garzik");
  48MODULE_LICENSE("GPL");
  49MODULE_DESCRIPTION("Promise SATA SX8 block driver");
  50MODULE_VERSION(DRV_VERSION);
  51
  52/*
  53 * SX8 hardware has a single message queue for all ATA ports.
  54 * When this driver was written, the hardware (firmware?) would
  55 * corrupt data eventually, if more than one request was outstanding.
  56 * As one can imagine, having 8 ports bottlenecking on a single
  57 * command hurts performance.
  58 *
  59 * Based on user reports, later versions of the hardware (firmware?)
  60 * seem to be able to survive with more than one command queued.
  61 *
  62 * Therefore, we default to the safe option -- 1 command -- but
  63 * allow the user to increase this.
  64 *
  65 * SX8 should be able to support up to ~60 queued commands (CARM_MAX_REQ),
  66 * but problems seem to occur when you exceed ~30, even on newer hardware.
  67 */
  68static int max_queue = 1;
  69module_param(max_queue, int, 0444);
  70MODULE_PARM_DESC(max_queue, "Maximum number of queued commands. (min==1, max==30, safe==1)");
  71
  72
  73#define NEXT_RESP(idx)  ((idx + 1) % RMSG_Q_LEN)
  74
  75/* 0xf is just arbitrary, non-zero noise; this is sorta like poisoning */
  76#define TAG_ENCODE(tag) (((tag) << 16) | 0xf)
  77#define TAG_DECODE(tag) (((tag) >> 16) & 0x1f)
  78#define TAG_VALID(tag)  ((((tag) & 0xf) == 0xf) && (TAG_DECODE(tag) < 32))
  79
  80/* note: prints function name for you */
  81#ifdef CARM_DEBUG
  82#define DPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __func__, ## args)
  83#ifdef CARM_VERBOSE_DEBUG
  84#define VPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __func__, ## args)
  85#else
  86#define VPRINTK(fmt, args...)
  87#endif  /* CARM_VERBOSE_DEBUG */
  88#else
  89#define DPRINTK(fmt, args...)
  90#define VPRINTK(fmt, args...)
  91#endif  /* CARM_DEBUG */
  92
  93#ifdef CARM_NDEBUG
  94#define assert(expr)
  95#else
  96#define assert(expr) \
  97        if(unlikely(!(expr))) {                                   \
  98        printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
  99        #expr, __FILE__, __func__, __LINE__);          \
 100        }
 101#endif
 102
 103/* defines only for the constants which don't work well as enums */
 104struct carm_host;
 105
 106enum {
 107        /* adapter-wide limits */
 108        CARM_MAX_PORTS          = 8,
 109        CARM_SHM_SIZE           = (4096 << 7),
 110        CARM_MINORS_PER_MAJOR   = 256 / CARM_MAX_PORTS,
 111        CARM_MAX_WAIT_Q         = CARM_MAX_PORTS + 1,
 112
 113        /* command message queue limits */
 114        CARM_MAX_REQ            = 64,          /* max command msgs per host */
 115        CARM_MSG_LOW_WATER      = (CARM_MAX_REQ / 4),        /* refill mark */
 116
 117        /* S/G limits, host-wide and per-request */
 118        CARM_MAX_REQ_SG         = 32,        /* max s/g entries per request */
 119        CARM_MAX_HOST_SG        = 600,          /* max s/g entries per host */
 120        CARM_SG_LOW_WATER       = (CARM_MAX_HOST_SG / 4),   /* re-fill mark */
 121
 122        /* hardware registers */
 123        CARM_IHQP               = 0x1c,
 124        CARM_INT_STAT           = 0x10, /* interrupt status */
 125        CARM_INT_MASK           = 0x14, /* interrupt mask */
 126        CARM_HMUC               = 0x18, /* host message unit control */
 127        RBUF_ADDR_LO            = 0x20, /* response msg DMA buf low 32 bits */
 128        RBUF_ADDR_HI            = 0x24, /* response msg DMA buf high 32 bits */
 129        RBUF_BYTE_SZ            = 0x28,
 130        CARM_RESP_IDX           = 0x2c,
 131        CARM_CMS0               = 0x30, /* command message size reg 0 */
 132        CARM_LMUC               = 0x48,
 133        CARM_HMPHA              = 0x6c,
 134        CARM_INITC              = 0xb5,
 135
 136        /* bits in CARM_INT_{STAT,MASK} */
 137        INT_RESERVED            = 0xfffffff0,
 138        INT_WATCHDOG            = (1 << 3),     /* watchdog timer */
 139        INT_Q_OVERFLOW          = (1 << 2),     /* cmd msg q overflow */
 140        INT_Q_AVAILABLE         = (1 << 1),     /* cmd msg q has free space */
 141        INT_RESPONSE            = (1 << 0),     /* response msg available */
 142        INT_ACK_MASK            = INT_WATCHDOG | INT_Q_OVERFLOW,
 143        INT_DEF_MASK            = INT_RESERVED | INT_Q_OVERFLOW |
 144                                  INT_RESPONSE,
 145
 146        /* command messages, and related register bits */
 147        CARM_HAVE_RESP          = 0x01,
 148        CARM_MSG_READ           = 1,
 149        CARM_MSG_WRITE          = 2,
 150        CARM_MSG_VERIFY         = 3,
 151        CARM_MSG_GET_CAPACITY   = 4,
 152        CARM_MSG_FLUSH          = 5,
 153        CARM_MSG_IOCTL          = 6,
 154        CARM_MSG_ARRAY          = 8,
 155        CARM_MSG_MISC           = 9,
 156        CARM_CME                = (1 << 2),
 157        CARM_RME                = (1 << 1),
 158        CARM_WZBC               = (1 << 0),
 159        CARM_RMI                = (1 << 0),
 160        CARM_Q_FULL             = (1 << 3),
 161        CARM_MSG_SIZE           = 288,
 162        CARM_Q_LEN              = 48,
 163
 164        /* CARM_MSG_IOCTL messages */
 165        CARM_IOC_SCAN_CHAN      = 5,    /* scan channels for devices */
 166        CARM_IOC_GET_TCQ        = 13,   /* get tcq/ncq depth */
 167        CARM_IOC_SET_TCQ        = 14,   /* set tcq/ncq depth */
 168
 169        IOC_SCAN_CHAN_NODEV     = 0x1f,
 170        IOC_SCAN_CHAN_OFFSET    = 0x40,
 171
 172        /* CARM_MSG_ARRAY messages */
 173        CARM_ARRAY_INFO         = 0,
 174
 175        ARRAY_NO_EXIST          = (1 << 31),
 176
 177        /* response messages */
 178        RMSG_SZ                 = 8,    /* sizeof(struct carm_response) */
 179        RMSG_Q_LEN              = 48,   /* resp. msg list length */
 180        RMSG_OK                 = 1,    /* bit indicating msg was successful */
 181                                        /* length of entire resp. msg buffer */
 182        RBUF_LEN                = RMSG_SZ * RMSG_Q_LEN,
 183
 184        PDC_SHM_SIZE            = (4096 << 7), /* length of entire h/w buffer */
 185
 186        /* CARM_MSG_MISC messages */
 187        MISC_GET_FW_VER         = 2,
 188        MISC_ALLOC_MEM          = 3,
 189        MISC_SET_TIME           = 5,
 190
 191        /* MISC_GET_FW_VER feature bits */
 192        FW_VER_4PORT            = (1 << 2), /* 1=4 ports, 0=8 ports */
 193        FW_VER_NON_RAID         = (1 << 1), /* 1=non-RAID firmware, 0=RAID */
 194        FW_VER_ZCR              = (1 << 0), /* zero channel RAID (whatever that is) */
 195
 196        /* carm_host flags */
 197        FL_NON_RAID             = FW_VER_NON_RAID,
 198        FL_4PORT                = FW_VER_4PORT,
 199        FL_FW_VER_MASK          = (FW_VER_NON_RAID | FW_VER_4PORT),
 200        FL_DYN_MAJOR            = (1 << 17),
 201};
 202
 203enum {
 204        CARM_SG_BOUNDARY        = 0xffffUL,         /* s/g segment boundary */
 205};
 206
 207enum scatter_gather_types {
 208        SGT_32BIT               = 0,
 209        SGT_64BIT               = 1,
 210};
 211
 212enum host_states {
 213        HST_INVALID,            /* invalid state; never used */
 214        HST_ALLOC_BUF,          /* setting up master SHM area */
 215        HST_ERROR,              /* we never leave here */
 216        HST_PORT_SCAN,          /* start dev scan */
 217        HST_DEV_SCAN_START,     /* start per-device probe */
 218        HST_DEV_SCAN,           /* continue per-device probe */
 219        HST_DEV_ACTIVATE,       /* activate devices we found */
 220        HST_PROBE_FINISHED,     /* probe is complete */
 221        HST_PROBE_START,        /* initiate probe */
 222        HST_SYNC_TIME,          /* tell firmware what time it is */
 223        HST_GET_FW_VER,         /* get firmware version, adapter port cnt */
 224};
 225
 226#ifdef CARM_DEBUG
 227static const char *state_name[] = {
 228        "HST_INVALID",
 229        "HST_ALLOC_BUF",
 230        "HST_ERROR",
 231        "HST_PORT_SCAN",
 232        "HST_DEV_SCAN_START",
 233        "HST_DEV_SCAN",
 234        "HST_DEV_ACTIVATE",
 235        "HST_PROBE_FINISHED",
 236        "HST_PROBE_START",
 237        "HST_SYNC_TIME",
 238        "HST_GET_FW_VER",
 239};
 240#endif
 241
 242struct carm_port {
 243        unsigned int                    port_no;
 244        struct gendisk                  *disk;
 245        struct carm_host                *host;
 246
 247        /* attached device characteristics */
 248        u64                             capacity;
 249        char                            name[41];
 250        u16                             dev_geom_head;
 251        u16                             dev_geom_sect;
 252        u16                             dev_geom_cyl;
 253};
 254
 255struct carm_request {
 256        int                             n_elem;
 257        unsigned int                    msg_type;
 258        unsigned int                    msg_subtype;
 259        unsigned int                    msg_bucket;
 260        struct scatterlist              sg[CARM_MAX_REQ_SG];
 261};
 262
 263struct carm_host {
 264        unsigned long                   flags;
 265        void                            __iomem *mmio;
 266        void                            *shm;
 267        dma_addr_t                      shm_dma;
 268
 269        int                             major;
 270        int                             id;
 271        char                            name[32];
 272
 273        spinlock_t                      lock;
 274        struct pci_dev                  *pdev;
 275        unsigned int                    state;
 276        u32                             fw_ver;
 277
 278        struct blk_mq_tag_set           tag_set;
 279        struct request_queue            *oob_q;
 280        unsigned int                    n_oob;
 281
 282        unsigned int                    hw_sg_used;
 283
 284        unsigned int                    resp_idx;
 285
 286        unsigned int                    wait_q_prod;
 287        unsigned int                    wait_q_cons;
 288        struct request_queue            *wait_q[CARM_MAX_WAIT_Q];
 289
 290        void                            *msg_base;
 291        dma_addr_t                      msg_dma;
 292
 293        int                             cur_scan_dev;
 294        unsigned long                   dev_active;
 295        unsigned long                   dev_present;
 296        struct carm_port                port[CARM_MAX_PORTS];
 297
 298        struct work_struct              fsm_task;
 299
 300        struct completion               probe_comp;
 301};
 302
 303struct carm_response {
 304        __le32 ret_handle;
 305        __le32 status;
 306}  __attribute__((packed));
 307
 308struct carm_msg_sg {
 309        __le32 start;
 310        __le32 len;
 311}  __attribute__((packed));
 312
 313struct carm_msg_rw {
 314        u8 type;
 315        u8 id;
 316        u8 sg_count;
 317        u8 sg_type;
 318        __le32 handle;
 319        __le32 lba;
 320        __le16 lba_count;
 321        __le16 lba_high;
 322        struct carm_msg_sg sg[32];
 323}  __attribute__((packed));
 324
 325struct carm_msg_allocbuf {
 326        u8 type;
 327        u8 subtype;
 328        u8 n_sg;
 329        u8 sg_type;
 330        __le32 handle;
 331        __le32 addr;
 332        __le32 len;
 333        __le32 evt_pool;
 334        __le32 n_evt;
 335        __le32 rbuf_pool;
 336        __le32 n_rbuf;
 337        __le32 msg_pool;
 338        __le32 n_msg;
 339        struct carm_msg_sg sg[8];
 340}  __attribute__((packed));
 341
 342struct carm_msg_ioctl {
 343        u8 type;
 344        u8 subtype;
 345        u8 array_id;
 346        u8 reserved1;
 347        __le32 handle;
 348        __le32 data_addr;
 349        u32 reserved2;
 350}  __attribute__((packed));
 351
 352struct carm_msg_sync_time {
 353        u8 type;
 354        u8 subtype;
 355        u16 reserved1;
 356        __le32 handle;
 357        u32 reserved2;
 358        __le32 timestamp;
 359}  __attribute__((packed));
 360
 361struct carm_msg_get_fw_ver {
 362        u8 type;
 363        u8 subtype;
 364        u16 reserved1;
 365        __le32 handle;
 366        __le32 data_addr;
 367        u32 reserved2;
 368}  __attribute__((packed));
 369
 370struct carm_fw_ver {
 371        __le32 version;
 372        u8 features;
 373        u8 reserved1;
 374        u16 reserved2;
 375}  __attribute__((packed));
 376
 377struct carm_array_info {
 378        __le32 size;
 379
 380        __le16 size_hi;
 381        __le16 stripe_size;
 382
 383        __le32 mode;
 384
 385        __le16 stripe_blk_sz;
 386        __le16 reserved1;
 387
 388        __le16 cyl;
 389        __le16 head;
 390
 391        __le16 sect;
 392        u8 array_id;
 393        u8 reserved2;
 394
 395        char name[40];
 396
 397        __le32 array_status;
 398
 399        /* device list continues beyond this point? */
 400}  __attribute__((packed));
 401
 402static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
 403static void carm_remove_one (struct pci_dev *pdev);
 404static int carm_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo);
 405
 406static const struct pci_device_id carm_pci_tbl[] = {
 407        { PCI_VENDOR_ID_PROMISE, 0x8000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
 408        { PCI_VENDOR_ID_PROMISE, 0x8002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
 409        { }     /* terminate list */
 410};
 411MODULE_DEVICE_TABLE(pci, carm_pci_tbl);
 412
 413static struct pci_driver carm_driver = {
 414        .name           = DRV_NAME,
 415        .id_table       = carm_pci_tbl,
 416        .probe          = carm_init_one,
 417        .remove         = carm_remove_one,
 418};
 419
 420static const struct block_device_operations carm_bd_ops = {
 421        .owner          = THIS_MODULE,
 422        .getgeo         = carm_bdev_getgeo,
 423};
 424
 425static unsigned int carm_host_id;
 426static unsigned long carm_major_alloc;
 427
 428
 429
 430static int carm_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 431{
 432        struct carm_port *port = bdev->bd_disk->private_data;
 433
 434        geo->heads = (u8) port->dev_geom_head;
 435        geo->sectors = (u8) port->dev_geom_sect;
 436        geo->cylinders = port->dev_geom_cyl;
 437        return 0;
 438}
 439
 440static const u32 msg_sizes[] = { 32, 64, 128, CARM_MSG_SIZE };
 441
 442static inline int carm_lookup_bucket(u32 msg_size)
 443{
 444        int i;
 445
 446        for (i = 0; i < ARRAY_SIZE(msg_sizes); i++)
 447                if (msg_size <= msg_sizes[i])
 448                        return i;
 449
 450        return -ENOENT;
 451}
 452
 453static void carm_init_buckets(void __iomem *mmio)
 454{
 455        unsigned int i;
 456
 457        for (i = 0; i < ARRAY_SIZE(msg_sizes); i++)
 458                writel(msg_sizes[i], mmio + CARM_CMS0 + (4 * i));
 459}
 460
 461static inline void *carm_ref_msg(struct carm_host *host,
 462                                 unsigned int msg_idx)
 463{
 464        return host->msg_base + (msg_idx * CARM_MSG_SIZE);
 465}
 466
 467static inline dma_addr_t carm_ref_msg_dma(struct carm_host *host,
 468                                          unsigned int msg_idx)
 469{
 470        return host->msg_dma + (msg_idx * CARM_MSG_SIZE);
 471}
 472
 473static int carm_send_msg(struct carm_host *host,
 474                         struct carm_request *crq, unsigned tag)
 475{
 476        void __iomem *mmio = host->mmio;
 477        u32 msg = (u32) carm_ref_msg_dma(host, tag);
 478        u32 cm_bucket = crq->msg_bucket;
 479        u32 tmp;
 480        int rc = 0;
 481
 482        VPRINTK("ENTER\n");
 483
 484        tmp = readl(mmio + CARM_HMUC);
 485        if (tmp & CARM_Q_FULL) {
 486#if 0
 487                tmp = readl(mmio + CARM_INT_MASK);
 488                tmp |= INT_Q_AVAILABLE;
 489                writel(tmp, mmio + CARM_INT_MASK);
 490                readl(mmio + CARM_INT_MASK);    /* flush */
 491#endif
 492                DPRINTK("host msg queue full\n");
 493                rc = -EBUSY;
 494        } else {
 495                writel(msg | (cm_bucket << 1), mmio + CARM_IHQP);
 496                readl(mmio + CARM_IHQP);        /* flush */
 497        }
 498
 499        return rc;
 500}
 501
 502static int carm_array_info (struct carm_host *host, unsigned int array_idx)
 503{
 504        struct carm_msg_ioctl *ioc;
 505        u32 msg_data;
 506        dma_addr_t msg_dma;
 507        struct carm_request *crq;
 508        struct request *rq;
 509        int rc;
 510
 511        rq = blk_mq_alloc_request(host->oob_q, REQ_OP_DRV_OUT, 0);
 512        if (IS_ERR(rq)) {
 513                rc = -ENOMEM;
 514                goto err_out;
 515        }
 516        crq = blk_mq_rq_to_pdu(rq);
 517
 518        ioc = carm_ref_msg(host, rq->tag);
 519        msg_dma = carm_ref_msg_dma(host, rq->tag);
 520        msg_data = (u32) (msg_dma + sizeof(struct carm_array_info));
 521
 522        crq->msg_type = CARM_MSG_ARRAY;
 523        crq->msg_subtype = CARM_ARRAY_INFO;
 524        rc = carm_lookup_bucket(sizeof(struct carm_msg_ioctl) +
 525                                sizeof(struct carm_array_info));
 526        BUG_ON(rc < 0);
 527        crq->msg_bucket = (u32) rc;
 528
 529        memset(ioc, 0, sizeof(*ioc));
 530        ioc->type       = CARM_MSG_ARRAY;
 531        ioc->subtype    = CARM_ARRAY_INFO;
 532        ioc->array_id   = (u8) array_idx;
 533        ioc->handle     = cpu_to_le32(TAG_ENCODE(rq->tag));
 534        ioc->data_addr  = cpu_to_le32(msg_data);
 535
 536        spin_lock_irq(&host->lock);
 537        assert(host->state == HST_DEV_SCAN_START ||
 538               host->state == HST_DEV_SCAN);
 539        spin_unlock_irq(&host->lock);
 540
 541        DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag);
 542        blk_execute_rq_nowait(NULL, rq, true, NULL);
 543
 544        return 0;
 545
 546err_out:
 547        spin_lock_irq(&host->lock);
 548        host->state = HST_ERROR;
 549        spin_unlock_irq(&host->lock);
 550        return rc;
 551}
 552
 553typedef unsigned int (*carm_sspc_t)(struct carm_host *, unsigned int, void *);
 554
 555static int carm_send_special (struct carm_host *host, carm_sspc_t func)
 556{
 557        struct request *rq;
 558        struct carm_request *crq;
 559        struct carm_msg_ioctl *ioc;
 560        void *mem;
 561        unsigned int msg_size;
 562        int rc;
 563
 564        rq = blk_mq_alloc_request(host->oob_q, REQ_OP_DRV_OUT, 0);
 565        if (IS_ERR(rq))
 566                return -ENOMEM;
 567        crq = blk_mq_rq_to_pdu(rq);
 568
 569        mem = carm_ref_msg(host, rq->tag);
 570
 571        msg_size = func(host, rq->tag, mem);
 572
 573        ioc = mem;
 574        crq->msg_type = ioc->type;
 575        crq->msg_subtype = ioc->subtype;
 576        rc = carm_lookup_bucket(msg_size);
 577        BUG_ON(rc < 0);
 578        crq->msg_bucket = (u32) rc;
 579
 580        DPRINTK("blk_execute_rq_nowait, tag == %u\n", rq->tag);
 581        blk_execute_rq_nowait(NULL, rq, true, NULL);
 582
 583        return 0;
 584}
 585
 586static unsigned int carm_fill_sync_time(struct carm_host *host,
 587                                        unsigned int idx, void *mem)
 588{
 589        struct carm_msg_sync_time *st = mem;
 590
 591        time64_t tv = ktime_get_real_seconds();
 592
 593        memset(st, 0, sizeof(*st));
 594        st->type        = CARM_MSG_MISC;
 595        st->subtype     = MISC_SET_TIME;
 596        st->handle      = cpu_to_le32(TAG_ENCODE(idx));
 597        st->timestamp   = cpu_to_le32(tv);
 598
 599        return sizeof(struct carm_msg_sync_time);
 600}
 601
 602static unsigned int carm_fill_alloc_buf(struct carm_host *host,
 603                                        unsigned int idx, void *mem)
 604{
 605        struct carm_msg_allocbuf *ab = mem;
 606
 607        memset(ab, 0, sizeof(*ab));
 608        ab->type        = CARM_MSG_MISC;
 609        ab->subtype     = MISC_ALLOC_MEM;
 610        ab->handle      = cpu_to_le32(TAG_ENCODE(idx));
 611        ab->n_sg        = 1;
 612        ab->sg_type     = SGT_32BIT;
 613        ab->addr        = cpu_to_le32(host->shm_dma + (PDC_SHM_SIZE >> 1));
 614        ab->len         = cpu_to_le32(PDC_SHM_SIZE >> 1);
 615        ab->evt_pool    = cpu_to_le32(host->shm_dma + (16 * 1024));
 616        ab->n_evt       = cpu_to_le32(1024);
 617        ab->rbuf_pool   = cpu_to_le32(host->shm_dma);
 618        ab->n_rbuf      = cpu_to_le32(RMSG_Q_LEN);
 619        ab->msg_pool    = cpu_to_le32(host->shm_dma + RBUF_LEN);
 620        ab->n_msg       = cpu_to_le32(CARM_Q_LEN);
 621        ab->sg[0].start = cpu_to_le32(host->shm_dma + (PDC_SHM_SIZE >> 1));
 622        ab->sg[0].len   = cpu_to_le32(65536);
 623
 624        return sizeof(struct carm_msg_allocbuf);
 625}
 626
 627static unsigned int carm_fill_scan_channels(struct carm_host *host,
 628                                            unsigned int idx, void *mem)
 629{
 630        struct carm_msg_ioctl *ioc = mem;
 631        u32 msg_data = (u32) (carm_ref_msg_dma(host, idx) +
 632                              IOC_SCAN_CHAN_OFFSET);
 633
 634        memset(ioc, 0, sizeof(*ioc));
 635        ioc->type       = CARM_MSG_IOCTL;
 636        ioc->subtype    = CARM_IOC_SCAN_CHAN;
 637        ioc->handle     = cpu_to_le32(TAG_ENCODE(idx));
 638        ioc->data_addr  = cpu_to_le32(msg_data);
 639
 640        /* fill output data area with "no device" default values */
 641        mem += IOC_SCAN_CHAN_OFFSET;
 642        memset(mem, IOC_SCAN_CHAN_NODEV, CARM_MAX_PORTS);
 643
 644        return IOC_SCAN_CHAN_OFFSET + CARM_MAX_PORTS;
 645}
 646
 647static unsigned int carm_fill_get_fw_ver(struct carm_host *host,
 648                                         unsigned int idx, void *mem)
 649{
 650        struct carm_msg_get_fw_ver *ioc = mem;
 651        u32 msg_data = (u32) (carm_ref_msg_dma(host, idx) + sizeof(*ioc));
 652
 653        memset(ioc, 0, sizeof(*ioc));
 654        ioc->type       = CARM_MSG_MISC;
 655        ioc->subtype    = MISC_GET_FW_VER;
 656        ioc->handle     = cpu_to_le32(TAG_ENCODE(idx));
 657        ioc->data_addr  = cpu_to_le32(msg_data);
 658
 659        return sizeof(struct carm_msg_get_fw_ver) +
 660               sizeof(struct carm_fw_ver);
 661}
 662
 663static inline void carm_push_q (struct carm_host *host, struct request_queue *q)
 664{
 665        unsigned int idx = host->wait_q_prod % CARM_MAX_WAIT_Q;
 666
 667        blk_mq_stop_hw_queues(q);
 668        VPRINTK("STOPPED QUEUE %p\n", q);
 669
 670        host->wait_q[idx] = q;
 671        host->wait_q_prod++;
 672        BUG_ON(host->wait_q_prod == host->wait_q_cons); /* overrun */
 673}
 674
 675static inline struct request_queue *carm_pop_q(struct carm_host *host)
 676{
 677        unsigned int idx;
 678
 679        if (host->wait_q_prod == host->wait_q_cons)
 680                return NULL;
 681
 682        idx = host->wait_q_cons % CARM_MAX_WAIT_Q;
 683        host->wait_q_cons++;
 684
 685        return host->wait_q[idx];
 686}
 687
 688static inline void carm_round_robin(struct carm_host *host)
 689{
 690        struct request_queue *q = carm_pop_q(host);
 691        if (q) {
 692                blk_mq_start_hw_queues(q);
 693                VPRINTK("STARTED QUEUE %p\n", q);
 694        }
 695}
 696
 697static inline enum dma_data_direction carm_rq_dir(struct request *rq)
 698{
 699        return op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 700}
 701
 702static blk_status_t carm_queue_rq(struct blk_mq_hw_ctx *hctx,
 703                                  const struct blk_mq_queue_data *bd)
 704{
 705        struct request_queue *q = hctx->queue;
 706        struct request *rq = bd->rq;
 707        struct carm_port *port = q->queuedata;
 708        struct carm_host *host = port->host;
 709        struct carm_request *crq = blk_mq_rq_to_pdu(rq);
 710        struct carm_msg_rw *msg;
 711        struct scatterlist *sg;
 712        int i, n_elem = 0, rc;
 713        unsigned int msg_size;
 714        u32 tmp;
 715
 716        crq->n_elem = 0;
 717        sg_init_table(crq->sg, CARM_MAX_REQ_SG);
 718
 719        blk_mq_start_request(rq);
 720
 721        spin_lock_irq(&host->lock);
 722        if (req_op(rq) == REQ_OP_DRV_OUT)
 723                goto send_msg;
 724
 725        /* get scatterlist from block layer */
 726        sg = &crq->sg[0];
 727        n_elem = blk_rq_map_sg(q, rq, sg);
 728        if (n_elem <= 0)
 729                goto out_ioerr;
 730
 731        /* map scatterlist to PCI bus addresses */
 732        n_elem = dma_map_sg(&host->pdev->dev, sg, n_elem, carm_rq_dir(rq));
 733        if (n_elem <= 0)
 734                goto out_ioerr;
 735
 736        /* obey global hardware limit on S/G entries */
 737        if (host->hw_sg_used >= CARM_MAX_HOST_SG - n_elem)
 738                goto out_resource;
 739
 740        crq->n_elem = n_elem;
 741        host->hw_sg_used += n_elem;
 742
 743        /*
 744         * build read/write message
 745         */
 746
 747        VPRINTK("build msg\n");
 748        msg = (struct carm_msg_rw *) carm_ref_msg(host, rq->tag);
 749
 750        if (rq_data_dir(rq) == WRITE) {
 751                msg->type = CARM_MSG_WRITE;
 752                crq->msg_type = CARM_MSG_WRITE;
 753        } else {
 754                msg->type = CARM_MSG_READ;
 755                crq->msg_type = CARM_MSG_READ;
 756        }
 757
 758        msg->id         = port->port_no;
 759        msg->sg_count   = n_elem;
 760        msg->sg_type    = SGT_32BIT;
 761        msg->handle     = cpu_to_le32(TAG_ENCODE(rq->tag));
 762        msg->lba        = cpu_to_le32(blk_rq_pos(rq) & 0xffffffff);
 763        tmp             = (blk_rq_pos(rq) >> 16) >> 16;
 764        msg->lba_high   = cpu_to_le16( (u16) tmp );
 765        msg->lba_count  = cpu_to_le16(blk_rq_sectors(rq));
 766
 767        msg_size = sizeof(struct carm_msg_rw) - sizeof(msg->sg);
 768        for (i = 0; i < n_elem; i++) {
 769                struct carm_msg_sg *carm_sg = &msg->sg[i];
 770                carm_sg->start = cpu_to_le32(sg_dma_address(&crq->sg[i]));
 771                carm_sg->len = cpu_to_le32(sg_dma_len(&crq->sg[i]));
 772                msg_size += sizeof(struct carm_msg_sg);
 773        }
 774
 775        rc = carm_lookup_bucket(msg_size);
 776        BUG_ON(rc < 0);
 777        crq->msg_bucket = (u32) rc;
 778send_msg:
 779        /*
 780         * queue read/write message to hardware
 781         */
 782        VPRINTK("send msg, tag == %u\n", rq->tag);
 783        rc = carm_send_msg(host, crq, rq->tag);
 784        if (rc) {
 785                host->hw_sg_used -= n_elem;
 786                goto out_resource;
 787        }
 788
 789        spin_unlock_irq(&host->lock);
 790        return BLK_STS_OK;
 791out_resource:
 792        dma_unmap_sg(&host->pdev->dev, &crq->sg[0], n_elem, carm_rq_dir(rq));
 793        carm_push_q(host, q);
 794        spin_unlock_irq(&host->lock);
 795        return BLK_STS_DEV_RESOURCE;
 796out_ioerr:
 797        carm_round_robin(host);
 798        spin_unlock_irq(&host->lock);
 799        return BLK_STS_IOERR;
 800}
 801
 802static void carm_handle_array_info(struct carm_host *host,
 803                                   struct carm_request *crq, u8 *mem,
 804                                   blk_status_t error)
 805{
 806        struct carm_port *port;
 807        u8 *msg_data = mem + sizeof(struct carm_array_info);
 808        struct carm_array_info *desc = (struct carm_array_info *) msg_data;
 809        u64 lo, hi;
 810        int cur_port;
 811        size_t slen;
 812
 813        DPRINTK("ENTER\n");
 814
 815        if (error)
 816                goto out;
 817        if (le32_to_cpu(desc->array_status) & ARRAY_NO_EXIST)
 818                goto out;
 819
 820        cur_port = host->cur_scan_dev;
 821
 822        /* should never occur */
 823        if ((cur_port < 0) || (cur_port >= CARM_MAX_PORTS)) {
 824                printk(KERN_ERR PFX "BUG: cur_scan_dev==%d, array_id==%d\n",
 825                       cur_port, (int) desc->array_id);
 826                goto out;
 827        }
 828
 829        port = &host->port[cur_port];
 830
 831        lo = (u64) le32_to_cpu(desc->size);
 832        hi = (u64) le16_to_cpu(desc->size_hi);
 833
 834        port->capacity = lo | (hi << 32);
 835        port->dev_geom_head = le16_to_cpu(desc->head);
 836        port->dev_geom_sect = le16_to_cpu(desc->sect);
 837        port->dev_geom_cyl = le16_to_cpu(desc->cyl);
 838
 839        host->dev_active |= (1 << cur_port);
 840
 841        strncpy(port->name, desc->name, sizeof(port->name));
 842        port->name[sizeof(port->name) - 1] = 0;
 843        slen = strlen(port->name);
 844        while (slen && (port->name[slen - 1] == ' ')) {
 845                port->name[slen - 1] = 0;
 846                slen--;
 847        }
 848
 849        printk(KERN_INFO DRV_NAME "(%s): port %u device %Lu sectors\n",
 850               pci_name(host->pdev), port->port_no,
 851               (unsigned long long) port->capacity);
 852        printk(KERN_INFO DRV_NAME "(%s): port %u device \"%s\"\n",
 853               pci_name(host->pdev), port->port_no, port->name);
 854
 855out:
 856        assert(host->state == HST_DEV_SCAN);
 857        schedule_work(&host->fsm_task);
 858}
 859
 860static void carm_handle_scan_chan(struct carm_host *host,
 861                                  struct carm_request *crq, u8 *mem,
 862                                  blk_status_t error)
 863{
 864        u8 *msg_data = mem + IOC_SCAN_CHAN_OFFSET;
 865        unsigned int i, dev_count = 0;
 866        int new_state = HST_DEV_SCAN_START;
 867
 868        DPRINTK("ENTER\n");
 869
 870        if (error) {
 871                new_state = HST_ERROR;
 872                goto out;
 873        }
 874
 875        /* TODO: scan and support non-disk devices */
 876        for (i = 0; i < 8; i++)
 877                if (msg_data[i] == 0) { /* direct-access device (disk) */
 878                        host->dev_present |= (1 << i);
 879                        dev_count++;
 880                }
 881
 882        printk(KERN_INFO DRV_NAME "(%s): found %u interesting devices\n",
 883               pci_name(host->pdev), dev_count);
 884
 885out:
 886        assert(host->state == HST_PORT_SCAN);
 887        host->state = new_state;
 888        schedule_work(&host->fsm_task);
 889}
 890
 891static void carm_handle_generic(struct carm_host *host,
 892                                struct carm_request *crq, blk_status_t error,
 893                                int cur_state, int next_state)
 894{
 895        DPRINTK("ENTER\n");
 896
 897        assert(host->state == cur_state);
 898        if (error)
 899                host->state = HST_ERROR;
 900        else
 901                host->state = next_state;
 902        schedule_work(&host->fsm_task);
 903}
 904
 905static inline void carm_handle_resp(struct carm_host *host,
 906                                    __le32 ret_handle_le, u32 status)
 907{
 908        u32 handle = le32_to_cpu(ret_handle_le);
 909        unsigned int msg_idx;
 910        struct request *rq;
 911        struct carm_request *crq;
 912        blk_status_t error = (status == RMSG_OK) ? 0 : BLK_STS_IOERR;
 913        u8 *mem;
 914
 915        VPRINTK("ENTER, handle == 0x%x\n", handle);
 916
 917        if (unlikely(!TAG_VALID(handle))) {
 918                printk(KERN_ERR DRV_NAME "(%s): BUG: invalid tag 0x%x\n",
 919                       pci_name(host->pdev), handle);
 920                return;
 921        }
 922
 923        msg_idx = TAG_DECODE(handle);
 924        VPRINTK("tag == %u\n", msg_idx);
 925
 926        rq = blk_mq_tag_to_rq(host->tag_set.tags[0], msg_idx);
 927        crq = blk_mq_rq_to_pdu(rq);
 928
 929        /* fast path */
 930        if (likely(crq->msg_type == CARM_MSG_READ ||
 931                   crq->msg_type == CARM_MSG_WRITE)) {
 932                dma_unmap_sg(&host->pdev->dev, &crq->sg[0], crq->n_elem,
 933                             carm_rq_dir(rq));
 934                goto done;
 935        }
 936
 937        mem = carm_ref_msg(host, msg_idx);
 938
 939        switch (crq->msg_type) {
 940        case CARM_MSG_IOCTL: {
 941                switch (crq->msg_subtype) {
 942                case CARM_IOC_SCAN_CHAN:
 943                        carm_handle_scan_chan(host, crq, mem, error);
 944                        goto done;
 945                default:
 946                        /* unknown / invalid response */
 947                        goto err_out;
 948                }
 949                break;
 950        }
 951
 952        case CARM_MSG_MISC: {
 953                switch (crq->msg_subtype) {
 954                case MISC_ALLOC_MEM:
 955                        carm_handle_generic(host, crq, error,
 956                                            HST_ALLOC_BUF, HST_SYNC_TIME);
 957                        goto done;
 958                case MISC_SET_TIME:
 959                        carm_handle_generic(host, crq, error,
 960                                            HST_SYNC_TIME, HST_GET_FW_VER);
 961                        goto done;
 962                case MISC_GET_FW_VER: {
 963                        struct carm_fw_ver *ver = (struct carm_fw_ver *)
 964                                (mem + sizeof(struct carm_msg_get_fw_ver));
 965                        if (!error) {
 966                                host->fw_ver = le32_to_cpu(ver->version);
 967                                host->flags |= (ver->features & FL_FW_VER_MASK);
 968                        }
 969                        carm_handle_generic(host, crq, error,
 970                                            HST_GET_FW_VER, HST_PORT_SCAN);
 971                        goto done;
 972                }
 973                default:
 974                        /* unknown / invalid response */
 975                        goto err_out;
 976                }
 977                break;
 978        }
 979
 980        case CARM_MSG_ARRAY: {
 981                switch (crq->msg_subtype) {
 982                case CARM_ARRAY_INFO:
 983                        carm_handle_array_info(host, crq, mem, error);
 984                        break;
 985                default:
 986                        /* unknown / invalid response */
 987                        goto err_out;
 988                }
 989                break;
 990        }
 991
 992        default:
 993                /* unknown / invalid response */
 994                goto err_out;
 995        }
 996
 997        return;
 998
 999err_out:
1000        printk(KERN_WARNING DRV_NAME "(%s): BUG: unhandled message type %d/%d\n",
1001               pci_name(host->pdev), crq->msg_type, crq->msg_subtype);
1002        error = BLK_STS_IOERR;
1003done:
1004        host->hw_sg_used -= crq->n_elem;
1005        blk_mq_end_request(blk_mq_rq_from_pdu(crq), error);
1006
1007        if (host->hw_sg_used <= CARM_SG_LOW_WATER)
1008                carm_round_robin(host);
1009}
1010
1011static inline void carm_handle_responses(struct carm_host *host)
1012{
1013        void __iomem *mmio = host->mmio;
1014        struct carm_response *resp = (struct carm_response *) host->shm;
1015        unsigned int work = 0;
1016        unsigned int idx = host->resp_idx % RMSG_Q_LEN;
1017
1018        while (1) {
1019                u32 status = le32_to_cpu(resp[idx].status);
1020
1021                if (status == 0xffffffff) {
1022                        VPRINTK("ending response on index %u\n", idx);
1023                        writel(idx << 3, mmio + CARM_RESP_IDX);
1024                        break;
1025                }
1026
1027                /* response to a message we sent */
1028                else if ((status & (1 << 31)) == 0) {
1029                        VPRINTK("handling msg response on index %u\n", idx);
1030                        carm_handle_resp(host, resp[idx].ret_handle, status);
1031                        resp[idx].status = cpu_to_le32(0xffffffff);
1032                }
1033
1034                /* asynchronous events the hardware throws our way */
1035                else if ((status & 0xff000000) == (1 << 31)) {
1036                        u8 *evt_type_ptr = (u8 *) &resp[idx];
1037                        u8 evt_type = *evt_type_ptr;
1038                        printk(KERN_WARNING DRV_NAME "(%s): unhandled event type %d\n",
1039                               pci_name(host->pdev), (int) evt_type);
1040                        resp[idx].status = cpu_to_le32(0xffffffff);
1041                }
1042
1043                idx = NEXT_RESP(idx);
1044                work++;
1045        }
1046
1047        VPRINTK("EXIT, work==%u\n", work);
1048        host->resp_idx += work;
1049}
1050
1051static irqreturn_t carm_interrupt(int irq, void *__host)
1052{
1053        struct carm_host *host = __host;
1054        void __iomem *mmio;
1055        u32 mask;
1056        int handled = 0;
1057        unsigned long flags;
1058
1059        if (!host) {
1060                VPRINTK("no host\n");
1061                return IRQ_NONE;
1062        }
1063
1064        spin_lock_irqsave(&host->lock, flags);
1065
1066        mmio = host->mmio;
1067
1068        /* reading should also clear interrupts */
1069        mask = readl(mmio + CARM_INT_STAT);
1070
1071        if (mask == 0 || mask == 0xffffffff) {
1072                VPRINTK("no work, mask == 0x%x\n", mask);
1073                goto out;
1074        }
1075
1076        if (mask & INT_ACK_MASK)
1077                writel(mask, mmio + CARM_INT_STAT);
1078
1079        if (unlikely(host->state == HST_INVALID)) {
1080                VPRINTK("not initialized yet, mask = 0x%x\n", mask);
1081                goto out;
1082        }
1083
1084        if (mask & CARM_HAVE_RESP) {
1085                handled = 1;
1086                carm_handle_responses(host);
1087        }
1088
1089out:
1090        spin_unlock_irqrestore(&host->lock, flags);
1091        VPRINTK("EXIT\n");
1092        return IRQ_RETVAL(handled);
1093}
1094
1095static void carm_fsm_task (struct work_struct *work)
1096{
1097        struct carm_host *host =
1098                container_of(work, struct carm_host, fsm_task);
1099        unsigned long flags;
1100        unsigned int state;
1101        int rc, i, next_dev;
1102        int reschedule = 0;
1103        int new_state = HST_INVALID;
1104
1105        spin_lock_irqsave(&host->lock, flags);
1106        state = host->state;
1107        spin_unlock_irqrestore(&host->lock, flags);
1108
1109        DPRINTK("ENTER, state == %s\n", state_name[state]);
1110
1111        switch (state) {
1112        case HST_PROBE_START:
1113                new_state = HST_ALLOC_BUF;
1114                reschedule = 1;
1115                break;
1116
1117        case HST_ALLOC_BUF:
1118                rc = carm_send_special(host, carm_fill_alloc_buf);
1119                if (rc) {
1120                        new_state = HST_ERROR;
1121                        reschedule = 1;
1122                }
1123                break;
1124
1125        case HST_SYNC_TIME:
1126                rc = carm_send_special(host, carm_fill_sync_time);
1127                if (rc) {
1128                        new_state = HST_ERROR;
1129                        reschedule = 1;
1130                }
1131                break;
1132
1133        case HST_GET_FW_VER:
1134                rc = carm_send_special(host, carm_fill_get_fw_ver);
1135                if (rc) {
1136                        new_state = HST_ERROR;
1137                        reschedule = 1;
1138                }
1139                break;
1140
1141        case HST_PORT_SCAN:
1142                rc = carm_send_special(host, carm_fill_scan_channels);
1143                if (rc) {
1144                        new_state = HST_ERROR;
1145                        reschedule = 1;
1146                }
1147                break;
1148
1149        case HST_DEV_SCAN_START:
1150                host->cur_scan_dev = -1;
1151                new_state = HST_DEV_SCAN;
1152                reschedule = 1;
1153                break;
1154
1155        case HST_DEV_SCAN:
1156                next_dev = -1;
1157                for (i = host->cur_scan_dev + 1; i < CARM_MAX_PORTS; i++)
1158                        if (host->dev_present & (1 << i)) {
1159                                next_dev = i;
1160                                break;
1161                        }
1162
1163                if (next_dev >= 0) {
1164                        host->cur_scan_dev = next_dev;
1165                        rc = carm_array_info(host, next_dev);
1166                        if (rc) {
1167                                new_state = HST_ERROR;
1168                                reschedule = 1;
1169                        }
1170                } else {
1171                        new_state = HST_DEV_ACTIVATE;
1172                        reschedule = 1;
1173                }
1174                break;
1175
1176        case HST_DEV_ACTIVATE: {
1177                int activated = 0;
1178                for (i = 0; i < CARM_MAX_PORTS; i++)
1179                        if (host->dev_active & (1 << i)) {
1180                                struct carm_port *port = &host->port[i];
1181                                struct gendisk *disk = port->disk;
1182
1183                                set_capacity(disk, port->capacity);
1184                                add_disk(disk);
1185                                activated++;
1186                        }
1187
1188                printk(KERN_INFO DRV_NAME "(%s): %d ports activated\n",
1189                       pci_name(host->pdev), activated);
1190
1191                new_state = HST_PROBE_FINISHED;
1192                reschedule = 1;
1193                break;
1194        }
1195
1196        case HST_PROBE_FINISHED:
1197                complete(&host->probe_comp);
1198                break;
1199
1200        case HST_ERROR:
1201                /* FIXME: TODO */
1202                break;
1203
1204        default:
1205                /* should never occur */
1206                printk(KERN_ERR PFX "BUG: unknown state %d\n", state);
1207                assert(0);
1208                break;
1209        }
1210
1211        if (new_state != HST_INVALID) {
1212                spin_lock_irqsave(&host->lock, flags);
1213                host->state = new_state;
1214                spin_unlock_irqrestore(&host->lock, flags);
1215        }
1216        if (reschedule)
1217                schedule_work(&host->fsm_task);
1218}
1219
1220static int carm_init_wait(void __iomem *mmio, u32 bits, unsigned int test_bit)
1221{
1222        unsigned int i;
1223
1224        for (i = 0; i < 50000; i++) {
1225                u32 tmp = readl(mmio + CARM_LMUC);
1226                udelay(100);
1227
1228                if (test_bit) {
1229                        if ((tmp & bits) == bits)
1230                                return 0;
1231                } else {
1232                        if ((tmp & bits) == 0)
1233                                return 0;
1234                }
1235
1236                cond_resched();
1237        }
1238
1239        printk(KERN_ERR PFX "carm_init_wait timeout, bits == 0x%x, test_bit == %s\n",
1240               bits, test_bit ? "yes" : "no");
1241        return -EBUSY;
1242}
1243
1244static void carm_init_responses(struct carm_host *host)
1245{
1246        void __iomem *mmio = host->mmio;
1247        unsigned int i;
1248        struct carm_response *resp = (struct carm_response *) host->shm;
1249
1250        for (i = 0; i < RMSG_Q_LEN; i++)
1251                resp[i].status = cpu_to_le32(0xffffffff);
1252
1253        writel(0, mmio + CARM_RESP_IDX);
1254}
1255
1256static int carm_init_host(struct carm_host *host)
1257{
1258        void __iomem *mmio = host->mmio;
1259        u32 tmp;
1260        u8 tmp8;
1261        int rc;
1262
1263        DPRINTK("ENTER\n");
1264
1265        writel(0, mmio + CARM_INT_MASK);
1266
1267        tmp8 = readb(mmio + CARM_INITC);
1268        if (tmp8 & 0x01) {
1269                tmp8 &= ~0x01;
1270                writeb(tmp8, mmio + CARM_INITC);
1271                readb(mmio + CARM_INITC);       /* flush */
1272
1273                DPRINTK("snooze...\n");
1274                msleep(5000);
1275        }
1276
1277        tmp = readl(mmio + CARM_HMUC);
1278        if (tmp & CARM_CME) {
1279                DPRINTK("CME bit present, waiting\n");
1280                rc = carm_init_wait(mmio, CARM_CME, 1);
1281                if (rc) {
1282                        DPRINTK("EXIT, carm_init_wait 1 failed\n");
1283                        return rc;
1284                }
1285        }
1286        if (tmp & CARM_RME) {
1287                DPRINTK("RME bit present, waiting\n");
1288                rc = carm_init_wait(mmio, CARM_RME, 1);
1289                if (rc) {
1290                        DPRINTK("EXIT, carm_init_wait 2 failed\n");
1291                        return rc;
1292                }
1293        }
1294
1295        tmp &= ~(CARM_RME | CARM_CME);
1296        writel(tmp, mmio + CARM_HMUC);
1297        readl(mmio + CARM_HMUC);        /* flush */
1298
1299        rc = carm_init_wait(mmio, CARM_RME | CARM_CME, 0);
1300        if (rc) {
1301                DPRINTK("EXIT, carm_init_wait 3 failed\n");
1302                return rc;
1303        }
1304
1305        carm_init_buckets(mmio);
1306
1307        writel(host->shm_dma & 0xffffffff, mmio + RBUF_ADDR_LO);
1308        writel((host->shm_dma >> 16) >> 16, mmio + RBUF_ADDR_HI);
1309        writel(RBUF_LEN, mmio + RBUF_BYTE_SZ);
1310
1311        tmp = readl(mmio + CARM_HMUC);
1312        tmp |= (CARM_RME | CARM_CME | CARM_WZBC);
1313        writel(tmp, mmio + CARM_HMUC);
1314        readl(mmio + CARM_HMUC);        /* flush */
1315
1316        rc = carm_init_wait(mmio, CARM_RME | CARM_CME, 1);
1317        if (rc) {
1318                DPRINTK("EXIT, carm_init_wait 4 failed\n");
1319                return rc;
1320        }
1321
1322        writel(0, mmio + CARM_HMPHA);
1323        writel(INT_DEF_MASK, mmio + CARM_INT_MASK);
1324
1325        carm_init_responses(host);
1326
1327        /* start initialization, probing state machine */
1328        spin_lock_irq(&host->lock);
1329        assert(host->state == HST_INVALID);
1330        host->state = HST_PROBE_START;
1331        spin_unlock_irq(&host->lock);
1332        schedule_work(&host->fsm_task);
1333
1334        DPRINTK("EXIT\n");
1335        return 0;
1336}
1337
1338static const struct blk_mq_ops carm_mq_ops = {
1339        .queue_rq       = carm_queue_rq,
1340};
1341
1342static int carm_init_disk(struct carm_host *host, unsigned int port_no)
1343{
1344        struct carm_port *port = &host->port[port_no];
1345        struct gendisk *disk;
1346
1347        port->host = host;
1348        port->port_no = port_no;
1349
1350        disk = blk_mq_alloc_disk(&host->tag_set, port);
1351        if (IS_ERR(disk))
1352                return PTR_ERR(disk);
1353
1354        port->disk = disk;
1355        sprintf(disk->disk_name, DRV_NAME "/%u",
1356                (unsigned int)host->id * CARM_MAX_PORTS + port_no);
1357        disk->major = host->major;
1358        disk->first_minor = port_no * CARM_MINORS_PER_MAJOR;
1359        disk->minors = CARM_MINORS_PER_MAJOR;
1360        disk->fops = &carm_bd_ops;
1361        disk->private_data = port;
1362
1363        blk_queue_max_segments(disk->queue, CARM_MAX_REQ_SG);
1364        blk_queue_segment_boundary(disk->queue, CARM_SG_BOUNDARY);
1365        return 0;
1366}
1367
1368static void carm_free_disk(struct carm_host *host, unsigned int port_no)
1369{
1370        struct carm_port *port = &host->port[port_no];
1371        struct gendisk *disk = port->disk;
1372
1373        if (!disk)
1374                return;
1375
1376        if (disk->flags & GENHD_FL_UP)
1377                del_gendisk(disk);
1378        blk_cleanup_disk(disk);
1379}
1380
1381static int carm_init_shm(struct carm_host *host)
1382{
1383        host->shm = dma_alloc_coherent(&host->pdev->dev, CARM_SHM_SIZE,
1384                                       &host->shm_dma, GFP_KERNEL);
1385        if (!host->shm)
1386                return -ENOMEM;
1387
1388        host->msg_base = host->shm + RBUF_LEN;
1389        host->msg_dma = host->shm_dma + RBUF_LEN;
1390
1391        memset(host->shm, 0xff, RBUF_LEN);
1392        memset(host->msg_base, 0, PDC_SHM_SIZE - RBUF_LEN);
1393
1394        return 0;
1395}
1396
1397static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
1398{
1399        struct carm_host *host;
1400        int rc;
1401        struct request_queue *q;
1402        unsigned int i;
1403
1404        printk_once(KERN_DEBUG DRV_NAME " version " DRV_VERSION "\n");
1405
1406        rc = pci_enable_device(pdev);
1407        if (rc)
1408                return rc;
1409
1410        rc = pci_request_regions(pdev, DRV_NAME);
1411        if (rc)
1412                goto err_out;
1413
1414        rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1415        if (rc) {
1416                printk(KERN_ERR DRV_NAME "(%s): DMA mask failure\n",
1417                        pci_name(pdev));
1418                goto err_out_regions;
1419        }
1420
1421        host = kzalloc(sizeof(*host), GFP_KERNEL);
1422        if (!host) {
1423                rc = -ENOMEM;
1424                goto err_out_regions;
1425        }
1426
1427        host->pdev = pdev;
1428        spin_lock_init(&host->lock);
1429        INIT_WORK(&host->fsm_task, carm_fsm_task);
1430        init_completion(&host->probe_comp);
1431
1432        host->mmio = ioremap(pci_resource_start(pdev, 0),
1433                             pci_resource_len(pdev, 0));
1434        if (!host->mmio) {
1435                printk(KERN_ERR DRV_NAME "(%s): MMIO alloc failure\n",
1436                       pci_name(pdev));
1437                rc = -ENOMEM;
1438                goto err_out_kfree;
1439        }
1440
1441        rc = carm_init_shm(host);
1442        if (rc) {
1443                printk(KERN_ERR DRV_NAME "(%s): DMA SHM alloc failure\n",
1444                       pci_name(pdev));
1445                goto err_out_iounmap;
1446        }
1447
1448        memset(&host->tag_set, 0, sizeof(host->tag_set));
1449        host->tag_set.ops = &carm_mq_ops;
1450        host->tag_set.cmd_size = sizeof(struct carm_request);
1451        host->tag_set.nr_hw_queues = 1;
1452        host->tag_set.nr_maps = 1;
1453        host->tag_set.queue_depth = max_queue;
1454        host->tag_set.numa_node = NUMA_NO_NODE;
1455        host->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
1456
1457        rc = blk_mq_alloc_tag_set(&host->tag_set);
1458        if (rc)
1459                goto err_out_dma_free;
1460
1461        q = blk_mq_init_queue(&host->tag_set);
1462        if (IS_ERR(q)) {
1463                rc = PTR_ERR(q);
1464                blk_mq_free_tag_set(&host->tag_set);
1465                goto err_out_dma_free;
1466        }
1467
1468        host->oob_q = q;
1469        q->queuedata = host;
1470
1471        /*
1472         * Figure out which major to use: 160, 161, or dynamic
1473         */
1474        if (!test_and_set_bit(0, &carm_major_alloc))
1475                host->major = 160;
1476        else if (!test_and_set_bit(1, &carm_major_alloc))
1477                host->major = 161;
1478        else
1479                host->flags |= FL_DYN_MAJOR;
1480
1481        host->id = carm_host_id;
1482        sprintf(host->name, DRV_NAME "%d", carm_host_id);
1483
1484        rc = register_blkdev(host->major, host->name);
1485        if (rc < 0)
1486                goto err_out_free_majors;
1487        if (host->flags & FL_DYN_MAJOR)
1488                host->major = rc;
1489
1490        for (i = 0; i < CARM_MAX_PORTS; i++) {
1491                rc = carm_init_disk(host, i);
1492                if (rc)
1493                        goto err_out_blkdev_disks;
1494        }
1495
1496        pci_set_master(pdev);
1497
1498        rc = request_irq(pdev->irq, carm_interrupt, IRQF_SHARED, DRV_NAME, host);
1499        if (rc) {
1500                printk(KERN_ERR DRV_NAME "(%s): irq alloc failure\n",
1501                       pci_name(pdev));
1502                goto err_out_blkdev_disks;
1503        }
1504
1505        rc = carm_init_host(host);
1506        if (rc)
1507                goto err_out_free_irq;
1508
1509        DPRINTK("waiting for probe_comp\n");
1510        wait_for_completion(&host->probe_comp);
1511
1512        printk(KERN_INFO "%s: pci %s, ports %d, io %llx, irq %u, major %d\n",
1513               host->name, pci_name(pdev), (int) CARM_MAX_PORTS,
1514               (unsigned long long)pci_resource_start(pdev, 0),
1515                   pdev->irq, host->major);
1516
1517        carm_host_id++;
1518        pci_set_drvdata(pdev, host);
1519        return 0;
1520
1521err_out_free_irq:
1522        free_irq(pdev->irq, host);
1523err_out_blkdev_disks:
1524        for (i = 0; i < CARM_MAX_PORTS; i++)
1525                carm_free_disk(host, i);
1526        unregister_blkdev(host->major, host->name);
1527err_out_free_majors:
1528        if (host->major == 160)
1529                clear_bit(0, &carm_major_alloc);
1530        else if (host->major == 161)
1531                clear_bit(1, &carm_major_alloc);
1532        blk_cleanup_queue(host->oob_q);
1533        blk_mq_free_tag_set(&host->tag_set);
1534err_out_dma_free:
1535        dma_free_coherent(&pdev->dev, CARM_SHM_SIZE, host->shm, host->shm_dma);
1536err_out_iounmap:
1537        iounmap(host->mmio);
1538err_out_kfree:
1539        kfree(host);
1540err_out_regions:
1541        pci_release_regions(pdev);
1542err_out:
1543        pci_disable_device(pdev);
1544        return rc;
1545}
1546
1547static void carm_remove_one (struct pci_dev *pdev)
1548{
1549        struct carm_host *host = pci_get_drvdata(pdev);
1550        unsigned int i;
1551
1552        if (!host) {
1553                printk(KERN_ERR PFX "BUG: no host data for PCI(%s)\n",
1554                       pci_name(pdev));
1555                return;
1556        }
1557
1558        free_irq(pdev->irq, host);
1559        for (i = 0; i < CARM_MAX_PORTS; i++)
1560                carm_free_disk(host, i);
1561        unregister_blkdev(host->major, host->name);
1562        if (host->major == 160)
1563                clear_bit(0, &carm_major_alloc);
1564        else if (host->major == 161)
1565                clear_bit(1, &carm_major_alloc);
1566        blk_cleanup_queue(host->oob_q);
1567        blk_mq_free_tag_set(&host->tag_set);
1568        dma_free_coherent(&pdev->dev, CARM_SHM_SIZE, host->shm, host->shm_dma);
1569        iounmap(host->mmio);
1570        kfree(host);
1571        pci_release_regions(pdev);
1572        pci_disable_device(pdev);
1573}
1574
1575module_pci_driver(carm_driver);
1576