linux/drivers/block/mtip32xx/mtip32xx.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Driver for the Micron P320 SSD
   4 *   Copyright (C) 2011 Micron Technology, Inc.
   5 *
   6 * Portions of this code were derived from works subjected to the
   7 * following copyright:
   8 *    Copyright (C) 2009 Integrated Device Technology, Inc.
   9 */
  10
  11#include <linux/pci.h>
  12#include <linux/interrupt.h>
  13#include <linux/ata.h>
  14#include <linux/delay.h>
  15#include <linux/hdreg.h>
  16#include <linux/uaccess.h>
  17#include <linux/random.h>
  18#include <linux/smp.h>
  19#include <linux/compat.h>
  20#include <linux/fs.h>
  21#include <linux/module.h>
  22#include <linux/genhd.h>
  23#include <linux/blkdev.h>
  24#include <linux/blk-mq.h>
  25#include <linux/bio.h>
  26#include <linux/dma-mapping.h>
  27#include <linux/idr.h>
  28#include <linux/kthread.h>
  29#include <../drivers/ata/ahci.h>
  30#include <linux/export.h>
  31#include <linux/debugfs.h>
  32#include <linux/prefetch.h>
  33#include <linux/numa.h>
  34#include "mtip32xx.h"
  35
  36#define HW_CMD_SLOT_SZ          (MTIP_MAX_COMMAND_SLOTS * 32)
  37
  38/* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */
  39#define AHCI_RX_FIS_SZ          0x100
  40#define AHCI_RX_FIS_OFFSET      0x0
  41#define AHCI_IDFY_SZ            ATA_SECT_SIZE
  42#define AHCI_IDFY_OFFSET        0x400
  43#define AHCI_SECTBUF_SZ         ATA_SECT_SIZE
  44#define AHCI_SECTBUF_OFFSET     0x800
  45#define AHCI_SMARTBUF_SZ        ATA_SECT_SIZE
  46#define AHCI_SMARTBUF_OFFSET    0xC00
  47/* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */
  48#define BLOCK_DMA_ALLOC_SZ      4096
  49
  50/* DMA region containing command table (should be 8192 bytes) */
  51#define AHCI_CMD_SLOT_SZ        sizeof(struct mtip_cmd_hdr)
  52#define AHCI_CMD_TBL_SZ         (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ)
  53#define AHCI_CMD_TBL_OFFSET     0x0
  54
  55/* DMA region per command (contains header and SGL) */
  56#define AHCI_CMD_TBL_HDR_SZ     0x80
  57#define AHCI_CMD_TBL_HDR_OFFSET 0x0
  58#define AHCI_CMD_TBL_SGL_SZ     (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg))
  59#define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ
  60#define CMD_DMA_ALLOC_SZ        (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ)
  61
  62
  63#define HOST_CAP_NZDMA          (1 << 19)
  64#define HOST_HSORG              0xFC
  65#define HSORG_DISABLE_SLOTGRP_INTR (1<<24)
  66#define HSORG_DISABLE_SLOTGRP_PXIS (1<<16)
  67#define HSORG_HWREV             0xFF00
  68#define HSORG_STYLE             0x8
  69#define HSORG_SLOTGROUPS        0x7
  70
  71#define PORT_COMMAND_ISSUE      0x38
  72#define PORT_SDBV               0x7C
  73
  74#define PORT_OFFSET             0x100
  75#define PORT_MEM_SIZE           0x80
  76
  77#define PORT_IRQ_ERR \
  78        (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \
  79         PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \
  80         PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \
  81         PORT_IRQ_OVERFLOW)
  82#define PORT_IRQ_LEGACY \
  83        (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS)
  84#define PORT_IRQ_HANDLED \
  85        (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \
  86         PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \
  87         PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)
  88#define DEF_PORT_IRQ \
  89        (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS)
  90
  91/* product numbers */
  92#define MTIP_PRODUCT_UNKNOWN    0x00
  93#define MTIP_PRODUCT_ASICFPGA   0x11
  94
  95/* Device instance number, incremented each time a device is probed. */
  96static int instance;
  97
  98static LIST_HEAD(online_list);
  99static LIST_HEAD(removing_list);
 100static DEFINE_SPINLOCK(dev_lock);
 101
 102/*
 103 * Global variable used to hold the major block device number
 104 * allocated in mtip_init().
 105 */
 106static int mtip_major;
 107static struct dentry *dfs_parent;
 108static struct dentry *dfs_device_status;
 109
 110static u32 cpu_use[NR_CPUS];
 111
 112static DEFINE_IDA(rssd_index_ida);
 113
 114static int mtip_block_initialize(struct driver_data *dd);
 115
 116#ifdef CONFIG_COMPAT
 117struct mtip_compat_ide_task_request_s {
 118        __u8            io_ports[8];
 119        __u8            hob_ports[8];
 120        ide_reg_valid_t out_flags;
 121        ide_reg_valid_t in_flags;
 122        int             data_phase;
 123        int             req_cmd;
 124        compat_ulong_t  out_size;
 125        compat_ulong_t  in_size;
 126};
 127#endif
 128
 129/*
 130 * This function check_for_surprise_removal is called
 131 * while card is removed from the system and it will
 132 * read the vendor id from the configuration space
 133 *
 134 * @pdev Pointer to the pci_dev structure.
 135 *
 136 * return value
 137 *       true if device removed, else false
 138 */
 139static bool mtip_check_surprise_removal(struct pci_dev *pdev)
 140{
 141        u16 vendor_id = 0;
 142        struct driver_data *dd = pci_get_drvdata(pdev);
 143
 144        if (dd->sr)
 145                return true;
 146
 147       /* Read the vendorID from the configuration space */
 148        pci_read_config_word(pdev, 0x00, &vendor_id);
 149        if (vendor_id == 0xFFFF) {
 150                dd->sr = true;
 151                if (dd->queue)
 152                        blk_queue_flag_set(QUEUE_FLAG_DEAD, dd->queue);
 153                else
 154                        dev_warn(&dd->pdev->dev,
 155                                "%s: dd->queue is NULL\n", __func__);
 156                return true; /* device removed */
 157        }
 158
 159        return false; /* device present */
 160}
 161
 162static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd,
 163                                          unsigned int tag)
 164{
 165        struct blk_mq_hw_ctx *hctx = dd->queue->queue_hw_ctx[0];
 166
 167        return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(hctx->tags, tag));
 168}
 169
 170/*
 171 * Reset the HBA (without sleeping)
 172 *
 173 * @dd Pointer to the driver data structure.
 174 *
 175 * return value
 176 *      0       The reset was successful.
 177 *      -1      The HBA Reset bit did not clear.
 178 */
 179static int mtip_hba_reset(struct driver_data *dd)
 180{
 181        unsigned long timeout;
 182
 183        /* Set the reset bit */
 184        writel(HOST_RESET, dd->mmio + HOST_CTL);
 185
 186        /* Flush */
 187        readl(dd->mmio + HOST_CTL);
 188
 189        /*
 190         * Spin for up to 10 seconds waiting for reset acknowledgement. Spec
 191         * is 1 sec but in LUN failure conditions, up to 10 secs are required
 192         */
 193        timeout = jiffies + msecs_to_jiffies(10000);
 194        do {
 195                mdelay(10);
 196                if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
 197                        return -1;
 198
 199        } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET)
 200                 && time_before(jiffies, timeout));
 201
 202        if (readl(dd->mmio + HOST_CTL) & HOST_RESET)
 203                return -1;
 204
 205        return 0;
 206}
 207
 208/*
 209 * Issue a command to the hardware.
 210 *
 211 * Set the appropriate bit in the s_active and Command Issue hardware
 212 * registers, causing hardware command processing to begin.
 213 *
 214 * @port Pointer to the port structure.
 215 * @tag  The tag of the command to be issued.
 216 *
 217 * return value
 218 *      None
 219 */
 220static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag)
 221{
 222        int group = tag >> 5;
 223
 224        /* guard SACT and CI registers */
 225        spin_lock(&port->cmd_issue_lock[group]);
 226        writel((1 << MTIP_TAG_BIT(tag)),
 227                        port->s_active[MTIP_TAG_INDEX(tag)]);
 228        writel((1 << MTIP_TAG_BIT(tag)),
 229                        port->cmd_issue[MTIP_TAG_INDEX(tag)]);
 230        spin_unlock(&port->cmd_issue_lock[group]);
 231}
 232
 233/*
 234 * Enable/disable the reception of FIS
 235 *
 236 * @port   Pointer to the port data structure
 237 * @enable 1 to enable, 0 to disable
 238 *
 239 * return value
 240 *      Previous state: 1 enabled, 0 disabled
 241 */
 242static int mtip_enable_fis(struct mtip_port *port, int enable)
 243{
 244        u32 tmp;
 245
 246        /* enable FIS reception */
 247        tmp = readl(port->mmio + PORT_CMD);
 248        if (enable)
 249                writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
 250        else
 251                writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
 252
 253        /* Flush */
 254        readl(port->mmio + PORT_CMD);
 255
 256        return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX));
 257}
 258
 259/*
 260 * Enable/disable the DMA engine
 261 *
 262 * @port   Pointer to the port data structure
 263 * @enable 1 to enable, 0 to disable
 264 *
 265 * return value
 266 *      Previous state: 1 enabled, 0 disabled.
 267 */
 268static int mtip_enable_engine(struct mtip_port *port, int enable)
 269{
 270        u32 tmp;
 271
 272        /* enable FIS reception */
 273        tmp = readl(port->mmio + PORT_CMD);
 274        if (enable)
 275                writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD);
 276        else
 277                writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD);
 278
 279        readl(port->mmio + PORT_CMD);
 280        return (((tmp & PORT_CMD_START) == PORT_CMD_START));
 281}
 282
 283/*
 284 * Enables the port DMA engine and FIS reception.
 285 *
 286 * return value
 287 *      None
 288 */
 289static inline void mtip_start_port(struct mtip_port *port)
 290{
 291        /* Enable FIS reception */
 292        mtip_enable_fis(port, 1);
 293
 294        /* Enable the DMA engine */
 295        mtip_enable_engine(port, 1);
 296}
 297
 298/*
 299 * Deinitialize a port by disabling port interrupts, the DMA engine,
 300 * and FIS reception.
 301 *
 302 * @port Pointer to the port structure
 303 *
 304 * return value
 305 *      None
 306 */
 307static inline void mtip_deinit_port(struct mtip_port *port)
 308{
 309        /* Disable interrupts on this port */
 310        writel(0, port->mmio + PORT_IRQ_MASK);
 311
 312        /* Disable the DMA engine */
 313        mtip_enable_engine(port, 0);
 314
 315        /* Disable FIS reception */
 316        mtip_enable_fis(port, 0);
 317}
 318
 319/*
 320 * Initialize a port.
 321 *
 322 * This function deinitializes the port by calling mtip_deinit_port() and
 323 * then initializes it by setting the command header and RX FIS addresses,
 324 * clearing the SError register and any pending port interrupts before
 325 * re-enabling the default set of port interrupts.
 326 *
 327 * @port Pointer to the port structure.
 328 *
 329 * return value
 330 *      None
 331 */
 332static void mtip_init_port(struct mtip_port *port)
 333{
 334        int i;
 335        mtip_deinit_port(port);
 336
 337        /* Program the command list base and FIS base addresses */
 338        if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) {
 339                writel((port->command_list_dma >> 16) >> 16,
 340                         port->mmio + PORT_LST_ADDR_HI);
 341                writel((port->rxfis_dma >> 16) >> 16,
 342                         port->mmio + PORT_FIS_ADDR_HI);
 343                set_bit(MTIP_PF_HOST_CAP_64, &port->flags);
 344        }
 345
 346        writel(port->command_list_dma & 0xFFFFFFFF,
 347                        port->mmio + PORT_LST_ADDR);
 348        writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR);
 349
 350        /* Clear SError */
 351        writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
 352
 353        /* reset the completed registers.*/
 354        for (i = 0; i < port->dd->slot_groups; i++)
 355                writel(0xFFFFFFFF, port->completed[i]);
 356
 357        /* Clear any pending interrupts for this port */
 358        writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT);
 359
 360        /* Clear any pending interrupts on the HBA. */
 361        writel(readl(port->dd->mmio + HOST_IRQ_STAT),
 362                                        port->dd->mmio + HOST_IRQ_STAT);
 363
 364        /* Enable port interrupts */
 365        writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK);
 366}
 367
 368/*
 369 * Restart a port
 370 *
 371 * @port Pointer to the port data structure.
 372 *
 373 * return value
 374 *      None
 375 */
 376static void mtip_restart_port(struct mtip_port *port)
 377{
 378        unsigned long timeout;
 379
 380        /* Disable the DMA engine */
 381        mtip_enable_engine(port, 0);
 382
 383        /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */
 384        timeout = jiffies + msecs_to_jiffies(500);
 385        while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON)
 386                 && time_before(jiffies, timeout))
 387                ;
 388
 389        if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
 390                return;
 391
 392        /*
 393         * Chip quirk: escalate to hba reset if
 394         * PxCMD.CR not clear after 500 ms
 395         */
 396        if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) {
 397                dev_warn(&port->dd->pdev->dev,
 398                        "PxCMD.CR not clear, escalating reset\n");
 399
 400                if (mtip_hba_reset(port->dd))
 401                        dev_err(&port->dd->pdev->dev,
 402                                "HBA reset escalation failed.\n");
 403
 404                /* 30 ms delay before com reset to quiesce chip */
 405                mdelay(30);
 406        }
 407
 408        dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n");
 409
 410        /* Set PxSCTL.DET */
 411        writel(readl(port->mmio + PORT_SCR_CTL) |
 412                         1, port->mmio + PORT_SCR_CTL);
 413        readl(port->mmio + PORT_SCR_CTL);
 414
 415        /* Wait 1 ms to quiesce chip function */
 416        timeout = jiffies + msecs_to_jiffies(1);
 417        while (time_before(jiffies, timeout))
 418                ;
 419
 420        if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
 421                return;
 422
 423        /* Clear PxSCTL.DET */
 424        writel(readl(port->mmio + PORT_SCR_CTL) & ~1,
 425                         port->mmio + PORT_SCR_CTL);
 426        readl(port->mmio + PORT_SCR_CTL);
 427
 428        /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */
 429        timeout = jiffies + msecs_to_jiffies(500);
 430        while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
 431                         && time_before(jiffies, timeout))
 432                ;
 433
 434        if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
 435                return;
 436
 437        if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
 438                dev_warn(&port->dd->pdev->dev,
 439                        "COM reset failed\n");
 440
 441        mtip_init_port(port);
 442        mtip_start_port(port);
 443
 444}
 445
 446static int mtip_device_reset(struct driver_data *dd)
 447{
 448        int rv = 0;
 449
 450        if (mtip_check_surprise_removal(dd->pdev))
 451                return 0;
 452
 453        if (mtip_hba_reset(dd) < 0)
 454                rv = -EFAULT;
 455
 456        mdelay(1);
 457        mtip_init_port(dd->port);
 458        mtip_start_port(dd->port);
 459
 460        /* Enable interrupts on the HBA. */
 461        writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
 462                                        dd->mmio + HOST_CTL);
 463        return rv;
 464}
 465
 466/*
 467 * Helper function for tag logging
 468 */
 469static void print_tags(struct driver_data *dd,
 470                        char *msg,
 471                        unsigned long *tagbits,
 472                        int cnt)
 473{
 474        unsigned char tagmap[128];
 475        int group, tagmap_len = 0;
 476
 477        memset(tagmap, 0, sizeof(tagmap));
 478        for (group = SLOTBITS_IN_LONGS; group > 0; group--)
 479                tagmap_len += sprintf(tagmap + tagmap_len, "%016lX ",
 480                                                tagbits[group-1]);
 481        dev_warn(&dd->pdev->dev,
 482                        "%d command(s) %s: tagmap [%s]", cnt, msg, tagmap);
 483}
 484
 485static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
 486                                dma_addr_t buffer_dma, unsigned int sectors);
 487static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
 488                                                struct smart_attr *attrib);
 489
 490static void mtip_complete_command(struct mtip_cmd *cmd, blk_status_t status)
 491{
 492        struct request *req = blk_mq_rq_from_pdu(cmd);
 493
 494        cmd->status = status;
 495        if (likely(!blk_should_fake_timeout(req->q)))
 496                blk_mq_complete_request(req);
 497}
 498
 499/*
 500 * Handle an error.
 501 *
 502 * @dd Pointer to the DRIVER_DATA structure.
 503 *
 504 * return value
 505 *      None
 506 */
 507static void mtip_handle_tfe(struct driver_data *dd)
 508{
 509        int group, tag, bit, reissue, rv;
 510        struct mtip_port *port;
 511        struct mtip_cmd  *cmd;
 512        u32 completed;
 513        struct host_to_dev_fis *fis;
 514        unsigned long tagaccum[SLOTBITS_IN_LONGS];
 515        unsigned int cmd_cnt = 0;
 516        unsigned char *buf;
 517        char *fail_reason = NULL;
 518        int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0;
 519
 520        dev_warn(&dd->pdev->dev, "Taskfile error\n");
 521
 522        port = dd->port;
 523
 524        if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) {
 525                cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
 526                dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n");
 527                mtip_complete_command(cmd, BLK_STS_IOERR);
 528                return;
 529        }
 530
 531        /* clear the tag accumulator */
 532        memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
 533
 534        /* Loop through all the groups */
 535        for (group = 0; group < dd->slot_groups; group++) {
 536                completed = readl(port->completed[group]);
 537
 538                dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed);
 539
 540                /* clear completed status register in the hardware.*/
 541                writel(completed, port->completed[group]);
 542
 543                /* Process successfully completed commands */
 544                for (bit = 0; bit < 32 && completed; bit++) {
 545                        if (!(completed & (1<<bit)))
 546                                continue;
 547                        tag = (group << 5) + bit;
 548
 549                        /* Skip the internal command slot */
 550                        if (tag == MTIP_TAG_INTERNAL)
 551                                continue;
 552
 553                        cmd = mtip_cmd_from_tag(dd, tag);
 554                        mtip_complete_command(cmd, 0);
 555                        set_bit(tag, tagaccum);
 556                        cmd_cnt++;
 557                }
 558        }
 559
 560        print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt);
 561
 562        /* Restart the port */
 563        mdelay(20);
 564        mtip_restart_port(port);
 565
 566        /* Trying to determine the cause of the error */
 567        rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
 568                                dd->port->log_buf,
 569                                dd->port->log_buf_dma, 1);
 570        if (rv) {
 571                dev_warn(&dd->pdev->dev,
 572                        "Error in READ LOG EXT (10h) command\n");
 573                /* non-critical error, don't fail the load */
 574        } else {
 575                buf = (unsigned char *)dd->port->log_buf;
 576                if (buf[259] & 0x1) {
 577                        dev_info(&dd->pdev->dev,
 578                                "Write protect bit is set.\n");
 579                        set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
 580                        fail_all_ncq_write = 1;
 581                        fail_reason = "write protect";
 582                }
 583                if (buf[288] == 0xF7) {
 584                        dev_info(&dd->pdev->dev,
 585                                "Exceeded Tmax, drive in thermal shutdown.\n");
 586                        set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
 587                        fail_all_ncq_cmds = 1;
 588                        fail_reason = "thermal shutdown";
 589                }
 590                if (buf[288] == 0xBF) {
 591                        set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
 592                        dev_info(&dd->pdev->dev,
 593                                "Drive indicates rebuild has failed. Secure erase required.\n");
 594                        fail_all_ncq_cmds = 1;
 595                        fail_reason = "rebuild failed";
 596                }
 597        }
 598
 599        /* clear the tag accumulator */
 600        memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
 601
 602        /* Loop through all the groups */
 603        for (group = 0; group < dd->slot_groups; group++) {
 604                for (bit = 0; bit < 32; bit++) {
 605                        reissue = 1;
 606                        tag = (group << 5) + bit;
 607                        cmd = mtip_cmd_from_tag(dd, tag);
 608
 609                        fis = (struct host_to_dev_fis *)cmd->command;
 610
 611                        /* Should re-issue? */
 612                        if (tag == MTIP_TAG_INTERNAL ||
 613                            fis->command == ATA_CMD_SET_FEATURES)
 614                                reissue = 0;
 615                        else {
 616                                if (fail_all_ncq_cmds ||
 617                                        (fail_all_ncq_write &&
 618                                        fis->command == ATA_CMD_FPDMA_WRITE)) {
 619                                        dev_warn(&dd->pdev->dev,
 620                                        "  Fail: %s w/tag %d [%s].\n",
 621                                        fis->command == ATA_CMD_FPDMA_WRITE ?
 622                                                "write" : "read",
 623                                        tag,
 624                                        fail_reason != NULL ?
 625                                                fail_reason : "unknown");
 626                                        mtip_complete_command(cmd, BLK_STS_MEDIUM);
 627                                        continue;
 628                                }
 629                        }
 630
 631                        /*
 632                         * First check if this command has
 633                         *  exceeded its retries.
 634                         */
 635                        if (reissue && (cmd->retries-- > 0)) {
 636
 637                                set_bit(tag, tagaccum);
 638
 639                                /* Re-issue the command. */
 640                                mtip_issue_ncq_command(port, tag);
 641
 642                                continue;
 643                        }
 644
 645                        /* Retire a command that will not be reissued */
 646                        dev_warn(&port->dd->pdev->dev,
 647                                "retiring tag %d\n", tag);
 648
 649                        mtip_complete_command(cmd, BLK_STS_IOERR);
 650                }
 651        }
 652        print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt);
 653}
 654
 655/*
 656 * Handle a set device bits interrupt
 657 */
 658static inline void mtip_workq_sdbfx(struct mtip_port *port, int group,
 659                                                        u32 completed)
 660{
 661        struct driver_data *dd = port->dd;
 662        int tag, bit;
 663        struct mtip_cmd *command;
 664
 665        if (!completed) {
 666                WARN_ON_ONCE(!completed);
 667                return;
 668        }
 669        /* clear completed status register in the hardware.*/
 670        writel(completed, port->completed[group]);
 671
 672        /* Process completed commands. */
 673        for (bit = 0; (bit < 32) && completed; bit++) {
 674                if (completed & 0x01) {
 675                        tag = (group << 5) | bit;
 676
 677                        /* skip internal command slot. */
 678                        if (unlikely(tag == MTIP_TAG_INTERNAL))
 679                                continue;
 680
 681                        command = mtip_cmd_from_tag(dd, tag);
 682                        mtip_complete_command(command, 0);
 683                }
 684                completed >>= 1;
 685        }
 686
 687        /* If last, re-enable interrupts */
 688        if (atomic_dec_return(&dd->irq_workers_active) == 0)
 689                writel(0xffffffff, dd->mmio + HOST_IRQ_STAT);
 690}
 691
 692/*
 693 * Process legacy pio and d2h interrupts
 694 */
 695static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat)
 696{
 697        struct mtip_port *port = dd->port;
 698        struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
 699
 700        if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && cmd) {
 701                int group = MTIP_TAG_INDEX(MTIP_TAG_INTERNAL);
 702                int status = readl(port->cmd_issue[group]);
 703
 704                if (!(status & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))))
 705                        mtip_complete_command(cmd, 0);
 706        }
 707}
 708
 709/*
 710 * Demux and handle errors
 711 */
 712static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat)
 713{
 714        if (unlikely(port_stat & PORT_IRQ_CONNECT)) {
 715                dev_warn(&dd->pdev->dev,
 716                        "Clearing PxSERR.DIAG.x\n");
 717                writel((1 << 26), dd->port->mmio + PORT_SCR_ERR);
 718        }
 719
 720        if (unlikely(port_stat & PORT_IRQ_PHYRDY)) {
 721                dev_warn(&dd->pdev->dev,
 722                        "Clearing PxSERR.DIAG.n\n");
 723                writel((1 << 16), dd->port->mmio + PORT_SCR_ERR);
 724        }
 725
 726        if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) {
 727                dev_warn(&dd->pdev->dev,
 728                        "Port stat errors %x unhandled\n",
 729                        (port_stat & ~PORT_IRQ_HANDLED));
 730                if (mtip_check_surprise_removal(dd->pdev))
 731                        return;
 732        }
 733        if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) {
 734                set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags);
 735                wake_up_interruptible(&dd->port->svc_wait);
 736        }
 737}
 738
 739static inline irqreturn_t mtip_handle_irq(struct driver_data *data)
 740{
 741        struct driver_data *dd = (struct driver_data *) data;
 742        struct mtip_port *port = dd->port;
 743        u32 hba_stat, port_stat;
 744        int rv = IRQ_NONE;
 745        int do_irq_enable = 1, i, workers;
 746        struct mtip_work *twork;
 747
 748        hba_stat = readl(dd->mmio + HOST_IRQ_STAT);
 749        if (hba_stat) {
 750                rv = IRQ_HANDLED;
 751
 752                /* Acknowledge the interrupt status on the port.*/
 753                port_stat = readl(port->mmio + PORT_IRQ_STAT);
 754                if (unlikely(port_stat == 0xFFFFFFFF)) {
 755                        mtip_check_surprise_removal(dd->pdev);
 756                        return IRQ_HANDLED;
 757                }
 758                writel(port_stat, port->mmio + PORT_IRQ_STAT);
 759
 760                /* Demux port status */
 761                if (likely(port_stat & PORT_IRQ_SDB_FIS)) {
 762                        do_irq_enable = 0;
 763                        WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0);
 764
 765                        /* Start at 1: group zero is always local? */
 766                        for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS;
 767                                                                        i++) {
 768                                twork = &dd->work[i];
 769                                twork->completed = readl(port->completed[i]);
 770                                if (twork->completed)
 771                                        workers++;
 772                        }
 773
 774                        atomic_set(&dd->irq_workers_active, workers);
 775                        if (workers) {
 776                                for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) {
 777                                        twork = &dd->work[i];
 778                                        if (twork->completed)
 779                                                queue_work_on(
 780                                                        twork->cpu_binding,
 781                                                        dd->isr_workq,
 782                                                        &twork->work);
 783                                }
 784
 785                                if (likely(dd->work[0].completed))
 786                                        mtip_workq_sdbfx(port, 0,
 787                                                        dd->work[0].completed);
 788
 789                        } else {
 790                                /*
 791                                 * Chip quirk: SDB interrupt but nothing
 792                                 * to complete
 793                                 */
 794                                do_irq_enable = 1;
 795                        }
 796                }
 797
 798                if (unlikely(port_stat & PORT_IRQ_ERR)) {
 799                        if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
 800                                /* don't proceed further */
 801                                return IRQ_HANDLED;
 802                        }
 803                        if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
 804                                                        &dd->dd_flag))
 805                                return rv;
 806
 807                        mtip_process_errors(dd, port_stat & PORT_IRQ_ERR);
 808                }
 809
 810                if (unlikely(port_stat & PORT_IRQ_LEGACY))
 811                        mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY);
 812        }
 813
 814        /* acknowledge interrupt */
 815        if (unlikely(do_irq_enable))
 816                writel(hba_stat, dd->mmio + HOST_IRQ_STAT);
 817
 818        return rv;
 819}
 820
 821/*
 822 * HBA interrupt subroutine.
 823 *
 824 * @irq         IRQ number.
 825 * @instance    Pointer to the driver data structure.
 826 *
 827 * return value
 828 *      IRQ_HANDLED     A HBA interrupt was pending and handled.
 829 *      IRQ_NONE        This interrupt was not for the HBA.
 830 */
 831static irqreturn_t mtip_irq_handler(int irq, void *instance)
 832{
 833        struct driver_data *dd = instance;
 834
 835        return mtip_handle_irq(dd);
 836}
 837
 838static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag)
 839{
 840        writel(1 << MTIP_TAG_BIT(tag), port->cmd_issue[MTIP_TAG_INDEX(tag)]);
 841}
 842
 843static bool mtip_pause_ncq(struct mtip_port *port,
 844                                struct host_to_dev_fis *fis)
 845{
 846        unsigned long task_file_data;
 847
 848        task_file_data = readl(port->mmio+PORT_TFDATA);
 849        if ((task_file_data & 1))
 850                return false;
 851
 852        if (fis->command == ATA_CMD_SEC_ERASE_PREP) {
 853                port->ic_pause_timer = jiffies;
 854                return true;
 855        } else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) &&
 856                                        (fis->features == 0x03)) {
 857                set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
 858                port->ic_pause_timer = jiffies;
 859                return true;
 860        } else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) ||
 861                ((fis->command == 0xFC) &&
 862                        (fis->features == 0x27 || fis->features == 0x72 ||
 863                         fis->features == 0x62 || fis->features == 0x26))) {
 864                clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
 865                clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag);
 866                /* Com reset after secure erase or lowlevel format */
 867                mtip_restart_port(port);
 868                clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
 869                return false;
 870        }
 871
 872        return false;
 873}
 874
 875static bool mtip_commands_active(struct mtip_port *port)
 876{
 877        unsigned int active;
 878        unsigned int n;
 879
 880        /*
 881         * Ignore s_active bit 0 of array element 0.
 882         * This bit will always be set
 883         */
 884        active = readl(port->s_active[0]) & 0xFFFFFFFE;
 885        for (n = 1; n < port->dd->slot_groups; n++)
 886                active |= readl(port->s_active[n]);
 887
 888        return active != 0;
 889}
 890
 891/*
 892 * Wait for port to quiesce
 893 *
 894 * @port    Pointer to port data structure
 895 * @timeout Max duration to wait (ms)
 896 *
 897 * return value
 898 *      0       Success
 899 *      -EBUSY  Commands still active
 900 */
 901static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout)
 902{
 903        unsigned long to;
 904        bool active = true;
 905
 906        blk_mq_quiesce_queue(port->dd->queue);
 907
 908        to = jiffies + msecs_to_jiffies(timeout);
 909        do {
 910                if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) &&
 911                        test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
 912                        msleep(20);
 913                        continue; /* svc thd is actively issuing commands */
 914                }
 915
 916                msleep(100);
 917
 918                if (mtip_check_surprise_removal(port->dd->pdev))
 919                        goto err_fault;
 920
 921                active = mtip_commands_active(port);
 922                if (!active)
 923                        break;
 924        } while (time_before(jiffies, to));
 925
 926        blk_mq_unquiesce_queue(port->dd->queue);
 927        return active ? -EBUSY : 0;
 928err_fault:
 929        blk_mq_unquiesce_queue(port->dd->queue);
 930        return -EFAULT;
 931}
 932
 933struct mtip_int_cmd {
 934        int fis_len;
 935        dma_addr_t buffer;
 936        int buf_len;
 937        u32 opts;
 938};
 939
 940/*
 941 * Execute an internal command and wait for the completion.
 942 *
 943 * @port    Pointer to the port data structure.
 944 * @fis     Pointer to the FIS that describes the command.
 945 * @fis_len  Length in WORDS of the FIS.
 946 * @buffer  DMA accessible for command data.
 947 * @buf_len  Length, in bytes, of the data buffer.
 948 * @opts    Command header options, excluding the FIS length
 949 *             and the number of PRD entries.
 950 * @timeout Time in ms to wait for the command to complete.
 951 *
 952 * return value
 953 *      0        Command completed successfully.
 954 *      -EFAULT  The buffer address is not correctly aligned.
 955 *      -EBUSY   Internal command or other IO in progress.
 956 *      -EAGAIN  Time out waiting for command to complete.
 957 */
 958static int mtip_exec_internal_command(struct mtip_port *port,
 959                                        struct host_to_dev_fis *fis,
 960                                        int fis_len,
 961                                        dma_addr_t buffer,
 962                                        int buf_len,
 963                                        u32 opts,
 964                                        unsigned long timeout)
 965{
 966        struct mtip_cmd *int_cmd;
 967        struct driver_data *dd = port->dd;
 968        struct request *rq;
 969        struct mtip_int_cmd icmd = {
 970                .fis_len = fis_len,
 971                .buffer = buffer,
 972                .buf_len = buf_len,
 973                .opts = opts
 974        };
 975        int rv = 0;
 976
 977        /* Make sure the buffer is 8 byte aligned. This is asic specific. */
 978        if (buffer & 0x00000007) {
 979                dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n");
 980                return -EFAULT;
 981        }
 982
 983        if (mtip_check_surprise_removal(dd->pdev))
 984                return -EFAULT;
 985
 986        rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED);
 987        if (IS_ERR(rq)) {
 988                dbg_printk(MTIP_DRV_NAME "Unable to allocate tag for PIO cmd\n");
 989                return -EFAULT;
 990        }
 991
 992        set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
 993
 994        if (fis->command == ATA_CMD_SEC_ERASE_PREP)
 995                set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
 996
 997        clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
 998
 999        if (fis->command != ATA_CMD_STANDBYNOW1) {
1000                /* wait for io to complete if non atomic */
1001                if (mtip_quiesce_io(port, MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) {
1002                        dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n");
1003                        blk_mq_free_request(rq);
1004                        clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
1005                        wake_up_interruptible(&port->svc_wait);
1006                        return -EBUSY;
1007                }
1008        }
1009
1010        /* Copy the command to the command table */
1011        int_cmd = blk_mq_rq_to_pdu(rq);
1012        int_cmd->icmd = &icmd;
1013        memcpy(int_cmd->command, fis, fis_len*4);
1014
1015        rq->timeout = timeout;
1016
1017        /* insert request and run queue */
1018        blk_execute_rq(NULL, rq, true);
1019
1020        if (int_cmd->status) {
1021                dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n",
1022                                fis->command, int_cmd->status);
1023                rv = -EIO;
1024
1025                if (mtip_check_surprise_removal(dd->pdev) ||
1026                        test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
1027                                        &dd->dd_flag)) {
1028                        dev_err(&dd->pdev->dev,
1029                                "Internal command [%02X] wait returned due to SR\n",
1030                                fis->command);
1031                        rv = -ENXIO;
1032                        goto exec_ic_exit;
1033                }
1034                mtip_device_reset(dd); /* recover from timeout issue */
1035                rv = -EAGAIN;
1036                goto exec_ic_exit;
1037        }
1038
1039        if (readl(port->cmd_issue[MTIP_TAG_INDEX(MTIP_TAG_INTERNAL)])
1040                        & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))) {
1041                rv = -ENXIO;
1042                if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
1043                        mtip_device_reset(dd);
1044                        rv = -EAGAIN;
1045                }
1046        }
1047exec_ic_exit:
1048        /* Clear the allocated and active bits for the internal command. */
1049        blk_mq_free_request(rq);
1050        clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
1051        if (rv >= 0 && mtip_pause_ncq(port, fis)) {
1052                /* NCQ paused */
1053                return rv;
1054        }
1055        wake_up_interruptible(&port->svc_wait);
1056
1057        return rv;
1058}
1059
1060/*
1061 * Byte-swap ATA ID strings.
1062 *
1063 * ATA identify data contains strings in byte-swapped 16-bit words.
1064 * They must be swapped (on all architectures) to be usable as C strings.
1065 * This function swaps bytes in-place.
1066 *
1067 * @buf The buffer location of the string
1068 * @len The number of bytes to swap
1069 *
1070 * return value
1071 *      None
1072 */
1073static inline void ata_swap_string(u16 *buf, unsigned int len)
1074{
1075        int i;
1076        for (i = 0; i < (len/2); i++)
1077                be16_to_cpus(&buf[i]);
1078}
1079
1080static void mtip_set_timeout(struct driver_data *dd,
1081                                        struct host_to_dev_fis *fis,
1082                                        unsigned int *timeout, u8 erasemode)
1083{
1084        switch (fis->command) {
1085        case ATA_CMD_DOWNLOAD_MICRO:
1086                *timeout = 120000; /* 2 minutes */
1087                break;
1088        case ATA_CMD_SEC_ERASE_UNIT:
1089        case 0xFC:
1090                if (erasemode)
1091                        *timeout = ((*(dd->port->identify + 90) * 2) * 60000);
1092                else
1093                        *timeout = ((*(dd->port->identify + 89) * 2) * 60000);
1094                break;
1095        case ATA_CMD_STANDBYNOW1:
1096                *timeout = 120000;  /* 2 minutes */
1097                break;
1098        case 0xF7:
1099        case 0xFA:
1100                *timeout = 60000;  /* 60 seconds */
1101                break;
1102        case ATA_CMD_SMART:
1103                *timeout = 15000;  /* 15 seconds */
1104                break;
1105        default:
1106                *timeout = MTIP_IOCTL_CMD_TIMEOUT_MS;
1107                break;
1108        }
1109}
1110
1111/*
1112 * Request the device identity information.
1113 *
1114 * If a user space buffer is not specified, i.e. is NULL, the
1115 * identify information is still read from the drive and placed
1116 * into the identify data buffer (@e port->identify) in the
1117 * port data structure.
1118 * When the identify buffer contains valid identify information @e
1119 * port->identify_valid is non-zero.
1120 *
1121 * @port         Pointer to the port structure.
1122 * @user_buffer  A user space buffer where the identify data should be
1123 *                    copied.
1124 *
1125 * return value
1126 *      0       Command completed successfully.
1127 *      -EFAULT An error occurred while coping data to the user buffer.
1128 *      -1      Command failed.
1129 */
1130static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer)
1131{
1132        int rv = 0;
1133        struct host_to_dev_fis fis;
1134
1135        if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
1136                return -EFAULT;
1137
1138        /* Build the FIS. */
1139        memset(&fis, 0, sizeof(struct host_to_dev_fis));
1140        fis.type        = 0x27;
1141        fis.opts        = 1 << 7;
1142        fis.command     = ATA_CMD_ID_ATA;
1143
1144        /* Set the identify information as invalid. */
1145        port->identify_valid = 0;
1146
1147        /* Clear the identify information. */
1148        memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS);
1149
1150        /* Execute the command. */
1151        if (mtip_exec_internal_command(port,
1152                                &fis,
1153                                5,
1154                                port->identify_dma,
1155                                sizeof(u16) * ATA_ID_WORDS,
1156                                0,
1157                                MTIP_INT_CMD_TIMEOUT_MS)
1158                                < 0) {
1159                rv = -1;
1160                goto out;
1161        }
1162
1163        /*
1164         * Perform any necessary byte-swapping.  Yes, the kernel does in fact
1165         * perform field-sensitive swapping on the string fields.
1166         * See the kernel use of ata_id_string() for proof of this.
1167         */
1168#ifdef __LITTLE_ENDIAN
1169        ata_swap_string(port->identify + 27, 40);  /* model string*/
1170        ata_swap_string(port->identify + 23, 8);   /* firmware string*/
1171        ata_swap_string(port->identify + 10, 20);  /* serial# string*/
1172#else
1173        {
1174                int i;
1175                for (i = 0; i < ATA_ID_WORDS; i++)
1176                        port->identify[i] = le16_to_cpu(port->identify[i]);
1177        }
1178#endif
1179
1180        /* Check security locked state */
1181        if (port->identify[128] & 0x4)
1182                set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
1183        else
1184                clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
1185
1186        /* Set the identify buffer as valid. */
1187        port->identify_valid = 1;
1188
1189        if (user_buffer) {
1190                if (copy_to_user(
1191                        user_buffer,
1192                        port->identify,
1193                        ATA_ID_WORDS * sizeof(u16))) {
1194                        rv = -EFAULT;
1195                        goto out;
1196                }
1197        }
1198
1199out:
1200        return rv;
1201}
1202
1203/*
1204 * Issue a standby immediate command to the device.
1205 *
1206 * @port Pointer to the port structure.
1207 *
1208 * return value
1209 *      0       Command was executed successfully.
1210 *      -1      An error occurred while executing the command.
1211 */
1212static int mtip_standby_immediate(struct mtip_port *port)
1213{
1214        int rv;
1215        struct host_to_dev_fis  fis;
1216        unsigned long __maybe_unused start;
1217        unsigned int timeout;
1218
1219        /* Build the FIS. */
1220        memset(&fis, 0, sizeof(struct host_to_dev_fis));
1221        fis.type        = 0x27;
1222        fis.opts        = 1 << 7;
1223        fis.command     = ATA_CMD_STANDBYNOW1;
1224
1225        mtip_set_timeout(port->dd, &fis, &timeout, 0);
1226
1227        start = jiffies;
1228        rv = mtip_exec_internal_command(port,
1229                                        &fis,
1230                                        5,
1231                                        0,
1232                                        0,
1233                                        0,
1234                                        timeout);
1235        dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n",
1236                        jiffies_to_msecs(jiffies - start));
1237        if (rv)
1238                dev_warn(&port->dd->pdev->dev,
1239                        "STANDBY IMMEDIATE command failed.\n");
1240
1241        return rv;
1242}
1243
1244/*
1245 * Issue a READ LOG EXT command to the device.
1246 *
1247 * @port        pointer to the port structure.
1248 * @page        page number to fetch
1249 * @buffer      pointer to buffer
1250 * @buffer_dma  dma address corresponding to @buffer
1251 * @sectors     page length to fetch, in sectors
1252 *
1253 * return value
1254 *      @rv     return value from mtip_exec_internal_command()
1255 */
1256static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
1257                                dma_addr_t buffer_dma, unsigned int sectors)
1258{
1259        struct host_to_dev_fis fis;
1260
1261        memset(&fis, 0, sizeof(struct host_to_dev_fis));
1262        fis.type        = 0x27;
1263        fis.opts        = 1 << 7;
1264        fis.command     = ATA_CMD_READ_LOG_EXT;
1265        fis.sect_count  = sectors & 0xFF;
1266        fis.sect_cnt_ex = (sectors >> 8) & 0xFF;
1267        fis.lba_low     = page;
1268        fis.lba_mid     = 0;
1269        fis.device      = ATA_DEVICE_OBS;
1270
1271        memset(buffer, 0, sectors * ATA_SECT_SIZE);
1272
1273        return mtip_exec_internal_command(port,
1274                                        &fis,
1275                                        5,
1276                                        buffer_dma,
1277                                        sectors * ATA_SECT_SIZE,
1278                                        0,
1279                                        MTIP_INT_CMD_TIMEOUT_MS);
1280}
1281
1282/*
1283 * Issue a SMART READ DATA command to the device.
1284 *
1285 * @port        pointer to the port structure.
1286 * @buffer      pointer to buffer
1287 * @buffer_dma  dma address corresponding to @buffer
1288 *
1289 * return value
1290 *      @rv     return value from mtip_exec_internal_command()
1291 */
1292static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer,
1293                                        dma_addr_t buffer_dma)
1294{
1295        struct host_to_dev_fis fis;
1296
1297        memset(&fis, 0, sizeof(struct host_to_dev_fis));
1298        fis.type        = 0x27;
1299        fis.opts        = 1 << 7;
1300        fis.command     = ATA_CMD_SMART;
1301        fis.features    = 0xD0;
1302        fis.sect_count  = 1;
1303        fis.lba_mid     = 0x4F;
1304        fis.lba_hi      = 0xC2;
1305        fis.device      = ATA_DEVICE_OBS;
1306
1307        return mtip_exec_internal_command(port,
1308                                        &fis,
1309                                        5,
1310                                        buffer_dma,
1311                                        ATA_SECT_SIZE,
1312                                        0,
1313                                        15000);
1314}
1315
1316/*
1317 * Get the value of a smart attribute
1318 *
1319 * @port        pointer to the port structure
1320 * @id          attribute number
1321 * @attrib      pointer to return attrib information corresponding to @id
1322 *
1323 * return value
1324 *      -EINVAL NULL buffer passed or unsupported attribute @id.
1325 *      -EPERM  Identify data not valid, SMART not supported or not enabled
1326 */
1327static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
1328                                                struct smart_attr *attrib)
1329{
1330        int rv, i;
1331        struct smart_attr *pattr;
1332
1333        if (!attrib)
1334                return -EINVAL;
1335
1336        if (!port->identify_valid) {
1337                dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n");
1338                return -EPERM;
1339        }
1340        if (!(port->identify[82] & 0x1)) {
1341                dev_warn(&port->dd->pdev->dev, "SMART not supported\n");
1342                return -EPERM;
1343        }
1344        if (!(port->identify[85] & 0x1)) {
1345                dev_warn(&port->dd->pdev->dev, "SMART not enabled\n");
1346                return -EPERM;
1347        }
1348
1349        memset(port->smart_buf, 0, ATA_SECT_SIZE);
1350        rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma);
1351        if (rv) {
1352                dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n");
1353                return rv;
1354        }
1355
1356        pattr = (struct smart_attr *)(port->smart_buf + 2);
1357        for (i = 0; i < 29; i++, pattr++)
1358                if (pattr->attr_id == id) {
1359                        memcpy(attrib, pattr, sizeof(struct smart_attr));
1360                        break;
1361                }
1362
1363        if (i == 29) {
1364                dev_warn(&port->dd->pdev->dev,
1365                        "Query for invalid SMART attribute ID\n");
1366                rv = -EINVAL;
1367        }
1368
1369        return rv;
1370}
1371
1372/*
1373 * Get the drive capacity.
1374 *
1375 * @dd      Pointer to the device data structure.
1376 * @sectors Pointer to the variable that will receive the sector count.
1377 *
1378 * return value
1379 *      1 Capacity was returned successfully.
1380 *      0 The identify information is invalid.
1381 */
1382static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors)
1383{
1384        struct mtip_port *port = dd->port;
1385        u64 total, raw0, raw1, raw2, raw3;
1386        raw0 = port->identify[100];
1387        raw1 = port->identify[101];
1388        raw2 = port->identify[102];
1389        raw3 = port->identify[103];
1390        total = raw0 | raw1<<16 | raw2<<32 | raw3<<48;
1391        *sectors = total;
1392        return (bool) !!port->identify_valid;
1393}
1394
1395/*
1396 * Display the identify command data.
1397 *
1398 * @port Pointer to the port data structure.
1399 *
1400 * return value
1401 *      None
1402 */
1403static void mtip_dump_identify(struct mtip_port *port)
1404{
1405        sector_t sectors;
1406        unsigned short revid;
1407        char cbuf[42];
1408
1409        if (!port->identify_valid)
1410                return;
1411
1412        strlcpy(cbuf, (char *)(port->identify+10), 21);
1413        dev_info(&port->dd->pdev->dev,
1414                "Serial No.: %s\n", cbuf);
1415
1416        strlcpy(cbuf, (char *)(port->identify+23), 9);
1417        dev_info(&port->dd->pdev->dev,
1418                "Firmware Ver.: %s\n", cbuf);
1419
1420        strlcpy(cbuf, (char *)(port->identify+27), 41);
1421        dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf);
1422
1423        dev_info(&port->dd->pdev->dev, "Security: %04x %s\n",
1424                port->identify[128],
1425                port->identify[128] & 0x4 ? "(LOCKED)" : "");
1426
1427        if (mtip_hw_get_capacity(port->dd, &sectors))
1428                dev_info(&port->dd->pdev->dev,
1429                        "Capacity: %llu sectors (%llu MB)\n",
1430                         (u64)sectors,
1431                         ((u64)sectors) * ATA_SECT_SIZE >> 20);
1432
1433        pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid);
1434        switch (revid & 0xFF) {
1435        case 0x1:
1436                strlcpy(cbuf, "A0", 3);
1437                break;
1438        case 0x3:
1439                strlcpy(cbuf, "A2", 3);
1440                break;
1441        default:
1442                strlcpy(cbuf, "?", 2);
1443                break;
1444        }
1445        dev_info(&port->dd->pdev->dev,
1446                "Card Type: %s\n", cbuf);
1447}
1448
1449/*
1450 * Map the commands scatter list into the command table.
1451 *
1452 * @command Pointer to the command.
1453 * @nents Number of scatter list entries.
1454 *
1455 * return value
1456 *      None
1457 */
1458static inline void fill_command_sg(struct driver_data *dd,
1459                                struct mtip_cmd *command,
1460                                int nents)
1461{
1462        int n;
1463        unsigned int dma_len;
1464        struct mtip_cmd_sg *command_sg;
1465        struct scatterlist *sg;
1466
1467        command_sg = command->command + AHCI_CMD_TBL_HDR_SZ;
1468
1469        for_each_sg(command->sg, sg, nents, n) {
1470                dma_len = sg_dma_len(sg);
1471                if (dma_len > 0x400000)
1472                        dev_err(&dd->pdev->dev,
1473                                "DMA segment length truncated\n");
1474                command_sg->info = cpu_to_le32((dma_len-1) & 0x3FFFFF);
1475                command_sg->dba =  cpu_to_le32(sg_dma_address(sg));
1476                command_sg->dba_upper =
1477                        cpu_to_le32((sg_dma_address(sg) >> 16) >> 16);
1478                command_sg++;
1479        }
1480}
1481
1482/*
1483 * @brief Execute a drive command.
1484 *
1485 * return value 0 The command completed successfully.
1486 * return value -1 An error occurred while executing the command.
1487 */
1488static int exec_drive_task(struct mtip_port *port, u8 *command)
1489{
1490        struct host_to_dev_fis  fis;
1491        struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG);
1492        unsigned int to;
1493
1494        /* Build the FIS. */
1495        memset(&fis, 0, sizeof(struct host_to_dev_fis));
1496        fis.type        = 0x27;
1497        fis.opts        = 1 << 7;
1498        fis.command     = command[0];
1499        fis.features    = command[1];
1500        fis.sect_count  = command[2];
1501        fis.sector      = command[3];
1502        fis.cyl_low     = command[4];
1503        fis.cyl_hi      = command[5];
1504        fis.device      = command[6] & ~0x10; /* Clear the dev bit*/
1505
1506        mtip_set_timeout(port->dd, &fis, &to, 0);
1507
1508        dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n",
1509                __func__,
1510                command[0],
1511                command[1],
1512                command[2],
1513                command[3],
1514                command[4],
1515                command[5],
1516                command[6]);
1517
1518        /* Execute the command. */
1519        if (mtip_exec_internal_command(port,
1520                                 &fis,
1521                                 5,
1522                                 0,
1523                                 0,
1524                                 0,
1525                                 to) < 0) {
1526                return -1;
1527        }
1528
1529        command[0] = reply->command; /* Status*/
1530        command[1] = reply->features; /* Error*/
1531        command[4] = reply->cyl_low;
1532        command[5] = reply->cyl_hi;
1533
1534        dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n",
1535                __func__,
1536                command[0],
1537                command[1],
1538                command[4],
1539                command[5]);
1540
1541        return 0;
1542}
1543
1544/*
1545 * @brief Execute a drive command.
1546 *
1547 * @param port Pointer to the port data structure.
1548 * @param command Pointer to the user specified command parameters.
1549 * @param user_buffer Pointer to the user space buffer where read sector
1550 *                   data should be copied.
1551 *
1552 * return value 0 The command completed successfully.
1553 * return value -EFAULT An error occurred while copying the completion
1554 *                 data to the user space buffer.
1555 * return value -1 An error occurred while executing the command.
1556 */
1557static int exec_drive_command(struct mtip_port *port, u8 *command,
1558                                void __user *user_buffer)
1559{
1560        struct host_to_dev_fis  fis;
1561        struct host_to_dev_fis *reply;
1562        u8 *buf = NULL;
1563        dma_addr_t dma_addr = 0;
1564        int rv = 0, xfer_sz = command[3];
1565        unsigned int to;
1566
1567        if (xfer_sz) {
1568                if (!user_buffer)
1569                        return -EFAULT;
1570
1571                buf = dma_alloc_coherent(&port->dd->pdev->dev,
1572                                ATA_SECT_SIZE * xfer_sz,
1573                                &dma_addr,
1574                                GFP_KERNEL);
1575                if (!buf) {
1576                        dev_err(&port->dd->pdev->dev,
1577                                "Memory allocation failed (%d bytes)\n",
1578                                ATA_SECT_SIZE * xfer_sz);
1579                        return -ENOMEM;
1580                }
1581        }
1582
1583        /* Build the FIS. */
1584        memset(&fis, 0, sizeof(struct host_to_dev_fis));
1585        fis.type        = 0x27;
1586        fis.opts        = 1 << 7;
1587        fis.command     = command[0];
1588        fis.features    = command[2];
1589        fis.sect_count  = command[3];
1590        if (fis.command == ATA_CMD_SMART) {
1591                fis.sector      = command[1];
1592                fis.cyl_low     = 0x4F;
1593                fis.cyl_hi      = 0xC2;
1594        }
1595
1596        mtip_set_timeout(port->dd, &fis, &to, 0);
1597
1598        if (xfer_sz)
1599                reply = (port->rxfis + RX_FIS_PIO_SETUP);
1600        else
1601                reply = (port->rxfis + RX_FIS_D2H_REG);
1602
1603        dbg_printk(MTIP_DRV_NAME
1604                " %s: User Command: cmd %x, sect %x, "
1605                "feat %x, sectcnt %x\n",
1606                __func__,
1607                command[0],
1608                command[1],
1609                command[2],
1610                command[3]);
1611
1612        /* Execute the command. */
1613        if (mtip_exec_internal_command(port,
1614                                &fis,
1615                                 5,
1616                                 (xfer_sz ? dma_addr : 0),
1617                                 (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0),
1618                                 0,
1619                                 to)
1620                                 < 0) {
1621                rv = -EFAULT;
1622                goto exit_drive_command;
1623        }
1624
1625        /* Collect the completion status. */
1626        command[0] = reply->command; /* Status*/
1627        command[1] = reply->features; /* Error*/
1628        command[2] = reply->sect_count;
1629
1630        dbg_printk(MTIP_DRV_NAME
1631                " %s: Completion Status: stat %x, "
1632                "err %x, nsect %x\n",
1633                __func__,
1634                command[0],
1635                command[1],
1636                command[2]);
1637
1638        if (xfer_sz) {
1639                if (copy_to_user(user_buffer,
1640                                 buf,
1641                                 ATA_SECT_SIZE * command[3])) {
1642                        rv = -EFAULT;
1643                        goto exit_drive_command;
1644                }
1645        }
1646exit_drive_command:
1647        if (buf)
1648                dma_free_coherent(&port->dd->pdev->dev,
1649                                ATA_SECT_SIZE * xfer_sz, buf, dma_addr);
1650        return rv;
1651}
1652
1653/*
1654 *  Indicates whether a command has a single sector payload.
1655 *
1656 *  @command passed to the device to perform the certain event.
1657 *  @features passed to the device to perform the certain event.
1658 *
1659 *  return value
1660 *      1       command is one that always has a single sector payload,
1661 *              regardless of the value in the Sector Count field.
1662 *      0       otherwise
1663 *
1664 */
1665static unsigned int implicit_sector(unsigned char command,
1666                                    unsigned char features)
1667{
1668        unsigned int rv = 0;
1669
1670        /* list of commands that have an implicit sector count of 1 */
1671        switch (command) {
1672        case ATA_CMD_SEC_SET_PASS:
1673        case ATA_CMD_SEC_UNLOCK:
1674        case ATA_CMD_SEC_ERASE_PREP:
1675        case ATA_CMD_SEC_ERASE_UNIT:
1676        case ATA_CMD_SEC_FREEZE_LOCK:
1677        case ATA_CMD_SEC_DISABLE_PASS:
1678        case ATA_CMD_PMP_READ:
1679        case ATA_CMD_PMP_WRITE:
1680                rv = 1;
1681                break;
1682        case ATA_CMD_SET_MAX:
1683                if (features == ATA_SET_MAX_UNLOCK)
1684                        rv = 1;
1685                break;
1686        case ATA_CMD_SMART:
1687                if ((features == ATA_SMART_READ_VALUES) ||
1688                                (features == ATA_SMART_READ_THRESHOLDS))
1689                        rv = 1;
1690                break;
1691        case ATA_CMD_CONF_OVERLAY:
1692                if ((features == ATA_DCO_IDENTIFY) ||
1693                                (features == ATA_DCO_SET))
1694                        rv = 1;
1695                break;
1696        }
1697        return rv;
1698}
1699
1700/*
1701 * Executes a taskfile
1702 * See ide_taskfile_ioctl() for derivation
1703 */
1704static int exec_drive_taskfile(struct driver_data *dd,
1705                               void __user *buf,
1706                               ide_task_request_t *req_task,
1707                               int outtotal)
1708{
1709        struct host_to_dev_fis  fis;
1710        struct host_to_dev_fis *reply;
1711        u8 *outbuf = NULL;
1712        u8 *inbuf = NULL;
1713        dma_addr_t outbuf_dma = 0;
1714        dma_addr_t inbuf_dma = 0;
1715        dma_addr_t dma_buffer = 0;
1716        int err = 0;
1717        unsigned int taskin = 0;
1718        unsigned int taskout = 0;
1719        u8 nsect = 0;
1720        unsigned int timeout;
1721        unsigned int force_single_sector;
1722        unsigned int transfer_size;
1723        unsigned long task_file_data;
1724        int intotal = outtotal + req_task->out_size;
1725        int erasemode = 0;
1726
1727        taskout = req_task->out_size;
1728        taskin = req_task->in_size;
1729        /* 130560 = 512 * 0xFF*/
1730        if (taskin > 130560 || taskout > 130560)
1731                return -EINVAL;
1732
1733        if (taskout) {
1734                outbuf = memdup_user(buf + outtotal, taskout);
1735                if (IS_ERR(outbuf))
1736                        return PTR_ERR(outbuf);
1737
1738                outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf,
1739                                            taskout, DMA_TO_DEVICE);
1740                if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) {
1741                        err = -ENOMEM;
1742                        goto abort;
1743                }
1744                dma_buffer = outbuf_dma;
1745        }
1746
1747        if (taskin) {
1748                inbuf = memdup_user(buf + intotal, taskin);
1749                if (IS_ERR(inbuf)) {
1750                        err = PTR_ERR(inbuf);
1751                        inbuf = NULL;
1752                        goto abort;
1753                }
1754                inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf,
1755                                           taskin, DMA_FROM_DEVICE);
1756                if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) {
1757                        err = -ENOMEM;
1758                        goto abort;
1759                }
1760                dma_buffer = inbuf_dma;
1761        }
1762
1763        /* only supports PIO and non-data commands from this ioctl. */
1764        switch (req_task->data_phase) {
1765        case TASKFILE_OUT:
1766                nsect = taskout / ATA_SECT_SIZE;
1767                reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1768                break;
1769        case TASKFILE_IN:
1770                reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1771                break;
1772        case TASKFILE_NO_DATA:
1773                reply = (dd->port->rxfis + RX_FIS_D2H_REG);
1774                break;
1775        default:
1776                err = -EINVAL;
1777                goto abort;
1778        }
1779
1780        /* Build the FIS. */
1781        memset(&fis, 0, sizeof(struct host_to_dev_fis));
1782
1783        fis.type        = 0x27;
1784        fis.opts        = 1 << 7;
1785        fis.command     = req_task->io_ports[7];
1786        fis.features    = req_task->io_ports[1];
1787        fis.sect_count  = req_task->io_ports[2];
1788        fis.lba_low     = req_task->io_ports[3];
1789        fis.lba_mid     = req_task->io_ports[4];
1790        fis.lba_hi      = req_task->io_ports[5];
1791         /* Clear the dev bit*/
1792        fis.device      = req_task->io_ports[6] & ~0x10;
1793
1794        if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) {
1795                req_task->in_flags.all  =
1796                        IDE_TASKFILE_STD_IN_FLAGS |
1797                        (IDE_HOB_STD_IN_FLAGS << 8);
1798                fis.lba_low_ex          = req_task->hob_ports[3];
1799                fis.lba_mid_ex          = req_task->hob_ports[4];
1800                fis.lba_hi_ex           = req_task->hob_ports[5];
1801                fis.features_ex         = req_task->hob_ports[1];
1802                fis.sect_cnt_ex         = req_task->hob_ports[2];
1803
1804        } else {
1805                req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS;
1806        }
1807
1808        force_single_sector = implicit_sector(fis.command, fis.features);
1809
1810        if ((taskin || taskout) && (!fis.sect_count)) {
1811                if (nsect)
1812                        fis.sect_count = nsect;
1813                else {
1814                        if (!force_single_sector) {
1815                                dev_warn(&dd->pdev->dev,
1816                                        "data movement but "
1817                                        "sect_count is 0\n");
1818                                err = -EINVAL;
1819                                goto abort;
1820                        }
1821                }
1822        }
1823
1824        dbg_printk(MTIP_DRV_NAME
1825                " %s: cmd %x, feat %x, nsect %x,"
1826                " sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x,"
1827                " head/dev %x\n",
1828                __func__,
1829                fis.command,
1830                fis.features,
1831                fis.sect_count,
1832                fis.lba_low,
1833                fis.lba_mid,
1834                fis.lba_hi,
1835                fis.device);
1836
1837        /* check for erase mode support during secure erase.*/
1838        if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf &&
1839                                        (outbuf[0] & MTIP_SEC_ERASE_MODE)) {
1840                erasemode = 1;
1841        }
1842
1843        mtip_set_timeout(dd, &fis, &timeout, erasemode);
1844
1845        /* Determine the correct transfer size.*/
1846        if (force_single_sector)
1847                transfer_size = ATA_SECT_SIZE;
1848        else
1849                transfer_size = ATA_SECT_SIZE * fis.sect_count;
1850
1851        /* Execute the command.*/
1852        if (mtip_exec_internal_command(dd->port,
1853                                 &fis,
1854                                 5,
1855                                 dma_buffer,
1856                                 transfer_size,
1857                                 0,
1858                                 timeout) < 0) {
1859                err = -EIO;
1860                goto abort;
1861        }
1862
1863        task_file_data = readl(dd->port->mmio+PORT_TFDATA);
1864
1865        if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) {
1866                reply = dd->port->rxfis + RX_FIS_PIO_SETUP;
1867                req_task->io_ports[7] = reply->control;
1868        } else {
1869                reply = dd->port->rxfis + RX_FIS_D2H_REG;
1870                req_task->io_ports[7] = reply->command;
1871        }
1872
1873        /* reclaim the DMA buffers.*/
1874        if (inbuf_dma)
1875                dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
1876                                 DMA_FROM_DEVICE);
1877        if (outbuf_dma)
1878                dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
1879                                 DMA_TO_DEVICE);
1880        inbuf_dma  = 0;
1881        outbuf_dma = 0;
1882
1883        /* return the ATA registers to the caller.*/
1884        req_task->io_ports[1] = reply->features;
1885        req_task->io_ports[2] = reply->sect_count;
1886        req_task->io_ports[3] = reply->lba_low;
1887        req_task->io_ports[4] = reply->lba_mid;
1888        req_task->io_ports[5] = reply->lba_hi;
1889        req_task->io_ports[6] = reply->device;
1890
1891        if (req_task->out_flags.all & 1)  {
1892
1893                req_task->hob_ports[3] = reply->lba_low_ex;
1894                req_task->hob_ports[4] = reply->lba_mid_ex;
1895                req_task->hob_ports[5] = reply->lba_hi_ex;
1896                req_task->hob_ports[1] = reply->features_ex;
1897                req_task->hob_ports[2] = reply->sect_cnt_ex;
1898        }
1899        dbg_printk(MTIP_DRV_NAME
1900                " %s: Completion: stat %x,"
1901                "err %x, sect_cnt %x, lbalo %x,"
1902                "lbamid %x, lbahi %x, dev %x\n",
1903                __func__,
1904                req_task->io_ports[7],
1905                req_task->io_ports[1],
1906                req_task->io_ports[2],
1907                req_task->io_ports[3],
1908                req_task->io_ports[4],
1909                req_task->io_ports[5],
1910                req_task->io_ports[6]);
1911
1912        if (taskout) {
1913                if (copy_to_user(buf + outtotal, outbuf, taskout)) {
1914                        err = -EFAULT;
1915                        goto abort;
1916                }
1917        }
1918        if (taskin) {
1919                if (copy_to_user(buf + intotal, inbuf, taskin)) {
1920                        err = -EFAULT;
1921                        goto abort;
1922                }
1923        }
1924abort:
1925        if (inbuf_dma)
1926                dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
1927                                 DMA_FROM_DEVICE);
1928        if (outbuf_dma)
1929                dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
1930                                 DMA_TO_DEVICE);
1931        kfree(outbuf);
1932        kfree(inbuf);
1933
1934        return err;
1935}
1936
1937/*
1938 * Handle IOCTL calls from the Block Layer.
1939 *
1940 * This function is called by the Block Layer when it receives an IOCTL
1941 * command that it does not understand. If the IOCTL command is not supported
1942 * this function returns -ENOTTY.
1943 *
1944 * @dd  Pointer to the driver data structure.
1945 * @cmd IOCTL command passed from the Block Layer.
1946 * @arg IOCTL argument passed from the Block Layer.
1947 *
1948 * return value
1949 *      0       The IOCTL completed successfully.
1950 *      -ENOTTY The specified command is not supported.
1951 *      -EFAULT An error occurred copying data to a user space buffer.
1952 *      -EIO    An error occurred while executing the command.
1953 */
1954static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd,
1955                         unsigned long arg)
1956{
1957        switch (cmd) {
1958        case HDIO_GET_IDENTITY:
1959        {
1960                if (copy_to_user((void __user *)arg, dd->port->identify,
1961                                                sizeof(u16) * ATA_ID_WORDS))
1962                        return -EFAULT;
1963                break;
1964        }
1965        case HDIO_DRIVE_CMD:
1966        {
1967                u8 drive_command[4];
1968
1969                /* Copy the user command info to our buffer. */
1970                if (copy_from_user(drive_command,
1971                                         (void __user *) arg,
1972                                         sizeof(drive_command)))
1973                        return -EFAULT;
1974
1975                /* Execute the drive command. */
1976                if (exec_drive_command(dd->port,
1977                                         drive_command,
1978                                         (void __user *) (arg+4)))
1979                        return -EIO;
1980
1981                /* Copy the status back to the users buffer. */
1982                if (copy_to_user((void __user *) arg,
1983                                         drive_command,
1984                                         sizeof(drive_command)))
1985                        return -EFAULT;
1986
1987                break;
1988        }
1989        case HDIO_DRIVE_TASK:
1990        {
1991                u8 drive_command[7];
1992
1993                /* Copy the user command info to our buffer. */
1994                if (copy_from_user(drive_command,
1995                                         (void __user *) arg,
1996                                         sizeof(drive_command)))
1997                        return -EFAULT;
1998
1999                /* Execute the drive command. */
2000                if (exec_drive_task(dd->port, drive_command))
2001                        return -EIO;
2002
2003                /* Copy the status back to the users buffer. */
2004                if (copy_to_user((void __user *) arg,
2005                                         drive_command,
2006                                         sizeof(drive_command)))
2007                        return -EFAULT;
2008
2009                break;
2010        }
2011        case HDIO_DRIVE_TASKFILE: {
2012                ide_task_request_t req_task;
2013                int ret, outtotal;
2014
2015                if (copy_from_user(&req_task, (void __user *) arg,
2016                                        sizeof(req_task)))
2017                        return -EFAULT;
2018
2019                outtotal = sizeof(req_task);
2020
2021                ret = exec_drive_taskfile(dd, (void __user *) arg,
2022                                                &req_task, outtotal);
2023
2024                if (copy_to_user((void __user *) arg, &req_task,
2025                                                        sizeof(req_task)))
2026                        return -EFAULT;
2027
2028                return ret;
2029        }
2030
2031        default:
2032                return -EINVAL;
2033        }
2034        return 0;
2035}
2036
2037/*
2038 * Submit an IO to the hw
2039 *
2040 * This function is called by the block layer to issue an io
2041 * to the device. Upon completion, the callback function will
2042 * be called with the data parameter passed as the callback data.
2043 *
2044 * @dd       Pointer to the driver data structure.
2045 * @start    First sector to read.
2046 * @nsect    Number of sectors to read.
2047 * @tag      The tag of this read command.
2048 * @callback Pointer to the function that should be called
2049 *           when the read completes.
2050 * @data     Callback data passed to the callback function
2051 *           when the read completes.
2052 * @dir      Direction (read or write)
2053 *
2054 * return value
2055 *      None
2056 */
2057static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq,
2058                              struct mtip_cmd *command,
2059                              struct blk_mq_hw_ctx *hctx)
2060{
2061        struct mtip_cmd_hdr *hdr =
2062                dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
2063        struct host_to_dev_fis  *fis;
2064        struct mtip_port *port = dd->port;
2065        int dma_dir = rq_data_dir(rq) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
2066        u64 start = blk_rq_pos(rq);
2067        unsigned int nsect = blk_rq_sectors(rq);
2068        unsigned int nents;
2069
2070        /* Map the scatter list for DMA access */
2071        nents = blk_rq_map_sg(hctx->queue, rq, command->sg);
2072        nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir);
2073
2074        prefetch(&port->flags);
2075
2076        command->scatter_ents = nents;
2077
2078        /*
2079         * The number of retries for this command before it is
2080         * reported as a failure to the upper layers.
2081         */
2082        command->retries = MTIP_MAX_RETRIES;
2083
2084        /* Fill out fis */
2085        fis = command->command;
2086        fis->type        = 0x27;
2087        fis->opts        = 1 << 7;
2088        if (dma_dir == DMA_FROM_DEVICE)
2089                fis->command = ATA_CMD_FPDMA_READ;
2090        else
2091                fis->command = ATA_CMD_FPDMA_WRITE;
2092        fis->lba_low     = start & 0xFF;
2093        fis->lba_mid     = (start >> 8) & 0xFF;
2094        fis->lba_hi      = (start >> 16) & 0xFF;
2095        fis->lba_low_ex  = (start >> 24) & 0xFF;
2096        fis->lba_mid_ex  = (start >> 32) & 0xFF;
2097        fis->lba_hi_ex   = (start >> 40) & 0xFF;
2098        fis->device      = 1 << 6;
2099        fis->features    = nsect & 0xFF;
2100        fis->features_ex = (nsect >> 8) & 0xFF;
2101        fis->sect_count  = ((rq->tag << 3) | (rq->tag >> 5));
2102        fis->sect_cnt_ex = 0;
2103        fis->control     = 0;
2104        fis->res2        = 0;
2105        fis->res3        = 0;
2106        fill_command_sg(dd, command, nents);
2107
2108        if (unlikely(command->unaligned))
2109                fis->device |= 1 << 7;
2110
2111        /* Populate the command header */
2112        hdr->ctba = cpu_to_le32(command->command_dma & 0xFFFFFFFF);
2113        if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
2114                hdr->ctbau = cpu_to_le32((command->command_dma >> 16) >> 16);
2115        hdr->opts = cpu_to_le32((nents << 16) | 5 | AHCI_CMD_PREFETCH);
2116        hdr->byte_count = 0;
2117
2118        command->direction = dma_dir;
2119
2120        /*
2121         * To prevent this command from being issued
2122         * if an internal command is in progress or error handling is active.
2123         */
2124        if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) {
2125                set_bit(rq->tag, port->cmds_to_issue);
2126                set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
2127                return;
2128        }
2129
2130        /* Issue the command to the hardware */
2131        mtip_issue_ncq_command(port, rq->tag);
2132}
2133
2134/*
2135 * Sysfs status dump.
2136 *
2137 * @dev  Pointer to the device structure, passed by the kernrel.
2138 * @attr Pointer to the device_attribute structure passed by the kernel.
2139 * @buf  Pointer to the char buffer that will receive the stats info.
2140 *
2141 * return value
2142 *      The size, in bytes, of the data copied into buf.
2143 */
2144static ssize_t mtip_hw_show_status(struct device *dev,
2145                                struct device_attribute *attr,
2146                                char *buf)
2147{
2148        struct driver_data *dd = dev_to_disk(dev)->private_data;
2149        int size = 0;
2150
2151        if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
2152                size += sprintf(buf, "%s", "thermal_shutdown\n");
2153        else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag))
2154                size += sprintf(buf, "%s", "write_protect\n");
2155        else
2156                size += sprintf(buf, "%s", "online\n");
2157
2158        return size;
2159}
2160
2161static DEVICE_ATTR(status, 0444, mtip_hw_show_status, NULL);
2162
2163/* debugsfs entries */
2164
2165static ssize_t show_device_status(struct device_driver *drv, char *buf)
2166{
2167        int size = 0;
2168        struct driver_data *dd, *tmp;
2169        unsigned long flags;
2170        char id_buf[42];
2171        u16 status = 0;
2172
2173        spin_lock_irqsave(&dev_lock, flags);
2174        size += sprintf(&buf[size], "Devices Present:\n");
2175        list_for_each_entry_safe(dd, tmp, &online_list, online_list) {
2176                if (dd->pdev) {
2177                        if (dd->port &&
2178                            dd->port->identify &&
2179                            dd->port->identify_valid) {
2180                                strlcpy(id_buf,
2181                                        (char *) (dd->port->identify + 10), 21);
2182                                status = *(dd->port->identify + 141);
2183                        } else {
2184                                memset(id_buf, 0, 42);
2185                                status = 0;
2186                        }
2187
2188                        if (dd->port &&
2189                            test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
2190                                size += sprintf(&buf[size],
2191                                        " device %s %s (ftl rebuild %d %%)\n",
2192                                        dev_name(&dd->pdev->dev),
2193                                        id_buf,
2194                                        status);
2195                        } else {
2196                                size += sprintf(&buf[size],
2197                                        " device %s %s\n",
2198                                        dev_name(&dd->pdev->dev),
2199                                        id_buf);
2200                        }
2201                }
2202        }
2203
2204        size += sprintf(&buf[size], "Devices Being Removed:\n");
2205        list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) {
2206                if (dd->pdev) {
2207                        if (dd->port &&
2208                            dd->port->identify &&
2209                            dd->port->identify_valid) {
2210                                strlcpy(id_buf,
2211                                        (char *) (dd->port->identify+10), 21);
2212                                status = *(dd->port->identify + 141);
2213                        } else {
2214                                memset(id_buf, 0, 42);
2215                                status = 0;
2216                        }
2217
2218                        if (dd->port &&
2219                            test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
2220                                size += sprintf(&buf[size],
2221                                        " device %s %s (ftl rebuild %d %%)\n",
2222                                        dev_name(&dd->pdev->dev),
2223                                        id_buf,
2224                                        status);
2225                        } else {
2226                                size += sprintf(&buf[size],
2227                                        " device %s %s\n",
2228                                        dev_name(&dd->pdev->dev),
2229                                        id_buf);
2230                        }
2231                }
2232        }
2233        spin_unlock_irqrestore(&dev_lock, flags);
2234
2235        return size;
2236}
2237
2238static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf,
2239                                                size_t len, loff_t *offset)
2240{
2241        struct driver_data *dd =  (struct driver_data *)f->private_data;
2242        int size = *offset;
2243        char *buf;
2244        int rv = 0;
2245
2246        if (!len || *offset)
2247                return 0;
2248
2249        buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2250        if (!buf) {
2251                dev_err(&dd->pdev->dev,
2252                        "Memory allocation: status buffer\n");
2253                return -ENOMEM;
2254        }
2255
2256        size += show_device_status(NULL, buf);
2257
2258        *offset = size <= len ? size : len;
2259        size = copy_to_user(ubuf, buf, *offset);
2260        if (size)
2261                rv = -EFAULT;
2262
2263        kfree(buf);
2264        return rv ? rv : *offset;
2265}
2266
2267static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf,
2268                                  size_t len, loff_t *offset)
2269{
2270        struct driver_data *dd =  (struct driver_data *)f->private_data;
2271        char *buf;
2272        u32 group_allocated;
2273        int size = *offset;
2274        int n, rv = 0;
2275
2276        if (!len || size)
2277                return 0;
2278
2279        buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2280        if (!buf) {
2281                dev_err(&dd->pdev->dev,
2282                        "Memory allocation: register buffer\n");
2283                return -ENOMEM;
2284        }
2285
2286        size += sprintf(&buf[size], "H/ S ACTive      : [ 0x");
2287
2288        for (n = dd->slot_groups-1; n >= 0; n--)
2289                size += sprintf(&buf[size], "%08X ",
2290                                         readl(dd->port->s_active[n]));
2291
2292        size += sprintf(&buf[size], "]\n");
2293        size += sprintf(&buf[size], "H/ Command Issue : [ 0x");
2294
2295        for (n = dd->slot_groups-1; n >= 0; n--)
2296                size += sprintf(&buf[size], "%08X ",
2297                                        readl(dd->port->cmd_issue[n]));
2298
2299        size += sprintf(&buf[size], "]\n");
2300        size += sprintf(&buf[size], "H/ Completed     : [ 0x");
2301
2302        for (n = dd->slot_groups-1; n >= 0; n--)
2303                size += sprintf(&buf[size], "%08X ",
2304                                readl(dd->port->completed[n]));
2305
2306        size += sprintf(&buf[size], "]\n");
2307        size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n",
2308                                readl(dd->port->mmio + PORT_IRQ_STAT));
2309        size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n",
2310                                readl(dd->mmio + HOST_IRQ_STAT));
2311        size += sprintf(&buf[size], "\n");
2312
2313        size += sprintf(&buf[size], "L/ Commands in Q : [ 0x");
2314
2315        for (n = dd->slot_groups-1; n >= 0; n--) {
2316                if (sizeof(long) > sizeof(u32))
2317                        group_allocated =
2318                                dd->port->cmds_to_issue[n/2] >> (32*(n&1));
2319                else
2320                        group_allocated = dd->port->cmds_to_issue[n];
2321                size += sprintf(&buf[size], "%08X ", group_allocated);
2322        }
2323        size += sprintf(&buf[size], "]\n");
2324
2325        *offset = size <= len ? size : len;
2326        size = copy_to_user(ubuf, buf, *offset);
2327        if (size)
2328                rv = -EFAULT;
2329
2330        kfree(buf);
2331        return rv ? rv : *offset;
2332}
2333
2334static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf,
2335                                  size_t len, loff_t *offset)
2336{
2337        struct driver_data *dd =  (struct driver_data *)f->private_data;
2338        char *buf;
2339        int size = *offset;
2340        int rv = 0;
2341
2342        if (!len || size)
2343                return 0;
2344
2345        buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2346        if (!buf) {
2347                dev_err(&dd->pdev->dev,
2348                        "Memory allocation: flag buffer\n");
2349                return -ENOMEM;
2350        }
2351
2352        size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n",
2353                                                        dd->port->flags);
2354        size += sprintf(&buf[size], "Flag-dd   : [ %08lX ]\n",
2355                                                        dd->dd_flag);
2356
2357        *offset = size <= len ? size : len;
2358        size = copy_to_user(ubuf, buf, *offset);
2359        if (size)
2360                rv = -EFAULT;
2361
2362        kfree(buf);
2363        return rv ? rv : *offset;
2364}
2365
2366static const struct file_operations mtip_device_status_fops = {
2367        .owner  = THIS_MODULE,
2368        .open   = simple_open,
2369        .read   = mtip_hw_read_device_status,
2370        .llseek = no_llseek,
2371};
2372
2373static const struct file_operations mtip_regs_fops = {
2374        .owner  = THIS_MODULE,
2375        .open   = simple_open,
2376        .read   = mtip_hw_read_registers,
2377        .llseek = no_llseek,
2378};
2379
2380static const struct file_operations mtip_flags_fops = {
2381        .owner  = THIS_MODULE,
2382        .open   = simple_open,
2383        .read   = mtip_hw_read_flags,
2384        .llseek = no_llseek,
2385};
2386
2387/*
2388 * Create the sysfs related attributes.
2389 *
2390 * @dd   Pointer to the driver data structure.
2391 * @kobj Pointer to the kobj for the block device.
2392 *
2393 * return value
2394 *      0       Operation completed successfully.
2395 *      -EINVAL Invalid parameter.
2396 */
2397static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj)
2398{
2399        if (!kobj || !dd)
2400                return -EINVAL;
2401
2402        if (sysfs_create_file(kobj, &dev_attr_status.attr))
2403                dev_warn(&dd->pdev->dev,
2404                        "Error creating 'status' sysfs entry\n");
2405        return 0;
2406}
2407
2408/*
2409 * Remove the sysfs related attributes.
2410 *
2411 * @dd   Pointer to the driver data structure.
2412 * @kobj Pointer to the kobj for the block device.
2413 *
2414 * return value
2415 *      0       Operation completed successfully.
2416 *      -EINVAL Invalid parameter.
2417 */
2418static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj)
2419{
2420        if (!kobj || !dd)
2421                return -EINVAL;
2422
2423        sysfs_remove_file(kobj, &dev_attr_status.attr);
2424
2425        return 0;
2426}
2427
2428static int mtip_hw_debugfs_init(struct driver_data *dd)
2429{
2430        if (!dfs_parent)
2431                return -1;
2432
2433        dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent);
2434        if (IS_ERR_OR_NULL(dd->dfs_node)) {
2435                dev_warn(&dd->pdev->dev,
2436                        "Error creating node %s under debugfs\n",
2437                                                dd->disk->disk_name);
2438                dd->dfs_node = NULL;
2439                return -1;
2440        }
2441
2442        debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops);
2443        debugfs_create_file("registers", 0444, dd->dfs_node, dd,
2444                            &mtip_regs_fops);
2445
2446        return 0;
2447}
2448
2449static void mtip_hw_debugfs_exit(struct driver_data *dd)
2450{
2451        debugfs_remove_recursive(dd->dfs_node);
2452}
2453
2454/*
2455 * Perform any init/resume time hardware setup
2456 *
2457 * @dd Pointer to the driver data structure.
2458 *
2459 * return value
2460 *      None
2461 */
2462static inline void hba_setup(struct driver_data *dd)
2463{
2464        u32 hwdata;
2465        hwdata = readl(dd->mmio + HOST_HSORG);
2466
2467        /* interrupt bug workaround: use only 1 IS bit.*/
2468        writel(hwdata |
2469                HSORG_DISABLE_SLOTGRP_INTR |
2470                HSORG_DISABLE_SLOTGRP_PXIS,
2471                dd->mmio + HOST_HSORG);
2472}
2473
2474static int mtip_device_unaligned_constrained(struct driver_data *dd)
2475{
2476        return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0);
2477}
2478
2479/*
2480 * Detect the details of the product, and store anything needed
2481 * into the driver data structure.  This includes product type and
2482 * version and number of slot groups.
2483 *
2484 * @dd Pointer to the driver data structure.
2485 *
2486 * return value
2487 *      None
2488 */
2489static void mtip_detect_product(struct driver_data *dd)
2490{
2491        u32 hwdata;
2492        unsigned int rev, slotgroups;
2493
2494        /*
2495         * HBA base + 0xFC [15:0] - vendor-specific hardware interface
2496         * info register:
2497         * [15:8] hardware/software interface rev#
2498         * [   3] asic-style interface
2499         * [ 2:0] number of slot groups, minus 1 (only valid for asic-style).
2500         */
2501        hwdata = readl(dd->mmio + HOST_HSORG);
2502
2503        dd->product_type = MTIP_PRODUCT_UNKNOWN;
2504        dd->slot_groups = 1;
2505
2506        if (hwdata & 0x8) {
2507                dd->product_type = MTIP_PRODUCT_ASICFPGA;
2508                rev = (hwdata & HSORG_HWREV) >> 8;
2509                slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1;
2510                dev_info(&dd->pdev->dev,
2511                        "ASIC-FPGA design, HS rev 0x%x, "
2512                        "%i slot groups [%i slots]\n",
2513                         rev,
2514                         slotgroups,
2515                         slotgroups * 32);
2516
2517                if (slotgroups > MTIP_MAX_SLOT_GROUPS) {
2518                        dev_warn(&dd->pdev->dev,
2519                                "Warning: driver only supports "
2520                                "%i slot groups.\n", MTIP_MAX_SLOT_GROUPS);
2521                        slotgroups = MTIP_MAX_SLOT_GROUPS;
2522                }
2523                dd->slot_groups = slotgroups;
2524                return;
2525        }
2526
2527        dev_warn(&dd->pdev->dev, "Unrecognized product id\n");
2528}
2529
2530/*
2531 * Blocking wait for FTL rebuild to complete
2532 *
2533 * @dd Pointer to the DRIVER_DATA structure.
2534 *
2535 * return value
2536 *      0       FTL rebuild completed successfully
2537 *      -EFAULT FTL rebuild error/timeout/interruption
2538 */
2539static int mtip_ftl_rebuild_poll(struct driver_data *dd)
2540{
2541        unsigned long timeout, cnt = 0, start;
2542
2543        dev_warn(&dd->pdev->dev,
2544                "FTL rebuild in progress. Polling for completion.\n");
2545
2546        start = jiffies;
2547        timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS);
2548
2549        do {
2550                if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
2551                                &dd->dd_flag)))
2552                        return -EFAULT;
2553                if (mtip_check_surprise_removal(dd->pdev))
2554                        return -EFAULT;
2555
2556                if (mtip_get_identify(dd->port, NULL) < 0)
2557                        return -EFAULT;
2558
2559                if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2560                        MTIP_FTL_REBUILD_MAGIC) {
2561                        ssleep(1);
2562                        /* Print message every 3 minutes */
2563                        if (cnt++ >= 180) {
2564                                dev_warn(&dd->pdev->dev,
2565                                "FTL rebuild in progress (%d secs).\n",
2566                                jiffies_to_msecs(jiffies - start) / 1000);
2567                                cnt = 0;
2568                        }
2569                } else {
2570                        dev_warn(&dd->pdev->dev,
2571                                "FTL rebuild complete (%d secs).\n",
2572                        jiffies_to_msecs(jiffies - start) / 1000);
2573                        mtip_block_initialize(dd);
2574                        return 0;
2575                }
2576        } while (time_before(jiffies, timeout));
2577
2578        /* Check for timeout */
2579        dev_err(&dd->pdev->dev,
2580                "Timed out waiting for FTL rebuild to complete (%d secs).\n",
2581                jiffies_to_msecs(jiffies - start) / 1000);
2582        return -EFAULT;
2583}
2584
2585static void mtip_softirq_done_fn(struct request *rq)
2586{
2587        struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
2588        struct driver_data *dd = rq->q->queuedata;
2589
2590        /* Unmap the DMA scatter list entries */
2591        dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents,
2592                                                        cmd->direction);
2593
2594        if (unlikely(cmd->unaligned))
2595                atomic_inc(&dd->port->cmd_slot_unal);
2596
2597        blk_mq_end_request(rq, cmd->status);
2598}
2599
2600static bool mtip_abort_cmd(struct request *req, void *data, bool reserved)
2601{
2602        struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
2603        struct driver_data *dd = data;
2604
2605        dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag);
2606
2607        clear_bit(req->tag, dd->port->cmds_to_issue);
2608        cmd->status = BLK_STS_IOERR;
2609        mtip_softirq_done_fn(req);
2610        return true;
2611}
2612
2613static bool mtip_queue_cmd(struct request *req, void *data, bool reserved)
2614{
2615        struct driver_data *dd = data;
2616
2617        set_bit(req->tag, dd->port->cmds_to_issue);
2618        blk_abort_request(req);
2619        return true;
2620}
2621
2622/*
2623 * service thread to issue queued commands
2624 *
2625 * @data Pointer to the driver data structure.
2626 *
2627 * return value
2628 *      0
2629 */
2630
2631static int mtip_service_thread(void *data)
2632{
2633        struct driver_data *dd = (struct driver_data *)data;
2634        unsigned long slot, slot_start, slot_wrap, to;
2635        unsigned int num_cmd_slots = dd->slot_groups * 32;
2636        struct mtip_port *port = dd->port;
2637
2638        while (1) {
2639                if (kthread_should_stop() ||
2640                        test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
2641                        goto st_out;
2642                clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
2643
2644                /*
2645                 * the condition is to check neither an internal command is
2646                 * is in progress nor error handling is active
2647                 */
2648                wait_event_interruptible(port->svc_wait, (port->flags) &&
2649                        (port->flags & MTIP_PF_SVC_THD_WORK));
2650
2651                if (kthread_should_stop() ||
2652                        test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
2653                        goto st_out;
2654
2655                if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
2656                                &dd->dd_flag)))
2657                        goto st_out;
2658
2659                set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
2660
2661restart_eh:
2662                /* Demux bits: start with error handling */
2663                if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) {
2664                        mtip_handle_tfe(dd);
2665                        clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags);
2666                }
2667
2668                if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags))
2669                        goto restart_eh;
2670
2671                if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) {
2672                        to = jiffies + msecs_to_jiffies(5000);
2673
2674                        do {
2675                                mdelay(100);
2676                        } while (atomic_read(&dd->irq_workers_active) != 0 &&
2677                                time_before(jiffies, to));
2678
2679                        if (atomic_read(&dd->irq_workers_active) != 0)
2680                                dev_warn(&dd->pdev->dev,
2681                                        "Completion workers still active!");
2682
2683                        blk_mq_quiesce_queue(dd->queue);
2684
2685                        blk_mq_tagset_busy_iter(&dd->tags, mtip_queue_cmd, dd);
2686
2687                        set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags);
2688
2689                        if (mtip_device_reset(dd))
2690                                blk_mq_tagset_busy_iter(&dd->tags,
2691                                                        mtip_abort_cmd, dd);
2692
2693                        clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags);
2694
2695                        blk_mq_unquiesce_queue(dd->queue);
2696                }
2697
2698                if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
2699                        slot = 1;
2700                        /* used to restrict the loop to one iteration */
2701                        slot_start = num_cmd_slots;
2702                        slot_wrap = 0;
2703                        while (1) {
2704                                slot = find_next_bit(port->cmds_to_issue,
2705                                                num_cmd_slots, slot);
2706                                if (slot_wrap == 1) {
2707                                        if ((slot_start >= slot) ||
2708                                                (slot >= num_cmd_slots))
2709                                                break;
2710                                }
2711                                if (unlikely(slot_start == num_cmd_slots))
2712                                        slot_start = slot;
2713
2714                                if (unlikely(slot == num_cmd_slots)) {
2715                                        slot = 1;
2716                                        slot_wrap = 1;
2717                                        continue;
2718                                }
2719
2720                                /* Issue the command to the hardware */
2721                                mtip_issue_ncq_command(port, slot);
2722
2723                                clear_bit(slot, port->cmds_to_issue);
2724                        }
2725
2726                        clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
2727                }
2728
2729                if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) {
2730                        if (mtip_ftl_rebuild_poll(dd) == 0)
2731                                clear_bit(MTIP_PF_REBUILD_BIT, &port->flags);
2732                }
2733        }
2734
2735st_out:
2736        return 0;
2737}
2738
2739/*
2740 * DMA region teardown
2741 *
2742 * @dd Pointer to driver_data structure
2743 *
2744 * return value
2745 *      None
2746 */
2747static void mtip_dma_free(struct driver_data *dd)
2748{
2749        struct mtip_port *port = dd->port;
2750
2751        if (port->block1)
2752                dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2753                                        port->block1, port->block1_dma);
2754
2755        if (port->command_list) {
2756                dma_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
2757                                port->command_list, port->command_list_dma);
2758        }
2759}
2760
2761/*
2762 * DMA region setup
2763 *
2764 * @dd Pointer to driver_data structure
2765 *
2766 * return value
2767 *      -ENOMEM Not enough free DMA region space to initialize driver
2768 */
2769static int mtip_dma_alloc(struct driver_data *dd)
2770{
2771        struct mtip_port *port = dd->port;
2772
2773        /* Allocate dma memory for RX Fis, Identify, and Sector Bufffer */
2774        port->block1 =
2775                dma_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2776                                        &port->block1_dma, GFP_KERNEL);
2777        if (!port->block1)
2778                return -ENOMEM;
2779
2780        /* Allocate dma memory for command list */
2781        port->command_list =
2782                dma_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
2783                                        &port->command_list_dma, GFP_KERNEL);
2784        if (!port->command_list) {
2785                dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2786                                        port->block1, port->block1_dma);
2787                port->block1 = NULL;
2788                port->block1_dma = 0;
2789                return -ENOMEM;
2790        }
2791
2792        /* Setup all pointers into first DMA region */
2793        port->rxfis         = port->block1 + AHCI_RX_FIS_OFFSET;
2794        port->rxfis_dma     = port->block1_dma + AHCI_RX_FIS_OFFSET;
2795        port->identify      = port->block1 + AHCI_IDFY_OFFSET;
2796        port->identify_dma  = port->block1_dma + AHCI_IDFY_OFFSET;
2797        port->log_buf       = port->block1 + AHCI_SECTBUF_OFFSET;
2798        port->log_buf_dma   = port->block1_dma + AHCI_SECTBUF_OFFSET;
2799        port->smart_buf     = port->block1 + AHCI_SMARTBUF_OFFSET;
2800        port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET;
2801
2802        return 0;
2803}
2804
2805static int mtip_hw_get_identify(struct driver_data *dd)
2806{
2807        struct smart_attr attr242;
2808        unsigned char *buf;
2809        int rv;
2810
2811        if (mtip_get_identify(dd->port, NULL) < 0)
2812                return -EFAULT;
2813
2814        if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2815                MTIP_FTL_REBUILD_MAGIC) {
2816                set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags);
2817                return MTIP_FTL_REBUILD_MAGIC;
2818        }
2819        mtip_dump_identify(dd->port);
2820
2821        /* check write protect, over temp and rebuild statuses */
2822        rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
2823                                dd->port->log_buf,
2824                                dd->port->log_buf_dma, 1);
2825        if (rv) {
2826                dev_warn(&dd->pdev->dev,
2827                        "Error in READ LOG EXT (10h) command\n");
2828                /* non-critical error, don't fail the load */
2829        } else {
2830                buf = (unsigned char *)dd->port->log_buf;
2831                if (buf[259] & 0x1) {
2832                        dev_info(&dd->pdev->dev,
2833                                "Write protect bit is set.\n");
2834                        set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
2835                }
2836                if (buf[288] == 0xF7) {
2837                        dev_info(&dd->pdev->dev,
2838                                "Exceeded Tmax, drive in thermal shutdown.\n");
2839                        set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
2840                }
2841                if (buf[288] == 0xBF) {
2842                        dev_info(&dd->pdev->dev,
2843                                "Drive indicates rebuild has failed.\n");
2844                        set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
2845                }
2846        }
2847
2848        /* get write protect progess */
2849        memset(&attr242, 0, sizeof(struct smart_attr));
2850        if (mtip_get_smart_attr(dd->port, 242, &attr242))
2851                dev_warn(&dd->pdev->dev,
2852                                "Unable to check write protect progress\n");
2853        else
2854                dev_info(&dd->pdev->dev,
2855                                "Write protect progress: %u%% (%u blocks)\n",
2856                                attr242.cur, le32_to_cpu(attr242.data));
2857
2858        return rv;
2859}
2860
2861/*
2862 * Called once for each card.
2863 *
2864 * @dd Pointer to the driver data structure.
2865 *
2866 * return value
2867 *      0 on success, else an error code.
2868 */
2869static int mtip_hw_init(struct driver_data *dd)
2870{
2871        int i;
2872        int rv;
2873        unsigned long timeout, timetaken;
2874
2875        dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR];
2876
2877        mtip_detect_product(dd);
2878        if (dd->product_type == MTIP_PRODUCT_UNKNOWN) {
2879                rv = -EIO;
2880                goto out1;
2881        }
2882
2883        hba_setup(dd);
2884
2885        dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL,
2886                                dd->numa_node);
2887        if (!dd->port) {
2888                dev_err(&dd->pdev->dev,
2889                        "Memory allocation: port structure\n");
2890                return -ENOMEM;
2891        }
2892
2893        /* Continue workqueue setup */
2894        for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
2895                dd->work[i].port = dd->port;
2896
2897        /* Enable unaligned IO constraints for some devices */
2898        if (mtip_device_unaligned_constrained(dd))
2899                dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS;
2900        else
2901                dd->unal_qdepth = 0;
2902
2903        atomic_set(&dd->port->cmd_slot_unal, dd->unal_qdepth);
2904
2905        /* Spinlock to prevent concurrent issue */
2906        for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
2907                spin_lock_init(&dd->port->cmd_issue_lock[i]);
2908
2909        /* Set the port mmio base address. */
2910        dd->port->mmio  = dd->mmio + PORT_OFFSET;
2911        dd->port->dd    = dd;
2912
2913        /* DMA allocations */
2914        rv = mtip_dma_alloc(dd);
2915        if (rv < 0)
2916                goto out1;
2917
2918        /* Setup the pointers to the extended s_active and CI registers. */
2919        for (i = 0; i < dd->slot_groups; i++) {
2920                dd->port->s_active[i] =
2921                        dd->port->mmio + i*0x80 + PORT_SCR_ACT;
2922                dd->port->cmd_issue[i] =
2923                        dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE;
2924                dd->port->completed[i] =
2925                        dd->port->mmio + i*0x80 + PORT_SDBV;
2926        }
2927
2928        timetaken = jiffies;
2929        timeout = jiffies + msecs_to_jiffies(30000);
2930        while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) &&
2931                 time_before(jiffies, timeout)) {
2932                mdelay(100);
2933        }
2934        if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
2935                timetaken = jiffies - timetaken;
2936                dev_warn(&dd->pdev->dev,
2937                        "Surprise removal detected at %u ms\n",
2938                        jiffies_to_msecs(timetaken));
2939                rv = -ENODEV;
2940                goto out2 ;
2941        }
2942        if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) {
2943                timetaken = jiffies - timetaken;
2944                dev_warn(&dd->pdev->dev,
2945                        "Removal detected at %u ms\n",
2946                        jiffies_to_msecs(timetaken));
2947                rv = -EFAULT;
2948                goto out2;
2949        }
2950
2951        /* Conditionally reset the HBA. */
2952        if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) {
2953                if (mtip_hba_reset(dd) < 0) {
2954                        dev_err(&dd->pdev->dev,
2955                                "Card did not reset within timeout\n");
2956                        rv = -EIO;
2957                        goto out2;
2958                }
2959        } else {
2960                /* Clear any pending interrupts on the HBA */
2961                writel(readl(dd->mmio + HOST_IRQ_STAT),
2962                        dd->mmio + HOST_IRQ_STAT);
2963        }
2964
2965        mtip_init_port(dd->port);
2966        mtip_start_port(dd->port);
2967
2968        /* Setup the ISR and enable interrupts. */
2969        rv = request_irq(dd->pdev->irq, mtip_irq_handler, IRQF_SHARED,
2970                         dev_driver_string(&dd->pdev->dev), dd);
2971        if (rv) {
2972                dev_err(&dd->pdev->dev,
2973                        "Unable to allocate IRQ %d\n", dd->pdev->irq);
2974                goto out2;
2975        }
2976        irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding));
2977
2978        /* Enable interrupts on the HBA. */
2979        writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
2980                                        dd->mmio + HOST_CTL);
2981
2982        init_waitqueue_head(&dd->port->svc_wait);
2983
2984        if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
2985                rv = -EFAULT;
2986                goto out3;
2987        }
2988
2989        return rv;
2990
2991out3:
2992        /* Disable interrupts on the HBA. */
2993        writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
2994                        dd->mmio + HOST_CTL);
2995
2996        /* Release the IRQ. */
2997        irq_set_affinity_hint(dd->pdev->irq, NULL);
2998        free_irq(dd->pdev->irq, dd);
2999
3000out2:
3001        mtip_deinit_port(dd->port);
3002        mtip_dma_free(dd);
3003
3004out1:
3005        /* Free the memory allocated for the for structure. */
3006        kfree(dd->port);
3007
3008        return rv;
3009}
3010
3011static int mtip_standby_drive(struct driver_data *dd)
3012{
3013        int rv = 0;
3014
3015        if (dd->sr || !dd->port)
3016                return -ENODEV;
3017        /*
3018         * Send standby immediate (E0h) to the drive so that it
3019         * saves its state.
3020         */
3021        if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) &&
3022            !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) &&
3023            !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) {
3024                rv = mtip_standby_immediate(dd->port);
3025                if (rv)
3026                        dev_warn(&dd->pdev->dev,
3027                                "STANDBY IMMEDIATE failed\n");
3028        }
3029        return rv;
3030}
3031
3032/*
3033 * Called to deinitialize an interface.
3034 *
3035 * @dd Pointer to the driver data structure.
3036 *
3037 * return value
3038 *      0
3039 */
3040static int mtip_hw_exit(struct driver_data *dd)
3041{
3042        if (!dd->sr) {
3043                /* de-initialize the port. */
3044                mtip_deinit_port(dd->port);
3045
3046                /* Disable interrupts on the HBA. */
3047                writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
3048                                dd->mmio + HOST_CTL);
3049        }
3050
3051        /* Release the IRQ. */
3052        irq_set_affinity_hint(dd->pdev->irq, NULL);
3053        free_irq(dd->pdev->irq, dd);
3054        msleep(1000);
3055
3056        /* Free dma regions */
3057        mtip_dma_free(dd);
3058
3059        /* Free the memory allocated for the for structure. */
3060        kfree(dd->port);
3061        dd->port = NULL;
3062
3063        return 0;
3064}
3065
3066/*
3067 * Issue a Standby Immediate command to the device.
3068 *
3069 * This function is called by the Block Layer just before the
3070 * system powers off during a shutdown.
3071 *
3072 * @dd Pointer to the driver data structure.
3073 *
3074 * return value
3075 *      0
3076 */
3077static int mtip_hw_shutdown(struct driver_data *dd)
3078{
3079        /*
3080         * Send standby immediate (E0h) to the drive so that it
3081         * saves its state.
3082         */
3083        mtip_standby_drive(dd);
3084
3085        return 0;
3086}
3087
3088/*
3089 * Suspend function
3090 *
3091 * This function is called by the Block Layer just before the
3092 * system hibernates.
3093 *
3094 * @dd Pointer to the driver data structure.
3095 *
3096 * return value
3097 *      0       Suspend was successful
3098 *      -EFAULT Suspend was not successful
3099 */
3100static int mtip_hw_suspend(struct driver_data *dd)
3101{
3102        /*
3103         * Send standby immediate (E0h) to the drive
3104         * so that it saves its state.
3105         */
3106        if (mtip_standby_drive(dd) != 0) {
3107                dev_err(&dd->pdev->dev,
3108                        "Failed standby-immediate command\n");
3109                return -EFAULT;
3110        }
3111
3112        /* Disable interrupts on the HBA.*/
3113        writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
3114                        dd->mmio + HOST_CTL);
3115        mtip_deinit_port(dd->port);
3116
3117        return 0;
3118}
3119
3120/*
3121 * Resume function
3122 *
3123 * This function is called by the Block Layer as the
3124 * system resumes.
3125 *
3126 * @dd Pointer to the driver data structure.
3127 *
3128 * return value
3129 *      0       Resume was successful
3130 *      -EFAULT Resume was not successful
3131 */
3132static int mtip_hw_resume(struct driver_data *dd)
3133{
3134        /* Perform any needed hardware setup steps */
3135        hba_setup(dd);
3136
3137        /* Reset the HBA */
3138        if (mtip_hba_reset(dd) != 0) {
3139                dev_err(&dd->pdev->dev,
3140                        "Unable to reset the HBA\n");
3141                return -EFAULT;
3142        }
3143
3144        /*
3145         * Enable the port, DMA engine, and FIS reception specific
3146         * h/w in controller.
3147         */
3148        mtip_init_port(dd->port);
3149        mtip_start_port(dd->port);
3150
3151        /* Enable interrupts on the HBA.*/
3152        writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
3153                        dd->mmio + HOST_CTL);
3154
3155        return 0;
3156}
3157
3158/*
3159 * Helper function for reusing disk name
3160 * upon hot insertion.
3161 */
3162static int rssd_disk_name_format(char *prefix,
3163                                 int index,
3164                                 char *buf,
3165                                 int buflen)
3166{
3167        const int base = 'z' - 'a' + 1;
3168        char *begin = buf + strlen(prefix);
3169        char *end = buf + buflen;
3170        char *p;
3171        int unit;
3172
3173        p = end - 1;
3174        *p = '\0';
3175        unit = base;
3176        do {
3177                if (p == begin)
3178                        return -EINVAL;
3179                *--p = 'a' + (index % unit);
3180                index = (index / unit) - 1;
3181        } while (index >= 0);
3182
3183        memmove(begin, p, end - p);
3184        memcpy(buf, prefix, strlen(prefix));
3185
3186        return 0;
3187}
3188
3189/*
3190 * Block layer IOCTL handler.
3191 *
3192 * @dev Pointer to the block_device structure.
3193 * @mode ignored
3194 * @cmd IOCTL command passed from the user application.
3195 * @arg Argument passed from the user application.
3196 *
3197 * return value
3198 *      0        IOCTL completed successfully.
3199 *      -ENOTTY  IOCTL not supported or invalid driver data
3200 *                 structure pointer.
3201 */
3202static int mtip_block_ioctl(struct block_device *dev,
3203                            fmode_t mode,
3204                            unsigned cmd,
3205                            unsigned long arg)
3206{
3207        struct driver_data *dd = dev->bd_disk->private_data;
3208
3209        if (!capable(CAP_SYS_ADMIN))
3210                return -EACCES;
3211
3212        if (!dd)
3213                return -ENOTTY;
3214
3215        if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
3216                return -ENOTTY;
3217
3218        switch (cmd) {
3219        case BLKFLSBUF:
3220                return -ENOTTY;
3221        default:
3222                return mtip_hw_ioctl(dd, cmd, arg);
3223        }
3224}
3225
3226#ifdef CONFIG_COMPAT
3227/*
3228 * Block layer compat IOCTL handler.
3229 *
3230 * @dev Pointer to the block_device structure.
3231 * @mode ignored
3232 * @cmd IOCTL command passed from the user application.
3233 * @arg Argument passed from the user application.
3234 *
3235 * return value
3236 *      0        IOCTL completed successfully.
3237 *      -ENOTTY  IOCTL not supported or invalid driver data
3238 *                 structure pointer.
3239 */
3240static int mtip_block_compat_ioctl(struct block_device *dev,
3241                            fmode_t mode,
3242                            unsigned cmd,
3243                            unsigned long arg)
3244{
3245        struct driver_data *dd = dev->bd_disk->private_data;
3246
3247        if (!capable(CAP_SYS_ADMIN))
3248                return -EACCES;
3249
3250        if (!dd)
3251                return -ENOTTY;
3252
3253        if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
3254                return -ENOTTY;
3255
3256        switch (cmd) {
3257        case BLKFLSBUF:
3258                return -ENOTTY;
3259        case HDIO_DRIVE_TASKFILE: {
3260                struct mtip_compat_ide_task_request_s __user *compat_req_task;
3261                ide_task_request_t req_task;
3262                int compat_tasksize, outtotal, ret;
3263
3264                compat_tasksize =
3265                        sizeof(struct mtip_compat_ide_task_request_s);
3266
3267                compat_req_task =
3268                        (struct mtip_compat_ide_task_request_s __user *) arg;
3269
3270                if (copy_from_user(&req_task, (void __user *) arg,
3271                        compat_tasksize - (2 * sizeof(compat_long_t))))
3272                        return -EFAULT;
3273
3274                if (get_user(req_task.out_size, &compat_req_task->out_size))
3275                        return -EFAULT;
3276
3277                if (get_user(req_task.in_size, &compat_req_task->in_size))
3278                        return -EFAULT;
3279
3280                outtotal = sizeof(struct mtip_compat_ide_task_request_s);
3281
3282                ret = exec_drive_taskfile(dd, (void __user *) arg,
3283                                                &req_task, outtotal);
3284
3285                if (copy_to_user((void __user *) arg, &req_task,
3286                                compat_tasksize -
3287                                (2 * sizeof(compat_long_t))))
3288                        return -EFAULT;
3289
3290                if (put_user(req_task.out_size, &compat_req_task->out_size))
3291                        return -EFAULT;
3292
3293                if (put_user(req_task.in_size, &compat_req_task->in_size))
3294                        return -EFAULT;
3295
3296                return ret;
3297        }
3298        default:
3299                return mtip_hw_ioctl(dd, cmd, arg);
3300        }
3301}
3302#endif
3303
3304/*
3305 * Obtain the geometry of the device.
3306 *
3307 * You may think that this function is obsolete, but some applications,
3308 * fdisk for example still used CHS values. This function describes the
3309 * device as having 224 heads and 56 sectors per cylinder. These values are
3310 * chosen so that each cylinder is aligned on a 4KB boundary. Since a
3311 * partition is described in terms of a start and end cylinder this means
3312 * that each partition is also 4KB aligned. Non-aligned partitions adversely
3313 * affects performance.
3314 *
3315 * @dev Pointer to the block_device strucutre.
3316 * @geo Pointer to a hd_geometry structure.
3317 *
3318 * return value
3319 *      0       Operation completed successfully.
3320 *      -ENOTTY An error occurred while reading the drive capacity.
3321 */
3322static int mtip_block_getgeo(struct block_device *dev,
3323                                struct hd_geometry *geo)
3324{
3325        struct driver_data *dd = dev->bd_disk->private_data;
3326        sector_t capacity;
3327
3328        if (!dd)
3329                return -ENOTTY;
3330
3331        if (!(mtip_hw_get_capacity(dd, &capacity))) {
3332                dev_warn(&dd->pdev->dev,
3333                        "Could not get drive capacity.\n");
3334                return -ENOTTY;
3335        }
3336
3337        geo->heads = 224;
3338        geo->sectors = 56;
3339        sector_div(capacity, (geo->heads * geo->sectors));
3340        geo->cylinders = capacity;
3341        return 0;
3342}
3343
3344static int mtip_block_open(struct block_device *dev, fmode_t mode)
3345{
3346        struct driver_data *dd;
3347
3348        if (dev && dev->bd_disk) {
3349                dd = (struct driver_data *) dev->bd_disk->private_data;
3350
3351                if (dd) {
3352                        if (test_bit(MTIP_DDF_REMOVAL_BIT,
3353                                                        &dd->dd_flag)) {
3354                                return -ENODEV;
3355                        }
3356                        return 0;
3357                }
3358        }
3359        return -ENODEV;
3360}
3361
3362static void mtip_block_release(struct gendisk *disk, fmode_t mode)
3363{
3364}
3365
3366/*
3367 * Block device operation function.
3368 *
3369 * This structure contains pointers to the functions required by the block
3370 * layer.
3371 */
3372static const struct block_device_operations mtip_block_ops = {
3373        .open           = mtip_block_open,
3374        .release        = mtip_block_release,
3375        .ioctl          = mtip_block_ioctl,
3376#ifdef CONFIG_COMPAT
3377        .compat_ioctl   = mtip_block_compat_ioctl,
3378#endif
3379        .getgeo         = mtip_block_getgeo,
3380        .owner          = THIS_MODULE
3381};
3382
3383static inline bool is_se_active(struct driver_data *dd)
3384{
3385        if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) {
3386                if (dd->port->ic_pause_timer) {
3387                        unsigned long to = dd->port->ic_pause_timer +
3388                                                        msecs_to_jiffies(1000);
3389                        if (time_after(jiffies, to)) {
3390                                clear_bit(MTIP_PF_SE_ACTIVE_BIT,
3391                                                        &dd->port->flags);
3392                                clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag);
3393                                dd->port->ic_pause_timer = 0;
3394                                wake_up_interruptible(&dd->port->svc_wait);
3395                                return false;
3396                        }
3397                }
3398                return true;
3399        }
3400        return false;
3401}
3402
3403static inline bool is_stopped(struct driver_data *dd, struct request *rq)
3404{
3405        if (likely(!(dd->dd_flag & MTIP_DDF_STOP_IO)))
3406                return false;
3407
3408        if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
3409                return true;
3410        if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
3411                return true;
3412        if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag) &&
3413            rq_data_dir(rq))
3414                return true;
3415        if (test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))
3416                return true;
3417        if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))
3418                return true;
3419
3420        return false;
3421}
3422
3423static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx,
3424                                  struct request *rq)
3425{
3426        struct driver_data *dd = hctx->queue->queuedata;
3427        struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3428
3429        if (rq_data_dir(rq) == READ || !dd->unal_qdepth)
3430                return false;
3431
3432        /*
3433         * If unaligned depth must be limited on this controller, mark it
3434         * as unaligned if the IO isn't on a 4k boundary (start of length).
3435         */
3436        if (blk_rq_sectors(rq) <= 64) {
3437                if ((blk_rq_pos(rq) & 7) || (blk_rq_sectors(rq) & 7))
3438                        cmd->unaligned = 1;
3439        }
3440
3441        if (cmd->unaligned && atomic_dec_if_positive(&dd->port->cmd_slot_unal) >= 0)
3442                return true;
3443
3444        return false;
3445}
3446
3447static blk_status_t mtip_issue_reserved_cmd(struct blk_mq_hw_ctx *hctx,
3448                struct request *rq)
3449{
3450        struct driver_data *dd = hctx->queue->queuedata;
3451        struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3452        struct mtip_int_cmd *icmd = cmd->icmd;
3453        struct mtip_cmd_hdr *hdr =
3454                dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
3455        struct mtip_cmd_sg *command_sg;
3456
3457        if (mtip_commands_active(dd->port))
3458                return BLK_STS_DEV_RESOURCE;
3459
3460        hdr->ctba = cpu_to_le32(cmd->command_dma & 0xFFFFFFFF);
3461        if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
3462                hdr->ctbau = cpu_to_le32((cmd->command_dma >> 16) >> 16);
3463        /* Populate the SG list */
3464        hdr->opts = cpu_to_le32(icmd->opts | icmd->fis_len);
3465        if (icmd->buf_len) {
3466                command_sg = cmd->command + AHCI_CMD_TBL_HDR_SZ;
3467
3468                command_sg->info = cpu_to_le32((icmd->buf_len-1) & 0x3FFFFF);
3469                command_sg->dba = cpu_to_le32(icmd->buffer & 0xFFFFFFFF);
3470                command_sg->dba_upper =
3471                        cpu_to_le32((icmd->buffer >> 16) >> 16);
3472
3473                hdr->opts |= cpu_to_le32((1 << 16));
3474        }
3475
3476        /* Populate the command header */
3477        hdr->byte_count = 0;
3478
3479        blk_mq_start_request(rq);
3480        mtip_issue_non_ncq_command(dd->port, rq->tag);
3481        return 0;
3482}
3483
3484static blk_status_t mtip_queue_rq(struct blk_mq_hw_ctx *hctx,
3485                         const struct blk_mq_queue_data *bd)
3486{
3487        struct driver_data *dd = hctx->queue->queuedata;
3488        struct request *rq = bd->rq;
3489        struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3490
3491        if (blk_rq_is_passthrough(rq))
3492                return mtip_issue_reserved_cmd(hctx, rq);
3493
3494        if (unlikely(mtip_check_unal_depth(hctx, rq)))
3495                return BLK_STS_DEV_RESOURCE;
3496
3497        if (is_se_active(dd) || is_stopped(dd, rq))
3498                return BLK_STS_IOERR;
3499
3500        blk_mq_start_request(rq);
3501
3502        mtip_hw_submit_io(dd, rq, cmd, hctx);
3503        return BLK_STS_OK;
3504}
3505
3506static void mtip_free_cmd(struct blk_mq_tag_set *set, struct request *rq,
3507                          unsigned int hctx_idx)
3508{
3509        struct driver_data *dd = set->driver_data;
3510        struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3511
3512        if (!cmd->command)
3513                return;
3514
3515        dma_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, cmd->command,
3516                          cmd->command_dma);
3517}
3518
3519static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq,
3520                         unsigned int hctx_idx, unsigned int numa_node)
3521{
3522        struct driver_data *dd = set->driver_data;
3523        struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3524
3525        cmd->command = dma_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ,
3526                        &cmd->command_dma, GFP_KERNEL);
3527        if (!cmd->command)
3528                return -ENOMEM;
3529
3530        sg_init_table(cmd->sg, MTIP_MAX_SG);
3531        return 0;
3532}
3533
3534static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req,
3535                                                                bool reserved)
3536{
3537        struct driver_data *dd = req->q->queuedata;
3538
3539        if (reserved) {
3540                struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
3541
3542                cmd->status = BLK_STS_TIMEOUT;
3543                blk_mq_complete_request(req);
3544                return BLK_EH_DONE;
3545        }
3546
3547        if (test_bit(req->tag, dd->port->cmds_to_issue))
3548                goto exit_handler;
3549
3550        if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags))
3551                goto exit_handler;
3552
3553        wake_up_interruptible(&dd->port->svc_wait);
3554exit_handler:
3555        return BLK_EH_RESET_TIMER;
3556}
3557
3558static const struct blk_mq_ops mtip_mq_ops = {
3559        .queue_rq       = mtip_queue_rq,
3560        .init_request   = mtip_init_cmd,
3561        .exit_request   = mtip_free_cmd,
3562        .complete       = mtip_softirq_done_fn,
3563        .timeout        = mtip_cmd_timeout,
3564};
3565
3566/*
3567 * Block layer initialization function.
3568 *
3569 * This function is called once by the PCI layer for each P320
3570 * device that is connected to the system.
3571 *
3572 * @dd Pointer to the driver data structure.
3573 *
3574 * return value
3575 *      0 on success else an error code.
3576 */
3577static int mtip_block_initialize(struct driver_data *dd)
3578{
3579        int rv = 0, wait_for_rebuild = 0;
3580        sector_t capacity;
3581        unsigned int index = 0;
3582        struct kobject *kobj;
3583
3584        if (dd->disk)
3585                goto skip_create_disk; /* hw init done, before rebuild */
3586
3587        if (mtip_hw_init(dd)) {
3588                rv = -EINVAL;
3589                goto protocol_init_error;
3590        }
3591
3592        dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node);
3593        if (dd->disk  == NULL) {
3594                dev_err(&dd->pdev->dev,
3595                        "Unable to allocate gendisk structure\n");
3596                rv = -EINVAL;
3597                goto alloc_disk_error;
3598        }
3599
3600        rv = ida_alloc(&rssd_index_ida, GFP_KERNEL);
3601        if (rv < 0)
3602                goto ida_get_error;
3603        index = rv;
3604
3605        rv = rssd_disk_name_format("rssd",
3606                                index,
3607                                dd->disk->disk_name,
3608                                DISK_NAME_LEN);
3609        if (rv)
3610                goto disk_index_error;
3611
3612        dd->disk->major         = dd->major;
3613        dd->disk->first_minor   = index * MTIP_MAX_MINORS;
3614        dd->disk->minors        = MTIP_MAX_MINORS;
3615        dd->disk->fops          = &mtip_block_ops;
3616        dd->disk->private_data  = dd;
3617        dd->index               = index;
3618
3619        mtip_hw_debugfs_init(dd);
3620
3621        memset(&dd->tags, 0, sizeof(dd->tags));
3622        dd->tags.ops = &mtip_mq_ops;
3623        dd->tags.nr_hw_queues = 1;
3624        dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS;
3625        dd->tags.reserved_tags = 1;
3626        dd->tags.cmd_size = sizeof(struct mtip_cmd);
3627        dd->tags.numa_node = dd->numa_node;
3628        dd->tags.flags = BLK_MQ_F_SHOULD_MERGE;
3629        dd->tags.driver_data = dd;
3630        dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS;
3631
3632        rv = blk_mq_alloc_tag_set(&dd->tags);
3633        if (rv) {
3634                dev_err(&dd->pdev->dev,
3635                        "Unable to allocate request queue\n");
3636                goto block_queue_alloc_tag_error;
3637        }
3638
3639        /* Allocate the request queue. */
3640        dd->queue = blk_mq_init_queue(&dd->tags);
3641        if (IS_ERR(dd->queue)) {
3642                dev_err(&dd->pdev->dev,
3643                        "Unable to allocate request queue\n");
3644                rv = -ENOMEM;
3645                goto block_queue_alloc_init_error;
3646        }
3647
3648        dd->disk->queue         = dd->queue;
3649        dd->queue->queuedata    = dd;
3650
3651skip_create_disk:
3652        /* Initialize the protocol layer. */
3653        wait_for_rebuild = mtip_hw_get_identify(dd);
3654        if (wait_for_rebuild < 0) {
3655                dev_err(&dd->pdev->dev,
3656                        "Protocol layer initialization failed\n");
3657                rv = -EINVAL;
3658                goto init_hw_cmds_error;
3659        }
3660
3661        /*
3662         * if rebuild pending, start the service thread, and delay the block
3663         * queue creation and device_add_disk()
3664         */
3665        if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
3666                goto start_service_thread;
3667
3668        /* Set device limits. */
3669        blk_queue_flag_set(QUEUE_FLAG_NONROT, dd->queue);
3670        blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dd->queue);
3671        blk_queue_max_segments(dd->queue, MTIP_MAX_SG);
3672        blk_queue_physical_block_size(dd->queue, 4096);
3673        blk_queue_max_hw_sectors(dd->queue, 0xffff);
3674        blk_queue_max_segment_size(dd->queue, 0x400000);
3675        dma_set_max_seg_size(&dd->pdev->dev, 0x400000);
3676        blk_queue_io_min(dd->queue, 4096);
3677
3678        /* Set the capacity of the device in 512 byte sectors. */
3679        if (!(mtip_hw_get_capacity(dd, &capacity))) {
3680                dev_warn(&dd->pdev->dev,
3681                        "Could not read drive capacity\n");
3682                rv = -EIO;
3683                goto read_capacity_error;
3684        }
3685        set_capacity(dd->disk, capacity);
3686
3687        /* Enable the block device and add it to /dev */
3688        device_add_disk(&dd->pdev->dev, dd->disk, NULL);
3689
3690        /*
3691         * Now that the disk is active, initialize any sysfs attributes
3692         * managed by the protocol layer.
3693         */
3694        kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
3695        if (kobj) {
3696                mtip_hw_sysfs_init(dd, kobj);
3697                kobject_put(kobj);
3698        }
3699
3700        if (dd->mtip_svc_handler) {
3701                set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
3702                return rv; /* service thread created for handling rebuild */
3703        }
3704
3705start_service_thread:
3706        dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread,
3707                                                dd, dd->numa_node,
3708                                                "mtip_svc_thd_%02d", index);
3709
3710        if (IS_ERR(dd->mtip_svc_handler)) {
3711                dev_err(&dd->pdev->dev, "service thread failed to start\n");
3712                dd->mtip_svc_handler = NULL;
3713                rv = -EFAULT;
3714                goto kthread_run_error;
3715        }
3716        wake_up_process(dd->mtip_svc_handler);
3717        if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
3718                rv = wait_for_rebuild;
3719
3720        return rv;
3721
3722kthread_run_error:
3723        /* Delete our gendisk. This also removes the device from /dev */
3724        del_gendisk(dd->disk);
3725
3726read_capacity_error:
3727init_hw_cmds_error:
3728        blk_cleanup_queue(dd->queue);
3729block_queue_alloc_init_error:
3730        blk_mq_free_tag_set(&dd->tags);
3731block_queue_alloc_tag_error:
3732        mtip_hw_debugfs_exit(dd);
3733disk_index_error:
3734        ida_free(&rssd_index_ida, index);
3735
3736ida_get_error:
3737        put_disk(dd->disk);
3738
3739alloc_disk_error:
3740        mtip_hw_exit(dd); /* De-initialize the protocol layer. */
3741
3742protocol_init_error:
3743        return rv;
3744}
3745
3746static bool mtip_no_dev_cleanup(struct request *rq, void *data, bool reserv)
3747{
3748        struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3749
3750        cmd->status = BLK_STS_IOERR;
3751        blk_mq_complete_request(rq);
3752        return true;
3753}
3754
3755/*
3756 * Block layer deinitialization function.
3757 *
3758 * Called by the PCI layer as each P320 device is removed.
3759 *
3760 * @dd Pointer to the driver data structure.
3761 *
3762 * return value
3763 *      0
3764 */
3765static int mtip_block_remove(struct driver_data *dd)
3766{
3767        struct kobject *kobj;
3768
3769        mtip_hw_debugfs_exit(dd);
3770
3771        if (dd->mtip_svc_handler) {
3772                set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags);
3773                wake_up_interruptible(&dd->port->svc_wait);
3774                kthread_stop(dd->mtip_svc_handler);
3775        }
3776
3777        /* Clean up the sysfs attributes, if created */
3778        if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) {
3779                kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
3780                if (kobj) {
3781                        mtip_hw_sysfs_exit(dd, kobj);
3782                        kobject_put(kobj);
3783                }
3784        }
3785
3786        if (!dd->sr) {
3787                /*
3788                 * Explicitly wait here for IOs to quiesce,
3789                 * as mtip_standby_drive usually won't wait for IOs.
3790                 */
3791                if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS))
3792                        mtip_standby_drive(dd);
3793        }
3794        else
3795                dev_info(&dd->pdev->dev, "device %s surprise removal\n",
3796                                                dd->disk->disk_name);
3797
3798        blk_freeze_queue_start(dd->queue);
3799        blk_mq_quiesce_queue(dd->queue);
3800        blk_mq_tagset_busy_iter(&dd->tags, mtip_no_dev_cleanup, dd);
3801        blk_mq_unquiesce_queue(dd->queue);
3802
3803        if (dd->disk) {
3804                if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3805                        del_gendisk(dd->disk);
3806                if (dd->disk->queue) {
3807                        blk_cleanup_queue(dd->queue);
3808                        blk_mq_free_tag_set(&dd->tags);
3809                        dd->queue = NULL;
3810                }
3811                put_disk(dd->disk);
3812        }
3813        dd->disk  = NULL;
3814
3815        ida_free(&rssd_index_ida, dd->index);
3816
3817        /* De-initialize the protocol layer. */
3818        mtip_hw_exit(dd);
3819
3820        return 0;
3821}
3822
3823/*
3824 * Function called by the PCI layer when just before the
3825 * machine shuts down.
3826 *
3827 * If a protocol layer shutdown function is present it will be called
3828 * by this function.
3829 *
3830 * @dd Pointer to the driver data structure.
3831 *
3832 * return value
3833 *      0
3834 */
3835static int mtip_block_shutdown(struct driver_data *dd)
3836{
3837        mtip_hw_shutdown(dd);
3838
3839        /* Delete our gendisk structure, and cleanup the blk queue. */
3840        if (dd->disk) {
3841                dev_info(&dd->pdev->dev,
3842                        "Shutting down %s ...\n", dd->disk->disk_name);
3843
3844                if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3845                        del_gendisk(dd->disk);
3846                if (dd->disk->queue) {
3847                        blk_cleanup_queue(dd->queue);
3848                        blk_mq_free_tag_set(&dd->tags);
3849                }
3850                put_disk(dd->disk);
3851                dd->disk  = NULL;
3852                dd->queue = NULL;
3853        }
3854
3855        ida_free(&rssd_index_ida, dd->index);
3856        return 0;
3857}
3858
3859static int mtip_block_suspend(struct driver_data *dd)
3860{
3861        dev_info(&dd->pdev->dev,
3862                "Suspending %s ...\n", dd->disk->disk_name);
3863        mtip_hw_suspend(dd);
3864        return 0;
3865}
3866
3867static int mtip_block_resume(struct driver_data *dd)
3868{
3869        dev_info(&dd->pdev->dev, "Resuming %s ...\n",
3870                dd->disk->disk_name);
3871        mtip_hw_resume(dd);
3872        return 0;
3873}
3874
3875static void drop_cpu(int cpu)
3876{
3877        cpu_use[cpu]--;
3878}
3879
3880static int get_least_used_cpu_on_node(int node)
3881{
3882        int cpu, least_used_cpu, least_cnt;
3883        const struct cpumask *node_mask;
3884
3885        node_mask = cpumask_of_node(node);
3886        least_used_cpu = cpumask_first(node_mask);
3887        least_cnt = cpu_use[least_used_cpu];
3888        cpu = least_used_cpu;
3889
3890        for_each_cpu(cpu, node_mask) {
3891                if (cpu_use[cpu] < least_cnt) {
3892                        least_used_cpu = cpu;
3893                        least_cnt = cpu_use[cpu];
3894                }
3895        }
3896        cpu_use[least_used_cpu]++;
3897        return least_used_cpu;
3898}
3899
3900/* Helper for selecting a node in round robin mode */
3901static inline int mtip_get_next_rr_node(void)
3902{
3903        static int next_node = NUMA_NO_NODE;
3904
3905        if (next_node == NUMA_NO_NODE) {
3906                next_node = first_online_node;
3907                return next_node;
3908        }
3909
3910        next_node = next_online_node(next_node);
3911        if (next_node == MAX_NUMNODES)
3912                next_node = first_online_node;
3913        return next_node;
3914}
3915
3916static DEFINE_HANDLER(0);
3917static DEFINE_HANDLER(1);
3918static DEFINE_HANDLER(2);
3919static DEFINE_HANDLER(3);
3920static DEFINE_HANDLER(4);
3921static DEFINE_HANDLER(5);
3922static DEFINE_HANDLER(6);
3923static DEFINE_HANDLER(7);
3924
3925static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev)
3926{
3927        unsigned short pcie_dev_ctrl;
3928
3929        if (pci_is_pcie(pdev)) {
3930                pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &pcie_dev_ctrl);
3931                if (pcie_dev_ctrl & PCI_EXP_DEVCTL_NOSNOOP_EN ||
3932                    pcie_dev_ctrl & PCI_EXP_DEVCTL_RELAX_EN) {
3933                        dev_info(&dd->pdev->dev,
3934                                "Disabling ERO/No-Snoop on bridge device %04x:%04x\n",
3935                                        pdev->vendor, pdev->device);
3936                        pcie_dev_ctrl &= ~(PCI_EXP_DEVCTL_NOSNOOP_EN |
3937                                                PCI_EXP_DEVCTL_RELAX_EN);
3938                        pcie_capability_write_word(pdev, PCI_EXP_DEVCTL,
3939                                pcie_dev_ctrl);
3940                }
3941        }
3942}
3943
3944static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev)
3945{
3946        /*
3947         * This workaround is specific to AMD/ATI chipset with a PCI upstream
3948         * device with device id 0x5aXX
3949         */
3950        if (pdev->bus && pdev->bus->self) {
3951                if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI &&
3952                    ((pdev->bus->self->device & 0xff00) == 0x5a00)) {
3953                        mtip_disable_link_opts(dd, pdev->bus->self);
3954                } else {
3955                        /* Check further up the topology */
3956                        struct pci_dev *parent_dev = pdev->bus->self;
3957                        if (parent_dev->bus &&
3958                                parent_dev->bus->parent &&
3959                                parent_dev->bus->parent->self &&
3960                                parent_dev->bus->parent->self->vendor ==
3961                                         PCI_VENDOR_ID_ATI &&
3962                                (parent_dev->bus->parent->self->device &
3963                                        0xff00) == 0x5a00) {
3964                                mtip_disable_link_opts(dd,
3965                                        parent_dev->bus->parent->self);
3966                        }
3967                }
3968        }
3969}
3970
3971/*
3972 * Called for each supported PCI device detected.
3973 *
3974 * This function allocates the private data structure, enables the
3975 * PCI device and then calls the block layer initialization function.
3976 *
3977 * return value
3978 *      0 on success else an error code.
3979 */
3980static int mtip_pci_probe(struct pci_dev *pdev,
3981                        const struct pci_device_id *ent)
3982{
3983        int rv = 0;
3984        struct driver_data *dd = NULL;
3985        char cpu_list[256];
3986        const struct cpumask *node_mask;
3987        int cpu, i = 0, j = 0;
3988        int my_node = NUMA_NO_NODE;
3989        unsigned long flags;
3990
3991        /* Allocate memory for this devices private data. */
3992        my_node = pcibus_to_node(pdev->bus);
3993        if (my_node != NUMA_NO_NODE) {
3994                if (!node_online(my_node))
3995                        my_node = mtip_get_next_rr_node();
3996        } else {
3997                dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n");
3998                my_node = mtip_get_next_rr_node();
3999        }
4000        dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n",
4001                my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev),
4002                cpu_to_node(raw_smp_processor_id()), raw_smp_processor_id());
4003
4004        dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node);
4005        if (dd == NULL) {
4006                dev_err(&pdev->dev,
4007                        "Unable to allocate memory for driver data\n");
4008                return -ENOMEM;
4009        }
4010
4011        /* Attach the private data to this PCI device.  */
4012        pci_set_drvdata(pdev, dd);
4013
4014        rv = pcim_enable_device(pdev);
4015        if (rv < 0) {
4016                dev_err(&pdev->dev, "Unable to enable device\n");
4017                goto iomap_err;
4018        }
4019
4020        /* Map BAR5 to memory. */
4021        rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME);
4022        if (rv < 0) {
4023                dev_err(&pdev->dev, "Unable to map regions\n");
4024                goto iomap_err;
4025        }
4026
4027        rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4028        if (rv) {
4029                dev_warn(&pdev->dev, "64-bit DMA enable failed\n");
4030                goto setmask_err;
4031        }
4032
4033        /* Copy the info we may need later into the private data structure. */
4034        dd->major       = mtip_major;
4035        dd->instance    = instance;
4036        dd->pdev        = pdev;
4037        dd->numa_node   = my_node;
4038
4039        INIT_LIST_HEAD(&dd->online_list);
4040        INIT_LIST_HEAD(&dd->remove_list);
4041
4042        memset(dd->workq_name, 0, 32);
4043        snprintf(dd->workq_name, 31, "mtipq%d", dd->instance);
4044
4045        dd->isr_workq = create_workqueue(dd->workq_name);
4046        if (!dd->isr_workq) {
4047                dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance);
4048                rv = -ENOMEM;
4049                goto setmask_err;
4050        }
4051
4052        memset(cpu_list, 0, sizeof(cpu_list));
4053
4054        node_mask = cpumask_of_node(dd->numa_node);
4055        if (!cpumask_empty(node_mask)) {
4056                for_each_cpu(cpu, node_mask)
4057                {
4058                        snprintf(&cpu_list[j], 256 - j, "%d ", cpu);
4059                        j = strlen(cpu_list);
4060                }
4061
4062                dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n",
4063                        dd->numa_node,
4064                        topology_physical_package_id(cpumask_first(node_mask)),
4065                        nr_cpus_node(dd->numa_node),
4066                        cpu_list);
4067        } else
4068                dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n");
4069
4070        dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node);
4071        dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n",
4072                cpu_to_node(dd->isr_binding), dd->isr_binding);
4073
4074        /* first worker context always runs in ISR */
4075        dd->work[0].cpu_binding = dd->isr_binding;
4076        dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
4077        dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
4078        dd->work[3].cpu_binding = dd->work[0].cpu_binding;
4079        dd->work[4].cpu_binding = dd->work[1].cpu_binding;
4080        dd->work[5].cpu_binding = dd->work[2].cpu_binding;
4081        dd->work[6].cpu_binding = dd->work[2].cpu_binding;
4082        dd->work[7].cpu_binding = dd->work[1].cpu_binding;
4083
4084        /* Log the bindings */
4085        for_each_present_cpu(cpu) {
4086                memset(cpu_list, 0, sizeof(cpu_list));
4087                for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) {
4088                        if (dd->work[i].cpu_binding == cpu) {
4089                                snprintf(&cpu_list[j], 256 - j, "%d ", i);
4090                                j = strlen(cpu_list);
4091                        }
4092                }
4093                if (j)
4094                        dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list);
4095        }
4096
4097        INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0);
4098        INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1);
4099        INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2);
4100        INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3);
4101        INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4);
4102        INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5);
4103        INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6);
4104        INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7);
4105
4106        pci_set_master(pdev);
4107        rv = pci_enable_msi(pdev);
4108        if (rv) {
4109                dev_warn(&pdev->dev,
4110                        "Unable to enable MSI interrupt.\n");
4111                goto msi_initialize_err;
4112        }
4113
4114        mtip_fix_ero_nosnoop(dd, pdev);
4115
4116        /* Initialize the block layer. */
4117        rv = mtip_block_initialize(dd);
4118        if (rv < 0) {
4119                dev_err(&pdev->dev,
4120                        "Unable to initialize block layer\n");
4121                goto block_initialize_err;
4122        }
4123
4124        /*
4125         * Increment the instance count so that each device has a unique
4126         * instance number.
4127         */
4128        instance++;
4129        if (rv != MTIP_FTL_REBUILD_MAGIC)
4130                set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
4131        else
4132                rv = 0; /* device in rebuild state, return 0 from probe */
4133
4134        /* Add to online list even if in ftl rebuild */
4135        spin_lock_irqsave(&dev_lock, flags);
4136        list_add(&dd->online_list, &online_list);
4137        spin_unlock_irqrestore(&dev_lock, flags);
4138
4139        goto done;
4140
4141block_initialize_err:
4142        pci_disable_msi(pdev);
4143
4144msi_initialize_err:
4145        if (dd->isr_workq) {
4146                flush_workqueue(dd->isr_workq);
4147                destroy_workqueue(dd->isr_workq);
4148                drop_cpu(dd->work[0].cpu_binding);
4149                drop_cpu(dd->work[1].cpu_binding);
4150                drop_cpu(dd->work[2].cpu_binding);
4151        }