linux/drivers/ata/sata_mv.c
<<
>>
Prefs
   1/*
   2 * sata_mv.c - Marvell SATA support
   3 *
   4 * Copyright 2008-2009: Marvell Corporation, all rights reserved.
   5 * Copyright 2005: EMC Corporation, all rights reserved.
   6 * Copyright 2005 Red Hat, Inc.  All rights reserved.
   7 *
   8 * Originally written by Brett Russ.
   9 * Extensive overhaul and enhancement by Mark Lord <mlord@pobox.com>.
  10 *
  11 * Please ALWAYS copy linux-ide@vger.kernel.org on emails.
  12 *
  13 * This program is free software; you can redistribute it and/or modify
  14 * it under the terms of the GNU General Public License as published by
  15 * the Free Software Foundation; version 2 of the License.
  16 *
  17 * This program is distributed in the hope that it will be useful,
  18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 * GNU General Public License for more details.
  21 *
  22 * You should have received a copy of the GNU General Public License
  23 * along with this program; if not, write to the Free Software
  24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  25 *
  26 */
  27
  28/*
  29 * sata_mv TODO list:
  30 *
  31 * --> Develop a low-power-consumption strategy, and implement it.
  32 *
  33 * --> Add sysfs attributes for per-chip / per-HC IRQ coalescing thresholds.
  34 *
  35 * --> [Experiment, Marvell value added] Is it possible to use target
  36 *       mode to cross-connect two Linux boxes with Marvell cards?  If so,
  37 *       creating LibATA target mode support would be very interesting.
  38 *
  39 *       Target mode, for those without docs, is the ability to directly
  40 *       connect two SATA ports.
  41 */
  42
  43/*
  44 * 80x1-B2 errata PCI#11:
  45 *
  46 * Users of the 6041/6081 Rev.B2 chips (current is C0)
  47 * should be careful to insert those cards only onto PCI-X bus #0,
  48 * and only in device slots 0..7, not higher.  The chips may not
  49 * work correctly otherwise  (note: this is a pretty rare condition).
  50 */
  51
  52#include <linux/kernel.h>
  53#include <linux/module.h>
  54#include <linux/pci.h>
  55#include <linux/init.h>
  56#include <linux/blkdev.h>
  57#include <linux/delay.h>
  58#include <linux/interrupt.h>
  59#include <linux/dmapool.h>
  60#include <linux/dma-mapping.h>
  61#include <linux/device.h>
  62#include <linux/clk.h>
  63#include <linux/platform_device.h>
  64#include <linux/ata_platform.h>
  65#include <linux/mbus.h>
  66#include <linux/bitops.h>
  67#include <linux/gfp.h>
  68#include <linux/of.h>
  69#include <linux/of_irq.h>
  70#include <scsi/scsi_host.h>
  71#include <scsi/scsi_cmnd.h>
  72#include <scsi/scsi_device.h>
  73#include <linux/libata.h>
  74
  75#define DRV_NAME        "sata_mv"
  76#define DRV_VERSION     "1.28"
  77
  78/*
  79 * module options
  80 */
  81
  82#ifdef CONFIG_PCI
  83static int msi;
  84module_param(msi, int, S_IRUGO);
  85MODULE_PARM_DESC(msi, "Enable use of PCI MSI (0=off, 1=on)");
  86#endif
  87
  88static int irq_coalescing_io_count;
  89module_param(irq_coalescing_io_count, int, S_IRUGO);
  90MODULE_PARM_DESC(irq_coalescing_io_count,
  91                 "IRQ coalescing I/O count threshold (0..255)");
  92
  93static int irq_coalescing_usecs;
  94module_param(irq_coalescing_usecs, int, S_IRUGO);
  95MODULE_PARM_DESC(irq_coalescing_usecs,
  96                 "IRQ coalescing time threshold in usecs");
  97
  98enum {
  99        /* BAR's are enumerated in terms of pci_resource_start() terms */
 100        MV_PRIMARY_BAR          = 0,    /* offset 0x10: memory space */
 101        MV_IO_BAR               = 2,    /* offset 0x18: IO space */
 102        MV_MISC_BAR             = 3,    /* offset 0x1c: FLASH, NVRAM, SRAM */
 103
 104        MV_MAJOR_REG_AREA_SZ    = 0x10000,      /* 64KB */
 105        MV_MINOR_REG_AREA_SZ    = 0x2000,       /* 8KB */
 106
 107        /* For use with both IRQ coalescing methods ("all ports" or "per-HC" */
 108        COAL_CLOCKS_PER_USEC    = 150,          /* for calculating COAL_TIMEs */
 109        MAX_COAL_TIME_THRESHOLD = ((1 << 24) - 1), /* internal clocks count */
 110        MAX_COAL_IO_COUNT       = 255,          /* completed I/O count */
 111
 112        MV_PCI_REG_BASE         = 0,
 113
 114        /*
 115         * Per-chip ("all ports") interrupt coalescing feature.
 116         * This is only for GEN_II / GEN_IIE hardware.
 117         *
 118         * Coalescing defers the interrupt until either the IO_THRESHOLD
 119         * (count of completed I/Os) is met, or the TIME_THRESHOLD is met.
 120         */
 121        COAL_REG_BASE           = 0x18000,
 122        IRQ_COAL_CAUSE          = (COAL_REG_BASE + 0x08),
 123        ALL_PORTS_COAL_IRQ      = (1 << 4),     /* all ports irq event */
 124
 125        IRQ_COAL_IO_THRESHOLD   = (COAL_REG_BASE + 0xcc),
 126        IRQ_COAL_TIME_THRESHOLD = (COAL_REG_BASE + 0xd0),
 127
 128        /*
 129         * Registers for the (unused here) transaction coalescing feature:
 130         */
 131        TRAN_COAL_CAUSE_LO      = (COAL_REG_BASE + 0x88),
 132        TRAN_COAL_CAUSE_HI      = (COAL_REG_BASE + 0x8c),
 133
 134        SATAHC0_REG_BASE        = 0x20000,
 135        FLASH_CTL               = 0x1046c,
 136        GPIO_PORT_CTL           = 0x104f0,
 137        RESET_CFG               = 0x180d8,
 138
 139        MV_PCI_REG_SZ           = MV_MAJOR_REG_AREA_SZ,
 140        MV_SATAHC_REG_SZ        = MV_MAJOR_REG_AREA_SZ,
 141        MV_SATAHC_ARBTR_REG_SZ  = MV_MINOR_REG_AREA_SZ,         /* arbiter */
 142        MV_PORT_REG_SZ          = MV_MINOR_REG_AREA_SZ,
 143
 144        MV_MAX_Q_DEPTH          = 32,
 145        MV_MAX_Q_DEPTH_MASK     = MV_MAX_Q_DEPTH - 1,
 146
 147        /* CRQB needs alignment on a 1KB boundary. Size == 1KB
 148         * CRPB needs alignment on a 256B boundary. Size == 256B
 149         * ePRD (SG) entries need alignment on a 16B boundary. Size == 16B
 150         */
 151        MV_CRQB_Q_SZ            = (32 * MV_MAX_Q_DEPTH),
 152        MV_CRPB_Q_SZ            = (8 * MV_MAX_Q_DEPTH),
 153        MV_MAX_SG_CT            = 256,
 154        MV_SG_TBL_SZ            = (16 * MV_MAX_SG_CT),
 155
 156        /* Determine hc from 0-7 port: hc = port >> MV_PORT_HC_SHIFT */
 157        MV_PORT_HC_SHIFT        = 2,
 158        MV_PORTS_PER_HC         = (1 << MV_PORT_HC_SHIFT), /* 4 */
 159        /* Determine hc port from 0-7 port: hardport = port & MV_PORT_MASK */
 160        MV_PORT_MASK            = (MV_PORTS_PER_HC - 1),   /* 3 */
 161
 162        /* Host Flags */
 163        MV_FLAG_DUAL_HC         = (1 << 30),  /* two SATA Host Controllers */
 164
 165        MV_COMMON_FLAGS         = ATA_FLAG_SATA | ATA_FLAG_PIO_POLLING,
 166
 167        MV_GEN_I_FLAGS          = MV_COMMON_FLAGS | ATA_FLAG_NO_ATAPI,
 168
 169        MV_GEN_II_FLAGS         = MV_COMMON_FLAGS | ATA_FLAG_NCQ |
 170                                  ATA_FLAG_PMP | ATA_FLAG_ACPI_SATA,
 171
 172        MV_GEN_IIE_FLAGS        = MV_GEN_II_FLAGS | ATA_FLAG_AN,
 173
 174        CRQB_FLAG_READ          = (1 << 0),
 175        CRQB_TAG_SHIFT          = 1,
 176        CRQB_IOID_SHIFT         = 6,    /* CRQB Gen-II/IIE IO Id shift */
 177        CRQB_PMP_SHIFT          = 12,   /* CRQB Gen-II/IIE PMP shift */
 178        CRQB_HOSTQ_SHIFT        = 17,   /* CRQB Gen-II/IIE HostQueTag shift */
 179        CRQB_CMD_ADDR_SHIFT     = 8,
 180        CRQB_CMD_CS             = (0x2 << 11),
 181        CRQB_CMD_LAST           = (1 << 15),
 182
 183        CRPB_FLAG_STATUS_SHIFT  = 8,
 184        CRPB_IOID_SHIFT_6       = 5,    /* CRPB Gen-II IO Id shift */
 185        CRPB_IOID_SHIFT_7       = 7,    /* CRPB Gen-IIE IO Id shift */
 186
 187        EPRD_FLAG_END_OF_TBL    = (1 << 31),
 188
 189        /* PCI interface registers */
 190
 191        MV_PCI_COMMAND          = 0xc00,
 192        MV_PCI_COMMAND_MWRCOM   = (1 << 4),     /* PCI Master Write Combining */
 193        MV_PCI_COMMAND_MRDTRIG  = (1 << 7),     /* PCI Master Read Trigger */
 194
 195        PCI_MAIN_CMD_STS        = 0xd30,
 196        STOP_PCI_MASTER         = (1 << 2),
 197        PCI_MASTER_EMPTY        = (1 << 3),
 198        GLOB_SFT_RST            = (1 << 4),
 199
 200        MV_PCI_MODE             = 0xd00,
 201        MV_PCI_MODE_MASK        = 0x30,
 202
 203        MV_PCI_EXP_ROM_BAR_CTL  = 0xd2c,
 204        MV_PCI_DISC_TIMER       = 0xd04,
 205        MV_PCI_MSI_TRIGGER      = 0xc38,
 206        MV_PCI_SERR_MASK        = 0xc28,
 207        MV_PCI_XBAR_TMOUT       = 0x1d04,
 208        MV_PCI_ERR_LOW_ADDRESS  = 0x1d40,
 209        MV_PCI_ERR_HIGH_ADDRESS = 0x1d44,
 210        MV_PCI_ERR_ATTRIBUTE    = 0x1d48,
 211        MV_PCI_ERR_COMMAND      = 0x1d50,
 212
 213        PCI_IRQ_CAUSE           = 0x1d58,
 214        PCI_IRQ_MASK            = 0x1d5c,
 215        PCI_UNMASK_ALL_IRQS     = 0x7fffff,     /* bits 22-0 */
 216
 217        PCIE_IRQ_CAUSE          = 0x1900,
 218        PCIE_IRQ_MASK           = 0x1910,
 219        PCIE_UNMASK_ALL_IRQS    = 0x40a,        /* assorted bits */
 220
 221        /* Host Controller Main Interrupt Cause/Mask registers (1 per-chip) */
 222        PCI_HC_MAIN_IRQ_CAUSE   = 0x1d60,
 223        PCI_HC_MAIN_IRQ_MASK    = 0x1d64,
 224        SOC_HC_MAIN_IRQ_CAUSE   = 0x20020,
 225        SOC_HC_MAIN_IRQ_MASK    = 0x20024,
 226        ERR_IRQ                 = (1 << 0),     /* shift by (2 * port #) */
 227        DONE_IRQ                = (1 << 1),     /* shift by (2 * port #) */
 228        HC0_IRQ_PEND            = 0x1ff,        /* bits 0-8 = HC0's ports */
 229        HC_SHIFT                = 9,            /* bits 9-17 = HC1's ports */
 230        DONE_IRQ_0_3            = 0x000000aa,   /* DONE_IRQ ports 0,1,2,3 */
 231        DONE_IRQ_4_7            = (DONE_IRQ_0_3 << HC_SHIFT),  /* 4,5,6,7 */
 232        PCI_ERR                 = (1 << 18),
 233        TRAN_COAL_LO_DONE       = (1 << 19),    /* transaction coalescing */
 234        TRAN_COAL_HI_DONE       = (1 << 20),    /* transaction coalescing */
 235        PORTS_0_3_COAL_DONE     = (1 << 8),     /* HC0 IRQ coalescing */
 236        PORTS_4_7_COAL_DONE     = (1 << 17),    /* HC1 IRQ coalescing */
 237        ALL_PORTS_COAL_DONE     = (1 << 21),    /* GEN_II(E) IRQ coalescing */
 238        GPIO_INT                = (1 << 22),
 239        SELF_INT                = (1 << 23),
 240        TWSI_INT                = (1 << 24),
 241        HC_MAIN_RSVD            = (0x7f << 25), /* bits 31-25 */
 242        HC_MAIN_RSVD_5          = (0x1fff << 19), /* bits 31-19 */
 243        HC_MAIN_RSVD_SOC        = (0x3fffffb << 6),     /* bits 31-9, 7-6 */
 244
 245        /* SATAHC registers */
 246        HC_CFG                  = 0x00,
 247
 248        HC_IRQ_CAUSE            = 0x14,
 249        DMA_IRQ                 = (1 << 0),     /* shift by port # */
 250        HC_COAL_IRQ             = (1 << 4),     /* IRQ coalescing */
 251        DEV_IRQ                 = (1 << 8),     /* shift by port # */
 252
 253        /*
 254         * Per-HC (Host-Controller) interrupt coalescing feature.
 255         * This is present on all chip generations.
 256         *
 257         * Coalescing defers the interrupt until either the IO_THRESHOLD
 258         * (count of completed I/Os) is met, or the TIME_THRESHOLD is met.
 259         */
 260        HC_IRQ_COAL_IO_THRESHOLD        = 0x000c,
 261        HC_IRQ_COAL_TIME_THRESHOLD      = 0x0010,
 262
 263        SOC_LED_CTRL            = 0x2c,
 264        SOC_LED_CTRL_BLINK      = (1 << 0),     /* Active LED blink */
 265        SOC_LED_CTRL_ACT_PRESENCE = (1 << 2),   /* Multiplex dev presence */
 266                                                /*  with dev activity LED */
 267
 268        /* Shadow block registers */
 269        SHD_BLK                 = 0x100,
 270        SHD_CTL_AST             = 0x20,         /* ofs from SHD_BLK */
 271
 272        /* SATA registers */
 273        SATA_STATUS             = 0x300,  /* ctrl, err regs follow status */
 274        SATA_ACTIVE             = 0x350,
 275        FIS_IRQ_CAUSE           = 0x364,
 276        FIS_IRQ_CAUSE_AN        = (1 << 9),     /* async notification */
 277
 278        LTMODE                  = 0x30c,        /* requires read-after-write */
 279        LTMODE_BIT8             = (1 << 8),     /* unknown, but necessary */
 280
 281        PHY_MODE2               = 0x330,
 282        PHY_MODE3               = 0x310,
 283
 284        PHY_MODE4               = 0x314,        /* requires read-after-write */
 285        PHY_MODE4_CFG_MASK      = 0x00000003,   /* phy internal config field */
 286        PHY_MODE4_CFG_VALUE     = 0x00000001,   /* phy internal config field */
 287        PHY_MODE4_RSVD_ZEROS    = 0x5de3fffa,   /* Gen2e always write zeros */
 288        PHY_MODE4_RSVD_ONES     = 0x00000005,   /* Gen2e always write ones */
 289
 290        SATA_IFCTL              = 0x344,
 291        SATA_TESTCTL            = 0x348,
 292        SATA_IFSTAT             = 0x34c,
 293        VENDOR_UNIQUE_FIS       = 0x35c,
 294
 295        FISCFG                  = 0x360,
 296        FISCFG_WAIT_DEV_ERR     = (1 << 8),     /* wait for host on DevErr */
 297        FISCFG_SINGLE_SYNC      = (1 << 16),    /* SYNC on DMA activation */
 298
 299        PHY_MODE9_GEN2          = 0x398,
 300        PHY_MODE9_GEN1          = 0x39c,
 301        PHYCFG_OFS              = 0x3a0,        /* only in 65n devices */
 302
 303        MV5_PHY_MODE            = 0x74,
 304        MV5_LTMODE              = 0x30,
 305        MV5_PHY_CTL             = 0x0C,
 306        SATA_IFCFG              = 0x050,
 307
 308        MV_M2_PREAMP_MASK       = 0x7e0,
 309
 310        /* Port registers */
 311        EDMA_CFG                = 0,
 312        EDMA_CFG_Q_DEPTH        = 0x1f,         /* max device queue depth */
 313        EDMA_CFG_NCQ            = (1 << 5),     /* for R/W FPDMA queued */
 314        EDMA_CFG_NCQ_GO_ON_ERR  = (1 << 14),    /* continue on error */
 315        EDMA_CFG_RD_BRST_EXT    = (1 << 11),    /* read burst 512B */
 316        EDMA_CFG_WR_BUFF_LEN    = (1 << 13),    /* write buffer 512B */
 317        EDMA_CFG_EDMA_FBS       = (1 << 16),    /* EDMA FIS-Based Switching */
 318        EDMA_CFG_FBS            = (1 << 26),    /* FIS-Based Switching */
 319
 320        EDMA_ERR_IRQ_CAUSE      = 0x8,
 321        EDMA_ERR_IRQ_MASK       = 0xc,
 322        EDMA_ERR_D_PAR          = (1 << 0),     /* UDMA data parity err */
 323        EDMA_ERR_PRD_PAR        = (1 << 1),     /* UDMA PRD parity err */
 324        EDMA_ERR_DEV            = (1 << 2),     /* device error */
 325        EDMA_ERR_DEV_DCON       = (1 << 3),     /* device disconnect */
 326        EDMA_ERR_DEV_CON        = (1 << 4),     /* device connected */
 327        EDMA_ERR_SERR           = (1 << 5),     /* SError bits [WBDST] raised */
 328        EDMA_ERR_SELF_DIS       = (1 << 7),     /* Gen II/IIE self-disable */
 329        EDMA_ERR_SELF_DIS_5     = (1 << 8),     /* Gen I self-disable */
 330        EDMA_ERR_BIST_ASYNC     = (1 << 8),     /* BIST FIS or Async Notify */
 331        EDMA_ERR_TRANS_IRQ_7    = (1 << 8),     /* Gen IIE transprt layer irq */
 332        EDMA_ERR_CRQB_PAR       = (1 << 9),     /* CRQB parity error */
 333        EDMA_ERR_CRPB_PAR       = (1 << 10),    /* CRPB parity error */
 334        EDMA_ERR_INTRL_PAR      = (1 << 11),    /* internal parity error */
 335        EDMA_ERR_IORDY          = (1 << 12),    /* IORdy timeout */
 336
 337        EDMA_ERR_LNK_CTRL_RX    = (0xf << 13),  /* link ctrl rx error */
 338        EDMA_ERR_LNK_CTRL_RX_0  = (1 << 13),    /* transient: CRC err */
 339        EDMA_ERR_LNK_CTRL_RX_1  = (1 << 14),    /* transient: FIFO err */
 340        EDMA_ERR_LNK_CTRL_RX_2  = (1 << 15),    /* fatal: caught SYNC */
 341        EDMA_ERR_LNK_CTRL_RX_3  = (1 << 16),    /* transient: FIS rx err */
 342
 343        EDMA_ERR_LNK_DATA_RX    = (0xf << 17),  /* link data rx error */
 344
 345        EDMA_ERR_LNK_CTRL_TX    = (0x1f << 21), /* link ctrl tx error */
 346        EDMA_ERR_LNK_CTRL_TX_0  = (1 << 21),    /* transient: CRC err */
 347        EDMA_ERR_LNK_CTRL_TX_1  = (1 << 22),    /* transient: FIFO err */
 348        EDMA_ERR_LNK_CTRL_TX_2  = (1 << 23),    /* transient: caught SYNC */
 349        EDMA_ERR_LNK_CTRL_TX_3  = (1 << 24),    /* transient: caught DMAT */
 350        EDMA_ERR_LNK_CTRL_TX_4  = (1 << 25),    /* transient: FIS collision */
 351
 352        EDMA_ERR_LNK_DATA_TX    = (0x1f << 26), /* link data tx error */
 353
 354        EDMA_ERR_TRANS_PROTO    = (1 << 31),    /* transport protocol error */
 355        EDMA_ERR_OVERRUN_5      = (1 << 5),
 356        EDMA_ERR_UNDERRUN_5     = (1 << 6),
 357
 358        EDMA_ERR_IRQ_TRANSIENT  = EDMA_ERR_LNK_CTRL_RX_0 |
 359                                  EDMA_ERR_LNK_CTRL_RX_1 |
 360                                  EDMA_ERR_LNK_CTRL_RX_3 |
 361                                  EDMA_ERR_LNK_CTRL_TX,
 362
 363        EDMA_EH_FREEZE          = EDMA_ERR_D_PAR |
 364                                  EDMA_ERR_PRD_PAR |
 365                                  EDMA_ERR_DEV_DCON |
 366                                  EDMA_ERR_DEV_CON |
 367                                  EDMA_ERR_SERR |
 368                                  EDMA_ERR_SELF_DIS |
 369                                  EDMA_ERR_CRQB_PAR |
 370                                  EDMA_ERR_CRPB_PAR |
 371                                  EDMA_ERR_INTRL_PAR |
 372                                  EDMA_ERR_IORDY |
 373                                  EDMA_ERR_LNK_CTRL_RX_2 |
 374                                  EDMA_ERR_LNK_DATA_RX |
 375                                  EDMA_ERR_LNK_DATA_TX |
 376                                  EDMA_ERR_TRANS_PROTO,
 377
 378        EDMA_EH_FREEZE_5        = EDMA_ERR_D_PAR |
 379                                  EDMA_ERR_PRD_PAR |
 380                                  EDMA_ERR_DEV_DCON |
 381                                  EDMA_ERR_DEV_CON |
 382                                  EDMA_ERR_OVERRUN_5 |
 383                                  EDMA_ERR_UNDERRUN_5 |
 384                                  EDMA_ERR_SELF_DIS_5 |
 385                                  EDMA_ERR_CRQB_PAR |
 386                                  EDMA_ERR_CRPB_PAR |
 387                                  EDMA_ERR_INTRL_PAR |
 388                                  EDMA_ERR_IORDY,
 389
 390        EDMA_REQ_Q_BASE_HI      = 0x10,
 391        EDMA_REQ_Q_IN_PTR       = 0x14,         /* also contains BASE_LO */
 392
 393        EDMA_REQ_Q_OUT_PTR      = 0x18,
 394        EDMA_REQ_Q_PTR_SHIFT    = 5,
 395
 396        EDMA_RSP_Q_BASE_HI      = 0x1c,
 397        EDMA_RSP_Q_IN_PTR       = 0x20,
 398        EDMA_RSP_Q_OUT_PTR      = 0x24,         /* also contains BASE_LO */
 399        EDMA_RSP_Q_PTR_SHIFT    = 3,
 400
 401        EDMA_CMD                = 0x28,         /* EDMA command register */
 402        EDMA_EN                 = (1 << 0),     /* enable EDMA */
 403        EDMA_DS                 = (1 << 1),     /* disable EDMA; self-negated */
 404        EDMA_RESET              = (1 << 2),     /* reset eng/trans/link/phy */
 405
 406        EDMA_STATUS             = 0x30,         /* EDMA engine status */
 407        EDMA_STATUS_CACHE_EMPTY = (1 << 6),     /* GenIIe command cache empty */
 408        EDMA_STATUS_IDLE        = (1 << 7),     /* GenIIe EDMA enabled/idle */
 409
 410        EDMA_IORDY_TMOUT        = 0x34,
 411        EDMA_ARB_CFG            = 0x38,
 412
 413        EDMA_HALTCOND           = 0x60,         /* GenIIe halt conditions */
 414        EDMA_UNKNOWN_RSVD       = 0x6C,         /* GenIIe unknown/reserved */
 415
 416        BMDMA_CMD               = 0x224,        /* bmdma command register */
 417        BMDMA_STATUS            = 0x228,        /* bmdma status register */
 418        BMDMA_PRD_LOW           = 0x22c,        /* bmdma PRD addr 31:0 */
 419        BMDMA_PRD_HIGH          = 0x230,        /* bmdma PRD addr 63:32 */
 420
 421        /* Host private flags (hp_flags) */
 422        MV_HP_FLAG_MSI          = (1 << 0),
 423        MV_HP_ERRATA_50XXB0     = (1 << 1),
 424        MV_HP_ERRATA_50XXB2     = (1 << 2),
 425        MV_HP_ERRATA_60X1B2     = (1 << 3),
 426        MV_HP_ERRATA_60X1C0     = (1 << 4),
 427        MV_HP_GEN_I             = (1 << 6),     /* Generation I: 50xx */
 428        MV_HP_GEN_II            = (1 << 7),     /* Generation II: 60xx */
 429        MV_HP_GEN_IIE           = (1 << 8),     /* Generation IIE: 6042/7042 */
 430        MV_HP_PCIE              = (1 << 9),     /* PCIe bus/regs: 7042 */
 431        MV_HP_CUT_THROUGH       = (1 << 10),    /* can use EDMA cut-through */
 432        MV_HP_FLAG_SOC          = (1 << 11),    /* SystemOnChip, no PCI */
 433        MV_HP_QUIRK_LED_BLINK_EN = (1 << 12),   /* is led blinking enabled? */
 434
 435        /* Port private flags (pp_flags) */
 436        MV_PP_FLAG_EDMA_EN      = (1 << 0),     /* is EDMA engine enabled? */
 437        MV_PP_FLAG_NCQ_EN       = (1 << 1),     /* is EDMA set up for NCQ? */
 438        MV_PP_FLAG_FBS_EN       = (1 << 2),     /* is EDMA set up for FBS? */
 439        MV_PP_FLAG_DELAYED_EH   = (1 << 3),     /* delayed dev err handling */
 440        MV_PP_FLAG_FAKE_ATA_BUSY = (1 << 4),    /* ignore initial ATA_DRDY */
 441};
 442
 443#define IS_GEN_I(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_I)
 444#define IS_GEN_II(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_II)
 445#define IS_GEN_IIE(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_IIE)
 446#define IS_PCIE(hpriv) ((hpriv)->hp_flags & MV_HP_PCIE)
 447#define IS_SOC(hpriv) ((hpriv)->hp_flags & MV_HP_FLAG_SOC)
 448
 449#define WINDOW_CTRL(i)          (0x20030 + ((i) << 4))
 450#define WINDOW_BASE(i)          (0x20034 + ((i) << 4))
 451
 452enum {
 453        /* DMA boundary 0xffff is required by the s/g splitting
 454         * we need on /length/ in mv_fill-sg().
 455         */
 456        MV_DMA_BOUNDARY         = 0xffffU,
 457
 458        /* mask of register bits containing lower 32 bits
 459         * of EDMA request queue DMA address
 460         */
 461        EDMA_REQ_Q_BASE_LO_MASK = 0xfffffc00U,
 462
 463        /* ditto, for response queue */
 464        EDMA_RSP_Q_BASE_LO_MASK = 0xffffff00U,
 465};
 466
 467enum chip_type {
 468        chip_504x,
 469        chip_508x,
 470        chip_5080,
 471        chip_604x,
 472        chip_608x,
 473        chip_6042,
 474        chip_7042,
 475        chip_soc,
 476};
 477
 478/* Command ReQuest Block: 32B */
 479struct mv_crqb {
 480        __le32                  sg_addr;
 481        __le32                  sg_addr_hi;
 482        __le16                  ctrl_flags;
 483        __le16                  ata_cmd[11];
 484};
 485
 486struct mv_crqb_iie {
 487        __le32                  addr;
 488        __le32                  addr_hi;
 489        __le32                  flags;
 490        __le32                  len;
 491        __le32                  ata_cmd[4];
 492};
 493
 494/* Command ResPonse Block: 8B */
 495struct mv_crpb {
 496        __le16                  id;
 497        __le16                  flags;
 498        __le32                  tmstmp;
 499};
 500
 501/* EDMA Physical Region Descriptor (ePRD); A.K.A. SG */
 502struct mv_sg {
 503        __le32                  addr;
 504        __le32                  flags_size;
 505        __le32                  addr_hi;
 506        __le32                  reserved;
 507};
 508
 509/*
 510 * We keep a local cache of a few frequently accessed port
 511 * registers here, to avoid having to read them (very slow)
 512 * when switching between EDMA and non-EDMA modes.
 513 */
 514struct mv_cached_regs {
 515        u32                     fiscfg;
 516        u32                     ltmode;
 517        u32                     haltcond;
 518        u32                     unknown_rsvd;
 519};
 520
 521struct mv_port_priv {
 522        struct mv_crqb          *crqb;
 523        dma_addr_t              crqb_dma;
 524        struct mv_crpb          *crpb;
 525        dma_addr_t              crpb_dma;
 526        struct mv_sg            *sg_tbl[MV_MAX_Q_DEPTH];
 527        dma_addr_t              sg_tbl_dma[MV_MAX_Q_DEPTH];
 528
 529        unsigned int            req_idx;
 530        unsigned int            resp_idx;
 531
 532        u32                     pp_flags;
 533        struct mv_cached_regs   cached;
 534        unsigned int            delayed_eh_pmp_map;
 535};
 536
 537struct mv_port_signal {
 538        u32                     amps;
 539        u32                     pre;
 540};
 541
 542struct mv_host_priv {
 543        u32                     hp_flags;
 544        unsigned int            board_idx;
 545        u32                     main_irq_mask;
 546        struct mv_port_signal   signal[8];
 547        const struct mv_hw_ops  *ops;
 548        int                     n_ports;
 549        void __iomem            *base;
 550        void __iomem            *main_irq_cause_addr;
 551        void __iomem            *main_irq_mask_addr;
 552        u32                     irq_cause_offset;
 553        u32                     irq_mask_offset;
 554        u32                     unmask_all_irqs;
 555
 556#if defined(CONFIG_HAVE_CLK)
 557        struct clk              *clk;
 558        struct clk              **port_clks;
 559#endif
 560        /*
 561         * These consistent DMA memory pools give us guaranteed
 562         * alignment for hardware-accessed data structures,
 563         * and less memory waste in accomplishing the alignment.
 564         */
 565        struct dma_pool         *crqb_pool;
 566        struct dma_pool         *crpb_pool;
 567        struct dma_pool         *sg_tbl_pool;
 568};
 569
 570struct mv_hw_ops {
 571        void (*phy_errata)(struct mv_host_priv *hpriv, void __iomem *mmio,
 572                           unsigned int port);
 573        void (*enable_leds)(struct mv_host_priv *hpriv, void __iomem *mmio);
 574        void (*read_preamp)(struct mv_host_priv *hpriv, int idx,
 575                           void __iomem *mmio);
 576        int (*reset_hc)(struct mv_host_priv *hpriv, void __iomem *mmio,
 577                        unsigned int n_hc);
 578        void (*reset_flash)(struct mv_host_priv *hpriv, void __iomem *mmio);
 579        void (*reset_bus)(struct ata_host *host, void __iomem *mmio);
 580};
 581
 582static int mv_scr_read(struct ata_link *link, unsigned int sc_reg_in, u32 *val);
 583static int mv_scr_write(struct ata_link *link, unsigned int sc_reg_in, u32 val);
 584static int mv5_scr_read(struct ata_link *link, unsigned int sc_reg_in, u32 *val);
 585static int mv5_scr_write(struct ata_link *link, unsigned int sc_reg_in, u32 val);
 586static int mv_port_start(struct ata_port *ap);
 587static void mv_port_stop(struct ata_port *ap);
 588static int mv_qc_defer(struct ata_queued_cmd *qc);
 589static void mv_qc_prep(struct ata_queued_cmd *qc);
 590static void mv_qc_prep_iie(struct ata_queued_cmd *qc);
 591static unsigned int mv_qc_issue(struct ata_queued_cmd *qc);
 592static int mv_hardreset(struct ata_link *link, unsigned int *class,
 593                        unsigned long deadline);
 594static void mv_eh_freeze(struct ata_port *ap);
 595static void mv_eh_thaw(struct ata_port *ap);
 596static void mv6_dev_config(struct ata_device *dev);
 597
 598static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
 599                           unsigned int port);
 600static void mv5_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio);
 601static void mv5_read_preamp(struct mv_host_priv *hpriv, int idx,
 602                           void __iomem *mmio);
 603static int mv5_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
 604                        unsigned int n_hc);
 605static void mv5_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio);
 606static void mv5_reset_bus(struct ata_host *host, void __iomem *mmio);
 607
 608static void mv6_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
 609                           unsigned int port);
 610static void mv6_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio);
 611static void mv6_read_preamp(struct mv_host_priv *hpriv, int idx,
 612                           void __iomem *mmio);
 613static int mv6_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
 614                        unsigned int n_hc);
 615static void mv6_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio);
 616static void mv_soc_enable_leds(struct mv_host_priv *hpriv,
 617                                      void __iomem *mmio);
 618static void mv_soc_read_preamp(struct mv_host_priv *hpriv, int idx,
 619                                      void __iomem *mmio);
 620static int mv_soc_reset_hc(struct mv_host_priv *hpriv,
 621                                  void __iomem *mmio, unsigned int n_hc);
 622static void mv_soc_reset_flash(struct mv_host_priv *hpriv,
 623                                      void __iomem *mmio);
 624static void mv_soc_reset_bus(struct ata_host *host, void __iomem *mmio);
 625static void mv_soc_65n_phy_errata(struct mv_host_priv *hpriv,
 626                                  void __iomem *mmio, unsigned int port);
 627static void mv_reset_pci_bus(struct ata_host *host, void __iomem *mmio);
 628static void mv_reset_channel(struct mv_host_priv *hpriv, void __iomem *mmio,
 629                             unsigned int port_no);
 630static int mv_stop_edma(struct ata_port *ap);
 631static int mv_stop_edma_engine(void __iomem *port_mmio);
 632static void mv_edma_cfg(struct ata_port *ap, int want_ncq, int want_edma);
 633
 634static void mv_pmp_select(struct ata_port *ap, int pmp);
 635static int mv_pmp_hardreset(struct ata_link *link, unsigned int *class,
 636                                unsigned long deadline);
 637static int  mv_softreset(struct ata_link *link, unsigned int *class,
 638                                unsigned long deadline);
 639static void mv_pmp_error_handler(struct ata_port *ap);
 640static void mv_process_crpb_entries(struct ata_port *ap,
 641                                        struct mv_port_priv *pp);
 642
 643static void mv_sff_irq_clear(struct ata_port *ap);
 644static int mv_check_atapi_dma(struct ata_queued_cmd *qc);
 645static void mv_bmdma_setup(struct ata_queued_cmd *qc);
 646static void mv_bmdma_start(struct ata_queued_cmd *qc);
 647static void mv_bmdma_stop(struct ata_queued_cmd *qc);
 648static u8   mv_bmdma_status(struct ata_port *ap);
 649static u8 mv_sff_check_status(struct ata_port *ap);
 650
 651/* .sg_tablesize is (MV_MAX_SG_CT / 2) in the structures below
 652 * because we have to allow room for worst case splitting of
 653 * PRDs for 64K boundaries in mv_fill_sg().
 654 */
 655#ifdef CONFIG_PCI
 656static struct scsi_host_template mv5_sht = {
 657        ATA_BASE_SHT(DRV_NAME),
 658        .sg_tablesize           = MV_MAX_SG_CT / 2,
 659        .dma_boundary           = MV_DMA_BOUNDARY,
 660};
 661#endif
 662static struct scsi_host_template mv6_sht = {
 663        ATA_NCQ_SHT(DRV_NAME),
 664        .can_queue              = MV_MAX_Q_DEPTH - 1,
 665        .sg_tablesize           = MV_MAX_SG_CT / 2,
 666        .dma_boundary           = MV_DMA_BOUNDARY,
 667};
 668
 669static struct ata_port_operations mv5_ops = {
 670        .inherits               = &ata_sff_port_ops,
 671
 672        .lost_interrupt         = ATA_OP_NULL,
 673
 674        .qc_defer               = mv_qc_defer,
 675        .qc_prep                = mv_qc_prep,
 676        .qc_issue               = mv_qc_issue,
 677
 678        .freeze                 = mv_eh_freeze,
 679        .thaw                   = mv_eh_thaw,
 680        .hardreset              = mv_hardreset,
 681
 682        .scr_read               = mv5_scr_read,
 683        .scr_write              = mv5_scr_write,
 684
 685        .port_start             = mv_port_start,
 686        .port_stop              = mv_port_stop,
 687};
 688
 689static struct ata_port_operations mv6_ops = {
 690        .inherits               = &ata_bmdma_port_ops,
 691
 692        .lost_interrupt         = ATA_OP_NULL,
 693
 694        .qc_defer               = mv_qc_defer,
 695        .qc_prep                = mv_qc_prep,
 696        .qc_issue               = mv_qc_issue,
 697
 698        .dev_config             = mv6_dev_config,
 699
 700        .freeze                 = mv_eh_freeze,
 701        .thaw                   = mv_eh_thaw,
 702        .hardreset              = mv_hardreset,
 703        .softreset              = mv_softreset,
 704        .pmp_hardreset          = mv_pmp_hardreset,
 705        .pmp_softreset          = mv_softreset,
 706        .error_handler          = mv_pmp_error_handler,
 707
 708        .scr_read               = mv_scr_read,
 709        .scr_write              = mv_scr_write,
 710
 711        .sff_check_status       = mv_sff_check_status,
 712        .sff_irq_clear          = mv_sff_irq_clear,
 713        .check_atapi_dma        = mv_check_atapi_dma,
 714        .bmdma_setup            = mv_bmdma_setup,
 715        .bmdma_start            = mv_bmdma_start,
 716        .bmdma_stop             = mv_bmdma_stop,
 717        .bmdma_status           = mv_bmdma_status,
 718
 719        .port_start             = mv_port_start,
 720        .port_stop              = mv_port_stop,
 721};
 722
 723static struct ata_port_operations mv_iie_ops = {
 724        .inherits               = &mv6_ops,
 725        .dev_config             = ATA_OP_NULL,
 726        .qc_prep                = mv_qc_prep_iie,
 727};
 728
 729static const struct ata_port_info mv_port_info[] = {
 730        {  /* chip_504x */
 731                .flags          = MV_GEN_I_FLAGS,
 732                .pio_mask       = ATA_PIO4,
 733                .udma_mask      = ATA_UDMA6,
 734                .port_ops       = &mv5_ops,
 735        },
 736        {  /* chip_508x */
 737                .flags          = MV_GEN_I_FLAGS | MV_FLAG_DUAL_HC,
 738                .pio_mask       = ATA_PIO4,
 739                .udma_mask      = ATA_UDMA6,
 740                .port_ops       = &mv5_ops,
 741        },
 742        {  /* chip_5080 */
 743                .flags          = MV_GEN_I_FLAGS | MV_FLAG_DUAL_HC,
 744                .pio_mask       = ATA_PIO4,
 745                .udma_mask      = ATA_UDMA6,
 746                .port_ops       = &mv5_ops,
 747        },
 748        {  /* chip_604x */
 749                .flags          = MV_GEN_II_FLAGS,
 750                .pio_mask       = ATA_PIO4,
 751                .udma_mask      = ATA_UDMA6,
 752                .port_ops       = &mv6_ops,
 753        },
 754        {  /* chip_608x */
 755                .flags          = MV_GEN_II_FLAGS | MV_FLAG_DUAL_HC,
 756                .pio_mask       = ATA_PIO4,
 757                .udma_mask      = ATA_UDMA6,
 758                .port_ops       = &mv6_ops,
 759        },
 760        {  /* chip_6042 */
 761                .flags          = MV_GEN_IIE_FLAGS,
 762                .pio_mask       = ATA_PIO4,
 763                .udma_mask      = ATA_UDMA6,
 764                .port_ops       = &mv_iie_ops,
 765        },
 766        {  /* chip_7042 */
 767                .flags          = MV_GEN_IIE_FLAGS,
 768                .pio_mask       = ATA_PIO4,
 769                .udma_mask      = ATA_UDMA6,
 770                .port_ops       = &mv_iie_ops,
 771        },
 772        {  /* chip_soc */
 773                .flags          = MV_GEN_IIE_FLAGS,
 774                .pio_mask       = ATA_PIO4,
 775                .udma_mask      = ATA_UDMA6,
 776                .port_ops       = &mv_iie_ops,
 777        },
 778};
 779
 780static const struct pci_device_id mv_pci_tbl[] = {
 781        { PCI_VDEVICE(MARVELL, 0x5040), chip_504x },
 782        { PCI_VDEVICE(MARVELL, 0x5041), chip_504x },
 783        { PCI_VDEVICE(MARVELL, 0x5080), chip_5080 },
 784        { PCI_VDEVICE(MARVELL, 0x5081), chip_508x },
 785        /* RocketRAID 1720/174x have different identifiers */
 786        { PCI_VDEVICE(TTI, 0x1720), chip_6042 },
 787        { PCI_VDEVICE(TTI, 0x1740), chip_6042 },
 788        { PCI_VDEVICE(TTI, 0x1742), chip_6042 },
 789
 790        { PCI_VDEVICE(MARVELL, 0x6040), chip_604x },
 791        { PCI_VDEVICE(MARVELL, 0x6041), chip_604x },
 792        { PCI_VDEVICE(MARVELL, 0x6042), chip_6042 },
 793        { PCI_VDEVICE(MARVELL, 0x6080), chip_608x },
 794        { PCI_VDEVICE(MARVELL, 0x6081), chip_608x },
 795
 796        { PCI_VDEVICE(ADAPTEC2, 0x0241), chip_604x },
 797
 798        /* Adaptec 1430SA */
 799        { PCI_VDEVICE(ADAPTEC2, 0x0243), chip_7042 },
 800
 801        /* Marvell 7042 support */
 802        { PCI_VDEVICE(MARVELL, 0x7042), chip_7042 },
 803
 804        /* Highpoint RocketRAID PCIe series */
 805        { PCI_VDEVICE(TTI, 0x2300), chip_7042 },
 806        { PCI_VDEVICE(TTI, 0x2310), chip_7042 },
 807
 808        { }                     /* terminate list */
 809};
 810
 811static const struct mv_hw_ops mv5xxx_ops = {
 812        .phy_errata             = mv5_phy_errata,
 813        .enable_leds            = mv5_enable_leds,
 814        .read_preamp            = mv5_read_preamp,
 815        .reset_hc               = mv5_reset_hc,
 816        .reset_flash            = mv5_reset_flash,
 817        .reset_bus              = mv5_reset_bus,
 818};
 819
 820static const struct mv_hw_ops mv6xxx_ops = {
 821        .phy_errata             = mv6_phy_errata,
 822        .enable_leds            = mv6_enable_leds,
 823        .read_preamp            = mv6_read_preamp,
 824        .reset_hc               = mv6_reset_hc,
 825        .reset_flash            = mv6_reset_flash,
 826        .reset_bus              = mv_reset_pci_bus,
 827};
 828
 829static const struct mv_hw_ops mv_soc_ops = {
 830        .phy_errata             = mv6_phy_errata,
 831        .enable_leds            = mv_soc_enable_leds,
 832        .read_preamp            = mv_soc_read_preamp,
 833        .reset_hc               = mv_soc_reset_hc,
 834        .reset_flash            = mv_soc_reset_flash,
 835        .reset_bus              = mv_soc_reset_bus,
 836};
 837
 838static const struct mv_hw_ops mv_soc_65n_ops = {
 839        .phy_errata             = mv_soc_65n_phy_errata,
 840        .enable_leds            = mv_soc_enable_leds,
 841        .reset_hc               = mv_soc_reset_hc,
 842        .reset_flash            = mv_soc_reset_flash,
 843        .reset_bus              = mv_soc_reset_bus,
 844};
 845
 846/*
 847 * Functions
 848 */
 849
 850static inline void writelfl(unsigned long data, void __iomem *addr)
 851{
 852        writel(data, addr);
 853        (void) readl(addr);     /* flush to avoid PCI posted write */
 854}
 855
 856static inline unsigned int mv_hc_from_port(unsigned int port)
 857{
 858        return port >> MV_PORT_HC_SHIFT;
 859}
 860
 861static inline unsigned int mv_hardport_from_port(unsigned int port)
 862{
 863        return port & MV_PORT_MASK;
 864}
 865
 866/*
 867 * Consolidate some rather tricky bit shift calculations.
 868 * This is hot-path stuff, so not a function.
 869 * Simple code, with two return values, so macro rather than inline.
 870 *
 871 * port is the sole input, in range 0..7.
 872 * shift is one output, for use with main_irq_cause / main_irq_mask registers.
 873 * hardport is the other output, in range 0..3.
 874 *
 875 * Note that port and hardport may be the same variable in some cases.
 876 */
 877#define MV_PORT_TO_SHIFT_AND_HARDPORT(port, shift, hardport)    \
 878{                                                               \
 879        shift    = mv_hc_from_port(port) * HC_SHIFT;            \
 880        hardport = mv_hardport_from_port(port);                 \
 881        shift   += hardport * 2;                                \
 882}
 883
 884static inline void __iomem *mv_hc_base(void __iomem *base, unsigned int hc)
 885{
 886        return (base + SATAHC0_REG_BASE + (hc * MV_SATAHC_REG_SZ));
 887}
 888
 889static inline void __iomem *mv_hc_base_from_port(void __iomem *base,
 890                                                 unsigned int port)
 891{
 892        return mv_hc_base(base, mv_hc_from_port(port));
 893}
 894
 895static inline void __iomem *mv_port_base(void __iomem *base, unsigned int port)
 896{
 897        return  mv_hc_base_from_port(base, port) +
 898                MV_SATAHC_ARBTR_REG_SZ +
 899                (mv_hardport_from_port(port) * MV_PORT_REG_SZ);
 900}
 901
 902static void __iomem *mv5_phy_base(void __iomem *mmio, unsigned int port)
 903{
 904        void __iomem *hc_mmio = mv_hc_base_from_port(mmio, port);
 905        unsigned long ofs = (mv_hardport_from_port(port) + 1) * 0x100UL;
 906
 907        return hc_mmio + ofs;
 908}
 909
 910static inline void __iomem *mv_host_base(struct ata_host *host)
 911{
 912        struct mv_host_priv *hpriv = host->private_data;
 913        return hpriv->base;
 914}
 915
 916static inline void __iomem *mv_ap_base(struct ata_port *ap)
 917{
 918        return mv_port_base(mv_host_base(ap->host), ap->port_no);
 919}
 920
 921static inline int mv_get_hc_count(unsigned long port_flags)
 922{
 923        return ((port_flags & MV_FLAG_DUAL_HC) ? 2 : 1);
 924}
 925
 926/**
 927 *      mv_save_cached_regs - (re-)initialize cached port registers
 928 *      @ap: the port whose registers we are caching
 929 *
 930 *      Initialize the local cache of port registers,
 931 *      so that reading them over and over again can
 932 *      be avoided on the hotter paths of this driver.
 933 *      This saves a few microseconds each time we switch
 934 *      to/from EDMA mode to perform (eg.) a drive cache flush.
 935 */
 936static void mv_save_cached_regs(struct ata_port *ap)
 937{
 938        void __iomem *port_mmio = mv_ap_base(ap);
 939        struct mv_port_priv *pp = ap->private_data;
 940
 941        pp->cached.fiscfg = readl(port_mmio + FISCFG);
 942        pp->cached.ltmode = readl(port_mmio + LTMODE);
 943        pp->cached.haltcond = readl(port_mmio + EDMA_HALTCOND);
 944        pp->cached.unknown_rsvd = readl(port_mmio + EDMA_UNKNOWN_RSVD);
 945}
 946
 947/**
 948 *      mv_write_cached_reg - write to a cached port register
 949 *      @addr: hardware address of the register
 950 *      @old: pointer to cached value of the register
 951 *      @new: new value for the register
 952 *
 953 *      Write a new value to a cached register,
 954 *      but only if the value is different from before.
 955 */
 956static inline void mv_write_cached_reg(void __iomem *addr, u32 *old, u32 new)
 957{
 958        if (new != *old) {
 959                unsigned long laddr;
 960                *old = new;
 961                /*
 962                 * Workaround for 88SX60x1-B2 FEr SATA#13:
 963                 * Read-after-write is needed to prevent generating 64-bit
 964                 * write cycles on the PCI bus for SATA interface registers
 965                 * at offsets ending in 0x4 or 0xc.
 966                 *
 967                 * Looks like a lot of fuss, but it avoids an unnecessary
 968                 * +1 usec read-after-write delay for unaffected registers.
 969                 */
 970                laddr = (long)addr & 0xffff;
 971                if (laddr >= 0x300 && laddr <= 0x33c) {
 972                        laddr &= 0x000f;
 973                        if (laddr == 0x4 || laddr == 0xc) {
 974                                writelfl(new, addr); /* read after write */
 975                                return;
 976                        }
 977                }
 978                writel(new, addr); /* unaffected by the errata */
 979        }
 980}
 981
 982static void mv_set_edma_ptrs(void __iomem *port_mmio,
 983                             struct mv_host_priv *hpriv,
 984                             struct mv_port_priv *pp)
 985{
 986        u32 index;
 987
 988        /*
 989         * initialize request queue
 990         */
 991        pp->req_idx &= MV_MAX_Q_DEPTH_MASK;     /* paranoia */
 992        index = pp->req_idx << EDMA_REQ_Q_PTR_SHIFT;
 993
 994        WARN_ON(pp->crqb_dma & 0x3ff);
 995        writel((pp->crqb_dma >> 16) >> 16, port_mmio + EDMA_REQ_Q_BASE_HI);
 996        writelfl((pp->crqb_dma & EDMA_REQ_Q_BASE_LO_MASK) | index,
 997                 port_mmio + EDMA_REQ_Q_IN_PTR);
 998        writelfl(index, port_mmio + EDMA_REQ_Q_OUT_PTR);
 999
1000        /*
1001         * initialize response queue
1002         */
1003        pp->resp_idx &= MV_MAX_Q_DEPTH_MASK;    /* paranoia */
1004        index = pp->resp_idx << EDMA_RSP_Q_PTR_SHIFT;
1005
1006        WARN_ON(pp->crpb_dma & 0xff);
1007        writel((pp->crpb_dma >> 16) >> 16, port_mmio + EDMA_RSP_Q_BASE_HI);
1008        writelfl(index, port_mmio + EDMA_RSP_Q_IN_PTR);
1009        writelfl((pp->crpb_dma & EDMA_RSP_Q_BASE_LO_MASK) | index,
1010                 port_mmio + EDMA_RSP_Q_OUT_PTR);
1011}
1012
1013static void mv_write_main_irq_mask(u32 mask, struct mv_host_priv *hpriv)
1014{
1015        /*
1016         * When writing to the main_irq_mask in hardware,
1017         * we must ensure exclusivity between the interrupt coalescing bits
1018         * and the corresponding individual port DONE_IRQ bits.
1019         *
1020         * Note that this register is really an "IRQ enable" register,
1021         * not an "IRQ mask" register as Marvell's naming might suggest.
1022         */
1023        if (mask & (ALL_PORTS_COAL_DONE | PORTS_0_3_COAL_DONE))
1024                mask &= ~DONE_IRQ_0_3;
1025        if (mask & (ALL_PORTS_COAL_DONE | PORTS_4_7_COAL_DONE))
1026                mask &= ~DONE_IRQ_4_7;
1027        writelfl(mask, hpriv->main_irq_mask_addr);
1028}
1029
1030static void mv_set_main_irq_mask(struct ata_host *host,
1031                                 u32 disable_bits, u32 enable_bits)
1032{
1033        struct mv_host_priv *hpriv = host->private_data;
1034        u32 old_mask, new_mask;
1035
1036        old_mask = hpriv->main_irq_mask;
1037        new_mask = (old_mask & ~disable_bits) | enable_bits;
1038        if (new_mask != old_mask) {
1039                hpriv->main_irq_mask = new_mask;
1040                mv_write_main_irq_mask(new_mask, hpriv);
1041        }
1042}
1043
1044static void mv_enable_port_irqs(struct ata_port *ap,
1045                                     unsigned int port_bits)
1046{
1047        unsigned int shift, hardport, port = ap->port_no;
1048        u32 disable_bits, enable_bits;
1049
1050        MV_PORT_TO_SHIFT_AND_HARDPORT(port, shift, hardport);
1051
1052        disable_bits = (DONE_IRQ | ERR_IRQ) << shift;
1053        enable_bits  = port_bits << shift;
1054        mv_set_main_irq_mask(ap->host, disable_bits, enable_bits);
1055}
1056
1057static void mv_clear_and_enable_port_irqs(struct ata_port *ap,
1058                                          void __iomem *port_mmio,
1059                                          unsigned int port_irqs)
1060{
1061        struct mv_host_priv *hpriv = ap->host->private_data;
1062        int hardport = mv_hardport_from_port(ap->port_no);
1063        void __iomem *hc_mmio = mv_hc_base_from_port(
1064                                mv_host_base(ap->host), ap->port_no);
1065        u32 hc_irq_cause;
1066
1067        /* clear EDMA event indicators, if any */
1068        writelfl(0, port_mmio + EDMA_ERR_IRQ_CAUSE);
1069
1070        /* clear pending irq events */
1071        hc_irq_cause = ~((DEV_IRQ | DMA_IRQ) << hardport);
1072        writelfl(hc_irq_cause, hc_mmio + HC_IRQ_CAUSE);
1073
1074        /* clear FIS IRQ Cause */
1075        if (IS_GEN_IIE(hpriv))
1076                writelfl(0, port_mmio + FIS_IRQ_CAUSE);
1077
1078        mv_enable_port_irqs(ap, port_irqs);
1079}
1080
1081static void mv_set_irq_coalescing(struct ata_host *host,
1082                                  unsigned int count, unsigned int usecs)
1083{
1084        struct mv_host_priv *hpriv = host->private_data;
1085        void __iomem *mmio = hpriv->base, *hc_mmio;
1086        u32 coal_enable = 0;
1087        unsigned long flags;
1088        unsigned int clks, is_dual_hc = hpriv->n_ports > MV_PORTS_PER_HC;
1089        const u32 coal_disable = PORTS_0_3_COAL_DONE | PORTS_4_7_COAL_DONE |
1090                                                        ALL_PORTS_COAL_DONE;
1091
1092        /* Disable IRQ coalescing if either threshold is zero */
1093        if (!usecs || !count) {
1094                clks = count = 0;
1095        } else {
1096                /* Respect maximum limits of the hardware */
1097                clks = usecs * COAL_CLOCKS_PER_USEC;
1098                if (clks > MAX_COAL_TIME_THRESHOLD)
1099                        clks = MAX_COAL_TIME_THRESHOLD;
1100                if (count > MAX_COAL_IO_COUNT)
1101                        count = MAX_COAL_IO_COUNT;
1102        }
1103
1104        spin_lock_irqsave(&host->lock, flags);
1105        mv_set_main_irq_mask(host, coal_disable, 0);
1106
1107        if (is_dual_hc && !IS_GEN_I(hpriv)) {
1108                /*
1109                 * GEN_II/GEN_IIE with dual host controllers:
1110                 * one set of global thresholds for the entire chip.
1111                 */
1112                writel(clks,  mmio + IRQ_COAL_TIME_THRESHOLD);
1113                writel(count, mmio + IRQ_COAL_IO_THRESHOLD);
1114                /* clear leftover coal IRQ bit */
1115                writel(~ALL_PORTS_COAL_IRQ, mmio + IRQ_COAL_CAUSE);
1116                if (count)
1117                        coal_enable = ALL_PORTS_COAL_DONE;
1118                clks = count = 0; /* force clearing of regular regs below */
1119        }
1120
1121        /*
1122         * All chips: independent thresholds for each HC on the chip.
1123         */
1124        hc_mmio = mv_hc_base_from_port(mmio, 0);
1125        writel(clks,  hc_mmio + HC_IRQ_COAL_TIME_THRESHOLD);
1126        writel(count, hc_mmio + HC_IRQ_COAL_IO_THRESHOLD);
1127        writel(~HC_COAL_IRQ, hc_mmio + HC_IRQ_CAUSE);
1128        if (count)
1129                coal_enable |= PORTS_0_3_COAL_DONE;
1130        if (is_dual_hc) {
1131                hc_mmio = mv_hc_base_from_port(mmio, MV_PORTS_PER_HC);
1132                writel(clks,  hc_mmio + HC_IRQ_COAL_TIME_THRESHOLD);
1133                writel(count, hc_mmio + HC_IRQ_COAL_IO_THRESHOLD);
1134                writel(~HC_COAL_IRQ, hc_mmio + HC_IRQ_CAUSE);
1135                if (count)
1136                        coal_enable |= PORTS_4_7_COAL_DONE;
1137        }
1138
1139        mv_set_main_irq_mask(host, 0, coal_enable);
1140        spin_unlock_irqrestore(&host->lock, flags);
1141}
1142
1143/**
1144 *      mv_start_edma - Enable eDMA engine
1145 *      @base: port base address
1146 *      @pp: port private data
1147 *
1148 *      Verify the local cache of the eDMA state is accurate with a
1149 *      WARN_ON.
1150 *
1151 *      LOCKING:
1152 *      Inherited from caller.
1153 */
1154static void mv_start_edma(struct ata_port *ap, void __iomem *port_mmio,
1155                         struct mv_port_priv *pp, u8 protocol)
1156{
1157        int want_ncq = (protocol == ATA_PROT_NCQ);
1158
1159        if (pp->pp_flags & MV_PP_FLAG_EDMA_EN) {
1160                int using_ncq = ((pp->pp_flags & MV_PP_FLAG_NCQ_EN) != 0);
1161                if (want_ncq != using_ncq)
1162                        mv_stop_edma(ap);
1163        }
1164        if (!(pp->pp_flags & MV_PP_FLAG_EDMA_EN)) {
1165                struct mv_host_priv *hpriv = ap->host->private_data;
1166
1167                mv_edma_cfg(ap, want_ncq, 1);
1168
1169                mv_set_edma_ptrs(port_mmio, hpriv, pp);
1170                mv_clear_and_enable_port_irqs(ap, port_mmio, DONE_IRQ|ERR_IRQ);
1171
1172                writelfl(EDMA_EN, port_mmio + EDMA_CMD);
1173                pp->pp_flags |= MV_PP_FLAG_EDMA_EN;
1174        }
1175}
1176
1177static void mv_wait_for_edma_empty_idle(struct ata_port *ap)
1178{
1179        void __iomem *port_mmio = mv_ap_base(ap);
1180        const u32 empty_idle = (EDMA_STATUS_CACHE_EMPTY | EDMA_STATUS_IDLE);
1181        const int per_loop = 5, timeout = (15 * 1000 / per_loop);
1182        int i;
1183
1184        /*
1185         * Wait for the EDMA engine to finish transactions in progress.
1186         * No idea what a good "timeout" value might be, but measurements
1187         * indicate that it often requires hundreds of microseconds
1188         * with two drives in-use.  So we use the 15msec value above
1189         * as a rough guess at what even more drives might require.
1190         */
1191        for (i = 0; i < timeout; ++i) {
1192                u32 edma_stat = readl(port_mmio + EDMA_STATUS);
1193                if ((edma_stat & empty_idle) == empty_idle)
1194                        break;
1195                udelay(per_loop);
1196        }
1197        /* ata_port_info(ap, "%s: %u+ usecs\n", __func__, i); */
1198}
1199
1200/**
1201 *      mv_stop_edma_engine - Disable eDMA engine
1202 *      @port_mmio: io base address
1203 *
1204 *      LOCKING:
1205 *      Inherited from caller.
1206 */
1207static int mv_stop_edma_engine(void __iomem *port_mmio)
1208{
1209        int i;
1210
1211        /* Disable eDMA.  The disable bit auto clears. */
1212        writelfl(EDMA_DS, port_mmio + EDMA_CMD);
1213
1214        /* Wait for the chip to confirm eDMA is off. */
1215        for (i = 10000; i > 0; i--) {
1216                u32 reg = readl(port_mmio + EDMA_CMD);
1217                if (!(reg & EDMA_EN))
1218                        return 0;
1219                udelay(10);
1220        }
1221        return -EIO;
1222}
1223
1224static int mv_stop_edma(struct ata_port *ap)
1225{
1226        void __iomem *port_mmio = mv_ap_base(ap);
1227        struct mv_port_priv *pp = ap->private_data;
1228        int err = 0;
1229
1230        if (!(pp->pp_flags & MV_PP_FLAG_EDMA_EN))
1231                return 0;
1232        pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN;
1233        mv_wait_for_edma_empty_idle(ap);
1234        if (mv_stop_edma_engine(port_mmio)) {
1235                ata_port_err(ap, "Unable to stop eDMA\n");
1236                err = -EIO;
1237        }
1238        mv_edma_cfg(ap, 0, 0);
1239        return err;
1240}
1241
1242#ifdef ATA_DEBUG
1243static void mv_dump_mem(void __iomem *start, unsigned bytes)
1244{
1245        int b, w;
1246        for (b = 0; b < bytes; ) {
1247                DPRINTK("%p: ", start + b);
1248                for (w = 0; b < bytes && w < 4; w++) {
1249                        printk("%08x ", readl(start + b));
1250                        b += sizeof(u32);
1251                }
1252                printk("\n");
1253        }
1254}
1255#endif
1256#if defined(ATA_DEBUG) || defined(CONFIG_PCI)
1257static void mv_dump_pci_cfg(struct pci_dev *pdev, unsigned bytes)
1258{
1259#ifdef ATA_DEBUG
1260        int b, w;
1261        u32 dw;
1262        for (b = 0; b < bytes; ) {
1263                DPRINTK("%02x: ", b);
1264                for (w = 0; b < bytes && w < 4; w++) {
1265                        (void) pci_read_config_dword(pdev, b, &dw);
1266                        printk("%08x ", dw);
1267                        b += sizeof(u32);
1268                }
1269                printk("\n");
1270        }
1271#endif
1272}
1273#endif
1274static void mv_dump_all_regs(void __iomem *mmio_base, int port,
1275                             struct pci_dev *pdev)
1276{
1277#ifdef ATA_DEBUG
1278        void __iomem *hc_base = mv_hc_base(mmio_base,
1279                                           port >> MV_PORT_HC_SHIFT);
1280        void __iomem *port_base;
1281        int start_port, num_ports, p, start_hc, num_hcs, hc;
1282
1283        if (0 > port) {
1284                start_hc = start_port = 0;
1285                num_ports = 8;          /* shld be benign for 4 port devs */
1286                num_hcs = 2;
1287        } else {
1288                start_hc = port >> MV_PORT_HC_SHIFT;
1289                start_port = port;
1290                num_ports = num_hcs = 1;
1291        }
1292        DPRINTK("All registers for port(s) %u-%u:\n", start_port,
1293                num_ports > 1 ? num_ports - 1 : start_port);
1294
1295        if (NULL != pdev) {
1296                DPRINTK("PCI config space regs:\n");
1297                mv_dump_pci_cfg(pdev, 0x68);
1298        }
1299        DPRINTK("PCI regs:\n");
1300        mv_dump_mem(mmio_base+0xc00, 0x3c);
1301        mv_dump_mem(mmio_base+0xd00, 0x34);
1302        mv_dump_mem(mmio_base+0xf00, 0x4);
1303        mv_dump_mem(mmio_base+0x1d00, 0x6c);
1304        for (hc = start_hc; hc < start_hc + num_hcs; hc++) {
1305                hc_base = mv_hc_base(mmio_base, hc);
1306                DPRINTK("HC regs (HC %i):\n", hc);
1307                mv_dump_mem(hc_base, 0x1c);
1308        }
1309        for (p = start_port; p < start_port + num_ports; p++) {
1310                port_base = mv_port_base(mmio_base, p);
1311                DPRINTK("EDMA regs (port %i):\n", p);
1312                mv_dump_mem(port_base, 0x54);
1313                DPRINTK("SATA regs (port %i):\n", p);
1314                mv_dump_mem(port_base+0x300, 0x60);
1315        }
1316#endif
1317}
1318
1319static unsigned int mv_scr_offset(unsigned int sc_reg_in)
1320{
1321        unsigned int ofs;
1322
1323        switch (sc_reg_in) {
1324        case SCR_STATUS:
1325        case SCR_CONTROL:
1326        case SCR_ERROR:
1327                ofs = SATA_STATUS + (sc_reg_in * sizeof(u32));
1328                break;
1329        case SCR_ACTIVE:
1330                ofs = SATA_ACTIVE;   /* active is not with the others */
1331                break;
1332        default:
1333                ofs = 0xffffffffU;
1334                break;
1335        }
1336        return ofs;
1337}
1338
1339static int mv_scr_read(struct ata_link *link, unsigned int sc_reg_in, u32 *val)
1340{
1341        unsigned int ofs = mv_scr_offset(sc_reg_in);
1342
1343        if (ofs != 0xffffffffU) {
1344                *val = readl(mv_ap_base(link->ap) + ofs);
1345                return 0;
1346        } else
1347                return -EINVAL;
1348}
1349
1350static int mv_scr_write(struct ata_link *link, unsigned int sc_reg_in, u32 val)
1351{
1352        unsigned int ofs = mv_scr_offset(sc_reg_in);
1353
1354        if (ofs != 0xffffffffU) {
1355                void __iomem *addr = mv_ap_base(link->ap) + ofs;
1356                if (sc_reg_in == SCR_CONTROL) {
1357                        /*
1358                         * Workaround for 88SX60x1 FEr SATA#26:
1359                         *
1360                         * COMRESETs have to take care not to accidentally
1361                         * put the drive to sleep when writing SCR_CONTROL.
1362                         * Setting bits 12..15 prevents this problem.
1363                         *
1364                         * So if we see an outbound COMMRESET, set those bits.
1365                         * Ditto for the followup write that clears the reset.
1366                         *
1367                         * The proprietary driver does this for
1368                         * all chip versions, and so do we.
1369                         */
1370                        if ((val & 0xf) == 1 || (readl(addr) & 0xf) == 1)
1371                                val |= 0xf000;
1372                }
1373                writelfl(val, addr);
1374                return 0;
1375        } else
1376                return -EINVAL;
1377}
1378
1379static void mv6_dev_config(struct ata_device *adev)
1380{
1381        /*
1382         * Deal with Gen-II ("mv6") hardware quirks/restrictions:
1383         *
1384         * Gen-II does not support NCQ over a port multiplier
1385         *  (no FIS-based switching).
1386         */
1387        if (adev->flags & ATA_DFLAG_NCQ) {
1388                if (sata_pmp_attached(adev->link->ap)) {
1389                        adev->flags &= ~ATA_DFLAG_NCQ;
1390                        ata_dev_info(adev,
1391                                "NCQ disabled for command-based switching\n");
1392                }
1393        }
1394}
1395
1396static int mv_qc_defer(struct ata_queued_cmd *qc)
1397{
1398        struct ata_link *link = qc->dev->link;
1399        struct ata_port *ap = link->ap;
1400        struct mv_port_priv *pp = ap->private_data;
1401
1402        /*
1403         * Don't allow new commands if we're in a delayed EH state
1404         * for NCQ and/or FIS-based switching.
1405         */
1406        if (pp->pp_flags & MV_PP_FLAG_DELAYED_EH)
1407                return ATA_DEFER_PORT;
1408
1409        /* PIO commands need exclusive link: no other commands [DMA or PIO]
1410         * can run concurrently.
1411         * set excl_link when we want to send a PIO command in DMA mode
1412         * or a non-NCQ command in NCQ mode.
1413         * When we receive a command from that link, and there are no
1414         * outstanding commands, mark a flag to clear excl_link and let
1415         * the command go through.
1416         */
1417        if (unlikely(ap->excl_link)) {
1418                if (link == ap->excl_link) {
1419                        if (ap->nr_active_links)
1420                                return ATA_DEFER_PORT;
1421                        qc->flags |= ATA_QCFLAG_CLEAR_EXCL;
1422                        return 0;
1423                } else
1424                        return ATA_DEFER_PORT;
1425        }
1426
1427        /*
1428         * If the port is completely idle, then allow the new qc.
1429         */
1430        if (ap->nr_active_links == 0)
1431                return 0;
1432
1433        /*
1434         * The port is operating in host queuing mode (EDMA) with NCQ
1435         * enabled, allow multiple NCQ commands.  EDMA also allows
1436         * queueing multiple DMA commands but libata core currently
1437         * doesn't allow it.
1438         */
1439        if ((pp->pp_flags & MV_PP_FLAG_EDMA_EN) &&
1440            (pp->pp_flags & MV_PP_FLAG_NCQ_EN)) {
1441                if (ata_is_ncq(qc->tf.protocol))
1442                        return 0;
1443                else {
1444                        ap->excl_link = link;
1445                        return ATA_DEFER_PORT;
1446                }
1447        }
1448
1449        return ATA_DEFER_PORT;
1450}
1451
1452static void mv_config_fbs(struct ata_port *ap, int want_ncq, int want_fbs)
1453{
1454        struct mv_port_priv *pp = ap->private_data;
1455        void __iomem *port_mmio;
1456
1457        u32 fiscfg,   *old_fiscfg   = &pp->cached.fiscfg;
1458        u32 ltmode,   *old_ltmode   = &pp->cached.ltmode;
1459        u32 haltcond, *old_haltcond = &pp->cached.haltcond;
1460
1461        ltmode   = *old_ltmode & ~LTMODE_BIT8;
1462        haltcond = *old_haltcond | EDMA_ERR_DEV;
1463
1464        if (want_fbs) {
1465                fiscfg = *old_fiscfg | FISCFG_SINGLE_SYNC;
1466                ltmode = *old_ltmode | LTMODE_BIT8;
1467                if (want_ncq)
1468                        haltcond &= ~EDMA_ERR_DEV;
1469                else
1470                        fiscfg |=  FISCFG_WAIT_DEV_ERR;
1471        } else {
1472                fiscfg = *old_fiscfg & ~(FISCFG_SINGLE_SYNC | FISCFG_WAIT_DEV_ERR);
1473        }
1474
1475        port_mmio = mv_ap_base(ap);
1476        mv_write_cached_reg(port_mmio + FISCFG, old_fiscfg, fiscfg);
1477        mv_write_cached_reg(port_mmio + LTMODE, old_ltmode, ltmode);
1478        mv_write_cached_reg(port_mmio + EDMA_HALTCOND, old_haltcond, haltcond);
1479}
1480
1481static void mv_60x1_errata_sata25(struct ata_port *ap, int want_ncq)
1482{
1483        struct mv_host_priv *hpriv = ap->host->private_data;
1484        u32 old, new;
1485
1486        /* workaround for 88SX60x1 FEr SATA#25 (part 1) */
1487        old = readl(hpriv->base + GPIO_PORT_CTL);
1488        if (want_ncq)
1489                new = old | (1 << 22);
1490        else
1491                new = old & ~(1 << 22);
1492        if (new != old)
1493                writel(new, hpriv->base + GPIO_PORT_CTL);
1494}
1495
1496/**
1497 *      mv_bmdma_enable - set a magic bit on GEN_IIE to allow bmdma
1498 *      @ap: Port being initialized
1499 *
1500 *      There are two DMA modes on these chips:  basic DMA, and EDMA.
1501 *
1502 *      Bit-0 of the "EDMA RESERVED" register enables/disables use
1503 *      of basic DMA on the GEN_IIE versions of the chips.
1504 *
1505 *      This bit survives EDMA resets, and must be set for basic DMA
1506 *      to function, and should be cleared when EDMA is active.
1507 */
1508static void mv_bmdma_enable_iie(struct ata_port *ap, int enable_bmdma)
1509{
1510        struct mv_port_priv *pp = ap->private_data;
1511        u32 new, *old = &pp->cached.unknown_rsvd;
1512
1513        if (enable_bmdma)
1514                new = *old | 1;
1515        else
1516                new = *old & ~1;
1517        mv_write_cached_reg(mv_ap_base(ap) + EDMA_UNKNOWN_RSVD, old, new);
1518}
1519
1520/*
1521 * SOC chips have an issue whereby the HDD LEDs don't always blink
1522 * during I/O when NCQ is enabled. Enabling a special "LED blink" mode
1523 * of the SOC takes care of it, generating a steady blink rate when
1524 * any drive on the chip is active.
1525 *
1526 * Unfortunately, the blink mode is a global hardware setting for the SOC,
1527 * so we must use it whenever at least one port on the SOC has NCQ enabled.
1528 *
1529 * We turn "LED blink" off when NCQ is not in use anywhere, because the normal
1530 * LED operation works then, and provides better (more accurate) feedback.
1531 *
1532 * Note that this code assumes that an SOC never has more than one HC onboard.
1533 */
1534static void mv_soc_led_blink_enable(struct ata_port *ap)
1535{
1536        struct ata_host *host = ap->host;
1537        struct mv_host_priv *hpriv = host->private_data;
1538        void __iomem *hc_mmio;
1539        u32 led_ctrl;
1540
1541        if (hpriv->hp_flags & MV_HP_QUIRK_LED_BLINK_EN)
1542                return;
1543        hpriv->hp_flags |= MV_HP_QUIRK_LED_BLINK_EN;
1544        hc_mmio = mv_hc_base_from_port(mv_host_base(host), ap->port_no);
1545        led_ctrl = readl(hc_mmio + SOC_LED_CTRL);
1546        writel(led_ctrl | SOC_LED_CTRL_BLINK, hc_mmio + SOC_LED_CTRL);
1547}
1548
1549static void mv_soc_led_blink_disable(struct ata_port *ap)
1550{
1551        struct ata_host *host = ap->host;
1552        struct mv_host_priv *hpriv = host->private_data;
1553        void __iomem *hc_mmio;
1554        u32 led_ctrl;
1555        unsigned int port;
1556
1557        if (!(hpriv->hp_flags & MV_HP_QUIRK_LED_BLINK_EN))
1558                return;
1559
1560        /* disable led-blink only if no ports are using NCQ */
1561        for (port = 0; port < hpriv->n_ports; port++) {
1562                struct ata_port *this_ap = host->ports[port];
1563                struct mv_port_priv *pp = this_ap->private_data;
1564
1565                if (pp->pp_flags & MV_PP_FLAG_NCQ_EN)
1566                        return;
1567        }
1568
1569        hpriv->hp_flags &= ~MV_HP_QUIRK_LED_BLINK_EN;
1570        hc_mmio = mv_hc_base_from_port(mv_host_base(host), ap->port_no);
1571        led_ctrl = readl(hc_mmio + SOC_LED_CTRL);
1572        writel(led_ctrl & ~SOC_LED_CTRL_BLINK, hc_mmio + SOC_LED_CTRL);
1573}
1574
1575static void mv_edma_cfg(struct ata_port *ap, int want_ncq, int want_edma)
1576{
1577        u32 cfg;
1578        struct mv_port_priv *pp    = ap->private_data;
1579        struct mv_host_priv *hpriv = ap->host->private_data;
1580        void __iomem *port_mmio    = mv_ap_base(ap);
1581
1582        /* set up non-NCQ EDMA configuration */
1583        cfg = EDMA_CFG_Q_DEPTH;         /* always 0x1f for *all* chips */
1584        pp->pp_flags &=
1585          ~(MV_PP_FLAG_FBS_EN | MV_PP_FLAG_NCQ_EN | MV_PP_FLAG_FAKE_ATA_BUSY);
1586
1587        if (IS_GEN_I(hpriv))
1588                cfg |= (1 << 8);        /* enab config burst size mask */
1589
1590        else if (IS_GEN_II(hpriv)) {
1591                cfg |= EDMA_CFG_RD_BRST_EXT | EDMA_CFG_WR_BUFF_LEN;
1592                mv_60x1_errata_sata25(ap, want_ncq);
1593
1594        } else if (IS_GEN_IIE(hpriv)) {
1595                int want_fbs = sata_pmp_attached(ap);
1596                /*
1597                 * Possible future enhancement:
1598                 *
1599                 * The chip can use FBS with non-NCQ, if we allow it,
1600                 * But first we need to have the error handling in place
1601                 * for this mode (datasheet section 7.3.15.4.2.3).
1602                 * So disallow non-NCQ FBS for now.
1603                 */
1604                want_fbs &= want_ncq;
1605
1606                mv_config_fbs(ap, want_ncq, want_fbs);
1607
1608                if (want_fbs) {
1609                        pp->pp_flags |= MV_PP_FLAG_FBS_EN;
1610                        cfg |= EDMA_CFG_EDMA_FBS; /* FIS-based switching */
1611                }
1612
1613                cfg |= (1 << 23);       /* do not mask PM field in rx'd FIS */
1614                if (want_edma) {
1615                        cfg |= (1 << 22); /* enab 4-entry host queue cache */
1616                        if (!IS_SOC(hpriv))
1617                                cfg |= (1 << 18); /* enab early completion */
1618                }
1619                if (hpriv->hp_flags & MV_HP_CUT_THROUGH)
1620                        cfg |= (1 << 17); /* enab cut-thru (dis stor&forwrd) */
1621                mv_bmdma_enable_iie(ap, !want_edma);
1622
1623                if (IS_SOC(hpriv)) {
1624                        if (want_ncq)
1625                                mv_soc_led_blink_enable(ap);
1626                        else
1627                                mv_soc_led_blink_disable(ap);
1628                }
1629        }
1630
1631        if (want_ncq) {
1632                cfg |= EDMA_CFG_NCQ;
1633                pp->pp_flags |=  MV_PP_FLAG_NCQ_EN;
1634        }
1635
1636        writelfl(cfg, port_mmio + EDMA_CFG);
1637}
1638
1639static void mv_port_free_dma_mem(struct ata_port *ap)
1640{
1641        struct mv_host_priv *hpriv = ap->host->private_data;
1642        struct mv_port_priv *pp = ap->private_data;
1643        int tag;
1644
1645        if (pp->crqb) {
1646                dma_pool_free(hpriv->crqb_pool, pp->crqb, pp->crqb_dma);
1647                pp->crqb = NULL;
1648        }
1649        if (pp->crpb) {
1650                dma_pool_free(hpriv->crpb_pool, pp->crpb, pp->crpb_dma);
1651                pp->crpb = NULL;
1652        }
1653        /*
1654         * For GEN_I, there's no NCQ, so we have only a single sg_tbl.
1655         * For later hardware, we have one unique sg_tbl per NCQ tag.
1656         */
1657        for (tag = 0; tag < MV_MAX_Q_DEPTH; ++tag) {
1658                if (pp->sg_tbl[tag]) {
1659                        if (tag == 0 || !IS_GEN_I(hpriv))
1660                                dma_pool_free(hpriv->sg_tbl_pool,
1661                                              pp->sg_tbl[tag],
1662                                              pp->sg_tbl_dma[tag]);
1663                        pp->sg_tbl[tag] = NULL;
1664                }
1665        }
1666}
1667
1668/**
1669 *      mv_port_start - Port specific init/start routine.
1670 *      @ap: ATA channel to manipulate
1671 *
1672 *      Allocate and point to DMA memory, init port private memory,
1673 *      zero indices.
1674 *
1675 *      LOCKING:
1676 *      Inherited from caller.
1677 */
1678static int mv_port_start(struct ata_port *ap)
1679{
1680        struct device *dev = ap->host->dev;
1681        struct mv_host_priv *hpriv = ap->host->private_data;
1682        struct mv_port_priv *pp;
1683        unsigned long flags;
1684        int tag;
1685
1686        pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1687        if (!pp)
1688                return -ENOMEM;
1689        ap->private_data = pp;
1690
1691        pp->crqb = dma_pool_alloc(hpriv->crqb_pool, GFP_KERNEL, &pp->crqb_dma);
1692        if (!pp->crqb)
1693                return -ENOMEM;
1694        memset(pp->crqb, 0, MV_CRQB_Q_SZ);
1695
1696        pp->crpb = dma_pool_alloc(hpriv->crpb_pool, GFP_KERNEL, &pp->crpb_dma);
1697        if (!pp->crpb)
1698                goto out_port_free_dma_mem;
1699        memset(pp->crpb, 0, MV_CRPB_Q_SZ);
1700
1701        /* 6041/6081 Rev. "C0" (and newer) are okay with async notify */
1702        if (hpriv->hp_flags & MV_HP_ERRATA_60X1C0)
1703                ap->flags |= ATA_FLAG_AN;
1704        /*
1705         * For GEN_I, there's no NCQ, so we only allocate a single sg_tbl.
1706         * For later hardware, we need one unique sg_tbl per NCQ tag.
1707         */
1708        for (tag = 0; tag < MV_MAX_Q_DEPTH; ++tag) {
1709                if (tag == 0 || !IS_GEN_I(hpriv)) {
1710                        pp->sg_tbl[tag] = dma_pool_alloc(hpriv->sg_tbl_pool,
1711                                              GFP_KERNEL, &pp->sg_tbl_dma[tag]);
1712                        if (!pp->sg_tbl[tag])
1713                                goto out_port_free_dma_mem;
1714                } else {
1715                        pp->sg_tbl[tag]     = pp->sg_tbl[0];
1716                        pp->sg_tbl_dma[tag] = pp->sg_tbl_dma[0];
1717                }
1718        }
1719
1720        spin_lock_irqsave(ap->lock, flags);
1721        mv_save_cached_regs(ap);
1722        mv_edma_cfg(ap, 0, 0);
1723        spin_unlock_irqrestore(ap->lock, flags);
1724
1725        return 0;
1726
1727out_port_free_dma_mem:
1728        mv_port_free_dma_mem(ap);
1729        return -ENOMEM;
1730}
1731
1732/**
1733 *      mv_port_stop - Port specific cleanup/stop routine.
1734 *      @ap: ATA channel to manipulate
1735 *
1736 *      Stop DMA, cleanup port memory.
1737 *
1738 *      LOCKING:
1739 *      This routine uses the host lock to protect the DMA stop.
1740 */
1741static void mv_port_stop(struct ata_port *ap)
1742{
1743        unsigned long flags;
1744
1745        spin_lock_irqsave(ap->lock, flags);
1746        mv_stop_edma(ap);
1747        mv_enable_port_irqs(ap, 0);
1748        spin_unlock_irqrestore(ap->lock, flags);
1749        mv_port_free_dma_mem(ap);
1750}
1751
1752/**
1753 *      mv_fill_sg - Fill out the Marvell ePRD (scatter gather) entries
1754 *      @qc: queued command whose SG list to source from
1755 *
1756 *      Populate the SG list and mark the last entry.
1757 *
1758 *      LOCKING:
1759 *      Inherited from caller.
1760 */
1761static void mv_fill_sg(struct ata_queued_cmd *qc)
1762{
1763        struct mv_port_priv *pp = qc->ap->private_data;
1764        struct scatterlist *sg;
1765        struct mv_sg *mv_sg, *last_sg = NULL;
1766        unsigned int si;
1767
1768        mv_sg = pp->sg_tbl[qc->tag];
1769        for_each_sg(qc->sg, sg, qc->n_elem, si) {
1770                dma_addr_t addr = sg_dma_address(sg);
1771                u32 sg_len = sg_dma_len(sg);
1772
1773                while (sg_len) {
1774                        u32 offset = addr & 0xffff;
1775                        u32 len = sg_len;
1776
1777                        if (offset + len > 0x10000)
1778                                len = 0x10000 - offset;
1779
1780                        mv_sg->addr = cpu_to_le32(addr & 0xffffffff);
1781                        mv_sg->addr_hi = cpu_to_le32((addr >> 16) >> 16);
1782                        mv_sg->flags_size = cpu_to_le32(len & 0xffff);
1783                        mv_sg->reserved = 0;
1784
1785                        sg_len -= len;
1786                        addr += len;
1787
1788                        last_sg = mv_sg;
1789                        mv_sg++;
1790                }
1791        }
1792
1793        if (likely(last_sg))
1794                last_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
1795        mb(); /* ensure data structure is visible to the chipset */
1796}
1797
1798static void mv_crqb_pack_cmd(__le16 *cmdw, u8 data, u8 addr, unsigned last)
1799{
1800        u16 tmp = data | (addr << CRQB_CMD_ADDR_SHIFT) | CRQB_CMD_CS |
1801                (last ? CRQB_CMD_LAST : 0);
1802        *cmdw = cpu_to_le16(tmp);
1803}
1804
1805/**
1806 *      mv_sff_irq_clear - Clear hardware interrupt after DMA.
1807 *      @ap: Port associated with this ATA transaction.
1808 *
1809 *      We need this only for ATAPI bmdma transactions,
1810 *      as otherwise we experience spurious interrupts
1811 *      after libata-sff handles the bmdma interrupts.
1812 */
1813static void mv_sff_irq_clear(struct ata_port *ap)
1814{
1815        mv_clear_and_enable_port_irqs(ap, mv_ap_base(ap), ERR_IRQ);
1816}
1817
1818/**
1819 *      mv_check_atapi_dma - Filter ATAPI cmds which are unsuitable for DMA.
1820 *      @qc: queued command to check for chipset/DMA compatibility.
1821 *
1822 *      The bmdma engines cannot handle speculative data sizes
1823 *      (bytecount under/over flow).  So only allow DMA for
1824 *      data transfer commands with known data sizes.
1825 *
1826 *      LOCKING:
1827 *      Inherited from caller.
1828 */
1829static int mv_check_atapi_dma(struct ata_queued_cmd *qc)
1830{
1831        struct scsi_cmnd *scmd = qc->scsicmd;
1832
1833        if (scmd) {
1834                switch (scmd->cmnd[0]) {
1835                case READ_6:
1836                case READ_10:
1837                case READ_12:
1838                case WRITE_6:
1839                case WRITE_10:
1840                case WRITE_12:
1841                case GPCMD_READ_CD:
1842                case GPCMD_SEND_DVD_STRUCTURE:
1843                case GPCMD_SEND_CUE_SHEET:
1844                        return 0; /* DMA is safe */
1845                }
1846        }
1847        return -EOPNOTSUPP; /* use PIO instead */
1848}
1849
1850/**
1851 *      mv_bmdma_setup - Set up BMDMA transaction
1852 *      @qc: queued command to prepare DMA for.
1853 *
1854 *      LOCKING:
1855 *      Inherited from caller.
1856 */
1857static void mv_bmdma_setup(struct ata_queued_cmd *qc)
1858{
1859        struct ata_port *ap = qc->ap;
1860        void __iomem *port_mmio = mv_ap_base(ap);
1861        struct mv_port_priv *pp = ap->private_data;
1862
1863        mv_fill_sg(qc);
1864
1865        /* clear all DMA cmd bits */
1866        writel(0, port_mmio + BMDMA_CMD);
1867
1868        /* load PRD table addr. */
1869        writel((pp->sg_tbl_dma[qc->tag] >> 16) >> 16,
1870                port_mmio + BMDMA_PRD_HIGH);
1871        writelfl(pp->sg_tbl_dma[qc->tag],
1872                port_mmio + BMDMA_PRD_LOW);
1873
1874        /* issue r/w command */
1875        ap->ops->sff_exec_command(ap, &qc->tf);
1876}
1877
1878/**
1879 *      mv_bmdma_start - Start a BMDMA transaction
1880 *      @qc: queued command to start DMA on.
1881 *
1882 *      LOCKING:
1883 *      Inherited from caller.
1884 */
1885static void mv_bmdma_start(struct ata_queued_cmd *qc)
1886{
1887        struct ata_port *ap = qc->ap;
1888        void __iomem *port_mmio = mv_ap_base(ap);
1889        unsigned int rw = (qc->tf.flags & ATA_TFLAG_WRITE);
1890        u32 cmd = (rw ? 0 : ATA_DMA_WR) | ATA_DMA_START;
1891
1892        /* start host DMA transaction */
1893        writelfl(cmd, port_mmio + BMDMA_CMD);
1894}
1895
1896/**
1897 *      mv_bmdma_stop - Stop BMDMA transfer
1898 *      @qc: queued command to stop DMA on.
1899 *
1900 *      Clears the ATA_DMA_START flag in the bmdma control register
1901 *
1902 *      LOCKING:
1903 *      Inherited from caller.
1904 */
1905static void mv_bmdma_stop_ap(struct ata_port *ap)
1906{
1907        void __iomem *port_mmio = mv_ap_base(ap);
1908        u32 cmd;
1909
1910        /* clear start/stop bit */
1911        cmd = readl(port_mmio + BMDMA_CMD);
1912        if (cmd & ATA_DMA_START) {
1913                cmd &= ~ATA_DMA_START;
1914                writelfl(cmd, port_mmio + BMDMA_CMD);
1915
1916                /* one-PIO-cycle guaranteed wait, per spec, for HDMA1:0 transition */
1917                ata_sff_dma_pause(ap);
1918        }
1919}
1920
1921static void mv_bmdma_stop(struct ata_queued_cmd *qc)
1922{
1923        mv_bmdma_stop_ap(qc->ap);
1924}
1925
1926/**
1927 *      mv_bmdma_status - Read BMDMA status
1928 *      @ap: port for which to retrieve DMA status.
1929 *
1930 *      Read and return equivalent of the sff BMDMA status register.
1931 *
1932 *      LOCKING:
1933 *      Inherited from caller.
1934 */
1935static u8 mv_bmdma_status(struct ata_port *ap)
1936{
1937        void __iomem *port_mmio = mv_ap_base(ap);
1938        u32 reg, status;
1939
1940        /*
1941         * Other bits are valid only if ATA_DMA_ACTIVE==0,
1942         * and the ATA_DMA_INTR bit doesn't exist.
1943         */
1944        reg = readl(port_mmio + BMDMA_STATUS);
1945        if (reg & ATA_DMA_ACTIVE)
1946                status = ATA_DMA_ACTIVE;
1947        else if (reg & ATA_DMA_ERR)
1948                status = (reg & ATA_DMA_ERR) | ATA_DMA_INTR;
1949        else {
1950                /*
1951                 * Just because DMA_ACTIVE is 0 (DMA completed),
1952                 * this does _not_ mean the device is "done".
1953                 * So we should not yet be signalling ATA_DMA_INTR
1954                 * in some cases.  Eg. DSM/TRIM, and perhaps others.
1955                 */
1956                mv_bmdma_stop_ap(ap);
1957                if (ioread8(ap->ioaddr.altstatus_addr) & ATA_BUSY)
1958                        status = 0;
1959                else
1960                        status = ATA_DMA_INTR;
1961        }
1962        return status;
1963}
1964
1965static void mv_rw_multi_errata_sata24(struct ata_queued_cmd *qc)
1966{
1967        struct ata_taskfile *tf = &qc->tf;
1968        /*
1969         * Workaround for 88SX60x1 FEr SATA#24.
1970         *
1971         * Chip may corrupt WRITEs if multi_count >= 4kB.
1972         * Note that READs are unaffected.
1973         *
1974         * It's not clear if this errata really means "4K bytes",
1975         * or if it always happens for multi_count > 7
1976         * regardless of device sector_size.
1977         *
1978         * So, for safety, any write with multi_count > 7
1979         * gets converted here into a regular PIO write instead:
1980         */
1981        if ((tf->flags & ATA_TFLAG_WRITE) && is_multi_taskfile(tf)) {
1982                if (qc->dev->multi_count > 7) {
1983                        switch (tf->command) {
1984                        case ATA_CMD_WRITE_MULTI:
1985                                tf->command = ATA_CMD_PIO_WRITE;
1986                                break;
1987                        case ATA_CMD_WRITE_MULTI_FUA_EXT:
1988                                tf->flags &= ~ATA_TFLAG_FUA; /* ugh */
1989                                /* fall through */
1990                        case ATA_CMD_WRITE_MULTI_EXT:
1991                                tf->command = ATA_CMD_PIO_WRITE_EXT;
1992                                break;
1993                        }
1994                }
1995        }
1996}
1997
1998/**
1999 *      mv_qc_prep - Host specific command preparation.
2000 *      @qc: queued command to prepare
2001 *
2002 *      This routine simply redirects to the general purpose routine
2003 *      if command is not DMA.  Else, it handles prep of the CRQB
2004 *      (command request block), does some sanity checking, and calls
2005 *      the SG load routine.
2006 *
2007 *      LOCKING:
2008 *      Inherited from caller.
2009 */
2010static void mv_qc_prep(struct ata_queued_cmd *qc)
2011{
2012        struct ata_port *ap = qc->ap;
2013        struct mv_port_priv *pp = ap->private_data;
2014        __le16 *cw;
2015        struct ata_taskfile *tf = &qc->tf;
2016        u16 flags = 0;
2017        unsigned in_index;
2018
2019        switch (tf->protocol) {
2020        case ATA_PROT_DMA:
2021                if (tf->command == ATA_CMD_DSM)
2022                        return;
2023                /* fall-thru */
2024        case ATA_PROT_NCQ:
2025                break;  /* continue below */
2026        case ATA_PROT_PIO:
2027                mv_rw_multi_errata_sata24(qc);
2028                return;
2029        default:
2030                return;
2031        }
2032
2033        /* Fill in command request block
2034         */
2035        if (!(tf->flags & ATA_TFLAG_WRITE))
2036                flags |= CRQB_FLAG_READ;
2037        WARN_ON(MV_MAX_Q_DEPTH <= qc->tag);
2038        flags |= qc->tag << CRQB_TAG_SHIFT;
2039        flags |= (qc->dev->link->pmp & 0xf) << CRQB_PMP_SHIFT;
2040
2041        /* get current queue index from software */
2042        in_index = pp->req_idx;
2043
2044        pp->crqb[in_index].sg_addr =
2045                cpu_to_le32(pp->sg_tbl_dma[qc->tag] & 0xffffffff);
2046        pp->crqb[in_index].sg_addr_hi =
2047                cpu_to_le32((pp->sg_tbl_dma[qc->tag] >> 16) >> 16);
2048        pp->crqb[in_index].ctrl_flags = cpu_to_le16(flags);
2049
2050        cw = &pp->crqb[in_index].ata_cmd[0];
2051
2052        /* Sadly, the CRQB cannot accommodate all registers--there are
2053         * only 11 bytes...so we must pick and choose required
2054         * registers based on the command.  So, we drop feature and
2055         * hob_feature for [RW] DMA commands, but they are needed for
2056         * NCQ.  NCQ will drop hob_nsect, which is not needed there
2057         * (nsect is used only for the tag; feat/hob_feat hold true nsect).
2058         */
2059        switch (tf->command) {
2060        case ATA_CMD_READ:
2061        case ATA_CMD_READ_EXT:
2062        case ATA_CMD_WRITE:
2063        case ATA_CMD_WRITE_EXT:
2064        case ATA_CMD_WRITE_FUA_EXT:
2065                mv_crqb_pack_cmd(cw++, tf->hob_nsect, ATA_REG_NSECT, 0);
2066                break;
2067        case ATA_CMD_FPDMA_READ:
2068        case ATA_CMD_FPDMA_WRITE:
2069                mv_crqb_pack_cmd(cw++, tf->hob_feature, ATA_REG_FEATURE, 0);
2070                mv_crqb_pack_cmd(cw++, tf->feature, ATA_REG_FEATURE, 0);
2071                break;
2072        default:
2073                /* The only other commands EDMA supports in non-queued and
2074                 * non-NCQ mode are: [RW] STREAM DMA and W DMA FUA EXT, none
2075                 * of which are defined/used by Linux.  If we get here, this
2076                 * driver needs work.
2077                 *
2078                 * FIXME: modify libata to give qc_prep a return value and
2079                 * return error here.
2080                 */
2081                BUG_ON(tf->command);
2082                break;
2083        }
2084        mv_crqb_pack_cmd(cw++, tf->nsect, ATA_REG_NSECT, 0);
2085        mv_crqb_pack_cmd(cw++, tf->hob_lbal, ATA_REG_LBAL, 0);
2086        mv_crqb_pack_cmd(cw++, tf->lbal, ATA_REG_LBAL, 0);
2087        mv_crqb_pack_cmd(cw++, tf->hob_lbam, ATA_REG_LBAM, 0);
2088        mv_crqb_pack_cmd(cw++, tf->lbam, ATA_REG_LBAM, 0);
2089        mv_crqb_pack_cmd(cw++, tf->hob_lbah, ATA_REG_LBAH, 0);
2090        mv_crqb_pack_cmd(cw++, tf->lbah, ATA_REG_LBAH, 0);
2091        mv_crqb_pack_cmd(cw++, tf->device, ATA_REG_DEVICE, 0);
2092        mv_crqb_pack_cmd(cw++, tf->command, ATA_REG_CMD, 1);    /* last */
2093
2094        if (!(qc->flags & ATA_QCFLAG_DMAMAP))
2095                return;
2096        mv_fill_sg(qc);
2097}
2098
2099/**
2100 *      mv_qc_prep_iie - Host specific command preparation.
2101 *      @qc: queued command to prepare
2102 *
2103 *      This routine simply redirects to the general purpose routine
2104 *      if command is not DMA.  Else, it handles prep of the CRQB
2105 *      (command request block), does some sanity checking, and calls
2106 *      the SG load routine.
2107 *
2108 *      LOCKING:
2109 *      Inherited from caller.
2110 */
2111static void mv_qc_prep_iie(struct ata_queued_cmd *qc)
2112{
2113        struct ata_port *ap = qc->ap;
2114        struct mv_port_priv *pp = ap->private_data;
2115        struct mv_crqb_iie *crqb;
2116        struct ata_taskfile *tf = &qc->tf;
2117        unsigned in_index;
2118        u32 flags = 0;
2119
2120        if ((tf->protocol != ATA_PROT_DMA) &&
2121            (tf->protocol != ATA_PROT_NCQ))
2122                return;
2123        if (tf->command == ATA_CMD_DSM)
2124                return;  /* use bmdma for this */
2125
2126        /* Fill in Gen IIE command request block */
2127        if (!(tf->flags & ATA_TFLAG_WRITE))
2128                flags |= CRQB_FLAG_READ;
2129
2130        WARN_ON(MV_MAX_Q_DEPTH <= qc->tag);
2131        flags |= qc->tag << CRQB_TAG_SHIFT;
2132        flags |= qc->tag << CRQB_HOSTQ_SHIFT;
2133        flags |= (qc->dev->link->pmp & 0xf) << CRQB_PMP_SHIFT;
2134
2135        /* get current queue index from software */
2136        in_index = pp->req_idx;
2137
2138        crqb = (struct mv_crqb_iie *) &pp->crqb[in_index];
2139        crqb->addr = cpu_to_le32(pp->sg_tbl_dma[qc->tag] & 0xffffffff);
2140        crqb->addr_hi = cpu_to_le32((pp->sg_tbl_dma[qc->tag] >> 16) >> 16);
2141        crqb->flags = cpu_to_le32(flags);
2142
2143        crqb->ata_cmd[0] = cpu_to_le32(
2144                        (tf->command << 16) |
2145                        (tf->feature << 24)
2146                );
2147        crqb->ata_cmd[1] = cpu_to_le32(
2148                        (tf->lbal << 0) |
2149                        (tf->lbam << 8) |
2150                        (tf->lbah << 16) |
2151                        (tf->device << 24)
2152                );
2153        crqb->ata_cmd[2] = cpu_to_le32(
2154                        (tf->hob_lbal << 0) |
2155                        (tf->hob_lbam << 8) |
2156                        (tf->hob_lbah << 16) |
2157                        (tf->hob_feature << 24)
2158                );
2159        crqb->ata_cmd[3] = cpu_to_le32(
2160                        (tf->nsect << 0) |
2161                        (tf->hob_nsect << 8)
2162                );
2163
2164        if (!(qc->flags & ATA_QCFLAG_DMAMAP))
2165                return;
2166        mv_fill_sg(qc);
2167}
2168
2169/**
2170 *      mv_sff_check_status - fetch device status, if valid
2171 *      @ap: ATA port to fetch status from
2172 *
2173 *      When using command issue via mv_qc_issue_fis(),
2174 *      the initial ATA_BUSY state does not show up in the
2175 *      ATA status (shadow) register.  This can confuse libata!
2176 *
2177 *      So we have a hook here to fake ATA_BUSY for that situation,
2178 *      until the first time a BUSY, DRQ, or ERR bit is seen.
2179 *
2180 *      The rest of the time, it simply returns the ATA status register.
2181 */
2182static u8 mv_sff_check_status(struct ata_port *ap)
2183{
2184        u8 stat = ioread8(ap->ioaddr.status_addr);
2185        struct mv_port_priv *pp = ap->private_data;
2186
2187        if (pp->pp_flags & MV_PP_FLAG_FAKE_ATA_BUSY) {
2188                if (stat & (ATA_BUSY | ATA_DRQ | ATA_ERR))
2189                        pp->pp_flags &= ~MV_PP_FLAG_FAKE_ATA_BUSY;
2190                else
2191                        stat = ATA_BUSY;
2192        }
2193        return stat;
2194}
2195
2196/**
2197 *      mv_send_fis - Send a FIS, using the "Vendor-Unique FIS" register
2198 *      @fis: fis to be sent
2199 *      @nwords: number of 32-bit words in the fis
2200 */
2201static unsigned int mv_send_fis(struct ata_port *ap, u32 *fis, int nwords)
2202{
2203        void __iomem *port_mmio = mv_ap_base(ap);
2204        u32 ifctl, old_ifctl, ifstat;
2205        int i, timeout = 200, final_word = nwords - 1;
2206
2207        /* Initiate FIS transmission mode */
2208        old_ifctl = readl(port_mmio + SATA_IFCTL);
2209        ifctl = 0x100 | (old_ifctl & 0xf);
2210        writelfl(ifctl, port_mmio + SATA_IFCTL);
2211
2212        /* Send all words of the FIS except for the final word */
2213        for (i = 0; i < final_word; ++i)
2214                writel(fis[i], port_mmio + VENDOR_UNIQUE_FIS);
2215
2216        /* Flag end-of-transmission, and then send the final word */
2217        writelfl(ifctl | 0x200, port_mmio + SATA_IFCTL);
2218        writelfl(fis[final_word], port_mmio + VENDOR_UNIQUE_FIS);
2219
2220        /*
2221         * Wait for FIS transmission to complete.
2222         * This typically takes just a single iteration.
2223         */
2224        do {
2225                ifstat = readl(port_mmio + SATA_IFSTAT);
2226        } while (!(ifstat & 0x1000) && --timeout);
2227
2228        /* Restore original port configuration */
2229        writelfl(old_ifctl, port_mmio + SATA_IFCTL);
2230
2231        /* See if it worked */
2232        if ((ifstat & 0x3000) != 0x1000) {
2233                ata_port_warn(ap, "%s transmission error, ifstat=%08x\n",
2234                              __func__, ifstat);
2235                return AC_ERR_OTHER;
2236        }
2237        return 0;
2238}
2239
2240/**
2241 *      mv_qc_issue_fis - Issue a command directly as a FIS
2242 *      @qc: queued command to start
2243 *
2244 *      Note that the ATA shadow registers are not updated
2245 *      after command issue, so the device will appear "READY"
2246 *      if polled, even while it is BUSY processing the command.
2247 *
2248 *      So we use a status hook to fake ATA_BUSY until the drive changes state.
2249 *
2250 *      Note: we don't get updated shadow regs on *completion*
2251 *      of non-data commands. So avoid sending them via this function,
2252 *      as they will appear to have completed immediately.
2253 *
2254 *      GEN_IIE has special registers that we could get the result tf from,
2255 *      but earlier chipsets do not.  For now, we ignore those registers.
2256 */
2257static unsigned int mv_qc_issue_fis(struct ata_queued_cmd *qc)
2258{
2259        struct ata_port *ap = qc->ap;
2260        struct mv_port_priv *pp = ap->private_data;
2261        struct ata_link *link = qc->dev->link;
2262        u32 fis[5];
2263        int err = 0;
2264
2265        ata_tf_to_fis(&qc->tf, link->pmp, 1, (void *)fis);
2266        err = mv_send_fis(ap, fis, ARRAY_SIZE(fis));
2267        if (err)
2268                return err;
2269
2270        switch (qc->tf.protocol) {
2271        case ATAPI_PROT_PIO:
2272                pp->pp_flags |= MV_PP_FLAG_FAKE_ATA_BUSY;
2273                /* fall through */
2274        case ATAPI_PROT_NODATA:
2275                ap->hsm_task_state = HSM_ST_FIRST;
2276                break;
2277        case ATA_PROT_PIO:
2278                pp->pp_flags |= MV_PP_FLAG_FAKE_ATA_BUSY;
2279                if (qc->tf.flags & ATA_TFLAG_WRITE)
2280                        ap->hsm_task_state = HSM_ST_FIRST;
2281                else
2282                        ap->hsm_task_state = HSM_ST;
2283                break;
2284        default:
2285                ap->hsm_task_state = HSM_ST_LAST;
2286                break;
2287        }
2288
2289        if (qc->tf.flags & ATA_TFLAG_POLLING)
2290                ata_sff_queue_pio_task(link, 0);
2291        return 0;
2292}
2293
2294/**
2295 *      mv_qc_issue - Initiate a command to the host
2296 *      @qc: queued command to start
2297 *
2298 *      This routine simply redirects to the general purpose routine
2299 *      if command is not DMA.  Else, it sanity checks our local
2300 *      caches of the request producer/consumer indices then enables
2301 *      DMA and bumps the request producer index.
2302 *
2303 *      LOCKING:
2304 *      Inherited from caller.
2305 */
2306static unsigned int mv_qc_issue(struct ata_queued_cmd *qc)
2307{
2308        static int limit_warnings = 10;
2309        struct ata_port *ap = qc->ap;
2310        void __iomem *port_mmio = mv_ap_base(ap);
2311        struct mv_port_priv *pp = ap->private_data;
2312        u32 in_index;
2313        unsigned int port_irqs;
2314
2315        pp->pp_flags &= ~MV_PP_FLAG_FAKE_ATA_BUSY; /* paranoia */
2316
2317        switch (qc->tf.protocol) {
2318        case ATA_PROT_DMA:
2319                if (qc->tf.command == ATA_CMD_DSM) {
2320                        if (!ap->ops->bmdma_setup)  /* no bmdma on GEN_I */
2321                                return AC_ERR_OTHER;
2322                        break;  /* use bmdma for this */
2323                }
2324                /* fall thru */
2325        case ATA_PROT_NCQ:
2326                mv_start_edma(ap, port_mmio, pp, qc->tf.protocol);
2327                pp->req_idx = (pp->req_idx + 1) & MV_MAX_Q_DEPTH_MASK;
2328                in_index = pp->req_idx << EDMA_REQ_Q_PTR_SHIFT;
2329
2330                /* Write the request in pointer to kick the EDMA to life */
2331                writelfl((pp->crqb_dma & EDMA_REQ_Q_BASE_LO_MASK) | in_index,
2332                                        port_mmio + EDMA_REQ_Q_IN_PTR);
2333                return 0;
2334
2335        case ATA_PROT_PIO:
2336                /*
2337                 * Errata SATA#16, SATA#24: warn if multiple DRQs expected.
2338                 *
2339                 * Someday, we might implement special polling workarounds
2340                 * for these, but it all seems rather unnecessary since we
2341                 * normally use only DMA for commands which transfer more
2342                 * than a single block of data.
2343                 *
2344                 * Much of the time, this could just work regardless.
2345                 * So for now, just log the incident, and allow the attempt.
2346                 */
2347                if (limit_warnings > 0 && (qc->nbytes / qc->sect_size) > 1) {
2348                        --limit_warnings;
2349                        ata_link_warn(qc->dev->link, DRV_NAME
2350                                      ": attempting PIO w/multiple DRQ: "
2351                                      "this may fail due to h/w errata\n");
2352                }
2353                /* drop through */
2354        case ATA_PROT_NODATA:
2355        case ATAPI_PROT_PIO:
2356        case ATAPI_PROT_NODATA:
2357                if (ap->flags & ATA_FLAG_PIO_POLLING)
2358                        qc->tf.flags |= ATA_TFLAG_POLLING;
2359                break;
2360        }
2361
2362        if (qc->tf.flags & ATA_TFLAG_POLLING)
2363                port_irqs = ERR_IRQ;    /* mask device interrupt when polling */
2364        else
2365                port_irqs = ERR_IRQ | DONE_IRQ; /* unmask all interrupts */
2366
2367        /*
2368         * We're about to send a non-EDMA capable command to the
2369         * port.  Turn off EDMA so there won't be problems accessing
2370         * shadow block, etc registers.
2371         */
2372        mv_stop_edma(ap);
2373        mv_clear_and_enable_port_irqs(ap, mv_ap_base(ap), port_irqs);
2374        mv_pmp_select(ap, qc->dev->link->pmp);
2375
2376        if (qc->tf.command == ATA_CMD_READ_LOG_EXT) {
2377                struct mv_host_priv *hpriv = ap->host->private_data;
2378                /*
2379                 * Workaround for 88SX60x1 FEr SATA#25 (part 2).
2380                 *
2381                 * After any NCQ error, the READ_LOG_EXT command
2382                 * from libata-eh *must* use mv_qc_issue_fis().
2383                 * Otherwise it might fail, due to chip errata.
2384                 *
2385                 * Rather than special-case it, we'll just *always*
2386                 * use this method here for READ_LOG_EXT, making for
2387                 * easier testing.
2388                 */
2389                if (IS_GEN_II(hpriv))
2390                        return mv_qc_issue_fis(qc);
2391        }
2392        return ata_bmdma_qc_issue(qc);
2393}
2394
2395static struct ata_queued_cmd *mv_get_active_qc(struct ata_port *ap)
2396{
2397        struct mv_port_priv *pp = ap->private_data;
2398        struct ata_queued_cmd *qc;
2399
2400        if (pp->pp_flags & MV_PP_FLAG_NCQ_EN)
2401                return NULL;
2402        qc = ata_qc_from_tag(ap, ap->link.active_tag);
2403        if (qc && !(qc->tf.flags & ATA_TFLAG_POLLING))
2404                return qc;
2405        return NULL;
2406}
2407
2408static void mv_pmp_error_handler(struct ata_port *ap)
2409{
2410        unsigned int pmp, pmp_map;
2411        struct mv_port_priv *pp = ap->private_data;
2412
2413        if (pp->pp_flags & MV_PP_FLAG_DELAYED_EH) {
2414                /*
2415                 * Perform NCQ error analysis on failed PMPs
2416                 * before we freeze the port entirely.
2417                 *
2418                 * The failed PMPs are marked earlier by mv_pmp_eh_prep().
2419                 */
2420                pmp_map = pp->delayed_eh_pmp_map;
2421                pp->pp_flags &= ~MV_PP_FLAG_DELAYED_EH;
2422                for (pmp = 0; pmp_map != 0; pmp++) {
2423                        unsigned int this_pmp = (1 << pmp);
2424                        if (pmp_map & this_pmp) {
2425                                struct ata_link *link = &ap->pmp_link[pmp];
2426                                pmp_map &= ~this_pmp;
2427                                ata_eh_analyze_ncq_error(link);
2428                        }
2429                }
2430                ata_port_freeze(ap);
2431        }
2432        sata_pmp_error_handler(ap);
2433}
2434
2435static unsigned int mv_get_err_pmp_map(struct ata_port *ap)
2436{
2437        void __iomem *port_mmio = mv_ap_base(ap);
2438
2439        return readl(port_mmio + SATA_TESTCTL) >> 16;
2440}
2441
2442static void mv_pmp_eh_prep(struct ata_port *ap, unsigned int pmp_map)
2443{
2444        struct ata_eh_info *ehi;
2445        unsigned int pmp;
2446
2447        /*
2448         * Initialize EH info for PMPs which saw device errors
2449         */
2450        ehi = &ap->link.eh_info;
2451        for (pmp = 0; pmp_map != 0; pmp++) {
2452                unsigned int this_pmp = (1 << pmp);
2453                if (pmp_map & this_pmp) {
2454                        struct ata_link *link = &ap->pmp_link[pmp];
2455
2456                        pmp_map &= ~this_pmp;
2457                        ehi = &link->eh_info;
2458                        ata_ehi_clear_desc(ehi);
2459                        ata_ehi_push_desc(ehi, "dev err");
2460                        ehi->err_mask |= AC_ERR_DEV;
2461                        ehi->action |= ATA_EH_RESET;
2462                        ata_link_abort(link);
2463                }
2464        }
2465}
2466
2467static int mv_req_q_empty(struct ata_port *ap)
2468{
2469        void __iomem *port_mmio = mv_ap_base(ap);
2470        u32 in_ptr, out_ptr;
2471
2472        in_ptr  = (readl(port_mmio + EDMA_REQ_Q_IN_PTR)
2473                        >> EDMA_REQ_Q_PTR_SHIFT) & MV_MAX_Q_DEPTH_MASK;
2474        out_ptr = (readl(port_mmio + EDMA_REQ_Q_OUT_PTR)
2475                        >> EDMA_REQ_Q_PTR_SHIFT) & MV_MAX_Q_DEPTH_MASK;
2476        return (in_ptr == out_ptr);     /* 1 == queue_is_empty */
2477}
2478
2479static int mv_handle_fbs_ncq_dev_err(struct ata_port *ap)
2480{
2481        struct mv_port_priv *pp = ap->private_data;
2482        int failed_links;
2483        unsigned int old_map, new_map;
2484
2485        /*
2486         * Device error during FBS+NCQ operation:
2487         *
2488         * Set a port flag to prevent further I/O being enqueued.
2489         * Leave the EDMA running to drain outstanding commands from this port.
2490         * Perform the post-mortem/EH only when all responses are complete.
2491         * Follow recovery sequence from 6042/7042 datasheet (7.3.15.4.2.2).
2492         */
2493        if (!(pp->pp_flags & MV_PP_FLAG_DELAYED_EH)) {
2494                pp->pp_flags |= MV_PP_FLAG_DELAYED_EH;
2495                pp->delayed_eh_pmp_map = 0;
2496        }
2497        old_map = pp->delayed_eh_pmp_map;
2498        new_map = old_map | mv_get_err_pmp_map(ap);
2499
2500        if (old_map != new_map) {
2501                pp->delayed_eh_pmp_map = new_map;
2502                mv_pmp_eh_prep(ap, new_map & ~old_map);
2503        }
2504        failed_links = hweight16(new_map);
2505
2506        ata_port_info(ap,
2507                      "%s: pmp_map=%04x qc_map=%04x failed_links=%d nr_active_links=%d\n",
2508                      __func__, pp->delayed_eh_pmp_map,
2509                      ap->qc_active, failed_links,
2510                      ap->nr_active_links);
2511
2512        if (ap->nr_active_links <= failed_links && mv_req_q_empty(ap)) {
2513                mv_process_crpb_entries(ap, pp);
2514                mv_stop_edma(ap);
2515                mv_eh_freeze(ap);
2516                ata_port_info(ap, "%s: done\n", __func__);
2517                return 1;       /* handled */
2518        }
2519        ata_port_info(ap, "%s: waiting\n", __func__);
2520        return 1;       /* handled */
2521}
2522
2523static int mv_handle_fbs_non_ncq_dev_err(struct ata_port *ap)
2524{
2525        /*
2526         * Possible future enhancement:
2527         *
2528         * FBS+non-NCQ operation is not yet implemented.
2529         * See related notes in mv_edma_cfg().
2530         *
2531         * Device error during FBS+non-NCQ operation:
2532         *
2533         * We need to snapshot the shadow registers for each failed command.
2534         * Follow recovery sequence from 6042/7042 datasheet (7.3.15.4.2.3).
2535         */
2536        return 0;       /* not handled */
2537}
2538
2539static int mv_handle_dev_err(struct ata_port *ap, u32 edma_err_cause)
2540{
2541        struct mv_port_priv *pp = ap->private_data;
2542
2543        if (!(pp->pp_flags & MV_PP_FLAG_EDMA_EN))
2544                return 0;       /* EDMA was not active: not handled */
2545        if (!(pp->pp_flags & MV_PP_FLAG_FBS_EN))
2546                return 0;       /* FBS was not active: not handled */
2547
2548        if (!(edma_err_cause & EDMA_ERR_DEV))
2549                return 0;       /* non DEV error: not handled */
2550        edma_err_cause &= ~EDMA_ERR_IRQ_TRANSIENT;
2551        if (edma_err_cause & ~(EDMA_ERR_DEV | EDMA_ERR_SELF_DIS))
2552                return 0;       /* other problems: not handled */
2553
2554        if (pp->pp_flags & MV_PP_FLAG_NCQ_EN) {
2555                /*
2556                 * EDMA should NOT have self-disabled for this case.
2557                 * If it did, then something is wrong elsewhere,
2558                 * and we cannot handle it here.
2559                 */
2560                if (edma_err_cause & EDMA_ERR_SELF_DIS) {
2561                        ata_port_warn(ap, "%s: err_cause=0x%x pp_flags=0x%x\n",
2562                                      __func__, edma_err_cause, pp->pp_flags);
2563                        return 0; /* not handled */
2564                }
2565                return mv_handle_fbs_ncq_dev_err(ap);
2566        } else {
2567                /*
2568                 * EDMA should have self-disabled for this case.
2569                 * If it did not, then something is wrong elsewhere,
2570                 * and we cannot handle it here.
2571                 */
2572                if (!(edma_err_cause & EDMA_ERR_SELF_DIS)) {
2573                        ata_port_warn(ap, "%s: err_cause=0x%x pp_flags=0x%x\n",
2574                                      __func__, edma_err_cause, pp->pp_flags);
2575                        return 0; /* not handled */
2576                }
2577                return mv_handle_fbs_non_ncq_dev_err(ap);
2578        }
2579        return 0;       /* not handled */
2580}
2581
2582static void mv_unexpected_intr(struct ata_port *ap, int edma_was_enabled)
2583{
2584        struct ata_eh_info *ehi = &ap->link.eh_info;
2585        char *when = "idle";
2586
2587        ata_ehi_clear_desc(ehi);
2588        if (edma_was_enabled) {
2589                when = "EDMA enabled";
2590        } else {
2591                struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
2592                if (qc && (qc->tf.flags & ATA_TFLAG_POLLING))
2593                        when = "polling";
2594        }
2595        ata_ehi_push_desc(ehi, "unexpected device interrupt while %s", when);
2596        ehi->err_mask |= AC_ERR_OTHER;
2597        ehi->action   |= ATA_EH_RESET;
2598        ata_port_freeze(ap);
2599}
2600
2601/**
2602 *      mv_err_intr - Handle error interrupts on the port
2603 *      @ap: ATA channel to manipulate
2604 *
2605 *      Most cases require a full reset of the chip's state machine,
2606 *      which also performs a COMRESET.
2607 *      Also, if the port disabled DMA, update our cached copy to match.
2608 *
2609 *      LOCKING:
2610 *      Inherited from caller.
2611 */
2612static void mv_err_intr(struct ata_port *ap)
2613{
2614        void __iomem *port_mmio = mv_ap_base(ap);
2615        u32 edma_err_cause, eh_freeze_mask, serr = 0;
2616        u32 fis_cause = 0;
2617        struct mv_port_priv *pp = ap->private_data;
2618        struct mv_host_priv *hpriv = ap->host->private_data;
2619        unsigned int action = 0, err_mask = 0;
2620        struct ata_eh_info *ehi = &ap->link.eh_info;
2621        struct ata_queued_cmd *qc;
2622        int abort = 0;
2623
2624        /*
2625         * Read and clear the SError and err_cause bits.
2626         * For GenIIe, if EDMA_ERR_TRANS_IRQ_7 is set, we also must read/clear
2627         * the FIS_IRQ_CAUSE register before clearing edma_err_cause.
2628         */
2629        sata_scr_read(&ap->link, SCR_ERROR, &serr);
2630        sata_scr_write_flush(&ap->link, SCR_ERROR, serr);
2631
2632        edma_err_cause = readl(port_mmio + EDMA_ERR_IRQ_CAUSE);
2633        if (IS_GEN_IIE(hpriv) && (edma_err_cause & EDMA_ERR_TRANS_IRQ_7)) {
2634                fis_cause = readl(port_mmio + FIS_IRQ_CAUSE);
2635                writelfl(~fis_cause, port_mmio + FIS_IRQ_CAUSE);
2636        }
2637        writelfl(~edma_err_cause, port_mmio + EDMA_ERR_IRQ_CAUSE);
2638
2639        if (edma_err_cause & EDMA_ERR_DEV) {
2640                /*
2641                 * Device errors during FIS-based switching operation
2642                 * require special handling.
2643                 */
2644                if (mv_handle_dev_err(ap, edma_err_cause))
2645                        return;
2646        }
2647
2648        qc = mv_get_active_qc(ap);
2649        ata_ehi_clear_desc(ehi);
2650        ata_ehi_push_desc(ehi, "edma_err_cause=%08x pp_flags=%08x",
2651                          edma_err_cause, pp->pp_flags);
2652
2653        if (IS_GEN_IIE(hpriv) && (edma_err_cause & EDMA_ERR_TRANS_IRQ_7)) {
2654                ata_ehi_push_desc(ehi, "fis_cause=%08x", fis_cause);
2655                if (fis_cause & FIS_IRQ_CAUSE_AN) {
2656                        u32 ec = edma_err_cause &
2657                               ~(EDMA_ERR_TRANS_IRQ_7 | EDMA_ERR_IRQ_TRANSIENT);
2658                        sata_async_notification(ap);
2659                        if (!ec)
2660                                return; /* Just an AN; no need for the nukes */
2661                        ata_ehi_push_desc(ehi, "SDB notify");
2662                }
2663        }
2664        /*
2665         * All generations share these EDMA error cause bits:
2666         */
2667        if (edma_err_cause & EDMA_ERR_DEV) {
2668                err_mask |= AC_ERR_DEV;
2669                action |= ATA_EH_RESET;
2670                ata_ehi_push_desc(ehi, "dev error");
2671        }
2672        if (edma_err_cause & (EDMA_ERR_D_PAR | EDMA_ERR_PRD_PAR |
2673                        EDMA_ERR_CRQB_PAR | EDMA_ERR_CRPB_PAR |
2674                        EDMA_ERR_INTRL_PAR)) {
2675                err_mask |= AC_ERR_ATA_BUS;
2676                action |= ATA_EH_RESET;
2677                ata_ehi_push_desc(ehi, "parity error");
2678        }
2679        if (edma_err_cause & (EDMA_ERR_DEV_DCON | EDMA_ERR_DEV_CON)) {
2680                ata_ehi_hotplugged(ehi);
2681                ata_ehi_push_desc(ehi, edma_err_cause & EDMA_ERR_DEV_DCON ?
2682                        "dev disconnect" : "dev connect");
2683                action |= ATA_EH_RESET;
2684        }
2685
2686        /*
2687         * Gen-I has a different SELF_DIS bit,
2688         * different FREEZE bits, and no SERR bit:
2689         */
2690        if (IS_GEN_I(hpriv)) {
2691                eh_freeze_mask = EDMA_EH_FREEZE_5;
2692                if (edma_err_cause & EDMA_ERR_SELF_DIS_5) {
2693                        pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN;
2694                        ata_ehi_push_desc(ehi, "EDMA self-disable");
2695                }
2696        } else {
2697                eh_freeze_mask = EDMA_EH_FREEZE;
2698                if (edma_err_cause & EDMA_ERR_SELF_DIS) {
2699                        pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN;
2700                        ata_ehi_push_desc(ehi, "EDMA self-disable");
2701                }
2702                if (edma_err_cause & EDMA_ERR_SERR) {
2703                        ata_ehi_push_desc(ehi, "SError=%08x", serr);
2704                        err_mask |= AC_ERR_ATA_BUS;
2705                        action |= ATA_EH_RESET;
2706                }
2707        }
2708
2709        if (!err_mask) {
2710                err_mask = AC_ERR_OTHER;
2711                action |= ATA_EH_RESET;
2712        }
2713
2714        ehi->serror |= serr;
2715        ehi->action |= action;
2716
2717        if (qc)
2718                qc->err_mask |= err_mask;
2719        else
2720                ehi->err_mask |= err_mask;
2721
2722        if (err_mask == AC_ERR_DEV) {
2723                /*
2724                 * Cannot do ata_port_freeze() here,
2725                 * because it would kill PIO access,
2726                 * which is needed for further diagnosis.
2727                 */
2728                mv_eh_freeze(ap);
2729                abort = 1;
2730        } else if (edma_err_cause & eh_freeze_mask) {
2731                /*
2732                 * Note to self: ata_port_freeze() calls ata_port_abort()
2733                 */
2734                ata_port_freeze(ap);
2735        } else {
2736                abort = 1;
2737        }
2738
2739        if (abort) {
2740                if (qc)
2741                        ata_link_abort(qc->dev->link);
2742                else
2743                        ata_port_abort(ap);
2744        }
2745}
2746
2747static bool mv_process_crpb_response(struct ata_port *ap,
2748                struct mv_crpb *response, unsigned int tag, int ncq_enabled)
2749{
2750        u8 ata_status;
2751        u16 edma_status = le16_to_cpu(response->flags);
2752
2753        /*
2754         * edma_status from a response queue entry:
2755         *   LSB is from EDMA_ERR_IRQ_CAUSE (non-NCQ only).
2756         *   MSB is saved ATA status from command completion.
2757         */
2758        if (!ncq_enabled) {
2759                u8 err_cause = edma_status & 0xff & ~EDMA_ERR_DEV;
2760                if (err_cause) {
2761                        /*
2762                         * Error will be seen/handled by
2763                         * mv_err_intr().  So do nothing at all here.
2764                         */
2765                        return false;
2766                }
2767        }
2768        ata_status = edma_status >> CRPB_FLAG_STATUS_SHIFT;
2769        if (!ac_err_mask(ata_status))
2770                return true;
2771        /* else: leave it for mv_err_intr() */
2772        return false;
2773}
2774
2775static void mv_process_crpb_entries(struct ata_port *ap, struct mv_port_priv *pp)
2776{
2777        void __iomem *port_mmio = mv_ap_base(ap);
2778        struct mv_host_priv *hpriv = ap->host->private_data;
2779        u32 in_index;
2780        bool work_done = false;
2781        u32 done_mask = 0;
2782        int ncq_enabled = (pp->pp_flags & MV_PP_FLAG_NCQ_EN);
2783
2784        /* Get the hardware queue position index */
2785        in_index = (readl(port_mmio + EDMA_RSP_Q_IN_PTR)
2786                        >> EDMA_RSP_Q_PTR_SHIFT) & MV_MAX_Q_DEPTH_MASK;
2787
2788        /* Process new responses from since the last time we looked */
2789        while (in_index != pp->resp_idx) {
2790                unsigned int tag;
2791                struct mv_crpb *response = &pp->crpb[pp->resp_idx];
2792
2793                pp->resp_idx = (pp->resp_idx + 1) & MV_MAX_Q_DEPTH_MASK;
2794
2795                if (IS_GEN_I(hpriv)) {
2796                        /* 50xx: no NCQ, only one command active at a time */
2797                        tag = ap->link.active_tag;
2798                } else {
2799                        /* Gen II/IIE: get command tag from CRPB entry */
2800                        tag = le16_to_cpu(response->id) & 0x1f;
2801                }
2802                if (mv_process_crpb_response(ap, response, tag, ncq_enabled))
2803                        done_mask |= 1 << tag;
2804                work_done = true;
2805        }
2806
2807        if (work_done) {
2808                ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
2809
2810                /* Update the software queue position index in hardware */
2811                writelfl((pp->crpb_dma & EDMA_RSP_Q_BASE_LO_MASK) |
2812                         (pp->resp_idx << EDMA_RSP_Q_PTR_SHIFT),
2813                         port_mmio + EDMA_RSP_Q_OUT_PTR);
2814        }
2815}
2816
2817static void mv_port_intr(struct ata_port *ap, u32 port_cause)
2818{
2819        struct mv_port_priv *pp;
2820        int edma_was_enabled;
2821
2822        /*
2823         * Grab a snapshot of the EDMA_EN flag setting,
2824         * so that we have a consistent view for this port,
2825         * even if something we call of our routines changes it.
2826         */
2827        pp = ap->private_data;
2828        edma_was_enabled = (pp->pp_flags & MV_PP_FLAG_EDMA_EN);
2829        /*
2830         * Process completed CRPB response(s) before other events.
2831         */
2832        if (edma_was_enabled && (port_cause & DONE_IRQ)) {
2833                mv_process_crpb_entries(ap, pp);
2834                if (pp->pp_flags & MV_PP_FLAG_DELAYED_EH)
2835                        mv_handle_fbs_ncq_dev_err(ap);
2836        }
2837        /*
2838         * Handle chip-reported errors, or continue on to handle PIO.
2839         */
2840        if (unlikely(port_cause & ERR_IRQ)) {
2841                mv_err_intr(ap);
2842        } else if (!edma_was_enabled) {
2843                struct ata_queued_cmd *qc = mv_get_active_qc(ap);
2844                if (qc)
2845                        ata_bmdma_port_intr(ap, qc);
2846                else
2847                        mv_unexpected_intr(ap, edma_was_enabled);
2848        }
2849}
2850
2851/**
2852 *      mv_host_intr - Handle all interrupts on the given host controller
2853 *      @host: host specific structure
2854 *      @main_irq_cause: Main interrupt cause register for the chip.
2855 *
2856 *      LOCKING:
2857 *      Inherited from caller.
2858 */
2859static int mv_host_intr(struct ata_host *host, u32 main_irq_cause)
2860{
2861        struct mv_host_priv *hpriv = host->private_data;
2862        void __iomem *mmio = hpriv->base, *hc_mmio;
2863        unsigned int handled = 0, port;
2864
2865        /* If asserted, clear the "all ports" IRQ coalescing bit */
2866        if (main_irq_cause & ALL_PORTS_COAL_DONE)
2867                writel(~ALL_PORTS_COAL_IRQ, mmio + IRQ_COAL_CAUSE);
2868
2869        for (port = 0; port < hpriv->n_ports; port++) {
2870                struct ata_port *ap = host->ports[port];
2871                unsigned int p, shift, hardport, port_cause;
2872
2873                MV_PORT_TO_SHIFT_AND_HARDPORT(port, shift, hardport);
2874                /*
2875                 * Each hc within the host has its own hc_irq_cause register,
2876                 * where the interrupting ports bits get ack'd.
2877                 */
2878                if (hardport == 0) {    /* first port on this hc ? */
2879                        u32 hc_cause = (main_irq_cause >> shift) & HC0_IRQ_PEND;
2880                        u32 port_mask, ack_irqs;
2881                        /*
2882                         * Skip this entire hc if nothing pending for any ports
2883                         */
2884                        if (!hc_cause) {
2885                                port += MV_PORTS_PER_HC - 1;
2886                                continue;
2887                        }
2888                        /*
2889                         * We don't need/want to read the hc_irq_cause register,
2890                         * because doing so hurts performance, and
2891                         * main_irq_cause already gives us everything we need.
2892                         *
2893                         * But we do have to *write* to the hc_irq_cause to ack
2894                         * the ports that we are handling this time through.
2895                         *
2896                         * This requires that we create a bitmap for those
2897                         * ports which interrupted us, and use that bitmap
2898                         * to ack (only) those ports via hc_irq_cause.
2899                         */
2900                        ack_irqs = 0;
2901                        if (hc_cause & PORTS_0_3_COAL_DONE)
2902                                ack_irqs = HC_COAL_IRQ;
2903                        for (p = 0; p < MV_PORTS_PER_HC; ++p) {
2904                                if ((port + p) >= hpriv->n_ports)
2905                                        break;
2906                                port_mask = (DONE_IRQ | ERR_IRQ) << (p * 2);
2907                                if (hc_cause & port_mask)
2908                                        ack_irqs |= (DMA_IRQ | DEV_IRQ) << p;
2909                        }
2910                        hc_mmio = mv_hc_base_from_port(mmio, port);
2911                        writelfl(~ack_irqs, hc_mmio + HC_IRQ_CAUSE);
2912                        handled = 1;
2913                }
2914                /*
2915                 * Handle interrupts signalled for this port:
2916                 */
2917                port_cause = (main_irq_cause >> shift) & (DONE_IRQ | ERR_IRQ);
2918                if (port_cause)
2919                        mv_port_intr(ap, port_cause);
2920        }
2921        return handled;
2922}
2923
2924static int mv_pci_error(struct ata_host *host, void __iomem *mmio)
2925{
2926        struct mv_host_priv *hpriv = host->private_data;
2927        struct ata_port *ap;
2928        struct ata_queued_cmd *qc;
2929        struct ata_eh_info *ehi;
2930        unsigned int i, err_mask, printed = 0;
2931        u32 err_cause;
2932
2933        err_cause = readl(mmio + hpriv->irq_cause_offset);
2934
2935        dev_err(host->dev, "PCI ERROR; PCI IRQ cause=0x%08x\n", err_cause);
2936
2937        DPRINTK("All regs @ PCI error\n");
2938        mv_dump_all_regs(mmio, -1, to_pci_dev(host->dev));
2939
2940        writelfl(0, mmio + hpriv->irq_cause_offset);
2941
2942        for (i = 0; i < host->n_ports; i++) {
2943                ap = host->ports[i];
2944                if (!ata_link_offline(&ap->link)) {
2945                        ehi = &ap->link.eh_info;
2946                        ata_ehi_clear_desc(ehi);
2947                        if (!printed++)
2948                                ata_ehi_push_desc(ehi,
2949                                        "PCI err cause 0x%08x", err_cause);
2950                        err_mask = AC_ERR_HOST_BUS;
2951                        ehi->action = ATA_EH_RESET;
2952                        qc = ata_qc_from_tag(ap, ap->link.active_tag);
2953                        if (qc)
2954                                qc->err_mask |= err_mask;
2955                        else
2956                                ehi->err_mask |= err_mask;
2957
2958                        ata_port_freeze(ap);
2959                }
2960        }
2961        return 1;       /* handled */
2962}
2963
2964/**
2965 *      mv_interrupt - Main interrupt event handler
2966 *      @irq: unused
2967 *      @dev_instance: private data; in this case the host structure
2968 *
2969 *      Read the read only register to determine if any host
2970 *      controllers have pending interrupts.  If so, call lower level
2971 *      routine to handle.  Also check for PCI errors which are only
2972 *      reported here.
2973 *
2974 *      LOCKING:
2975 *      This routine holds the host lock while processing pending
2976 *      interrupts.
2977 */
2978static irqreturn_t mv_interrupt(int irq, void *dev_instance)
2979{
2980        struct ata_host *host = dev_instance;
2981        struct mv_host_priv *hpriv = host->private_data;
2982        unsigned int handled = 0;
2983        int using_msi = hpriv->hp_flags & MV_HP_FLAG_MSI;
2984        u32 main_irq_cause, pending_irqs;
2985
2986        spin_lock(&host->lock);
2987
2988        /* for MSI:  block new interrupts while in here */
2989        if (using_msi)
2990                mv_write_main_irq_mask(0, hpriv);
2991
2992        main_irq_cause = readl(hpriv->main_irq_cause_addr);
2993        pending_irqs   = main_irq_cause & hpriv->main_irq_mask;
2994        /*
2995         * Deal with cases where we either have nothing pending, or have read
2996         * a bogus register value which can indicate HW removal or PCI fault.
2997         */
2998        if (pending_irqs && main_irq_cause != 0xffffffffU) {
2999                if (unlikely((pending_irqs & PCI_ERR) && !IS_SOC(hpriv)))
3000                        handled = mv_pci_error(host, hpriv->base);
3001                else
3002                        handled = mv_host_intr(host, pending_irqs);
3003        }
3004
3005        /* for MSI: unmask; interrupt cause bits will retrigger now */
3006        if (using_msi)
3007                mv_write_main_irq_mask(hpriv->main_irq_mask, hpriv);
3008
3009        spin_unlock(&host->lock);
3010
3011        return IRQ_RETVAL(handled);
3012}
3013
3014static unsigned int mv5_scr_offset(unsigned int sc_reg_in)
3015{
3016        unsigned int ofs;
3017
3018        switch (sc_reg_in) {
3019        case SCR_STATUS:
3020        case SCR_ERROR:
3021        case SCR_CONTROL:
3022                ofs = sc_reg_in * sizeof(u32);
3023                break;
3024        default:
3025                ofs = 0xffffffffU;
3026                break;
3027        }
3028        return ofs;
3029}
3030
3031static int mv5_scr_read(struct ata_link *link, unsigned int sc_reg_in, u32 *val)
3032{
3033        struct mv_host_priv *hpriv = link->ap->host->private_data;
3034        void __iomem *mmio = hpriv->base;
3035        void __iomem *addr = mv5_phy_base(mmio, link->ap->port_no);
3036        unsigned int ofs = mv5_scr_offset(sc_reg_in);
3037
3038        if (ofs != 0xffffffffU) {
3039                *val = readl(addr + ofs);
3040                return 0;
3041        } else
3042                return -EINVAL;
3043}
3044
3045static int mv5_scr_write(struct ata_link *link, unsigned int sc_reg_in, u32 val)
3046{
3047        struct mv_host_priv *hpriv = link->ap->host->private_data;
3048        void __iomem *mmio = hpriv->base;
3049        void __iomem *addr = mv5_phy_base(mmio, link->ap->port_no);
3050        unsigned int ofs = mv5_scr_offset(sc_reg_in);
3051
3052        if (ofs != 0xffffffffU) {
3053                writelfl(val, addr + ofs);
3054                return 0;
3055        } else
3056                return -EINVAL;
3057}
3058
3059static void mv5_reset_bus(struct ata_host *host, void __iomem *mmio)
3060{
3061        struct pci_dev *pdev = to_pci_dev(host->dev);
3062        int early_5080;
3063
3064        early_5080 = (pdev->device == 0x5080) && (pdev->revision == 0);
3065
3066        if (!early_5080) {
3067                u32 tmp = readl(mmio + MV_PCI_EXP_ROM_BAR_CTL);
3068                tmp |= (1 << 0);
3069                writel(tmp, mmio + MV_PCI_EXP_ROM_BAR_CTL);
3070        }
3071
3072        mv_reset_pci_bus(host, mmio);
3073}
3074
3075static void mv5_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio)
3076{
3077        writel(0x0fcfffff, mmio + FLASH_CTL);
3078}
3079
3080static void mv5_read_preamp(struct mv_host_priv *hpriv, int idx,
3081                           void __iomem *mmio)
3082{
3083        void __iomem *phy_mmio = mv5_phy_base(mmio, idx);
3084        u32 tmp;
3085
3086        tmp = readl(phy_mmio + MV5_PHY_MODE);
3087
3088        hpriv->signal[idx].pre = tmp & 0x1800;  /* bits 12:11 */
3089        hpriv->signal[idx].amps = tmp & 0xe0;   /* bits 7:5 */
3090}
3091
3092static void mv5_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio)
3093{
3094        u32 tmp;
3095
3096        writel(0, mmio + GPIO_PORT_CTL);
3097
3098        /* FIXME: handle MV_HP_ERRATA_50XXB2 errata */
3099
3100        tmp = readl(mmio + MV_PCI_EXP_ROM_BAR_CTL);
3101        tmp |= ~(1 << 0);
3102        writel(tmp, mmio + MV_PCI_EXP_ROM_BAR_CTL);
3103}
3104
3105static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
3106                           unsigned int port)
3107{
3108        void __iomem *phy_mmio = mv5_phy_base(mmio, port);
3109        const u32 mask = (1<<12) | (1<<11) | (1<<7) | (1<<6) | (1<<5);
3110        u32 tmp;
3111        int fix_apm_sq = (hpriv->hp_flags & MV_HP_ERRATA_50XXB0);
3112
3113        if (fix_apm_sq) {
3114                tmp = readl(phy_mmio + MV5_LTMODE);
3115                tmp |= (1 << 19);
3116                writel(tmp, phy_mmio + MV5_LTMODE);
3117
3118                tmp = readl(phy_mmio + MV5_PHY_CTL);
3119                tmp &= ~0x3;
3120                tmp |= 0x1;
3121                writel(tmp, phy_mmio + MV5_PHY_CTL);
3122        }
3123
3124        tmp = readl(phy_mmio + MV5_PHY_MODE);
3125        tmp &= ~mask;
3126        tmp |= hpriv->signal[port].pre;
3127        tmp |= hpriv->signal[port].amps;
3128        writel(tmp, phy_mmio + MV5_PHY_MODE);
3129}
3130
3131
3132#undef ZERO
3133#define ZERO(reg) writel(0, port_mmio + (reg))
3134static void mv5_reset_hc_port(struct mv_host_priv *hpriv, void __iomem *mmio,
3135                             unsigned int port)
3136{
3137        void __iomem *port_mmio = mv_port_base(mmio, port);
3138
3139        mv_reset_channel(hpriv, mmio, port);
3140
3141        ZERO(0x028);    /* command */
3142        writel(0x11f, port_mmio + EDMA_CFG);
3143        ZERO(0x004);    /* timer */
3144        ZERO(0x008);    /* irq err cause */
3145        ZERO(0x00c);    /* irq err mask */
3146        ZERO(0x010);    /* rq bah */
3147        ZERO(0x014);    /* rq inp */
3148        ZERO(0x018);    /* rq outp */
3149        ZERO(0x01c);    /* respq bah */
3150        ZERO(0x024);    /* respq outp */
3151        ZERO(0x020);    /* respq inp */
3152        ZERO(0x02c);    /* test control */
3153        writel(0xbc, port_mmio + EDMA_IORDY_TMOUT);
3154}
3155#undef ZERO
3156
3157#define ZERO(reg) writel(0, hc_mmio + (reg))
3158static void mv5_reset_one_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
3159                        unsigned int hc)
3160{
3161        void __iomem *hc_mmio = mv_hc_base(mmio, hc);
3162        u32 tmp;
3163
3164        ZERO(0x00c);
3165        ZERO(0x010);
3166        ZERO(0x014);
3167        ZERO(0x018);
3168
3169        tmp = readl(hc_mmio + 0x20);
3170        tmp &= 0x1c1c1c1c;
3171        tmp |= 0x03030303;
3172        writel(tmp, hc_mmio + 0x20);
3173}
3174#undef ZERO
3175
3176static int mv5_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
3177                        unsigned int n_hc)
3178{
3179        unsigned int hc, port;
3180
3181        for (hc = 0; hc < n_hc; hc++) {
3182                for (port = 0; port < MV_PORTS_PER_HC; port++)
3183                        mv5_reset_hc_port(hpriv, mmio,
3184                                          (hc * MV_PORTS_PER_HC) + port);
3185
3186                mv5_reset_one_hc(hpriv, mmio, hc);
3187        }
3188
3189        return 0;
3190}
3191
3192#undef ZERO
3193#define ZERO(reg) writel(0, mmio + (reg))
3194static void mv_reset_pci_bus(struct ata_host *host, void __iomem *mmio)
3195{
3196        struct mv_host_priv *hpriv = host->private_data;
3197        u32 tmp;
3198
3199        tmp = readl(mmio + MV_PCI_MODE);
3200        tmp &= 0xff00ffff;
3201        writel(tmp, mmio + MV_PCI_MODE);
3202
3203        ZERO(MV_PCI_DISC_TIMER);
3204        ZERO(MV_PCI_MSI_TRIGGER);
3205        writel(0x000100ff, mmio + MV_PCI_XBAR_TMOUT);
3206        ZERO(MV_PCI_SERR_MASK);
3207        ZERO(hpriv->irq_cause_offset);
3208        ZERO(hpriv->irq_mask_offset);
3209        ZERO(MV_PCI_ERR_LOW_ADDRESS);
3210        ZERO(MV_PCI_ERR_HIGH_ADDRESS);
3211        ZERO(MV_PCI_ERR_ATTRIBUTE);
3212        ZERO(MV_PCI_ERR_COMMAND);
3213}
3214#undef ZERO
3215
3216static void mv6_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio)
3217{
3218        u32 tmp;
3219
3220        mv5_reset_flash(hpriv, mmio);
3221
3222        tmp = readl(mmio + GPIO_PORT_CTL);
3223        tmp &= 0x3;
3224        tmp |= (1 << 5) | (1 << 6);
3225        writel(tmp, mmio + GPIO_PORT_CTL);
3226}
3227
3228/**
3229 *      mv6_reset_hc - Perform the 6xxx global soft reset
3230 *      @mmio: base address of the HBA
3231 *
3232 *      This routine only applies to 6xxx parts.
3233 *
3234 *      LOCKING:
3235 *      Inherited from caller.
3236 */
3237static int mv6_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
3238                        unsigned int n_hc)
3239{
3240        void __iomem *reg = mmio + PCI_MAIN_CMD_STS;
3241        int i, rc = 0;
3242        u32 t;
3243
3244        /* Following procedure defined in PCI "main command and status
3245         * register" table.
3246         */
3247        t = readl(reg);
3248        writel(t | STOP_PCI_MASTER, reg);
3249
3250        for (i = 0; i < 1000; i++) {
3251                udelay(1);
3252                t = readl(reg);
3253                if (PCI_MASTER_EMPTY & t)
3254                        break;
3255        }
3256        if (!(PCI_MASTER_EMPTY & t)) {
3257                printk(KERN_ERR DRV_NAME ": PCI master won't flush\n");
3258                rc = 1;
3259                goto done;
3260        }
3261
3262        /* set reset */
3263        i = 5;
3264        do {
3265                writel(t | GLOB_SFT_RST, reg);
3266                t = readl(reg);
3267                udelay(1);
3268        } while (!(GLOB_SFT_RST & t) && (i-- > 0));
3269
3270        if (!(GLOB_SFT_RST & t)) {
3271                printk(KERN_ERR DRV_NAME ": can't set global reset\n");
3272                rc = 1;
3273                goto done;
3274        }
3275
3276        /* clear reset and *reenable the PCI master* (not mentioned in spec) */
3277        i = 5;
3278        do {
3279                writel(t & ~(GLOB_SFT_RST | STOP_PCI_MASTER), reg);
3280                t = readl(reg);
3281                udelay(1);
3282        } while ((GLOB_SFT_RST & t) && (i-- > 0));
3283
3284        if (GLOB_SFT_RST & t) {
3285                printk(KERN_ERR DRV_NAME ": can't clear global reset\n");
3286                rc = 1;
3287        }
3288done:
3289        return rc;
3290}
3291
3292static void mv6_read_preamp(struct mv_host_priv *hpriv, int idx,
3293                           void __iomem *mmio)
3294{
3295        void __iomem *port_mmio;
3296        u32 tmp;
3297
3298        tmp = readl(mmio + RESET_CFG);
3299        if ((tmp & (1 << 0)) == 0) {
3300                hpriv->signal[idx].amps = 0x7 << 8;
3301                hpriv->signal[idx].pre = 0x1 << 5;
3302                return;
3303        }
3304
3305        port_mmio = mv_port_base(mmio, idx);
3306        tmp = readl(port_mmio + PHY_MODE2);
3307
3308        hpriv->signal[idx].amps = tmp & 0x700;  /* bits 10:8 */
3309        hpriv->signal[idx].pre = tmp & 0xe0;    /* bits 7:5 */
3310}
3311
3312static void mv6_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio)
3313{
3314        writel(0x00000060, mmio + GPIO_PORT_CTL);
3315}
3316
3317static void mv6_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
3318                           unsigned int port)
3319{
3320        void __iomem *port_mmio = mv_port_base(mmio, port);
3321
3322        u32 hp_flags = hpriv->hp_flags;
3323        int fix_phy_mode2 =
3324                hp_flags & (MV_HP_ERRATA_60X1B2 | MV_HP_ERRATA_60X1C0);
3325        int fix_phy_mode4 =
3326                hp_flags & (MV_HP_ERRATA_60X1B2 | MV_HP_ERRATA_60X1C0);
3327        u32 m2, m3;
3328
3329        if (fix_phy_mode2) {
3330                m2 = readl(port_mmio + PHY_MODE2);
3331                m2 &= ~(1 << 16);
3332                m2 |= (1 << 31);
3333                writel(m2, port_mmio + PHY_MODE2);
3334
3335                udelay(200);
3336
3337                m2 = readl(port_mmio + PHY_MODE2);
3338                m2 &= ~((1 << 16) | (1 << 31));
3339                writel(m2, port_mmio + PHY_MODE2);
3340
3341                udelay(200);
3342        }
3343
3344        /*
3345         * Gen-II/IIe PHY_MODE3 errata RM#2:
3346         * Achieves better receiver noise performance than the h/w default:
3347         */
3348        m3 = readl(port_mmio + PHY_MODE3);
3349        m3 = (m3 & 0x1f) | (0x5555601 << 5);
3350
3351        /* Guideline 88F5182 (GL# SATA-S11) */
3352        if (IS_SOC(hpriv))
3353                m3 &= ~0x1c;
3354
3355        if (fix_phy_mode4) {
3356                u32 m4 = readl(port_mmio + PHY_MODE4);
3357                /*
3358                 * Enforce reserved-bit restrictions on GenIIe devices only.
3359                 * For earlier chipsets, force only the internal config field
3360                 *  (workaround for errata FEr SATA#10 part 1).
3361                 */
3362                if (IS_GEN_IIE(hpriv))
3363                        m4 = (m4 & ~PHY_MODE4_RSVD_ZEROS) | PHY_MODE4_RSVD_ONES;
3364                else
3365                        m4 = (m4 & ~PHY_MODE4_CFG_MASK) | PHY_MODE4_CFG_VALUE;
3366                writel(m4, port_mmio + PHY_MODE4);
3367        }
3368        /*
3369         * Workaround for 60x1-B2 errata SATA#13:
3370         * Any write to PHY_MODE4 (above) may corrupt PHY_MODE3,
3371         * so we must always rewrite PHY_MODE3 after PHY_MODE4.
3372         * Or ensure we use writelfl() when writing PHY_MODE4.
3373         */
3374        writel(m3, port_mmio + PHY_MODE3);
3375
3376        /* Revert values of pre-emphasis and signal amps to the saved ones */
3377        m2 = readl(port_mmio + PHY_MODE2);
3378
3379        m2 &= ~MV_M2_PREAMP_MASK;
3380        m2 |= hpriv->signal[port].amps;
3381        m2 |= hpriv->signal[port].pre;
3382        m2 &= ~(1 << 16);
3383
3384        /* according to mvSata 3.6.1, some IIE values are fixed */
3385        if (IS_GEN_IIE(hpriv)) {
3386                m2 &= ~0xC30FF01F;
3387                m2 |= 0x0000900F;
3388        }
3389
3390        writel(m2, port_mmio + PHY_MODE2);
3391}
3392
3393/* TODO: use the generic LED interface to configure the SATA Presence */
3394/* & Acitivy LEDs on the board */
3395static void mv_soc_enable_leds(struct mv_host_priv *hpriv,
3396                                      void __iomem *mmio)
3397{
3398        return;
3399}
3400
3401static void mv_soc_read_preamp(struct mv_host_priv *hpriv, int idx,
3402                           void __iomem *mmio)
3403{
3404        void __iomem *port_mmio;
3405        u32 tmp;
3406
3407        port_mmio = mv_port_base(mmio, idx);
3408        tmp = readl(port_mmio + PHY_MODE2);
3409
3410        hpriv->signal[idx].amps = tmp & 0x700;  /* bits 10:8 */
3411        hpriv->signal[idx].pre = tmp & 0xe0;    /* bits 7:5 */
3412}
3413
3414#undef ZERO
3415#define ZERO(reg) writel(0, port_mmio + (reg))
3416static void mv_soc_reset_hc_port(struct mv_host_priv *hpriv,
3417                                        void __iomem *mmio, unsigned int port)
3418{
3419        void __iomem *port_mmio = mv_port_base(mmio, port);
3420
3421        mv_reset_channel(hpriv, mmio, port);
3422
3423        ZERO(0x028);            /* command */
3424        writel(0x101f, port_mmio + EDMA_CFG);
3425        ZERO(0x004);            /* timer */
3426        ZERO(0x008);            /* irq err cause */
3427        ZERO(0x00c);            /* irq err mask */
3428        ZERO(0x010);            /* rq bah */
3429        ZERO(0x014);            /* rq inp */
3430        ZERO(0x018);            /* rq outp */
3431        ZERO(0x01c);            /* respq bah */
3432        ZERO(0x024);            /* respq outp */
3433        ZERO(0x020);            /* respq inp */
3434        ZERO(0x02c);            /* test control */
3435        writel(0x800, port_mmio + EDMA_IORDY_TMOUT);
3436}
3437
3438#undef ZERO
3439
3440#define ZERO(reg) writel(0, hc_mmio + (reg))
3441static void mv_soc_reset_one_hc(struct mv_host_priv *hpriv,
3442                                       void __iomem *mmio)
3443{
3444        void __iomem *hc_mmio = mv_hc_base(mmio, 0);
3445
3446        ZERO(0x00c);
3447        ZERO(0x010);
3448        ZERO(0x014);
3449
3450}
3451
3452#undef ZERO
3453
3454static int mv_soc_reset_hc(struct mv_host_priv *hpriv,
3455                                  void __iomem *mmio, unsigned int n_hc)
3456{
3457        unsigned int port;
3458
3459        for (port = 0; port < hpriv->n_ports; port++)
3460                mv_soc_reset_hc_port(hpriv, mmio, port);
3461
3462        mv_soc_reset_one_hc(hpriv, mmio);
3463
3464        return 0;
3465}
3466
3467static void mv_soc_reset_flash(struct mv_host_priv *hpriv,
3468                                      void __iomem *mmio)
3469{
3470        return;
3471}
3472
3473static void mv_soc_reset_bus(struct ata_host *host, void __iomem *mmio)
3474{
3475        return;
3476}
3477
3478static void mv_soc_65n_phy_errata(struct mv_host_priv *hpriv,
3479                                  void __iomem *mmio, unsigned int port)
3480{
3481        void __iomem *port_mmio = mv_port_base(mmio, port);
3482        u32     reg;
3483
3484        reg = readl(port_mmio + PHY_MODE3);
3485        reg &= ~(0x3 << 27);    /* SELMUPF (bits 28:27) to 1 */
3486        reg |= (0x1 << 27);
3487        reg &= ~(0x3 << 29);    /* SELMUPI (bits 30:29) to 1 */
3488        reg |= (0x1 << 29);
3489        writel(reg, port_mmio + PHY_MODE3);
3490
3491        reg = readl(port_mmio + PHY_MODE4);
3492        reg &= ~0x1;    /* SATU_OD8 (bit 0) to 0, reserved bit 16 must be set */
3493        reg |= (0x1 << 16);
3494        writel(reg, port_mmio + PHY_MODE4);
3495
3496        reg = readl(port_mmio + PHY_MODE9_GEN2);
3497        reg &= ~0xf;    /* TXAMP[3:0] (bits 3:0) to 8 */
3498        reg |= 0x8;
3499        reg &= ~(0x1 << 14);    /* TXAMP[4] (bit 14) to 0 */
3500        writel(reg, port_mmio + PHY_MODE9_GEN2);
3501
3502        reg = readl(port_mmio + PHY_MODE9_GEN1);
3503        reg &= ~0xf;    /* TXAMP[3:0] (bits 3:0) to 8 */
3504        reg |= 0x8;
3505        reg &= ~(0x1 << 14);    /* TXAMP[4] (bit 14) to 0 */
3506        writel(reg, port_mmio + PHY_MODE9_GEN1);
3507}
3508
3509/**
3510 *      soc_is_65 - check if the soc is 65 nano device
3511 *
3512 *      Detect the type of the SoC, this is done by reading the PHYCFG_OFS
3513 *      register, this register should contain non-zero value and it exists only
3514 *      in the 65 nano devices, when reading it from older devices we get 0.
3515 */
3516static bool soc_is_65n(struct mv_host_priv *hpriv)
3517{
3518        void __iomem *port0_mmio = mv_port_base(hpriv->base, 0);
3519
3520        if (readl(port0_mmio + PHYCFG_OFS))
3521                return true;
3522        return false;
3523}
3524
3525static void mv_setup_ifcfg(void __iomem *port_mmio, int want_gen2i)
3526{
3527        u32 ifcfg = readl(port_mmio + SATA_IFCFG);
3528
3529        ifcfg = (ifcfg & 0xf7f) | 0x9b1000;     /* from chip spec */
3530        if (want_gen2i)
3531                ifcfg |= (1 << 7);              /* enable gen2i speed */
3532        writelfl(ifcfg, port_mmio + SATA_IFCFG);
3533}
3534
3535static void mv_reset_channel(struct mv_host_priv *hpriv, void __iomem *mmio,
3536                             unsigned int port_no)
3537{
3538        void __iomem *port_mmio = mv_port_base(mmio, port_no);
3539
3540        /*
3541         * The datasheet warns against setting EDMA_RESET when EDMA is active
3542         * (but doesn't say what the problem might be).  So we first try
3543         * to disable the EDMA engine before doing the EDMA_RESET operation.
3544         */
3545        mv_stop_edma_engine(port_mmio);
3546        writelfl(EDMA_RESET, port_mmio + EDMA_CMD);
3547
3548        if (!IS_GEN_I(hpriv)) {
3549                /* Enable 3.0gb/s link speed: this survives EDMA_RESET */
3550                mv_setup_ifcfg(port_mmio, 1);
3551        }
3552        /*
3553         * Strobing EDMA_RESET here causes a hard reset of the SATA transport,
3554         * link, and physical layers.  It resets all SATA interface registers
3555         * (except for SATA_IFCFG), and issues a COMRESET to the dev.
3556         */
3557        writelfl(EDMA_RESET, port_mmio + EDMA_CMD);
3558        udelay(25);     /* allow reset propagation */
3559        writelfl(0, port_mmio + EDMA_CMD);
3560
3561        hpriv->ops->phy_errata(hpriv, mmio, port_no);
3562
3563        if (IS_GEN_I(hpriv))
3564                mdelay(1);
3565}
3566
3567static void mv_pmp_select(struct ata_port *ap, int pmp)
3568{
3569        if (sata_pmp_supported(ap)) {
3570                void __iomem *port_mmio = mv_ap_base(ap);
3571                u32 reg = readl(port_mmio + SATA_IFCTL);
3572                int old = reg & 0xf;
3573
3574                if (old != pmp) {
3575                        reg = (reg & ~0xf) | pmp;
3576                        writelfl(reg, port_mmio + SATA_IFCTL);
3577                }
3578        }
3579}
3580
3581static int mv_pmp_hardreset(struct ata_link *link, unsigned int *class,
3582                                unsigned long deadline)
3583{
3584        mv_pmp_select(link->ap, sata_srst_pmp(link));
3585        return sata_std_hardreset(link, class, deadline);
3586}
3587
3588static int mv_softreset(struct ata_link *link, unsigned int *class,
3589                                unsigned long deadline)
3590{
3591        mv_pmp_select(link->ap, sata_srst_pmp(link));
3592        return ata_sff_softreset(link, class, deadline);
3593}
3594
3595static int mv_hardreset(struct ata_link *link, unsigned int *class,
3596                        unsigned long deadline)
3597{
3598        struct ata_port *ap = link->ap;
3599        struct mv_host_priv *hpriv = ap->host->private_data;
3600        struct mv_port_priv *pp = ap->private_data;
3601        void __iomem *mmio = hpriv->base;
3602        int rc, attempts = 0, extra = 0;
3603        u32 sstatus;
3604        bool online;
3605
3606        mv_reset_channel(hpriv, mmio, ap->port_no);
3607        pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN;
3608        pp->pp_flags &=
3609          ~(MV_PP_FLAG_FBS_EN | MV_PP_FLAG_NCQ_EN | MV_PP_FLAG_FAKE_ATA_BUSY);
3610
3611        /* Workaround for errata FEr SATA#10 (part 2) */
3612        do {
3613                const unsigned long *timing =
3614                                sata_ehc_deb_timing(&link->eh_context);
3615
3616                rc = sata_link_hardreset(link, timing, deadline + extra,
3617                                         &online, NULL);
3618                rc = online ? -EAGAIN : rc;
3619                if (rc)
3620                        return rc;
3621                sata_scr_read(link, SCR_STATUS, &sstatus);
3622                if (!IS_GEN_I(hpriv) && ++attempts >= 5 && sstatus == 0x121) {
3623                        /* Force 1.5gb/s link speed and try again */
3624                        mv_setup_ifcfg(mv_ap_base(ap), 0);
3625                        if (time_after(jiffies + HZ, deadline))
3626                                extra = HZ; /* only extend it once, max */
3627                }
3628        } while (sstatus != 0x0 && sstatus != 0x113 && sstatus != 0x123);
3629        mv_save_cached_regs(ap);
3630        mv_edma_cfg(ap, 0, 0);
3631
3632        return rc;
3633}
3634
3635static void mv_eh_freeze(struct ata_port *ap)
3636{
3637        mv_stop_edma(ap);
3638        mv_enable_port_irqs(ap, 0);
3639}
3640
3641static void mv_eh_thaw(struct ata_port *ap)
3642{
3643        struct mv_host_priv *hpriv = ap->host->private_data;
3644        unsigned int port = ap->port_no;
3645        unsigned int hardport = mv_hardport_from_port(port);
3646        void __iomem *hc_mmio = mv_hc_base_from_port(hpriv->base, port);
3647        void __iomem *port_mmio = mv_ap_base(ap);
3648        u32 hc_irq_cause;
3649
3650        /* clear EDMA errors on this port */
3651        writel(0, port_mmio + EDMA_ERR_IRQ_CAUSE);
3652
3653        /* clear pending irq events */
3654        hc_irq_cause = ~((DEV_IRQ | DMA_IRQ) << hardport);
3655        writelfl(hc_irq_cause, hc_mmio + HC_IRQ_CAUSE);
3656
3657        mv_enable_port_irqs(ap, ERR_IRQ);
3658}
3659
3660/**
3661 *      mv_port_init - Perform some early initialization on a single port.
3662 *      @port: libata data structure storing shadow register addresses
3663 *      @port_mmio: base address of the port
3664 *
3665 *      Initialize shadow register mmio addresses, clear outstanding
3666 *      interrupts on the port, and unmask interrupts for the future
3667 *      start of the port.
3668 *
3669 *      LOCKING:
3670 *      Inherited from caller.
3671 */
3672static void mv_port_init(struct ata_ioports *port,  void __iomem *port_mmio)
3673{
3674        void __iomem *serr, *shd_base = port_mmio + SHD_BLK;
3675
3676        /* PIO related setup
3677         */
3678        port->data_addr = shd_base + (sizeof(u32) * ATA_REG_DATA);
3679        port->error_addr =
3680                port->feature_addr = shd_base + (sizeof(u32) * ATA_REG_ERR);
3681        port->nsect_addr = shd_base + (sizeof(u32) * ATA_REG_NSECT);
3682        port->lbal_addr = shd_base + (sizeof(u32) * ATA_REG_LBAL);
3683        port->lbam_addr = shd_base + (sizeof(u32) * ATA_REG_LBAM);
3684        port->lbah_addr = shd_base + (sizeof(u32) * ATA_REG_LBAH);
3685        port->device_addr = shd_base + (sizeof(u32) * ATA_REG_DEVICE);
3686        port->status_addr =
3687                port->command_addr = shd_base + (sizeof(u32) * ATA_REG_STATUS);
3688        /* special case: control/altstatus doesn't have ATA_REG_ address */
3689        port->altstatus_addr = port->ctl_addr = shd_base + SHD_CTL_AST;
3690
3691        /* Clear any currently outstanding port interrupt conditions */
3692        serr = port_mmio + mv_scr_offset(SCR_ERROR);
3693        writelfl(readl(serr), serr);
3694        writelfl(0, port_mmio + EDMA_ERR_IRQ_CAUSE);
3695
3696        /* unmask all non-transient EDMA error interrupts */
3697        writelfl(~EDMA_ERR_IRQ_TRANSIENT, port_mmio + EDMA_ERR_IRQ_MASK);
3698
3699        VPRINTK("EDMA cfg=0x%08x EDMA IRQ err cause/mask=0x%08x/0x%08x\n",
3700                readl(port_mmio + EDMA_CFG),
3701                readl(port_mmio + EDMA_ERR_IRQ_CAUSE),
3702                readl(port_mmio + EDMA_ERR_IRQ_MASK));
3703}
3704
3705static unsigned int mv_in_pcix_mode(struct ata_host *host)
3706{
3707        struct mv_host_priv *hpriv = host->private_data;
3708        void __iomem *mmio = hpriv->base;
3709        u32 reg;
3710
3711        if (IS_SOC(hpriv) || !IS_PCIE(hpriv))
3712                return 0;       /* not PCI-X capable */
3713        reg = readl(mmio + MV_PCI_MODE);
3714        if ((reg & MV_PCI_MODE_MASK) == 0)
3715                return 0;       /* conventional PCI mode */
3716        return 1;       /* chip is in PCI-X mode */
3717}
3718
3719static int mv_pci_cut_through_okay(struct ata_host *host)
3720{
3721        struct mv_host_priv *hpriv = host->private_data;
3722        void __iomem *mmio = hpriv->base;
3723        u32 reg;
3724
3725        if (!mv_in_pcix_mode(host)) {
3726                reg = readl(mmio + MV_PCI_COMMAND);
3727                if (reg & MV_PCI_COMMAND_MRDTRIG)
3728                        return 0; /* not okay */
3729        }
3730        return 1; /* okay */
3731}
3732
3733static void mv_60x1b2_errata_pci7(struct ata_host *host)
3734{
3735        struct mv_host_priv *hpriv = host->private_data;
3736        void __iomem *mmio = hpriv->base;
3737
3738        /* workaround for 60x1-B2 errata PCI#7 */
3739        if (mv_in_pcix_mode(host)) {
3740                u32 reg = readl(mmio + MV_PCI_COMMAND);
3741                writelfl(reg & ~MV_PCI_COMMAND_MWRCOM, mmio + MV_PCI_COMMAND);
3742        }
3743}
3744
3745static int mv_chip_id(struct ata_host *host, unsigned int board_idx)
3746{
3747        struct pci_dev *pdev = to_pci_dev(host->dev);
3748        struct mv_host_priv *hpriv = host->private_data;
3749        u32 hp_flags = hpriv->hp_flags;
3750
3751        switch (board_idx) {
3752        case chip_5080:
3753                hpriv->ops = &mv5xxx_ops;
3754                hp_flags |= MV_HP_GEN_I;
3755
3756                switch (pdev->revision) {
3757                case 0x1:
3758                        hp_flags |= MV_HP_ERRATA_50XXB0;
3759                        break;
3760                case 0x3:
3761                        hp_flags |= MV_HP_ERRATA_50XXB2;
3762                        break;
3763                default:
3764                        dev_warn(&pdev->dev,
3765                                 "Applying 50XXB2 workarounds to unknown rev\n");
3766                        hp_flags |= MV_HP_ERRATA_50XXB2;
3767                        break;
3768                }
3769                break;
3770
3771        case chip_504x:
3772        case chip_508x:
3773                hpriv->ops = &mv5xxx_ops;
3774                hp_flags |= MV_HP_GEN_I;
3775
3776                switch (pdev->revision) {
3777                case 0x0:
3778                        hp_flags |= MV_HP_ERRATA_50XXB0;
3779                        break;
3780                case 0x3:
3781                        hp_flags |= MV_HP_ERRATA_50XXB2;
3782                        break;
3783                default:
3784                        dev_warn(&pdev->dev,
3785                                 "Applying B2 workarounds to unknown rev\n");
3786                        hp_flags |= MV_HP_ERRATA_50XXB2;
3787                        break;
3788                }
3789                break;
3790
3791        case chip_604x:
3792        case chip_608x:
3793                hpriv->ops = &mv6xxx_ops;
3794                hp_flags |= MV_HP_GEN_II;
3795
3796                switch (pdev->revision) {
3797                case 0x7:
3798                        mv_60x1b2_errata_pci7(host);
3799                        hp_flags |= MV_HP_ERRATA_60X1B2;
3800                        break;
3801                case 0x9:
3802                        hp_flags |= MV_HP_ERRATA_60X1C0;
3803                        break;
3804                default:
3805                        dev_warn(&pdev->dev,
3806                                 "Applying B2 workarounds to unknown rev\n");
3807                        hp_flags |= MV_HP_ERRATA_60X1B2;
3808                        break;
3809                }
3810                break;
3811
3812        case chip_7042:
3813                hp_flags |= MV_HP_PCIE | MV_HP_CUT_THROUGH;
3814                if (pdev->vendor == PCI_VENDOR_ID_TTI &&
3815                    (pdev->device == 0x2300 || pdev->device == 0x2310))
3816                {
3817                        /*
3818                         * Highpoint RocketRAID PCIe 23xx series cards:
3819                         *
3820                         * Unconfigured drives are treated as "Legacy"
3821                         * by the BIOS, and it overwrites sector 8 with
3822                         * a "Lgcy" metadata block prior to Linux boot.
3823                         *
3824                         * Configured drives (RAID or JBOD) leave sector 8
3825                         * alone, but instead overwrite a high numbered
3826                         * sector for the RAID metadata.  This sector can
3827                         * be determined exactly, by truncating the physical
3828                         * drive capacity to a nice even GB value.
3829                         *
3830                         * RAID metadata is at: (dev->n_sectors & ~0xfffff)
3831                         *
3832                         * Warn the user, lest they think we're just buggy.
3833                         */
3834                        printk(KERN_WARNING DRV_NAME ": Highpoint RocketRAID"
3835                                " BIOS CORRUPTS DATA on all attached drives,"
3836                                " regardless of if/how they are configured."
3837                                " BEWARE!\n");
3838                        printk(KERN_WARNING DRV_NAME ": For data safety, do not"
3839                                " use sectors 8-9 on \"Legacy\" drives,"
3840                                " and avoid the final two gigabytes on"
3841                                " all RocketRAID BIOS initialized drives.\n");
3842                }
3843                /* drop through */
3844        case chip_6042:
3845                hpriv->ops = &mv6xxx_ops;
3846                hp_flags |= MV_HP_GEN_IIE;
3847                if (board_idx == chip_6042 && mv_pci_cut_through_okay(host))
3848                        hp_flags |= MV_HP_CUT_THROUGH;
3849
3850                switch (pdev->revision) {
3851                case 0x2: /* Rev.B0: the first/only public release */
3852                        hp_flags |= MV_HP_ERRATA_60X1C0;
3853                        break;
3854                default:
3855                        dev_warn(&pdev->dev,
3856                                 "Applying 60X1C0 workarounds to unknown rev\n");
3857                        hp_flags |= MV_HP_ERRATA_60X1C0;
3858                        break;
3859                }
3860                break;
3861        case chip_soc:
3862                if (soc_is_65n(hpriv))
3863                        hpriv->ops = &mv_soc_65n_ops;
3864                else
3865                        hpriv->ops = &mv_soc_ops;
3866                hp_flags |= MV_HP_FLAG_SOC | MV_HP_GEN_IIE |
3867                        MV_HP_ERRATA_60X1C0;
3868                break;
3869
3870        default:
3871                dev_err(host->dev, "BUG: invalid board index %u\n", board_idx);
3872                return 1;
3873        }
3874
3875        hpriv->hp_flags = hp_flags;
3876        if (hp_flags & MV_HP_PCIE) {
3877                hpriv->irq_cause_offset = PCIE_IRQ_CAUSE;
3878                hpriv->irq_mask_offset  = PCIE_IRQ_MASK;
3879                hpriv->unmask_all_irqs  = PCIE_UNMASK_ALL_IRQS;
3880        } else {
3881                hpriv->irq_cause_offset = PCI_IRQ_CAUSE;
3882                hpriv->irq_mask_offset  = PCI_IRQ_MASK;
3883                hpriv->unmask_all_irqs  = PCI_UNMASK_ALL_IRQS;
3884        }
3885
3886        return 0;
3887}
3888
3889/**
3890 *      mv_init_host - Perform some early initialization of the host.
3891 *      @host: ATA host to initialize
3892 *
3893 *      If possible, do an early global reset of the host.  Then do
3894 *      our port init and clear/unmask all/relevant host interrupts.
3895 *
3896 *      LOCKING:
3897 *      Inherited from caller.
3898 */
3899static int mv_init_host(struct ata_host *host)
3900{
3901        int rc = 0, n_hc, port, hc;
3902        struct mv_host_priv *hpriv = host->private_data;
3903        void __iomem *mmio = hpriv->base;
3904
3905        rc = mv_chip_id(host, hpriv->board_idx);
3906        if (rc)
3907                goto done;
3908
3909        if (IS_SOC(hpriv)) {
3910                hpriv->main_irq_cause_addr = mmio + SOC_HC_MAIN_IRQ_CAUSE;
3911                hpriv->main_irq_mask_addr  = mmio + SOC_HC_MAIN_IRQ_MASK;
3912        } else {
3913                hpriv->main_irq_cause_addr = mmio + PCI_HC_MAIN_IRQ_CAUSE;
3914                hpriv->main_irq_mask_addr  = mmio + PCI_HC_MAIN_IRQ_MASK;
3915        }
3916
3917        /* initialize shadow irq mask with register's value */
3918        hpriv->main_irq_mask = readl(hpriv->main_irq_mask_addr);
3919
3920        /* global interrupt mask: 0 == mask everything */
3921        mv_set_main_irq_mask(host, ~0, 0);
3922
3923        n_hc = mv_get_hc_count(host->ports[0]->flags);
3924
3925        for (port = 0; port < host->n_ports; port++)
3926                if (hpriv->ops->read_preamp)
3927                        hpriv->ops->read_preamp(hpriv, port, mmio);
3928
3929        rc = hpriv->ops->reset_hc(hpriv, mmio, n_hc);
3930        if (rc)
3931                goto done;
3932
3933        hpriv->ops->reset_flash(hpriv, mmio);
3934        hpriv->ops->reset_bus(host, mmio);
3935        hpriv->ops->enable_leds(hpriv, mmio);
3936
3937        for (port = 0; port < host->n_ports; port++) {
3938                struct ata_port *ap = host->ports[port];
3939                void __iomem *port_mmio = mv_port_base(mmio, port);
3940
3941                mv_port_init(&ap->ioaddr, port_mmio);
3942        }
3943
3944        for (hc = 0; hc < n_hc; hc++) {
3945                void __iomem *hc_mmio = mv_hc_base(mmio, hc);
3946
3947                VPRINTK("HC%i: HC config=0x%08x HC IRQ cause "
3948                        "(before clear)=0x%08x\n", hc,
3949                        readl(hc_mmio + HC_CFG),
3950                        readl(hc_mmio + HC_IRQ_CAUSE));
3951
3952                /* Clear any currently outstanding hc interrupt conditions */
3953                writelfl(0, hc_mmio + HC_IRQ_CAUSE);
3954        }
3955
3956        if (!IS_SOC(hpriv)) {
3957                /* Clear any currently outstanding host interrupt conditions */
3958                writelfl(0, mmio + hpriv->irq_cause_offset);
3959
3960                /* and unmask interrupt generation for host regs */
3961                writelfl(hpriv->unmask_all_irqs, mmio + hpriv->irq_mask_offset);
3962        }
3963
3964        /*
3965         * enable only global host interrupts for now.
3966         * The per-port interrupts get done later as ports are set up.
3967         */
3968        mv_set_main_irq_mask(host, 0, PCI_ERR);
3969        mv_set_irq_coalescing(host, irq_coalescing_io_count,
3970                                    irq_coalescing_usecs);
3971done:
3972        return rc;
3973}
3974
3975static int mv_create_dma_pools(struct mv_host_priv *hpriv, struct device *dev)
3976{
3977        hpriv->crqb_pool   = dmam_pool_create("crqb_q", dev, MV_CRQB_Q_SZ,
3978                                                             MV_CRQB_Q_SZ, 0);
3979        if (!hpriv->crqb_pool)
3980                return -ENOMEM;
3981
3982        hpriv->crpb_pool   = dmam_pool_create("crpb_q", dev, MV_CRPB_Q_SZ,
3983                                                             MV_CRPB_Q_SZ, 0);
3984        if (!hpriv->crpb_pool)
3985                return -ENOMEM;
3986
3987        hpriv->sg_tbl_pool = dmam_pool_create("sg_tbl", dev, MV_SG_TBL_SZ,
3988                                                             MV_SG_TBL_SZ, 0);
3989        if (!hpriv->sg_tbl_pool)
3990                return -ENOMEM;
3991
3992        return 0;
3993}
3994
3995static void mv_conf_mbus_windows(struct mv_host_priv *hpriv,
3996                                 const struct mbus_dram_target_info *dram)
3997{
3998        int i;
3999
4000        for (i = 0; i < 4; i++) {
4001                writel(0, hpriv->base + WINDOW_CTRL(i));
4002                writel(0, hpriv->base + WINDOW_BASE(i));
4003        }
4004
4005        for (i = 0; i < dram->num_cs; i++) {
4006                const struct mbus_dram_window *cs = dram->cs + i;
4007
4008                writel(((cs->size - 1) & 0xffff0000) |
4009                        (cs->mbus_attr << 8) |
4010                        (dram->mbus_dram_target_id << 4) | 1,
4011                        hpriv->base + WINDOW_CTRL(i));
4012                writel(cs->base, hpriv->base + WINDOW_BASE(i));
4013        }
4014}
4015
4016/**
4017 *      mv_platform_probe - handle a positive probe of an soc Marvell
4018 *      host
4019 *      @pdev: platform device found
4020 *
4021 *      LOCKING:
4022 *      Inherited from caller.
4023 */
4024static int mv_platform_probe(struct platform_device *pdev)
4025{
4026        const struct mv_sata_platform_data *mv_platform_data;
4027        const struct mbus_dram_target_info *dram;
4028        const struct ata_port_info *ppi[] =
4029            { &mv_port_info[chip_soc], NULL };
4030        struct ata_host *host;
4031        struct mv_host_priv *hpriv;
4032        struct resource *res;
4033        int n_ports = 0, irq = 0;
4034        int rc;
4035#if defined(CONFIG_HAVE_CLK)
4036        int port;
4037#endif
4038
4039        ata_print_version_once(&pdev->dev, DRV_VERSION);
4040
4041        /*
4042         * Simple resource validation ..
4043         */
4044        if (unlikely(pdev->num_resources != 2)) {
4045                dev_err(&pdev->dev, "invalid number of resources\n");
4046                return -EINVAL;
4047        }
4048
4049        /*
4050         * Get the register base first
4051         */
4052        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4053        if (res == NULL)
4054                return -EINVAL;
4055
4056        /* allocate host */
4057        if (pdev->dev.of_node) {
4058                of_property_read_u32(pdev->dev.of_node, "nr-ports", &n_ports);
4059                irq = irq_of_parse_and_map(pdev->dev.of_node, 0);
4060        } else {
4061                mv_platform_data = pdev->dev.platform_data;
4062                n_ports = mv_platform_data->n_ports;
4063                irq = platform_get_irq(pdev, 0);
4064        }
4065
4066        host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports);
4067        hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
4068
4069        if (!host || !hpriv)
4070                return -ENOMEM;
4071#if defined(CONFIG_HAVE_CLK)
4072        hpriv->port_clks = devm_kzalloc(&pdev->dev,
4073                                        sizeof(struct clk *) * n_ports,
4074                                        GFP_KERNEL);
4075        if (!hpriv->port_clks)
4076                return -ENOMEM;
4077#endif
4078        host->private_data = hpriv;
4079        hpriv->n_ports = n_ports;
4080        hpriv->board_idx = chip_soc;
4081
4082        host->iomap = NULL;
4083        hpriv->base = devm_ioremap(&pdev->dev, res->start,
4084                                   resource_size(res));
4085        hpriv->base -= SATAHC0_REG_BASE;
4086
4087#if defined(CONFIG_HAVE_CLK)
4088        hpriv->clk = clk_get(&pdev->dev, NULL);
4089        if (IS_ERR(hpriv->clk))
4090                dev_notice(&pdev->dev, "cannot get optional clkdev\n");
4091        else
4092                clk_prepare_enable(hpriv->clk);
4093
4094        for (port = 0; port < n_ports; port++) {
4095                char port_number[16];
4096                sprintf(port_number, "%d", port);
4097                hpriv->port_clks[port] = clk_get(&pdev->dev, port_number);
4098                if (!IS_ERR(hpriv->port_clks[port]))
4099                        clk_prepare_enable(hpriv->port_clks[port]);
4100        }
4101#endif
4102
4103        /*
4104         * (Re-)program MBUS remapping windows if we are asked to.
4105         */
4106        dram = mv_mbus_dram_info();
4107        if (dram)
4108                mv_conf_mbus_windows(hpriv, dram);
4109
4110        rc = mv_create_dma_pools(hpriv, &pdev->dev);
4111        if (rc)
4112                goto err;
4113
4114        /* initialize adapter */
4115        rc = mv_init_host(host);
4116        if (rc)
4117                goto err;
4118
4119        dev_info(&pdev->dev, "slots %u ports %d\n",
4120                 (unsigned)MV_MAX_Q_DEPTH, host->n_ports);
4121
4122        rc = ata_host_activate(host, irq, mv_interrupt, IRQF_SHARED, &mv6_sht);
4123        if (!rc)
4124                return 0;
4125
4126err:
4127#if defined(CONFIG_HAVE_CLK)
4128        if (!IS_ERR(hpriv->clk)) {
4129                clk_disable_unprepare(hpriv->clk);
4130                clk_put(hpriv->clk);
4131        }
4132        for (port = 0; port < n_ports; port++) {
4133                if (!IS_ERR(hpriv->port_clks[port])) {
4134                        clk_disable_unprepare(hpriv->port_clks[port]);
4135                        clk_put(hpriv->port_clks[port]);
4136                }
4137        }
4138#endif
4139
4140        return rc;
4141}
4142
4143/*
4144 *
4145 *      mv_platform_remove    -       unplug a platform interface
4146 *      @pdev: platform device
4147 *
4148 *      A platform bus SATA device has been unplugged. Perform the needed
4149 *      cleanup. Also called on module unload for any active devices.
4150 */
4151static int __devexit mv_platform_remove(struct platform_device *pdev)
4152{
4153        struct ata_host *host = platform_get_drvdata(pdev);
4154#if defined(CONFIG_HAVE_CLK)
4155        struct mv_host_priv *hpriv = host->private_data;
4156        int port;
4157#endif
4158        ata_host_detach(host);
4159
4160#if defined(CONFIG_HAVE_CLK)
4161        if (!IS_ERR(hpriv->clk)) {
4162                clk_disable_unprepare(hpriv->clk);
4163                clk_put(hpriv->clk);
4164        }
4165        for (port = 0; port < host->n_ports; port++) {
4166                if (!IS_ERR(hpriv->port_clks[port])) {
4167                        clk_disable_unprepare(hpriv->port_clks[port]);
4168                        clk_put(hpriv->port_clks[port]);
4169                }
4170        }
4171#endif
4172        return 0;
4173}
4174
4175#ifdef CONFIG_PM
4176static int mv_platform_suspend(struct platform_device *pdev, pm_message_t state)
4177{
4178        struct ata_host *host = platform_get_drvdata(pdev);
4179        if (host)
4180                return ata_host_suspend(host, state);
4181        else
4182                return 0;
4183}
4184
4185static int mv_platform_resume(struct platform_device *pdev)
4186{
4187        struct ata_host *host = platform_get_drvdata(pdev);
4188        const struct mbus_dram_target_info *dram;
4189        int ret;
4190
4191        if (host) {
4192                struct mv_host_priv *hpriv = host->private_data;
4193
4194                /*
4195                 * (Re-)program MBUS remapping windows if we are asked to.
4196                 */
4197                dram = mv_mbus_dram_info();
4198                if (dram)
4199                        mv_conf_mbus_windows(hpriv, dram);
4200
4201                /* initialize adapter */
4202                ret = mv_init_host(host);
4203                if (ret) {
4204                        printk(KERN_ERR DRV_NAME ": Error during HW init\n");
4205                        return ret;
4206                }
4207                ata_host_resume(host);
4208        }
4209
4210        return 0;
4211}
4212#else
4213#define mv_platform_suspend NULL
4214#define mv_platform_resume NULL
4215#endif
4216
4217#ifdef CONFIG_OF
4218static struct of_device_id mv_sata_dt_ids[] __devinitdata = {
4219        { .compatible = "marvell,orion-sata", },
4220        {},
4221};
4222MODULE_DEVICE_TABLE(of, mv_sata_dt_ids);
4223#endif
4224
4225static struct platform_driver mv_platform_driver = {
4226        .probe          = mv_platform_probe,
4227        .remove         = __devexit_p(mv_platform_remove),
4228        .suspend        = mv_platform_suspend,
4229        .resume         = mv_platform_resume,
4230        .driver         = {
4231                .name = DRV_NAME,
4232                .owner = THIS_MODULE,
4233                .of_match_table = of_match_ptr(mv_sata_dt_ids),
4234        },
4235};
4236
4237
4238#ifdef CONFIG_PCI
4239static int mv_pci_init_one(struct pci_dev *pdev,
4240                           const struct pci_device_id *ent);
4241#ifdef CONFIG_PM
4242static int mv_pci_device_resume(struct pci_dev *pdev);
4243#endif
4244
4245
4246static struct pci_driver mv_pci_driver = {
4247        .name                   = DRV_NAME,
4248        .id_table               = mv_pci_tbl,
4249        .probe                  = mv_pci_init_one,
4250        .remove                 = ata_pci_remove_one,
4251#ifdef CONFIG_PM
4252        .suspend                = ata_pci_device_suspend,
4253        .resume                 = mv_pci_device_resume,
4254#endif
4255
4256};
4257
4258/* move to PCI layer or libata core? */
4259static int pci_go_64(struct pci_dev *pdev)
4260{
4261        int rc;
4262
4263        if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
4264                rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
4265                if (rc) {
4266                        rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
4267                        if (rc) {
4268                                dev_err(&pdev->dev,
4269                                        "64-bit DMA enable failed\n");
4270                                return rc;
4271                        }
4272                }
4273        } else {
4274                rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
4275                if (rc) {
4276                        dev_err(&pdev->dev, "32-bit DMA enable failed\n");
4277                        return rc;
4278                }
4279                rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
4280                if (rc) {
4281                        dev_err(&pdev->dev,
4282                                "32-bit consistent DMA enable failed\n");
4283                        return rc;
4284                }
4285        }
4286
4287        return rc;
4288}
4289
4290/**
4291 *      mv_print_info - Dump key info to kernel log for perusal.
4292 *      @host: ATA host to print info about
4293 *
4294 *      FIXME: complete this.
4295 *
4296 *      LOCKING:
4297 *      Inherited from caller.
4298 */
4299static void mv_print_info(struct ata_host *host)
4300{
4301        struct pci_dev *pdev = to_pci_dev(host->dev);
4302        struct mv_host_priv *hpriv = host->private_data;
4303        u8 scc;
4304        const char *scc_s, *gen;
4305
4306        /* Use this to determine the HW stepping of the chip so we know
4307         * what errata to workaround
4308         */
4309        pci_read_config_byte(pdev, PCI_CLASS_DEVICE, &scc);
4310        if (scc == 0)
4311                scc_s = "SCSI";
4312        else if (scc == 0x01)
4313                scc_s = "RAID";
4314        else
4315                scc_s = "?";
4316
4317        if (IS_GEN_I(hpriv))
4318                gen = "I";
4319        else if (IS_GEN_II(hpriv))
4320                gen = "II";
4321        else if (IS_GEN_IIE(hpriv))
4322                gen = "IIE";
4323        else
4324                gen = "?";
4325
4326        dev_info(&pdev->dev, "Gen-%s %u slots %u ports %s mode IRQ via %s\n",
4327                 gen, (unsigned)MV_MAX_Q_DEPTH, host->n_ports,
4328                 scc_s, (MV_HP_FLAG_MSI & hpriv->hp_flags) ? "MSI" : "INTx");
4329}
4330
4331/**
4332 *      mv_pci_init_one - handle a positive probe of a PCI Marvell host
4333 *      @pdev: PCI device found
4334 *      @ent: PCI device ID entry for the matched host
4335 *
4336 *      LOCKING:
4337 *      Inherited from caller.
4338 */
4339static int mv_pci_init_one(struct pci_dev *pdev,
4340                           const struct pci_device_id *ent)
4341{
4342        unsigned int board_idx = (unsigned int)ent->driver_data;
4343        const struct ata_port_info *ppi[] = { &mv_port_info[board_idx], NULL };
4344        struct ata_host *host;
4345        struct mv_host_priv *hpriv;
4346        int n_ports, port, rc;
4347
4348        ata_print_version_once(&pdev->dev, DRV_VERSION);
4349
4350        /* allocate host */
4351        n_ports = mv_get_hc_count(ppi[0]->flags) * MV_PORTS_PER_HC;
4352
4353        host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports);
4354        hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
4355        if (!host || !hpriv)
4356                return -ENOMEM;
4357        host->private_data = hpriv;
4358        hpriv->n_ports = n_ports;
4359        hpriv->board_idx = board_idx;
4360
4361        /* acquire resources */
4362        rc = pcim_enable_device(pdev);
4363        if (rc)
4364                return rc;
4365
4366        rc = pcim_iomap_regions(pdev, 1 << MV_PRIMARY_BAR, DRV_NAME);
4367        if (rc == -EBUSY)
4368                pcim_pin_device(pdev);
4369        if (rc)
4370                return rc;
4371        host->iomap = pcim_iomap_table(pdev);
4372        hpriv->base = host->iomap[MV_PRIMARY_BAR];
4373
4374        rc = pci_go_64(pdev);
4375        if (rc)
4376                return rc;
4377
4378        rc = mv_create_dma_pools(hpriv, &pdev->dev);
4379        if (rc)
4380                return rc;
4381
4382        for (port = 0; port < host->n_ports; port++) {
4383                struct ata_port *ap = host->ports[port];
4384                void __iomem *port_mmio = mv_port_base(hpriv->base, port);
4385                unsigned int offset = port_mmio - hpriv->base;
4386
4387                ata_port_pbar_desc(ap, MV_PRIMARY_BAR, -1, "mmio");
4388                ata_port_pbar_desc(ap, MV_PRIMARY_BAR, offset, "port");
4389        }
4390
4391        /* initialize adapter */
4392        rc = mv_init_host(host);
4393        if (rc)
4394                return rc;
4395
4396        /* Enable message-switched interrupts, if requested */
4397        if (msi && pci_enable_msi(pdev) == 0)
4398                hpriv->hp_flags |= MV_HP_FLAG_MSI;
4399
4400        mv_dump_pci_cfg(pdev, 0x68);
4401        mv_print_info(host);
4402
4403        pci_set_master(pdev);
4404        pci_try_set_mwi(pdev);
4405        return ata_host_activate(host, pdev->irq, mv_interrupt, IRQF_SHARED,
4406                                 IS_GEN_I(hpriv) ? &mv5_sht : &mv6_sht);
4407}
4408
4409#ifdef CONFIG_PM
4410static int mv_pci_device_resume(struct pci_dev *pdev)
4411{
4412        struct ata_host *host = pci_get_drvdata(pdev);
4413        int rc;
4414
4415        rc = ata_pci_device_do_resume(pdev);
4416        if (rc)
4417                return rc;
4418
4419        /* initialize adapter */
4420        rc = mv_init_host(host);
4421        if (rc)
4422                return rc;
4423
4424        ata_host_resume(host);
4425
4426        return 0;
4427}
4428#endif
4429#endif
4430
4431static int mv_platform_probe(struct platform_device *pdev);
4432static int __devexit mv_platform_remove(struct platform_device *pdev);
4433
4434static int __init mv_init(void)
4435{
4436        int rc = -ENODEV;
4437#ifdef CONFIG_PCI
4438        rc = pci_register_driver(&mv_pci_driver);
4439        if (rc < 0)
4440                return rc;
4441#endif
4442        rc = platform_driver_register(&mv_platform_driver);
4443
4444#ifdef CONFIG_PCI
4445        if (rc < 0)
4446                pci_unregister_driver(&mv_pci_driver);
4447#endif
4448        return rc;
4449}
4450
4451static void __exit mv_exit(void)
4452{
4453#ifdef CONFIG_PCI
4454        pci_unregister_driver(&mv_pci_driver);
4455#endif
4456        platform_driver_unregister(&mv_platform_driver);
4457}
4458
4459MODULE_AUTHOR("Brett Russ");
4460MODULE_DESCRIPTION("SCSI low-level driver for Marvell SATA controllers");
4461MODULE_LICENSE("GPL");
4462MODULE_DEVICE_TABLE(pci, mv_pci_tbl);
4463MODULE_VERSION(DRV_VERSION);
4464MODULE_ALIAS("platform:" DRV_NAME);
4465
4466module_init(mv_init);
4467module_exit(mv_exit);
4468
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.