linux/drivers/ata/sata_nv.c
<<
>>
Prefs
   1/*
   2 *  sata_nv.c - NVIDIA nForce SATA
   3 *
   4 *  Copyright 2004 NVIDIA Corp.  All rights reserved.
   5 *  Copyright 2004 Andrew Chew
   6 *
   7 *
   8 *  This program is free software; you can redistribute it and/or modify
   9 *  it under the terms of the GNU General Public License as published by
  10 *  the Free Software Foundation; either version 2, or (at your option)
  11 *  any later version.
  12 *
  13 *  This program is distributed in the hope that it will be useful,
  14 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 *  GNU General Public License for more details.
  17 *
  18 *  You should have received a copy of the GNU General Public License
  19 *  along with this program; see the file COPYING.  If not, write to
  20 *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  21 *
  22 *
  23 *  libata documentation is available via 'make {ps|pdf}docs',
  24 *  as Documentation/DocBook/libata.*
  25 *
  26 *  No hardware documentation available outside of NVIDIA.
  27 *  This driver programs the NVIDIA SATA controller in a similar
  28 *  fashion as with other PCI IDE BMDMA controllers, with a few
  29 *  NV-specific details such as register offsets, SATA phy location,
  30 *  hotplug info, etc.
  31 *
  32 *  CK804/MCP04 controllers support an alternate programming interface
  33 *  similar to the ADMA specification (with some modifications).
  34 *  This allows the use of NCQ. Non-DMA-mapped ATA commands are still
  35 *  sent through the legacy interface.
  36 *
  37 */
  38
  39#include <linux/kernel.h>
  40#include <linux/module.h>
  41#include <linux/gfp.h>
  42#include <linux/pci.h>
  43#include <linux/init.h>
  44#include <linux/blkdev.h>
  45#include <linux/delay.h>
  46#include <linux/interrupt.h>
  47#include <linux/device.h>
  48#include <scsi/scsi_host.h>
  49#include <scsi/scsi_device.h>
  50#include <linux/libata.h>
  51
  52#define DRV_NAME                        "sata_nv"
  53#define DRV_VERSION                     "3.5"
  54
  55#define NV_ADMA_DMA_BOUNDARY            0xffffffffUL
  56
  57enum {
  58        NV_MMIO_BAR                     = 5,
  59
  60        NV_PORTS                        = 2,
  61        NV_PIO_MASK                     = ATA_PIO4,
  62        NV_MWDMA_MASK                   = ATA_MWDMA2,
  63        NV_UDMA_MASK                    = ATA_UDMA6,
  64        NV_PORT0_SCR_REG_OFFSET         = 0x00,
  65        NV_PORT1_SCR_REG_OFFSET         = 0x40,
  66
  67        /* INT_STATUS/ENABLE */
  68        NV_INT_STATUS                   = 0x10,
  69        NV_INT_ENABLE                   = 0x11,
  70        NV_INT_STATUS_CK804             = 0x440,
  71        NV_INT_ENABLE_CK804             = 0x441,
  72
  73        /* INT_STATUS/ENABLE bits */
  74        NV_INT_DEV                      = 0x01,
  75        NV_INT_PM                       = 0x02,
  76        NV_INT_ADDED                    = 0x04,
  77        NV_INT_REMOVED                  = 0x08,
  78
  79        NV_INT_PORT_SHIFT               = 4,    /* each port occupies 4 bits */
  80
  81        NV_INT_ALL                      = 0x0f,
  82        NV_INT_MASK                     = NV_INT_DEV |
  83                                          NV_INT_ADDED | NV_INT_REMOVED,
  84
  85        /* INT_CONFIG */
  86        NV_INT_CONFIG                   = 0x12,
  87        NV_INT_CONFIG_METHD             = 0x01, // 0 = INT, 1 = SMI
  88
  89        // For PCI config register 20
  90        NV_MCP_SATA_CFG_20              = 0x50,
  91        NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
  92        NV_MCP_SATA_CFG_20_PORT0_EN     = (1 << 17),
  93        NV_MCP_SATA_CFG_20_PORT1_EN     = (1 << 16),
  94        NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
  95        NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
  96
  97        NV_ADMA_MAX_CPBS                = 32,
  98        NV_ADMA_CPB_SZ                  = 128,
  99        NV_ADMA_APRD_SZ                 = 16,
 100        NV_ADMA_SGTBL_LEN               = (1024 - NV_ADMA_CPB_SZ) /
 101                                           NV_ADMA_APRD_SZ,
 102        NV_ADMA_SGTBL_TOTAL_LEN         = NV_ADMA_SGTBL_LEN + 5,
 103        NV_ADMA_SGTBL_SZ                = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
 104        NV_ADMA_PORT_PRIV_DMA_SZ        = NV_ADMA_MAX_CPBS *
 105                                           (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
 106
 107        /* BAR5 offset to ADMA general registers */
 108        NV_ADMA_GEN                     = 0x400,
 109        NV_ADMA_GEN_CTL                 = 0x00,
 110        NV_ADMA_NOTIFIER_CLEAR          = 0x30,
 111
 112        /* BAR5 offset to ADMA ports */
 113        NV_ADMA_PORT                    = 0x480,
 114
 115        /* size of ADMA port register space  */
 116        NV_ADMA_PORT_SIZE               = 0x100,
 117
 118        /* ADMA port registers */
 119        NV_ADMA_CTL                     = 0x40,
 120        NV_ADMA_CPB_COUNT               = 0x42,
 121        NV_ADMA_NEXT_CPB_IDX            = 0x43,
 122        NV_ADMA_STAT                    = 0x44,
 123        NV_ADMA_CPB_BASE_LOW            = 0x48,
 124        NV_ADMA_CPB_BASE_HIGH           = 0x4C,
 125        NV_ADMA_APPEND                  = 0x50,
 126        NV_ADMA_NOTIFIER                = 0x68,
 127        NV_ADMA_NOTIFIER_ERROR          = 0x6C,
 128
 129        /* NV_ADMA_CTL register bits */
 130        NV_ADMA_CTL_HOTPLUG_IEN         = (1 << 0),
 131        NV_ADMA_CTL_CHANNEL_RESET       = (1 << 5),
 132        NV_ADMA_CTL_GO                  = (1 << 7),
 133        NV_ADMA_CTL_AIEN                = (1 << 8),
 134        NV_ADMA_CTL_READ_NON_COHERENT   = (1 << 11),
 135        NV_ADMA_CTL_WRITE_NON_COHERENT  = (1 << 12),
 136
 137        /* CPB response flag bits */
 138        NV_CPB_RESP_DONE                = (1 << 0),
 139        NV_CPB_RESP_ATA_ERR             = (1 << 3),
 140        NV_CPB_RESP_CMD_ERR             = (1 << 4),
 141        NV_CPB_RESP_CPB_ERR             = (1 << 7),
 142
 143        /* CPB control flag bits */
 144        NV_CPB_CTL_CPB_VALID            = (1 << 0),
 145        NV_CPB_CTL_QUEUE                = (1 << 1),
 146        NV_CPB_CTL_APRD_VALID           = (1 << 2),
 147        NV_CPB_CTL_IEN                  = (1 << 3),
 148        NV_CPB_CTL_FPDMA                = (1 << 4),
 149
 150        /* APRD flags */
 151        NV_APRD_WRITE                   = (1 << 1),
 152        NV_APRD_END                     = (1 << 2),
 153        NV_APRD_CONT                    = (1 << 3),
 154
 155        /* NV_ADMA_STAT flags */
 156        NV_ADMA_STAT_TIMEOUT            = (1 << 0),
 157        NV_ADMA_STAT_HOTUNPLUG          = (1 << 1),
 158        NV_ADMA_STAT_HOTPLUG            = (1 << 2),
 159        NV_ADMA_STAT_CPBERR             = (1 << 4),
 160        NV_ADMA_STAT_SERROR             = (1 << 5),
 161        NV_ADMA_STAT_CMD_COMPLETE       = (1 << 6),
 162        NV_ADMA_STAT_IDLE               = (1 << 8),
 163        NV_ADMA_STAT_LEGACY             = (1 << 9),
 164        NV_ADMA_STAT_STOPPED            = (1 << 10),
 165        NV_ADMA_STAT_DONE               = (1 << 12),
 166        NV_ADMA_STAT_ERR                = NV_ADMA_STAT_CPBERR |
 167                                          NV_ADMA_STAT_TIMEOUT,
 168
 169        /* port flags */
 170        NV_ADMA_PORT_REGISTER_MODE      = (1 << 0),
 171        NV_ADMA_ATAPI_SETUP_COMPLETE    = (1 << 1),
 172
 173        /* MCP55 reg offset */
 174        NV_CTL_MCP55                    = 0x400,
 175        NV_INT_STATUS_MCP55             = 0x440,
 176        NV_INT_ENABLE_MCP55             = 0x444,
 177        NV_NCQ_REG_MCP55                = 0x448,
 178
 179        /* MCP55 */
 180        NV_INT_ALL_MCP55                = 0xffff,
 181        NV_INT_PORT_SHIFT_MCP55         = 16,   /* each port occupies 16 bits */
 182        NV_INT_MASK_MCP55               = NV_INT_ALL_MCP55 & 0xfffd,
 183
 184        /* SWNCQ ENABLE BITS*/
 185        NV_CTL_PRI_SWNCQ                = 0x02,
 186        NV_CTL_SEC_SWNCQ                = 0x04,
 187
 188        /* SW NCQ status bits*/
 189        NV_SWNCQ_IRQ_DEV                = (1 << 0),
 190        NV_SWNCQ_IRQ_PM                 = (1 << 1),
 191        NV_SWNCQ_IRQ_ADDED              = (1 << 2),
 192        NV_SWNCQ_IRQ_REMOVED            = (1 << 3),
 193
 194        NV_SWNCQ_IRQ_BACKOUT            = (1 << 4),
 195        NV_SWNCQ_IRQ_SDBFIS             = (1 << 5),
 196        NV_SWNCQ_IRQ_DHREGFIS           = (1 << 6),
 197        NV_SWNCQ_IRQ_DMASETUP           = (1 << 7),
 198
 199        NV_SWNCQ_IRQ_HOTPLUG            = NV_SWNCQ_IRQ_ADDED |
 200                                          NV_SWNCQ_IRQ_REMOVED,
 201
 202};
 203
 204/* ADMA Physical Region Descriptor - one SG segment */
 205struct nv_adma_prd {
 206        __le64                  addr;
 207        __le32                  len;
 208        u8                      flags;
 209        u8                      packet_len;
 210        __le16                  reserved;
 211};
 212
 213enum nv_adma_regbits {
 214        CMDEND  = (1 << 15),            /* end of command list */
 215        WNB     = (1 << 14),            /* wait-not-BSY */
 216        IGN     = (1 << 13),            /* ignore this entry */
 217        CS1n    = (1 << (4 + 8)),       /* std. PATA signals follow... */
 218        DA2     = (1 << (2 + 8)),
 219        DA1     = (1 << (1 + 8)),
 220        DA0     = (1 << (0 + 8)),
 221};
 222
 223/* ADMA Command Parameter Block
 224   The first 5 SG segments are stored inside the Command Parameter Block itself.
 225   If there are more than 5 segments the remainder are stored in a separate
 226   memory area indicated by next_aprd. */
 227struct nv_adma_cpb {
 228        u8                      resp_flags;    /* 0 */
 229        u8                      reserved1;     /* 1 */
 230        u8                      ctl_flags;     /* 2 */
 231        /* len is length of taskfile in 64 bit words */
 232        u8                      len;            /* 3  */
 233        u8                      tag;           /* 4 */
 234        u8                      next_cpb_idx;  /* 5 */
 235        __le16                  reserved2;     /* 6-7 */
 236        __le16                  tf[12];        /* 8-31 */
 237        struct nv_adma_prd      aprd[5];       /* 32-111 */
 238        __le64                  next_aprd;     /* 112-119 */
 239        __le64                  reserved3;     /* 120-127 */
 240};
 241
 242
 243struct nv_adma_port_priv {
 244        struct nv_adma_cpb      *cpb;
 245        dma_addr_t              cpb_dma;
 246        struct nv_adma_prd      *aprd;
 247        dma_addr_t              aprd_dma;
 248        void __iomem            *ctl_block;
 249        void __iomem            *gen_block;
 250        void __iomem            *notifier_clear_block;
 251        u64                     adma_dma_mask;
 252        u8                      flags;
 253        int                     last_issue_ncq;
 254};
 255
 256struct nv_host_priv {
 257        unsigned long           type;
 258};
 259
 260struct defer_queue {
 261        u32             defer_bits;
 262        unsigned int    head;
 263        unsigned int    tail;
 264        unsigned int    tag[ATA_MAX_QUEUE];
 265};
 266
 267enum ncq_saw_flag_list {
 268        ncq_saw_d2h     = (1U << 0),
 269        ncq_saw_dmas    = (1U << 1),
 270        ncq_saw_sdb     = (1U << 2),
 271        ncq_saw_backout = (1U << 3),
 272};
 273
 274struct nv_swncq_port_priv {
 275        struct ata_bmdma_prd *prd;       /* our SG list */
 276        dma_addr_t      prd_dma; /* and its DMA mapping */
 277        void __iomem    *sactive_block;
 278        void __iomem    *irq_block;
 279        void __iomem    *tag_block;
 280        u32             qc_active;
 281
 282        unsigned int    last_issue_tag;
 283
 284        /* fifo circular queue to store deferral command */
 285        struct defer_queue defer_queue;
 286
 287        /* for NCQ interrupt analysis */
 288        u32             dhfis_bits;
 289        u32             dmafis_bits;
 290        u32             sdbfis_bits;
 291
 292        unsigned int    ncq_flags;
 293};
 294
 295
 296#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
 297
 298static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
 299#ifdef CONFIG_PM
 300static int nv_pci_device_resume(struct pci_dev *pdev);
 301#endif
 302static void nv_ck804_host_stop(struct ata_host *host);
 303static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
 304static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
 305static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
 306static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
 307static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
 308
 309static int nv_hardreset(struct ata_link *link, unsigned int *class,
 310                        unsigned long deadline);
 311static void nv_nf2_freeze(struct ata_port *ap);
 312static void nv_nf2_thaw(struct ata_port *ap);
 313static void nv_ck804_freeze(struct ata_port *ap);
 314static void nv_ck804_thaw(struct ata_port *ap);
 315static int nv_adma_slave_config(struct scsi_device *sdev);
 316static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
 317static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
 318static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
 319static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
 320static void nv_adma_irq_clear(struct ata_port *ap);
 321static int nv_adma_port_start(struct ata_port *ap);
 322static void nv_adma_port_stop(struct ata_port *ap);
 323#ifdef CONFIG_PM
 324static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
 325static int nv_adma_port_resume(struct ata_port *ap);
 326#endif
 327static void nv_adma_freeze(struct ata_port *ap);
 328static void nv_adma_thaw(struct ata_port *ap);
 329static void nv_adma_error_handler(struct ata_port *ap);
 330static void nv_adma_host_stop(struct ata_host *host);
 331static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
 332static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
 333
 334static void nv_mcp55_thaw(struct ata_port *ap);
 335static void nv_mcp55_freeze(struct ata_port *ap);
 336static void nv_swncq_error_handler(struct ata_port *ap);
 337static int nv_swncq_slave_config(struct scsi_device *sdev);
 338static int nv_swncq_port_start(struct ata_port *ap);
 339static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
 340static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
 341static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
 342static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
 343static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
 344#ifdef CONFIG_PM
 345static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
 346static int nv_swncq_port_resume(struct ata_port *ap);
 347#endif
 348
 349enum nv_host_type
 350{
 351        GENERIC,
 352        NFORCE2,
 353        NFORCE3 = NFORCE2,      /* NF2 == NF3 as far as sata_nv is concerned */
 354        CK804,
 355        ADMA,
 356        MCP5x,
 357        SWNCQ,
 358};
 359
 360static const struct pci_device_id nv_pci_tbl[] = {
 361        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
 362        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
 363        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
 364        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
 365        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
 366        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
 367        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
 368        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x },
 369        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x },
 370        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x },
 371        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x },
 372        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
 373        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
 374        { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
 375
 376        { } /* terminate list */
 377};
 378
 379static struct pci_driver nv_pci_driver = {
 380        .name                   = DRV_NAME,
 381        .id_table               = nv_pci_tbl,
 382        .probe                  = nv_init_one,
 383#ifdef CONFIG_PM
 384        .suspend                = ata_pci_device_suspend,
 385        .resume                 = nv_pci_device_resume,
 386#endif
 387        .remove                 = ata_pci_remove_one,
 388};
 389
 390static struct scsi_host_template nv_sht = {
 391        ATA_BMDMA_SHT(DRV_NAME),
 392};
 393
 394static struct scsi_host_template nv_adma_sht = {
 395        ATA_NCQ_SHT(DRV_NAME),
 396        .can_queue              = NV_ADMA_MAX_CPBS,
 397        .sg_tablesize           = NV_ADMA_SGTBL_TOTAL_LEN,
 398        .dma_boundary           = NV_ADMA_DMA_BOUNDARY,
 399        .slave_configure        = nv_adma_slave_config,
 400};
 401
 402static struct scsi_host_template nv_swncq_sht = {
 403        ATA_NCQ_SHT(DRV_NAME),
 404        .can_queue              = ATA_MAX_QUEUE,
 405        .sg_tablesize           = LIBATA_MAX_PRD,
 406        .dma_boundary           = ATA_DMA_BOUNDARY,
 407        .slave_configure        = nv_swncq_slave_config,
 408};
 409
 410/*
 411 * NV SATA controllers have various different problems with hardreset
 412 * protocol depending on the specific controller and device.
 413 *
 414 * GENERIC:
 415 *
 416 *  bko11195 reports that link doesn't come online after hardreset on
 417 *  generic nv's and there have been several other similar reports on
 418 *  linux-ide.
 419 *
 420 *  bko12351#c23 reports that warmplug on MCP61 doesn't work with
 421 *  softreset.
 422 *
 423 * NF2/3:
 424 *
 425 *  bko3352 reports nf2/3 controllers can't determine device signature
 426 *  reliably after hardreset.  The following thread reports detection
 427 *  failure on cold boot with the standard debouncing timing.
 428 *
 429 *  http://thread.gmane.org/gmane.linux.ide/34098
 430 *
 431 *  bko12176 reports that hardreset fails to bring up the link during
 432 *  boot on nf2.
 433 *
 434 * CK804:
 435 *
 436 *  For initial probing after boot and hot plugging, hardreset mostly
 437 *  works fine on CK804 but curiously, reprobing on the initial port
 438 *  by rescanning or rmmod/insmod fails to acquire the initial D2H Reg
 439 *  FIS in somewhat undeterministic way.
 440 *
 441 * SWNCQ:
 442 *
 443 *  bko12351 reports that when SWNCQ is enabled, for hotplug to work,
 444 *  hardreset should be used and hardreset can't report proper
 445 *  signature, which suggests that mcp5x is closer to nf2 as long as
 446 *  reset quirkiness is concerned.
 447 *
 448 *  bko12703 reports that boot probing fails for intel SSD with
 449 *  hardreset.  Link fails to come online.  Softreset works fine.
 450 *
 451 * The failures are varied but the following patterns seem true for
 452 * all flavors.
 453 *
 454 * - Softreset during boot always works.
 455 *
 456 * - Hardreset during boot sometimes fails to bring up the link on
 457 *   certain comibnations and device signature acquisition is
 458 *   unreliable.
 459 *
 460 * - Hardreset is often necessary after hotplug.
 461 *
 462 * So, preferring softreset for boot probing and error handling (as
 463 * hardreset might bring down the link) but using hardreset for
 464 * post-boot probing should work around the above issues in most
 465 * cases.  Define nv_hardreset() which only kicks in for post-boot
 466 * probing and use it for all variants.
 467 */
 468static struct ata_port_operations nv_generic_ops = {
 469        .inherits               = &ata_bmdma_port_ops,
 470        .lost_interrupt         = ATA_OP_NULL,
 471        .scr_read               = nv_scr_read,
 472        .scr_write              = nv_scr_write,
 473        .hardreset              = nv_hardreset,
 474};
 475
 476static struct ata_port_operations nv_nf2_ops = {
 477        .inherits               = &nv_generic_ops,
 478        .freeze                 = nv_nf2_freeze,
 479        .thaw                   = nv_nf2_thaw,
 480};
 481
 482static struct ata_port_operations nv_ck804_ops = {
 483        .inherits               = &nv_generic_ops,
 484        .freeze                 = nv_ck804_freeze,
 485        .thaw                   = nv_ck804_thaw,
 486        .host_stop              = nv_ck804_host_stop,
 487};
 488
 489static struct ata_port_operations nv_adma_ops = {
 490        .inherits               = &nv_ck804_ops,
 491
 492        .check_atapi_dma        = nv_adma_check_atapi_dma,
 493        .sff_tf_read            = nv_adma_tf_read,
 494        .qc_defer               = ata_std_qc_defer,
 495        .qc_prep                = nv_adma_qc_prep,
 496        .qc_issue               = nv_adma_qc_issue,
 497        .sff_irq_clear          = nv_adma_irq_clear,
 498
 499        .freeze                 = nv_adma_freeze,
 500        .thaw                   = nv_adma_thaw,
 501        .error_handler          = nv_adma_error_handler,
 502        .post_internal_cmd      = nv_adma_post_internal_cmd,
 503
 504        .port_start             = nv_adma_port_start,
 505        .port_stop              = nv_adma_port_stop,
 506#ifdef CONFIG_PM
 507        .port_suspend           = nv_adma_port_suspend,
 508        .port_resume            = nv_adma_port_resume,
 509#endif
 510        .host_stop              = nv_adma_host_stop,
 511};
 512
 513static struct ata_port_operations nv_swncq_ops = {
 514        .inherits               = &nv_generic_ops,
 515
 516        .qc_defer               = ata_std_qc_defer,
 517        .qc_prep                = nv_swncq_qc_prep,
 518        .qc_issue               = nv_swncq_qc_issue,
 519
 520        .freeze                 = nv_mcp55_freeze,
 521        .thaw                   = nv_mcp55_thaw,
 522        .error_handler          = nv_swncq_error_handler,
 523
 524#ifdef CONFIG_PM
 525        .port_suspend           = nv_swncq_port_suspend,
 526        .port_resume            = nv_swncq_port_resume,
 527#endif
 528        .port_start             = nv_swncq_port_start,
 529};
 530
 531struct nv_pi_priv {
 532        irq_handler_t                   irq_handler;
 533        struct scsi_host_template       *sht;
 534};
 535
 536#define NV_PI_PRIV(_irq_handler, _sht) \
 537        &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
 538
 539static const struct ata_port_info nv_port_info[] = {
 540        /* generic */
 541        {
 542                .flags          = ATA_FLAG_SATA,
 543                .pio_mask       = NV_PIO_MASK,
 544                .mwdma_mask     = NV_MWDMA_MASK,
 545                .udma_mask      = NV_UDMA_MASK,
 546                .port_ops       = &nv_generic_ops,
 547                .private_data   = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
 548        },
 549        /* nforce2/3 */
 550        {
 551                .flags          = ATA_FLAG_SATA,
 552                .pio_mask       = NV_PIO_MASK,
 553                .mwdma_mask     = NV_MWDMA_MASK,
 554                .udma_mask      = NV_UDMA_MASK,
 555                .port_ops       = &nv_nf2_ops,
 556                .private_data   = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
 557        },
 558        /* ck804 */
 559        {
 560                .flags          = ATA_FLAG_SATA,
 561                .pio_mask       = NV_PIO_MASK,
 562                .mwdma_mask     = NV_MWDMA_MASK,
 563                .udma_mask      = NV_UDMA_MASK,
 564                .port_ops       = &nv_ck804_ops,
 565                .private_data   = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
 566        },
 567        /* ADMA */
 568        {
 569                .flags          = ATA_FLAG_SATA | ATA_FLAG_NCQ,
 570                .pio_mask       = NV_PIO_MASK,
 571                .mwdma_mask     = NV_MWDMA_MASK,
 572                .udma_mask      = NV_UDMA_MASK,
 573                .port_ops       = &nv_adma_ops,
 574                .private_data   = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
 575        },
 576        /* MCP5x */
 577        {
 578                .flags          = ATA_FLAG_SATA,
 579                .pio_mask       = NV_PIO_MASK,
 580                .mwdma_mask     = NV_MWDMA_MASK,
 581                .udma_mask      = NV_UDMA_MASK,
 582                .port_ops       = &nv_generic_ops,
 583                .private_data   = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
 584        },
 585        /* SWNCQ */
 586        {
 587                .flags          = ATA_FLAG_SATA | ATA_FLAG_NCQ,
 588                .pio_mask       = NV_PIO_MASK,
 589                .mwdma_mask     = NV_MWDMA_MASK,
 590                .udma_mask      = NV_UDMA_MASK,
 591                .port_ops       = &nv_swncq_ops,
 592                .private_data   = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
 593        },
 594};
 595
 596MODULE_AUTHOR("NVIDIA");
 597MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
 598MODULE_LICENSE("GPL");
 599MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
 600MODULE_VERSION(DRV_VERSION);
 601
 602static bool adma_enabled;
 603static bool swncq_enabled = 1;
 604static bool msi_enabled;
 605
 606static void nv_adma_register_mode(struct ata_port *ap)
 607{
 608        struct nv_adma_port_priv *pp = ap->private_data;
 609        void __iomem *mmio = pp->ctl_block;
 610        u16 tmp, status;
 611        int count = 0;
 612
 613        if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
 614                return;
 615
 616        status = readw(mmio + NV_ADMA_STAT);
 617        while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
 618                ndelay(50);
 619                status = readw(mmio + NV_ADMA_STAT);
 620                count++;
 621        }
 622        if (count == 20)
 623                ata_port_warn(ap, "timeout waiting for ADMA IDLE, stat=0x%hx\n",
 624                              status);
 625
 626        tmp = readw(mmio + NV_ADMA_CTL);
 627        writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
 628
 629        count = 0;
 630        status = readw(mmio + NV_ADMA_STAT);
 631        while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
 632                ndelay(50);
 633                status = readw(mmio + NV_ADMA_STAT);
 634                count++;
 635        }
 636        if (count == 20)
 637                ata_port_warn(ap,
 638                              "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
 639                              status);
 640
 641        pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
 642}
 643
 644static void nv_adma_mode(struct ata_port *ap)
 645{
 646        struct nv_adma_port_priv *pp = ap->private_data;
 647        void __iomem *mmio = pp->ctl_block;
 648        u16 tmp, status;
 649        int count = 0;
 650
 651        if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
 652                return;
 653
 654        WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
 655
 656        tmp = readw(mmio + NV_ADMA_CTL);
 657        writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
 658
 659        status = readw(mmio + NV_ADMA_STAT);
 660        while (((status & NV_ADMA_STAT_LEGACY) ||
 661              !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
 662                ndelay(50);
 663                status = readw(mmio + NV_ADMA_STAT);
 664                count++;
 665        }
 666        if (count == 20)
 667                ata_port_warn(ap,
 668                        "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
 669                        status);
 670
 671        pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
 672}
 673
 674static int nv_adma_slave_config(struct scsi_device *sdev)
 675{
 676        struct ata_port *ap = ata_shost_to_port(sdev->host);
 677        struct nv_adma_port_priv *pp = ap->private_data;
 678        struct nv_adma_port_priv *port0, *port1;
 679        struct scsi_device *sdev0, *sdev1;
 680        struct pci_dev *pdev = to_pci_dev(ap->host->dev);
 681        unsigned long segment_boundary, flags;
 682        unsigned short sg_tablesize;
 683        int rc;
 684        int adma_enable;
 685        u32 current_reg, new_reg, config_mask;
 686
 687        rc = ata_scsi_slave_config(sdev);
 688
 689        if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
 690                /* Not a proper libata device, ignore */
 691                return rc;
 692
 693        spin_lock_irqsave(ap->lock, flags);
 694
 695        if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
 696                /*
 697                 * NVIDIA reports that ADMA mode does not support ATAPI commands.
 698                 * Therefore ATAPI commands are sent through the legacy interface.
 699                 * However, the legacy interface only supports 32-bit DMA.
 700                 * Restrict DMA parameters as required by the legacy interface
 701                 * when an ATAPI device is connected.
 702                 */
 703                segment_boundary = ATA_DMA_BOUNDARY;
 704                /* Subtract 1 since an extra entry may be needed for padding, see
 705                   libata-scsi.c */
 706                sg_tablesize = LIBATA_MAX_PRD - 1;
 707
 708                /* Since the legacy DMA engine is in use, we need to disable ADMA
 709                   on the port. */
 710                adma_enable = 0;
 711                nv_adma_register_mode(ap);
 712        } else {
 713                segment_boundary = NV_ADMA_DMA_BOUNDARY;
 714                sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
 715                adma_enable = 1;
 716        }
 717
 718        pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
 719
 720        if (ap->port_no == 1)
 721                config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
 722                              NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
 723        else
 724                config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
 725                              NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
 726
 727        if (adma_enable) {
 728                new_reg = current_reg | config_mask;
 729                pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
 730        } else {
 731                new_reg = current_reg & ~config_mask;
 732                pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
 733        }
 734
 735        if (current_reg != new_reg)
 736                pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
 737
 738        port0 = ap->host->ports[0]->private_data;
 739        port1 = ap->host->ports[1]->private_data;
 740        sdev0 = ap->host->ports[0]->link.device[0].sdev;
 741        sdev1 = ap->host->ports[1]->link.device[0].sdev;
 742        if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
 743            (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
 744                /** We have to set the DMA mask to 32-bit if either port is in
 745                    ATAPI mode, since they are on the same PCI device which is
 746                    used for DMA mapping. If we set the mask we also need to set
 747                    the bounce limit on both ports to ensure that the block
 748                    layer doesn't feed addresses that cause DMA mapping to
 749                    choke. If either SCSI device is not allocated yet, it's OK
 750                    since that port will discover its correct setting when it
 751                    does get allocated.
 752                    Note: Setting 32-bit mask should not fail. */
 753                if (sdev0)
 754                        blk_queue_bounce_limit(sdev0->request_queue,
 755                                               ATA_DMA_MASK);
 756                if (sdev1)
 757                        blk_queue_bounce_limit(sdev1->request_queue,
 758                                               ATA_DMA_MASK);
 759
 760                pci_set_dma_mask(pdev, ATA_DMA_MASK);
 761        } else {
 762                /** This shouldn't fail as it was set to this value before */
 763                pci_set_dma_mask(pdev, pp->adma_dma_mask);
 764                if (sdev0)
 765                        blk_queue_bounce_limit(sdev0->request_queue,
 766                                               pp->adma_dma_mask);
 767                if (sdev1)
 768                        blk_queue_bounce_limit(sdev1->request_queue,
 769                                               pp->adma_dma_mask);
 770        }
 771
 772        blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
 773        blk_queue_max_segments(sdev->request_queue, sg_tablesize);
 774        ata_port_info(ap,
 775                      "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
 776                      (unsigned long long)*ap->host->dev->dma_mask,
 777                      segment_boundary, sg_tablesize);
 778
 779        spin_unlock_irqrestore(ap->lock, flags);
 780
 781        return rc;
 782}
 783
 784static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
 785{
 786        struct nv_adma_port_priv *pp = qc->ap->private_data;
 787        return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
 788}
 789
 790static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
 791{
 792        /* Other than when internal or pass-through commands are executed,
 793           the only time this function will be called in ADMA mode will be
 794           if a command fails. In the failure case we don't care about going
 795           into register mode with ADMA commands pending, as the commands will
 796           all shortly be aborted anyway. We assume that NCQ commands are not
 797           issued via passthrough, which is the only way that switching into
 798           ADMA mode could abort outstanding commands. */
 799        nv_adma_register_mode(ap);
 800
 801        ata_sff_tf_read(ap, tf);
 802}
 803
 804static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
 805{
 806        unsigned int idx = 0;
 807
 808        if (tf->flags & ATA_TFLAG_ISADDR) {
 809                if (tf->flags & ATA_TFLAG_LBA48) {
 810                        cpb[idx++] = cpu_to_le16((ATA_REG_ERR   << 8) | tf->hob_feature | WNB);
 811                        cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
 812                        cpb[idx++] = cpu_to_le16((ATA_REG_LBAL  << 8) | tf->hob_lbal);
 813                        cpb[idx++] = cpu_to_le16((ATA_REG_LBAM  << 8) | tf->hob_lbam);
 814                        cpb[idx++] = cpu_to_le16((ATA_REG_LBAH  << 8) | tf->hob_lbah);
 815                        cpb[idx++] = cpu_to_le16((ATA_REG_ERR    << 8) | tf->feature);
 816                } else
 817                        cpb[idx++] = cpu_to_le16((ATA_REG_ERR    << 8) | tf->feature | WNB);
 818
 819                cpb[idx++] = cpu_to_le16((ATA_REG_NSECT  << 8) | tf->nsect);
 820                cpb[idx++] = cpu_to_le16((ATA_REG_LBAL   << 8) | tf->lbal);
 821                cpb[idx++] = cpu_to_le16((ATA_REG_LBAM   << 8) | tf->lbam);
 822                cpb[idx++] = cpu_to_le16((ATA_REG_LBAH   << 8) | tf->lbah);
 823        }
 824
 825        if (tf->flags & ATA_TFLAG_DEVICE)
 826                cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
 827
 828        cpb[idx++] = cpu_to_le16((ATA_REG_CMD    << 8) | tf->command | CMDEND);
 829
 830        while (idx < 12)
 831                cpb[idx++] = cpu_to_le16(IGN);
 832
 833        return idx;
 834}
 835
 836static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
 837{
 838        struct nv_adma_port_priv *pp = ap->private_data;
 839        u8 flags = pp->cpb[cpb_num].resp_flags;
 840
 841        VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
 842
 843        if (unlikely((force_err ||
 844                     flags & (NV_CPB_RESP_ATA_ERR |
 845                              NV_CPB_RESP_CMD_ERR |
 846                              NV_CPB_RESP_CPB_ERR)))) {
 847                struct ata_eh_info *ehi = &ap->link.eh_info;
 848                int freeze = 0;
 849
 850                ata_ehi_clear_desc(ehi);
 851                __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
 852                if (flags & NV_CPB_RESP_ATA_ERR) {
 853                        ata_ehi_push_desc(ehi, "ATA error");
 854                        ehi->err_mask |= AC_ERR_DEV;
 855                } else if (flags & NV_CPB_RESP_CMD_ERR) {
 856                        ata_ehi_push_desc(ehi, "CMD error");
 857                        ehi->err_mask |= AC_ERR_DEV;
 858                } else if (flags & NV_CPB_RESP_CPB_ERR) {
 859                        ata_ehi_push_desc(ehi, "CPB error");
 860                        ehi->err_mask |= AC_ERR_SYSTEM;
 861                        freeze = 1;
 862                } else {
 863                        /* notifier error, but no error in CPB flags? */
 864                        ata_ehi_push_desc(ehi, "unknown");
 865                        ehi->err_mask |= AC_ERR_OTHER;
 866                        freeze = 1;
 867                }
 868                /* Kill all commands. EH will determine what actually failed. */
 869                if (freeze)
 870                        ata_port_freeze(ap);
 871                else
 872                        ata_port_abort(ap);
 873                return -1;
 874        }
 875
 876        if (likely(flags & NV_CPB_RESP_DONE))
 877                return 1;
 878        return 0;
 879}
 880
 881static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
 882{
 883        struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
 884
 885        /* freeze if hotplugged */
 886        if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
 887                ata_port_freeze(ap);
 888                return 1;
 889        }
 890
 891        /* bail out if not our interrupt */
 892        if (!(irq_stat & NV_INT_DEV))
 893                return 0;
 894
 895        /* DEV interrupt w/ no active qc? */
 896        if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
 897                ata_sff_check_status(ap);
 898                return 1;
 899        }
 900
 901        /* handle interrupt */
 902        return ata_bmdma_port_intr(ap, qc);
 903}
 904
 905static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
 906{
 907        struct ata_host *host = dev_instance;
 908        int i, handled = 0;
 909        u32 notifier_clears[2];
 910
 911        spin_lock(&host->lock);
 912
 913        for (i = 0; i < host->n_ports; i++) {
 914                struct ata_port *ap = host->ports[i];
 915                struct nv_adma_port_priv *pp = ap->private_data;
 916                void __iomem *mmio = pp->ctl_block;
 917                u16 status;
 918                u32 gen_ctl;
 919                u32 notifier, notifier_error;
 920
 921                notifier_clears[i] = 0;
 922
 923                /* if ADMA is disabled, use standard ata interrupt handler */
 924                if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
 925                        u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
 926                                >> (NV_INT_PORT_SHIFT * i);
 927                        handled += nv_host_intr(ap, irq_stat);
 928                        continue;
 929                }
 930
 931                /* if in ATA register mode, check for standard interrupts */
 932                if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
 933                        u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
 934                                >> (NV_INT_PORT_SHIFT * i);
 935                        if (ata_tag_valid(ap->link.active_tag))
 936                                /** NV_INT_DEV indication seems unreliable
 937                                    at times at least in ADMA mode. Force it
 938                                    on always when a command is active, to
 939                                    prevent losing interrupts. */
 940                                irq_stat |= NV_INT_DEV;
 941                        handled += nv_host_intr(ap, irq_stat);
 942                }
 943
 944                notifier = readl(mmio + NV_ADMA_NOTIFIER);
 945                notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
 946                notifier_clears[i] = notifier | notifier_error;
 947
 948                gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
 949
 950                if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
 951                    !notifier_error)
 952                        /* Nothing to do */
 953                        continue;
 954
 955                status = readw(mmio + NV_ADMA_STAT);
 956
 957                /*
 958                 * Clear status. Ensure the controller sees the
 959                 * clearing before we start looking at any of the CPB
 960                 * statuses, so that any CPB completions after this
 961                 * point in the handler will raise another interrupt.
 962                 */
 963                writew(status, mmio + NV_ADMA_STAT);
 964                readw(mmio + NV_ADMA_STAT); /* flush posted write */
 965                rmb();
 966
 967                handled++; /* irq handled if we got here */
 968
 969                /* freeze if hotplugged or controller error */
 970                if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
 971                                       NV_ADMA_STAT_HOTUNPLUG |
 972                                       NV_ADMA_STAT_TIMEOUT |
 973                                       NV_ADMA_STAT_SERROR))) {
 974                        struct ata_eh_info *ehi = &ap->link.eh_info;
 975
 976                        ata_ehi_clear_desc(ehi);
 977                        __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
 978                        if (status & NV_ADMA_STAT_TIMEOUT) {
 979                                ehi->err_mask |= AC_ERR_SYSTEM;
 980                                ata_ehi_push_desc(ehi, "timeout");
 981                        } else if (status & NV_ADMA_STAT_HOTPLUG) {
 982                                ata_ehi_hotplugged(ehi);
 983                                ata_ehi_push_desc(ehi, "hotplug");
 984                        } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
 985                                ata_ehi_hotplugged(ehi);
 986                                ata_ehi_push_desc(ehi, "hot unplug");
 987                        } else if (status & NV_ADMA_STAT_SERROR) {
 988                                /* let EH analyze SError and figure out cause */
 989                                ata_ehi_push_desc(ehi, "SError");
 990                        } else
 991                                ata_ehi_push_desc(ehi, "unknown");
 992                        ata_port_freeze(ap);
 993                        continue;
 994                }
 995
 996                if (status & (NV_ADMA_STAT_DONE |
 997                              NV_ADMA_STAT_CPBERR |
 998                              NV_ADMA_STAT_CMD_COMPLETE)) {
 999                        u32 check_commands = notifier_clears[i];
1000                        u32 done_mask = 0;
1001                        int pos, rc;
1002
1003                        if (status & NV_ADMA_STAT_CPBERR) {
1004                                /* check all active commands */
1005                                if (ata_tag_valid(ap->link.active_tag))
1006                                        check_commands = 1 <<
1007                                                ap->link.active_tag;
1008                                else
1009                                        check_commands = ap->link.sactive;
1010                        }
1011
1012                        /* check CPBs for completed commands */
1013                        while ((pos = ffs(check_commands))) {
1014                                pos--;
1015                                rc = nv_adma_check_cpb(ap, pos,
1016                                                notifier_error & (1 << pos));
1017                                if (rc > 0)
1018                                        done_mask |= 1 << pos;
1019                                else if (unlikely(rc < 0))
1020                                        check_commands = 0;
1021                                check_commands &= ~(1 << pos);
1022                        }
1023                        ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
1024                }
1025        }
1026
1027        if (notifier_clears[0] || notifier_clears[1]) {
1028                /* Note: Both notifier clear registers must be written
1029                   if either is set, even if one is zero, according to NVIDIA. */
1030                struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1031                writel(notifier_clears[0], pp->notifier_clear_block);
1032                pp = host->ports[1]->private_data;
1033                writel(notifier_clears[1], pp->notifier_clear_block);
1034        }
1035
1036        spin_unlock(&host->lock);
1037
1038        return IRQ_RETVAL(handled);
1039}
1040
1041static void nv_adma_freeze(struct ata_port *ap)
1042{
1043        struct nv_adma_port_priv *pp = ap->private_data;
1044        void __iomem *mmio = pp->ctl_block;
1045        u16 tmp;
1046
1047        nv_ck804_freeze(ap);
1048
1049        if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1050                return;
1051
1052        /* clear any outstanding CK804 notifications */
1053        writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1054                ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1055
1056        /* Disable interrupt */
1057        tmp = readw(mmio + NV_ADMA_CTL);
1058        writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1059                mmio + NV_ADMA_CTL);
1060        readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1061}
1062
1063static void nv_adma_thaw(struct ata_port *ap)
1064{
1065        struct nv_adma_port_priv *pp = ap->private_data;
1066        void __iomem *mmio = pp->ctl_block;
1067        u16 tmp;
1068
1069        nv_ck804_thaw(ap);
1070
1071        if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1072                return;
1073
1074        /* Enable interrupt */
1075        tmp = readw(mmio + NV_ADMA_CTL);
1076        writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1077                mmio + NV_ADMA_CTL);
1078        readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1079}
1080
1081static void nv_adma_irq_clear(struct ata_port *ap)
1082{
1083        struct nv_adma_port_priv *pp = ap->private_data;
1084        void __iomem *mmio = pp->ctl_block;
1085        u32 notifier_clears[2];
1086
1087        if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1088                ata_bmdma_irq_clear(ap);
1089                return;
1090        }
1091
1092        /* clear any outstanding CK804 notifications */
1093        writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1094                ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1095
1096        /* clear ADMA status */
1097        writew(0xffff, mmio + NV_ADMA_STAT);
1098
1099        /* clear notifiers - note both ports need to be written with
1100           something even though we are only clearing on one */
1101        if (ap->port_no == 0) {
1102                notifier_clears[0] = 0xFFFFFFFF;
1103                notifier_clears[1] = 0;
1104        } else {
1105                notifier_clears[0] = 0;
1106                notifier_clears[1] = 0xFFFFFFFF;
1107        }
1108        pp = ap->host->ports[0]->private_data;
1109        writel(notifier_clears[0], pp->notifier_clear_block);
1110        pp = ap->host->ports[1]->private_data;
1111        writel(notifier_clears[1], pp->notifier_clear_block);
1112}
1113
1114static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
1115{
1116        struct nv_adma_port_priv *pp = qc->ap->private_data;
1117
1118        if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
1119                ata_bmdma_post_internal_cmd(qc);
1120}
1121
1122static int nv_adma_port_start(struct ata_port *ap)
1123{
1124        struct device *dev = ap->host->dev;
1125        struct nv_adma_port_priv *pp;
1126        int rc;
1127        void *mem;
1128        dma_addr_t mem_dma;
1129        void __iomem *mmio;
1130        struct pci_dev *pdev = to_pci_dev(dev);
1131        u16 tmp;
1132
1133        VPRINTK("ENTER\n");
1134
1135        /* Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1136           pad buffers */
1137        rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1138        if (rc)
1139                return rc;
1140        rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1141        if (rc)
1142                return rc;
1143
1144        /* we might fallback to bmdma, allocate bmdma resources */
1145        rc = ata_bmdma_port_start(ap);
1146        if (rc)
1147                return rc;
1148
1149        pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1150        if (!pp)
1151                return -ENOMEM;
1152
1153        mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
1154               ap->port_no * NV_ADMA_PORT_SIZE;
1155        pp->ctl_block = mmio;
1156        pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
1157        pp->notifier_clear_block = pp->gen_block +
1158               NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1159
1160        /* Now that the legacy PRD and padding buffer are allocated we can
1161           safely raise the DMA mask to allocate the CPB/APRD table.
1162           These are allowed to fail since we store the value that ends up
1163           being used to set as the bounce limit in slave_config later if
1164           needed. */
1165        pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1166        pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1167        pp->adma_dma_mask = *dev->dma_mask;
1168
1169        mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1170                                  &mem_dma, GFP_KERNEL);
1171        if (!mem)
1172                return -ENOMEM;
1173        memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1174
1175        /*
1176         * First item in chunk of DMA memory:
1177         * 128-byte command parameter block (CPB)
1178         * one for each command tag
1179         */
1180        pp->cpb     = mem;
1181        pp->cpb_dma = mem_dma;
1182
1183        writel(mem_dma & 0xFFFFFFFF,    mmio + NV_ADMA_CPB_BASE_LOW);
1184        writel((mem_dma >> 16) >> 16,   mmio + NV_ADMA_CPB_BASE_HIGH);
1185
1186        mem     += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1187        mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1188
1189        /*
1190         * Second item: block of ADMA_SGTBL_LEN s/g entries
1191         */
1192        pp->aprd = mem;
1193        pp->aprd_dma = mem_dma;
1194
1195        ap->private_data = pp;
1196
1197        /* clear any outstanding interrupt conditions */
1198        writew(0xffff, mmio + NV_ADMA_STAT);
1199
1200        /* initialize port variables */
1201        pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1202
1203        /* clear CPB fetch count */
1204        writew(0, mmio + NV_ADMA_CPB_COUNT);
1205
1206        /* clear GO for register mode, enable interrupt */
1207        tmp = readw(mmio + NV_ADMA_CTL);
1208        writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1209                NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1210
1211        tmp = readw(mmio + NV_ADMA_CTL);
1212        writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1213        readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1214        udelay(1);
1215        writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1216        readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1217
1218        return 0;
1219}
1220
1221static void nv_adma_port_stop(struct ata_port *ap)
1222{
1223        struct nv_adma_port_priv *pp = ap->private_data;
1224        void __iomem *mmio = pp->ctl_block;
1225
1226        VPRINTK("ENTER\n");
1227        writew(0, mmio + NV_ADMA_CTL);
1228}
1229
1230#ifdef CONFIG_PM
1231static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1232{
1233        struct nv_adma_port_priv *pp = ap->private_data;
1234        void __iomem *mmio = pp->ctl_block;
1235
1236        /* Go to register mode - clears GO */
1237        nv_adma_register_mode(ap);
1238
1239        /* clear CPB fetch count */
1240        writew(0, mmio + NV_ADMA_CPB_COUNT);
1241
1242        /* disable interrupt, shut down port */
1243        writew(0, mmio + NV_ADMA_CTL);
1244
1245        return 0;
1246}
1247
1248static int nv_adma_port_resume(struct ata_port *ap)
1249{
1250        struct nv_adma_port_priv *pp = ap->private_data;
1251        void __iomem *mmio = pp->ctl_block;
1252        u16 tmp;
1253
1254        /* set CPB block location */
1255        writel(pp->cpb_dma & 0xFFFFFFFF,        mmio + NV_ADMA_CPB_BASE_LOW);
1256        writel((pp->cpb_dma >> 16) >> 16,       mmio + NV_ADMA_CPB_BASE_HIGH);
1257
1258        /* clear any outstanding interrupt conditions */
1259        writew(0xffff, mmio + NV_ADMA_STAT);
1260
1261        /* initialize port variables */
1262        pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1263
1264        /* clear CPB fetch count */
1265        writew(0, mmio + NV_ADMA_CPB_COUNT);
1266
1267        /* clear GO for register mode, enable interrupt */
1268        tmp = readw(mmio + NV_ADMA_CTL);
1269        writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1270                NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1271
1272        tmp = readw(mmio + NV_ADMA_CTL);
1273        writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1274        readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1275        udelay(1);
1276        writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1277        readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1278
1279        return 0;
1280}
1281#endif
1282
1283static void nv_adma_setup_port(struct ata_port *ap)
1284{
1285        void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1286        struct ata_ioports *ioport = &ap->ioaddr;
1287
1288        VPRINTK("ENTER\n");
1289
1290        mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
1291
1292        ioport->cmd_addr        = mmio;
1293        ioport->data_addr       = mmio + (ATA_REG_DATA * 4);
1294        ioport->error_addr      =
1295        ioport->feature_addr    = mmio + (ATA_REG_ERR * 4);
1296        ioport->nsect_addr      = mmio + (ATA_REG_NSECT * 4);
1297        ioport->lbal_addr       = mmio + (ATA_REG_LBAL * 4);
1298        ioport->lbam_addr       = mmio + (ATA_REG_LBAM * 4);
1299        ioport->lbah_addr       = mmio + (ATA_REG_LBAH * 4);
1300        ioport->device_addr     = mmio + (ATA_REG_DEVICE * 4);
1301        ioport->status_addr     =
1302        ioport->command_addr    = mmio + (ATA_REG_STATUS * 4);
1303        ioport->altstatus_addr  =
1304        ioport->ctl_addr        = mmio + 0x20;
1305}
1306
1307static int nv_adma_host_init(struct ata_host *host)
1308{
1309        struct pci_dev *pdev = to_pci_dev(host->dev);
1310        unsigned int i;
1311        u32 tmp32;
1312
1313        VPRINTK("ENTER\n");
1314
1315        /* enable ADMA on the ports */
1316        pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1317        tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1318                 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1319                 NV_MCP_SATA_CFG_20_PORT1_EN |
1320                 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1321
1322        pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1323
1324        for (i = 0; i < host->n_ports; i++)
1325                nv_adma_setup_port(host->ports[i]);
1326
1327        return 0;
1328}
1329
1330static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1331                              struct scatterlist *sg,
1332                              int idx,
1333                              struct nv_adma_prd *aprd)
1334{
1335        u8 flags = 0;
1336        if (qc->tf.flags & ATA_TFLAG_WRITE)
1337                flags |= NV_APRD_WRITE;
1338        if (idx == qc->n_elem - 1)
1339                flags |= NV_APRD_END;
1340        else if (idx != 4)
1341                flags |= NV_APRD_CONT;
1342
1343        aprd->addr  = cpu_to_le64(((u64)sg_dma_address(sg)));
1344        aprd->len   = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1345        aprd->flags = flags;
1346        aprd->packet_len = 0;
1347}
1348
1349static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1350{
1351        struct nv_adma_port_priv *pp = qc->ap->private_data;
1352        struct nv_adma_prd *aprd;
1353        struct scatterlist *sg;
1354        unsigned int si;
1355
1356        VPRINTK("ENTER\n");
1357
1358        for_each_sg(qc->sg, sg, qc->n_elem, si) {
1359                aprd = (si < 5) ? &cpb->aprd[si] :
1360                               &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1361                nv_adma_fill_aprd(qc, sg, si, aprd);
1362        }
1363        if (si > 5)
1364                cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
1365        else
1366                cpb->next_aprd = cpu_to_le64(0);
1367}
1368
1369static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1370{
1371        struct nv_adma_port_priv *pp = qc->ap->private_data;
1372
1373        /* ADMA engine can only be used for non-ATAPI DMA commands,
1374           or interrupt-driven no-data commands. */
1375        if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
1376           (qc->tf.flags & ATA_TFLAG_POLLING))
1377                return 1;
1378
1379        if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
1380           (qc->tf.protocol == ATA_PROT_NODATA))
1381                return 0;
1382
1383        return 1;
1384}
1385
1386static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1387{
1388        struct nv_adma_port_priv *pp = qc->ap->private_data;
1389        struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1390        u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1391                       NV_CPB_CTL_IEN;
1392
1393        if (nv_adma_use_reg_mode(qc)) {
1394                BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1395                        (qc->flags & ATA_QCFLAG_DMAMAP));
1396                nv_adma_register_mode(qc->ap);
1397                ata_bmdma_qc_prep(qc);
1398                return;
1399        }
1400
1401        cpb->resp_flags = NV_CPB_RESP_DONE;
1402        wmb();
1403        cpb->ctl_flags = 0;
1404        wmb();
1405
1406        cpb->len                = 3;
1407        cpb->tag                = qc->tag;
1408        cpb->next_cpb_idx       = 0;
1409
1410        /* turn on NCQ flags for NCQ commands */
1411        if (qc->tf.protocol == ATA_PROT_NCQ)
1412                ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1413
1414        VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1415
1416        nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1417
1418        if (qc->flags & ATA_QCFLAG_DMAMAP) {
1419                nv_adma_fill_sg(qc, cpb);
1420                ctl_flags |= NV_CPB_CTL_APRD_VALID;
1421        } else
1422                memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
1423
1424        /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1425           until we are finished filling in all of the contents */
1426        wmb();
1427        cpb->ctl_flags = ctl_flags;
1428        wmb();
1429        cpb->resp_flags = 0;
1430}
1431
1432static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1433{
1434        struct nv_adma_port_priv *pp = qc->ap->private_data;
1435        void __iomem *mmio = pp->ctl_block;
1436        int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
1437
1438        VPRINTK("ENTER\n");
1439
1440        /* We can't handle result taskfile with NCQ commands, since
1441           retrieving the taskfile switches us out of ADMA mode and would abort
1442           existing commands. */
1443        if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1444                     (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1445                ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n");
1446                return AC_ERR_SYSTEM;
1447        }
1448
1449        if (nv_adma_use_reg_mode(qc)) {
1450                /* use ATA register mode */
1451                VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
1452                BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1453                        (qc->flags & ATA_QCFLAG_DMAMAP));
1454                nv_adma_register_mode(qc->ap);
1455                return ata_bmdma_qc_issue(qc);
1456        } else
1457                nv_adma_mode(qc->ap);
1458
1459        /* write append register, command tag in lower 8 bits
1460           and (number of cpbs to append -1) in top 8 bits */
1461        wmb();
1462
1463        if (curr_ncq != pp->last_issue_ncq) {
1464                /* Seems to need some delay before switching between NCQ and
1465                   non-NCQ commands, else we get command timeouts and such. */
1466                udelay(20);
1467                pp->last_issue_ncq = curr_ncq;
1468        }
1469
1470        writew(qc->tag, mmio + NV_ADMA_APPEND);
1471
1472        DPRINTK("Issued tag %u\n", qc->tag);
1473
1474        return 0;
1475}
1476
1477static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1478{
1479        struct ata_host *host = dev_instance;
1480        unsigned int i;
1481        unsigned int handled = 0;
1482        unsigned long flags;
1483
1484        spin_lock_irqsave(&host->lock, flags);
1485
1486        for (i = 0; i < host->n_ports; i++) {
1487                struct ata_port *ap = host->ports[i];
1488                struct ata_queued_cmd *qc;
1489
1490                qc = ata_qc_from_tag(ap, ap->link.active_tag);
1491                if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) {
1492                        handled += ata_bmdma_port_intr(ap, qc);
1493                } else {
1494                        /*
1495                         * No request pending?  Clear interrupt status
1496                         * anyway, in case there's one pending.
1497                         */
1498                        ap->ops->sff_check_status(ap);
1499                }
1500        }
1501
1502        spin_unlock_irqrestore(&host->lock, flags);
1503
1504        return IRQ_RETVAL(handled);
1505}
1506
1507static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1508{
1509        int i, handled = 0;
1510
1511        for (i = 0; i < host->n_ports; i++) {
1512                handled += nv_host_intr(host->ports[i], irq_stat);
1513                irq_stat >>= NV_INT_PORT_SHIFT;
1514        }
1515
1516        return IRQ_RETVAL(handled);
1517}
1518
1519static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1520{
1521        struct ata_host *host = dev_instance;
1522        u8 irq_stat;
1523        irqreturn_t ret;
1524
1525        spin_lock(&host->lock);
1526        irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1527        ret = nv_do_interrupt(host, irq_stat);
1528        spin_unlock(&host->lock);
1529
1530        return ret;
1531}
1532
1533static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1534{
1535        struct ata_host *host = dev_instance;
1536        u8 irq_stat;
1537        irqreturn_t ret;
1538
1539        spin_lock(&host->lock);
1540        irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1541        ret = nv_do_interrupt(host, irq_stat);
1542        spin_unlock(&host->lock);
1543
1544        return ret;
1545}
1546
1547static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1548{
1549        if (sc_reg > SCR_CONTROL)
1550                return -EINVAL;
1551
1552        *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
1553        return 0;
1554}
1555
1556static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
1557{
1558        if (sc_reg > SCR_CONTROL)
1559                return -EINVAL;
1560
1561        iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
1562        return 0;
1563}
1564
1565static int nv_hardreset(struct ata_link *link, unsigned int *class,
1566                        unsigned long deadline)
1567{
1568        struct ata_eh_context *ehc = &link->eh_context;
1569
1570        /* Do hardreset iff it's post-boot probing, please read the
1571         * comment above port ops for details.
1572         */
1573        if (!(link->ap->pflags & ATA_PFLAG_LOADING) &&
1574            !ata_dev_enabled(link->device))
1575                sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1576                                    NULL, NULL);
1577        else {
1578                const unsigned long *timing = sata_ehc_deb_timing(ehc);
1579                int rc;
1580
1581                if (!(ehc->i.flags & ATA_EHI_QUIET))
1582                        ata_link_info(link,
1583                                      "nv: skipping hardreset on occupied port\n");
1584
1585                /* make sure the link is online */
1586                rc = sata_link_resume(link, timing, deadline);
1587                /* whine about phy resume failure but proceed */
1588                if (rc && rc != -EOPNOTSUPP)
1589                        ata_link_warn(link, "failed to resume link (errno=%d)\n",
1590                                      rc);
1591        }
1592
1593        /* device signature acquisition is unreliable */
1594        return -EAGAIN;
1595}
1596
1597static void nv_nf2_freeze(struct ata_port *ap)
1598{
1599        void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1600        int shift = ap->port_no * NV_INT_PORT_SHIFT;
1601        u8 mask;
1602
1603        mask = ioread8(scr_addr + NV_INT_ENABLE);
1604        mask &= ~(NV_INT_ALL << shift);
1605        iowrite8(mask, scr_addr + NV_INT_ENABLE);
1606}
1607
1608static void nv_nf2_thaw(struct ata_port *ap)
1609{
1610        void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1611        int shift = ap->port_no * NV_INT_PORT_SHIFT;
1612        u8 mask;
1613
1614        iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1615
1616        mask = ioread8(scr_addr + NV_INT_ENABLE);
1617        mask |= (NV_INT_MASK << shift);
1618        iowrite8(mask, scr_addr + NV_INT_ENABLE);
1619}
1620
1621static void nv_ck804_freeze(struct ata_port *ap)
1622{
1623        void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1624        int shift = ap->port_no * NV_INT_PORT_SHIFT;
1625        u8 mask;
1626
1627        mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1628        mask &= ~(NV_INT_ALL << shift);
1629        writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1630}
1631
1632static void nv_ck804_thaw(struct ata_port *ap)
1633{
1634        void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1635        int shift = ap->port_no * NV_INT_PORT_SHIFT;
1636        u8 mask;
1637
1638        writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1639
1640        mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1641        mask |= (NV_INT_MASK << shift);
1642        writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1643}
1644
1645static void nv_mcp55_freeze(struct ata_port *ap)
1646{
1647        void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1648        int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1649        u32 mask;
1650
1651        writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1652
1653        mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1654        mask &= ~(NV_INT_ALL_MCP55 << shift);
1655        writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1656}
1657
1658static void nv_mcp55_thaw(struct ata_port *ap)
1659{
1660        void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1661        int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1662        u32 mask;
1663
1664        writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1665
1666        mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1667        mask |= (NV_INT_MASK_MCP55 << shift);
1668        writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1669}
1670
1671static void nv_adma_error_handler(struct ata_port *ap)
1672{
1673        struct nv_adma_port_priv *pp = ap->private_data;
1674        if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1675                void __iomem *mmio = pp->ctl_block;
1676                int i;
1677                u16 tmp;
1678
1679                if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
1680                        u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1681                        u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1682                        u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1683                        u32 status = readw(mmio + NV_ADMA_STAT);
1684                        u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1685                        u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
1686
1687                        ata_port_err(ap,
1688                                "EH in ADMA mode, notifier 0x%X "
1689                                "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1690                                "next cpb count 0x%X next cpb idx 0x%x\n",
1691                                notifier, notifier_error, gen_ctl, status,
1692                                cpb_count, next_cpb_idx);
1693
1694                        for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
1695                                struct nv_adma_cpb *cpb = &pp->cpb[i];
1696                                if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
1697                                    ap->link.sactive & (1 << i))
1698                                        ata_port_err(ap,
1699                                                "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1700                                                i, cpb->ctl_flags, cpb->resp_flags);
1701                        }
1702                }
1703
1704                /* Push us back into port register mode for error handling. */
1705                nv_adma_register_mode(ap);
1706
1707                /* Mark all of the CPBs as invalid to prevent them from
1708                   being executed */
1709                for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
1710                        pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1711
1712                /* clear CPB fetch count */
1713                writew(0, mmio + NV_ADMA_CPB_COUNT);
1714
1715                /* Reset channel */
1716                tmp = readw(mmio + NV_ADMA_CTL);
1717                writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1718                readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1719                udelay(1);
1720                writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1721                readw(mmio + NV_ADMA_CTL);      /* flush posted write */
1722        }
1723
1724        ata_bmdma_error_handler(ap);
1725}
1726
1727static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1728{
1729        struct nv_swncq_port_priv *pp = ap->private_data;
1730        struct defer_queue *dq = &pp->defer_queue;
1731
1732        /* queue is full */
1733        WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1734        dq->defer_bits |= (1 << qc->tag);
1735        dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1736}
1737
1738static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1739{
1740        struct nv_swncq_port_priv *pp = ap->private_data;
1741        struct defer_queue *dq = &pp->defer_queue;
1742        unsigned int tag;
1743
1744        if (dq->head == dq->tail)       /* null queue */
1745                return NULL;
1746
1747        tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1748        dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1749        WARN_ON(!(dq->defer_bits & (1 << tag)));
1750        dq->defer_bits &= ~(1 << tag);
1751
1752        return ata_qc_from_tag(ap, tag);
1753}
1754
1755static void nv_swncq_fis_reinit(struct ata_port *ap)
1756{
1757        struct nv_swncq_port_priv *pp = ap->private_data;
1758
1759        pp->dhfis_bits = 0;
1760        pp->dmafis_bits = 0;
1761        pp->sdbfis_bits = 0;
1762        pp->ncq_flags = 0;
1763}
1764
1765static void nv_swncq_pp_reinit(struct ata_port *ap)
1766{
1767        struct nv_swncq_port_priv *pp = ap->private_data;
1768        struct defer_queue *dq = &pp->defer_queue;
1769
1770        dq->head = 0;
1771        dq->tail = 0;
1772        dq->defer_bits = 0;
1773        pp->qc_active = 0;
1774        pp->last_issue_tag = ATA_TAG_POISON;
1775        nv_swncq_fis_reinit(ap);
1776}
1777
1778static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1779{
1780        struct nv_swncq_port_priv *pp = ap->private_data;
1781
1782        writew(fis, pp->irq_block);
1783}
1784
1785static void __ata_bmdma_stop(struct ata_port *ap)
1786{
1787        struct ata_queued_cmd qc;
1788
1789        qc.ap = ap;
1790        ata_bmdma_stop(&qc);
1791}
1792
1793static void nv_swncq_ncq_stop(struct ata_port *ap)
1794{
1795        struct nv_swncq_port_priv *pp = ap->private_data;
1796        unsigned int i;
1797        u32 sactive;
1798        u32 done_mask;
1799
1800        ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1801                     ap->qc_active, ap->link.sactive);
1802        ata_port_err(ap,
1803                "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n  "
1804                "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1805                pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1806                pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1807
1808        ata_port_err(ap, "ATA_REG 0x%X ERR_REG 0x%X\n",
1809                     ap->ops->sff_check_status(ap),
1810                     ioread8(ap->ioaddr.error_addr));
1811
1812        sactive = readl(pp->sactive_block);
1813        done_mask = pp->qc_active ^ sactive;
1814
1815        ata_port_err(ap, "tag : dhfis dmafis sdbfis sactive\n");
1816        for (i = 0; i < ATA_MAX_QUEUE; i++) {
1817                u8 err = 0;
1818                if (pp->qc_active & (1 << i))
1819                        err = 0;
1820                else if (done_mask & (1 << i))
1821                        err = 1;
1822                else
1823                        continue;
1824
1825                ata_port_err(ap,
1826                             "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1827                             (pp->dhfis_bits >> i) & 0x1,
1828                             (pp->dmafis_bits >> i) & 0x1,
1829                             (pp->sdbfis_bits >> i) & 0x1,
1830                             (sactive >> i) & 0x1,
1831                             (err ? "error! tag doesn't exit" : " "));
1832        }
1833
1834        nv_swncq_pp_reinit(ap);
1835        ap->ops->sff_irq_clear(ap);
1836        __ata_bmdma_stop(ap);
1837        nv_swncq_irq_clear(ap, 0xffff);
1838}
1839
1840static void nv_swncq_error_handler(struct ata_port *ap)
1841{
1842        struct ata_eh_context *ehc = &ap->link.eh_context;
1843
1844        if (ap->link.sactive) {
1845                nv_swncq_ncq_stop(ap);
1846                ehc->i.action |= ATA_EH_RESET;
1847        }
1848
1849        ata_bmdma_error_handler(ap);
1850}
1851
1852#ifdef CONFIG_PM
1853static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1854{
1855        void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1856        u32 tmp;
1857
1858        /* clear irq */
1859        writel(~0, mmio + NV_INT_STATUS_MCP55);
1860
1861        /* disable irq */
1862        writel(0, mmio + NV_INT_ENABLE_MCP55);
1863
1864        /* disable swncq */
1865        tmp = readl(mmio + NV_CTL_MCP55);
1866        tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1867        writel(tmp, mmio + NV_CTL_MCP55);
1868
1869        return 0;
1870}
1871
1872static int nv_swncq_port_resume(struct ata_port *ap)
1873{
1874        void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1875        u32 tmp;
1876
1877        /* clear irq */
1878        writel(~0, mmio + NV_INT_STATUS_MCP55);
1879
1880        /* enable irq */
1881        writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1882
1883        /* enable swncq */
1884        tmp = readl(mmio + NV_CTL_MCP55);
1885        writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1886
1887        return 0;
1888}
1889#endif
1890
1891static void nv_swncq_host_init(struct ata_host *host)
1892{
1893        u32 tmp;
1894        void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1895        struct pci_dev *pdev = to_pci_dev(host->dev);
1896        u8 regval;
1897
1898        /* disable  ECO 398 */
1899        pci_read_config_byte(pdev, 0x7f, &regval);
1900        regval &= ~(1 << 7);
1901        pci_write_config_byte(pdev, 0x7f, regval);
1902
1903        /* enable swncq */
1904        tmp = readl(mmio + NV_CTL_MCP55);
1905        VPRINTK("HOST_CTL:0x%X\n", tmp);
1906        writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1907
1908        /* enable irq intr */
1909        tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1910        VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1911        writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1912
1913        /*  clear port irq */
1914        writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1915}
1916
1917static int nv_swncq_slave_config(struct scsi_device *sdev)
1918{
1919        struct ata_port *ap = ata_shost_to_port(sdev->host);
1920        struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1921        struct ata_device *dev;
1922        int rc;
1923        u8 rev;
1924        u8 check_maxtor = 0;
1925        unsigned char model_num[ATA_ID_PROD_LEN + 1];
1926
1927        rc = ata_scsi_slave_config(sdev);
1928        if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1929                /* Not a proper libata device, ignore */
1930                return rc;
1931
1932        dev = &ap->link.device[sdev->id];
1933        if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1934                return rc;
1935
1936        /* if MCP51 and Maxtor, then disable ncq */
1937        if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1938                pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1939                check_maxtor = 1;
1940
1941        /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1942        if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1943                pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1944                pci_read_config_byte(pdev, 0x8, &rev);
1945                if (rev <= 0xa2)
1946                        check_maxtor = 1;
1947        }
1948
1949        if (!check_maxtor)
1950                return rc;
1951
1952        ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1953
1954        if (strncmp(model_num, "Maxtor", 6) == 0) {
1955                ata_scsi_change_queue_depth(sdev, 1, SCSI_QDEPTH_DEFAULT);
1956                ata_dev_notice(dev, "Disabling SWNCQ mode (depth %x)\n",
1957                               sdev->queue_depth);
1958        }
1959
1960        return rc;
1961}
1962
1963static int nv_swncq_port_start(struct ata_port *ap)
1964{
1965        struct device *dev = ap->host->dev;
1966        void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1967        struct nv_swncq_port_priv *pp;
1968        int rc;
1969
1970        /* we might fallback to bmdma, allocate bmdma resources */
1971        rc = ata_bmdma_port_start(ap);
1972        if (rc)
1973                return rc;
1974
1975        pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1976        if (!pp)
1977                return -ENOMEM;
1978
1979        pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1980                                      &pp->prd_dma, GFP_KERNEL);
1981        if (!pp->prd)
1982                return -ENOMEM;
1983        memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1984
1985        ap->private_data = pp;
1986        pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1987        pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1988        pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1989
1990        return 0;
1991}
1992
1993static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1994{
1995        if (qc->tf.protocol != ATA_PROT_NCQ) {
1996                ata_bmdma_qc_prep(qc);
1997                return;
1998        }
1999
2000        if (!(qc->flags & ATA_QCFLAG_DMAMAP))
2001                return;
2002
2003        nv_swncq_fill_sg(qc);
2004}
2005
2006static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
2007{
2008        struct ata_port *ap = qc->ap;
2009        struct scatterlist *sg;
2010        struct nv_swncq_port_priv *pp = ap->private_data;
2011        struct ata_bmdma_prd *prd;
2012        unsigned int si, idx;
2013
2014        prd = pp->prd + ATA_MAX_PRD * qc->tag;
2015
2016        idx = 0;
2017        for_each_sg(qc->sg, sg, qc->n_elem, si) {
2018                u32 addr, offset;
2019                u32 sg_len, len;
2020
2021                addr = (u32)sg_dma_address(sg);
2022                sg_len = sg_dma_len(sg);
2023
2024                while (sg_len) {
2025                        offset = addr & 0xffff;
2026                        len = sg_len;
2027                        if ((offset + sg_len) > 0x10000)
2028                                len = 0x10000 - offset;
2029
2030                        prd[idx].addr = cpu_to_le32(addr);
2031                        prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2032
2033                        idx++;
2034                        sg_len -= len;
2035                        addr += len;
2036                }
2037        }
2038
2039        prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2040}
2041
2042static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2043                                          struct ata_queued_cmd *qc)
2044{
2045        struct nv_swncq_port_priv *pp = ap->private_data;
2046
2047        if (qc == NULL)
2048                return 0;
2049
2050        DPRINTK("Enter\n");
2051
2052        writel((1 << qc->tag), pp->sactive_block);
2053        pp->last_issue_tag = qc->tag;
2054        pp->dhfis_bits &= ~(1 << qc->tag);
2055        pp->dmafis_bits &= ~(1 << qc->tag);
2056        pp->qc_active |= (0x1 << qc->tag);
2057
2058        ap->ops->sff_tf_load(ap, &qc->tf);       /* load tf registers */
2059        ap->ops->sff_exec_command(ap, &qc->tf);
2060
2061        DPRINTK("Issued tag %u\n", qc->tag);
2062
2063        return 0;
2064}
2065
2066static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2067{
2068        struct ata_port *ap = qc->ap;
2069        struct nv_swncq_port_priv *pp = ap->private_data;
2070
2071        if (qc->tf.protocol != ATA_PROT_NCQ)
2072                return ata_bmdma_qc_issue(qc);
2073
2074        DPRINTK("Enter\n");
2075
2076        if (!pp->qc_active)
2077                nv_swncq_issue_atacmd(ap, qc);
2078        else
2079                nv_swncq_qc_to_dq(ap, qc);      /* add qc to defer queue */
2080
2081        return 0;
2082}
2083
2084static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2085{
2086        u32 serror;
2087        struct ata_eh_info *ehi = &ap->link.eh_info;
2088
2089        ata_ehi_clear_desc(ehi);
2090
2091        /* AHCI needs SError cleared; otherwise, it might lock up */
2092        sata_scr_read(&ap->link, SCR_ERROR, &serror);
2093        sata_scr_write(&ap->link, SCR_ERROR, serror);
2094
2095        /* analyze @irq_stat */
2096        if (fis & NV_SWNCQ_IRQ_ADDED)
2097                ata_ehi_push_desc(ehi, "hot plug");
2098        else if (fis & NV_SWNCQ_IRQ_REMOVED)
2099                ata_ehi_push_desc(ehi, "hot unplug");
2100
2101        ata_ehi_hotplugged(ehi);
2102
2103        /* okay, let's hand over to EH */
2104        ehi->serror |= serror;
2105
2106        ata_port_freeze(ap);
2107}
2108
2109static int nv_swncq_sdbfis(struct ata_port *ap)
2110{
2111        struct ata_queued_cmd *qc;
2112        struct nv_swncq_port_priv *pp = ap->private_data;
2113        struct ata_eh_info *ehi = &ap->link.eh_info;
2114        u32 sactive;
2115        u32 done_mask;
2116        u8 host_stat;
2117        u8 lack_dhfis = 0;
2118
2119        host_stat = ap->ops->bmdma_status(ap);
2120        if (unlikely(host_stat & ATA_DMA_ERR)) {
2121                /* error when transferring data to/from memory */
2122                ata_ehi_clear_desc(ehi);
2123                ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2124                ehi->err_mask |= AC_ERR_HOST_BUS;
2125                ehi->action |= ATA_EH_RESET;
2126                return -EINVAL;
2127        }
2128
2129        ap->ops->sff_irq_clear(ap);
2130        __ata_bmdma_stop(ap);
2131
2132        sactive = readl(pp->sactive_block);
2133        done_mask = pp->qc_active ^ sactive;
2134
2135        pp->qc_active &= ~done_mask;
2136        pp->dhfis_bits &= ~done_mask;
2137        pp->dmafis_bits &= ~done_mask;
2138        pp->sdbfis_bits |= done_mask;
2139        ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
2140
2141        if (!ap->qc_active) {
2142                DPRINTK("over\n");
2143                nv_swncq_pp_reinit(ap);
2144                return 0;
2145        }
2146
2147        if (pp->qc_active & pp->dhfis_bits)
2148                return 0;
2149
2150        if ((pp->ncq_flags & ncq_saw_backout) ||
2151            (pp->qc_active ^ pp->dhfis_bits))
2152                /* if the controller can't get a device to host register FIS,
2153                 * The driver needs to reissue the new command.
2154                 */
2155                lack_dhfis = 1;
2156
2157        DPRINTK("id 0x%x QC: qc_active 0x%x,"
2158                "SWNCQ:qc_active 0x%X defer_bits %X "
2159                "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2160                ap->print_id, ap->qc_active, pp->qc_active,
2161                pp->defer_queue.defer_bits, pp->dhfis_bits,
2162                pp->dmafis_bits, pp->last_issue_tag);
2163
2164        nv_swncq_fis_reinit(ap);
2165
2166        if (lack_dhfis) {
2167                qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2168                nv_swncq_issue_atacmd(ap, qc);
2169                return 0;
2170        }
2171
2172        if (pp->defer_queue.defer_bits) {
2173                /* send deferral queue command */
2174                qc = nv_swncq_qc_from_dq(ap);
2175                WARN_ON(qc == NULL);
2176                nv_swncq_issue_atacmd(ap, qc);
2177        }
2178
2179        return 0;
2180}
2181
2182static inline u32 nv_swncq_tag(struct ata_port *ap)
2183{
2184        struct nv_swncq_port_priv *pp = ap->private_data;
2185        u32 tag;
2186
2187        tag = readb(pp->tag_block) >> 2;
2188        return (tag & 0x1f);
2189}
2190
2191static void nv_swncq_dmafis(struct ata_port *ap)
2192{
2193        struct ata_queued_cmd *qc;
2194        unsigned int rw;
2195        u8 dmactl;
2196        u32 tag;
2197        struct nv_swncq_port_priv *pp = ap->private_data;
2198
2199        __ata_bmdma_stop(ap);
2200        tag = nv_swncq_tag(ap);
2201
2202        DPRINTK("dma setup tag 0x%x\n", tag);
2203        qc = ata_qc_from_tag(ap, tag);
2204
2205        if (unlikely(!qc))
2206                return;
2207
2208        rw = qc->tf.flags & ATA_TFLAG_WRITE;
2209
2210        /* load PRD table addr. */
2211        iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2212                  ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2213
2214        /* specify data direction, triple-check start bit is clear */
2215        dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2216        dmactl &= ~ATA_DMA_WR;
2217        if (!rw)
2218                dmactl |= ATA_DMA_WR;
2219
2220        iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2221}
2222
2223static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2224{
2225        struct nv_swncq_port_priv *pp = ap->private_data;
2226        struct ata_queued_cmd *qc;
2227        struct ata_eh_info *ehi = &ap->link.eh_info;
2228        u32 serror;
2229        u8 ata_stat;
2230
2231        ata_stat = ap->ops->sff_check_status(ap);
2232        nv_swncq_irq_clear(ap, fis);
2233        if (!fis)
2234                return;
2235
2236        if (ap->pflags & ATA_PFLAG_FROZEN)
2237                return;
2238
2239        if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2240                nv_swncq_hotplug(ap, fis);
2241                return;
2242        }
2243
2244        if (!pp->qc_active)
2245                return;
2246
2247        if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
2248                return;
2249        ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
2250
2251        if (ata_stat & ATA_ERR) {
2252                ata_ehi_clear_desc(ehi);
2253                ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2254                ehi->err_mask |= AC_ERR_DEV;
2255                ehi->serror |= serror;
2256                ehi->action |= ATA_EH_RESET;
2257                ata_port_freeze(ap);
2258                return;
2259        }
2260
2261        if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2262                /* If the IRQ is backout, driver must issue
2263                 * the new command again some time later.
2264                 */
2265                pp->ncq_flags |= ncq_saw_backout;
2266        }
2267
2268        if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2269                pp->ncq_flags |= ncq_saw_sdb;
2270                DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2271                        "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2272                        ap->print_id, pp->qc_active, pp->dhfis_bits,
2273                        pp->dmafis_bits, readl(pp->sactive_block));
2274                if (nv_swncq_sdbfis(ap) < 0)
2275                        goto irq_error;
2276        }
2277
2278        if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2279                /* The interrupt indicates the new command
2280                 * was transmitted correctly to the drive.
2281                 */
2282                pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2283                pp->ncq_flags |= ncq_saw_d2h;
2284                if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2285                        ata_ehi_push_desc(ehi, "illegal fis transaction");
2286                        ehi->err_mask |= AC_ERR_HSM;
2287                        ehi->action |= ATA_EH_RESET;
2288                        goto irq_error;
2289                }
2290
2291                if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2292                    !(pp->ncq_flags & ncq_saw_dmas)) {
2293                        ata_stat = ap->ops->sff_check_status(ap);
2294                        if (ata_stat & ATA_BUSY)
2295                                goto irq_exit;
2296
2297                        if (pp->defer_queue.defer_bits) {
2298                                DPRINTK("send next command\n");
2299                                qc = nv_swncq_qc_from_dq(ap);
2300                                nv_swncq_issue_atacmd(ap, qc);
2301                        }
2302                }
2303        }
2304
2305        if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2306                /* program the dma controller with appropriate PRD buffers
2307                 * and start the DMA transfer for requested command.
2308                 */
2309                pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2310                pp->ncq_flags |= ncq_saw_dmas;
2311                nv_swncq_dmafis(ap);
2312        }
2313
2314irq_exit:
2315        return;
2316irq_error:
2317        ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2318        ata_port_freeze(ap);
2319        return;
2320}
2321
2322static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2323{
2324        struct ata_host *host = dev_instance;
2325        unsigned int i;
2326        unsigned int handled = 0;
2327        unsigned long flags;
2328        u32 irq_stat;
2329
2330        spin_lock_irqsave(&host->lock, flags);
2331
2332        irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2333
2334        for (i = 0; i < host->n_ports; i++) {
2335                struct ata_port *ap = host->ports[i];
2336
2337                if (ap->link.sactive) {
2338                        nv_swncq_host_interrupt(ap, (u16)irq_stat);
2339                        handled = 1;
2340                } else {
2341                        if (irq_stat)   /* reserve Hotplug */
2342                                nv_swncq_irq_clear(ap, 0xfff0);
2343
2344                        handled += nv_host_intr(ap, (u8)irq_stat);
2345                }
2346                irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2347        }
2348
2349        spin_unlock_irqrestore(&host->lock, flags);
2350
2351        return IRQ_RETVAL(handled);
2352}
2353
2354static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
2355{
2356        const struct ata_port_info *ppi[] = { NULL, NULL };
2357        struct nv_pi_priv *ipriv;
2358        struct ata_host *host;
2359        struct nv_host_priv *hpriv;
2360        int rc;
2361        u32 bar;
2362        void __iomem *base;
2363        unsigned long type = ent->driver_data;
2364
2365        // Make sure this is a SATA controller by counting the number of bars
2366        // (NVIDIA SATA controllers will always have six bars).  Otherwise,
2367        // it's an IDE controller and we ignore it.
2368        for (bar = 0; bar < 6; bar++)
2369                if (pci_resource_start(pdev, bar) == 0)
2370                        return -ENODEV;
2371
2372        ata_print_version_once(&pdev->dev, DRV_VERSION);
2373
2374        rc = pcim_enable_device(pdev);
2375        if (rc)
2376                return rc;
2377
2378        /* determine type and allocate host */
2379        if (type == CK804 && adma_enabled) {
2380                dev_notice(&pdev->dev, "Using ADMA mode\n");
2381                type = ADMA;
2382        } else if (type == MCP5x && swncq_enabled) {
2383                dev_notice(&pdev->dev, "Using SWNCQ mode\n");
2384                type = SWNCQ;
2385        }
2386
2387        ppi[0] = &nv_port_info[type];
2388        ipriv = ppi[0]->private_data;
2389        rc = ata_pci_bmdma_prepare_host(pdev, ppi, &host);
2390        if (rc)
2391                return rc;
2392
2393        hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2394        if (!hpriv)
2395                return -ENOMEM;
2396        hpriv->type = type;
2397        host->private_data = hpriv;
2398
2399        /* request and iomap NV_MMIO_BAR */
2400        rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2401        if (rc)
2402                return rc;
2403
2404        /* configure SCR access */
2405        base = host->iomap[NV_MMIO_BAR];
2406        host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2407        host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
2408
2409        /* enable SATA space for CK804 */
2410        if (type >= CK804) {
2411                u8 regval;
2412
2413                pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2414                regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2415                pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2416        }
2417
2418        /* init ADMA */
2419        if (type == ADMA) {
2420                rc = nv_adma_host_init(host);
2421                if (rc)
2422                        return rc;
2423        } else if (type == SWNCQ)
2424                nv_swncq_host_init(host);
2425
2426        if (msi_enabled) {
2427                dev_notice(&pdev->dev, "Using MSI\n");
2428                pci_enable_msi(pdev);
2429        }
2430
2431        pci_set_master(pdev);
2432        return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht);
2433}
2434
2435#ifdef CONFIG_PM
2436static int nv_pci_device_resume(struct pci_dev *pdev)
2437{
2438        struct ata_host *host = dev_get_drvdata(&pdev->dev);
2439        struct nv_host_priv *hpriv = host->private_data;
2440        int rc;
2441
2442        rc = ata_pci_device_do_resume(pdev);
2443        if (rc)
2444                return rc;
2445
2446        if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2447                if (hpriv->type >= CK804) {
2448                        u8 regval;
2449
2450                        pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2451                        regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2452                        pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2453                }
2454                if (hpriv->type == ADMA) {
2455                        u32 tmp32;
2456                        struct nv_adma_port_priv *pp;
2457                        /* enable/disable ADMA on the ports appropriately */
2458                        pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2459
2460                        pp = host->ports[0]->private_data;
2461                        if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2462                                tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2463                                           NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2464                        else
2465                                tmp32 |=  (NV_MCP_SATA_CFG_20_PORT0_EN |
2466                                           NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2467                        pp = host->ports[1]->private_data;
2468                        if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2469                                tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
2470                                           NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2471                        else
2472                                tmp32 |=  (NV_MCP_SATA_CFG_20_PORT1_EN |
2473                                           NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2474
2475                        pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2476                }
2477        }
2478
2479        ata_host_resume(host);
2480
2481        return 0;
2482}
2483#endif
2484
2485static void nv_ck804_host_stop(struct ata_host *host)
2486{
2487        struct pci_dev *pdev = to_pci_dev(host->dev);
2488        u8 regval;
2489
2490        /* disable SATA space for CK804 */
2491        pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2492        regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2493        pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2494}
2495
2496static void nv_adma_host_stop(struct ata_host *host)
2497{
2498        struct pci_dev *pdev = to_pci_dev(host->dev);
2499        u32 tmp32;
2500
2501        /* disable ADMA on the ports */
2502        pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2503        tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2504                   NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2505                   NV_MCP_SATA_CFG_20_PORT1_EN |
2506                   NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2507
2508        pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2509
2510        nv_ck804_host_stop(host);
2511}
2512
2513module_pci_driver(nv_pci_driver);
2514
2515module_param_named(adma, adma_enabled, bool, 0444);
2516MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)");
2517module_param_named(swncq, swncq_enabled, bool, 0444);
2518MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
2519module_param_named(msi, msi_enabled, bool, 0444);
2520MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)");
2521
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.