linux/drivers/spi/spi-cadence-quadspi.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2//
   3// Driver for Cadence QSPI Controller
   4//
   5// Copyright Altera Corporation (C) 2012-2014. All rights reserved.
   6// Copyright Intel Corporation (C) 2019-2020. All rights reserved.
   7// Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com
   8
   9#include <linux/clk.h>
  10#include <linux/completion.h>
  11#include <linux/delay.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/dmaengine.h>
  14#include <linux/err.h>
  15#include <linux/errno.h>
  16#include <linux/interrupt.h>
  17#include <linux/io.h>
  18#include <linux/iopoll.h>
  19#include <linux/jiffies.h>
  20#include <linux/kernel.h>
  21#include <linux/module.h>
  22#include <linux/of_device.h>
  23#include <linux/of.h>
  24#include <linux/platform_device.h>
  25#include <linux/pm_runtime.h>
  26#include <linux/reset.h>
  27#include <linux/sched.h>
  28#include <linux/spi/spi.h>
  29#include <linux/spi/spi-mem.h>
  30#include <linux/timer.h>
  31
  32#define CQSPI_NAME                      "cadence-qspi"
  33#define CQSPI_MAX_CHIPSELECT            16
  34
  35/* Quirks */
  36#define CQSPI_NEEDS_WR_DELAY            BIT(0)
  37#define CQSPI_DISABLE_DAC_MODE          BIT(1)
  38
  39/* Capabilities */
  40#define CQSPI_SUPPORTS_OCTAL            BIT(0)
  41
  42struct cqspi_st;
  43
  44struct cqspi_flash_pdata {
  45        struct cqspi_st *cqspi;
  46        u32             clk_rate;
  47        u32             read_delay;
  48        u32             tshsl_ns;
  49        u32             tsd2d_ns;
  50        u32             tchsh_ns;
  51        u32             tslch_ns;
  52        u8              inst_width;
  53        u8              addr_width;
  54        u8              data_width;
  55        bool            dtr;
  56        u8              cs;
  57};
  58
  59struct cqspi_st {
  60        struct platform_device  *pdev;
  61
  62        struct clk              *clk;
  63        unsigned int            sclk;
  64
  65        void __iomem            *iobase;
  66        void __iomem            *ahb_base;
  67        resource_size_t         ahb_size;
  68        struct completion       transfer_complete;
  69
  70        struct dma_chan         *rx_chan;
  71        struct completion       rx_dma_complete;
  72        dma_addr_t              mmap_phys_base;
  73
  74        int                     current_cs;
  75        unsigned long           master_ref_clk_hz;
  76        bool                    is_decoded_cs;
  77        u32                     fifo_depth;
  78        u32                     fifo_width;
  79        u32                     num_chipselect;
  80        bool                    rclk_en;
  81        u32                     trigger_address;
  82        u32                     wr_delay;
  83        bool                    use_direct_mode;
  84        struct cqspi_flash_pdata f_pdata[CQSPI_MAX_CHIPSELECT];
  85};
  86
  87struct cqspi_driver_platdata {
  88        u32 hwcaps_mask;
  89        u8 quirks;
  90};
  91
  92/* Operation timeout value */
  93#define CQSPI_TIMEOUT_MS                        500
  94#define CQSPI_READ_TIMEOUT_MS                   10
  95
  96/* Instruction type */
  97#define CQSPI_INST_TYPE_SINGLE                  0
  98#define CQSPI_INST_TYPE_DUAL                    1
  99#define CQSPI_INST_TYPE_QUAD                    2
 100#define CQSPI_INST_TYPE_OCTAL                   3
 101
 102#define CQSPI_DUMMY_CLKS_PER_BYTE               8
 103#define CQSPI_DUMMY_BYTES_MAX                   4
 104#define CQSPI_DUMMY_CLKS_MAX                    31
 105
 106#define CQSPI_STIG_DATA_LEN_MAX                 8
 107
 108/* Register map */
 109#define CQSPI_REG_CONFIG                        0x00
 110#define CQSPI_REG_CONFIG_ENABLE_MASK            BIT(0)
 111#define CQSPI_REG_CONFIG_ENB_DIR_ACC_CTRL       BIT(7)
 112#define CQSPI_REG_CONFIG_DECODE_MASK            BIT(9)
 113#define CQSPI_REG_CONFIG_CHIPSELECT_LSB         10
 114#define CQSPI_REG_CONFIG_DMA_MASK               BIT(15)
 115#define CQSPI_REG_CONFIG_BAUD_LSB               19
 116#define CQSPI_REG_CONFIG_DTR_PROTO              BIT(24)
 117#define CQSPI_REG_CONFIG_DUAL_OPCODE            BIT(30)
 118#define CQSPI_REG_CONFIG_IDLE_LSB               31
 119#define CQSPI_REG_CONFIG_CHIPSELECT_MASK        0xF
 120#define CQSPI_REG_CONFIG_BAUD_MASK              0xF
 121
 122#define CQSPI_REG_RD_INSTR                      0x04
 123#define CQSPI_REG_RD_INSTR_OPCODE_LSB           0
 124#define CQSPI_REG_RD_INSTR_TYPE_INSTR_LSB       8
 125#define CQSPI_REG_RD_INSTR_TYPE_ADDR_LSB        12
 126#define CQSPI_REG_RD_INSTR_TYPE_DATA_LSB        16
 127#define CQSPI_REG_RD_INSTR_MODE_EN_LSB          20
 128#define CQSPI_REG_RD_INSTR_DUMMY_LSB            24
 129#define CQSPI_REG_RD_INSTR_TYPE_INSTR_MASK      0x3
 130#define CQSPI_REG_RD_INSTR_TYPE_ADDR_MASK       0x3
 131#define CQSPI_REG_RD_INSTR_TYPE_DATA_MASK       0x3
 132#define CQSPI_REG_RD_INSTR_DUMMY_MASK           0x1F
 133
 134#define CQSPI_REG_WR_INSTR                      0x08
 135#define CQSPI_REG_WR_INSTR_OPCODE_LSB           0
 136#define CQSPI_REG_WR_INSTR_TYPE_ADDR_LSB        12
 137#define CQSPI_REG_WR_INSTR_TYPE_DATA_LSB        16
 138
 139#define CQSPI_REG_DELAY                         0x0C
 140#define CQSPI_REG_DELAY_TSLCH_LSB               0
 141#define CQSPI_REG_DELAY_TCHSH_LSB               8
 142#define CQSPI_REG_DELAY_TSD2D_LSB               16
 143#define CQSPI_REG_DELAY_TSHSL_LSB               24
 144#define CQSPI_REG_DELAY_TSLCH_MASK              0xFF
 145#define CQSPI_REG_DELAY_TCHSH_MASK              0xFF
 146#define CQSPI_REG_DELAY_TSD2D_MASK              0xFF
 147#define CQSPI_REG_DELAY_TSHSL_MASK              0xFF
 148
 149#define CQSPI_REG_READCAPTURE                   0x10
 150#define CQSPI_REG_READCAPTURE_BYPASS_LSB        0
 151#define CQSPI_REG_READCAPTURE_DELAY_LSB         1
 152#define CQSPI_REG_READCAPTURE_DELAY_MASK        0xF
 153
 154#define CQSPI_REG_SIZE                          0x14
 155#define CQSPI_REG_SIZE_ADDRESS_LSB              0
 156#define CQSPI_REG_SIZE_PAGE_LSB                 4
 157#define CQSPI_REG_SIZE_BLOCK_LSB                16
 158#define CQSPI_REG_SIZE_ADDRESS_MASK             0xF
 159#define CQSPI_REG_SIZE_PAGE_MASK                0xFFF
 160#define CQSPI_REG_SIZE_BLOCK_MASK               0x3F
 161
 162#define CQSPI_REG_SRAMPARTITION                 0x18
 163#define CQSPI_REG_INDIRECTTRIGGER               0x1C
 164
 165#define CQSPI_REG_DMA                           0x20
 166#define CQSPI_REG_DMA_SINGLE_LSB                0
 167#define CQSPI_REG_DMA_BURST_LSB                 8
 168#define CQSPI_REG_DMA_SINGLE_MASK               0xFF
 169#define CQSPI_REG_DMA_BURST_MASK                0xFF
 170
 171#define CQSPI_REG_REMAP                         0x24
 172#define CQSPI_REG_MODE_BIT                      0x28
 173
 174#define CQSPI_REG_SDRAMLEVEL                    0x2C
 175#define CQSPI_REG_SDRAMLEVEL_RD_LSB             0
 176#define CQSPI_REG_SDRAMLEVEL_WR_LSB             16
 177#define CQSPI_REG_SDRAMLEVEL_RD_MASK            0xFFFF
 178#define CQSPI_REG_SDRAMLEVEL_WR_MASK            0xFFFF
 179
 180#define CQSPI_REG_WR_COMPLETION_CTRL            0x38
 181#define CQSPI_REG_WR_DISABLE_AUTO_POLL          BIT(14)
 182
 183#define CQSPI_REG_IRQSTATUS                     0x40
 184#define CQSPI_REG_IRQMASK                       0x44
 185
 186#define CQSPI_REG_INDIRECTRD                    0x60
 187#define CQSPI_REG_INDIRECTRD_START_MASK         BIT(0)
 188#define CQSPI_REG_INDIRECTRD_CANCEL_MASK        BIT(1)
 189#define CQSPI_REG_INDIRECTRD_DONE_MASK          BIT(5)
 190
 191#define CQSPI_REG_INDIRECTRDWATERMARK           0x64
 192#define CQSPI_REG_INDIRECTRDSTARTADDR           0x68
 193#define CQSPI_REG_INDIRECTRDBYTES               0x6C
 194
 195#define CQSPI_REG_CMDCTRL                       0x90
 196#define CQSPI_REG_CMDCTRL_EXECUTE_MASK          BIT(0)
 197#define CQSPI_REG_CMDCTRL_INPROGRESS_MASK       BIT(1)
 198#define CQSPI_REG_CMDCTRL_DUMMY_LSB             7
 199#define CQSPI_REG_CMDCTRL_WR_BYTES_LSB          12
 200#define CQSPI_REG_CMDCTRL_WR_EN_LSB             15
 201#define CQSPI_REG_CMDCTRL_ADD_BYTES_LSB         16
 202#define CQSPI_REG_CMDCTRL_ADDR_EN_LSB           19
 203#define CQSPI_REG_CMDCTRL_RD_BYTES_LSB          20
 204#define CQSPI_REG_CMDCTRL_RD_EN_LSB             23
 205#define CQSPI_REG_CMDCTRL_OPCODE_LSB            24
 206#define CQSPI_REG_CMDCTRL_WR_BYTES_MASK         0x7
 207#define CQSPI_REG_CMDCTRL_ADD_BYTES_MASK        0x3
 208#define CQSPI_REG_CMDCTRL_RD_BYTES_MASK         0x7
 209#define CQSPI_REG_CMDCTRL_DUMMY_MASK            0x1F
 210
 211#define CQSPI_REG_INDIRECTWR                    0x70
 212#define CQSPI_REG_INDIRECTWR_START_MASK         BIT(0)
 213#define CQSPI_REG_INDIRECTWR_CANCEL_MASK        BIT(1)
 214#define CQSPI_REG_INDIRECTWR_DONE_MASK          BIT(5)
 215
 216#define CQSPI_REG_INDIRECTWRWATERMARK           0x74
 217#define CQSPI_REG_INDIRECTWRSTARTADDR           0x78
 218#define CQSPI_REG_INDIRECTWRBYTES               0x7C
 219
 220#define CQSPI_REG_CMDADDRESS                    0x94
 221#define CQSPI_REG_CMDREADDATALOWER              0xA0
 222#define CQSPI_REG_CMDREADDATAUPPER              0xA4
 223#define CQSPI_REG_CMDWRITEDATALOWER             0xA8
 224#define CQSPI_REG_CMDWRITEDATAUPPER             0xAC
 225
 226#define CQSPI_REG_POLLING_STATUS                0xB0
 227#define CQSPI_REG_POLLING_STATUS_DUMMY_LSB      16
 228
 229#define CQSPI_REG_OP_EXT_LOWER                  0xE0
 230#define CQSPI_REG_OP_EXT_READ_LSB               24
 231#define CQSPI_REG_OP_EXT_WRITE_LSB              16
 232#define CQSPI_REG_OP_EXT_STIG_LSB               0
 233
 234/* Interrupt status bits */
 235#define CQSPI_REG_IRQ_MODE_ERR                  BIT(0)
 236#define CQSPI_REG_IRQ_UNDERFLOW                 BIT(1)
 237#define CQSPI_REG_IRQ_IND_COMP                  BIT(2)
 238#define CQSPI_REG_IRQ_IND_RD_REJECT             BIT(3)
 239#define CQSPI_REG_IRQ_WR_PROTECTED_ERR          BIT(4)
 240#define CQSPI_REG_IRQ_ILLEGAL_AHB_ERR           BIT(5)
 241#define CQSPI_REG_IRQ_WATERMARK                 BIT(6)
 242#define CQSPI_REG_IRQ_IND_SRAM_FULL             BIT(12)
 243
 244#define CQSPI_IRQ_MASK_RD               (CQSPI_REG_IRQ_WATERMARK        | \
 245                                         CQSPI_REG_IRQ_IND_SRAM_FULL    | \
 246                                         CQSPI_REG_IRQ_IND_COMP)
 247
 248#define CQSPI_IRQ_MASK_WR               (CQSPI_REG_IRQ_IND_COMP         | \
 249                                         CQSPI_REG_IRQ_WATERMARK        | \
 250                                         CQSPI_REG_IRQ_UNDERFLOW)
 251
 252#define CQSPI_IRQ_STATUS_MASK           0x1FFFF
 253
 254static int cqspi_wait_for_bit(void __iomem *reg, const u32 mask, bool clr)
 255{
 256        u32 val;
 257
 258        return readl_relaxed_poll_timeout(reg, val,
 259                                          (((clr ? ~val : val) & mask) == mask),
 260                                          10, CQSPI_TIMEOUT_MS * 1000);
 261}
 262
 263static bool cqspi_is_idle(struct cqspi_st *cqspi)
 264{
 265        u32 reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
 266
 267        return reg & (1UL << CQSPI_REG_CONFIG_IDLE_LSB);
 268}
 269
 270static u32 cqspi_get_rd_sram_level(struct cqspi_st *cqspi)
 271{
 272        u32 reg = readl(cqspi->iobase + CQSPI_REG_SDRAMLEVEL);
 273
 274        reg >>= CQSPI_REG_SDRAMLEVEL_RD_LSB;
 275        return reg & CQSPI_REG_SDRAMLEVEL_RD_MASK;
 276}
 277
 278static irqreturn_t cqspi_irq_handler(int this_irq, void *dev)
 279{
 280        struct cqspi_st *cqspi = dev;
 281        unsigned int irq_status;
 282
 283        /* Read interrupt status */
 284        irq_status = readl(cqspi->iobase + CQSPI_REG_IRQSTATUS);
 285
 286        /* Clear interrupt */
 287        writel(irq_status, cqspi->iobase + CQSPI_REG_IRQSTATUS);
 288
 289        irq_status &= CQSPI_IRQ_MASK_RD | CQSPI_IRQ_MASK_WR;
 290
 291        if (irq_status)
 292                complete(&cqspi->transfer_complete);
 293
 294        return IRQ_HANDLED;
 295}
 296
 297static unsigned int cqspi_calc_rdreg(struct cqspi_flash_pdata *f_pdata)
 298{
 299        u32 rdreg = 0;
 300
 301        rdreg |= f_pdata->inst_width << CQSPI_REG_RD_INSTR_TYPE_INSTR_LSB;
 302        rdreg |= f_pdata->addr_width << CQSPI_REG_RD_INSTR_TYPE_ADDR_LSB;
 303        rdreg |= f_pdata->data_width << CQSPI_REG_RD_INSTR_TYPE_DATA_LSB;
 304
 305        return rdreg;
 306}
 307
 308static unsigned int cqspi_calc_dummy(const struct spi_mem_op *op, bool dtr)
 309{
 310        unsigned int dummy_clk;
 311
 312        if (!op->dummy.nbytes)
 313                return 0;
 314
 315        dummy_clk = op->dummy.nbytes * (8 / op->dummy.buswidth);
 316        if (dtr)
 317                dummy_clk /= 2;
 318
 319        return dummy_clk;
 320}
 321
 322static int cqspi_set_protocol(struct cqspi_flash_pdata *f_pdata,
 323                              const struct spi_mem_op *op)
 324{
 325        f_pdata->inst_width = CQSPI_INST_TYPE_SINGLE;
 326        f_pdata->addr_width = CQSPI_INST_TYPE_SINGLE;
 327        f_pdata->data_width = CQSPI_INST_TYPE_SINGLE;
 328
 329        /*
 330         * For an op to be DTR, cmd phase along with every other non-empty
 331         * phase should have dtr field set to 1. If an op phase has zero
 332         * nbytes, ignore its dtr field; otherwise, check its dtr field.
 333         */
 334        f_pdata->dtr = op->cmd.dtr &&
 335                       (!op->addr.nbytes || op->addr.dtr) &&
 336                       (!op->data.nbytes || op->data.dtr);
 337
 338        switch (op->data.buswidth) {
 339        case 0:
 340                break;
 341        case 1:
 342                f_pdata->data_width = CQSPI_INST_TYPE_SINGLE;
 343                break;
 344        case 2:
 345                f_pdata->data_width = CQSPI_INST_TYPE_DUAL;
 346                break;
 347        case 4:
 348                f_pdata->data_width = CQSPI_INST_TYPE_QUAD;
 349                break;
 350        case 8:
 351                f_pdata->data_width = CQSPI_INST_TYPE_OCTAL;
 352                break;
 353        default:
 354                return -EINVAL;
 355        }
 356
 357        /* Right now we only support 8-8-8 DTR mode. */
 358        if (f_pdata->dtr) {
 359                switch (op->cmd.buswidth) {
 360                case 0:
 361                        break;
 362                case 8:
 363                        f_pdata->inst_width = CQSPI_INST_TYPE_OCTAL;
 364                        break;
 365                default:
 366                        return -EINVAL;
 367                }
 368
 369                switch (op->addr.buswidth) {
 370                case 0:
 371                        break;
 372                case 8:
 373                        f_pdata->addr_width = CQSPI_INST_TYPE_OCTAL;
 374                        break;
 375                default:
 376                        return -EINVAL;
 377                }
 378
 379                switch (op->data.buswidth) {
 380                case 0:
 381                        break;
 382                case 8:
 383                        f_pdata->data_width = CQSPI_INST_TYPE_OCTAL;
 384                        break;
 385                default:
 386                        return -EINVAL;
 387                }
 388        }
 389
 390        return 0;
 391}
 392
 393static int cqspi_wait_idle(struct cqspi_st *cqspi)
 394{
 395        const unsigned int poll_idle_retry = 3;
 396        unsigned int count = 0;
 397        unsigned long timeout;
 398
 399        timeout = jiffies + msecs_to_jiffies(CQSPI_TIMEOUT_MS);
 400        while (1) {
 401                /*
 402                 * Read few times in succession to ensure the controller
 403                 * is indeed idle, that is, the bit does not transition
 404                 * low again.
 405                 */
 406                if (cqspi_is_idle(cqspi))
 407                        count++;
 408                else
 409                        count = 0;
 410
 411                if (count >= poll_idle_retry)
 412                        return 0;
 413
 414                if (time_after(jiffies, timeout)) {
 415                        /* Timeout, in busy mode. */
 416                        dev_err(&cqspi->pdev->dev,
 417                                "QSPI is still busy after %dms timeout.\n",
 418                                CQSPI_TIMEOUT_MS);
 419                        return -ETIMEDOUT;
 420                }
 421
 422                cpu_relax();
 423        }
 424}
 425
 426static int cqspi_exec_flash_cmd(struct cqspi_st *cqspi, unsigned int reg)
 427{
 428        void __iomem *reg_base = cqspi->iobase;
 429        int ret;
 430
 431        /* Write the CMDCTRL without start execution. */
 432        writel(reg, reg_base + CQSPI_REG_CMDCTRL);
 433        /* Start execute */
 434        reg |= CQSPI_REG_CMDCTRL_EXECUTE_MASK;
 435        writel(reg, reg_base + CQSPI_REG_CMDCTRL);
 436
 437        /* Polling for completion. */
 438        ret = cqspi_wait_for_bit(reg_base + CQSPI_REG_CMDCTRL,
 439                                 CQSPI_REG_CMDCTRL_INPROGRESS_MASK, 1);
 440        if (ret) {
 441                dev_err(&cqspi->pdev->dev,
 442                        "Flash command execution timed out.\n");
 443                return ret;
 444        }
 445
 446        /* Polling QSPI idle status. */
 447        return cqspi_wait_idle(cqspi);
 448}
 449
 450static int cqspi_setup_opcode_ext(struct cqspi_flash_pdata *f_pdata,
 451                                  const struct spi_mem_op *op,
 452                                  unsigned int shift)
 453{
 454        struct cqspi_st *cqspi = f_pdata->cqspi;
 455        void __iomem *reg_base = cqspi->iobase;
 456        unsigned int reg;
 457        u8 ext;
 458
 459        if (op->cmd.nbytes != 2)
 460                return -EINVAL;
 461
 462        /* Opcode extension is the LSB. */
 463        ext = op->cmd.opcode & 0xff;
 464
 465        reg = readl(reg_base + CQSPI_REG_OP_EXT_LOWER);
 466        reg &= ~(0xff << shift);
 467        reg |= ext << shift;
 468        writel(reg, reg_base + CQSPI_REG_OP_EXT_LOWER);
 469
 470        return 0;
 471}
 472
 473static int cqspi_enable_dtr(struct cqspi_flash_pdata *f_pdata,
 474                            const struct spi_mem_op *op, unsigned int shift,
 475                            bool enable)
 476{
 477        struct cqspi_st *cqspi = f_pdata->cqspi;
 478        void __iomem *reg_base = cqspi->iobase;
 479        unsigned int reg;
 480        int ret;
 481
 482        reg = readl(reg_base + CQSPI_REG_CONFIG);
 483
 484        /*
 485         * We enable dual byte opcode here. The callers have to set up the
 486         * extension opcode based on which type of operation it is.
 487         */
 488        if (enable) {
 489                reg |= CQSPI_REG_CONFIG_DTR_PROTO;
 490                reg |= CQSPI_REG_CONFIG_DUAL_OPCODE;
 491
 492                /* Set up command opcode extension. */
 493                ret = cqspi_setup_opcode_ext(f_pdata, op, shift);
 494                if (ret)
 495                        return ret;
 496        } else {
 497                reg &= ~CQSPI_REG_CONFIG_DTR_PROTO;
 498                reg &= ~CQSPI_REG_CONFIG_DUAL_OPCODE;
 499        }
 500
 501        writel(reg, reg_base + CQSPI_REG_CONFIG);
 502
 503        return cqspi_wait_idle(cqspi);
 504}
 505
 506static int cqspi_command_read(struct cqspi_flash_pdata *f_pdata,
 507                              const struct spi_mem_op *op)
 508{
 509        struct cqspi_st *cqspi = f_pdata->cqspi;
 510        void __iomem *reg_base = cqspi->iobase;
 511        u8 *rxbuf = op->data.buf.in;
 512        u8 opcode;
 513        size_t n_rx = op->data.nbytes;
 514        unsigned int rdreg;
 515        unsigned int reg;
 516        unsigned int dummy_clk;
 517        size_t read_len;
 518        int status;
 519
 520        status = cqspi_set_protocol(f_pdata, op);
 521        if (status)
 522                return status;
 523
 524        status = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_STIG_LSB,
 525                                  f_pdata->dtr);
 526        if (status)
 527                return status;
 528
 529        if (!n_rx || n_rx > CQSPI_STIG_DATA_LEN_MAX || !rxbuf) {
 530                dev_err(&cqspi->pdev->dev,
 531                        "Invalid input argument, len %zu rxbuf 0x%p\n",
 532                        n_rx, rxbuf);
 533                return -EINVAL;
 534        }
 535
 536        if (f_pdata->dtr)
 537                opcode = op->cmd.opcode >> 8;
 538        else
 539                opcode = op->cmd.opcode;
 540
 541        reg = opcode << CQSPI_REG_CMDCTRL_OPCODE_LSB;
 542
 543        rdreg = cqspi_calc_rdreg(f_pdata);
 544        writel(rdreg, reg_base + CQSPI_REG_RD_INSTR);
 545
 546        dummy_clk = cqspi_calc_dummy(op, f_pdata->dtr);
 547        if (dummy_clk > CQSPI_DUMMY_CLKS_MAX)
 548                return -EOPNOTSUPP;
 549
 550        if (dummy_clk)
 551                reg |= (dummy_clk & CQSPI_REG_CMDCTRL_DUMMY_MASK)
 552                     << CQSPI_REG_CMDCTRL_DUMMY_LSB;
 553
 554        reg |= (0x1 << CQSPI_REG_CMDCTRL_RD_EN_LSB);
 555
 556        /* 0 means 1 byte. */
 557        reg |= (((n_rx - 1) & CQSPI_REG_CMDCTRL_RD_BYTES_MASK)
 558                << CQSPI_REG_CMDCTRL_RD_BYTES_LSB);
 559        status = cqspi_exec_flash_cmd(cqspi, reg);
 560        if (status)
 561                return status;
 562
 563        reg = readl(reg_base + CQSPI_REG_CMDREADDATALOWER);
 564
 565        /* Put the read value into rx_buf */
 566        read_len = (n_rx > 4) ? 4 : n_rx;
 567        memcpy(rxbuf, &reg, read_len);
 568        rxbuf += read_len;
 569
 570        if (n_rx > 4) {
 571                reg = readl(reg_base + CQSPI_REG_CMDREADDATAUPPER);
 572
 573                read_len = n_rx - read_len;
 574                memcpy(rxbuf, &reg, read_len);
 575        }
 576
 577        return 0;
 578}
 579
 580static int cqspi_command_write(struct cqspi_flash_pdata *f_pdata,
 581                               const struct spi_mem_op *op)
 582{
 583        struct cqspi_st *cqspi = f_pdata->cqspi;
 584        void __iomem *reg_base = cqspi->iobase;
 585        u8 opcode;
 586        const u8 *txbuf = op->data.buf.out;
 587        size_t n_tx = op->data.nbytes;
 588        unsigned int reg;
 589        unsigned int data;
 590        size_t write_len;
 591        int ret;
 592
 593        ret = cqspi_set_protocol(f_pdata, op);
 594        if (ret)
 595                return ret;
 596
 597        ret = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_STIG_LSB,
 598                               f_pdata->dtr);
 599        if (ret)
 600                return ret;
 601
 602        if (n_tx > CQSPI_STIG_DATA_LEN_MAX || (n_tx && !txbuf)) {
 603                dev_err(&cqspi->pdev->dev,
 604                        "Invalid input argument, cmdlen %zu txbuf 0x%p\n",
 605                        n_tx, txbuf);
 606                return -EINVAL;
 607        }
 608
 609        reg = cqspi_calc_rdreg(f_pdata);
 610        writel(reg, reg_base + CQSPI_REG_RD_INSTR);
 611
 612        if (f_pdata->dtr)
 613                opcode = op->cmd.opcode >> 8;
 614        else
 615                opcode = op->cmd.opcode;
 616
 617        reg = opcode << CQSPI_REG_CMDCTRL_OPCODE_LSB;
 618
 619        if (op->addr.nbytes) {
 620                reg |= (0x1 << CQSPI_REG_CMDCTRL_ADDR_EN_LSB);
 621                reg |= ((op->addr.nbytes - 1) &
 622                        CQSPI_REG_CMDCTRL_ADD_BYTES_MASK)
 623                        << CQSPI_REG_CMDCTRL_ADD_BYTES_LSB;
 624
 625                writel(op->addr.val, reg_base + CQSPI_REG_CMDADDRESS);
 626        }
 627
 628        if (n_tx) {
 629                reg |= (0x1 << CQSPI_REG_CMDCTRL_WR_EN_LSB);
 630                reg |= ((n_tx - 1) & CQSPI_REG_CMDCTRL_WR_BYTES_MASK)
 631                        << CQSPI_REG_CMDCTRL_WR_BYTES_LSB;
 632                data = 0;
 633                write_len = (n_tx > 4) ? 4 : n_tx;
 634                memcpy(&data, txbuf, write_len);
 635                txbuf += write_len;
 636                writel(data, reg_base + CQSPI_REG_CMDWRITEDATALOWER);
 637
 638                if (n_tx > 4) {
 639                        data = 0;
 640                        write_len = n_tx - 4;
 641                        memcpy(&data, txbuf, write_len);
 642                        writel(data, reg_base + CQSPI_REG_CMDWRITEDATAUPPER);
 643                }
 644        }
 645
 646        return cqspi_exec_flash_cmd(cqspi, reg);
 647}
 648
 649static int cqspi_read_setup(struct cqspi_flash_pdata *f_pdata,
 650                            const struct spi_mem_op *op)
 651{
 652        struct cqspi_st *cqspi = f_pdata->cqspi;
 653        void __iomem *reg_base = cqspi->iobase;
 654        unsigned int dummy_clk = 0;
 655        unsigned int reg;
 656        int ret;
 657        u8 opcode;
 658
 659        ret = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_READ_LSB,
 660                               f_pdata->dtr);
 661        if (ret)
 662                return ret;
 663
 664        if (f_pdata->dtr)
 665                opcode = op->cmd.opcode >> 8;
 666        else
 667                opcode = op->cmd.opcode;
 668
 669        reg = opcode << CQSPI_REG_RD_INSTR_OPCODE_LSB;
 670        reg |= cqspi_calc_rdreg(f_pdata);
 671
 672        /* Setup dummy clock cycles */
 673        dummy_clk = cqspi_calc_dummy(op, f_pdata->dtr);
 674
 675        if (dummy_clk > CQSPI_DUMMY_CLKS_MAX)
 676                return -EOPNOTSUPP;
 677
 678        if (dummy_clk)
 679                reg |= (dummy_clk & CQSPI_REG_RD_INSTR_DUMMY_MASK)
 680                       << CQSPI_REG_RD_INSTR_DUMMY_LSB;
 681
 682        writel(reg, reg_base + CQSPI_REG_RD_INSTR);
 683
 684        /* Set address width */
 685        reg = readl(reg_base + CQSPI_REG_SIZE);
 686        reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK;
 687        reg |= (op->addr.nbytes - 1);
 688        writel(reg, reg_base + CQSPI_REG_SIZE);
 689        return 0;
 690}
 691
 692static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata,
 693                                       u8 *rxbuf, loff_t from_addr,
 694                                       const size_t n_rx)
 695{
 696        struct cqspi_st *cqspi = f_pdata->cqspi;
 697        struct device *dev = &cqspi->pdev->dev;
 698        void __iomem *reg_base = cqspi->iobase;
 699        void __iomem *ahb_base = cqspi->ahb_base;
 700        unsigned int remaining = n_rx;
 701        unsigned int mod_bytes = n_rx % 4;
 702        unsigned int bytes_to_read = 0;
 703        u8 *rxbuf_end = rxbuf + n_rx;
 704        int ret = 0;
 705
 706        writel(from_addr, reg_base + CQSPI_REG_INDIRECTRDSTARTADDR);
 707        writel(remaining, reg_base + CQSPI_REG_INDIRECTRDBYTES);
 708
 709        /* Clear all interrupts. */
 710        writel(CQSPI_IRQ_STATUS_MASK, reg_base + CQSPI_REG_IRQSTATUS);
 711
 712        writel(CQSPI_IRQ_MASK_RD, reg_base + CQSPI_REG_IRQMASK);
 713
 714        reinit_completion(&cqspi->transfer_complete);
 715        writel(CQSPI_REG_INDIRECTRD_START_MASK,
 716               reg_base + CQSPI_REG_INDIRECTRD);
 717
 718        while (remaining > 0) {
 719                if (!wait_for_completion_timeout(&cqspi->transfer_complete,
 720                                                 msecs_to_jiffies(CQSPI_READ_TIMEOUT_MS)))
 721                        ret = -ETIMEDOUT;
 722
 723                bytes_to_read = cqspi_get_rd_sram_level(cqspi);
 724
 725                if (ret && bytes_to_read == 0) {
 726                        dev_err(dev, "Indirect read timeout, no bytes\n");
 727                        goto failrd;
 728                }
 729
 730                while (bytes_to_read != 0) {
 731                        unsigned int word_remain = round_down(remaining, 4);
 732
 733                        bytes_to_read *= cqspi->fifo_width;
 734                        bytes_to_read = bytes_to_read > remaining ?
 735                                        remaining : bytes_to_read;
 736                        bytes_to_read = round_down(bytes_to_read, 4);
 737                        /* Read 4 byte word chunks then single bytes */
 738                        if (bytes_to_read) {
 739                                ioread32_rep(ahb_base, rxbuf,
 740                                             (bytes_to_read / 4));
 741                        } else if (!word_remain && mod_bytes) {
 742                                unsigned int temp = ioread32(ahb_base);
 743
 744                                bytes_to_read = mod_bytes;
 745                                memcpy(rxbuf, &temp, min((unsigned int)
 746                                                         (rxbuf_end - rxbuf),
 747                                                         bytes_to_read));
 748                        }
 749                        rxbuf += bytes_to_read;
 750                        remaining -= bytes_to_read;
 751                        bytes_to_read = cqspi_get_rd_sram_level(cqspi);
 752                }
 753
 754                if (remaining > 0)
 755                        reinit_completion(&cqspi->transfer_complete);
 756        }
 757
 758        /* Check indirect done status */
 759        ret = cqspi_wait_for_bit(reg_base + CQSPI_REG_INDIRECTRD,
 760                                 CQSPI_REG_INDIRECTRD_DONE_MASK, 0);
 761        if (ret) {
 762                dev_err(dev, "Indirect read completion error (%i)\n", ret);
 763                goto failrd;
 764        }
 765
 766        /* Disable interrupt */
 767        writel(0, reg_base + CQSPI_REG_IRQMASK);
 768
 769        /* Clear indirect completion status */
 770        writel(CQSPI_REG_INDIRECTRD_DONE_MASK, reg_base + CQSPI_REG_INDIRECTRD);
 771
 772        return 0;
 773
 774failrd:
 775        /* Disable interrupt */
 776        writel(0, reg_base + CQSPI_REG_IRQMASK);
 777
 778        /* Cancel the indirect read */
 779        writel(CQSPI_REG_INDIRECTWR_CANCEL_MASK,
 780               reg_base + CQSPI_REG_INDIRECTRD);
 781        return ret;
 782}
 783
 784static int cqspi_write_setup(struct cqspi_flash_pdata *f_pdata,
 785                             const struct spi_mem_op *op)
 786{
 787        unsigned int reg;
 788        int ret;
 789        struct cqspi_st *cqspi = f_pdata->cqspi;
 790        void __iomem *reg_base = cqspi->iobase;
 791        u8 opcode;
 792
 793        ret = cqspi_enable_dtr(f_pdata, op, CQSPI_REG_OP_EXT_WRITE_LSB,
 794                               f_pdata->dtr);
 795        if (ret)
 796                return ret;
 797
 798        if (f_pdata->dtr)
 799                opcode = op->cmd.opcode >> 8;
 800        else
 801                opcode = op->cmd.opcode;
 802
 803        /* Set opcode. */
 804        reg = opcode << CQSPI_REG_WR_INSTR_OPCODE_LSB;
 805        reg |= f_pdata->data_width << CQSPI_REG_WR_INSTR_TYPE_DATA_LSB;
 806        reg |= f_pdata->addr_width << CQSPI_REG_WR_INSTR_TYPE_ADDR_LSB;
 807        writel(reg, reg_base + CQSPI_REG_WR_INSTR);
 808        reg = cqspi_calc_rdreg(f_pdata);
 809        writel(reg, reg_base + CQSPI_REG_RD_INSTR);
 810
 811        /*
 812         * SPI NAND flashes require the address of the status register to be
 813         * passed in the Read SR command. Also, some SPI NOR flashes like the
 814         * cypress Semper flash expect a 4-byte dummy address in the Read SR
 815         * command in DTR mode.
 816         *
 817         * But this controller does not support address phase in the Read SR
 818         * command when doing auto-HW polling. So, disable write completion
 819         * polling on the controller's side. spinand and spi-nor will take
 820         * care of polling the status register.
 821         */
 822        reg = readl(reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
 823        reg |= CQSPI_REG_WR_DISABLE_AUTO_POLL;
 824        writel(reg, reg_base + CQSPI_REG_WR_COMPLETION_CTRL);
 825
 826        reg = readl(reg_base + CQSPI_REG_SIZE);
 827        reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK;
 828        reg |= (op->addr.nbytes - 1);
 829        writel(reg, reg_base + CQSPI_REG_SIZE);
 830        return 0;
 831}
 832
 833static int cqspi_indirect_write_execute(struct cqspi_flash_pdata *f_pdata,
 834                                        loff_t to_addr, const u8 *txbuf,
 835                                        const size_t n_tx)
 836{
 837        struct cqspi_st *cqspi = f_pdata->cqspi;
 838        struct device *dev = &cqspi->pdev->dev;
 839        void __iomem *reg_base = cqspi->iobase;
 840        unsigned int remaining = n_tx;
 841        unsigned int write_bytes;
 842        int ret;
 843
 844        writel(to_addr, reg_base + CQSPI_REG_INDIRECTWRSTARTADDR);
 845        writel(remaining, reg_base + CQSPI_REG_INDIRECTWRBYTES);
 846
 847        /* Clear all interrupts. */
 848        writel(CQSPI_IRQ_STATUS_MASK, reg_base + CQSPI_REG_IRQSTATUS);
 849
 850        writel(CQSPI_IRQ_MASK_WR, reg_base + CQSPI_REG_IRQMASK);
 851
 852        reinit_completion(&cqspi->transfer_complete);
 853        writel(CQSPI_REG_INDIRECTWR_START_MASK,
 854               reg_base + CQSPI_REG_INDIRECTWR);
 855        /*
 856         * As per 66AK2G02 TRM SPRUHY8F section 11.15.5.3 Indirect Access
 857         * Controller programming sequence, couple of cycles of
 858         * QSPI_REF_CLK delay is required for the above bit to
 859         * be internally synchronized by the QSPI module. Provide 5
 860         * cycles of delay.
 861         */
 862        if (cqspi->wr_delay)
 863                ndelay(cqspi->wr_delay);
 864
 865        while (remaining > 0) {
 866                size_t write_words, mod_bytes;
 867
 868                write_bytes = remaining;
 869                write_words = write_bytes / 4;
 870                mod_bytes = write_bytes % 4;
 871                /* Write 4 bytes at a time then single bytes. */
 872                if (write_words) {
 873                        iowrite32_rep(cqspi->ahb_base, txbuf, write_words);
 874                        txbuf += (write_words * 4);
 875                }
 876                if (mod_bytes) {
 877                        unsigned int temp = 0xFFFFFFFF;
 878
 879                        memcpy(&temp, txbuf, mod_bytes);
 880                        iowrite32(temp, cqspi->ahb_base);
 881                        txbuf += mod_bytes;
 882                }
 883
 884                if (!wait_for_completion_timeout(&cqspi->transfer_complete,
 885                                                 msecs_to_jiffies(CQSPI_TIMEOUT_MS))) {
 886                        dev_err(dev, "Indirect write timeout\n");
 887                        ret = -ETIMEDOUT;
 888                        goto failwr;
 889                }
 890
 891                remaining -= write_bytes;
 892
 893                if (remaining > 0)
 894                        reinit_completion(&cqspi->transfer_complete);
 895        }
 896
 897        /* Check indirect done status */
 898        ret = cqspi_wait_for_bit(reg_base + CQSPI_REG_INDIRECTWR,
 899                                 CQSPI_REG_INDIRECTWR_DONE_MASK, 0);
 900        if (ret) {
 901                dev_err(dev, "Indirect write completion error (%i)\n", ret);
 902                goto failwr;
 903        }
 904
 905        /* Disable interrupt. */
 906        writel(0, reg_base + CQSPI_REG_IRQMASK);
 907
 908        /* Clear indirect completion status */
 909        writel(CQSPI_REG_INDIRECTWR_DONE_MASK, reg_base + CQSPI_REG_INDIRECTWR);
 910
 911        cqspi_wait_idle(cqspi);
 912
 913        return 0;
 914
 915failwr:
 916        /* Disable interrupt. */
 917        writel(0, reg_base + CQSPI_REG_IRQMASK);
 918
 919        /* Cancel the indirect write */
 920        writel(CQSPI_REG_INDIRECTWR_CANCEL_MASK,
 921               reg_base + CQSPI_REG_INDIRECTWR);
 922        return ret;
 923}
 924
 925static void cqspi_chipselect(struct cqspi_flash_pdata *f_pdata)
 926{
 927        struct cqspi_st *cqspi = f_pdata->cqspi;
 928        void __iomem *reg_base = cqspi->iobase;
 929        unsigned int chip_select = f_pdata->cs;
 930        unsigned int reg;
 931
 932        reg = readl(reg_base + CQSPI_REG_CONFIG);
 933        if (cqspi->is_decoded_cs) {
 934                reg |= CQSPI_REG_CONFIG_DECODE_MASK;
 935        } else {
 936                reg &= ~CQSPI_REG_CONFIG_DECODE_MASK;
 937
 938                /* Convert CS if without decoder.
 939                 * CS0 to 4b'1110
 940                 * CS1 to 4b'1101
 941                 * CS2 to 4b'1011
 942                 * CS3 to 4b'0111
 943                 */
 944                chip_select = 0xF & ~(1 << chip_select);
 945        }
 946
 947        reg &= ~(CQSPI_REG_CONFIG_CHIPSELECT_MASK
 948                 << CQSPI_REG_CONFIG_CHIPSELECT_LSB);
 949        reg |= (chip_select & CQSPI_REG_CONFIG_CHIPSELECT_MASK)
 950            << CQSPI_REG_CONFIG_CHIPSELECT_LSB;
 951        writel(reg, reg_base + CQSPI_REG_CONFIG);
 952}
 953
 954static unsigned int calculate_ticks_for_ns(const unsigned int ref_clk_hz,
 955                                           const unsigned int ns_val)
 956{
 957        unsigned int ticks;
 958
 959        ticks = ref_clk_hz / 1000;      /* kHz */
 960        ticks = DIV_ROUND_UP(ticks * ns_val, 1000000);
 961
 962        return ticks;
 963}
 964
 965static void cqspi_delay(struct cqspi_flash_pdata *f_pdata)
 966{
 967        struct cqspi_st *cqspi = f_pdata->cqspi;
 968        void __iomem *iobase = cqspi->iobase;
 969        const unsigned int ref_clk_hz = cqspi->master_ref_clk_hz;
 970        unsigned int tshsl, tchsh, tslch, tsd2d;
 971        unsigned int reg;
 972        unsigned int tsclk;
 973
 974        /* calculate the number of ref ticks for one sclk tick */
 975        tsclk = DIV_ROUND_UP(ref_clk_hz, cqspi->sclk);
 976
 977        tshsl = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tshsl_ns);
 978        /* this particular value must be at least one sclk */
 979        if (tshsl < tsclk)
 980                tshsl = tsclk;
 981
 982        tchsh = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tchsh_ns);
 983        tslch = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tslch_ns);
 984        tsd2d = calculate_ticks_for_ns(ref_clk_hz, f_pdata->tsd2d_ns);
 985
 986        reg = (tshsl & CQSPI_REG_DELAY_TSHSL_MASK)
 987               << CQSPI_REG_DELAY_TSHSL_LSB;
 988        reg |= (tchsh & CQSPI_REG_DELAY_TCHSH_MASK)
 989                << CQSPI_REG_DELAY_TCHSH_LSB;
 990        reg |= (tslch & CQSPI_REG_DELAY_TSLCH_MASK)
 991                << CQSPI_REG_DELAY_TSLCH_LSB;
 992        reg |= (tsd2d & CQSPI_REG_DELAY_TSD2D_MASK)
 993                << CQSPI_REG_DELAY_TSD2D_LSB;
 994        writel(reg, iobase + CQSPI_REG_DELAY);
 995}
 996
 997static void cqspi_config_baudrate_div(struct cqspi_st *cqspi)
 998{
 999        const unsigned int ref_clk_hz = cqspi->master_ref_clk_hz;
1000        void __iomem *reg_base = cqspi->iobase;
1001        u32 reg, div;
1002
1003        /* Recalculate the baudrate divisor based on QSPI specification. */
1004        div = DIV_ROUND_UP(ref_clk_hz, 2 * cqspi->sclk) - 1;
1005
1006        reg = readl(reg_base + CQSPI_REG_CONFIG);
1007        reg &= ~(CQSPI_REG_CONFIG_BAUD_MASK << CQSPI_REG_CONFIG_BAUD_LSB);
1008        reg |= (div & CQSPI_REG_CONFIG_BAUD_MASK) << CQSPI_REG_CONFIG_BAUD_LSB;
1009        writel(reg, reg_base + CQSPI_REG_CONFIG);
1010}
1011
1012static void cqspi_readdata_capture(struct cqspi_st *cqspi,
1013                                   const bool bypass,
1014                                   const unsigned int delay)
1015{
1016        void __iomem *reg_base = cqspi->iobase;
1017        unsigned int reg;
1018
1019        reg = readl(reg_base + CQSPI_REG_READCAPTURE);
1020
1021        if (bypass)
1022                reg |= (1 << CQSPI_REG_READCAPTURE_BYPASS_LSB);
1023        else
1024                reg &= ~(1 << CQSPI_REG_READCAPTURE_BYPASS_LSB);
1025
1026        reg &= ~(CQSPI_REG_READCAPTURE_DELAY_MASK
1027                 << CQSPI_REG_READCAPTURE_DELAY_LSB);
1028
1029        reg |= (delay & CQSPI_REG_READCAPTURE_DELAY_MASK)
1030                << CQSPI_REG_READCAPTURE_DELAY_LSB;
1031
1032        writel(reg, reg_base + CQSPI_REG_READCAPTURE);
1033}
1034
1035static void cqspi_controller_enable(struct cqspi_st *cqspi, bool enable)
1036{
1037        void __iomem *reg_base = cqspi->iobase;
1038        unsigned int reg;
1039
1040        reg = readl(reg_base + CQSPI_REG_CONFIG);
1041
1042        if (enable)
1043                reg |= CQSPI_REG_CONFIG_ENABLE_MASK;
1044        else
1045                reg &= ~CQSPI_REG_CONFIG_ENABLE_MASK;
1046
1047        writel(reg, reg_base + CQSPI_REG_CONFIG);
1048}
1049
1050static void cqspi_configure(struct cqspi_flash_pdata *f_pdata,
1051                            unsigned long sclk)
1052{
1053        struct cqspi_st *cqspi = f_pdata->cqspi;
1054        int switch_cs = (cqspi->current_cs != f_pdata->cs);
1055        int switch_ck = (cqspi->sclk != sclk);
1056
1057        if (switch_cs || switch_ck)
1058                cqspi_controller_enable(cqspi, 0);
1059
1060        /* Switch chip select. */
1061        if (switch_cs) {
1062                cqspi->current_cs = f_pdata->cs;
1063                cqspi_chipselect(f_pdata);
1064        }
1065
1066        /* Setup baudrate divisor and delays */
1067        if (switch_ck) {
1068                cqspi->sclk = sclk;
1069                cqspi_config_baudrate_div(cqspi);
1070                cqspi_delay(f_pdata);
1071                cqspi_readdata_capture(cqspi, !cqspi->rclk_en,
1072                                       f_pdata->read_delay);
1073        }
1074
1075        if (switch_cs || switch_ck)
1076                cqspi_controller_enable(cqspi, 1);
1077}
1078
1079static ssize_t cqspi_write(struct cqspi_flash_pdata *f_pdata,
1080                           const struct spi_mem_op *op)
1081{
1082        struct cqspi_st *cqspi = f_pdata->cqspi;
1083        loff_t to = op->addr.val;
1084        size_t len = op->data.nbytes;
1085        const u_char *buf = op->data.buf.out;
1086        int ret;
1087
1088        ret = cqspi_set_protocol(f_pdata, op);
1089        if (ret)
1090                return ret;
1091
1092        ret = cqspi_write_setup(f_pdata, op);
1093        if (ret)
1094                return ret;
1095
1096        /*
1097         * Some flashes like the Cypress Semper flash expect a dummy 4-byte
1098         * address (all 0s) with the read status register command in DTR mode.
1099         * But this controller does not support sending dummy address bytes to
1100         * the flash when it is polling the write completion register in DTR
1101         * mode. So, we can not use direct mode when in DTR mode for writing
1102         * data.
1103         */
1104        if (!f_pdata->dtr && cqspi->use_direct_mode &&
1105            ((to + len) <= cqspi->ahb_size)) {
1106                memcpy_toio(cqspi->ahb_base + to, buf, len);
1107                return cqspi_wait_idle(cqspi);
1108        }
1109
1110        return cqspi_indirect_write_execute(f_pdata, to, buf, len);
1111}
1112
1113static void cqspi_rx_dma_callback(void *param)
1114{
1115        struct cqspi_st *cqspi = param;
1116
1117        complete(&cqspi->rx_dma_complete);
1118}
1119
1120static int cqspi_direct_read_execute(struct cqspi_flash_pdata *f_pdata,
1121                                     u_char *buf, loff_t from, size_t len)
1122{
1123        struct cqspi_st *cqspi = f_pdata->cqspi;
1124        struct device *dev = &cqspi->pdev->dev;
1125        enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
1126        dma_addr_t dma_src = (dma_addr_t)cqspi->mmap_phys_base + from;
1127        int ret = 0;
1128        struct dma_async_tx_descriptor *tx;
1129        dma_cookie_t cookie;
1130        dma_addr_t dma_dst;
1131        struct device *ddev;
1132
1133        if (!cqspi->rx_chan || !virt_addr_valid(buf)) {
1134                memcpy_fromio(buf, cqspi->ahb_base + from, len);
1135                return 0;
1136        }
1137
1138        ddev = cqspi->rx_chan->device->dev;
1139        dma_dst = dma_map_single(ddev, buf, len, DMA_FROM_DEVICE);
1140        if (dma_mapping_error(ddev, dma_dst)) {
1141                dev_err(dev, "dma mapping failed\n");
1142                return -ENOMEM;
1143        }
1144        tx = dmaengine_prep_dma_memcpy(cqspi->rx_chan, dma_dst, dma_src,
1145                                       len, flags);
1146        if (!tx) {
1147                dev_err(dev, "device_prep_dma_memcpy error\n");
1148                ret = -EIO;
1149                goto err_unmap;
1150        }
1151
1152        tx->callback = cqspi_rx_dma_callback;
1153        tx->callback_param = cqspi;
1154        cookie = tx->tx_submit(tx);
1155        reinit_completion(&cqspi->rx_dma_complete);
1156
1157        ret = dma_submit_error(cookie);
1158        if (ret) {
1159                dev_err(dev, "dma_submit_error %d\n", cookie);
1160                ret = -EIO;
1161                goto err_unmap;
1162        }
1163
1164        dma_async_issue_pending(cqspi->rx_chan);
1165        if (!wait_for_completion_timeout(&cqspi->rx_dma_complete,
1166                                         msecs_to_jiffies(max_t(size_t, len, 500)))) {
1167                dmaengine_terminate_sync(cqspi->rx_chan);
1168                dev_err(dev, "DMA wait_for_completion_timeout\n");
1169                ret = -ETIMEDOUT;
1170                goto err_unmap;
1171        }
1172
1173err_unmap:
1174        dma_unmap_single(ddev, dma_dst, len, DMA_FROM_DEVICE);
1175
1176        return ret;
1177}
1178
1179static ssize_t cqspi_read(struct cqspi_flash_pdata *f_pdata,
1180                          const struct spi_mem_op *op)
1181{
1182        struct cqspi_st *cqspi = f_pdata->cqspi;
1183        loff_t from = op->addr.val;
1184        size_t len = op->data.nbytes;
1185        u_char *buf = op->data.buf.in;
1186        int ret;
1187
1188        ret = cqspi_set_protocol(f_pdata, op);
1189        if (ret)
1190                return ret;
1191
1192        ret = cqspi_read_setup(f_pdata, op);
1193        if (ret)
1194                return ret;
1195
1196        if (cqspi->use_direct_mode && ((from + len) <= cqspi->ahb_size))
1197                return cqspi_direct_read_execute(f_pdata, buf, from, len);
1198
1199        return cqspi_indirect_read_execute(f_pdata, buf, from, len);
1200}
1201
1202static int cqspi_mem_process(struct spi_mem *mem, const struct spi_mem_op *op)
1203{
1204        struct cqspi_st *cqspi = spi_master_get_devdata(mem->spi->master);
1205        struct cqspi_flash_pdata *f_pdata;
1206
1207        f_pdata = &cqspi->f_pdata[mem->spi->chip_select];
1208        cqspi_configure(f_pdata, mem->spi->max_speed_hz);
1209
1210        if (op->data.dir == SPI_MEM_DATA_IN && op->data.buf.in) {
1211                if (!op->addr.nbytes)
1212                        return cqspi_command_read(f_pdata, op);
1213
1214                return cqspi_read(f_pdata, op);
1215        }
1216
1217        if (!op->addr.nbytes || !op->data.buf.out)
1218                return cqspi_command_write(f_pdata, op);
1219
1220        return cqspi_write(f_pdata, op);
1221}
1222
1223static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
1224{
1225        int ret;
1226
1227        ret = cqspi_mem_process(mem, op);
1228        if (ret)
1229                dev_err(&mem->spi->dev, "operation failed with %d\n", ret);
1230
1231        return ret;
1232}
1233
1234static bool cqspi_supports_mem_op(struct spi_mem *mem,
1235                                  const struct spi_mem_op *op)
1236{
1237        bool all_true, all_false;
1238
1239        /*
1240         * op->dummy.dtr is required for converting nbytes into ncycles.
1241         * Also, don't check the dtr field of the op phase having zero nbytes.
1242         */
1243        all_true = op->cmd.dtr &&
1244                   (!op->addr.nbytes || op->addr.dtr) &&
1245                   (!op->dummy.nbytes || op->dummy.dtr) &&
1246                   (!op->data.nbytes || op->data.dtr);
1247
1248        all_false = !op->cmd.dtr && !op->addr.dtr && !op->dummy.dtr &&
1249                    !op->data.dtr;
1250
1251        /* Mixed DTR modes not supported. */
1252        if (!(all_true || all_false))
1253                return false;
1254
1255        if (all_true)
1256                return spi_mem_dtr_supports_op(mem, op);
1257        else
1258                return spi_mem_default_supports_op(mem, op);
1259}
1260
1261static int cqspi_of_get_flash_pdata(struct platform_device *pdev,
1262                                    struct cqspi_flash_pdata *f_pdata,
1263                                    struct device_node *np)
1264{
1265        if (of_property_read_u32(np, "cdns,read-delay", &f_pdata->read_delay)) {
1266                dev_err(&pdev->dev, "couldn't determine read-delay\n");
1267                return -ENXIO;
1268        }
1269
1270        if (of_property_read_u32(np, "cdns,tshsl-ns", &f_pdata->tshsl_ns)) {
1271                dev_err(&pdev->dev, "couldn't determine tshsl-ns\n");
1272                return -ENXIO;
1273        }
1274
1275        if (of_property_read_u32(np, "cdns,tsd2d-ns", &f_pdata->tsd2d_ns)) {
1276                dev_err(&pdev->dev, "couldn't determine tsd2d-ns\n");
1277                return -ENXIO;
1278        }
1279
1280        if (of_property_read_u32(np, "cdns,tchsh-ns", &f_pdata->tchsh_ns)) {
1281                dev_err(&pdev->dev, "couldn't determine tchsh-ns\n");
1282                return -ENXIO;
1283        }
1284
1285        if (of_property_read_u32(np, "cdns,tslch-ns", &f_pdata->tslch_ns)) {
1286                dev_err(&pdev->dev, "couldn't determine tslch-ns\n");
1287                return -ENXIO;
1288        }
1289
1290        if (of_property_read_u32(np, "spi-max-frequency", &f_pdata->clk_rate)) {
1291                dev_err(&pdev->dev, "couldn't determine spi-max-frequency\n");
1292                return -ENXIO;
1293        }
1294
1295        return 0;
1296}
1297
1298static int cqspi_of_get_pdata(struct cqspi_st *cqspi)
1299{
1300        struct device *dev = &cqspi->pdev->dev;
1301        struct device_node *np = dev->of_node;
1302
1303        cqspi->is_decoded_cs = of_property_read_bool(np, "cdns,is-decoded-cs");
1304
1305        if (of_property_read_u32(np, "cdns,fifo-depth", &cqspi->fifo_depth)) {
1306                dev_err(dev, "couldn't determine fifo-depth\n");
1307                return -ENXIO;
1308        }
1309
1310        if (of_property_read_u32(np, "cdns,fifo-width", &cqspi->fifo_width)) {
1311                dev_err(dev, "couldn't determine fifo-width\n");
1312                return -ENXIO;
1313        }
1314
1315        if (of_property_read_u32(np, "cdns,trigger-address",
1316                                 &cqspi->trigger_address)) {
1317                dev_err(dev, "couldn't determine trigger-address\n");
1318                return -ENXIO;
1319        }
1320
1321        if (of_property_read_u32(np, "num-cs", &cqspi->num_chipselect))
1322                cqspi->num_chipselect = CQSPI_MAX_CHIPSELECT;
1323
1324        cqspi->rclk_en = of_property_read_bool(np, "cdns,rclk-en");
1325
1326        return 0;
1327}
1328
1329static void cqspi_controller_init(struct cqspi_st *cqspi)
1330{
1331        u32 reg;
1332
1333        cqspi_controller_enable(cqspi, 0);
1334
1335        /* Configure the remap address register, no remap */
1336        writel(0, cqspi->iobase + CQSPI_REG_REMAP);
1337
1338        /* Disable all interrupts. */
1339        writel(0, cqspi->iobase + CQSPI_REG_IRQMASK);
1340
1341        /* Configure the SRAM split to 1:1 . */
1342        writel(cqspi->fifo_depth / 2, cqspi->iobase + CQSPI_REG_SRAMPARTITION);
1343
1344        /* Load indirect trigger address. */
1345        writel(cqspi->trigger_address,
1346               cqspi->iobase + CQSPI_REG_INDIRECTTRIGGER);
1347
1348        /* Program read watermark -- 1/2 of the FIFO. */
1349        writel(cqspi->fifo_depth * cqspi->fifo_width / 2,
1350               cqspi->iobase + CQSPI_REG_INDIRECTRDWATERMARK);
1351        /* Program write watermark -- 1/8 of the FIFO. */
1352        writel(cqspi->fifo_depth * cqspi->fifo_width / 8,
1353               cqspi->iobase + CQSPI_REG_INDIRECTWRWATERMARK);
1354
1355        /* Disable direct access controller */
1356        if (!cqspi->use_direct_mode) {
1357                reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
1358                reg &= ~CQSPI_REG_CONFIG_ENB_DIR_ACC_CTRL;
1359                writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
1360        }
1361
1362        cqspi_controller_enable(cqspi, 1);
1363}
1364
1365static int cqspi_request_mmap_dma(struct cqspi_st *cqspi)
1366{
1367        dma_cap_mask_t mask;
1368
1369        dma_cap_zero(mask);
1370        dma_cap_set(DMA_MEMCPY, mask);
1371
1372        cqspi->rx_chan = dma_request_chan_by_mask(&mask);
1373        if (IS_ERR(cqspi->rx_chan)) {
1374                int ret = PTR_ERR(cqspi->rx_chan);
1375                cqspi->rx_chan = NULL;
1376                return dev_err_probe(&cqspi->pdev->dev, ret, "No Rx DMA available\n");
1377        }
1378        init_completion(&cqspi->rx_dma_complete);
1379
1380        return 0;
1381}
1382
1383static const char *cqspi_get_name(struct spi_mem *mem)
1384{
1385        struct cqspi_st *cqspi = spi_master_get_devdata(mem->spi->master);
1386        struct device *dev = &cqspi->pdev->dev;
1387
1388        return devm_kasprintf(dev, GFP_KERNEL, "%s.%d", dev_name(dev), mem->spi->chip_select);
1389}
1390
1391static const struct spi_controller_mem_ops cqspi_mem_ops = {
1392        .exec_op = cqspi_exec_mem_op,
1393        .get_name = cqspi_get_name,
1394        .supports_op = cqspi_supports_mem_op,
1395};
1396
1397static int cqspi_setup_flash(struct cqspi_st *cqspi)
1398{
1399        struct platform_device *pdev = cqspi->pdev;
1400        struct device *dev = &pdev->dev;
1401        struct device_node *np = dev->of_node;
1402        struct cqspi_flash_pdata *f_pdata;
1403        unsigned int cs;
1404        int ret;
1405
1406        /* Get flash device data */
1407        for_each_available_child_of_node(dev->of_node, np) {
1408                ret = of_property_read_u32(np, "reg", &cs);
1409                if (ret) {
1410                        dev_err(dev, "Couldn't determine chip select.\n");
1411                        of_node_put(np);
1412                        return ret;
1413                }
1414
1415                if (cs >= CQSPI_MAX_CHIPSELECT) {
1416                        dev_err(dev, "Chip select %d out of range.\n", cs);
1417                        of_node_put(np);
1418                        return -EINVAL;
1419                }
1420
1421                f_pdata = &cqspi->f_pdata[cs];
1422                f_pdata->cqspi = cqspi;
1423                f_pdata->cs = cs;
1424
1425                ret = cqspi_of_get_flash_pdata(pdev, f_pdata, np);
1426                if (ret) {
1427                        of_node_put(np);
1428                        return ret;
1429                }
1430        }
1431
1432        return 0;
1433}
1434
1435static int cqspi_probe(struct platform_device *pdev)
1436{
1437        const struct cqspi_driver_platdata *ddata;
1438        struct reset_control *rstc, *rstc_ocp;
1439        struct device *dev = &pdev->dev;
1440        struct spi_master *master;
1441        struct resource *res_ahb;
1442        struct cqspi_st *cqspi;
1443        struct resource *res;
1444        int ret;
1445        int irq;
1446
1447        master = spi_alloc_master(&pdev->dev, sizeof(*cqspi));
1448        if (!master) {
1449                dev_err(&pdev->dev, "spi_alloc_master failed\n");
1450                return -ENOMEM;
1451        }
1452        master->mode_bits = SPI_RX_QUAD | SPI_RX_DUAL;
1453        master->mem_ops = &cqspi_mem_ops;
1454        master->dev.of_node = pdev->dev.of_node;
1455
1456        cqspi = spi_master_get_devdata(master);
1457
1458        cqspi->pdev = pdev;
1459        platform_set_drvdata(pdev, cqspi);
1460
1461        /* Obtain configuration from OF. */
1462        ret = cqspi_of_get_pdata(cqspi);
1463        if (ret) {
1464                dev_err(dev, "Cannot get mandatory OF data.\n");
1465                ret = -ENODEV;
1466                goto probe_master_put;
1467        }
1468
1469        /* Obtain QSPI clock. */
1470        cqspi->clk = devm_clk_get(dev, NULL);
1471        if (IS_ERR(cqspi->clk)) {
1472                dev_err(dev, "Cannot claim QSPI clock.\n");
1473                ret = PTR_ERR(cqspi->clk);
1474                goto probe_master_put;
1475        }
1476
1477        /* Obtain and remap controller address. */
1478        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1479        cqspi->iobase = devm_ioremap_resource(dev, res);
1480        if (IS_ERR(cqspi->iobase)) {
1481                dev_err(dev, "Cannot remap controller address.\n");
1482                ret = PTR_ERR(cqspi->iobase);
1483                goto probe_master_put;
1484        }
1485
1486        /* Obtain and remap AHB address. */
1487        res_ahb = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1488        cqspi->ahb_base = devm_ioremap_resource(dev, res_ahb);
1489        if (IS_ERR(cqspi->ahb_base)) {
1490                dev_err(dev, "Cannot remap AHB address.\n");
1491                ret = PTR_ERR(cqspi->ahb_base);
1492                goto probe_master_put;
1493        }
1494        cqspi->mmap_phys_base = (dma_addr_t)res_ahb->start;
1495        cqspi->ahb_size = resource_size(res_ahb);
1496
1497        init_completion(&cqspi->transfer_complete);
1498
1499        /* Obtain IRQ line. */
1500        irq = platform_get_irq(pdev, 0);
1501        if (irq < 0) {
1502                ret = -ENXIO;
1503                goto probe_master_put;
1504        }
1505
1506        pm_runtime_enable(dev);
1507        ret = pm_runtime_get_sync(dev);
1508        if (ret < 0) {
1509                pm_runtime_put_noidle(dev);
1510                goto probe_master_put;
1511        }
1512
1513        ret = clk_prepare_enable(cqspi->clk);
1514        if (ret) {
1515                dev_err(dev, "Cannot enable QSPI clock.\n");
1516                goto probe_clk_failed;
1517        }
1518
1519        /* Obtain QSPI reset control */
1520        rstc = devm_reset_control_get_optional_exclusive(dev, "qspi");
1521        if (IS_ERR(rstc)) {
1522                ret = PTR_ERR(rstc);
1523                dev_err(dev, "Cannot get QSPI reset.\n");
1524                goto probe_reset_failed;
1525        }
1526
1527        rstc_ocp = devm_reset_control_get_optional_exclusive(dev, "qspi-ocp");
1528        if (IS_ERR(rstc_ocp)) {
1529                ret = PTR_ERR(rstc_ocp);
1530                dev_err(dev, "Cannot get QSPI OCP reset.\n");
1531                goto probe_reset_failed;
1532        }
1533
1534        reset_control_assert(rstc);
1535        reset_control_deassert(rstc);
1536
1537        reset_control_assert(rstc_ocp);
1538        reset_control_deassert(rstc_ocp);
1539
1540        cqspi->master_ref_clk_hz = clk_get_rate(cqspi->clk);
1541        master->max_speed_hz = cqspi->master_ref_clk_hz;
1542        ddata  = of_device_get_match_data(dev);
1543        if (ddata) {
1544                if (ddata->quirks & CQSPI_NEEDS_WR_DELAY)
1545                        cqspi->wr_delay = 50 * DIV_ROUND_UP(NSEC_PER_SEC,
1546                                                cqspi->master_ref_clk_hz);
1547                if (ddata->hwcaps_mask & CQSPI_SUPPORTS_OCTAL)
1548                        master->mode_bits |= SPI_RX_OCTAL | SPI_TX_OCTAL;
1549                if (!(ddata->quirks & CQSPI_DISABLE_DAC_MODE))
1550                        cqspi->use_direct_mode = true;
1551        }
1552
1553        ret = devm_request_irq(dev, irq, cqspi_irq_handler, 0,
1554                               pdev->name, cqspi);
1555        if (ret) {
1556                dev_err(dev, "Cannot request IRQ.\n");
1557                goto probe_reset_failed;
1558        }
1559
1560        cqspi_wait_idle(cqspi);
1561        cqspi_controller_init(cqspi);
1562        cqspi->current_cs = -1;
1563        cqspi->sclk = 0;
1564
1565        master->num_chipselect = cqspi->num_chipselect;
1566
1567        ret = cqspi_setup_flash(cqspi);
1568        if (ret) {
1569                dev_err(dev, "failed to setup flash parameters %d\n", ret);
1570                goto probe_setup_failed;
1571        }
1572
1573        if (cqspi->use_direct_mode) {
1574                ret = cqspi_request_mmap_dma(cqspi);
1575                if (ret == -EPROBE_DEFER)
1576                        goto probe_setup_failed;
1577        }
1578
1579        ret = devm_spi_register_master(dev, master);
1580        if (ret) {
1581                dev_err(&pdev->dev, "failed to register SPI ctlr %d\n", ret);
1582                goto probe_setup_failed;
1583        }
1584
1585        return 0;
1586probe_setup_failed:
1587        cqspi_controller_enable(cqspi, 0);
1588probe_reset_failed:
1589        clk_disable_unprepare(cqspi->clk);
1590probe_clk_failed:
1591        pm_runtime_put_sync(dev);
1592        pm_runtime_disable(dev);
1593probe_master_put:
1594        spi_master_put(master);
1595        return ret;
1596}
1597
1598static int cqspi_remove(struct platform_device *pdev)
1599{
1600        struct cqspi_st *cqspi = platform_get_drvdata(pdev);
1601
1602        cqspi_controller_enable(cqspi, 0);
1603
1604        if (cqspi->rx_chan)
1605                dma_release_channel(cqspi->rx_chan);
1606
1607        clk_disable_unprepare(cqspi->clk);
1608
1609        pm_runtime_put_sync(&pdev->dev);
1610        pm_runtime_disable(&pdev->dev);
1611
1612        return 0;
1613}
1614
1615#ifdef CONFIG_PM_SLEEP
1616static int cqspi_suspend(struct device *dev)
1617{
1618        struct cqspi_st *cqspi = dev_get_drvdata(dev);
1619
1620        cqspi_controller_enable(cqspi, 0);
1621        return 0;
1622}
1623
1624static int cqspi_resume(struct device *dev)
1625{
1626        struct cqspi_st *cqspi = dev_get_drvdata(dev);
1627
1628        cqspi_controller_enable(cqspi, 1);
1629        return 0;
1630}
1631
1632static const struct dev_pm_ops cqspi__dev_pm_ops = {
1633        .suspend = cqspi_suspend,
1634        .resume = cqspi_resume,
1635};
1636
1637#define CQSPI_DEV_PM_OPS        (&cqspi__dev_pm_ops)
1638#else
1639#define CQSPI_DEV_PM_OPS        NULL
1640#endif
1641
1642static const struct cqspi_driver_platdata cdns_qspi = {
1643        .quirks = CQSPI_DISABLE_DAC_MODE,
1644};
1645
1646static const struct cqspi_driver_platdata k2g_qspi = {
1647        .quirks = CQSPI_NEEDS_WR_DELAY,
1648};
1649
1650static const struct cqspi_driver_platdata am654_ospi = {
1651        .hwcaps_mask = CQSPI_SUPPORTS_OCTAL,
1652        .quirks = CQSPI_NEEDS_WR_DELAY,
1653};
1654
1655static const struct cqspi_driver_platdata intel_lgm_qspi = {
1656        .quirks = CQSPI_DISABLE_DAC_MODE,
1657};
1658
1659static const struct of_device_id cqspi_dt_ids[] = {
1660        {
1661                .compatible = "cdns,qspi-nor",
1662                .data = &cdns_qspi,
1663        },
1664        {
1665                .compatible = "ti,k2g-qspi",
1666                .data = &k2g_qspi,
1667        },
1668        {
1669                .compatible = "ti,am654-ospi",
1670                .data = &am654_ospi,
1671        },
1672        {
1673                .compatible = "intel,lgm-qspi",
1674                .data = &intel_lgm_qspi,
1675        },
1676        { /* end of table */ }
1677};
1678
1679MODULE_DEVICE_TABLE(of, cqspi_dt_ids);
1680
1681static struct platform_driver cqspi_platform_driver = {
1682        .probe = cqspi_probe,
1683        .remove = cqspi_remove,
1684        .driver = {
1685                .name = CQSPI_NAME,
1686                .pm = CQSPI_DEV_PM_OPS,
1687                .of_match_table = cqspi_dt_ids,
1688        },
1689};
1690
1691module_platform_driver(cqspi_platform_driver);
1692
1693MODULE_DESCRIPTION("Cadence QSPI Controller Driver");
1694MODULE_LICENSE("GPL v2");
1695MODULE_ALIAS("platform:" CQSPI_NAME);
1696MODULE_AUTHOR("Ley Foon Tan <lftan@altera.com>");
1697MODULE_AUTHOR("Graham Moore <grmoore@opensource.altera.com>");
1698MODULE_AUTHOR("Vadivel Murugan R <vadivel.muruganx.ramuthevar@intel.com>");
1699MODULE_AUTHOR("Vignesh Raghavendra <vigneshr@ti.com>");
1700MODULE_AUTHOR("Pratyush Yadav <p.yadav@ti.com>");
1701