linux/drivers/mtd/nand/raw/arasan-nand-controller.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Arasan NAND Flash Controller Driver
   4 *
   5 * Copyright (C) 2014 - 2020 Xilinx, Inc.
   6 * Author:
   7 *   Miquel Raynal <miquel.raynal@bootlin.com>
   8 * Original work (fully rewritten):
   9 *   Punnaiah Choudary Kalluri <punnaia@xilinx.com>
  10 *   Naga Sureshkumar Relli <nagasure@xilinx.com>
  11 */
  12
  13#include <linux/bch.h>
  14#include <linux/bitfield.h>
  15#include <linux/clk.h>
  16#include <linux/delay.h>
  17#include <linux/dma-mapping.h>
  18#include <linux/gpio/consumer.h>
  19#include <linux/interrupt.h>
  20#include <linux/iopoll.h>
  21#include <linux/module.h>
  22#include <linux/mtd/mtd.h>
  23#include <linux/mtd/partitions.h>
  24#include <linux/mtd/rawnand.h>
  25#include <linux/of.h>
  26#include <linux/platform_device.h>
  27#include <linux/slab.h>
  28
  29#define PKT_REG                         0x00
  30#define   PKT_SIZE(x)                   FIELD_PREP(GENMASK(10, 0), (x))
  31#define   PKT_STEPS(x)                  FIELD_PREP(GENMASK(23, 12), (x))
  32
  33#define MEM_ADDR1_REG                   0x04
  34
  35#define MEM_ADDR2_REG                   0x08
  36#define   ADDR2_STRENGTH(x)             FIELD_PREP(GENMASK(27, 25), (x))
  37#define   ADDR2_CS(x)                   FIELD_PREP(GENMASK(31, 30), (x))
  38
  39#define CMD_REG                         0x0C
  40#define   CMD_1(x)                      FIELD_PREP(GENMASK(7, 0), (x))
  41#define   CMD_2(x)                      FIELD_PREP(GENMASK(15, 8), (x))
  42#define   CMD_PAGE_SIZE(x)              FIELD_PREP(GENMASK(25, 23), (x))
  43#define   CMD_DMA_ENABLE                BIT(27)
  44#define   CMD_NADDRS(x)                 FIELD_PREP(GENMASK(30, 28), (x))
  45#define   CMD_ECC_ENABLE                BIT(31)
  46
  47#define PROG_REG                        0x10
  48#define   PROG_PGRD                     BIT(0)
  49#define   PROG_ERASE                    BIT(2)
  50#define   PROG_STATUS                   BIT(3)
  51#define   PROG_PGPROG                   BIT(4)
  52#define   PROG_RDID                     BIT(6)
  53#define   PROG_RDPARAM                  BIT(7)
  54#define   PROG_RST                      BIT(8)
  55#define   PROG_GET_FEATURE              BIT(9)
  56#define   PROG_SET_FEATURE              BIT(10)
  57#define   PROG_CHG_RD_COL_ENH           BIT(14)
  58
  59#define INTR_STS_EN_REG                 0x14
  60#define INTR_SIG_EN_REG                 0x18
  61#define INTR_STS_REG                    0x1C
  62#define   WRITE_READY                   BIT(0)
  63#define   READ_READY                    BIT(1)
  64#define   XFER_COMPLETE                 BIT(2)
  65#define   DMA_BOUNDARY                  BIT(6)
  66#define   EVENT_MASK                    GENMASK(7, 0)
  67
  68#define READY_STS_REG                   0x20
  69
  70#define DMA_ADDR0_REG                   0x50
  71#define DMA_ADDR1_REG                   0x24
  72
  73#define FLASH_STS_REG                   0x28
  74
  75#define TIMING_REG                      0x2C
  76#define   TCCS_TIME_500NS               0
  77#define   TCCS_TIME_300NS               3
  78#define   TCCS_TIME_200NS               2
  79#define   TCCS_TIME_100NS               1
  80#define   FAST_TCAD                     BIT(2)
  81#define   DQS_BUFF_SEL_IN(x)            FIELD_PREP(GENMASK(6, 3), (x))
  82#define   DQS_BUFF_SEL_OUT(x)           FIELD_PREP(GENMASK(18, 15), (x))
  83
  84#define DATA_PORT_REG                   0x30
  85
  86#define ECC_CONF_REG                    0x34
  87#define   ECC_CONF_COL(x)               FIELD_PREP(GENMASK(15, 0), (x))
  88#define   ECC_CONF_LEN(x)               FIELD_PREP(GENMASK(26, 16), (x))
  89#define   ECC_CONF_BCH_EN               BIT(27)
  90
  91#define ECC_ERR_CNT_REG                 0x38
  92#define   GET_PKT_ERR_CNT(x)            FIELD_GET(GENMASK(7, 0), (x))
  93#define   GET_PAGE_ERR_CNT(x)           FIELD_GET(GENMASK(16, 8), (x))
  94
  95#define ECC_SP_REG                      0x3C
  96#define   ECC_SP_CMD1(x)                FIELD_PREP(GENMASK(7, 0), (x))
  97#define   ECC_SP_CMD2(x)                FIELD_PREP(GENMASK(15, 8), (x))
  98#define   ECC_SP_ADDRS(x)               FIELD_PREP(GENMASK(30, 28), (x))
  99
 100#define ECC_1ERR_CNT_REG                0x40
 101#define ECC_2ERR_CNT_REG                0x44
 102
 103#define DATA_INTERFACE_REG              0x6C
 104#define   DIFACE_SDR_MODE(x)            FIELD_PREP(GENMASK(2, 0), (x))
 105#define   DIFACE_DDR_MODE(x)            FIELD_PREP(GENMASK(5, 3), (x))
 106#define   DIFACE_SDR                    0
 107#define   DIFACE_NVDDR                  BIT(9)
 108
 109#define ANFC_MAX_CS                     2
 110#define ANFC_DFLT_TIMEOUT_US            1000000
 111#define ANFC_MAX_CHUNK_SIZE             SZ_1M
 112#define ANFC_MAX_PARAM_SIZE             SZ_4K
 113#define ANFC_MAX_STEPS                  SZ_2K
 114#define ANFC_MAX_PKT_SIZE               (SZ_2K - 1)
 115#define ANFC_MAX_ADDR_CYC               5U
 116#define ANFC_RSVD_ECC_BYTES             21
 117
 118#define ANFC_XLNX_SDR_DFLT_CORE_CLK     100000000
 119#define ANFC_XLNX_SDR_HS_CORE_CLK       80000000
 120
 121static struct gpio_desc *anfc_default_cs_array[2] = {NULL, NULL};
 122
 123/**
 124 * struct anfc_op - Defines how to execute an operation
 125 * @pkt_reg: Packet register
 126 * @addr1_reg: Memory address 1 register
 127 * @addr2_reg: Memory address 2 register
 128 * @cmd_reg: Command register
 129 * @prog_reg: Program register
 130 * @steps: Number of "packets" to read/write
 131 * @rdy_timeout_ms: Timeout for waits on Ready/Busy pin
 132 * @len: Data transfer length
 133 * @read: Data transfer direction from the controller point of view
 134 * @buf: Data buffer
 135 */
 136struct anfc_op {
 137        u32 pkt_reg;
 138        u32 addr1_reg;
 139        u32 addr2_reg;
 140        u32 cmd_reg;
 141        u32 prog_reg;
 142        int steps;
 143        unsigned int rdy_timeout_ms;
 144        unsigned int len;
 145        bool read;
 146        u8 *buf;
 147};
 148
 149/**
 150 * struct anand - Defines the NAND chip related information
 151 * @node:               Used to store NAND chips into a list
 152 * @chip:               NAND chip information structure
 153 * @rb:                 Ready-busy line
 154 * @page_sz:            Register value of the page_sz field to use
 155 * @clk:                Expected clock frequency to use
 156 * @data_iface:         Data interface timing mode to use
 157 * @timings:            NV-DDR specific timings to use
 158 * @ecc_conf:           Hardware ECC configuration value
 159 * @strength:           Register value of the ECC strength
 160 * @raddr_cycles:       Row address cycle information
 161 * @caddr_cycles:       Column address cycle information
 162 * @ecc_bits:           Exact number of ECC bits per syndrome
 163 * @ecc_total:          Total number of ECC bytes
 164 * @errloc:             Array of errors located with soft BCH
 165 * @hw_ecc:             Buffer to store syndromes computed by hardware
 166 * @bch:                BCH structure
 167 * @cs_idx:             Array of chip-select for this device, values are indexes
 168 *                      of the controller structure @gpio_cs array
 169 * @ncs_idx:            Size of the @cs_idx array
 170 */
 171struct anand {
 172        struct list_head node;
 173        struct nand_chip chip;
 174        unsigned int rb;
 175        unsigned int page_sz;
 176        unsigned long clk;
 177        u32 data_iface;
 178        u32 timings;
 179        u32 ecc_conf;
 180        u32 strength;
 181        u16 raddr_cycles;
 182        u16 caddr_cycles;
 183        unsigned int ecc_bits;
 184        unsigned int ecc_total;
 185        unsigned int *errloc;
 186        u8 *hw_ecc;
 187        struct bch_control *bch;
 188        int *cs_idx;
 189        int ncs_idx;
 190};
 191
 192/**
 193 * struct arasan_nfc - Defines the Arasan NAND flash controller driver instance
 194 * @dev:                Pointer to the device structure
 195 * @base:               Remapped register area
 196 * @controller_clk:             Pointer to the system clock
 197 * @bus_clk:            Pointer to the flash clock
 198 * @controller:         Base controller structure
 199 * @chips:              List of all NAND chips attached to the controller
 200 * @cur_clk:            Current clock rate
 201 * @cs_array:           CS array. Native CS are left empty, the other cells are
 202 *                      populated with their corresponding GPIO descriptor.
 203 * @ncs:                Size of @cs_array
 204 * @cur_cs:             Index in @cs_array of the currently in use CS
 205 * @native_cs:          Currently selected native CS
 206 * @spare_cs:           Native CS that is not wired (may be selected when a GPIO
 207 *                      CS is in use)
 208 */
 209struct arasan_nfc {
 210        struct device *dev;
 211        void __iomem *base;
 212        struct clk *controller_clk;
 213        struct clk *bus_clk;
 214        struct nand_controller controller;
 215        struct list_head chips;
 216        unsigned int cur_clk;
 217        struct gpio_desc **cs_array;
 218        unsigned int ncs;
 219        int cur_cs;
 220        unsigned int native_cs;
 221        unsigned int spare_cs;
 222};
 223
 224static struct anand *to_anand(struct nand_chip *nand)
 225{
 226        return container_of(nand, struct anand, chip);
 227}
 228
 229static struct arasan_nfc *to_anfc(struct nand_controller *ctrl)
 230{
 231        return container_of(ctrl, struct arasan_nfc, controller);
 232}
 233
 234static int anfc_wait_for_event(struct arasan_nfc *nfc, unsigned int event)
 235{
 236        u32 val;
 237        int ret;
 238
 239        ret = readl_relaxed_poll_timeout(nfc->base + INTR_STS_REG, val,
 240                                         val & event, 0,
 241                                         ANFC_DFLT_TIMEOUT_US);
 242        if (ret) {
 243                dev_err(nfc->dev, "Timeout waiting for event 0x%x\n", event);
 244                return -ETIMEDOUT;
 245        }
 246
 247        writel_relaxed(event, nfc->base + INTR_STS_REG);
 248
 249        return 0;
 250}
 251
 252static int anfc_wait_for_rb(struct arasan_nfc *nfc, struct nand_chip *chip,
 253                            unsigned int timeout_ms)
 254{
 255        struct anand *anand = to_anand(chip);
 256        u32 val;
 257        int ret;
 258
 259        /* There is no R/B interrupt, we must poll a register */
 260        ret = readl_relaxed_poll_timeout(nfc->base + READY_STS_REG, val,
 261                                         val & BIT(anand->rb),
 262                                         1, timeout_ms * 1000);
 263        if (ret) {
 264                dev_err(nfc->dev, "Timeout waiting for R/B 0x%x\n",
 265                        readl_relaxed(nfc->base + READY_STS_REG));
 266                return -ETIMEDOUT;
 267        }
 268
 269        return 0;
 270}
 271
 272static void anfc_trigger_op(struct arasan_nfc *nfc, struct anfc_op *nfc_op)
 273{
 274        writel_relaxed(nfc_op->pkt_reg, nfc->base + PKT_REG);
 275        writel_relaxed(nfc_op->addr1_reg, nfc->base + MEM_ADDR1_REG);
 276        writel_relaxed(nfc_op->addr2_reg, nfc->base + MEM_ADDR2_REG);
 277        writel_relaxed(nfc_op->cmd_reg, nfc->base + CMD_REG);
 278        writel_relaxed(nfc_op->prog_reg, nfc->base + PROG_REG);
 279}
 280
 281static int anfc_pkt_len_config(unsigned int len, unsigned int *steps,
 282                               unsigned int *pktsize)
 283{
 284        unsigned int nb, sz;
 285
 286        for (nb = 1; nb < ANFC_MAX_STEPS; nb *= 2) {
 287                sz = len / nb;
 288                if (sz <= ANFC_MAX_PKT_SIZE)
 289                        break;
 290        }
 291
 292        if (sz * nb != len)
 293                return -ENOTSUPP;
 294
 295        if (steps)
 296                *steps = nb;
 297
 298        if (pktsize)
 299                *pktsize = sz;
 300
 301        return 0;
 302}
 303
 304static bool anfc_is_gpio_cs(struct arasan_nfc *nfc, int nfc_cs)
 305{
 306        return nfc_cs >= 0 && nfc->cs_array[nfc_cs];
 307}
 308
 309static int anfc_relative_to_absolute_cs(struct anand *anand, int num)
 310{
 311        return anand->cs_idx[num];
 312}
 313
 314static void anfc_assert_cs(struct arasan_nfc *nfc, unsigned int nfc_cs_idx)
 315{
 316        /* CS did not change: do nothing */
 317        if (nfc->cur_cs == nfc_cs_idx)
 318                return;
 319
 320        /* Deassert the previous CS if it was a GPIO */
 321        if (anfc_is_gpio_cs(nfc, nfc->cur_cs))
 322                gpiod_set_value_cansleep(nfc->cs_array[nfc->cur_cs], 1);
 323
 324        /* Assert the new one */
 325        if (anfc_is_gpio_cs(nfc, nfc_cs_idx)) {
 326                nfc->native_cs = nfc->spare_cs;
 327                gpiod_set_value_cansleep(nfc->cs_array[nfc_cs_idx], 0);
 328        } else {
 329                nfc->native_cs = nfc_cs_idx;
 330        }
 331
 332        nfc->cur_cs = nfc_cs_idx;
 333}
 334
 335static int anfc_select_target(struct nand_chip *chip, int target)
 336{
 337        struct anand *anand = to_anand(chip);
 338        struct arasan_nfc *nfc = to_anfc(chip->controller);
 339        unsigned int nfc_cs_idx = anfc_relative_to_absolute_cs(anand, target);
 340        int ret;
 341
 342        anfc_assert_cs(nfc, nfc_cs_idx);
 343
 344        /* Update the controller timings and the potential ECC configuration */
 345        writel_relaxed(anand->data_iface, nfc->base + DATA_INTERFACE_REG);
 346        writel_relaxed(anand->timings, nfc->base + TIMING_REG);
 347
 348        /* Update clock frequency */
 349        if (nfc->cur_clk != anand->clk) {
 350                clk_disable_unprepare(nfc->controller_clk);
 351                ret = clk_set_rate(nfc->controller_clk, anand->clk);
 352                if (ret) {
 353                        dev_err(nfc->dev, "Failed to change clock rate\n");
 354                        return ret;
 355                }
 356
 357                ret = clk_prepare_enable(nfc->controller_clk);
 358                if (ret) {
 359                        dev_err(nfc->dev,
 360                                "Failed to re-enable the controller clock\n");
 361                        return ret;
 362                }
 363
 364                nfc->cur_clk = anand->clk;
 365        }
 366
 367        return 0;
 368}
 369
 370/*
 371 * When using the embedded hardware ECC engine, the controller is in charge of
 372 * feeding the engine with, first, the ECC residue present in the data array.
 373 * A typical read operation is:
 374 * 1/ Assert the read operation by sending the relevant command/address cycles
 375 *    but targeting the column of the first ECC bytes in the OOB area instead of
 376 *    the main data directly.
 377 * 2/ After having read the relevant number of ECC bytes, the controller uses
 378 *    the RNDOUT/RNDSTART commands which are set into the "ECC Spare Command
 379 *    Register" to move the pointer back at the beginning of the main data.
 380 * 3/ It will read the content of the main area for a given size (pktsize) and
 381 *    will feed the ECC engine with this buffer again.
 382 * 4/ The ECC engine derives the ECC bytes for the given data and compare them
 383 *    with the ones already received. It eventually trigger status flags and
 384 *    then set the "Buffer Read Ready" flag.
 385 * 5/ The corrected data is then available for reading from the data port
 386 *    register.
 387 *
 388 * The hardware BCH ECC engine is known to be inconstent in BCH mode and never
 389 * reports uncorrectable errors. Because of this bug, we have to use the
 390 * software BCH implementation in the read path.
 391 */
 392static int anfc_read_page_hw_ecc(struct nand_chip *chip, u8 *buf,
 393                                 int oob_required, int page)
 394{
 395        struct arasan_nfc *nfc = to_anfc(chip->controller);
 396        struct mtd_info *mtd = nand_to_mtd(chip);
 397        struct anand *anand = to_anand(chip);
 398        unsigned int len = mtd->writesize + (oob_required ? mtd->oobsize : 0);
 399        unsigned int max_bitflips = 0;
 400        dma_addr_t dma_addr;
 401        int step, ret;
 402        struct anfc_op nfc_op = {
 403                .pkt_reg =
 404                        PKT_SIZE(chip->ecc.size) |
 405                        PKT_STEPS(chip->ecc.steps),
 406                .addr1_reg =
 407                        (page & 0xFF) << (8 * (anand->caddr_cycles)) |
 408                        (((page >> 8) & 0xFF) << (8 * (1 + anand->caddr_cycles))),
 409                .addr2_reg =
 410                        ((page >> 16) & 0xFF) |
 411                        ADDR2_STRENGTH(anand->strength) |
 412                        ADDR2_CS(nfc->native_cs),
 413                .cmd_reg =
 414                        CMD_1(NAND_CMD_READ0) |
 415                        CMD_2(NAND_CMD_READSTART) |
 416                        CMD_PAGE_SIZE(anand->page_sz) |
 417                        CMD_DMA_ENABLE |
 418                        CMD_NADDRS(anand->caddr_cycles +
 419                                   anand->raddr_cycles),
 420                .prog_reg = PROG_PGRD,
 421        };
 422
 423        dma_addr = dma_map_single(nfc->dev, (void *)buf, len, DMA_FROM_DEVICE);
 424        if (dma_mapping_error(nfc->dev, dma_addr)) {
 425                dev_err(nfc->dev, "Buffer mapping error");
 426                return -EIO;
 427        }
 428
 429        writel_relaxed(lower_32_bits(dma_addr), nfc->base + DMA_ADDR0_REG);
 430        writel_relaxed(upper_32_bits(dma_addr), nfc->base + DMA_ADDR1_REG);
 431
 432        anfc_trigger_op(nfc, &nfc_op);
 433
 434        ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
 435        dma_unmap_single(nfc->dev, dma_addr, len, DMA_FROM_DEVICE);
 436        if (ret) {
 437                dev_err(nfc->dev, "Error reading page %d\n", page);
 438                return ret;
 439        }
 440
 441        /* Store the raw OOB bytes as well */
 442        ret = nand_change_read_column_op(chip, mtd->writesize, chip->oob_poi,
 443                                         mtd->oobsize, 0);
 444        if (ret)
 445                return ret;
 446
 447        /*
 448         * For each step, compute by softare the BCH syndrome over the raw data.
 449         * Compare the theoretical amount of errors and compare with the
 450         * hardware engine feedback.
 451         */
 452        for (step = 0; step < chip->ecc.steps; step++) {
 453                u8 *raw_buf = &buf[step * chip->ecc.size];
 454                unsigned int bit, byte;
 455                int bf, i;
 456
 457                /* Extract the syndrome, it is not necessarily aligned */
 458                memset(anand->hw_ecc, 0, chip->ecc.bytes);
 459                nand_extract_bits(anand->hw_ecc, 0,
 460                                  &chip->oob_poi[mtd->oobsize - anand->ecc_total],
 461                                  anand->ecc_bits * step, anand->ecc_bits);
 462
 463                bf = bch_decode(anand->bch, raw_buf, chip->ecc.size,
 464                                anand->hw_ecc, NULL, NULL, anand->errloc);
 465                if (!bf) {
 466                        continue;
 467                } else if (bf > 0) {
 468                        for (i = 0; i < bf; i++) {
 469                                /* Only correct the data, not the syndrome */
 470                                if (anand->errloc[i] < (chip->ecc.size * 8)) {
 471                                        bit = BIT(anand->errloc[i] & 7);
 472                                        byte = anand->errloc[i] >> 3;
 473                                        raw_buf[byte] ^= bit;
 474                                }
 475                        }
 476
 477                        mtd->ecc_stats.corrected += bf;
 478                        max_bitflips = max_t(unsigned int, max_bitflips, bf);
 479
 480                        continue;
 481                }
 482
 483                bf = nand_check_erased_ecc_chunk(raw_buf, chip->ecc.size,
 484                                                 NULL, 0, NULL, 0,
 485                                                 chip->ecc.strength);
 486                if (bf > 0) {
 487                        mtd->ecc_stats.corrected += bf;
 488                        max_bitflips = max_t(unsigned int, max_bitflips, bf);
 489                        memset(raw_buf, 0xFF, chip->ecc.size);
 490                } else if (bf < 0) {
 491                        mtd->ecc_stats.failed++;
 492                }
 493        }
 494
 495        return 0;
 496}
 497
 498static int anfc_sel_read_page_hw_ecc(struct nand_chip *chip, u8 *buf,
 499                                     int oob_required, int page)
 500{
 501        int ret;
 502
 503        ret = anfc_select_target(chip, chip->cur_cs);
 504        if (ret)
 505                return ret;
 506
 507        return anfc_read_page_hw_ecc(chip, buf, oob_required, page);
 508};
 509
 510static int anfc_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf,
 511                                  int oob_required, int page)
 512{
 513        struct anand *anand = to_anand(chip);
 514        struct arasan_nfc *nfc = to_anfc(chip->controller);
 515        struct mtd_info *mtd = nand_to_mtd(chip);
 516        unsigned int len = mtd->writesize + (oob_required ? mtd->oobsize : 0);
 517        dma_addr_t dma_addr;
 518        int ret;
 519        struct anfc_op nfc_op = {
 520                .pkt_reg =
 521                        PKT_SIZE(chip->ecc.size) |
 522                        PKT_STEPS(chip->ecc.steps),
 523                .addr1_reg =
 524                        (page & 0xFF) << (8 * (anand->caddr_cycles)) |
 525                        (((page >> 8) & 0xFF) << (8 * (1 + anand->caddr_cycles))),
 526                .addr2_reg =
 527                        ((page >> 16) & 0xFF) |
 528                        ADDR2_STRENGTH(anand->strength) |
 529                        ADDR2_CS(nfc->native_cs),
 530                .cmd_reg =
 531                        CMD_1(NAND_CMD_SEQIN) |
 532                        CMD_2(NAND_CMD_PAGEPROG) |
 533                        CMD_PAGE_SIZE(anand->page_sz) |
 534                        CMD_DMA_ENABLE |
 535                        CMD_NADDRS(anand->caddr_cycles +
 536                                   anand->raddr_cycles) |
 537                        CMD_ECC_ENABLE,
 538                .prog_reg = PROG_PGPROG,
 539        };
 540
 541        writel_relaxed(anand->ecc_conf, nfc->base + ECC_CONF_REG);
 542        writel_relaxed(ECC_SP_CMD1(NAND_CMD_RNDIN) |
 543                       ECC_SP_ADDRS(anand->caddr_cycles),
 544                       nfc->base + ECC_SP_REG);
 545
 546        dma_addr = dma_map_single(nfc->dev, (void *)buf, len, DMA_TO_DEVICE);
 547        if (dma_mapping_error(nfc->dev, dma_addr)) {
 548                dev_err(nfc->dev, "Buffer mapping error");
 549                return -EIO;
 550        }
 551
 552        writel_relaxed(lower_32_bits(dma_addr), nfc->base + DMA_ADDR0_REG);
 553        writel_relaxed(upper_32_bits(dma_addr), nfc->base + DMA_ADDR1_REG);
 554
 555        anfc_trigger_op(nfc, &nfc_op);
 556        ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
 557        dma_unmap_single(nfc->dev, dma_addr, len, DMA_TO_DEVICE);
 558        if (ret) {
 559                dev_err(nfc->dev, "Error writing page %d\n", page);
 560                return ret;
 561        }
 562
 563        /* Spare data is not protected */
 564        if (oob_required)
 565                ret = nand_write_oob_std(chip, page);
 566
 567        return ret;
 568}
 569
 570static int anfc_sel_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf,
 571                                      int oob_required, int page)
 572{
 573        int ret;
 574
 575        ret = anfc_select_target(chip, chip->cur_cs);
 576        if (ret)
 577                return ret;
 578
 579        return anfc_write_page_hw_ecc(chip, buf, oob_required, page);
 580};
 581
 582/* NAND framework ->exec_op() hooks and related helpers */
 583static int anfc_parse_instructions(struct nand_chip *chip,
 584                                   const struct nand_subop *subop,
 585                                   struct anfc_op *nfc_op)
 586{
 587        struct arasan_nfc *nfc = to_anfc(chip->controller);
 588        struct anand *anand = to_anand(chip);
 589        const struct nand_op_instr *instr = NULL;
 590        bool first_cmd = true;
 591        unsigned int op_id;
 592        int ret, i;
 593
 594        memset(nfc_op, 0, sizeof(*nfc_op));
 595        nfc_op->addr2_reg = ADDR2_CS(nfc->native_cs);
 596        nfc_op->cmd_reg = CMD_PAGE_SIZE(anand->page_sz);
 597
 598        for (op_id = 0; op_id < subop->ninstrs; op_id++) {
 599                unsigned int offset, naddrs, pktsize;
 600                const u8 *addrs;
 601                u8 *buf;
 602
 603                instr = &subop->instrs[op_id];
 604
 605                switch (instr->type) {
 606                case NAND_OP_CMD_INSTR:
 607                        if (first_cmd)
 608                                nfc_op->cmd_reg |= CMD_1(instr->ctx.cmd.opcode);
 609                        else
 610                                nfc_op->cmd_reg |= CMD_2(instr->ctx.cmd.opcode);
 611
 612                        first_cmd = false;
 613                        break;
 614
 615                case NAND_OP_ADDR_INSTR:
 616                        offset = nand_subop_get_addr_start_off(subop, op_id);
 617                        naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
 618                        addrs = &instr->ctx.addr.addrs[offset];
 619                        nfc_op->cmd_reg |= CMD_NADDRS(naddrs);
 620
 621                        for (i = 0; i < min(ANFC_MAX_ADDR_CYC, naddrs); i++) {
 622                                if (i < 4)
 623                                        nfc_op->addr1_reg |= (u32)addrs[i] << i * 8;
 624                                else
 625                                        nfc_op->addr2_reg |= addrs[i];
 626                        }
 627
 628                        break;
 629                case NAND_OP_DATA_IN_INSTR:
 630                        nfc_op->read = true;
 631                        fallthrough;
 632                case NAND_OP_DATA_OUT_INSTR:
 633                        offset = nand_subop_get_data_start_off(subop, op_id);
 634                        buf = instr->ctx.data.buf.in;
 635                        nfc_op->buf = &buf[offset];
 636                        nfc_op->len = nand_subop_get_data_len(subop, op_id);
 637                        ret = anfc_pkt_len_config(nfc_op->len, &nfc_op->steps,
 638                                                  &pktsize);
 639                        if (ret)
 640                                return ret;
 641
 642                        /*
 643                         * Number of DATA cycles must be aligned on 4, this
 644                         * means the controller might read/write more than
 645                         * requested. This is harmless most of the time as extra
 646                         * DATA are discarded in the write path and read pointer
 647                         * adjusted in the read path.
 648                         *
 649                         * FIXME: The core should mark operations where
 650                         * reading/writing more is allowed so the exec_op()
 651                         * implementation can take the right decision when the
 652                         * alignment constraint is not met: adjust the number of
 653                         * DATA cycles when it's allowed, reject the operation
 654                         * otherwise.
 655                         */
 656                        nfc_op->pkt_reg |= PKT_SIZE(round_up(pktsize, 4)) |
 657                                           PKT_STEPS(nfc_op->steps);
 658                        break;
 659                case NAND_OP_WAITRDY_INSTR:
 660                        nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
 661                        break;
 662                }
 663        }
 664
 665        return 0;
 666}
 667
 668static int anfc_rw_pio_op(struct arasan_nfc *nfc, struct anfc_op *nfc_op)
 669{
 670        unsigned int dwords = (nfc_op->len / 4) / nfc_op->steps;
 671        unsigned int last_len = nfc_op->len % 4;
 672        unsigned int offset, dir;
 673        u8 *buf = nfc_op->buf;
 674        int ret, i;
 675
 676        for (i = 0; i < nfc_op->steps; i++) {
 677                dir = nfc_op->read ? READ_READY : WRITE_READY;
 678                ret = anfc_wait_for_event(nfc, dir);
 679                if (ret) {
 680                        dev_err(nfc->dev, "PIO %s ready signal not received\n",
 681                                nfc_op->read ? "Read" : "Write");
 682                        return ret;
 683                }
 684
 685                offset = i * (dwords * 4);
 686                if (nfc_op->read)
 687                        ioread32_rep(nfc->base + DATA_PORT_REG, &buf[offset],
 688                                     dwords);
 689                else
 690                        iowrite32_rep(nfc->base + DATA_PORT_REG, &buf[offset],
 691                                      dwords);
 692        }
 693
 694        if (last_len) {
 695                u32 remainder;
 696
 697                offset = nfc_op->len - last_len;
 698
 699                if (nfc_op->read) {
 700                        remainder = readl_relaxed(nfc->base + DATA_PORT_REG);
 701                        memcpy(&buf[offset], &remainder, last_len);
 702                } else {
 703                        memcpy(&remainder, &buf[offset], last_len);
 704                        writel_relaxed(remainder, nfc->base + DATA_PORT_REG);
 705                }
 706        }
 707
 708        return anfc_wait_for_event(nfc, XFER_COMPLETE);
 709}
 710
 711static int anfc_misc_data_type_exec(struct nand_chip *chip,
 712                                    const struct nand_subop *subop,
 713                                    u32 prog_reg)
 714{
 715        struct arasan_nfc *nfc = to_anfc(chip->controller);
 716        struct anfc_op nfc_op = {};
 717        int ret;
 718
 719        ret = anfc_parse_instructions(chip, subop, &nfc_op);
 720        if (ret)
 721                return ret;
 722
 723        nfc_op.prog_reg = prog_reg;
 724        anfc_trigger_op(nfc, &nfc_op);
 725
 726        if (nfc_op.rdy_timeout_ms) {
 727                ret = anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
 728                if (ret)
 729                        return ret;
 730        }
 731
 732        return anfc_rw_pio_op(nfc, &nfc_op);
 733}
 734
 735static int anfc_param_read_type_exec(struct nand_chip *chip,
 736                                     const struct nand_subop *subop)
 737{
 738        return anfc_misc_data_type_exec(chip, subop, PROG_RDPARAM);
 739}
 740
 741static int anfc_data_read_type_exec(struct nand_chip *chip,
 742                                    const struct nand_subop *subop)
 743{
 744        u32 prog_reg = PROG_PGRD;
 745
 746        /*
 747         * Experience shows that while in SDR mode sending a CHANGE READ COLUMN
 748         * command through the READ PAGE "type" always works fine, when in
 749         * NV-DDR mode the same command simply fails. However, it was also
 750         * spotted that any CHANGE READ COLUMN command sent through the CHANGE
 751         * READ COLUMN ENHANCED "type" would correctly work in both cases (SDR
 752         * and NV-DDR). So, for simplicity, let's program the controller with
 753         * the CHANGE READ COLUMN ENHANCED "type" whenever we are requested to
 754         * perform a CHANGE READ COLUMN operation.
 755         */
 756        if (subop->instrs[0].ctx.cmd.opcode == NAND_CMD_RNDOUT &&
 757            subop->instrs[2].ctx.cmd.opcode == NAND_CMD_RNDOUTSTART)
 758                prog_reg = PROG_CHG_RD_COL_ENH;
 759
 760        return anfc_misc_data_type_exec(chip, subop, prog_reg);
 761}
 762
 763static int anfc_param_write_type_exec(struct nand_chip *chip,
 764                                      const struct nand_subop *subop)
 765{
 766        return anfc_misc_data_type_exec(chip, subop, PROG_SET_FEATURE);
 767}
 768
 769static int anfc_data_write_type_exec(struct nand_chip *chip,
 770                                     const struct nand_subop *subop)
 771{
 772        return anfc_misc_data_type_exec(chip, subop, PROG_PGPROG);
 773}
 774
 775static int anfc_misc_zerolen_type_exec(struct nand_chip *chip,
 776                                       const struct nand_subop *subop,
 777                                       u32 prog_reg)
 778{
 779        struct arasan_nfc *nfc = to_anfc(chip->controller);
 780        struct anfc_op nfc_op = {};
 781        int ret;
 782
 783        ret = anfc_parse_instructions(chip, subop, &nfc_op);
 784        if (ret)
 785                return ret;
 786
 787        nfc_op.prog_reg = prog_reg;
 788        anfc_trigger_op(nfc, &nfc_op);
 789
 790        ret = anfc_wait_for_event(nfc, XFER_COMPLETE);
 791        if (ret)
 792                return ret;
 793
 794        if (nfc_op.rdy_timeout_ms)
 795                ret = anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
 796
 797        return ret;
 798}
 799
 800static int anfc_status_type_exec(struct nand_chip *chip,
 801                                 const struct nand_subop *subop)
 802{
 803        struct arasan_nfc *nfc = to_anfc(chip->controller);
 804        u32 tmp;
 805        int ret;
 806
 807        /* See anfc_check_op() for details about this constraint */
 808        if (subop->instrs[0].ctx.cmd.opcode != NAND_CMD_STATUS)
 809                return -ENOTSUPP;
 810
 811        ret = anfc_misc_zerolen_type_exec(chip, subop, PROG_STATUS);
 812        if (ret)
 813                return ret;
 814
 815        tmp = readl_relaxed(nfc->base + FLASH_STS_REG);
 816        memcpy(subop->instrs[1].ctx.data.buf.in, &tmp, 1);
 817
 818        return 0;
 819}
 820
 821static int anfc_reset_type_exec(struct nand_chip *chip,
 822                                const struct nand_subop *subop)
 823{
 824        return anfc_misc_zerolen_type_exec(chip, subop, PROG_RST);
 825}
 826
 827static int anfc_erase_type_exec(struct nand_chip *chip,
 828                                const struct nand_subop *subop)
 829{
 830        return anfc_misc_zerolen_type_exec(chip, subop, PROG_ERASE);
 831}
 832
 833static int anfc_wait_type_exec(struct nand_chip *chip,
 834                               const struct nand_subop *subop)
 835{
 836        struct arasan_nfc *nfc = to_anfc(chip->controller);
 837        struct anfc_op nfc_op = {};
 838        int ret;
 839
 840        ret = anfc_parse_instructions(chip, subop, &nfc_op);
 841        if (ret)
 842                return ret;
 843
 844        return anfc_wait_for_rb(nfc, chip, nfc_op.rdy_timeout_ms);
 845}
 846
 847static const struct nand_op_parser anfc_op_parser = NAND_OP_PARSER(
 848        NAND_OP_PARSER_PATTERN(
 849                anfc_param_read_type_exec,
 850                NAND_OP_PARSER_PAT_CMD_ELEM(false),
 851                NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 852                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
 853                NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, ANFC_MAX_CHUNK_SIZE)),
 854        NAND_OP_PARSER_PATTERN(
 855                anfc_param_write_type_exec,
 856                NAND_OP_PARSER_PAT_CMD_ELEM(false),
 857                NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 858                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, ANFC_MAX_PARAM_SIZE)),
 859        NAND_OP_PARSER_PATTERN(
 860                anfc_data_read_type_exec,
 861                NAND_OP_PARSER_PAT_CMD_ELEM(false),
 862                NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 863                NAND_OP_PARSER_PAT_CMD_ELEM(false),
 864                NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
 865                NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, ANFC_MAX_CHUNK_SIZE)),
 866        NAND_OP_PARSER_PATTERN(
 867                anfc_data_write_type_exec,
 868                NAND_OP_PARSER_PAT_CMD_ELEM(false),
 869                NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 870                NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, ANFC_MAX_CHUNK_SIZE),
 871                NAND_OP_PARSER_PAT_CMD_ELEM(false)),
 872        NAND_OP_PARSER_PATTERN(
 873                anfc_reset_type_exec,
 874                NAND_OP_PARSER_PAT_CMD_ELEM(false),
 875                NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
 876        NAND_OP_PARSER_PATTERN(
 877                anfc_erase_type_exec,
 878                NAND_OP_PARSER_PAT_CMD_ELEM(false),
 879                NAND_OP_PARSER_PAT_ADDR_ELEM(false, ANFC_MAX_ADDR_CYC),
 880                NAND_OP_PARSER_PAT_CMD_ELEM(false),
 881                NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
 882        NAND_OP_PARSER_PATTERN(
 883                anfc_status_type_exec,
 884                NAND_OP_PARSER_PAT_CMD_ELEM(false),
 885                NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, ANFC_MAX_CHUNK_SIZE)),
 886        NAND_OP_PARSER_PATTERN(
 887                anfc_wait_type_exec,
 888                NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
 889        );
 890
 891static int anfc_check_op(struct nand_chip *chip,
 892                         const struct nand_operation *op)
 893{
 894        const struct nand_op_instr *instr;
 895        int op_id;
 896
 897        /*
 898         * The controller abstracts all the NAND operations and do not support
 899         * data only operations.
 900         *
 901         * TODO: The nand_op_parser framework should be extended to
 902         * support custom checks on DATA instructions.
 903         */
 904        for (op_id = 0; op_id < op->ninstrs; op_id++) {
 905                instr = &op->instrs[op_id];
 906
 907                switch (instr->type) {
 908                case NAND_OP_ADDR_INSTR:
 909                        if (instr->ctx.addr.naddrs > ANFC_MAX_ADDR_CYC)
 910                                return -ENOTSUPP;
 911
 912                        break;
 913                case NAND_OP_DATA_IN_INSTR:
 914                case NAND_OP_DATA_OUT_INSTR:
 915                        if (instr->ctx.data.len > ANFC_MAX_CHUNK_SIZE)
 916                                return -ENOTSUPP;
 917
 918                        if (anfc_pkt_len_config(instr->ctx.data.len, 0, 0))
 919                                return -ENOTSUPP;
 920
 921                        break;
 922                default:
 923                        break;
 924                }
 925        }
 926
 927        /*
 928         * The controller does not allow to proceed with a CMD+DATA_IN cycle
 929         * manually on the bus by reading data from the data register. Instead,
 930         * the controller abstract a status read operation with its own status
 931         * register after ordering a read status operation. Hence, we cannot
 932         * support any CMD+DATA_IN operation other than a READ STATUS.
 933         *
 934         * TODO: The nand_op_parser() framework should be extended to describe
 935         * fixed patterns instead of open-coding this check here.
 936         */
 937        if (op->ninstrs == 2 &&
 938            op->instrs[0].type == NAND_OP_CMD_INSTR &&
 939            op->instrs[0].ctx.cmd.opcode != NAND_CMD_STATUS &&
 940            op->instrs[1].type == NAND_OP_DATA_IN_INSTR)
 941                return -ENOTSUPP;
 942
 943        return nand_op_parser_exec_op(chip, &anfc_op_parser, op, true);
 944}
 945
 946static int anfc_exec_op(struct nand_chip *chip,
 947                        const struct nand_operation *op,
 948                        bool check_only)
 949{
 950        int ret;
 951
 952        if (check_only)
 953                return anfc_check_op(chip, op);
 954
 955        ret = anfc_select_target(chip, op->cs);
 956        if (ret)
 957                return ret;
 958
 959        return nand_op_parser_exec_op(chip, &anfc_op_parser, op, check_only);
 960}
 961
 962static int anfc_setup_interface(struct nand_chip *chip, int target,
 963                                const struct nand_interface_config *conf)
 964{
 965        struct anand *anand = to_anand(chip);
 966        struct arasan_nfc *nfc = to_anfc(chip->controller);
 967        struct device_node *np = nfc->dev->of_node;
 968        const struct nand_sdr_timings *sdr;
 969        const struct nand_nvddr_timings *nvddr;
 970        unsigned int tccs_min, dqs_mode, fast_tcad;
 971
 972        if (nand_interface_is_nvddr(conf)) {
 973                nvddr = nand_get_nvddr_timings(conf);
 974                if (IS_ERR(nvddr))
 975                        return PTR_ERR(nvddr);
 976        } else {
 977                sdr = nand_get_sdr_timings(conf);
 978                if (IS_ERR(sdr))
 979                        return PTR_ERR(sdr);
 980        }
 981
 982        if (target < 0)
 983                return 0;
 984
 985        if (nand_interface_is_sdr(conf)) {
 986                anand->data_iface = DIFACE_SDR |
 987                                    DIFACE_SDR_MODE(conf->timings.mode);
 988                anand->timings = 0;
 989        } else {
 990                anand->data_iface = DIFACE_NVDDR |
 991                                    DIFACE_DDR_MODE(conf->timings.mode);
 992
 993                if (conf->timings.nvddr.tCCS_min <= 100000)
 994                        tccs_min = TCCS_TIME_100NS;
 995                else if (conf->timings.nvddr.tCCS_min <= 200000)
 996                        tccs_min = TCCS_TIME_200NS;
 997                else if (conf->timings.nvddr.tCCS_min <= 300000)
 998                        tccs_min = TCCS_TIME_300NS;
 999                else
1000                        tccs_min = TCCS_TIME_500NS;
1001
1002                fast_tcad = 0;
1003                if (conf->timings.nvddr.tCAD_min < 45000)
1004                        fast_tcad = FAST_TCAD;
1005
1006                switch (conf->timings.mode) {
1007                case 5:
1008                case 4:
1009                        dqs_mode = 2;
1010                        break;
1011                case 3:
1012                        dqs_mode = 3;
1013                        break;
1014                case 2:
1015                        dqs_mode = 4;
1016                        break;
1017                case 1:
1018                        dqs_mode = 5;
1019                        break;
1020                case 0:
1021                default:
1022                        dqs_mode = 6;
1023                        break;
1024                }
1025
1026                anand->timings = tccs_min | fast_tcad |
1027                                 DQS_BUFF_SEL_IN(dqs_mode) |
1028                                 DQS_BUFF_SEL_OUT(dqs_mode);
1029        }
1030
1031        anand->clk = ANFC_XLNX_SDR_DFLT_CORE_CLK;
1032
1033        /*
1034         * Due to a hardware bug in the ZynqMP SoC, SDR timing modes 0-1 work
1035         * with f > 90MHz (default clock is 100MHz) but signals are unstable
1036         * with higher modes. Hence we decrease a little bit the clock rate to
1037         * 80MHz when using SDR modes 2-5 with this SoC.
1038         */
1039        if (of_device_is_compatible(np, "xlnx,zynqmp-nand-controller") &&
1040            nand_interface_is_sdr(conf) && conf->timings.mode >= 2)
1041                anand->clk = ANFC_XLNX_SDR_HS_CORE_CLK;
1042
1043        return 0;
1044}
1045
1046static int anfc_calc_hw_ecc_bytes(int step_size, int strength)
1047{
1048        unsigned int bch_gf_mag, ecc_bits;
1049
1050        switch (step_size) {
1051        case SZ_512:
1052                bch_gf_mag = 13;
1053                break;
1054        case SZ_1K:
1055                bch_gf_mag = 14;
1056                break;
1057        default:
1058                return -EINVAL;
1059        }
1060
1061        ecc_bits = bch_gf_mag * strength;
1062
1063        return DIV_ROUND_UP(ecc_bits, 8);
1064}
1065
1066static const int anfc_hw_ecc_512_strengths[] = {4, 8, 12};
1067
1068static const int anfc_hw_ecc_1024_strengths[] = {24};
1069
1070static const struct nand_ecc_step_info anfc_hw_ecc_step_infos[] = {
1071        {
1072                .stepsize = SZ_512,
1073                .strengths = anfc_hw_ecc_512_strengths,
1074                .nstrengths = ARRAY_SIZE(anfc_hw_ecc_512_strengths),
1075        },
1076        {
1077                .stepsize = SZ_1K,
1078                .strengths = anfc_hw_ecc_1024_strengths,
1079                .nstrengths = ARRAY_SIZE(anfc_hw_ecc_1024_strengths),
1080        },
1081};
1082
1083static const struct nand_ecc_caps anfc_hw_ecc_caps = {
1084        .stepinfos = anfc_hw_ecc_step_infos,
1085        .nstepinfos = ARRAY_SIZE(anfc_hw_ecc_step_infos),
1086        .calc_ecc_bytes = anfc_calc_hw_ecc_bytes,
1087};
1088
1089static int anfc_init_hw_ecc_controller(struct arasan_nfc *nfc,
1090                                       struct nand_chip *chip)
1091{
1092        struct anand *anand = to_anand(chip);
1093        struct mtd_info *mtd = nand_to_mtd(chip);
1094        struct nand_ecc_ctrl *ecc = &chip->ecc;
1095        unsigned int bch_prim_poly = 0, bch_gf_mag = 0, ecc_offset;
1096        int ret;
1097
1098        switch (mtd->writesize) {
1099        case SZ_512:
1100        case SZ_2K:
1101        case SZ_4K:
1102        case SZ_8K:
1103        case SZ_16K:
1104                break;
1105        default:
1106                dev_err(nfc->dev, "Unsupported page size %d\n", mtd->writesize);
1107                return -EINVAL;
1108        }
1109
1110        ret = nand_ecc_choose_conf(chip, &anfc_hw_ecc_caps, mtd->oobsize);
1111        if (ret)
1112                return ret;
1113
1114        switch (ecc->strength) {
1115        case 12:
1116                anand->strength = 0x1;
1117                break;
1118        case 8:
1119                anand->strength = 0x2;
1120                break;
1121        case 4:
1122                anand->strength = 0x3;
1123                break;
1124        case 24:
1125                anand->strength = 0x4;
1126                break;
1127        default:
1128                dev_err(nfc->dev, "Unsupported strength %d\n", ecc->strength);
1129                return -EINVAL;
1130        }
1131
1132        switch (ecc->size) {
1133        case SZ_512:
1134                bch_gf_mag = 13;
1135                bch_prim_poly = 0x201b;
1136                break;
1137        case SZ_1K:
1138                bch_gf_mag = 14;
1139                bch_prim_poly = 0x4443;
1140                break;
1141        default:
1142                dev_err(nfc->dev, "Unsupported step size %d\n", ecc->strength);
1143                return -EINVAL;
1144        }
1145
1146        mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout());
1147
1148        ecc->steps = mtd->writesize / ecc->size;
1149        ecc->algo = NAND_ECC_ALGO_BCH;
1150        anand->ecc_bits = bch_gf_mag * ecc->strength;
1151        ecc->bytes = DIV_ROUND_UP(anand->ecc_bits, 8);
1152        anand->ecc_total = DIV_ROUND_UP(anand->ecc_bits * ecc->steps, 8);
1153        ecc_offset = mtd->writesize + mtd->oobsize - anand->ecc_total;
1154        anand->ecc_conf = ECC_CONF_COL(ecc_offset) |
1155                          ECC_CONF_LEN(anand->ecc_total) |
1156                          ECC_CONF_BCH_EN;
1157
1158        anand->errloc = devm_kmalloc_array(nfc->dev, ecc->strength,
1159                                           sizeof(*anand->errloc), GFP_KERNEL);
1160        if (!anand->errloc)
1161                return -ENOMEM;
1162
1163        anand->hw_ecc = devm_kmalloc(nfc->dev, ecc->bytes, GFP_KERNEL);
1164        if (!anand->hw_ecc)
1165                return -ENOMEM;
1166
1167        /* Enforce bit swapping to fit the hardware */
1168        anand->bch = bch_init(bch_gf_mag, ecc->strength, bch_prim_poly, true);
1169        if (!anand->bch)
1170                return -EINVAL;
1171
1172        ecc->read_page = anfc_sel_read_page_hw_ecc;
1173        ecc->write_page = anfc_sel_write_page_hw_ecc;
1174
1175        return 0;
1176}
1177
1178static int anfc_attach_chip(struct nand_chip *chip)
1179{
1180        struct anand *anand = to_anand(chip);
1181        struct arasan_nfc *nfc = to_anfc(chip->controller);
1182        struct mtd_info *mtd = nand_to_mtd(chip);
1183        int ret = 0;
1184
1185        if (mtd->writesize <= SZ_512)
1186                anand->caddr_cycles = 1;
1187        else
1188                anand->caddr_cycles = 2;
1189
1190        if (chip->options & NAND_ROW_ADDR_3)
1191                anand->raddr_cycles = 3;
1192        else
1193                anand->raddr_cycles = 2;
1194
1195        switch (mtd->writesize) {
1196        case 512:
1197                anand->page_sz = 0;
1198                break;
1199        case 1024:
1200                anand->page_sz = 5;
1201                break;
1202        case 2048:
1203                anand->page_sz = 1;
1204                break;
1205        case 4096:
1206                anand->page_sz = 2;
1207                break;
1208        case 8192:
1209                anand->page_sz = 3;
1210                break;
1211        case 16384:
1212                anand->page_sz = 4;
1213                break;
1214        default:
1215                return -EINVAL;
1216        }
1217
1218        /* These hooks are valid for all ECC providers */
1219        chip->ecc.read_page_raw = nand_monolithic_read_page_raw;
1220        chip->ecc.write_page_raw = nand_monolithic_write_page_raw;
1221
1222        switch (chip->ecc.engine_type) {
1223        case NAND_ECC_ENGINE_TYPE_NONE:
1224        case NAND_ECC_ENGINE_TYPE_SOFT:
1225        case NAND_ECC_ENGINE_TYPE_ON_DIE:
1226                break;
1227        case NAND_ECC_ENGINE_TYPE_ON_HOST:
1228                ret = anfc_init_hw_ecc_controller(nfc, chip);
1229                break;
1230        default:
1231                dev_err(nfc->dev, "Unsupported ECC mode: %d\n",
1232                        chip->ecc.engine_type);
1233                return -EINVAL;
1234        }
1235
1236        return ret;
1237}
1238
1239static void anfc_detach_chip(struct nand_chip *chip)
1240{
1241        struct anand *anand = to_anand(chip);
1242
1243        if (anand->bch)
1244                bch_free(anand->bch);
1245}
1246
1247static const struct nand_controller_ops anfc_ops = {
1248        .exec_op = anfc_exec_op,
1249        .setup_interface = anfc_setup_interface,
1250        .attach_chip = anfc_attach_chip,
1251        .detach_chip = anfc_detach_chip,
1252};
1253
1254static int anfc_chip_init(struct arasan_nfc *nfc, struct device_node *np)
1255{
1256        struct anand *anand;
1257        struct nand_chip *chip;
1258        struct mtd_info *mtd;
1259        int rb, ret, i;
1260
1261        anand = devm_kzalloc(nfc->dev, sizeof(*anand), GFP_KERNEL);
1262        if (!anand)
1263                return -ENOMEM;
1264
1265        /* Chip-select init */
1266        anand->ncs_idx = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1267        if (anand->ncs_idx <= 0 || anand->ncs_idx > nfc->ncs) {
1268                dev_err(nfc->dev, "Invalid reg property\n");
1269                return -EINVAL;
1270        }
1271
1272        anand->cs_idx = devm_kcalloc(nfc->dev, anand->ncs_idx,
1273                                     sizeof(*anand->cs_idx), GFP_KERNEL);
1274        if (!anand->cs_idx)
1275                return -ENOMEM;
1276
1277        for (i = 0; i < anand->ncs_idx; i++) {
1278                ret = of_property_read_u32_index(np, "reg", i,
1279                                                 &anand->cs_idx[i]);
1280                if (ret) {
1281                        dev_err(nfc->dev, "invalid CS property: %d\n", ret);
1282                        return ret;
1283                }
1284        }
1285
1286        /* Ready-busy init */
1287        ret = of_property_read_u32(np, "nand-rb", &rb);
1288        if (ret)
1289                return ret;
1290
1291        if (rb >= ANFC_MAX_CS) {
1292                dev_err(nfc->dev, "Wrong RB %d\n", rb);
1293                return -EINVAL;
1294        }
1295
1296        anand->rb = rb;
1297
1298        chip = &anand->chip;
1299        mtd = nand_to_mtd(chip);
1300        mtd->dev.parent = nfc->dev;
1301        chip->controller = &nfc->controller;
1302        chip->options = NAND_BUSWIDTH_AUTO | NAND_NO_SUBPAGE_WRITE |
1303                        NAND_USES_DMA;
1304
1305        nand_set_flash_node(chip, np);
1306        if (!mtd->name) {
1307                dev_err(nfc->dev, "NAND label property is mandatory\n");
1308                return -EINVAL;
1309        }
1310
1311        ret = nand_scan(chip, anand->ncs_idx);
1312        if (ret) {
1313                dev_err(nfc->dev, "Scan operation failed\n");
1314                return ret;
1315        }
1316
1317        ret = mtd_device_register(mtd, NULL, 0);
1318        if (ret) {
1319                nand_cleanup(chip);
1320                return ret;
1321        }
1322
1323        list_add_tail(&anand->node, &nfc->chips);
1324
1325        return 0;
1326}
1327
1328static void anfc_chips_cleanup(struct arasan_nfc *nfc)
1329{
1330        struct anand *anand, *tmp;
1331        struct nand_chip *chip;
1332        int ret;
1333
1334        list_for_each_entry_safe(anand, tmp, &nfc->chips, node) {
1335                chip = &anand->chip;
1336                ret = mtd_device_unregister(nand_to_mtd(chip));
1337                WARN_ON(ret);
1338                nand_cleanup(chip);
1339                list_del(&anand->node);
1340        }
1341}
1342
1343static int anfc_chips_init(struct arasan_nfc *nfc)
1344{
1345        struct device_node *np = nfc->dev->of_node, *nand_np;
1346        int nchips = of_get_child_count(np);
1347        int ret;
1348
1349        if (!nchips) {
1350                dev_err(nfc->dev, "Incorrect number of NAND chips (%d)\n",
1351                        nchips);
1352                return -EINVAL;
1353        }
1354
1355        for_each_child_of_node(np, nand_np) {
1356                ret = anfc_chip_init(nfc, nand_np);
1357                if (ret) {
1358                        of_node_put(nand_np);
1359                        anfc_chips_cleanup(nfc);
1360                        break;
1361                }
1362        }
1363
1364        return ret;
1365}
1366
1367static void anfc_reset(struct arasan_nfc *nfc)
1368{
1369        /* Disable interrupt signals */
1370        writel_relaxed(0, nfc->base + INTR_SIG_EN_REG);
1371
1372        /* Enable interrupt status */
1373        writel_relaxed(EVENT_MASK, nfc->base + INTR_STS_EN_REG);
1374
1375        nfc->cur_cs = -1;
1376}
1377
1378static int anfc_parse_cs(struct arasan_nfc *nfc)
1379{
1380        int ret;
1381
1382        /* Check the gpio-cs property */
1383        ret = rawnand_dt_parse_gpio_cs(nfc->dev, &nfc->cs_array, &nfc->ncs);
1384        if (ret)
1385                return ret;
1386
1387        /*
1388         * The controller native CS cannot be both disabled at the same time.
1389         * Hence, only one native CS can be used if GPIO CS are needed, so that
1390         * the other is selected when a non-native CS must be asserted (not
1391         * wired physically or configured as GPIO instead of NAND CS). In this
1392         * case, the "not" chosen CS is assigned to nfc->spare_cs and selected
1393         * whenever a GPIO CS must be asserted.
1394         */
1395        if (nfc->cs_array && nfc->ncs > 2) {
1396                if (!nfc->cs_array[0] && !nfc->cs_array[1]) {
1397                        dev_err(nfc->dev,
1398                                "Assign a single native CS when using GPIOs\n");
1399                        return -EINVAL;
1400                }
1401
1402                if (nfc->cs_array[0])
1403                        nfc->spare_cs = 0;
1404                else
1405                        nfc->spare_cs = 1;
1406        }
1407
1408        if (!nfc->cs_array) {
1409                nfc->cs_array = anfc_default_cs_array;
1410                nfc->ncs = ANFC_MAX_CS;
1411                return 0;
1412        }
1413
1414        return 0;
1415}
1416
1417static int anfc_probe(struct platform_device *pdev)
1418{
1419        struct arasan_nfc *nfc;
1420        int ret;
1421
1422        nfc = devm_kzalloc(&pdev->dev, sizeof(*nfc), GFP_KERNEL);
1423        if (!nfc)
1424                return -ENOMEM;
1425
1426        nfc->dev = &pdev->dev;
1427        nand_controller_init(&nfc->controller);
1428        nfc->controller.ops = &anfc_ops;
1429        INIT_LIST_HEAD(&nfc->chips);
1430
1431        nfc->base = devm_platform_ioremap_resource(pdev, 0);
1432        if (IS_ERR(nfc->base))
1433                return PTR_ERR(nfc->base);
1434
1435        anfc_reset(nfc);
1436
1437        nfc->controller_clk = devm_clk_get(&pdev->dev, "controller");
1438        if (IS_ERR(nfc->controller_clk))
1439                return PTR_ERR(nfc->controller_clk);
1440
1441        nfc->bus_clk = devm_clk_get(&pdev->dev, "bus");
1442        if (IS_ERR(nfc->bus_clk))
1443                return PTR_ERR(nfc->bus_clk);
1444
1445        ret = clk_prepare_enable(nfc->controller_clk);
1446        if (ret)
1447                return ret;
1448
1449        ret = clk_prepare_enable(nfc->bus_clk);
1450        if (ret)
1451                goto disable_controller_clk;
1452
1453        ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1454        if (ret)
1455                goto disable_bus_clk;
1456
1457        ret = anfc_parse_cs(nfc);
1458        if (ret)
1459                goto disable_bus_clk;
1460
1461        ret = anfc_chips_init(nfc);
1462        if (ret)
1463                goto disable_bus_clk;
1464
1465        platform_set_drvdata(pdev, nfc);
1466
1467        return 0;
1468
1469disable_bus_clk:
1470        clk_disable_unprepare(nfc->bus_clk);
1471
1472disable_controller_clk:
1473        clk_disable_unprepare(nfc->controller_clk);
1474
1475        return ret;
1476}
1477
1478static int anfc_remove(struct platform_device *pdev)
1479{
1480        struct arasan_nfc *nfc = platform_get_drvdata(pdev);
1481
1482        anfc_chips_cleanup(nfc);
1483
1484        clk_disable_unprepare(nfc->bus_clk);
1485        clk_disable_unprepare(nfc->controller_clk);
1486
1487        return 0;
1488}
1489
1490static const struct of_device_id anfc_ids[] = {
1491        {
1492                .compatible = "xlnx,zynqmp-nand-controller",
1493        },
1494        {
1495                .compatible = "arasan,nfc-v3p10",
1496        },
1497        {}
1498};
1499MODULE_DEVICE_TABLE(of, anfc_ids);
1500
1501static struct platform_driver anfc_driver = {
1502        .driver = {
1503                .name = "arasan-nand-controller",
1504                .of_match_table = anfc_ids,
1505        },
1506        .probe = anfc_probe,
1507        .remove = anfc_remove,
1508};
1509module_platform_driver(anfc_driver);
1510
1511MODULE_LICENSE("GPL v2");
1512MODULE_AUTHOR("Punnaiah Choudary Kalluri <punnaia@xilinx.com>");
1513MODULE_AUTHOR("Naga Sureshkumar Relli <nagasure@xilinx.com>");
1514MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
1515MODULE_DESCRIPTION("Arasan NAND Flash Controller Driver");
1516